3 * Copyright © 2003-2008 Brooke Vibber <bvibber@wikimedia.org>
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
23 namespace MediaWiki\Specials
;
25 use MediaWiki\Export\WikiExporterFactory
;
26 use MediaWiki\HTMLForm\Field\HTMLTextAreaField
;
27 use MediaWiki\HTMLForm\HTMLForm
;
28 use MediaWiki\Linker\LinksMigration
;
29 use MediaWiki\Logger\LoggerFactory
;
30 use MediaWiki\MainConfigNames
;
31 use MediaWiki\Page\PageIdentity
;
32 use MediaWiki\SpecialPage\SpecialPage
;
33 use MediaWiki\Title\Title
;
34 use MediaWiki\Title\TitleFormatter
;
36 use Wikimedia\Rdbms\IConnectionProvider
;
37 use Wikimedia\Rdbms\SelectQueryBuilder
;
40 * A special page that allows users to export pages in a XML file
42 * @ingroup SpecialPage
45 class SpecialExport
extends SpecialPage
{
46 protected bool $curonly;
47 protected bool $doExport;
48 protected int $pageLinkDepth;
49 protected bool $templates;
51 private IConnectionProvider
$dbProvider;
52 private WikiExporterFactory
$wikiExporterFactory;
53 private TitleFormatter
$titleFormatter;
54 private LinksMigration
$linksMigration;
57 * @param IConnectionProvider $dbProvider
58 * @param WikiExporterFactory $wikiExporterFactory
59 * @param TitleFormatter $titleFormatter
60 * @param LinksMigration $linksMigration
62 public function __construct(
63 IConnectionProvider
$dbProvider,
64 WikiExporterFactory
$wikiExporterFactory,
65 TitleFormatter
$titleFormatter,
66 LinksMigration
$linksMigration
68 parent
::__construct( 'Export' );
69 $this->dbProvider
= $dbProvider;
70 $this->wikiExporterFactory
= $wikiExporterFactory;
71 $this->titleFormatter
= $titleFormatter;
72 $this->linksMigration
= $linksMigration;
75 public function execute( $par ) {
77 $this->outputHeader();
78 $config = $this->getConfig();
80 $this->curonly
= true;
81 $this->doExport
= false;
82 $request = $this->getRequest();
83 $this->templates
= $request->getCheck( 'templates' );
84 $this->pageLinkDepth
= $this->validateLinkDepth(
85 $request->getIntOrNull( 'pagelink-depth' )
90 if ( $request->getCheck( 'addcat' ) ) {
91 $page = $request->getText( 'pages' );
92 $catname = $request->getText( 'catname' );
94 if ( $catname !== '' && $catname !== null && $catname !== false ) {
95 $t = Title
::makeTitleSafe( NS_MAIN
, $catname );
98 * @todo FIXME: This can lead to hitting memory limit for very large
99 * categories. Ideally we would do the lookup synchronously
100 * during the export in a single query.
102 $catpages = $this->getPagesFromCategory( $t );
104 if ( $page !== '' ) {
107 $page .= implode( "\n", $catpages );
111 } elseif ( $request->getCheck( 'addns' ) && $config->get( MainConfigNames
::ExportFromNamespaces
) ) {
112 $page = $request->getText( 'pages' );
113 $nsindex = $request->getText( 'nsindex', '' );
115 if ( strval( $nsindex ) !== '' ) {
117 * Same implementation as above, so same @todo
119 $nspages = $this->getPagesFromNamespace( (int)$nsindex );
121 $page .= "\n" . implode( "\n", $nspages );
124 } elseif ( $request->getCheck( 'exportall' ) && $config->get( MainConfigNames
::ExportAllowAll
) ) {
125 $this->doExport
= true;
128 /* Although $page and $history are not used later on, we
129 nevertheless set them to avoid that PHP notices about using
130 undefined variables foul up our XML output (see call to
131 doExport(...) further down) */
134 } elseif ( $request->wasPosted() && $par == '' ) {
135 // Log to see if certain parameters are actually used.
136 // If not, we could deprecate them and do some cleanup, here and in WikiExporter.
137 LoggerFactory
::getInstance( 'export' )->debug(
138 'Special:Export POST, dir: [{dir}], offset: [{offset}], limit: [{limit}]', [
139 'dir' => $request->getRawVal( 'dir' ),
140 'offset' => $request->getRawVal( 'offset' ),
141 'limit' => $request->getRawVal( 'limit' ),
144 $page = $request->getText( 'pages' );
145 $this->curonly
= $request->getCheck( 'curonly' );
146 $rawOffset = $request->getVal( 'offset' );
149 $offset = wfTimestamp( TS_MW
, $rawOffset );
154 $maxHistory = $config->get( MainConfigNames
::ExportMaxHistory
);
155 $limit = $request->getInt( 'limit' );
156 $dir = $request->getVal( 'dir' );
160 'limit' => $maxHistory,
162 $historyCheck = $request->getCheck( 'history' );
164 if ( $this->curonly
) {
165 $history = WikiExporter
::CURRENT
;
166 } elseif ( !$historyCheck ) {
167 if ( $limit > 0 && ( $maxHistory == 0 ||
$limit < $maxHistory ) ) {
168 $history['limit'] = $limit;
171 if ( $offset !== null ) {
172 $history['offset'] = $offset;
175 if ( strtolower( $dir ??
'' ) == 'desc' ) {
176 $history['dir'] = 'desc';
181 $this->doExport
= true;
184 // Default to current-only for GET requests.
185 $page = $request->getText( 'pages', $par ??
'' );
186 $historyCheck = $request->getCheck( 'history' );
188 if ( $historyCheck ) {
189 $history = WikiExporter
::FULL
;
191 $history = WikiExporter
::CURRENT
;
195 $this->doExport
= true;
199 if ( !$config->get( MainConfigNames
::ExportAllowHistory
) ) {
201 $history = WikiExporter
::CURRENT
;
204 $list_authors = $request->getCheck( 'listauthors' );
205 if ( !$this->curonly ||
!$config->get( MainConfigNames
::ExportAllowListContributors
) ) {
206 $list_authors = false;
209 if ( $this->doExport
) {
210 $this->getOutput()->disable();
212 // Cancel output buffering and gzipping if set
213 // This should provide safer streaming for pages with history
214 wfResetOutputBuffers();
215 $request->response()->header( 'Content-type: application/xml; charset=utf-8' );
216 $request->response()->header( 'X-Robots-Tag: noindex,nofollow' );
218 if ( $request->getCheck( 'wpDownload' ) ) {
219 // Provide a sensible filename suggestion
220 $filename = urlencode( $config->get( MainConfigNames
::Sitename
) . '-' .
221 wfTimestampNow() . '.xml' );
222 $request->response()->header( "Content-disposition: attachment;filename={$filename}" );
225 // @phan-suppress-next-next-line PhanPossiblyUndeclaredVariable
226 // @phan-suppress-next-line PhanTypeMismatchArgumentNullable history is set when used
227 $this->doExport( $page, $history, $list_authors, $exportall );
232 $out = $this->getOutput();
233 $out->addWikiMsg( 'exporttext' );
236 $categoryName = $request->getText( 'catname' );
240 $canExportAll = $config->get( MainConfigNames
::ExportAllowAll
);
241 $hideIf = $canExportAll ?
[ 'hide-if' => [ '===', 'exportall', '1' ] ] : [];
245 'type' => 'textwithbutton',
247 'horizontal-label' => true,
248 'label-message' => 'export-addcattext',
249 'default' => $categoryName,
251 'buttontype' => 'submit',
252 'buttonname' => 'addcat',
253 'buttondefault' => $this->msg( 'export-addcat' )->text(),
256 if ( $config->get( MainConfigNames
::ExportFromNamespaces
) ) {
259 'type' => 'namespaceselectwithbutton',
260 'default' => $nsindex,
261 'label-message' => 'export-addnstext',
262 'horizontal-label' => true,
265 'cssclass' => 'namespaceselector',
266 'buttontype' => 'submit',
267 'buttonname' => 'addns',
268 'buttondefault' => $this->msg( 'export-addns' )->text(),
273 if ( $canExportAll ) {
277 'label-message' => 'exportall',
278 'name' => 'exportall',
280 'default' => $request->wasPosted() && $request->getCheck( 'exportall' ),
287 'class' => HTMLTextAreaField
::class,
289 'label-message' => 'export-manual',
296 if ( $config->get( MainConfigNames
::ExportAllowHistory
) ) {
300 'label-message' => 'exportcuronly',
303 'default' => !$request->wasPosted() ||
$request->getCheck( 'curonly' ),
307 $out->addWikiMsg( 'exportnohistory' );
313 'label-message' => 'export-templates',
314 'name' => 'templates',
315 'id' => 'wpExportTemplates',
316 'default' => $request->wasPosted() && $request->getCheck( 'templates' ),
320 if ( $config->get( MainConfigNames
::ExportMaxLinkDepth
) ||
$this->userCanOverrideExportDepth() ) {
322 'pagelink-depth' => [
324 'name' => 'pagelink-depth',
325 'id' => 'pagelink-depth',
326 'label-message' => 'export-pagelinks',
336 'name' => 'wpDownload',
337 'id' => 'wpDownload',
338 'default' => !$request->wasPosted() ||
$request->getCheck( 'wpDownload' ),
339 'label-message' => 'export-download',
343 if ( $config->get( MainConfigNames
::ExportAllowListContributors
) ) {
347 'label-message' => 'exportlistauthors',
348 'default' => $request->wasPosted() && $request->getCheck( 'listauthors' ),
349 'name' => 'listauthors',
350 'id' => 'listauthors',
355 $htmlForm = HTMLForm
::factory( 'ooui', $formDescriptor, $this->getContext() );
356 $htmlForm->setSubmitTextMsg( 'export-submit' );
357 $htmlForm->prepareForm()->displayForm( false );
358 $this->addHelpLink( 'Help:Export' );
364 protected function userCanOverrideExportDepth() {
365 return $this->getAuthority()->isAllowed( 'override-export-depth' );
369 * Do the actual page exporting
371 * @param string $page User input on what page(s) to export
372 * @param int $history One of the WikiExporter history export constants
373 * @param bool $list_authors Whether to add distinct author list (when
374 * not returning full history)
375 * @param bool $exportall Whether to export everything
377 protected function doExport( $page, $history, $list_authors, $exportall ) {
378 // If we are grabbing everything, enable full history and ignore the rest
380 $history = WikiExporter
::FULL
;
382 $pageSet = []; // Inverted index of all pages to look up
384 // Split up and normalize input
385 foreach ( explode( "\n", $page ) as $pageName ) {
386 $pageName = trim( $pageName );
387 $title = Title
::newFromText( $pageName );
388 if ( $title && !$title->isExternal() && $title->getText() !== '' ) {
389 // Only record each page once!
390 $pageSet[$title->getPrefixedText()] = true;
394 // Set of original pages to pass on to further manipulation...
395 $inputPages = array_keys( $pageSet );
397 // Look up any linked pages if asked...
398 if ( $this->templates
) {
399 $pageSet = $this->getTemplates( $inputPages, $pageSet );
401 $pageSet = $this->getExtraPages( $inputPages, $pageSet );
402 $linkDepth = $this->pageLinkDepth
;
404 $pageSet = $this->getPageLinks( $inputPages, $pageSet, $linkDepth );
407 $pages = array_keys( $pageSet );
409 // Normalize titles to the same format and remove dupes, see T19374
410 foreach ( $pages as $k => $v ) {
411 $pages[$k] = str_replace( ' ', '_', $v );
414 $pages = array_unique( $pages );
417 /* Ok, let's get to it... */
418 $db = $this->dbProvider
->getReplicaDatabase();
420 $exporter = $this->wikiExporterFactory
->getWikiExporter( $db, $history );
421 $exporter->list_authors
= $list_authors;
422 $exporter->openStream();
425 $exporter->allPages();
427 // @phan-suppress-next-line PhanPossiblyUndeclaredVariable
428 foreach ( $pages as $page ) {
429 # T10824: Only export pages the user can read
430 $title = Title
::newFromText( $page );
431 if ( $title === null ) {
432 // @todo Perhaps output an <error> tag or something.
436 if ( !$this->getAuthority()->authorizeRead( 'read', $title ) ) {
437 // @todo Perhaps output an <error> tag or something.
441 $exporter->pageByTitle( $title );
445 $exporter->closeStream();
449 * @param PageIdentity $page
452 protected function getPagesFromCategory( PageIdentity
$page ) {
453 $maxPages = $this->getConfig()->get( MainConfigNames
::ExportPagelistLimit
);
455 $name = $page->getDBkey();
457 $dbr = $this->dbProvider
->getReplicaDatabase();
458 $res = $dbr->newSelectQueryBuilder()
459 ->select( [ 'page_namespace', 'page_title' ] )
461 ->join( 'categorylinks', null, 'cl_from=page_id' )
462 ->where( [ 'cl_to' => $name ] )
464 ->caller( __METHOD__
)->fetchResultSet();
468 foreach ( $res as $row ) {
469 $pages[] = Title
::makeName( $row->page_namespace
, $row->page_title
);
476 * @param int $nsindex
479 protected function getPagesFromNamespace( $nsindex ) {
480 $maxPages = $this->getConfig()->get( MainConfigNames
::ExportPagelistLimit
);
482 $dbr = $this->dbProvider
->getReplicaDatabase();
483 $res = $dbr->newSelectQueryBuilder()
484 ->select( [ 'page_namespace', 'page_title' ] )
486 ->where( [ 'page_namespace' => $nsindex ] )
488 ->caller( __METHOD__
)->fetchResultSet();
492 foreach ( $res as $row ) {
493 $pages[] = Title
::makeName( $row->page_namespace
, $row->page_title
);
500 * Expand a list of pages to include templates used in those pages.
501 * @param array $inputPages List of titles to look up
502 * @param array $pageSet Associative array indexed by titles for output
503 * @return array Associative array index by titles
505 protected function getTemplates( $inputPages, $pageSet ) {
506 [ $nsField, $titleField ] = $this->linksMigration
->getTitleFields( 'templatelinks' );
507 $queryInfo = $this->linksMigration
->getQueryInfo( 'templatelinks' );
508 $dbr = $this->dbProvider
->getReplicaDatabase();
509 $queryBuilder = $dbr->newSelectQueryBuilder()
510 ->caller( __METHOD__
)
511 ->select( [ 'namespace' => $nsField, 'title' => $titleField ] )
513 ->join( 'templatelinks', null, 'page_id=tl_from' )
514 ->tables( array_diff( $queryInfo['tables'], [ 'templatelinks' ] ) )
515 ->joinConds( $queryInfo['joins'] );
516 return $this->getLinks( $inputPages, $pageSet, $queryBuilder );
520 * Add extra pages to the list of pages to export.
521 * @param string[] $inputPages List of page titles to export
522 * @param bool[] $pageSet Initial associative array indexed by string page titles
523 * @return bool[] Associative array indexed by string page titles including extra pages
525 private function getExtraPages( $inputPages, $pageSet ) {
527 $this->getHookRunner()->onSpecialExportGetExtraPages( $inputPages, $extraPages );
528 foreach ( $extraPages as $extraPage ) {
529 $pageSet[$this->titleFormatter
->getPrefixedText( $extraPage )] = true;
535 * Validate link depth setting, if available.
536 * @param int|null $depth
539 protected function validateLinkDepth( $depth ) {
540 if ( $depth === null ||
$depth < 0 ) {
544 if ( !$this->userCanOverrideExportDepth() ) {
545 $maxLinkDepth = $this->getConfig()->get( MainConfigNames
::ExportMaxLinkDepth
);
546 if ( $depth > $maxLinkDepth ) {
547 return $maxLinkDepth;
552 * There's a HARD CODED limit of 5 levels of recursion here to prevent a
553 * crazy-big export from being done by someone setting the depth
554 * number too high. In other words, last resort safety net.
557 return intval( min( $depth, 5 ) );
561 * Expand a list of pages to include pages linked to from that page.
562 * @param array $inputPages
563 * @param array $pageSet
567 protected function getPageLinks( $inputPages, $pageSet, $depth ) {
568 for ( ; $depth > 0; --$depth ) {
569 [ $nsField, $titleField ] = $this->linksMigration
->getTitleFields( 'pagelinks' );
570 $queryInfo = $this->linksMigration
->getQueryInfo( 'pagelinks' );
571 $dbr = $this->dbProvider
->getReplicaDatabase();
572 $queryBuilder = $dbr->newSelectQueryBuilder()
573 ->caller( __METHOD__
)
574 ->select( [ 'namespace' => $nsField, 'title' => $titleField ] )
576 ->join( 'pagelinks', null, 'page_id=pl_from' )
577 ->tables( array_diff( $queryInfo['tables'], [ 'pagelinks' ] ) )
578 ->joinConds( $queryInfo['joins'] );
579 $pageSet = $this->getLinks( $inputPages, $pageSet, $queryBuilder );
580 $inputPages = array_keys( $pageSet );
587 * Expand a list of pages to include items used in those pages.
588 * @param array $inputPages Array of page titles
589 * @param array $pageSet
590 * @param SelectQueryBuilder $queryBuilder
593 protected function getLinks( $inputPages, $pageSet, SelectQueryBuilder
$queryBuilder ) {
594 foreach ( $inputPages as $page ) {
595 $title = Title
::newFromText( $page );
597 $pageSet[$title->getPrefixedText()] = true;
598 /// @todo FIXME: May or may not be more efficient to batch these
599 /// by namespace when given multiple input pages.
600 $result = ( clone $queryBuilder )
602 'page_namespace' => $title->getNamespace(),
603 'page_title' => $title->getDBkey()
607 foreach ( $result as $row ) {
608 $template = Title
::makeTitle( $row->namespace, $row->title
);
609 $pageSet[$template->getPrefixedText()] = true;
617 protected function getGroupName() {
622 /** @deprecated class alias since 1.41 */
623 class_alias( SpecialExport
::class, 'SpecialExport' );