2 # Copyright (C) 2003, 2005, 2006 Brion Vibber <brion@pobox.com>
3 # http://www.mediawiki.org/
5 # This program is free software; you can redistribute it and/or modify
6 # it under the terms of the GNU General Public License as published by
7 # the Free Software Foundation; either version 2 of the License, or
8 # (at your option) any later version.
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
15 # You should have received a copy of the GNU General Public License along
16 # with this program; if not, write to the Free Software Foundation, Inc.,
17 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 # http://www.gnu.org/copyleft/gpl.html
25 * @ingroup SpecialPage Dump
28 var $list_authors = false ; # Return distinct author list (when not returning full history)
29 var $author_list = "" ;
31 var $dumpUploads = false;
43 * If using WikiExporter::STREAM to stream a large amount of data,
44 * provide a database connection which is not managed by
45 * LoadBalancer to read from: some history blob types will
46 * make additional queries to pull source data while the
47 * main query is still running.
50 * @param $history Mixed: one of WikiExporter::FULL or WikiExporter::CURRENT,
51 * or an associative array:
52 * offset: non-inclusive offset at which to start the query
53 * limit: maximum number of rows to return
54 * dir: "asc" or "desc" timestamp order
55 * @param $buffer Int: one of WikiExporter::BUFFER or WikiExporter::STREAM
57 function __construct( &$db, $history = WikiExporter
::CURRENT
,
58 $buffer = WikiExporter
::BUFFER
, $text = WikiExporter
::TEXT
) {
60 $this->history
= $history;
61 $this->buffer
= $buffer;
62 $this->writer
= new XmlDumpWriter();
63 $this->sink
= new DumpOutput();
68 * Set the DumpOutput or DumpFilter object which will receive
69 * various row objects and XML output for filtering. Filters
70 * can be chained or used as callbacks.
74 function setOutputSink( &$sink ) {
78 function openStream() {
79 $output = $this->writer
->openStream();
80 $this->sink
->writeOpenStream( $output );
83 function closeStream() {
84 $output = $this->writer
->closeStream();
85 $this->sink
->writeCloseStream( $output );
89 * Dumps a series of page and revision records for all pages
90 * in the database, either including complete history or only
91 * the most recent version.
94 return $this->dumpFrom( '' );
98 * Dumps a series of page and revision records for those pages
99 * in the database falling within the page_id range given.
100 * @param $start Int: inclusive lower limit (this id is included)
101 * @param $end Int: Exclusive upper limit (this id is not included)
102 * If 0, no upper limit.
104 function pagesByRange( $start, $end ) {
105 $condition = 'page_id >= ' . intval( $start );
107 $condition .= ' AND page_id < ' . intval( $end );
109 return $this->dumpFrom( $condition );
113 * @param $title Title
115 function pageByTitle( $title ) {
116 return $this->dumpFrom(
117 'page_namespace=' . $title->getNamespace() .
118 ' AND page_title=' . $this->db
->addQuotes( $title->getDBkey() ) );
121 function pageByName( $name ) {
122 $title = Title
::newFromText( $name );
123 if( is_null( $title ) ) {
124 return new WikiError( "Can't export invalid title" );
126 return $this->pageByTitle( $title );
130 function pagesByName( $names ) {
131 foreach( $names as $name ) {
132 $this->pageByName( $name );
137 // -------------------- private implementation below --------------------
139 # Generates the distinct list of authors of an article
140 # Not called by default (depends on $this->list_authors)
141 # Can be set by Special:Export when not exporting whole history
142 function do_list_authors ( $page , $revision , $cond ) {
143 $fname = "do_list_authors" ;
144 wfProfileIn( $fname );
145 $this->author_list
= "<contributors>";
147 $nothidden = '(rev_deleted & '.Revision
::DELETED_USER
.') = 0';
149 $sql = "SELECT DISTINCT rev_user_text,rev_user FROM {$page},{$revision} WHERE page_id=rev_page AND $nothidden AND " . $cond ;
150 $result = $this->db
->query( $sql, $fname );
151 $resultset = $this->db
->resultObject( $result );
152 while( $row = $resultset->fetchObject() ) {
153 $this->author_list
.= "<contributor>" .
155 htmlentities( $row->rev_user_text
) .
162 wfProfileOut( $fname );
163 $this->author_list
.= "</contributors>";
166 function dumpFrom( $cond = '' ) {
167 $fname = 'WikiExporter::dumpFrom';
168 wfProfileIn( $fname );
170 $page = $this->db
->tableName( 'page' );
171 $revision = $this->db
->tableName( 'revision' );
172 $text = $this->db
->tableName( 'text' );
174 $order = 'ORDER BY page_id';
177 if( $this->history
== WikiExporter
::FULL
) {
178 $join = 'page_id=rev_page';
179 } elseif( $this->history
== WikiExporter
::CURRENT
) {
180 if ( $this->list_authors
&& $cond != '' ) { // List authors, if so desired
181 $this->do_list_authors ( $page , $revision , $cond );
183 $join = 'page_id=rev_page AND page_latest=rev_id';
184 } elseif ( is_array( $this->history
) ) {
185 $join = 'page_id=rev_page';
186 if ( $this->history
['dir'] == 'asc' ) {
188 $order .= ', rev_timestamp';
191 $order .= ', rev_timestamp DESC';
193 if ( !empty( $this->history
['offset'] ) ) {
194 $join .= " AND rev_timestamp $op " . $this->db
->addQuotes(
195 $this->db
->timestamp( $this->history
['offset'] ) );
197 if ( !empty( $this->history
['limit'] ) ) {
198 $limitNum = intval( $this->history
['limit'] );
199 if ( $limitNum > 0 ) {
200 $limit = "LIMIT $limitNum";
204 wfProfileOut( $fname );
205 return new WikiError( "$fname given invalid history dump type." );
207 $where = ( $cond == '' ) ?
'' : "$cond AND";
209 if( $this->buffer
== WikiExporter
::STREAM
) {
210 $prev = $this->db
->bufferResults( false );
213 // Optimization hack for full-database dump
214 $revindex = $pageindex = $this->db
->useIndexClause("PRIMARY");
215 $straight = ' /*! STRAIGHT_JOIN */ ';
221 if( $this->text
== WikiExporter
::STUB
) {
222 $sql = "SELECT $straight * FROM
228 $sql = "SELECT $straight * FROM
232 WHERE $where $join AND rev_text_id=old_id
235 $result = $this->db
->query( $sql, $fname );
236 $wrapper = $this->db
->resultObject( $result );
237 $this->outputStream( $wrapper );
239 if ( $this->list_authors
) {
240 $this->outputStream( $wrapper );
243 if( $this->buffer
== WikiExporter
::STREAM
) {
244 $this->db
->bufferResults( $prev );
247 wfProfileOut( $fname );
251 * Runs through a query result set dumping page and revision records.
252 * The result set should be sorted/grouped by page to avoid duplicate
253 * page records in the output.
255 * The result set will be freed once complete. Should be safe for
256 * streaming (non-buffered) queries, as long as it was made on a
257 * separate database connection not managed by LoadBalancer; some
258 * blob storage types will make queries to pull source data.
260 * @param $resultset ResultWrapper
263 function outputStream( $resultset ) {
265 while( $row = $resultset->fetchObject() ) {
266 if( is_null( $last ) ||
267 $last->page_namespace
!= $row->page_namespace ||
268 $last->page_title
!= $row->page_title
) {
269 if( isset( $last ) ) {
271 if( $this->dumpUploads
) {
272 $output .= $this->writer
->writeUploads( $last );
274 $output .= $this->writer
->closePage();
275 $this->sink
->writeClosePage( $output );
277 $output = $this->writer
->openPage( $row );
278 $this->sink
->writeOpenPage( $row, $output );
281 $output = $this->writer
->writeRevision( $row );
282 $this->sink
->writeRevision( $row, $output );
284 if( isset( $last ) ) {
286 if( $this->dumpUploads
) {
287 $output .= $this->writer
->writeUploads( $last );
289 $output .= $this->author_list
;
290 $output .= $this->writer
->closePage();
291 $this->sink
->writeClosePage( $output );
300 class XmlDumpWriter
{
303 * Returns the export schema version.
306 function schemaVersion() {
307 return "0.3"; // FIXME: upgrade to 0.4 when updated XSD is ready, for the revision deletion bits
311 * Opens the XML output stream's root <mediawiki> element.
312 * This does not include an xml directive, so is safe to include
313 * as a subelement in a larger XML stream. Namespace and XML Schema
314 * references are included.
316 * Output will be encoded in UTF-8.
320 function openStream() {
321 global $wgContLanguageCode;
322 $ver = $this->schemaVersion();
323 return wfElement( 'mediawiki', array(
324 'xmlns' => "http://www.mediawiki.org/xml/export-$ver/",
325 'xmlns:xsi' => "http://www.w3.org/2001/XMLSchema-instance",
326 'xsi:schemaLocation' => "http://www.mediawiki.org/xml/export-$ver/ " .
327 "http://www.mediawiki.org/xml/export-$ver.xsd",
329 'xml:lang' => $wgContLanguageCode ),
335 function siteInfo() {
340 $this->caseSetting(),
341 $this->namespaces() );
342 return " <siteinfo>\n " .
343 implode( "\n ", $info ) .
347 function sitename() {
349 return wfElement( 'sitename', array(), $wgSitename );
352 function generator() {
354 return wfElement( 'generator', array(), "MediaWiki $wgVersion" );
357 function homelink() {
358 return wfElement( 'base', array(), Title
::newMainPage()->getFullUrl() );
361 function caseSetting() {
362 global $wgCapitalLinks;
363 // "case-insensitive" option is reserved for future
364 $sensitivity = $wgCapitalLinks ?
'first-letter' : 'case-sensitive';
365 return wfElement( 'case', array(), $sensitivity );
368 function namespaces() {
370 $spaces = " <namespaces>\n";
371 foreach( $wgContLang->getFormattedNamespaces() as $ns => $title ) {
372 $spaces .= ' ' . wfElement( 'namespace', array( 'key' => $ns ), $title ) . "\n";
374 $spaces .= " </namespaces>";
379 * Closes the output stream with the closing root element.
380 * Call when finished dumping things.
382 function closeStream() {
383 return "</mediawiki>\n";
388 * Opens a <page> section on the output stream, with data
389 * from the given database row.
395 function openPage( $row ) {
397 $title = Title
::makeTitle( $row->page_namespace
, $row->page_title
);
398 $out .= ' ' . wfElementClean( 'title', array(), $title->getPrefixedText() ) . "\n";
399 $out .= ' ' . wfElement( 'id', array(), strval( $row->page_id
) ) . "\n";
400 if( '' != $row->page_restrictions
) {
401 $out .= ' ' . wfElement( 'restrictions', array(),
402 strval( $row->page_restrictions
) ) . "\n";
408 * Closes a <page> section on the output stream.
412 function closePage() {
417 * Dumps a <revision> section on the output stream, with
418 * data filled in from the given database row.
424 function writeRevision( $row ) {
425 $fname = 'WikiExporter::dumpRev';
426 wfProfileIn( $fname );
428 $out = " <revision>\n";
429 $out .= " " . wfElement( 'id', null, strval( $row->rev_id
) ) . "\n";
431 $out .= $this->writeTimestamp( $row->rev_timestamp
);
433 if( $row->rev_deleted
& Revision
::DELETED_USER
) {
434 $out .= " " . wfElement( 'contributor', array( 'deleted' => 'deleted' ) ) . "\n";
436 $out .= $this->writeContributor( $row->rev_user
, $row->rev_user_text
);
439 if( $row->rev_minor_edit
) {
440 $out .= " <minor/>\n";
442 if( $row->rev_deleted
& Revision
::DELETED_COMMENT
) {
443 $out .= " " . wfElement( 'comment', array( 'deleted' => 'deleted' ) ) . "\n";
444 } elseif( $row->rev_comment
!= '' ) {
445 $out .= " " . wfElementClean( 'comment', null, strval( $row->rev_comment
) ) . "\n";
448 if( $row->rev_deleted
& Revision
::DELETED_TEXT
) {
449 $out .= " " . wfElement( 'text', array( 'deleted' => 'deleted' ) ) . "\n";
450 } elseif( isset( $row->old_text
) ) {
451 // Raw text from the database may have invalid chars
452 $text = strval( Revision
::getRevisionText( $row ) );
453 $out .= " " . wfElementClean( 'text',
454 array( 'xml:space' => 'preserve' ),
455 strval( $text ) ) . "\n";
458 $out .= " " . wfElement( 'text',
459 array( 'id' => $row->rev_text_id
),
463 $out .= " </revision>\n";
465 wfProfileOut( $fname );
469 function writeTimestamp( $timestamp ) {
470 $ts = wfTimestamp( TS_ISO_8601
, $timestamp );
471 return " " . wfElement( 'timestamp', null, $ts ) . "\n";
474 function writeContributor( $id, $text ) {
475 $out = " <contributor>\n";
477 $out .= " " . wfElementClean( 'username', null, strval( $text ) ) . "\n";
478 $out .= " " . wfElement( 'id', null, strval( $id ) ) . "\n";
480 $out .= " " . wfElementClean( 'ip', null, strval( $text ) ) . "\n";
482 $out .= " </contributor>\n";
487 * Warning! This data is potentially inconsistent. :(
489 function writeUploads( $row ) {
490 if( $row->page_namespace
== NS_IMAGE
) {
491 $img = wfFindFile( $row->page_title
);
494 foreach( array_reverse( $img->getHistory() ) as $ver ) {
495 $out .= $this->writeUpload( $ver );
497 $out .= $this->writeUpload( $img );
504 function writeUpload( $file ) {
505 return " <upload>\n" .
506 $this->writeTimestamp( $file->getTimestamp() ) .
507 $this->writeContributor( $file->getUser( 'id' ), $file->getUser( 'text' ) ) .
508 " " . wfElementClean( 'comment', null, $file->getDescription() ) . "\n" .
509 " " . wfElement( 'filename', null, $file->getName() ) . "\n" .
510 " " . wfElement( 'src', null, $file->getFullUrl() ) . "\n" .
511 " " . wfElement( 'size', null, $file->getSize() ) . "\n" .
519 * Base class for output stream; prints to stdout or buffer or whereever.
523 function writeOpenStream( $string ) {
524 $this->write( $string );
527 function writeCloseStream( $string ) {
528 $this->write( $string );
531 function writeOpenPage( $page, $string ) {
532 $this->write( $string );
535 function writeClosePage( $string ) {
536 $this->write( $string );
539 function writeRevision( $rev, $string ) {
540 $this->write( $string );
544 * Override to write to a different stream type.
547 function write( $string ) {
553 * Stream outputter to send data to a file.
556 class DumpFileOutput
extends DumpOutput
{
559 function DumpFileOutput( $file ) {
560 $this->handle
= fopen( $file, "wt" );
563 function write( $string ) {
564 fputs( $this->handle
, $string );
569 * Stream outputter to send data to a file via some filter program.
570 * Even if compression is available in a library, using a separate
571 * program can allow us to make use of a multi-processor system.
574 class DumpPipeOutput
extends DumpFileOutput
{
575 function DumpPipeOutput( $command, $file = null ) {
576 if( !is_null( $file ) ) {
577 $command .= " > " . wfEscapeShellArg( $file );
579 $this->handle
= popen( $command, "w" );
584 * Sends dump output via the gzip compressor.
587 class DumpGZipOutput
extends DumpPipeOutput
{
588 function DumpGZipOutput( $file ) {
589 parent
::DumpPipeOutput( "gzip", $file );
594 * Sends dump output via the bgzip2 compressor.
597 class DumpBZip2Output
extends DumpPipeOutput
{
598 function DumpBZip2Output( $file ) {
599 parent
::DumpPipeOutput( "bzip2", $file );
604 * Sends dump output via the p7zip compressor.
607 class Dump7ZipOutput
extends DumpPipeOutput
{
608 function Dump7ZipOutput( $file ) {
609 $command = "7za a -bd -si " . wfEscapeShellArg( $file );
610 // Suppress annoying useless crap from p7zip
611 // Unfortunately this could suppress real error messages too
612 $command .= ' >' . wfGetNull() . ' 2>&1';
613 parent
::DumpPipeOutput( $command );
620 * Dump output filter class.
621 * This just does output filtering and streaming; XML formatting is done
622 * higher up, so be careful in what you do.
626 function DumpFilter( &$sink ) {
627 $this->sink
=& $sink;
630 function writeOpenStream( $string ) {
631 $this->sink
->writeOpenStream( $string );
634 function writeCloseStream( $string ) {
635 $this->sink
->writeCloseStream( $string );
638 function writeOpenPage( $page, $string ) {
639 $this->sendingThisPage
= $this->pass( $page, $string );
640 if( $this->sendingThisPage
) {
641 $this->sink
->writeOpenPage( $page, $string );
645 function writeClosePage( $string ) {
646 if( $this->sendingThisPage
) {
647 $this->sink
->writeClosePage( $string );
648 $this->sendingThisPage
= false;
652 function writeRevision( $rev, $string ) {
653 if( $this->sendingThisPage
) {
654 $this->sink
->writeRevision( $rev, $string );
659 * Override for page-based filter types.
662 function pass( $page ) {
668 * Simple dump output filter to exclude all talk pages.
671 class DumpNotalkFilter
extends DumpFilter
{
672 function pass( $page ) {
673 return !MWNamespace
::isTalk( $page->page_namespace
);
678 * Dump output filter to include or exclude pages in a given set of namespaces.
681 class DumpNamespaceFilter
extends DumpFilter
{
683 var $namespaces = array();
685 function DumpNamespaceFilter( &$sink, $param ) {
686 parent
::DumpFilter( $sink );
689 "NS_MAIN" => NS_MAIN
,
690 "NS_TALK" => NS_TALK
,
691 "NS_USER" => NS_USER
,
692 "NS_USER_TALK" => NS_USER_TALK
,
693 "NS_PROJECT" => NS_PROJECT
,
694 "NS_PROJECT_TALK" => NS_PROJECT_TALK
,
695 "NS_IMAGE" => NS_IMAGE
,
696 "NS_IMAGE_TALK" => NS_IMAGE_TALK
,
697 "NS_MEDIAWIKI" => NS_MEDIAWIKI
,
698 "NS_MEDIAWIKI_TALK" => NS_MEDIAWIKI_TALK
,
699 "NS_TEMPLATE" => NS_TEMPLATE
,
700 "NS_TEMPLATE_TALK" => NS_TEMPLATE_TALK
,
701 "NS_HELP" => NS_HELP
,
702 "NS_HELP_TALK" => NS_HELP_TALK
,
703 "NS_CATEGORY" => NS_CATEGORY
,
704 "NS_CATEGORY_TALK" => NS_CATEGORY_TALK
);
706 if( $param{0} == '!' ) {
707 $this->invert
= true;
708 $param = substr( $param, 1 );
711 foreach( explode( ',', $param ) as $key ) {
713 if( isset( $constants[$key] ) ) {
714 $ns = $constants[$key];
715 $this->namespaces
[$ns] = true;
716 } elseif( is_numeric( $key ) ) {
717 $ns = intval( $key );
718 $this->namespaces
[$ns] = true;
720 throw new MWException( "Unrecognized namespace key '$key'\n" );
725 function pass( $page ) {
726 $match = isset( $this->namespaces
[$page->page_namespace
] );
727 return $this->invert
xor $match;
733 * Dump output filter to include only the last revision in each page sequence.
736 class DumpLatestFilter
extends DumpFilter
{
737 var $page, $pageString, $rev, $revString;
739 function writeOpenPage( $page, $string ) {
741 $this->pageString
= $string;
744 function writeClosePage( $string ) {
746 $this->sink
->writeOpenPage( $this->page
, $this->pageString
);
747 $this->sink
->writeRevision( $this->rev
, $this->revString
);
748 $this->sink
->writeClosePage( $string );
751 $this->revString
= null;
753 $this->pageString
= null;
756 function writeRevision( $rev, $string ) {
757 if( $rev->rev_id
== $this->page
->page_latest
) {
759 $this->revString
= $string;
765 * Base class for output stream; prints to stdout or buffer or whereever.
768 class DumpMultiWriter
{
769 function DumpMultiWriter( $sinks ) {
770 $this->sinks
= $sinks;
771 $this->count
= count( $sinks );
774 function writeOpenStream( $string ) {
775 for( $i = 0; $i < $this->count
; $i++
) {
776 $this->sinks
[$i]->writeOpenStream( $string );
780 function writeCloseStream( $string ) {
781 for( $i = 0; $i < $this->count
; $i++
) {
782 $this->sinks
[$i]->writeCloseStream( $string );
786 function writeOpenPage( $page, $string ) {
787 for( $i = 0; $i < $this->count
; $i++
) {
788 $this->sinks
[$i]->writeOpenPage( $page, $string );
792 function writeClosePage( $string ) {
793 for( $i = 0; $i < $this->count
; $i++
) {
794 $this->sinks
[$i]->writeClosePage( $string );
798 function writeRevision( $rev, $string ) {
799 for( $i = 0; $i < $this->count
; $i++
) {
800 $this->sinks
[$i]->writeRevision( $rev, $string );
805 function xmlsafe( $string ) {
807 wfProfileIn( $fname );
810 * The page may contain old data which has not been properly normalized.
811 * Invalid UTF-8 sequences or forbidden control characters will make our
812 * XML output invalid, so be sure to strip them out.
814 $string = UtfNormal
::cleanUp( $string );
816 $string = htmlspecialchars( $string );
817 wfProfileOut( $fname );