3 * Script that postprocesses XML dumps from dumpBackup.php to add page text
5 * Copyright (C) 2005 Brion Vibber <brion@pobox.com>
6 * http://www.mediawiki.org/
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License along
19 * with this program; if not, write to the Free Software Foundation, Inc.,
20 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
21 * http://www.gnu.org/copyleft/gpl.html
24 * @ingroup Maintenance
27 $originalDir = getcwd();
29 require_once( dirname( __FILE__
) . '/commandLine.inc' );
30 require_once( 'backup.inc' );
33 * @ingroup Maintenance
35 class TextPassDumper
extends BackupDumper
{
37 var $input = "php://stdin";
38 var $history = WikiExporter
::FULL
;
40 var $prefetchCount = 0;
42 var $pageCountLast = 0;
43 var $revCountLast = 0;
44 var $prefetchCountLast = 0;
45 var $fetchCountLast = 0;
49 var $failedTextRetrievals = 0;
50 var $maxConsecutiveFailedTextRetrievals = 200;
51 var $failureTimeout = 5; // Seconds to sleep after db failure
55 var $spawnProc = false;
56 var $spawnWrite = false;
57 var $spawnRead = false;
58 var $spawnErr = false;
62 function initProgress( $history ) {
63 parent
::initProgress();
64 $this->ID
= getmypid();
65 $this->lastTime
= $this->startTime
;
68 function dump( $history, $text = WikiExporter
::TEXT
) {
69 # This shouldn't happen if on console... ;)
70 header( 'Content-type: text/html; charset=UTF-8' );
72 # Notice messages will foul up your XML output even if they're
73 # relatively harmless.
74 if ( ini_get( 'display_errors' ) )
75 ini_set( 'display_errors', 'stderr' );
77 $this->initProgress( $this->history
);
79 $this->db
= $this->backupDb();
81 $this->egress
= new ExportProgressFilter( $this->sink
, $this );
83 $input = fopen( $this->input
, "rt" );
84 $result = $this->readDump( $input );
86 if ( WikiError
::isError( $result ) ) {
87 wfDie( $result->getMessage() );
90 if ( $this->spawnProc
) {
94 $this->report( true );
97 function processOption( $opt, $val, $param ) {
99 $url = $this->processFileOpt( $val, $param );
103 require_once "$IP/maintenance/backupPrefetch.inc";
104 $this->prefetch
= new BaseDump( $url );
110 $this->history
= WikiExporter
::CURRENT
;
113 $this->history
= WikiExporter
::FULL
;
124 function processFileOpt( $val, $param ) {
125 $fileURIs = explode(';',$param);
126 foreach ( $fileURIs as $URI ) {
132 $newURI = "compress.zlib://$URI";
135 $newURI = "compress.bzip2://$URI";
138 $newURI = "mediawiki.compress.7z://$URI";
143 $newFileURIs[] = $newURI;
145 $val = implode( ';', $newFileURIs );
150 * Overridden to include prefetch ratio if enabled.
152 function showReport() {
153 if ( !$this->prefetch
) {
154 return parent
::showReport();
157 if ( $this->reporting
) {
158 $now = wfTimestamp( TS_DB
);
159 $deltaAll = wfTime() - $this->startTime
;
160 $deltaPart = wfTime() - $this->lastTime
;
161 $this->pageCountPart
= $this->pageCount
- $this->pageCountLast
;
162 $this->revCountPart
= $this->revCount
- $this->revCountLast
;
165 $portion = $this->revCount
/ $this->maxCount
;
166 $eta = $this->startTime +
$deltaAll / $portion;
167 $etats = wfTimestamp( TS_DB
, intval( $eta ) );
168 if ( $this->fetchCount
) {
169 $fetchRate = 100.0 * $this->prefetchCount
/ $this->fetchCount
;
174 $pageRate = $this->pageCount
/ $deltaAll;
175 $revRate = $this->revCount
/ $deltaAll;
183 if ( $this->fetchCountLast
) {
184 $fetchRatePart = 100.0 * $this->prefetchCountLast
/ $this->fetchCountLast
;
187 $fetchRatePart = '-';
189 $pageRatePart = $this->pageCountPart
/ $deltaPart;
190 $revRatePart = $this->revCountPart
/ $deltaPart;
193 $fetchRatePart = '-';
197 $this->progress( sprintf( "%s: %s (ID %d) %d pages (%0.1f|%0.1f/sec all|curr), %d revs (%0.1f|%0.1f/sec all|curr), %0.1f%%|%0.1f%% prefetched (all|curr), ETA %s [max %d]",-
198 $now, wfWikiID(), $this->ID
, $this->pageCount
, $pageRate, $pageRatePart, $this->revCount
, $revRate, $revRatePart, $fetchRate, $fetchRatePart, $etats, $this->maxCount
) );
199 $this->lastTime
= $now;
200 $this->partCountLast
= $this->partCount
;
201 $this->revCountLast
= $this->revCount
;
202 $this->prefetchCountLast
= $this->prefetchCount
;
203 $this->fetchCountLast
= $this->fetchCount
;
207 function readDump( $input ) {
209 $this->openElement
= false;
210 $this->atStart
= true;
212 $this->lastName
= "";
216 $parser = xml_parser_create( "UTF-8" );
217 xml_parser_set_option( $parser, XML_OPTION_CASE_FOLDING
, false );
219 xml_set_element_handler( $parser, array( &$this, 'startElement' ), array( &$this, 'endElement' ) );
220 xml_set_character_data_handler( $parser, array( &$this, 'characterData' ) );
222 $offset = 0; // for context extraction on error reporting
223 $bufferSize = 512 * 1024;
225 $chunk = fread( $input, $bufferSize );
226 if ( !xml_parse( $parser, $chunk, feof( $input ) ) ) {
227 wfDebug( "TextDumpPass::readDump encountered XML parsing error\n" );
228 return new WikiXmlError( $parser, 'XML import parse failure', $chunk, $offset );
230 $offset +
= strlen( $chunk );
231 } while ( $chunk !== false && !feof( $input ) );
232 xml_parser_free( $parser );
237 function getText( $id ) {
239 if ( isset( $this->prefetch
) ) {
240 $text = $this->prefetch
->prefetch( $this->thisPage
, $this->thisRev
);
241 if ( $text !== null ) { // Entry missing from prefetch dump
242 $dbr = wfGetDB( DB_SLAVE
);
243 $revID = intval( $this->thisRev
);
244 $revLength = $dbr->selectField( 'revision', 'rev_len', array( 'rev_id' => $revID ) );
245 // if length of rev text in file doesn't match length in db, we reload
246 // this avoids carrying forward broken data from previous xml dumps
247 if( strlen( $text ) == $revLength ) {
248 $this->prefetchCount++
;
253 return $this->doGetText( $id );
256 private function doGetText( $id ) {
260 $ex = new MWException( "Graceful storage failure" );
262 if ( $this->spawn
) {
263 if ($this->failures
) {
264 // we don't know why it failed, could be the child process
265 // borked, could be db entry busted, could be db server out to lunch,
266 // so cover all bases
270 $text = $this->getTextSpawned( $id );
272 $text = $this->getTextDbSafe( $id );
274 if ( $text === false ) {
276 if ( $this->failures
> $this->maxFailures
) {
277 $this->progress( "Failed to retrieve revision text for text id ".
278 "$id after $this->maxFailures tries, giving up" );
279 // were there so many bad retrievals in a row we want to bail?
280 // at some point we have to declare the dump irretrievably broken
281 $this->failedTextRetrievals++
;
282 if ($this->failedTextRetrievals
> $this->maxConsecutiveFailedTextRetrievals
) {
286 // would be nice to return something better to the caller someday,
287 // log what we know about the failure and about the revision
291 $this->progress( "Error $this->failures " .
292 "of allowed $this->maxFailures retrieving revision text for text id $id! " .
293 "Pausing $this->failureTimeout seconds before retry..." );
294 sleep( $this->failureTimeout
);
297 $this->failedTextRetrievals
= 0;
305 * Fetch a text revision from the database, retrying in case of failure.
306 * This may survive some transitory errors by reconnecting, but
307 * may not survive a long-term server outage.
309 private function getTextDbSafe( $id ) {
312 $text = $this->getTextDb( $id );
313 } catch ( DBQueryError
$ex ) {
321 * May throw a database error if, say, the server dies during query.
323 private function getTextDb( $id ) {
325 $row = $this->db
->selectRow( 'text',
326 array( 'old_text', 'old_flags' ),
327 array( 'old_id' => $id ),
329 $text = Revision
::getRevisionText( $row );
330 if ( $text === false ) {
333 $stripped = str_replace( "\r", "", $text );
334 $normalized = $wgContLang->normalize( $stripped );
338 private function getTextSpawned( $id ) {
339 wfSuppressWarnings();
340 if ( !$this->spawnProc
) {
344 $text = $this->getTextSpawnedOnce( $id );
349 function openSpawn() {
353 array_map( 'wfEscapeShellArg',
356 "$IP/maintenance/fetchText.php",
357 '--wiki', wfWikiID() ) ) );
359 0 => array( "pipe", "r" ),
360 1 => array( "pipe", "w" ),
361 2 => array( "file", "/dev/null", "a" ) );
364 $this->progress( "Spawning database subprocess: $cmd" );
365 $this->spawnProc
= proc_open( $cmd, $spec, $pipes );
366 if ( !$this->spawnProc
) {
368 $this->progress( "Subprocess spawn failed." );
372 $this->spawnWrite
, // -> stdin
373 $this->spawnRead
, // <- stdout
379 private function closeSpawn() {
380 wfSuppressWarnings();
381 if ( $this->spawnRead
)
382 fclose( $this->spawnRead
);
383 $this->spawnRead
= false;
384 if ( $this->spawnWrite
)
385 fclose( $this->spawnWrite
);
386 $this->spawnWrite
= false;
387 if ( $this->spawnErr
)
388 fclose( $this->spawnErr
);
389 $this->spawnErr
= false;
390 if ( $this->spawnProc
)
391 pclose( $this->spawnProc
);
392 $this->spawnProc
= false;
396 private function getTextSpawnedOnce( $id ) {
399 $ok = fwrite( $this->spawnWrite
, "$id\n" );
400 // $this->progress( ">> $id" );
401 if ( !$ok ) return false;
403 $ok = fflush( $this->spawnWrite
);
404 // $this->progress( ">> [flush]" );
405 if ( !$ok ) return false;
407 // check that the text id they are sending is the one we asked for
408 // this avoids out of sync revision text errors we have encountered in the past
409 $newId = fgets( $this->spawnRead
);
410 if ( $newId === false ) {
413 if ( $id != intval( $newId ) ) {
417 $len = fgets( $this->spawnRead
);
418 // $this->progress( "<< " . trim( $len ) );
419 if ( $len === false ) return false;
421 $nbytes = intval( $len );
422 // actual error, not zero-length text
423 if ($nbytes < 0 ) return false;
427 // Subprocess may not send everything at once, we have to loop.
428 while ( $nbytes > strlen( $text ) ) {
429 $buffer = fread( $this->spawnRead
, $nbytes - strlen( $text ) );
430 if ( $buffer === false ) break;
434 $gotbytes = strlen( $text );
435 if ( $gotbytes != $nbytes ) {
436 $this->progress( "Expected $nbytes bytes from database subprocess, got $gotbytes " );
440 // Do normalization in the dump thread...
441 $stripped = str_replace( "\r", "", $text );
442 $normalized = $wgContLang->normalize( $stripped );
446 function startElement( $parser, $name, $attribs ) {
447 $this->clearOpenElement( null );
448 $this->lastName
= $name;
450 if ( $name == 'revision' ) {
451 $this->state
= $name;
452 $this->egress
->writeOpenPage( null, $this->buffer
);
454 } elseif ( $name == 'page' ) {
455 $this->state
= $name;
456 if ( $this->atStart
) {
457 $this->egress
->writeOpenStream( $this->buffer
);
459 $this->atStart
= false;
463 if ( $name == "text" && isset( $attribs['id'] ) ) {
464 $text = $this->getText( $attribs['id'] );
465 $this->openElement
= array( $name, array( 'xml:space' => 'preserve' ) );
466 if ( strlen( $text ) > 0 ) {
467 $this->characterData( $parser, $text );
470 $this->openElement
= array( $name, $attribs );
474 function endElement( $parser, $name ) {
475 if ( $this->openElement
) {
476 $this->clearOpenElement( "" );
478 $this->buffer
.= "</$name>";
481 if ( $name == 'revision' ) {
482 $this->egress
->writeRevision( null, $this->buffer
);
485 } elseif ( $name == 'page' ) {
486 $this->egress
->writeClosePage( $this->buffer
);
488 $this->thisPage
= "";
489 } elseif ( $name == 'mediawiki' ) {
490 $this->egress
->writeCloseStream( $this->buffer
);
495 function characterData( $parser, $data ) {
496 $this->clearOpenElement( null );
497 if ( $this->lastName
== "id" ) {
498 if ( $this->state
== "revision" ) {
499 $this->thisRev
.= $data;
500 } elseif ( $this->state
== "page" ) {
501 $this->thisPage
.= $data;
504 $this->buffer
.= htmlspecialchars( $data );
507 function clearOpenElement( $style ) {
508 if ( $this->openElement
) {
509 $this->buffer
.= Xml
::element( $this->openElement
[0], $this->openElement
[1], $style );
510 $this->openElement
= false;
516 $dumper = new TextPassDumper( $argv );
518 if ( !isset( $options['help'] ) ) {
519 $dumper->dump( true );
521 $dumper->progress( <<<ENDS
522 This script postprocesses XML dumps from dumpBackup.php to add
523 page text which was stubbed out (using --stub).
525 XML input is accepted on stdin.
526 XML output is sent to stdout; progress reports are sent to stderr.
528 Usage: php dumpTextPass.php [<options>]
530 --stub=<type>:<file> To load a compressed stub dump instead of stdin
531 --prefetch=<type>:<file> Use a prior dump file as a text source, to save
532 pressure on the database.
533 (Requires the XMLReader extension)
534 --quiet Don't dump status reports to stderr.
535 --report=n Report position and speed after every n pages processed.
537 --server=h Force reading from MySQL server h
538 --current Base ETA on number of pages in database instead of all revisions
539 --spawn Spawn a subprocess for loading text records
540 --help Display this help message