3 * Base classes for database dumpers
5 * Copyright © 2005 Brion Vibber <brion@pobox.com>
6 * http://www.mediawiki.org/
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License along
19 * with this program; if not, write to the Free Software Foundation, Inc.,
20 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
21 * http://www.gnu.org/copyleft/gpl.html
24 * @ingroup Dump Maintenance
28 * @ingroup Dump Maintenance
30 class DumpDBZip2Output extends DumpPipeOutput {
31 function DumpDBZip2Output( $file ) {
32 parent::__construct( "dbzip2", $file );
37 * @ingroup Dump Maintenance
40 var $reportingInterval = 100;
41 var $reporting = true;
44 var $server = null; // use default
45 var $pages = null; // all pages
46 var $skipHeader = false; // don't output <mediawiki> and <siteinfo>
47 var $skipFooter = false; // don't output </mediawiki>
50 var $sink = null; // Output filters
51 var $stubText = false; // include rev_text_id instead of text; for 2-pass dump
52 var $dumpUploads = false;
53 var $dumpUploadFileContents = false;
55 var $pageCountLast = 0;
56 var $revCountLast = 0;
59 var $outputTypes = array(), $filterTypes = array();
66 function __construct( $args ) {
67 $this->stderr = fopen( "php://stderr", "wt" );
69 // Built-in output and filter plugins
70 $this->registerOutput( 'file', 'DumpFileOutput' );
71 $this->registerOutput( 'gzip', 'DumpGZipOutput' );
72 $this->registerOutput( 'bzip2', 'DumpBZip2Output' );
73 $this->registerOutput( 'dbzip2', 'DumpDBZip2Output' );
74 $this->registerOutput( '7zip', 'Dump7ZipOutput' );
76 $this->registerFilter( 'latest', 'DumpLatestFilter' );
77 $this->registerFilter( 'notalk', 'DumpNotalkFilter' );
78 $this->registerFilter( 'namespace', 'DumpNamespaceFilter' );
80 $this->sink = $this->processArgs( $args );
85 * @param $class String: name of output filter plugin class
87 function registerOutput( $name, $class ) {
88 $this->outputTypes[$name] = $class;
93 * @param $class String: name of filter plugin class
95 function registerFilter( $name, $class ) {
96 $this->filterTypes[$name] = $class;
100 * Load a plugin and register it
102 * @param $class String: name of plugin class; must have a static 'register'
103 * method that takes a BackupDumper as a parameter.
104 * @param $file String: full or relative path to the PHP file to load, or empty
106 function loadPlugin( $class, $file ) {
108 require_once( $file );
110 $register = array( $class, 'register' );
111 call_user_func_array( $register, array( &$this ) );
118 function processArgs( $args ) {
121 foreach ( $args as $arg ) {
123 if ( preg_match( '/^--(.+?)(?:=(.+?)(?::(.+?))?)?$/', $arg, $matches ) ) {
124 @list( /* $full */ , $opt, $val, $param ) = $matches;
127 $this->loadPlugin( $val, $param );
130 if ( !is_null( $sink ) ) {
133 if ( !isset( $this->outputTypes[$val] ) ) {
134 $this->fatalError( "Unrecognized output sink type '$val'" );
136 $type = $this->outputTypes[$val];
137 $sink = new $type( $param );
140 if ( is_null( $sink ) ) {
141 $sink = new DumpOutput();
143 if ( !isset( $this->filterTypes[$val] ) ) {
144 $this->fatalError( "Unrecognized filter type '$val'" );
146 $type = $this->filterTypes[$val];
147 $filter = new $type( $sink, $param );
149 // references are lame in php...
155 $this->reportingInterval = intval( $val );
158 $this->server = $val;
161 if ( !function_exists( 'utf8_normalize' ) ) {
162 wfDl( "php_utfnormal.so" );
163 if ( !function_exists( 'utf8_normalize' ) ) {
164 $this->fatalError( "Failed to load UTF-8 normalization extension. " .
165 "Install or remove --force-normal parameter to use slower code." );
170 $this->processOption( $opt, $val, $param );
175 if ( is_null( $sink ) ) {
176 $sink = new DumpOutput();
180 if ( count( $sinks ) > 1 ) {
181 return new DumpMultiWriter( $sinks );
187 function processOption( $opt, $val, $param ) {
188 // extension point for subclasses to add options
191 function dump( $history, $text = WikiExporter::TEXT ) {
192 # Notice messages will foul up your XML output even if they're
193 # relatively harmless.
194 if ( ini_get( 'display_errors' ) )
195 ini_set( 'display_errors', 'stderr' );
197 $this->initProgress( $history );
199 $db = $this->backupDb();
200 $exporter = new WikiExporter( $db, $history, WikiExporter::STREAM, $text );
201 $exporter->dumpUploads = $this->dumpUploads;
202 $exporter->dumpUploadFileContents = $this->dumpUploadFileContents;
204 $wrapper = new ExportProgressFilter( $this->sink, $this );
205 $exporter->setOutputSink( $wrapper );
207 if ( !$this->skipHeader )
208 $exporter->openStream();
209 # Log item dumps: all or by range
210 if ( $history & WikiExporter::LOGS ) {
211 if ( $this->startId || $this->endId ) {
212 $exporter->logsByRange( $this->startId, $this->endId );
214 $exporter->allLogs();
216 # Page dumps: all or by page ID range
217 } else if ( is_null( $this->pages ) ) {
218 if ( $this->startId || $this->endId ) {
219 $exporter->pagesByRange( $this->startId, $this->endId );
221 $exporter->allPages();
223 # Dump of specific pages
225 $exporter->pagesByName( $this->pages );
228 if ( !$this->skipFooter )
229 $exporter->closeStream();
231 $this->report( true );
235 * Initialise starting time and maximum revision count.
236 * We'll make ETA calculations based an progress, assuming relatively
237 * constant per-revision rate.
238 * @param $history Integer: WikiExporter::CURRENT or WikiExporter::FULL
240 function initProgress( $history = WikiExporter::FULL ) {
241 $table = ( $history == WikiExporter::CURRENT ) ? 'page' : 'revision';
242 $field = ( $history == WikiExporter::CURRENT ) ? 'page_id' : 'rev_id';
244 $dbr = wfGetDB( DB_SLAVE );
245 $this->maxCount = $dbr->selectField( $table, "MAX($field)", '', __METHOD__ );
246 $this->startTime = wfTime();
247 $this->lastTime = $this->startTime;
248 $this->ID = getmypid();
252 * @todo Fixme: the --server parameter is currently not respected, as it
253 * doesn't seem terribly easy to ask the load balancer for a particular
254 * connection by name.
256 function backupDb() {
257 $this->lb = wfGetLBFactory()->newMainLB();
258 $db = $this->lb->getConnection( DB_SLAVE, 'backup' );
260 // Discourage the server from disconnecting us if it takes a long time
261 // to read out the big ol' batch query.
262 $db->setTimeout( 3600 * 24 );
267 function __destruct() {
268 if ( isset( $this->lb ) ) {
269 $this->lb->closeAll();
273 function backupServer() {
280 function reportPage() {
284 function revCount() {
289 function report( $final = false ) {
290 if ( $final xor ( $this->revCount % $this->reportingInterval == 0 ) ) {
295 function showReport() {
296 if ( $this->reporting ) {
297 $now = wfTimestamp( TS_DB );
299 $deltaAll = wfTime() - $this->startTime;
300 $deltaPart = wfTime() - $this->lastTime;
301 $this->pageCountPart = $this->pageCount - $this->pageCountLast;
302 $this->revCountPart = $this->revCount - $this->revCountLast;
305 $portion = $this->revCount / $this->maxCount;
306 $eta = $this->startTime + $deltaAll / $portion;
307 $etats = wfTimestamp( TS_DB, intval( $eta ) );
308 $pageRate = $this->pageCount / $deltaAll;
309 $revRate = $this->revCount / $deltaAll;
316 $pageRatePart = $this->pageCountPart / $deltaPart;
317 $revRatePart = $this->revCountPart / $deltaPart;
322 $this->progress( sprintf( "%s: %s (ID %d) %d pages (%0.1f|%0.1f/sec all|curr), %d revs (%0.1f|%0.1f/sec all|curr), ETA %s [max %d]",
323 $now, wfWikiID(), $this->ID, $this->pageCount, $pageRate, $pageRatePart, $this->revCount, $revRate, $revRatePart, $etats, $this->maxCount ) );
324 $this->lastTime = $nowts;
325 $this->revCountLast = $this->revCount;
329 function progress( $string ) {
330 fwrite( $this->stderr, $string . "\n" );
333 function fatalError( $msg ) {
334 $this->progress( "$msg\n" );
339 class ExportProgressFilter extends DumpFilter {
340 function __construct( &$sink, &$progress ) {
341 parent::__construct( $sink );
342 $this->progress = $progress;
345 function writeClosePage( $string ) {
346 parent::writeClosePage( $string );
347 $this->progress->reportPage();
350 function writeRevision( $rev, $string ) {
351 parent::writeRevision( $rev, $string );
352 $this->progress->revCount();