3 * Base classes for database dumpers
5 * Copyright © 2005 Brion Vibber <brion@pobox.com>
6 * http://www.mediawiki.org/
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License along
19 * with this program; if not, write to the Free Software Foundation, Inc.,
20 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
21 * http://www.gnu.org/copyleft/gpl.html
24 * @ingroup Dump Maintenance
28 * @ingroup Dump Maintenance
30 class DumpDBZip2Output extends DumpPipeOutput {
31 function DumpDBZip2Output( $file ) {
32 parent::__construct( "dbzip2", $file );
37 * @ingroup Dump Maintenance
40 var $reportingInterval = 100;
41 var $reporting = true;
44 var $server = null; // use default
45 var $pages = null; // all pages
46 var $skipHeader = false; // don't output <mediawiki> and <siteinfo>
47 var $skipFooter = false; // don't output </mediawiki>
50 var $sink = null; // Output filters
51 var $stubText = false; // include rev_text_id instead of text; for 2-pass dump
52 var $dumpUploads = false;
53 var $dumpUploadFileContents = false;
55 var $pageCountLast = 0;
56 var $revCountLast = 0;
59 function BackupDumper( $args ) {
60 $this->stderr = fopen( "php://stderr", "wt" );
62 // Built-in output and filter plugins
63 $this->registerOutput( 'file', 'DumpFileOutput' );
64 $this->registerOutput( 'gzip', 'DumpGZipOutput' );
65 $this->registerOutput( 'bzip2', 'DumpBZip2Output' );
66 $this->registerOutput( 'dbzip2', 'DumpDBZip2Output' );
67 $this->registerOutput( '7zip', 'Dump7ZipOutput' );
69 $this->registerFilter( 'latest', 'DumpLatestFilter' );
70 $this->registerFilter( 'notalk', 'DumpNotalkFilter' );
71 $this->registerFilter( 'namespace', 'DumpNamespaceFilter' );
73 $this->sink = $this->processArgs( $args );
78 * @param $class String: name of output filter plugin class
80 function registerOutput( $name, $class ) {
81 $this->outputTypes[$name] = $class;
86 * @param $class String: name of filter plugin class
88 function registerFilter( $name, $class ) {
89 $this->filterTypes[$name] = $class;
93 * Load a plugin and register it
95 * @param $class String: name of plugin class; must have a static 'register'
96 * method that takes a BackupDumper as a parameter.
97 * @param $file String: full or relative path to the PHP file to load, or empty
99 function loadPlugin( $class, $file ) {
101 require_once( $file );
103 $register = array( $class, 'register' );
104 call_user_func_array( $register, array( &$this ) );
111 function processArgs( $args ) {
114 foreach ( $args as $arg ) {
116 if ( preg_match( '/^--(.+?)(?:=(.+?)(?::(.+?))?)?$/', $arg, $matches ) ) {
117 @list( /* $full */ , $opt, $val, $param ) = $matches;
120 $this->loadPlugin( $val, $param );
123 if ( !is_null( $sink ) ) {
126 if ( !isset( $this->outputTypes[$val] ) ) {
127 $this->fatalError( "Unrecognized output sink type '$val'" );
129 $type = $this->outputTypes[$val];
130 $sink = new $type( $param );
133 if ( is_null( $sink ) ) {
134 $sink = new DumpOutput();
136 if ( !isset( $this->filterTypes[$val] ) ) {
137 $this->fatalError( "Unrecognized filter type '$val'" );
139 $type = $this->filterTypes[$val];
140 $filter = new $type( $sink, $param );
142 // references are lame in php...
148 $this->reportingInterval = intval( $val );
151 $this->server = $val;
154 if ( !function_exists( 'utf8_normalize' ) ) {
155 wfDl( "php_utfnormal.so" );
156 if ( !function_exists( 'utf8_normalize' ) ) {
157 $this->fatalError( "Failed to load UTF-8 normalization extension. " .
158 "Install or remove --force-normal parameter to use slower code." );
163 $this->processOption( $opt, $val, $param );
168 if ( is_null( $sink ) ) {
169 $sink = new DumpOutput();
173 if ( count( $sinks ) > 1 ) {
174 return new DumpMultiWriter( $sinks );
180 function processOption( $opt, $val, $param ) {
181 // extension point for subclasses to add options
184 function dump( $history, $text = WikiExporter::TEXT ) {
185 # Notice messages will foul up your XML output even if they're
186 # relatively harmless.
187 if ( ini_get( 'display_errors' ) )
188 ini_set( 'display_errors', 'stderr' );
190 $this->initProgress( $history );
192 $db = $this->backupDb();
193 $exporter = new WikiExporter( $db, $history, WikiExporter::STREAM, $text );
194 $exporter->dumpUploads = $this->dumpUploads;
195 $exporter->dumpUploadFileContents = $this->dumpUploadFileContents;
197 $wrapper = new ExportProgressFilter( $this->sink, $this );
198 $exporter->setOutputSink( $wrapper );
200 if ( !$this->skipHeader )
201 $exporter->openStream();
202 # Log item dumps: all or by range
203 if ( $history & WikiExporter::LOGS ) {
204 if ( $this->startId || $this->endId ) {
205 $exporter->logsByRange( $this->startId, $this->endId );
207 $exporter->allLogs();
209 # Page dumps: all or by page ID range
210 } else if ( is_null( $this->pages ) ) {
211 if ( $this->startId || $this->endId ) {
212 $exporter->pagesByRange( $this->startId, $this->endId );
214 $exporter->allPages();
216 # Dump of specific pages
218 $exporter->pagesByName( $this->pages );
221 if ( !$this->skipFooter )
222 $exporter->closeStream();
224 $this->report( true );
228 * Initialise starting time and maximum revision count.
229 * We'll make ETA calculations based an progress, assuming relatively
230 * constant per-revision rate.
231 * @param $history Integer: WikiExporter::CURRENT or WikiExporter::FULL
233 function initProgress( $history = WikiExporter::FULL ) {
234 $table = ( $history == WikiExporter::CURRENT ) ? 'page' : 'revision';
235 $field = ( $history == WikiExporter::CURRENT ) ? 'page_id' : 'rev_id';
237 $dbr = wfGetDB( DB_SLAVE );
238 $this->maxCount = $dbr->selectField( $table, "MAX($field)", '', __METHOD__ );
239 $this->startTime = wfTime();
240 $this->lastTime = $this->startTime;
241 $this->ID = getmypid();
245 * @todo Fixme: the --server parameter is currently not respected, as it
246 * doesn't seem terribly easy to ask the load balancer for a particular
247 * connection by name.
249 function backupDb() {
250 $this->lb = wfGetLBFactory()->newMainLB();
251 $db = $this->lb->getConnection( DB_SLAVE, 'backup' );
253 // Discourage the server from disconnecting us if it takes a long time
254 // to read out the big ol' batch query.
255 $db->setTimeout( 3600 * 24 );
260 function __destruct() {
261 if ( isset( $this->lb ) ) {
262 $this->lb->closeAll();
266 function backupServer() {
273 function reportPage() {
277 function revCount() {
282 function report( $final = false ) {
283 if ( $final xor ( $this->revCount % $this->reportingInterval == 0 ) ) {
288 function showReport() {
289 if ( $this->reporting ) {
290 $now = wfTimestamp( TS_DB );
292 $deltaAll = wfTime() - $this->startTime;
293 $deltaPart = wfTime() - $this->lastTime;
294 $this->pageCountPart = $this->pageCount - $this->pageCountLast;
295 $this->revCountPart = $this->revCount - $this->revCountLast;
298 $portion = $this->revCount / $this->maxCount;
299 $eta = $this->startTime + $deltaAll / $portion;
300 $etats = wfTimestamp( TS_DB, intval( $eta ) );
301 $pageRate = $this->pageCount / $deltaAll;
302 $revRate = $this->revCount / $deltaAll;
309 $pageRatePart = $this->pageCountPart / $deltaPart;
310 $revRatePart = $this->revCountPart / $deltaPart;
315 $this->progress( sprintf( "%s: %s (ID %d) %d pages (%0.1f|%0.1f/sec all|curr), %d revs (%0.1f|%0.1f/sec all|curr), ETA %s [max %d]",
316 $now, wfWikiID(), $this->ID, $this->pageCount, $pageRate, $pageRatePart, $this->revCount, $revRate, $revRatePart, $etats, $this->maxCount ) );
317 $this->lastTime = $nowts;
318 $this->revCountLast = $this->revCount;
322 function progress( $string ) {
323 fwrite( $this->stderr, $string . "\n" );
326 function fatalError( $msg ) {
327 $this->progress( "$msg\n" );
332 class ExportProgressFilter extends DumpFilter {
333 function ExportProgressFilter( &$sink, &$progress ) {
334 parent::__construct( $sink );
335 $this->progress = $progress;
338 function writeClosePage( $string ) {
339 parent::writeClosePage( $string );
340 $this->progress->reportPage();
343 function writeRevision( $rev, $string ) {
344 parent::writeRevision( $rev, $string );
345 $this->progress->revCount();