Make LBFactorySingle call initLoadBalancer() as the others do
[mediawiki.git] / maintenance / importImages.php
blob5a4ab394032d0aa00c36f63a91c6c1c483dfa42f
1 <?php
2 /**
3 * Import one or more images from the local file system into the wiki without
4 * using the web-based interface.
6 * "Smart import" additions:
7 * - aim: preserve the essential metadata (user, description) when importing media
8 * files from an existing wiki.
9 * - process:
10 * - interface with the source wiki, don't use bare files only (see --source-wiki-url).
11 * - fetch metadata from source wiki for each file to import.
12 * - commit the fetched metadata to the destination wiki while submitting.
14 * This program is free software; you can redistribute it and/or modify
15 * it under the terms of the GNU General Public License as published by
16 * the Free Software Foundation; either version 2 of the License, or
17 * (at your option) any later version.
19 * This program is distributed in the hope that it will be useful,
20 * but WITHOUT ANY WARRANTY; without even the implied warranty of
21 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
22 * GNU General Public License for more details.
24 * You should have received a copy of the GNU General Public License along
25 * with this program; if not, write to the Free Software Foundation, Inc.,
26 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
27 * http://www.gnu.org/copyleft/gpl.html
29 * @file
30 * @ingroup Maintenance
31 * @author Rob Church <robchur@gmail.com>
32 * @author Mij <mij@bitchx.it>
35 $optionsWithArgs = [
36 'extensions', 'comment', 'comment-file', 'comment-ext', 'summary', 'user',
37 'license', 'sleep', 'limit', 'from', 'source-wiki-url', 'timestamp',
40 $optionsWithoutArgs = [
41 'protect', 'unprotect', 'search-recursively', 'check-userblock', 'overwrite',
42 'skip-dupes', 'dry'
45 require_once __DIR__ . '/commandLine.inc';
46 require_once __DIR__ . '/importImages.inc';
47 $processed = $added = $ignored = $skipped = $overwritten = $failed = 0;
49 echo "Import Images\n\n";
51 # Need a path
52 if ( count( $args ) == 0 ) {
53 showUsage();
56 $dir = $args[0];
58 # Check Protection
59 if ( isset( $options['protect'] ) && isset( $options['unprotect'] ) ) {
60 die( "Cannot specify both protect and unprotect. Only 1 is allowed.\n" );
63 if ( isset( $options['protect'] ) && $options['protect'] == 1 ) {
64 die( "You must specify a protection option.\n" );
67 # Prepare the list of allowed extensions
68 global $wgFileExtensions;
69 $extensions = isset( $options['extensions'] )
70 ? explode( ',', strtolower( $options['extensions'] ) )
71 : $wgFileExtensions;
73 # Search the path provided for candidates for import
74 $files = findFiles( $dir, $extensions, isset( $options['search-recursively'] ) );
76 # Initialise the user for this operation
77 $user = isset( $options['user'] )
78 ? User::newFromName( $options['user'] )
79 : User::newSystemUser( 'Maintenance script', [ 'steal' => true ] );
80 if ( !$user instanceof User ) {
81 $user = User::newSystemUser( 'Maintenance script', [ 'steal' => true ] );
83 $wgUser = $user;
85 # Get block check. If a value is given, this specified how often the check is performed
86 if ( isset( $options['check-userblock'] ) ) {
87 if ( !$options['check-userblock'] ) {
88 $checkUserBlock = 1;
89 } else {
90 $checkUserBlock = (int)$options['check-userblock'];
92 } else {
93 $checkUserBlock = false;
96 # Get --from
97 MediaWiki\suppressWarnings();
98 $from = $options['from'];
99 MediaWiki\restoreWarnings();
101 # Get sleep time.
102 MediaWiki\suppressWarnings();
103 $sleep = $options['sleep'];
104 MediaWiki\restoreWarnings();
106 if ( $sleep ) {
107 $sleep = (int)$sleep;
110 # Get limit number
111 MediaWiki\suppressWarnings();
112 $limit = $options['limit'];
113 MediaWiki\restoreWarnings();
115 if ( $limit ) {
116 $limit = (int)$limit;
119 $timestamp = isset( $options['timestamp'] ) ? $options['timestamp'] : false;
121 # Get the upload comment. Provide a default one in case there's no comment given.
122 $comment = 'Importing file';
124 if ( isset( $options['comment-file'] ) ) {
125 $comment = file_get_contents( $options['comment-file'] );
126 if ( $comment === false || $comment === null ) {
127 die( "failed to read comment file: {$options['comment-file']}\n" );
129 } elseif ( isset( $options['comment'] ) ) {
130 $comment = $options['comment'];
133 $commentExt = isset( $options['comment-ext'] ) ? $options['comment-ext'] : false;
135 $summary = isset( $options['summary'] ) ? $options['summary'] : '';
137 # Get the license specifier
138 $license = isset( $options['license'] ) ? $options['license'] : '';
140 # Batch "upload" operation
141 $count = count( $files );
142 if ( $count > 0 ) {
144 foreach ( $files as $file ) {
145 $base = UtfNormal\Validator::cleanUp( wfBaseName( $file ) );
147 # Validate a title
148 $title = Title::makeTitleSafe( NS_FILE, $base );
149 if ( !is_object( $title ) ) {
150 echo "{$base} could not be imported; a valid title cannot be produced\n";
151 continue;
154 if ( $from ) {
155 if ( $from == $title->getDBkey() ) {
156 $from = null;
157 } else {
158 $ignored++;
159 continue;
163 if ( $checkUserBlock && ( ( $processed % $checkUserBlock ) == 0 ) ) {
164 $user->clearInstanceCache( 'name' ); // reload from DB!
165 if ( $user->isBlocked() ) {
166 echo $user->getName() . " was blocked! Aborting.\n";
167 break;
171 # Check existence
172 $image = wfLocalFile( $title );
173 if ( $image->exists() ) {
174 if ( isset( $options['overwrite'] ) ) {
175 echo "{$base} exists, overwriting...";
176 $svar = 'overwritten';
177 } else {
178 echo "{$base} exists, skipping\n";
179 $skipped++;
180 continue;
182 } else {
183 if ( isset( $options['skip-dupes'] ) ) {
184 $repo = $image->getRepo();
185 # XXX: we end up calculating this again when actually uploading. that sucks.
186 $sha1 = FSFile::getSha1Base36FromPath( $file );
188 $dupes = $repo->findBySha1( $sha1 );
190 if ( $dupes ) {
191 echo "{$base} already exists as " . $dupes[0]->getName() . ", skipping\n";
192 $skipped++;
193 continue;
197 echo "Importing {$base}...";
198 $svar = 'added';
201 if ( isset( $options['source-wiki-url'] ) ) {
202 /* find comment text directly from source wiki, through MW's API */
203 $real_comment = getFileCommentFromSourceWiki( $options['source-wiki-url'], $base );
204 if ( $real_comment === false ) {
205 $commentText = $comment;
206 } else {
207 $commentText = $real_comment;
210 /* find user directly from source wiki, through MW's API */
211 $real_user = getFileUserFromSourceWiki( $options['source-wiki-url'], $base );
212 if ( $real_user === false ) {
213 $wgUser = $user;
214 } else {
215 $wgUser = User::newFromName( $real_user );
216 if ( $wgUser === false ) {
217 # user does not exist in target wiki
218 echo "failed: user '$real_user' does not exist in target wiki.";
219 continue;
222 } else {
223 # Find comment text
224 $commentText = false;
226 if ( $commentExt ) {
227 $f = findAuxFile( $file, $commentExt );
228 if ( !$f ) {
229 echo " No comment file with extension {$commentExt} found "
230 . "for {$file}, using default comment. ";
231 } else {
232 $commentText = file_get_contents( $f );
233 if ( !$commentText ) {
234 echo " Failed to load comment file {$f}, using default comment. ";
239 if ( !$commentText ) {
240 $commentText = $comment;
244 # Import the file
245 if ( isset( $options['dry'] ) ) {
246 echo " publishing {$file} by '" . $wgUser->getName() . "', comment '$commentText'... ";
247 } else {
248 $mwProps = new MWFileProps( MimeMagic::singleton() );
249 $props = $mwProps->getPropsFromPath( $file, true );
250 $flags = 0;
251 $publishOptions = [];
252 $handler = MediaHandler::getHandler( $props['mime'] );
253 if ( $handler ) {
254 $publishOptions['headers'] = $handler->getStreamHeaders( $props['metadata'] );
255 } else {
256 $publishOptions['headers'] = [];
258 $archive = $image->publish( $file, $flags, $publishOptions );
259 if ( !$archive->isGood() ) {
260 echo "failed. (" .
261 $archive->getWikiText( false, false, 'en' ) .
262 ")\n";
263 $failed++;
264 continue;
268 $commentText = SpecialUpload::getInitialPageText( $commentText, $license );
269 if ( !isset( $options['summary'] ) ) {
270 $summary = $commentText;
273 if ( isset( $options['dry'] ) ) {
274 echo "done.\n";
275 } elseif ( $image->recordUpload2(
276 $archive->value,
277 $summary,
278 $commentText,
279 $props,
280 $timestamp
281 ) ) {
282 # We're done!
283 echo "done.\n";
285 $doProtect = false;
287 global $wgRestrictionLevels;
289 $protectLevel = isset( $options['protect'] ) ? $options['protect'] : null;
291 if ( $protectLevel && in_array( $protectLevel, $wgRestrictionLevels ) ) {
292 $doProtect = true;
294 if ( isset( $options['unprotect'] ) ) {
295 $protectLevel = '';
296 $doProtect = true;
299 if ( $doProtect ) {
300 # Protect the file
301 echo "\nWaiting for replica DBs...\n";
302 // Wait for replica DBs.
303 sleep( 2.0 ); # Why this sleep?
304 wfWaitForSlaves();
306 echo "\nSetting image restrictions ... ";
308 $cascade = false;
309 $restrictions = [];
310 foreach ( $title->getRestrictionTypes() as $type ) {
311 $restrictions[$type] = $protectLevel;
314 $page = WikiPage::factory( $title );
315 $status = $page->doUpdateRestrictions( $restrictions, [], $cascade, '', $user );
316 echo ( $status->isOK() ? 'done' : 'failed' ) . "\n";
318 } else {
319 echo "failed. (at recordUpload stage)\n";
320 $svar = 'failed';
323 $$svar++;
324 $processed++;
326 if ( $limit && $processed >= $limit ) {
327 break;
330 if ( $sleep ) {
331 sleep( $sleep );
335 # Print out some statistics
336 echo "\n";
337 foreach (
339 'count' => 'Found',
340 'limit' => 'Limit',
341 'ignored' => 'Ignored',
342 'added' => 'Added',
343 'skipped' => 'Skipped',
344 'overwritten' => 'Overwritten',
345 'failed' => 'Failed'
346 ] as $var => $desc
348 if ( $$var > 0 ) {
349 echo "{$desc}: {$$var}\n";
352 } else {
353 echo "No suitable files could be found for import.\n";
356 exit( 0 );
358 function showUsage( $reason = false ) {
359 if ( $reason ) {
360 echo $reason . "\n";
363 echo <<<TEXT
364 Imports images and other media files into the wiki
365 USAGE: php importImages.php [options] <dir>
367 <dir> : Path to the directory containing images to be imported
369 Options:
370 --extensions=<exts> Comma-separated list of allowable extensions, defaults
371 to \$wgFileExtensions.
372 --overwrite Overwrite existing images with the same name (default
373 is to skip them).
374 --limit=<num> Limit the number of images to process. Ignored or
375 skipped images are not counted.
376 --from=<name> Ignore all files until the one with the given name.
377 Useful for resuming aborted imports. <name> should be
378 the file's canonical database form.
379 --skip-dupes Skip images that were already uploaded under a different
380 name (check SHA1).
381 --search-recursively Search recursively for files in subdirectories.
382 --sleep=<sec> Sleep between files. Useful mostly for debugging.
383 --user=<username> Set username of uploader, default 'Maintenance script'.
384 --check-userblock Check if the user got blocked during import.
385 --comment=<text> Set file description, default 'Importing file'.
386 --comment-file=<file> Set description to the content of <file>.
387 --comment-ext=<ext> Causes the description for each file to be loaded from a
388 file with the same name, but the extension <ext>. If a
389 global description is also given, it is appended.
390 --license=<code> Use an optional license template.
391 --dry Dry run, don't import anything.
392 --protect=<protect> Specify the protect value (autoconfirmed,sysop).
393 --summary=<summary> Upload summary, description will be used if not
394 provided.
395 --timestamp=<timestamp> Override upload time/date, all MediaWiki timestamp
396 formats are accepted.
397 --unprotect Unprotects all uploaded images.
398 --source-wiki-url If specified, take User and Comment data for each
399 imported file from this URL. For example,
400 --source-wiki-url="http://en.wikipedia.org/."
402 TEXT;
403 exit( 1 );