maintenance/importImages.php

   1 <?php
   2
   3 /**
   4  * Maintenance script to import one or more images from the local file system into
   5  * the wiki without using the web-based interface.
   6  *
   7  * "Smart import" additions:
   8  * - aim: preserve the essential metadata (user, description) when importing medias from an existing wiki
   9  * - process:
  10  *      - interface with the source wiki, don't use bare files only (see --source-wiki-url).
  11  *      - fetch metadata from source wiki for each file to import.
  12  *      - commit the fetched metadata to the destination wiki while submitting.
  13  *
  14  * This program is free software; you can redistribute it and/or modify
  15  * it under the terms of the GNU General Public License as published by
  16  * the Free Software Foundation; either version 2 of the License, or
  17  * (at your option) any later version.
  18  *
  19  * This program is distributed in the hope that it will be useful,
  20  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  21  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  22  * GNU General Public License for more details.
  23  *
  24  * You should have received a copy of the GNU General Public License along
  25  * with this program; if not, write to the Free Software Foundation, Inc.,
  26  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  27  * http://www.gnu.org/copyleft/gpl.html
  28  *
  29  * @file
  30  * @ingroup Maintenance
  31  * @author Rob Church <robchur@gmail.com>
  32  * @author Mij <mij@bitchx.it>
  33  */
  34
  35 $optionsWithArgs = array( 'extensions', 'comment', 'comment-file', 'comment-ext', 'user', 'license', 'sleep', 'limit', 'from', 'source-wiki-url' );
  36 require_once( dirname( __FILE__ ) . '/commandLine.inc' );
  37 require_once( dirname( __FILE__ ) . '/importImages.inc' );
  38 $processed = $added = $ignored = $skipped = $overwritten = $failed = 0;
  39
  40 echo( "Import Images\n\n" );
  41
  42 # Need a path
  43 if ( count( $args ) == 0 ) {
  44         showUsage();
  45 }
  46
  47 $dir = $args[0];
  48
  49 # Check Protection
  50 if ( isset( $options['protect'] ) && isset( $options['unprotect'] ) ) {
  51         die( "Cannot specify both protect and unprotect.  Only 1 is allowed.\n" );
  52 }
  53
  54 if ( isset( $options['protect'] ) && $options['protect'] == 1 ) {
  55         die( "You must specify a protection option.\n" );
  56 }
  57
  58 # Prepare the list of allowed extensions
  59 global $wgFileExtensions;
  60 $extensions = isset( $options['extensions'] )
  61         ? explode( ',', strtolower( $options['extensions'] ) )
  62         : $wgFileExtensions;
  63
  64 # Search the path provided for candidates for import
  65 $files = findFiles( $dir, $extensions );
  66
  67 # Initialise the user for this operation
  68 $user = isset( $options['user'] )
  69         ? User::newFromName( $options['user'] )
  70         : User::newFromName( 'Maintenance script' );
  71 if ( !$user instanceof User ) {
  72         $user = User::newFromName( 'Maintenance script' );
  73 }
  74 $wgUser = $user;
  75
  76 # Get block check. If a value is given, this specified how often the check is performed
  77 if ( isset( $options['check-userblock'] ) ) {
  78         if ( !$options['check-userblock'] ) {
  79                 $checkUserBlock = 1;
  80         } else {
  81                 $checkUserBlock = (int)$options['check-userblock'];
  82         }
  83 } else {
  84         $checkUserBlock = false;
  85 }
  86
  87 # Get --from
  88 $from = @$options['from'];
  89
  90 # Get sleep time.
  91 $sleep = @$options['sleep'];
  92 if ( $sleep ) {
  93         $sleep = (int)$sleep;
  94 }
  95
  96 # Get limit number
  97 $limit = @$options['limit'];
  98 if ( $limit ) {
  99         $limit = (int)$limit;
 100 }
 101
 102 # Get the upload comment. Provide a default one in case there's no comment given.
 103 $comment = 'Importing image file';
 104
 105 if ( isset( $options['comment-file'] ) ) {
 106         $comment =  file_get_contents( $options['comment-file'] );
 107         if ( $comment === false || $comment === null ) {
 108                 die( "failed to read comment file: {$options['comment-file']}\n" );
 109         }
 110 } elseif ( isset( $options['comment'] ) ) {
 111         $comment =  $options['comment'];
 112 }
 113
 114 $commentExt = isset( $options['comment-ext'] ) ? $options['comment-ext'] : false;
 115
 116 # Get the license specifier
 117 $license = isset( $options['license'] ) ? $options['license'] : '';
 118
 119 # Batch "upload" operation
 120 $count = count( $files );
 121 if ( $count > 0 ) {
 122
 123         foreach ( $files as $file ) {
 124                 $base = wfBaseName( $file );
 125
 126                 # Validate a title
 127                 $title = Title::makeTitleSafe( NS_FILE, $base );
 128                 if ( !is_object( $title ) ) {
 129                         echo( "{$base} could not be imported; a valid title cannot be produced\n" );
 130                         continue;
 131                 }
 132
 133                 if ( $from ) {
 134                         if ( $from == $title->getDBkey() ) {
 135                                 $from = null;
 136                         } else {
 137                                 $ignored++;
 138                                 continue;
 139                         }
 140                 }
 141
 142                 if ( $checkUserBlock && ( ( $processed % $checkUserBlock ) == 0 ) ) {
 143                         $user->clearInstanceCache( 'name' ); // reload from DB!
 144                         if ( $user->isBlocked() ) {
 145                                 echo( $user->getName() . " was blocked! Aborting.\n" );
 146                                 break;
 147                         }
 148                 }
 149
 150                 # Check existence
 151                 $image = wfLocalFile( $title );
 152                 if ( $image->exists() ) {
 153                         if ( isset( $options['overwrite'] ) ) {
 154                                 echo( "{$base} exists, overwriting..." );
 155                                 $svar = 'overwritten';
 156                         } else {
 157                                 echo( "{$base} exists, skipping\n" );
 158                                 $skipped++;
 159                                 continue;
 160                         }
 161                 } else {
 162                         if ( isset( $options['skip-dupes'] ) ) {
 163                                 $repo = $image->getRepo();
 164                                 $sha1 = File::sha1Base36( $file ); # XXX: we end up calculating this again when actually uploading. that sucks.
 165
 166                                 $dupes = $repo->findBySha1( $sha1 );
 167
 168                                 if ( $dupes ) {
 169                                         echo( "{$base} already exists as " . $dupes[0]->getName() . ", skipping\n" );
 170                                         $skipped++;
 171                                         continue;
 172                                 }
 173                         }
 174
 175                         echo( "Importing {$base}..." );
 176                         $svar = 'added';
 177                 }
 178
 179                 if ( isset( $options['source-wiki-url'] ) ) {
 180                         /* find comment text directly from source wiki, through MW's API */
 181                         $real_comment = getFileCommentFromSourceWiki( $options['source-wiki-url'], $base );
 182                         if ( $real_comment === false )
 183                                 $commentText = $comment;
 184                         else
 185                                 $commentText = $real_comment;
 186
 187                         /* find user directly from source wiki, through MW's API */
 188                         $real_user = getFileUserFromSourceWiki( $options['source-wiki-url'], $base );
 189                         if ( $real_user === false ) {
 190                                 $wgUser = $user;
 191                         } else {
 192                                 $wgUser = User::newFromName( $real_user );
 193                                 if ( $wgUser === false ) {
 194                                         # user does not exist in target wiki
 195                                         echo ( "failed: user '$real_user' does not exist in target wiki." );
 196                                         continue;
 197                                 }
 198                         }
 199                 } else {
 200                         # Find comment text
 201                         $commentText = false;
 202
 203                         if ( $commentExt ) {
 204                                 $f = findAuxFile( $file, $commentExt );
 205                                 if ( !$f ) {
 206                                         echo( " No comment file with extension {$commentExt} found for {$file}, using default comment. " );
 207                                 } else {
 208                                         $commentText = file_get_contents( $f );
 209                                         if ( !$commentText ) {
 210                                                 echo( " Failed to load comment file {$f}, using default comment. " );
 211                                         }
 212                                 }
 213                         }
 214
 215                         if ( !$commentText ) {
 216                                 $commentText = $comment;
 217                         }
 218                 }
 219
 220                 # Import the file
 221                 if ( isset( $options['dry'] ) ) {
 222                         echo( " publishing {$file} by '" . $wgUser->getName() . "', comment '$commentText'... " );
 223                 } else {
 224                         $archive = $image->publish( $file );
 225                         if ( !$archive->isGood() ) {
 226                                 echo( "failed. (" .
 227                                         $archive->getWikiText() .
 228                                         ")\n" );
 229                                 $failed++;
 230                                 continue;
 231                         }
 232                 }
 233
 234                 if ( isset( $options['dry'] ) ) {
 235                         echo( "done.\n" );
 236                 } elseif ( $image->recordUpload( $archive->value, $commentText, $license ) ) {
 237                         # We're done!
 238                         echo( "done.\n" );
 239
 240                         $doProtect = false;
 241
 242                         global $wgRestrictionLevels;
 243
 244                         $protectLevel = isset( $options['protect'] ) ? $options['protect'] : null;
 245
 246                         if ( $protectLevel && in_array( $protectLevel, $wgRestrictionLevels ) ) {
 247                                 $doProtect = true;
 248                         }
 249                         if ( isset( $options['unprotect'] ) ) {
 250                                 $protectLevel = '';
 251                                 $doProtect = true;
 252                         }
 253
 254                         if ( $doProtect ) {
 255                                         # Protect the file
 256                                         echo "\nWaiting for slaves...\n";
 257                                         // Wait for slaves.
 258                                         sleep( 2.0 ); # Why this sleep?
 259                                         wfWaitForSlaves();
 260
 261                                         echo( "\nSetting image restrictions ... " );
 262
 263                                         $cascade = false;
 264                                         $restrictions = array();
 265                                         foreach( $title->getRestrictionTypes() as $type ) {
 266                                                 $restrictions[$type] = $protectLevel;
 267                                         }
 268
 269                                         $page = WikiPage::factory( $title );
 270                                         $status = $page->doUpdateRestrictions( $restrictions, array(), $cascade, '', $user );
 271                                         echo( ( $status->isOK() ? 'done' : 'failed' ) . "\n" );
 272                         }
 273
 274                 } else {
 275                         echo( "failed. (at recordUpload stage)\n" );
 276                         $svar = 'failed';
 277                 }
 278
 279                 $$svar++;
 280                 $processed++;
 281
 282                 if ( $limit && $processed >= $limit ) {
 283                         break;
 284                 }
 285
 286                 if ( $sleep ) {
 287                         sleep( $sleep );
 288                 }
 289         }
 290
 291         # Print out some statistics
 292         echo( "\n" );
 293         foreach ( array( 'count' => 'Found', 'limit' => 'Limit', 'ignored' => 'Ignored',
 294                 'added' => 'Added', 'skipped' => 'Skipped', 'overwritten' => 'Overwritten',
 295                 'failed' => 'Failed' ) as $var => $desc ) {
 296                 if ( $$var > 0 )
 297                         echo( "{$desc}: {$$var}\n" );
 298         }
 299
 300 } else {
 301         echo( "No suitable files could be found for import.\n" );
 302 }
 303
 304 exit( 0 );
 305
 306 function showUsage( $reason = false ) {
 307         if ( $reason ) {
 308                 echo( $reason . "\n" );
 309         }
 310
 311         echo <<<TEXT
 312 Imports images and other media files into the wiki
 313 USAGE: php importImages.php [options] <dir>
 314
 315 <dir> : Path to the directory containing images to be imported
 316
 317 Options:
 318 --extensions=<exts>     Comma-separated list of allowable extensions, defaults to \$wgFileExtensions
 319 --overwrite             Overwrite existing images with the same name (default is to skip them)
 320 --limit=<num>           Limit the number of images to process. Ignored or skipped images are not counted.
 321 --from=<name>           Ignore all files until the one with the given name. Useful for resuming
 322                                                 aborted imports. <name> should be the file's canonical database form.
 323 --skip-dupes            Skip images that were already uploaded under a different name (check SHA1)
 324 --sleep=<sec>           Sleep between files. Useful mostly for debugging.
 325 --user=<username>       Set username of uploader, default 'Maintenance script'
 326 --check-userblock       Check if the user got blocked during import.
 327 --comment=<text>        Set upload summary comment, default 'Importing image file'.
 328 --comment-file=<file>   Set upload summary comment the the content of <file>.
 329 --comment-ext=<ext>     Causes the comment for each file to be loaded from a file with the same name
 330                         but the extension <ext>. If a global comment is also given, it is appended.
 331 --license=<code>        Use an optional license template
 332 --dry                   Dry run, don't import anything
 333 --protect=<protect>     Specify the protect value (autoconfirmed,sysop)
 334 --unprotect             Unprotects all uploaded images
 335 --source-wiki-url   if specified, take User and Comment data for each imported file from this URL.
 336                                         For example, --source-wiki-url="http://en.wikipedia.org/"
 337
 338 TEXT;
 339         exit( 1 );
 340 }