Special case opus mime detction

[mediawiki.git] / maintenance / importImages.php

<?php
/**
 * Import one or more images from the local file system into the wiki without
 * using the web-based interface.
 *
 * "Smart import" additions:
 * - aim: preserve the essential metadata (user, description) when importing media
 *   files from an existing wiki.
 * - process:
 *      - interface with the source wiki, don't use bare files only (see --source-wiki-url).
 *      - fetch metadata from source wiki for each file to import.
 *      - commit the fetched metadata to the destination wiki while submitting.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 * http://www.gnu.org/copyleft/gpl.html
 *
 * @file
 * @ingroup Maintenance
 * @author Rob Church <robchur@gmail.com>
 * @author Mij <mij@bitchx.it>
 */

require_once __DIR__ . '/Maintenance.php';

class ImportImages extends Maintenance {

        public function __construct() {
                parent::__construct();

                $this->addDescription( 'Imports images and other media files into the wiki' );
                $this->addArg( 'dir', 'Path to the directory containing images to be imported' );

                $this->addOption( 'extensions',
                        'Comma-separated list of allowable extensions, defaults to $wgFileExtensions',
                        false,
                        true
                );
                $this->addOption( 'overwrite',
                        'Overwrite existing images with the same name (default is to skip them)' );
                $this->addOption( 'limit',
                        'Limit the number of images to process. Ignored or skipped images are not counted',
                        false,
                        true
                );
                $this->addOption( 'from',
                        "Ignore all files until the one with the given name. Useful for resuming aborted "
                                . "imports. The name should be the file's canonical database form.",
                        false,
                        true
                );
                $this->addOption( 'skip-dupes',
                        'Skip images that were already uploaded under a different name (check SHA1)' );
                $this->addOption( 'search-recursively', 'Search recursively for files in subdirectories' );
                $this->addOption( 'sleep',
                        'Sleep between files. Useful mostly for debugging',
                        false,
                        true
                );
                $this->addOption( 'user',
                        "Set username of uploader, default 'Maintenance script'",
                        false,
                        true
                );
                // This parameter can optionally have an argument. If none specified, getOption()
                // returns 1 which is precisely what we need.
                $this->addOption( 'check-userblock', 'Check if the user got blocked during import' );
                $this->addOption( 'comment',
                        "Set file description, default 'Importing file'",
                        false,
                        true
                );
                $this->addOption( 'comment-file',
                        'Set description to the content of this file',
                        false,
                        true
                );
                $this->addOption( 'comment-ext',
                        'Causes the description for each file to be loaded from a file with the same name, but '
                                . 'the extension provided. If a global description is also given, it is appended.',
                        false,
                        true
                );
                $this->addOption( 'summary',
                        'Upload summary, description will be used if not provided',
                        false,
                        true
                );
                $this->addOption( 'license',
                        'Use an optional license template',
                        false,
                        true
                );
                $this->addOption( 'timestamp',
                        'Override upload time/date, all MediaWiki timestamp formats are accepted',
                        false,
                        true
                );
                $this->addOption( 'protect',
                        'Specify the protect value (autoconfirmed,sysop)',
                        false,
                        true
                );
                $this->addOption( 'unprotect', 'Unprotects all uploaded images' );
                $this->addOption( 'source-wiki-url',
                        'If specified, take User and Comment data for each imported file from this URL. '
                                . 'For example, --source-wiki-url="http://en.wikipedia.org/',
                        false,
                        true
                );
                $this->addOption( 'dry', "Dry run, don't import anything" );
        }

        public function execute() {
                global $wgFileExtensions, $wgUser, $wgRestrictionLevels;

                $processed = $added = $ignored = $skipped = $overwritten = $failed = 0;

                $this->output( "Import Images\n\n" );

                $dir = $this->getArg( 0 );

                # Check Protection
                if ( $this->hasOption( 'protect' ) && $this->hasOption( 'unprotect' ) ) {
                        $this->error( "Cannot specify both protect and unprotect.  Only 1 is allowed.\n", 1 );
                }

                if ( $this->hasOption( 'protect' ) && trim( $this->getOption( 'protect' ) ) ) {
                        $this->error( "You must specify a protection option.\n", 1 );
                }

                # Prepare the list of allowed extensions
                $extensions = $this->hasOption( 'extensions' )
                        ? explode( ',', strtolower( $this->getOption( 'extensions' ) ) )
                        : $wgFileExtensions;

                # Search the path provided for candidates for import
                $files = $this->findFiles( $dir, $extensions, $this->hasOption( 'search-recursively' ) );

                # Initialise the user for this operation
                $user = $this->hasOption( 'user' )
                        ? User::newFromName( $this->getOption( 'user' ) )
                        : User::newSystemUser( 'Maintenance script', [ 'steal' => true ] );
                if ( !$user instanceof User ) {
                        $user = User::newSystemUser( 'Maintenance script', [ 'steal' => true ] );
                }
                $wgUser = $user;

                # Get block check. If a value is given, this specified how often the check is performed
                $checkUserBlock = (int)$this->getOption( 'check-userblock' );

                $from = $this->getOption( 'from' );
                $sleep = (int)$this->getOption( 'sleep' );
                $limit = (int)$this->getOption( 'limit' );
                $timestamp = $this->getOption( 'timestamp', false );

                # Get the upload comment. Provide a default one in case there's no comment given.
                $commentFile = $this->getOption( 'comment-file' );
                if ( $commentFile !== null ) {
                        $comment = file_get_contents( $commentFile );
                        if ( $comment === false || $comment === null ) {
                                $this->error( "failed to read comment file: {$commentFile}\n", 1 );
                        }
                } else {
                        $comment = $this->getOption( 'comment', 'Importing file' );
                }
                $commentExt = $this->getOption( 'comment-ext' );
                $summary = $this->getOption( 'summary', '' );

                $license = $this->getOption( 'license', '' );

                $sourceWikiUrl = $this->getOption( 'source-wiki-url' );

                # Batch "upload" operation
                $count = count( $files );
                if ( $count > 0 ) {

                        foreach ( $files as $file ) {
                                $base = UtfNormal\Validator::cleanUp( wfBaseName( $file ) );

                                # Validate a title
                                $title = Title::makeTitleSafe( NS_FILE, $base );
                                if ( !is_object( $title ) ) {
                                        $this->output(
                                                "{$base} could not be imported; a valid title cannot be produced\n" );
                                        continue;
                                }

                                if ( $from ) {
                                        if ( $from == $title->getDBkey() ) {
                                                $from = null;
                                        } else {
                                                $ignored++;
                                                continue;
                                        }
                                }

                                if ( $checkUserBlock && ( ( $processed % $checkUserBlock ) == 0 ) ) {
                                        $user->clearInstanceCache( 'name' ); // reload from DB!
                                        if ( $user->isBlocked() ) {
                                                $this->output( $user->getName() . " was blocked! Aborting.\n" );
                                                break;
                                        }
                                }

                                # Check existence
                                $image = wfLocalFile( $title );
                                if ( $image->exists() ) {
                                        if ( $this->hasOption( 'overwrite' ) ) {
                                                $this->output( "{$base} exists, overwriting..." );
                                                $svar = 'overwritten';
                                        } else {
                                                $this->output( "{$base} exists, skipping\n" );
                                                $skipped++;
                                                continue;
                                        }
                                } else {
                                        if ( $this->hasOption( 'skip-dupes' ) ) {
                                                $repo = $image->getRepo();
                                                # XXX: we end up calculating this again when actually uploading. that sucks.
                                                $sha1 = FSFile::getSha1Base36FromPath( $file );

                                                $dupes = $repo->findBySha1( $sha1 );

                                                if ( $dupes ) {
                                                        $this->output(
                                                                "{$base} already exists as {$dupes[0]->getName()}, skipping\n" );
                                                        $skipped++;
                                                        continue;
                                                }
                                        }

                                        $this->output( "Importing {$base}..." );
                                        $svar = 'added';
                                }

                                if ( $sourceWikiUrl ) {
                                        /* find comment text directly from source wiki, through MW's API */
                                        $real_comment = $this->getFileCommentFromSourceWiki( $sourceWikiUrl, $base );
                                        if ( $real_comment === false ) {
                                                $commentText = $comment;
                                        } else {
                                                $commentText = $real_comment;
                                        }

                                        /* find user directly from source wiki, through MW's API */
                                        $real_user = $this->getFileUserFromSourceWiki( $sourceWikiUrl, $base );
                                        if ( $real_user === false ) {
                                                $wgUser = $user;
                                        } else {
                                                $wgUser = User::newFromName( $real_user );
                                                if ( $wgUser === false ) {
                                                        # user does not exist in target wiki
                                                        $this->output(
                                                                "failed: user '$real_user' does not exist in target wiki." );
                                                        continue;
                                                }
                                        }
                                } else {
                                        # Find comment text
                                        $commentText = false;

                                        if ( $commentExt ) {
                                                $f = $this->findAuxFile( $file, $commentExt );
                                                if ( !$f ) {
                                                        $this->output( " No comment file with extension {$commentExt} found "
                                                                 . "for {$file}, using default comment. " );
                                                } else {
                                                        $commentText = file_get_contents( $f );
                                                        if ( !$commentText ) {
                                                                $this->output(
                                                                        " Failed to load comment file {$f}, using default comment. " );
                                                        }
                                                }
                                        }

                                        if ( !$commentText ) {
                                                $commentText = $comment;
                                        }
                                }

                                # Import the file
                                if ( $this->hasOption( 'dry' ) ) {
                                        $this->output(
                                                " publishing {$file} by '{$wgUser->getName()}', comment '$commentText'... "
                                        );
                                } else {
                                        $mwProps = new MWFileProps( MimeMagic::singleton() );
                                        $props = $mwProps->getPropsFromPath( $file, true );
                                        $flags = 0;
                                        $publishOptions = [];
                                        $handler = MediaHandler::getHandler( $props['mime'] );
                                        if ( $handler ) {
                                                $publishOptions['headers'] = $handler->getStreamHeaders( $props['metadata'] );
                                        } else {
                                                $publishOptions['headers'] = [];
                                        }
                                        $archive = $image->publish( $file, $flags, $publishOptions );
                                        if ( !$archive->isGood() ) {
                                                $this->output( "failed. (" .
                                                         $archive->getWikiText( false, false, 'en' ) .
                                                         ")\n" );
                                                $failed++;
                                                continue;
                                        }
                                }

                                $commentText = SpecialUpload::getInitialPageText( $commentText, $license );
                                if ( !$this->hasOption( 'summary' ) ) {
                                        $summary = $commentText;
                                }

                                if ( $this->hasOption( 'dry' ) ) {
                                        $this->output( "done.\n" );
                                } elseif ( $image->recordUpload2(
                                        $archive->value,
                                        $summary,
                                        $commentText,
                                        $props,
                                        $timestamp
                                ) ) {
                                        # We're done!
                                        $this->output( "done.\n" );

                                        $doProtect = false;

                                        $protectLevel = $this->getOption( 'protect' );

                                        if ( $protectLevel && in_array( $protectLevel, $wgRestrictionLevels ) ) {
                                                $doProtect = true;
                                        }
                                        if ( $this->hasOption( 'unprotect' ) ) {
                                                $protectLevel = '';
                                                $doProtect = true;
                                        }

                                        if ( $doProtect ) {
                                                # Protect the file
                                                $this->output( "\nWaiting for replica DBs...\n" );
                                                // Wait for replica DBs.
                                                sleep( 2.0 ); # Why this sleep?
                                                wfWaitForSlaves();

                                                $this->output( "\nSetting image restrictions ... " );

                                                $cascade = false;
                                                $restrictions = [];
                                                foreach ( $title->getRestrictionTypes() as $type ) {
                                                        $restrictions[$type] = $protectLevel;
                                                }

                                                $page = WikiPage::factory( $title );
                                                $status = $page->doUpdateRestrictions( $restrictions, [], $cascade, '', $user );
                                                $this->output( ( $status->isOK() ? 'done' : 'failed' ) . "\n" );
                                        }
                                } else {
                                        $this->output( "failed. (at recordUpload stage)\n" );
                                        $svar = 'failed';
                                }

                                $$svar++;
                                $processed++;

                                if ( $limit && $processed >= $limit ) {
                                        break;
                                }

                                if ( $sleep ) {
                                        sleep( $sleep );
                                }
                        }

                        # Print out some statistics
                        $this->output( "\n" );
                        foreach (
                                [
                                        'count' => 'Found',
                                        'limit' => 'Limit',
                                        'ignored' => 'Ignored',
                                        'added' => 'Added',
                                        'skipped' => 'Skipped',
                                        'overwritten' => 'Overwritten',
                                        'failed' => 'Failed'
                                ] as $var => $desc
                        ) {
                                if ( $$var > 0 ) {
                                        $this->output( "{$desc}: {$$var}\n" );
                                }
                        }
                } else {
                        $this->output( "No suitable files could be found for import.\n" );
                }
        }

        /**
         * Search a directory for files with one of a set of extensions
         *
         * @param string $dir Path to directory to search
         * @param array $exts Array of extensions to search for
         * @param bool $recurse Search subdirectories recursively
         * @return array|bool Array of filenames on success, or false on failure
         */
        private function findFiles( $dir, $exts, $recurse = false ) {
                if ( is_dir( $dir ) ) {
                        $dhl = opendir( $dir );
                        if ( $dhl ) {
                                $files = [];
                                while ( ( $file = readdir( $dhl ) ) !== false ) {
                                        if ( is_file( $dir . '/' . $file ) ) {
                                                list( /* $name */, $ext ) = $this->splitFilename( $dir . '/' . $file );
                                                if ( array_search( strtolower( $ext ), $exts ) !== false ) {
                                                        $files[] = $dir . '/' . $file;
                                                }
                                        } elseif ( $recurse && is_dir( $dir . '/' . $file ) && $file !== '..' && $file !== '.' ) {
                                                $files = array_merge( $files, $this->findFiles( $dir . '/' . $file, $exts, true ) );
                                        }
                                }

                                return $files;
                        } else {
                                return [];
                        }
                } else {
                        return [];
                }
        }

        /**
         * Split a filename into filename and extension
         *
         * @param string $filename Filename
         * @return array
         */
        private function splitFilename( $filename ) {
                $parts = explode( '.', $filename );
                $ext = $parts[count( $parts ) - 1];
                unset( $parts[count( $parts ) - 1] );
                $fname = implode( '.', $parts );

                return [ $fname, $ext ];
        }

        /**
         * Find an auxilliary file with the given extension, matching
         * the give base file path. $maxStrip determines how many extensions
         * may be stripped from the original file name before appending the
         * new extension. For example, with $maxStrip = 1 (the default),
         * file files acme.foo.bar.txt and acme.foo.txt would be auxilliary
         * files for acme.foo.bar and the extension ".txt". With $maxStrip = 2,
         * acme.txt would also be acceptable.
         *
         * @param string $file Base path
         * @param string $auxExtension The extension to be appended to the base path
         * @param int $maxStrip The maximum number of extensions to strip from the base path (default: 1)
         * @return string|bool
         */
        private function findAuxFile( $file, $auxExtension, $maxStrip = 1 ) {
                if ( strpos( $auxExtension, '.' ) !== 0 ) {
                        $auxExtension = '.' . $auxExtension;
                }

                $d = dirname( $file );
                $n = basename( $file );

                while ( $maxStrip >= 0 ) {
                        $f = $d . '/' . $n . $auxExtension;

                        if ( file_exists( $f ) ) {
                                return $f;
                        }

                        $idx = strrpos( $n, '.' );
                        if ( !$idx ) {
                                break;
                        }

                        $n = substr( $n, 0, $idx );
                        $maxStrip -= 1;
                }

                return false;
        }

        # @todo FIXME: Access the api in a saner way and performing just one query
        # (preferably batching files too).
        private function getFileCommentFromSourceWiki( $wiki_host, $file ) {
                $url = $wiki_host . '/api.php?action=query&format=xml&titles=File:'
                           . rawurlencode( $file ) . '&prop=imageinfo&&iiprop=comment';
                $body = Http::get( $url, [], __METHOD__ );
                if ( preg_match( '#<ii comment="([^"]*)" />#', $body, $matches ) == 0 ) {
                        return false;
                }

                return html_entity_decode( $matches[1] );
        }

        private function getFileUserFromSourceWiki( $wiki_host, $file ) {
                $url = $wiki_host . '/api.php?action=query&format=xml&titles=File:'
                           . rawurlencode( $file ) . '&prop=imageinfo&&iiprop=user';
                $body = Http::get( $url, [], __METHOD__ );
                if ( preg_match( '#<ii user="([^"]*)" />#', $body, $matches ) == 0 ) {
                        return false;
                }

                return html_entity_decode( $matches[1] );
        }

}

$maintClass = 'ImportImages';
require_once RUN_MAINTENANCE_IF_MAIN;