tests/parser/parserTests.php

   1 <?php
   2 /**
   3  * MediaWiki parser test suite
   4  *
   5  * Copyright © 2004 Brooke Vibber <bvibber@wikimedia.org>
   6  * https://www.mediawiki.org/
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License as published by
  10  * the Free Software Foundation; either version 2 of the License, or
  11  * (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License along
  19  * with this program; if not, write to the Free Software Foundation, Inc.,
  20  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  21  * http://www.gnu.org/copyleft/gpl.html
  22  *
  23  * @file
  24  * @ingroup Testing
  25  */
  26
  27 require_once __DIR__ . '/../../maintenance/Maintenance.php';
  28
  29 use MediaWiki\Maintenance\Maintenance;
  30 use MediaWiki\MediaWikiServices;
  31 use MediaWiki\Settings\SettingsBuilder;
  32 use MediaWiki\Specials\SpecialVersion;
  33 use MediaWiki\Tests\AnsiTermColorer;
  34 use MediaWiki\Tests\DummyTermColorer;
  35 use Wikimedia\Parsoid\Utils\ScriptUtils;
  36
  37 define( 'MW_AUTOLOAD_TEST_CLASSES', true );
  38 define( 'MW_PARSER_TEST', true );
  39
  40 class ParserTestsMaintenance extends Maintenance {
  41
  42         public function __construct() {
  43                 parent::__construct();
  44                 $this->addDescription( 'Run parser tests' );
  45
  46                 $this->addOption( 'quick', 'Suppress diff output of failed tests' );
  47                 $this->addOption( 'quiet', 'Suppress notification of passed tests (shows only failed tests)' );
  48                 $this->addOption( 'show-output', 'Show expected and actual output' );
  49                 $this->addOption( 'color', '[=yes|no] Override terminal detection and force ' .
  50                         'color output on or off. Use wgCommandLineDarkBg = true; if your term is dark',
  51                         false, true );
  52                 $this->addOption( 'regex', 'Only run tests whose descriptions which match given regex',
  53                         false, true );
  54                 $this->addOption( 'filter', 'Alias for --regex', false, true );
  55                 $this->addOption( 'file', 'Run test cases from a custom file instead of parserTests.txt',
  56                         false, true, false, true );
  57                 $this->addOption( 'dir', 'Run test cases for all *.txt files in a directory',
  58                         false, true, false, true );
  59                 $this->addOption( 'record', 'Record tests in database' );
  60                 $this->addOption( 'compare', 'Compare with recorded results, without updating the database.' );
  61                 $this->addOption( 'setversion', 'When using --record, set the version string to use (useful' .
  62                         'with "git rev-parse HEAD" to get the exact revision)',
  63                         false, true );
  64                 $this->addOption( 'keep-uploads', 'Re-use the same upload directory for each ' .
  65                         'test, don\'t delete it' );
  66                 $this->addOption( 'file-backend', 'Use the file backend with the given name,' .
  67                         'and upload files to it, instead of creating a mock file backend.', false, true );
  68                 $this->addOption( 'upload-dir', 'Specify the upload directory to use. Useful in ' .
  69                         'conjunction with --keep-uploads. Causes a real (non-mock) file backend to ' .
  70                         'be used.', false, true );
  71                 $this->addOption( 'run-disabled', 'run disabled tests' );
  72                 $this->addOption( 'disable-save-parse', 'Don\'t run the parser when ' .
  73                         'inserting articles into the database' );
  74                 $this->addOption( 'dwdiff', 'Use dwdiff to display diff output' );
  75                 $this->addOption( 'mark-ws', 'Mark whitespace in diffs by replacing it with symbols' );
  76                 $this->addOption( 'norm', 'Apply a comma-separated list of normalization functions to ' .
  77                         'both the expected and actual output in order to resolve ' .
  78                         'irrelevant differences. The accepted normalization functions ' .
  79                         'are: removeTbody to remove <tbody> tags; and trimWhitespace ' .
  80                         'to trim whitespace from the start and end of text nodes.',
  81                         false, true );
  82                 $this->addOption( 'wt2html', 'Parsoid: Wikitext -> HTML' );
  83                 $this->addOption( 'wt2wt',
  84                         'Parsoid Roundtrip testing: Wikitext -> HTML(DOM) -> Wikitext' );
  85                 $this->addOption( 'html2wt', 'Parsoid: HTML -> Wikitext' );
  86                 $this->addOption( 'numchanges',
  87                         'Max different selser edit tests to generate from the Parsoid DOM' );
  88                 $this->addOption( 'html2html',
  89                         'Parsoid Roundtrip testing: HTML -> Wikitext -> HTML' );
  90                 $this->addOption( 'selser',
  91                         'Parsoid Roundtrip testing: Wikitext -> DOM(HTML) -> Wikitext (with selective serialization). ' .
  92                                         'Set to "noauto" to just run the tests with manual selser changes.',
  93                         false, true );
  94                 $this->addOption( 'changetree',
  95                         'Changes to apply to Parsoid HTML to generate new HTML to be serialized (use with selser)',
  96                         false, true );
  97                 $this->addOption( 'parsoid', 'Run Parsoid tests' );
  98                 $this->addOption( 'trace', 'Use --trace=help for supported options (Parsoid only)', false, true );
  99                 $this->addOption( 'dump', 'Use --dump=help for supported options (Parsoid only)', false, true );
 100                 $this->addOption( 'updateKnownFailures', 'Update knownFailures.json with failing tests' );
 101                 $this->addOption( 'knownFailures',
 102                         'Compare against known failures (default: true). If false, ignores knownFailures.json file',
 103                         false, true );
 104                 $this->addOption( 'update-tests',
 105                         'Update parserTests.txt with results from wt2html fails.  Note that editTests.php exists ' .
 106                                 'for finer grained editing of tests.' );
 107         }
 108
 109         public function finalSetup( SettingsBuilder $settingsBuilder ) {
 110                 // Some methods which are discouraged for normal code throw exceptions unless
 111                 // we declare this is just a test.
 112                 define( 'MW_PARSER_TEST', true );
 113
 114                 parent::finalSetup( $settingsBuilder );
 115                 TestSetup::applyInitialConfig();
 116         }
 117
 118         public function execute() {
 119                 global $wgDBtype;
 120
 121                 // Cases of weird db corruption were encountered when running tests on earlyish
 122                 // versions of SQLite
 123                 if ( $wgDBtype == 'sqlite' ) {
 124                         $dbw = MediaWikiServices::getInstance()->getConnectionProvider()->getPrimaryDatabase();
 125                         $version = $dbw->getServerVersion();
 126                         if ( version_compare( $version, '3.6' ) < 0 ) {
 127                                 die( "Parser tests require SQLite version 3.6 or later, you have $version\n" );
 128                         }
 129                 }
 130
 131                 // Print out software version to assist with locating regressions
 132                 $version = SpecialVersion::getVersion( 'nodb' );
 133                 echo "This is MediaWiki version {$version}.\n\n";
 134
 135                 // Only colorize output if stdout is a terminal.
 136                 $color = !wfIsWindows() && Maintenance::posix_isatty( 1 );
 137
 138                 if ( $this->hasOption( 'color' ) ) {
 139                         switch ( $this->getOption( 'color' ) ) {
 140                                 case 'no':
 141                                         $color = false;
 142                                         break;
 143                                 case 'yes':
 144                                 default:
 145                                         $color = true;
 146                                         break;
 147                         }
 148                 }
 149
 150                 $record = $this->hasOption( 'record' );
 151                 $compare = $this->hasOption( 'compare' );
 152
 153                 $regex = $this->getOption( 'filter', $this->getOption( 'regex', false ) );
 154                 if ( $regex !== false ) {
 155                         $regex = "/$regex/i";
 156
 157                         if ( $record ) {
 158                                 echo "Warning: --record cannot be used with --regex, disabling --record\n";
 159                                 $record = false;
 160                         }
 161                 }
 162
 163                 $term = $color
 164                         ? new AnsiTermColorer()
 165                         : new DummyTermColorer();
 166
 167                 $recorder = new MultiTestRecorder;
 168
 169                 $recorder->addRecorder( new ParserTestPrinter(
 170                         $term,
 171                         [
 172                                 'showDiffs' => !$this->hasOption( 'quick' ),
 173                                 'showProgress' => !$this->hasOption( 'quiet' ),
 174                                 'showFailure' => !$this->hasOption( 'quiet' )
 175                                                 || ( !$record && !$compare ), // redundant output
 176                                 'showOutput' => $this->hasOption( 'show-output' ),
 177                                 'useDwdiff' => $this->hasOption( 'dwdiff' ),
 178                                 'markWhitespace' => $this->hasOption( 'mark-ws' ),
 179                         ]
 180                 ) );
 181
 182                 $traceFlags = array_fill_keys( explode( ',', $this->getOption( 'trace', '' ) ), true );
 183                 $dumpFlags = array_fill_keys( explode( ',', $this->getOption( 'dump', '' ) ), true );
 184                 if ( $traceFlags['help'] ?? false ) {
 185                         print "-------------- PARSOID ONLY --------------\n";
 186                         print ScriptUtils::traceUsageHelp();
 187                         exit( 1 );
 188                 }
 189                 if ( $dumpFlags['help'] ?? false ) {
 190                         print "-------------- PARSOID ONLY --------------\n";
 191                         print ScriptUtils::dumpUsageHelp();
 192                         exit( 1 );
 193                 }
 194
 195                 $recorderLB = false;
 196                 if ( $record || $compare ) {
 197                         // Make an untracked DB_PRIMARY connection (wiki's table prefix, not parsertest_)
 198                         $recorderLB = $this->getServiceContainer()->getDBLoadBalancerFactory()->newMainLB();
 199                         $recorderDB = $recorderLB->getMaintenanceConnectionRef( DB_PRIMARY );
 200                         // Add recorder before previewer because recorder will create the
 201                         // DB table if it doesn't exist
 202                         if ( $record ) {
 203                                 $recorder->addRecorder( new DbTestRecorder( $recorderDB ) );
 204                         }
 205                         $recorder->addRecorder( new DbTestPreviewer(
 206                                 $recorderDB,
 207                                 static function ( $name ) use ( $regex ) {
 208                                         // Filter reports of old tests by the filter regex
 209                                         return $regex === false || (bool)preg_match( $regex, $name );
 210                                 } ) );
 211                 }
 212
 213                 // Default parser tests and any set from extensions or local config
 214                 $dirs = $this->getOption( 'dir', [] );
 215                 $files = $this->getOption( 'file', ParserTestRunner::getParserTestFiles( $dirs ) );
 216                 $norm = $this->hasOption( 'norm' ) ? explode( ',', $this->getOption( 'norm' ) ) : [];
 217
 218                 $selserOpt = $this->getOption( 'selser', false ); /* can also be 'noauto' */
 219                 if ( $selserOpt !== 'noauto' ) {
 220                         $selserOpt = ScriptUtils::booleanOption( $selserOpt );
 221                 }
 222                 $tester = new ParserTestRunner( $recorder, [
 223                         'norm' => $norm,
 224                         'regex' => $regex,
 225                         'keep-uploads' => $this->hasOption( 'keep-uploads' ),
 226                         'run-disabled' => $this->hasOption( 'run-disabled' ),
 227                         'disable-save-parse' => $this->hasOption( 'disable-save-parse' ),
 228                         'file-backend' => $this->getOption( 'file-backend' ),
 229                         'upload-dir' => $this->getOption( 'upload-dir' ),
 230                         // Passing a parsoid-specific option implies --parsoid
 231                         'parsoid' => (
 232                                 $this->hasOption( 'parsoid' ) ||
 233                                 $this->hasOption( 'wt2html' ) ||
 234                                 $this->hasOption( 'wt2wt' ) ||
 235                                 $this->hasOption( 'html2wt' ) ||
 236                                 $this->hasOption( 'html2html' ) ||
 237                                 $selserOpt ),
 238                         'wt2html' => $this->hasOption( 'wt2html' ),
 239                         'wt2wt' => $this->hasOption( 'wt2wt' ),
 240                         'html2wt' => $this->hasOption( 'html2wt' ),
 241                         'html2html' => $this->hasOption( 'html2html' ),
 242                         'numchanges' => $this->getOption( 'numchanges', 20 ),
 243                         'selser' => $selserOpt,
 244                         'changetree' => json_decode( $this->getOption( 'changetree', '' ), true ),
 245                         'knownFailures' => ScriptUtils::booleanOption( $this->getOption( 'knownFailures', true ) ),
 246                         'updateKnownFailures' => $this->hasOption( 'updateKnownFailures' ),
 247                         'traceFlags' => $traceFlags,
 248                         'dumpFlags' => $dumpFlags,
 249                         'update-tests' => $this->hasOption( 'update-tests' ),
 250                 ] );
 251
 252                 $ok = $tester->runTestsFromFiles( $files );
 253                 if ( $recorderLB ) {
 254                         $recorderLB->closeAll( __METHOD__ );
 255                 }
 256                 if ( $tester->unexpectedTestPasses ) {
 257                         $recorder->warning( "There were some unexpected passing tests. " .
 258                                 "Please rerun with --updateKnownFailures option." );
 259                         $ok = false;
 260                 }
 261                 if ( !$ok ) {
 262                         exit( 1 );
 263                 }
 264         }
 265 }
 266
 267 $maintClass = ParserTestsMaintenance::class;
 268 require_once RUN_MAINTENANCE_IF_MAIN;