tests/parser/editTests.php

   1 <?php
   2
   3 use MediaWiki\Maintenance\Maintenance;
   4 use MediaWiki\Settings\SettingsBuilder;
   5 use MediaWiki\Tests\AnsiTermColorer;
   6 use Wikimedia\Diff\Diff;
   7 use Wikimedia\Diff\UnifiedDiffFormatter;
   8 use Wikimedia\Parsoid\ParserTests\Test as ParserTest;
   9 use Wikimedia\Parsoid\ParserTests\TestFileReader;
  10 use Wikimedia\Parsoid\ParserTests\TestMode as ParserTestMode;
  11 use Wikimedia\ScopedCallback;
  12
  13 require_once __DIR__ . '/../../maintenance/Maintenance.php';
  14
  15 define( 'MW_AUTOLOAD_TEST_CLASSES', true );
  16
  17 /**
  18  * Interactive parser test runner and test file editor
  19  */
  20 class ParserEditTests extends Maintenance {
  21         /** @var int */
  22         private $termWidth;
  23         /** @var TestFileReader[] */
  24         private $testFiles;
  25         /** @var int */
  26         private $testCount;
  27         /** @var TestRecorder */
  28         private $recorder;
  29         /** @var ParserTestRunner */
  30         private $runner;
  31         /** @var int */
  32         private $numExecuted;
  33         /** @var int */
  34         private $numSkipped;
  35         /** @var int */
  36         private $numFailed;
  37         /** @var array */
  38         private $results;
  39         /** @var array */
  40         private $session;
  41
  42         public function __construct() {
  43                 parent::__construct();
  44                 $this->addOption( 'session-data', 'internal option, do not use', false, true );
  45         }
  46
  47         public function finalSetup( SettingsBuilder $settingsBuilder ) {
  48                 // Some methods which are discouraged for normal code throw exceptions unless
  49                 // we declare this is just a test.
  50                 define( 'MW_PARSER_TEST', true );
  51
  52                 parent::finalSetup( $settingsBuilder );
  53                 TestSetup::applyInitialConfig();
  54         }
  55
  56         public function execute() {
  57                 $this->termWidth = $this->getTermSize()[0] - 1;
  58
  59                 $this->recorder = new TestRecorder();
  60                 $this->setupFileData();
  61
  62                 if ( $this->hasOption( 'session-data' ) ) {
  63                         $this->session = json_decode( $this->getOption( 'session-data' ), true );
  64                 } else {
  65                         $this->session = [ 'options' => [] ];
  66                 }
  67                 // @phan-suppress-next-line PhanTypeArraySuspiciousNullable options always set
  68                 $this->runner = new ParserTestRunner( $this->recorder, $this->session['options'] );
  69
  70                 $this->runTests();
  71
  72                 if ( $this->numFailed === 0 ) {
  73                         if ( $this->numSkipped === 0 ) {
  74                                 print "All tests passed!\n";
  75                         } else {
  76                                 print "All tests passed (but skipped {$this->numSkipped})\n";
  77                         }
  78                         return;
  79                 }
  80                 print "{$this->numFailed} test(s) failed.\n";
  81                 $this->showResults();
  82         }
  83
  84         protected function setupFileData() {
  85                 $this->testFiles = [];
  86                 $this->testCount = 0;
  87                 foreach ( ParserTestRunner::getParserTestFiles() as $file ) {
  88                         $fileInfo = TestFileReader::read( $file );
  89                         $this->testFiles[$file] = $fileInfo;
  90                         $this->testCount += count( $fileInfo->testCases );
  91                 }
  92         }
  93
  94         protected function getTestDesc( ParserTest $test ) {
  95                 return $test->testName; // could include mode here too
  96         }
  97
  98         protected function getResultSection( ParserTest $test ) {
  99                 // This used to switch between html and html+tidy, but we
 100                 // got rid of the "notidy" support some time ago.
 101                 // This should probably eventually support html+standalone
 102                 // and html+integrated in a similar way, but for now just
 103                 // walk the legacy HTML fallback set
 104                 $legacyHtmlKeys = [
 105                         'html/php', 'html/*', 'html',
 106                         # deprecated
 107                         'result',
 108                         'html/php+tidy',
 109                         'html/*+tidy',
 110                         'html+tidy',
 111                 ];
 112                 foreach ( $legacyHtmlKeys as $key ) {
 113                         if ( $test->sections[$key] ?? false ) {
 114                                 return $key;
 115                         }
 116                 }
 117                 return 'html';
 118         }
 119
 120         protected function runTests() {
 121                 $teardownGuard = null;
 122                 $teardownGuard = $this->runner->setupDatabase( $teardownGuard );
 123                 $teardownGuard = $this->runner->staticSetup( $teardownGuard );
 124                 $teardownGuard = $this->runner->setupUploads( $teardownGuard );
 125
 126                 print "Running tests...\n";
 127                 $this->results = [];
 128                 $this->numExecuted = 0;
 129                 $this->numSkipped = 0;
 130                 $this->numFailed = 0;
 131                 $mode = new ParserTestMode( 'legacy' );
 132                 foreach ( $this->testFiles as $fileName => $fileInfo ) {
 133                         $this->runner->addArticles( $fileInfo->articles );
 134                         foreach ( $fileInfo->testCases as $testInfo ) {
 135                                 $result = $this->runner->runTest( $testInfo, $mode );
 136                                 if ( $result === false ) {
 137                                         $this->numSkipped++;
 138                                 } elseif ( !$result->isSuccess() ) {
 139                                         $desc = $this->getTestDesc( $testInfo );
 140                                         $this->results[$fileName][$desc] = $result;
 141                                         $this->numFailed++;
 142                                 }
 143                                 $this->numExecuted++;
 144                                 $this->showProgress();
 145                         }
 146                 }
 147                 print "\n";
 148
 149                 ScopedCallback::consume( $teardownGuard );
 150         }
 151
 152         protected function showProgress() {
 153                 $done = $this->numExecuted;
 154                 $total = $this->testCount;
 155                 $width = $this->termWidth - 9;
 156                 $pos = (int)round( $width * $done / $total );
 157                 printf( '│' . str_repeat( '█', $pos ) . str_repeat( '-', $width - $pos ) .
 158                         "│ %5.1f%%\r", $done / $total * 100 );
 159         }
 160
 161         protected function showResults() {
 162                 if ( isset( $this->session['startFile'] ) ) {
 163                         $startFile = $this->session['startFile'];
 164                         $startTest = $this->session['startTest'];
 165                         $foundStart = false;
 166                 } else {
 167                         $startFile = false;
 168                         $startTest = false;
 169                         $foundStart = true;
 170                 }
 171
 172                 $testIndex = 0;
 173                 foreach ( $this->testFiles as $fileName => $fileInfo ) {
 174                         if ( !isset( $this->results[$fileName] ) ) {
 175                                 continue;
 176                         }
 177                         if ( !$foundStart && $startFile !== false && $fileName !== $startFile ) {
 178                                 $testIndex += count( $this->results[$fileName] );
 179                                 continue;
 180                         }
 181                         foreach ( $fileInfo->testCases as $testInfo ) {
 182                                 $desc = $this->getTestDesc( $testInfo );
 183                                 if ( !isset( $this->results[$fileName][$desc] ) ) {
 184                                         continue;
 185                                 }
 186                                 $result = $this->results[$fileName][$desc];
 187                                 $testIndex++;
 188                                 if ( !$foundStart && $startTest !== false ) {
 189                                         if ( $desc !== $startTest ) {
 190                                                 continue;
 191                                         }
 192                                         $foundStart = true;
 193                                 }
 194
 195                                 $this->handleFailure( $testIndex, $testInfo, $result );
 196                         }
 197                 }
 198
 199                 if ( !$foundStart ) {
 200                         print "Could not find the test after a restart, did you rename it?";
 201                         unset( $this->session['startFile'] );
 202                         unset( $this->session['startTest'] );
 203                         // @phan-suppress-next-line PhanPossiblyInfiniteRecursionSameParams
 204                         $this->showResults();
 205                 }
 206                 print "All done\n";
 207         }
 208
 209         protected function heading( $text ) {
 210                 $term = new AnsiTermColorer;
 211                 $heading = "─── $text ";
 212                 $heading .= str_repeat( '─', $this->termWidth - mb_strlen( $heading ) );
 213                 $heading = $term->color( '34' ) . $heading . $term->reset() . "\n";
 214                 return $heading;
 215         }
 216
 217         protected function unifiedDiff( $left, $right ) {
 218                 $fromLines = explode( "\n", $left );
 219                 $toLines = explode( "\n", $right );
 220                 $formatter = new UnifiedDiffFormatter;
 221                 return $formatter->format( new Diff( $fromLines, $toLines ) );
 222         }
 223
 224         protected function handleFailure( $index, $testInfo, $result ) {
 225                 $term = new AnsiTermColorer;
 226                 $div1 = $term->color( '34' ) . str_repeat( '━', $this->termWidth ) .
 227                         $term->reset() . "\n";
 228                 $div2 = $term->color( '34' ) . str_repeat( '─', $this->termWidth ) .
 229                         $term->reset() . "\n";
 230
 231                 $desc = $this->getTestDesc( $testInfo );
 232                 print $div1;
 233                 print "Failure $index/{$this->numFailed}: {$testInfo->filename} line {$testInfo->lineNumStart}\n" .
 234                         "{$desc}\n";
 235
 236                 print $this->heading( 'Input' );
 237                 print "{$testInfo->wikitext}\n";
 238
 239                 print $this->heading( 'Alternating expected/actual output' );
 240                 print $this->alternatingAligned( $result->expected, $result->actual );
 241
 242                 print $this->heading( 'Diff' );
 243
 244                 $dwdiff = $this->dwdiff( $result->expected, $result->actual );
 245                 if ( $dwdiff !== false ) {
 246                         $diff = $dwdiff;
 247                 } else {
 248                         $diff = $this->unifiedDiff( $result->expected, $result->actual );
 249                 }
 250                 print $diff;
 251
 252                 if ( $testInfo->options || $testInfo->config ) {
 253                         print $this->heading( 'Options / Config' );
 254                         if ( $testInfo->options ) {
 255                                 print json_encode( $testInfo->options ) . "\n";
 256                         }
 257                         if ( $testInfo->config ) {
 258                                 print json_encode( $testInfo->config ) . "\n";
 259                         }
 260                 }
 261
 262                 print $div2;
 263                 print "What do you want to do?\n";
 264                 $specs = [
 265                         '[R]eload code and run again',
 266                         '[U]pdate source file, copy actual to expected',
 267                         '[I]gnore' ];
 268
 269                 # XXX originally isSubtest was a way to edit the +tidy vs +untidy
 270                 # portions of the test separately (I believe)
 271                 // @phan-suppress-next-line PhanUndeclaredProperty
 272                 if ( !empty( $testInfo->isSubtest ) ) {
 273                         # FIXME: this is orphan code, will never be true
 274                         $specs[] = 'Delete [s]ubtest';
 275                 }
 276                 $specs[] = '[D]elete test';
 277                 $specs[] = '[Q]uit';
 278
 279                 $options = [];
 280                 foreach ( $specs as $spec ) {
 281                         if ( !preg_match( '/^(.*\[)(.)(\].*)$/', $spec, $m ) ) {
 282                                 throw new LogicException( 'Invalid option spec: ' . $spec );
 283                         }
 284                         print '* ' . $m[1] . $term->color( '35' ) . $m[2] . $term->color( '0' ) . $m[3] . "\n";
 285                         $options[strtoupper( $m[2] )] = true;
 286                 }
 287
 288                 do {
 289                         $response = self::readconsole();
 290                         $cmdResult = false;
 291                         if ( $response === false ) {
 292                                 exit( 0 );
 293                         }
 294
 295                         $response = strtoupper( trim( $response ) );
 296                         if ( !isset( $options[$response] ) ) {
 297                                 print "Invalid response, please enter a single letter from the list above\n";
 298                                 continue;
 299                         }
 300
 301                         switch ( strtoupper( trim( $response ) ) ) {
 302                                 case 'R':
 303                                         $cmdResult = $this->reload( $testInfo );
 304                                         break;
 305                                 case 'U':
 306                                         $cmdResult = $this->update( $testInfo, $result );
 307                                         break;
 308                                 case 'I':
 309                                         return;
 310                                 case 'T':
 311                                         $cmdResult = $this->switchTidy( $testInfo );
 312                                         break;
 313                                 case 'S':
 314                                         $cmdResult = $this->deleteSubtest( $testInfo );
 315                                         break;
 316                                 case 'D':
 317                                         $cmdResult = $this->deleteTest( $testInfo );
 318                                         break;
 319                                 case 'Q':
 320                                         exit( 0 );
 321                         }
 322                 } while ( !$cmdResult );
 323         }
 324
 325         protected function dwdiff( $expected, $actual ) {
 326                 if ( !is_executable( '/usr/bin/dwdiff' ) ) {
 327                         return false;
 328                 }
 329
 330                 $markers = [
 331                         "\n" => '¶',
 332                         ' ' => '·',
 333                         "\t" => '→'
 334                 ];
 335                 $markedExpected = strtr( $expected, $markers );
 336                 $markedActual = strtr( $actual, $markers );
 337                 $diff = $this->unifiedDiff( $markedExpected, $markedActual );
 338
 339                 $tempFile = tmpfile();
 340                 fwrite( $tempFile, $diff );
 341                 fseek( $tempFile, 0 );
 342                 $pipes = [];
 343                 $proc = proc_open( '/usr/bin/dwdiff -Pc --diff-input',
 344                         [ 0 => $tempFile, 1 => [ 'pipe', 'w' ], 2 => STDERR ],
 345                         $pipes );
 346
 347                 if ( !$proc ) {
 348                         return false;
 349                 }
 350
 351                 $result = stream_get_contents( $pipes[1] );
 352                 proc_close( $proc );
 353                 fclose( $tempFile );
 354                 return $result;
 355         }
 356
 357         protected function alternatingAligned( $expectedStr, $actualStr ) {
 358                 $expectedLines = explode( "\n", $expectedStr );
 359                 $actualLines = explode( "\n", $actualStr );
 360                 $maxLines = max( count( $expectedLines ), count( $actualLines ) );
 361                 $result = '';
 362                 for ( $i = 0; $i < $maxLines; $i++ ) {
 363                         if ( $i < count( $expectedLines ) ) {
 364                                 $expectedLine = $expectedLines[$i];
 365                                 $expectedChunks = str_split( $expectedLine, $this->termWidth - 3 );
 366                         } else {
 367                                 $expectedChunks = [];
 368                         }
 369
 370                         if ( $i < count( $actualLines ) ) {
 371                                 $actualLine = $actualLines[$i];
 372                                 $actualChunks = str_split( $actualLine, $this->termWidth - 3 );
 373                         } else {
 374                                 $actualChunks = [];
 375                         }
 376
 377                         $maxChunks = max( count( $expectedChunks ), count( $actualChunks ) );
 378
 379                         for ( $j = 0; $j < $maxChunks; $j++ ) {
 380                                 if ( isset( $expectedChunks[$j] ) ) {
 381                                         $result .= "E: " . $expectedChunks[$j];
 382                                         if ( $j === count( $expectedChunks ) - 1 ) {
 383                                                 $result .= "¶";
 384                                         }
 385                                         $result .= "\n";
 386                                 } else {
 387                                         $result .= "E:\n";
 388                                 }
 389                                 $result .= "\33[4m" . // underline
 390                                         "A: ";
 391                                 if ( isset( $actualChunks[$j] ) ) {
 392                                         $result .= $actualChunks[$j];
 393                                         if ( $j === count( $actualChunks ) - 1 ) {
 394                                                 $result .= "¶";
 395                                         }
 396                                 }
 397                                 $result .= "\33[0m\n"; // reset
 398                         }
 399                 }
 400                 return $result;
 401         }
 402
 403         protected function reload( $testInfo ) {
 404                 global $argv;
 405                 pcntl_exec( PHP_BINARY, [
 406                         $argv[0],
 407                         '--session-data',
 408                         json_encode( [
 409                                 'startFile' => $testInfo->filename,
 410                                 'startTest' => $this->getTestDesc( $testInfo ),
 411                         ] + $this->session ) ] );
 412
 413                 print "pcntl_exec() failed\n";
 414                 return false;
 415         }
 416
 417         protected function findTest( $file, $testInfo ) {
 418                 $initialPart = '';
 419                 for ( $i = 1; $i < $testInfo->lineNumStart; $i++ ) {
 420                         $line = fgets( $file );
 421                         if ( $line === false ) {
 422                                 print "Error reading from file\n";
 423                                 return false;
 424                         }
 425                         $initialPart .= $line;
 426                 }
 427
 428                 $line = fgets( $file );
 429                 if ( !preg_match( '/^!!\s*test/', $line ) ) {
 430                         print "Test has moved, cannot edit\n";
 431                         return false;
 432                 }
 433
 434                 $testPart = $line;
 435
 436                 $desc = fgets( $file );
 437                 if ( trim( $desc ) !== $this->getTestDesc( $testInfo ) ) {
 438                         print "Description does not match, cannot edit\n";
 439                         return false;
 440                 }
 441                 $testPart .= $desc;
 442                 return [ $initialPart, $testPart ];
 443         }
 444
 445         protected function getOutputFileName( $inputFileName ) {
 446                 if ( is_writable( $inputFileName ) ) {
 447                         $outputFileName = $inputFileName;
 448                 } else {
 449                         $outputFileName = wfTempDir() . '/' . basename( $inputFileName );
 450                         print "Cannot write to input file, writing to $outputFileName instead\n";
 451                 }
 452                 return $outputFileName;
 453         }
 454
 455         protected function editTest( $fileName, $deletions, $changes ) {
 456                 $text = file_get_contents( $fileName );
 457                 if ( $text === false ) {
 458                         print "Unable to open test file!";
 459                         return false;
 460                 }
 461                 $result = TestFileEditor::edit( $text, $deletions, $changes,
 462                         static function ( $msg ) {
 463                                 print "$msg\n";
 464                         }
 465                 );
 466                 if ( is_writable( $fileName ) ) {
 467                         file_put_contents( $fileName, $result );
 468                         print "Wrote updated file\n";
 469                 } else {
 470                         print "Cannot write updated file, here is a patch you can paste:\n\n";
 471                         print "--- {$fileName}\n" .
 472                                 "+++ {$fileName}~\n" .
 473                                 $this->unifiedDiff( $text, $result ) .
 474                                 "\n";
 475                 }
 476         }
 477
 478         protected function update( $testInfo, $result ) {
 479                 $resultSection = $this->getResultSection( $testInfo );
 480                 $this->editTest( $testInfo->filename,
 481                         [], // deletions
 482                         [ // changes
 483                                 $testInfo->testName => [
 484                                         $resultSection => [
 485                                                 'op' => 'update',
 486                                                 'value' => $result->actual . "\n"
 487                                         ]
 488                                 ]
 489                         ]
 490                 );
 491                 return false;
 492         }
 493
 494         protected function deleteTest( $testInfo ) {
 495                 $this->editTest( $testInfo->filename,
 496                         [ $testInfo->testName ], // deletions
 497                         [] // changes
 498                 );
 499                 return false;
 500         }
 501
 502         protected function switchTidy( $testInfo ) {
 503                 $resultSection = $this->getResultSection( $testInfo );
 504                 if ( in_array( $resultSection, [ 'html/php' ] ) ) {
 505                         $newSection = 'html/php';
 506                 } elseif ( in_array( $resultSection, [ 'html/*', 'html', 'result' ] ) ) {
 507                         $newSection = 'html';
 508                 } else {
 509                         print "Unrecognised result section name \"$resultSection\"";
 510                         return true;
 511                 }
 512
 513                 $this->editTest( $testInfo->filename,
 514                         [], // deletions
 515                         [ // changes
 516                                 $testInfo->testName => [
 517                                         $resultSection => [
 518                                                 'op' => 'rename',
 519                                                 'value' => $newSection
 520                                         ]
 521                                 ]
 522                         ]
 523                 );
 524                 return false;
 525         }
 526
 527         protected function deleteSubtest( $testInfo ) {
 528                 $resultSection = $this->getResultSection( $testInfo );
 529                 $this->editTest( $testInfo->filename,
 530                         [], // deletions
 531                         [ // changes
 532                                 $testInfo->testName => [
 533                                         $resultSection => [
 534                                                 'op' => 'delete'
 535                                         ]
 536                                 ]
 537                         ]
 538                 );
 539                 return false;
 540         }
 541 }
 542
 543 $maintClass = ParserEditTests::class;
 544 require_once RUN_MAINTENANCE_IF_MAIN;