tests/testHelpers.inc

   1 <?php
   2 /**
   3  * Recording for passing/failing tests.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup Testing
  22  */
  23
  24 /**
  25  * Interface to record parser test results.
  26  *
  27  * The ITestRecorder is a very simple interface to record the result of
  28  * MediaWiki parser tests. One should call start() before running the
  29  * full parser tests and end() once all the tests have been finished.
  30  * After each test, you should use record() to keep track of your tests
  31  * results. Finally, report() is used to generate a summary of your
  32  * test run, one could dump it to the console for human consumption or
  33  * register the result in a database for tracking purposes.
  34  *
  35  * @since 1.22
  36  */
  37 interface ITestRecorder {
  38
  39         /** Called at beginning of the parser test run */
  40         public function start();
  41
  42         /** Called after each test */
  43         public function record( $test, $result );
  44
  45         /** Called before finishing the test run */
  46         public function report();
  47
  48         /** Called at the end of the parser test run */
  49         public function end();
  50
  51 }
  52
  53 class TestRecorder implements ITestRecorder {
  54         var $parent;
  55         var $term;
  56
  57         function __construct( $parent ) {
  58                 $this->parent = $parent;
  59                 $this->term = $parent->term;
  60         }
  61
  62         function start() {
  63                 $this->total = 0;
  64                 $this->success = 0;
  65         }
  66
  67         function record( $test, $result ) {
  68                 $this->total++;
  69                 $this->success += ( $result ? 1 : 0 );
  70         }
  71
  72         function end() {
  73                 // dummy
  74         }
  75
  76         function report() {
  77                 if ( $this->total > 0 ) {
  78                         $this->reportPercentage( $this->success, $this->total );
  79                 } else {
  80                         throw new MWException( "No tests found.\n" );
  81                 }
  82         }
  83
  84         function reportPercentage( $success, $total ) {
  85                 $ratio = wfPercent( 100 * $success / $total );
  86                 print $this->term->color( 1 ) . "Passed $success of $total tests ($ratio)... ";
  87
  88                 if ( $success == $total ) {
  89                         print $this->term->color( 32 ) . "ALL TESTS PASSED!";
  90                 } else {
  91                         $failed = $total - $success;
  92                         print $this->term->color( 31 ) . "$failed tests failed!";
  93                 }
  94
  95                 print $this->term->reset() . "\n";
  96
  97                 return ( $success == $total );
  98         }
  99 }
 100
 101 class DbTestPreviewer extends TestRecorder {
 102         protected $lb; // /< Database load balancer
 103         protected $db; // /< Database connection to the main DB
 104         protected $curRun; // /< run ID number for the current run
 105         protected $prevRun; // /< run ID number for the previous run, if any
 106         protected $results; // /< Result array
 107
 108         /**
 109          * This should be called before the table prefix is changed
 110          */
 111         function __construct( $parent ) {
 112                 parent::__construct( $parent );
 113
 114                 $this->lb = wfGetLBFactory()->newMainLB();
 115                 // This connection will have the wiki's table prefix, not parsertest_
 116                 $this->db = $this->lb->getConnection( DB_MASTER );
 117         }
 118
 119         /**
 120          * Set up result recording; insert a record for the run with the date
 121          * and all that fun stuff
 122          */
 123         function start() {
 124                 parent::start();
 125
 126                 if ( !$this->db->tableExists( 'testrun', __METHOD__ )
 127                         || !$this->db->tableExists( 'testitem', __METHOD__ )
 128                 ) {
 129                         print "WARNING> `testrun` table not found in database.\n";
 130                         $this->prevRun = false;
 131                 } else {
 132                         // We'll make comparisons against the previous run later...
 133                         $this->prevRun = $this->db->selectField( 'testrun', 'MAX(tr_id)' );
 134                 }
 135
 136                 $this->results = array();
 137         }
 138
 139         function record( $test, $result ) {
 140                 parent::record( $test, $result );
 141                 $this->results[$test] = $result;
 142         }
 143
 144         function report() {
 145                 if ( $this->prevRun ) {
 146                         // f = fail, p = pass, n = nonexistent
 147                         // codes show before then after
 148                         $table = array(
 149                                 'fp' => 'previously failing test(s) now PASSING! :)',
 150                                 'pn' => 'previously PASSING test(s) removed o_O',
 151                                 'np' => 'new PASSING test(s) :)',
 152
 153                                 'pf' => 'previously passing test(s) now FAILING! :(',
 154                                 'fn' => 'previously FAILING test(s) removed O_o',
 155                                 'nf' => 'new FAILING test(s) :(',
 156                                 'ff' => 'still FAILING test(s) :(',
 157                         );
 158
 159                         $prevResults = array();
 160
 161                         $res = $this->db->select( 'testitem', array( 'ti_name', 'ti_success' ),
 162                                 array( 'ti_run' => $this->prevRun ), __METHOD__ );
 163
 164                         foreach ( $res as $row ) {
 165                                 if ( !$this->parent->regex
 166                                         || preg_match( "/{$this->parent->regex}/i", $row->ti_name )
 167                                 ) {
 168                                         $prevResults[$row->ti_name] = $row->ti_success;
 169                                 }
 170                         }
 171
 172                         $combined = array_keys( $this->results + $prevResults );
 173
 174                         # Determine breakdown by change type
 175                         $breakdown = array();
 176                         foreach ( $combined as $test ) {
 177                                 if ( !isset( $prevResults[$test] ) ) {
 178                                         $before = 'n';
 179                                 } elseif ( $prevResults[$test] == 1 ) {
 180                                         $before = 'p';
 181                                 } else /* if ( $prevResults[$test] == 0 )*/ {
 182                                         $before = 'f';
 183                                 }
 184
 185                                 if ( !isset( $this->results[$test] ) ) {
 186                                         $after = 'n';
 187                                 } elseif ( $this->results[$test] == 1 ) {
 188                                         $after = 'p';
 189                                 } else /*if ( $this->results[$test] == 0 ) */ {
 190                                         $after = 'f';
 191                                 }
 192
 193                                 $code = $before . $after;
 194
 195                                 if ( isset( $table[$code] ) ) {
 196                                         $breakdown[$code][$test] = $this->getTestStatusInfo( $test, $after );
 197                                 }
 198                         }
 199
 200                         # Write out results
 201                         foreach ( $table as $code => $label ) {
 202                                 if ( !empty( $breakdown[$code] ) ) {
 203                                         $count = count( $breakdown[$code] );
 204                                         printf( "\n%4d %s\n", $count, $label );
 205
 206                                         foreach ( $breakdown[$code] as $differing_test_name => $statusInfo ) {
 207                                                 print "      * $differing_test_name  [$statusInfo]\n";
 208                                         }
 209                                 }
 210                         }
 211                 } else {
 212                         print "No previous test runs to compare against.\n";
 213                 }
 214
 215                 print "\n";
 216                 parent::report();
 217         }
 218
 219         /**
 220          * Returns a string giving information about when a test last had a status change.
 221          * Could help to track down when regressions were introduced, as distinct from tests
 222          * which have never passed (which are more change requests than regressions).
 223          */
 224         private function getTestStatusInfo( $testname, $after ) {
 225                 // If we're looking at a test that has just been removed, then say when it first appeared.
 226                 if ( $after == 'n' ) {
 227                         $changedRun = $this->db->selectField( 'testitem',
 228                                 'MIN(ti_run)',
 229                                 array( 'ti_name' => $testname ),
 230                                 __METHOD__ );
 231                         $appear = $this->db->selectRow( 'testrun',
 232                                 array( 'tr_date', 'tr_mw_version' ),
 233                                 array( 'tr_id' => $changedRun ),
 234                                 __METHOD__ );
 235
 236                         return "First recorded appearance: "
 237                                 . date( "d-M-Y H:i:s", strtotime( $appear->tr_date ) )
 238                                 . ", " . $appear->tr_mw_version;
 239                 }
 240
 241                 // Otherwise, this test has previous recorded results.
 242                 // See when this test last had a different result to what we're seeing now.
 243                 $conds = array(
 244                         'ti_name' => $testname,
 245                         'ti_success' => ( $after == 'f' ? "1" : "0" ) );
 246
 247                 if ( $this->curRun ) {
 248                         $conds[] = "ti_run != " . $this->db->addQuotes( $this->curRun );
 249                 }
 250
 251                 $changedRun = $this->db->selectField( 'testitem', 'MAX(ti_run)', $conds, __METHOD__ );
 252
 253                 // If no record of ever having had a different result.
 254                 if ( is_null( $changedRun ) ) {
 255                         if ( $after == "f" ) {
 256                                 return "Has never passed";
 257                         } else {
 258                                 return "Has never failed";
 259                         }
 260                 }
 261
 262                 // Otherwise, we're looking at a test whose status has changed.
 263                 // (i.e. it used to work, but now doesn't; or used to fail, but is now fixed.)
 264                 // In this situation, give as much info as we can as to when it changed status.
 265                 $pre = $this->db->selectRow( 'testrun',
 266                         array( 'tr_date', 'tr_mw_version' ),
 267                         array( 'tr_id' => $changedRun ),
 268                         __METHOD__ );
 269                 $post = $this->db->selectRow( 'testrun',
 270                         array( 'tr_date', 'tr_mw_version' ),
 271                         array( "tr_id > " . $this->db->addQuotes( $changedRun ) ),
 272                         __METHOD__,
 273                         array( "LIMIT" => 1, "ORDER BY" => 'tr_id' )
 274                 );
 275
 276                 if ( $post ) {
 277                         $postDate = date( "d-M-Y H:i:s", strtotime( $post->tr_date ) ) . ", {$post->tr_mw_version}";
 278                 } else {
 279                         $postDate = 'now';
 280                 }
 281
 282                 return ( $after == "f" ? "Introduced" : "Fixed" ) . " between "
 283                         . date( "d-M-Y H:i:s", strtotime( $pre->tr_date ) ) . ", " . $pre->tr_mw_version
 284                         . " and $postDate";
 285         }
 286
 287         /**
 288          * Commit transaction and clean up for result recording
 289          */
 290         function end() {
 291                 $this->lb->commitMasterChanges();
 292                 $this->lb->closeAll();
 293                 parent::end();
 294         }
 295 }
 296
 297 class DbTestRecorder extends DbTestPreviewer {
 298         var $version;
 299
 300         /**
 301          * Set up result recording; insert a record for the run with the date
 302          * and all that fun stuff
 303          */
 304         function start() {
 305                 $this->db->begin( __METHOD__ );
 306
 307                 if ( !$this->db->tableExists( 'testrun' )
 308                         || !$this->db->tableExists( 'testitem' )
 309                 ) {
 310                         print "WARNING> `testrun` table not found in database. Trying to create table.\n";
 311                         $this->db->sourceFile( $this->db->patchPath( 'patch-testrun.sql' ) );
 312                         echo "OK, resuming.\n";
 313                 }
 314
 315                 parent::start();
 316
 317                 $this->db->insert( 'testrun',
 318                         array(
 319                                 'tr_date' => $this->db->timestamp(),
 320                                 'tr_mw_version' => $this->version,
 321                                 'tr_php_version' => phpversion(),
 322                                 'tr_db_version' => $this->db->getServerVersion(),
 323                                 'tr_uname' => php_uname()
 324                         ),
 325                         __METHOD__ );
 326                 if ( $this->db->getType() === 'postgres' ) {
 327                         $this->curRun = $this->db->currentSequenceValue( 'testrun_id_seq' );
 328                 } else {
 329                         $this->curRun = $this->db->insertId();
 330                 }
 331         }
 332
 333         /**
 334          * Record an individual test item's success or failure to the db
 335          *
 336          * @param $test String
 337          * @param $result Boolean
 338          */
 339         function record( $test, $result ) {
 340                 parent::record( $test, $result );
 341
 342                 $this->db->insert( 'testitem',
 343                         array(
 344                                 'ti_run' => $this->curRun,
 345                                 'ti_name' => $test,
 346                                 'ti_success' => $result ? 1 : 0,
 347                         ),
 348                         __METHOD__ );
 349         }
 350 }
 351
 352 class TestFileIterator implements Iterator {
 353         private $file;
 354         private $fh;
 355         private $parserTest; /* An instance of ParserTest (parserTests.php) or MediaWikiParserTest (phpunit) */
 356         private $index = 0;
 357         private $test;
 358         private $section = null;
 359         /** String|null: current test section being analyzed */
 360         private $sectionData = array();
 361         private $lineNum;
 362         private $eof;
 363
 364         function __construct( $file, $parserTest ) {
 365                 $this->file = $file;
 366                 $this->fh = fopen( $this->file, "rt" );
 367
 368                 if ( !$this->fh ) {
 369                         throw new MWException( "Couldn't open file '$file'\n" );
 370                 }
 371
 372                 $this->parserTest = $parserTest;
 373
 374                 $this->lineNum = $this->index = 0;
 375         }
 376
 377         function rewind() {
 378                 if ( fseek( $this->fh, 0 ) ) {
 379                         throw new MWException( "Couldn't fseek to the start of '$this->file'\n" );
 380                 }
 381
 382                 $this->index = -1;
 383                 $this->lineNum = 0;
 384                 $this->eof = false;
 385                 $this->next();
 386
 387                 return true;
 388         }
 389
 390         function current() {
 391                 return $this->test;
 392         }
 393
 394         function key() {
 395                 return $this->index;
 396         }
 397
 398         function next() {
 399                 if ( $this->readNextTest() ) {
 400                         $this->index++;
 401                         return true;
 402                 } else {
 403                         $this->eof = true;
 404                 }
 405         }
 406
 407         function valid() {
 408                 return $this->eof != true;
 409         }
 410
 411         function readNextTest() {
 412                 $this->clearSection();
 413
 414                 # Create a fake parser tests which never run anything unless
 415                 # asked to do so. This will avoid running hooks for a disabled test
 416                 $delayedParserTest = new DelayedParserTest();
 417
 418                 while ( false !== ( $line = fgets( $this->fh ) ) ) {
 419                         $this->lineNum++;
 420                         $matches = array();
 421
 422                         if ( preg_match( '/^!!\s*(\w+)/', $line, $matches ) ) {
 423                                 $this->section = strtolower( $matches[1] );
 424
 425                                 if ( $this->section == 'endarticle' ) {
 426                                         $this->checkSection( 'text' );
 427                                         $this->checkSection( 'article' );
 428
 429                                         $this->parserTest->addArticle( ParserTest::chomp( $this->sectionData['article'] ), $this->sectionData['text'], $this->lineNum );
 430
 431                                         $this->clearSection();
 432
 433                                         continue;
 434                                 }
 435
 436                                 if ( $this->section == 'endhooks' ) {
 437                                         $this->checkSection( 'hooks' );
 438
 439                                         foreach ( explode( "\n", $this->sectionData['hooks'] ) as $line ) {
 440                                                 $line = trim( $line );
 441
 442                                                 if ( $line ) {
 443                                                         $delayedParserTest->requireHook( $line );
 444                                                 }
 445                                         }
 446
 447                                         $this->clearSection();
 448
 449                                         continue;
 450                                 }
 451
 452                                 if ( $this->section == 'endfunctionhooks' ) {
 453                                         $this->checkSection( 'functionhooks' );
 454
 455                                         foreach ( explode( "\n", $this->sectionData['functionhooks'] ) as $line ) {
 456                                                 $line = trim( $line );
 457
 458                                                 if ( $line ) {
 459                                                         $delayedParserTest->requireFunctionHook( $line );
 460                                                 }
 461                                         }
 462
 463                                         $this->clearSection();
 464
 465                                         continue;
 466                                 }
 467
 468                                 if ( $this->section == 'end' ) {
 469                                         $this->checkSection( 'test' );
 470                                         $this->checkSection( 'input' );
 471                                         $this->checkSection( 'result' );
 472
 473                                         if ( !isset( $this->sectionData['options'] ) ) {
 474                                                 $this->sectionData['options'] = '';
 475                                         }
 476
 477                                         if ( !isset( $this->sectionData['config'] ) ) {
 478                                                 $this->sectionData['config'] = '';
 479                                         }
 480
 481                                         if ( ( ( preg_match( '/\\bdisabled\\b/i', $this->sectionData['options'] ) && !$this->parserTest->runDisabled )
 482                                                 || ( preg_match( '/\\bparsoid\\b/i', $this->sectionData['options'] ) && !$this->parserTest->runParsoid )
 483                                                 || !preg_match( "/" . $this->parserTest->regex . "/i", $this->sectionData['test'] ) )
 484                                         ) {
 485                                                 # disabled test
 486                                                 $this->clearSection();
 487
 488                                                 # Forget any pending hooks call since test is disabled
 489                                                 $delayedParserTest->reset();
 490
 491                                                 continue;
 492                                         }
 493
 494                                         # We are really going to run the test, run pending hooks and hooks function
 495                                         wfDebug( __METHOD__ . " unleashing delayed test for: {$this->sectionData['test']}" );
 496                                         $hooksResult = $delayedParserTest->unleash( $this->parserTest );
 497                                         if ( !$hooksResult ) {
 498                                                 # Some hook reported an issue. Abort.
 499                                                 return false;
 500                                         }
 501
 502                                         $this->test = array(
 503                                                 'test' => ParserTest::chomp( $this->sectionData['test'] ),
 504                                                 'input' => ParserTest::chomp( $this->sectionData['input'] ),
 505                                                 'result' => ParserTest::chomp( $this->sectionData['result'] ),
 506                                                 'options' => ParserTest::chomp( $this->sectionData['options'] ),
 507                                                 'config' => ParserTest::chomp( $this->sectionData['config'] ),
 508                                         );
 509
 510                                         return true;
 511                                 }
 512
 513                                 if ( isset( $this->sectionData[$this->section] ) ) {
 514                                         throw new MWException( "duplicate section '$this->section' at line {$this->lineNum} of $this->file\n" );
 515                                 }
 516
 517                                 $this->sectionData[$this->section] = '';
 518
 519                                 continue;
 520                         }
 521
 522                         if ( $this->section ) {
 523                                 $this->sectionData[$this->section] .= $line;
 524                         }
 525                 }
 526
 527                 return false;
 528         }
 529
 530
 531         /**
 532          * Clear section name and its data
 533          */
 534         private function clearSection() {
 535                 $this->sectionData = array();
 536                 $this->section = null;
 537
 538         }
 539
 540         /**
 541          * Verify the current section data has some value for the given token
 542          * name (first parameter).
 543          * Throw an exception if it is not set, referencing current section
 544          * and adding the current file name and line number
 545          *
 546          * @param $token String: expected token that should have been mentionned before closing this section
 547          */
 548         private function checkSection( $token ) {
 549                 if ( is_null( $this->section ) ) {
 550                         throw new MWException( __METHOD__ . " can not verify a null section!\n" );
 551                 }
 552
 553                 if ( !isset( $this->sectionData[$token] ) ) {
 554                         throw new MWException( sprintf(
 555                                 "'%s' without '%s' at line %s of %s\n",
 556                                 $this->section,
 557                                 $token,
 558                                 $this->lineNum,
 559                                 $this->file
 560                         ) );
 561                 }
 562                 return true;
 563         }
 564 }
 565
 566 /**
 567  * A class to delay execution of a parser test hooks.
 568  */
 569 class DelayedParserTest {
 570
 571         /** Initialized on construction */
 572         private $hooks;
 573         private $fnHooks;
 574
 575         public function __construct() {
 576                 $this->reset();
 577         }
 578
 579         /**
 580          * Init/reset or forgot about the current delayed test.
 581          * Call to this will erase any hooks function that were pending.
 582          */
 583         public function reset() {
 584                 $this->hooks = array();
 585                 $this->fnHooks = array();
 586         }
 587
 588         /**
 589          * Called whenever we actually want to run the hook.
 590          * Should be the case if we found the parserTest is not disabled
 591          */
 592         public function unleash( &$parserTest ) {
 593                 if ( !( $parserTest instanceof ParserTest || $parserTest instanceof NewParserTest )     ) {
 594                         throw new MWException( __METHOD__ . " must be passed an instance of ParserTest or NewParserTest classes\n" );
 595                 }
 596
 597                 # Trigger delayed hooks. Any failure will make us abort
 598                 foreach ( $this->hooks as $hook ) {
 599                         $ret = $parserTest->requireHook( $hook );
 600                         if ( !$ret ) {
 601                                 return false;
 602                         }
 603                 }
 604
 605                 # Trigger delayed function hooks. Any failure will make us abort
 606                 foreach ( $this->fnHooks as $fnHook ) {
 607                         $ret = $parserTest->requireFunctionHook( $fnHook );
 608                         if ( !$ret ) {
 609                                 return false;
 610                         }
 611                 }
 612
 613                 # Delayed execution was successful.
 614                 return true;
 615         }
 616
 617         /**
 618          * Similar to ParserTest object but does not run anything
 619          * Use unleash() to really execute the hook
 620          */
 621         public function requireHook( $hook ) {
 622                 $this->hooks[] = $hook;
 623         }
 624
 625         /**
 626          * Similar to ParserTest object but does not run anything
 627          * Use unleash() to really execute the hook function
 628          */
 629         public function requireFunctionHook( $fnHook ) {
 630                 $this->fnHooks[] = $fnHook;
 631         }
 632
 633 }