tests/testHelpers.inc

   1 <?php
   2
   3 class TestRecorder {
   4         var $parent;
   5         var $term;
   6
   7         function __construct( $parent ) {
   8                 $this->parent = $parent;
   9                 $this->term = $parent->term;
  10         }
  11
  12         function start() {
  13                 $this->total = 0;
  14                 $this->success = 0;
  15         }
  16
  17         function record( $test, $result ) {
  18                 $this->total++;
  19                 $this->success += ( $result ? 1 : 0 );
  20         }
  21
  22         function end() {
  23                 // dummy
  24         }
  25
  26         function report() {
  27                 if ( $this->total > 0 ) {
  28                         $this->reportPercentage( $this->success, $this->total );
  29                 } else {
  30                         throw new MWException( "No tests found.\n" );
  31                 }
  32         }
  33
  34         function reportPercentage( $success, $total ) {
  35                 $ratio = wfPercent( 100 * $success / $total );
  36                 print $this->term->color( 1 ) . "Passed $success of $total tests ($ratio)... ";
  37
  38                 if ( $success == $total ) {
  39                         print $this->term->color( 32 ) . "ALL TESTS PASSED!";
  40                 } else {
  41                         $failed = $total - $success ;
  42                         print $this->term->color( 31 ) . "$failed tests failed!";
  43                 }
  44
  45                 print $this->term->reset() . "\n";
  46
  47                 return ( $success == $total );
  48         }
  49 }
  50
  51 class DbTestPreviewer extends TestRecorder  {
  52         protected $lb;      // /< Database load balancer
  53         protected $db;      // /< Database connection to the main DB
  54         protected $curRun;  // /< run ID number for the current run
  55         protected $prevRun; // /< run ID number for the previous run, if any
  56         protected $results; // /< Result array
  57
  58         /**
  59          * This should be called before the table prefix is changed
  60          */
  61         function __construct( $parent ) {
  62                 parent::__construct( $parent );
  63
  64                 $this->lb = wfGetLBFactory()->newMainLB();
  65                 // This connection will have the wiki's table prefix, not parsertest_
  66                 $this->db = $this->lb->getConnection( DB_MASTER );
  67         }
  68
  69         /**
  70          * Set up result recording; insert a record for the run with the date
  71          * and all that fun stuff
  72          */
  73         function start() {
  74                 parent::start();
  75
  76                 if ( ! $this->db->tableExists( 'testrun', __METHOD__ )
  77                         || ! $this->db->tableExists( 'testitem', __METHOD__ ) )
  78                 {
  79                         print "WARNING> `testrun` table not found in database.\n";
  80                         $this->prevRun = false;
  81                 } else {
  82                         // We'll make comparisons against the previous run later...
  83                         $this->prevRun = $this->db->selectField( 'testrun', 'MAX(tr_id)' );
  84                 }
  85
  86                 $this->results = array();
  87         }
  88
  89         function record( $test, $result ) {
  90                 parent::record( $test, $result );
  91                 $this->results[$test] = $result;
  92         }
  93
  94         function report() {
  95                 if ( $this->prevRun ) {
  96                         // f = fail, p = pass, n = nonexistent
  97                         // codes show before then after
  98                         $table = array(
  99                                 'fp' => 'previously failing test(s) now PASSING! :)',
 100                                 'pn' => 'previously PASSING test(s) removed o_O',
 101                                 'np' => 'new PASSING test(s) :)',
 102
 103                                 'pf' => 'previously passing test(s) now FAILING! :(',
 104                                 'fn' => 'previously FAILING test(s) removed O_o',
 105                                 'nf' => 'new FAILING test(s) :(',
 106                                 'ff' => 'still FAILING test(s) :(',
 107                         );
 108
 109                         $prevResults = array();
 110
 111                         $res = $this->db->select( 'testitem', array( 'ti_name', 'ti_success' ),
 112                                 array( 'ti_run' => $this->prevRun ), __METHOD__ );
 113
 114                         foreach ( $res as $row ) {
 115                                 if ( !$this->parent->regex
 116                                         || preg_match( "/{$this->parent->regex}/i", $row->ti_name ) )
 117                                 {
 118                                         $prevResults[$row->ti_name] = $row->ti_success;
 119                                 }
 120                         }
 121
 122                         $combined = array_keys( $this->results + $prevResults );
 123
 124                         # Determine breakdown by change type
 125                         $breakdown = array();
 126                         foreach ( $combined as $test ) {
 127                                 if ( !isset( $prevResults[$test] ) ) {
 128                                         $before = 'n';
 129                                 } elseif ( $prevResults[$test] == 1 ) {
 130                                         $before = 'p';
 131                                 } else /* if ( $prevResults[$test] == 0 )*/ {
 132                                         $before = 'f';
 133                                 }
 134
 135                                 if ( !isset( $this->results[$test] ) ) {
 136                                         $after = 'n';
 137                                 } elseif ( $this->results[$test] == 1 ) {
 138                                         $after = 'p';
 139                                 } else /*if ( $this->results[$test] == 0 ) */ {
 140                                         $after = 'f';
 141                                 }
 142
 143                                 $code = $before . $after;
 144
 145                                 if ( isset( $table[$code] ) ) {
 146                                         $breakdown[$code][$test] = $this->getTestStatusInfo( $test, $after );
 147                                 }
 148                         }
 149
 150                         # Write out results
 151                         foreach ( $table as $code => $label ) {
 152                                 if ( !empty( $breakdown[$code] ) ) {
 153                                         $count = count( $breakdown[$code] );
 154                                         printf( "\n%4d %s\n", $count, $label );
 155
 156                                         foreach ( $breakdown[$code] as $differing_test_name => $statusInfo ) {
 157                                                 print "      * $differing_test_name  [$statusInfo]\n";
 158                                         }
 159                                 }
 160                         }
 161                 } else {
 162                         print "No previous test runs to compare against.\n";
 163                 }
 164
 165                 print "\n";
 166                 parent::report();
 167         }
 168
 169         /**
 170          * Returns a string giving information about when a test last had a status change.
 171          * Could help to track down when regressions were introduced, as distinct from tests
 172          * which have never passed (which are more change requests than regressions).
 173          */
 174         private function getTestStatusInfo( $testname, $after ) {
 175                 // If we're looking at a test that has just been removed, then say when it first appeared.
 176                 if ( $after == 'n' ) {
 177                         $changedRun = $this->db->selectField ( 'testitem',
 178                                 'MIN(ti_run)',
 179                                 array( 'ti_name' => $testname ),
 180                                 __METHOD__ );
 181                         $appear = $this->db->selectRow ( 'testrun',
 182                                 array( 'tr_date', 'tr_mw_version' ),
 183                                 array( 'tr_id' => $changedRun ),
 184                                 __METHOD__ );
 185
 186                         return "First recorded appearance: "
 187                                    . date( "d-M-Y H:i:s",  strtotime ( $appear->tr_date ) )
 188                                    .  ", " . $appear->tr_mw_version;
 189                 }
 190
 191                 // Otherwise, this test has previous recorded results.
 192                 // See when this test last had a different result to what we're seeing now.
 193                 $conds = array(
 194                         'ti_name'    => $testname,
 195                         'ti_success' => ( $after == 'f' ? "1" : "0" ) );
 196
 197                 if ( $this->curRun ) {
 198                         $conds[] = "ti_run != " . $this->db->addQuotes ( $this->curRun );
 199                 }
 200
 201                 $changedRun = $this->db->selectField ( 'testitem', 'MAX(ti_run)', $conds, __METHOD__ );
 202
 203                 // If no record of ever having had a different result.
 204                 if ( is_null ( $changedRun ) ) {
 205                         if ( $after == "f" ) {
 206                                 return "Has never passed";
 207                         } else {
 208                                 return "Has never failed";
 209                         }
 210                 }
 211
 212                 // Otherwise, we're looking at a test whose status has changed.
 213                 // (i.e. it used to work, but now doesn't; or used to fail, but is now fixed.)
 214                 // In this situation, give as much info as we can as to when it changed status.
 215                 $pre  = $this->db->selectRow ( 'testrun',
 216                         array( 'tr_date', 'tr_mw_version' ),
 217                         array( 'tr_id' => $changedRun ),
 218                         __METHOD__ );
 219                 $post = $this->db->selectRow ( 'testrun',
 220                         array( 'tr_date', 'tr_mw_version' ),
 221                         array( "tr_id > " . $this->db->addQuotes ( $changedRun ) ),
 222                         __METHOD__,
 223                         array( "LIMIT" => 1, "ORDER BY" => 'tr_id' )
 224                 );
 225
 226                 if ( $post ) {
 227                         $postDate = date( "d-M-Y H:i:s",  strtotime ( $post->tr_date  ) ) . ", {$post->tr_mw_version}";
 228                 } else {
 229                         $postDate = 'now';
 230                 }
 231
 232                 return ( $after == "f" ? "Introduced" : "Fixed" ) . " between "
 233                                 . date( "d-M-Y H:i:s",  strtotime ( $pre->tr_date ) ) .  ", " . $pre->tr_mw_version
 234                                 . " and $postDate";
 235
 236         }
 237
 238         /**
 239          * Commit transaction and clean up for result recording
 240          */
 241         function end() {
 242                 $this->lb->commitMasterChanges();
 243                 $this->lb->closeAll();
 244                 parent::end();
 245         }
 246
 247 }
 248
 249 class DbTestRecorder extends DbTestPreviewer  {
 250         var $version;
 251
 252         /**
 253          * Set up result recording; insert a record for the run with the date
 254          * and all that fun stuff
 255          */
 256         function start() {
 257                 $this->db->begin();
 258
 259                 if ( ! $this->db->tableExists( 'testrun' )
 260                         || ! $this->db->tableExists( 'testitem' ) )
 261                 {
 262                         print "WARNING> `testrun` table not found in database. Trying to create table.\n";
 263                         $this->db->sourceFile( $this->db->patchPath( 'patch-testrun.sql' ) );
 264                         echo "OK, resuming.\n";
 265                 }
 266
 267                 parent::start();
 268
 269                 $this->db->insert( 'testrun',
 270                         array(
 271                                 'tr_date'        => $this->db->timestamp(),
 272                                 'tr_mw_version'  => $this->version,
 273                                 'tr_php_version' => phpversion(),
 274                                 'tr_db_version'  => $this->db->getServerVersion(),
 275                                 'tr_uname'       => php_uname()
 276                         ),
 277                         __METHOD__ );
 278                         if ( $this->db->getType() === 'postgres' ) {
 279                                 $this->curRun = $this->db->currentSequenceValue( 'testrun_id_seq' );
 280                         } else {
 281                                 $this->curRun = $this->db->insertId();
 282                         }
 283         }
 284
 285         /**
 286          * Record an individual test item's success or failure to the db
 287          *
 288          * @param $test String
 289          * @param $result Boolean
 290          */
 291         function record( $test, $result ) {
 292                 parent::record( $test, $result );
 293
 294                 $this->db->insert( 'testitem',
 295                         array(
 296                                 'ti_run'     => $this->curRun,
 297                                 'ti_name'    => $test,
 298                                 'ti_success' => $result ? 1 : 0,
 299                         ),
 300                         __METHOD__ );
 301         }
 302 }
 303
 304 class TestFileIterator implements Iterator {
 305         private $file;
 306         private $fh;
 307         private $parserTest; /* An instance of ParserTest (parserTests.php) or MediaWikiParserTest (phpunit) */
 308         private $index = 0;
 309         private $test;
 310         private $section = null; /** String|null: current test section being analyzed */
 311         private $sectionData = array();
 312         private $lineNum;
 313         private $eof;
 314
 315         function __construct( $file, $parserTest ) {
 316                 $this->file = $file;
 317                 $this->fh = fopen( $this->file, "rt" );
 318
 319                 if ( !$this->fh ) {
 320                         throw new MWException( "Couldn't open file '$file'\n" );
 321                 }
 322
 323                 $this->parserTest = $parserTest;
 324
 325                 $this->lineNum = $this->index = 0;
 326         }
 327
 328         function rewind() {
 329                 if ( fseek( $this->fh, 0 ) ) {
 330                         throw new MWException( "Couldn't fseek to the start of '$this->file'\n" );
 331                 }
 332
 333                 $this->index = -1;
 334                 $this->lineNum = 0;
 335                 $this->eof = false;
 336                 $this->next();
 337
 338                 return true;
 339         }
 340
 341         function current() {
 342                 return $this->test;
 343         }
 344
 345         function key() {
 346                 return $this->index;
 347         }
 348
 349         function next() {
 350                 if ( $this->readNextTest() ) {
 351                         $this->index++;
 352                         return true;
 353                 } else {
 354                         $this->eof = true;
 355                 }
 356         }
 357
 358         function valid() {
 359                 return $this->eof != true;
 360         }
 361
 362         function readNextTest() {
 363                 $this->clearSection();
 364
 365                 # Create a fake parser tests which never run anything unless
 366                 # asked to do so. This will avoid running hooks for a disabled test
 367                 $delayedParserTest = new DelayedParserTest();
 368
 369                 while ( false !== ( $line = fgets( $this->fh ) ) ) {
 370                         $this->lineNum++;
 371                         $matches = array();
 372
 373                         if ( preg_match( '/^!!\s*(\w+)/', $line, $matches ) ) {
 374                                 $this->section = strtolower( $matches[1] );
 375
 376                                 if ( $this->section == 'endarticle' ) {
 377                                         $this->checkSection( 'text'    );
 378                                         $this->checkSection( 'article' );
 379
 380                                         $this->parserTest->addArticle( ParserTest::chomp( $this->sectionData['article'] ), $this->sectionData['text'], $this->lineNum );
 381
 382                                         $this->clearSection();
 383
 384                                         continue;
 385                                 }
 386
 387                                 if ( $this->section == 'endhooks' ) {
 388                                         $this->checkSection( 'hooks' );
 389
 390                                         foreach ( explode( "\n", $this->sectionData['hooks'] ) as $line ) {
 391                                                 $line = trim( $line );
 392
 393                                                 if ( $line ) {
 394                                                         $delayedParserTest->requireHook( $line );
 395                                                 }
 396                                         }
 397
 398                                         $this->clearSection();
 399
 400                                         continue;
 401                                 }
 402
 403                                 if ( $this->section == 'endfunctionhooks' ) {
 404                                         $this->checkSection( 'functionhooks' );
 405
 406                                         foreach ( explode( "\n", $this->sectionData['functionhooks'] ) as $line ) {
 407                                                 $line = trim( $line );
 408
 409                                                 if ( $line ) {
 410                                                         $delayedParserTest->requireFunctionHook( $line );
 411                                                 }
 412                                         }
 413
 414                                         $this->clearSection();
 415
 416                                         continue;
 417                                 }
 418
 419                                 if ( $this->section == 'end' ) {
 420                                         $this->checkSection( 'test'   );
 421                                         $this->checkSection( 'input'  );
 422                                         $this->checkSection( 'result' );
 423
 424                                         if ( !isset( $this->sectionData['options'] ) ) {
 425                                                 $this->sectionData['options'] = '';
 426                                         }
 427
 428                                         if ( !isset( $this->sectionData['config'] ) ) {
 429                                                 $this->sectionData['config'] = '';
 430                                         }
 431
 432                                         if ( ( ( preg_match( '/\\bdisabled\\b/i', $this->sectionData['options'] ) && !$this->parserTest->runDisabled )
 433                                                          || !preg_match( "/" . $this->parserTest->regex . "/i", $this->sectionData['test'] ) )  ) {
 434                                                 # disabled test
 435                                                 $this->clearSection();
 436
 437                                                 # Forget any pending hooks call since test is disabled
 438                                                 $delayedParserTest->reset();
 439
 440                                                 continue;
 441                                         }
 442
 443                                         # We are really going to run the test, run pending hooks and hooks function
 444                                         wfDebug( __METHOD__ . " unleashing delayed test for: {$this->sectionData['test']}" );
 445                                         $hooksResult = $delayedParserTest->unleash( $this->parserTest );
 446                                         if( !$hooksResult ) {
 447                                                 # Some hook reported an issue. Abort.
 448                                                 return false;
 449                                         }
 450
 451                                         $this->test = array(
 452                                                 'test'    => ParserTest::chomp( $this->sectionData['test']    ),
 453                                                 'input'   => ParserTest::chomp( $this->sectionData['input']   ),
 454                                                 'result'  => ParserTest::chomp( $this->sectionData['result']  ),
 455                                                 'options' => ParserTest::chomp( $this->sectionData['options'] ),
 456                                                 'config'  => ParserTest::chomp( $this->sectionData['config']  ),
 457                                         );
 458
 459                                         return true;
 460                                 }
 461
 462                                 if ( isset ( $this->sectionData[$this->section] ) ) {
 463                                         throw new MWException( "duplicate section '$this->section' at line {$this->lineNum} of $this->file\n" );
 464                                 }
 465
 466                                 $this->sectionData[$this->section] = '';
 467
 468                                 continue;
 469                         }
 470
 471                         if ( $this->section ) {
 472                                 $this->sectionData[$this->section] .= $line;
 473                         }
 474                 }
 475
 476                 return false;
 477         }
 478
 479
 480         /**
 481          * Clear section name and its data
 482          */
 483         private function clearSection() {
 484                 $this->sectionData = array();
 485                 $this->section = null;
 486
 487         }
 488
 489         /**
 490          * Verify the current section data has some value for the given token
 491          * name (first parameter).
 492          * Throw an exception if it is not set, referencing current section
 493          * and adding the current file name and line number
 494          *
 495          * @param $token String: expected token that should have been mentionned before closing this section
 496          */
 497         private function checkSection( $token ) {
 498                 if( is_null( $this->section ) ) {
 499                         throw new MWException( __METHOD__ . " can not verify a null section!\n" );
 500                 }
 501
 502                 if( !isset($this->sectionData[$token]) ) {
 503                         throw new MWException( sprintf(
 504                                 "'%s' without '%s' at line %s of %s\n",
 505                                 $this->section,
 506                                 $token,
 507                                 $this->lineNum,
 508                                 $this->file
 509                         ));
 510                 }
 511                 return true;
 512         }
 513 }
 514
 515 /**
 516  * A class to delay execution of a parser test hooks.
 517  */
 518 class DelayedParserTest {
 519
 520         /** Initialized on construction */
 521         private $hooks;
 522         private $fnHooks;
 523
 524         public function __construct() {
 525                 $this->reset();
 526         }
 527
 528         /**
 529          * Init/reset or forgot about the current delayed test.
 530          * Call to this will erase any hooks function that were pending.
 531          */
 532         public function reset() {
 533                 $this->hooks   = array();
 534                 $this->fnHooks = array();
 535         }
 536
 537         /**
 538          * Called whenever we actually want to run the hook.
 539          * Should be the case if we found the parserTest is not disabled
 540          */
 541         public function unleash( &$parserTest ) {
 542                 if( !($parserTest instanceof ParserTest || $parserTest instanceof NewParserTest
 543                 ) ) {
 544                         throw new MWException( __METHOD__ . " must be passed an instance of ParserTest or NewParserTest classes\n" );
 545                 }
 546
 547                 # Trigger delayed hooks. Any failure will make us abort
 548                 foreach( $this->hooks as $hook ) {
 549                         $ret = $parserTest->requireHook( $hook );
 550                         if( !$ret ) {
 551                                 return false;
 552                         }
 553                 }
 554
 555                 # Trigger delayed function hooks. Any failure will make us abort
 556                 foreach( $this->fnHooks as $fnHook ) {
 557                         $ret = $parserTest->requireFunctionHook( $fnHook );
 558                         if( !$ret ) {
 559                                 return false;
 560                         }
 561                 }
 562
 563                 # Delayed execution was successful.
 564                 return true;
 565         }
 566
 567         /**
 568          * Similar to ParserTest object but does not run anything
 569          * Use unleash() to really execute the hook
 570          */
 571         public function requireHook( $hook ) {
 572                 $this->hooks[] = $hook;
 573         }
 574         /**
 575          * Similar to ParserTest object but does not run anything
 576          * Use unleash() to really execute the hook function
 577          */
 578         public function requireFunctionHook( $fnHook ) {
 579                 $this->fnHooks[] = $fnHook;
 580         }
 581
 582 }