tests/testHelpers.inc

   1 <?php
   2
   3 class TestRecorder {
   4         var $parent;
   5         var $term;
   6
   7         function __construct( $parent ) {
   8                 $this->parent = $parent;
   9                 $this->term = $parent->term;
  10         }
  11
  12         function start() {
  13                 $this->total = 0;
  14                 $this->success = 0;
  15         }
  16
  17         function record( $test, $result ) {
  18                 $this->total++;
  19                 $this->success += ( $result ? 1 : 0 );
  20         }
  21
  22         function end() {
  23                 // dummy
  24         }
  25
  26         function report() {
  27                 if ( $this->total > 0 ) {
  28                         $this->reportPercentage( $this->success, $this->total );
  29                 } else {
  30                         throw new MWException( "No tests found.\n" );
  31                 }
  32         }
  33
  34         function reportPercentage( $success, $total ) {
  35                 $ratio = wfPercent( 100 * $success / $total );
  36                 print $this->term->color( 1 ) . "Passed $success of $total tests ($ratio)... ";
  37
  38                 if ( $success == $total ) {
  39                         print $this->term->color( 32 ) . "ALL TESTS PASSED!";
  40                 } else {
  41                         $failed = $total - $success ;
  42                         print $this->term->color( 31 ) . "$failed tests failed!";
  43                 }
  44
  45                 print $this->term->reset() . "\n";
  46
  47                 return ( $success == $total );
  48         }
  49 }
  50
  51 class DbTestPreviewer extends TestRecorder  {
  52         protected $lb;      // /< Database load balancer
  53         protected $db;      // /< Database connection to the main DB
  54         protected $curRun;  // /< run ID number for the current run
  55         protected $prevRun; // /< run ID number for the previous run, if any
  56         protected $results; // /< Result array
  57
  58         /**
  59          * This should be called before the table prefix is changed
  60          */
  61         function __construct( $parent ) {
  62                 parent::__construct( $parent );
  63
  64                 $this->lb = wfGetLBFactory()->newMainLB();
  65                 // This connection will have the wiki's table prefix, not parsertest_
  66                 $this->db = $this->lb->getConnection( DB_MASTER );
  67         }
  68
  69         /**
  70          * Set up result recording; insert a record for the run with the date
  71          * and all that fun stuff
  72          */
  73         function start() {
  74                 parent::start();
  75
  76                 if ( ! $this->db->tableExists( 'testrun', __METHOD__ )
  77                         || ! $this->db->tableExists( 'testitem', __METHOD__ ) )
  78                 {
  79                         print "WARNING> `testrun` table not found in database.\n";
  80                         $this->prevRun = false;
  81                 } else {
  82                         // We'll make comparisons against the previous run later...
  83                         $this->prevRun = $this->db->selectField( 'testrun', 'MAX(tr_id)' );
  84                 }
  85
  86                 $this->results = array();
  87         }
  88
  89         function record( $test, $result ) {
  90                 parent::record( $test, $result );
  91                 $this->results[$test] = $result;
  92         }
  93
  94         function report() {
  95                 if ( $this->prevRun ) {
  96                         // f = fail, p = pass, n = nonexistent
  97                         // codes show before then after
  98                         $table = array(
  99                                 'fp' => 'previously failing test(s) now PASSING! :)',
 100                                 'pn' => 'previously PASSING test(s) removed o_O',
 101                                 'np' => 'new PASSING test(s) :)',
 102
 103                                 'pf' => 'previously passing test(s) now FAILING! :(',
 104                                 'fn' => 'previously FAILING test(s) removed O_o',
 105                                 'nf' => 'new FAILING test(s) :(',
 106                                 'ff' => 'still FAILING test(s) :(',
 107                         );
 108
 109                         $prevResults = array();
 110
 111                         $res = $this->db->select( 'testitem', array( 'ti_name', 'ti_success' ),
 112                                 array( 'ti_run' => $this->prevRun ), __METHOD__ );
 113
 114                         foreach ( $res as $row ) {
 115                                 if ( !$this->parent->regex
 116                                         || preg_match( "/{$this->parent->regex}/i", $row->ti_name ) )
 117                                 {
 118                                         $prevResults[$row->ti_name] = $row->ti_success;
 119                                 }
 120                         }
 121
 122                         $combined = array_keys( $this->results + $prevResults );
 123
 124                         # Determine breakdown by change type
 125                         $breakdown = array();
 126                         foreach ( $combined as $test ) {
 127                                 if ( !isset( $prevResults[$test] ) ) {
 128                                         $before = 'n';
 129                                 } elseif ( $prevResults[$test] == 1 ) {
 130                                         $before = 'p';
 131                                 } else /* if ( $prevResults[$test] == 0 )*/ {
 132                                         $before = 'f';
 133                                 }
 134
 135                                 if ( !isset( $this->results[$test] ) ) {
 136                                         $after = 'n';
 137                                 } elseif ( $this->results[$test] == 1 ) {
 138                                         $after = 'p';
 139                                 } else /*if ( $this->results[$test] == 0 ) */ {
 140                                         $after = 'f';
 141                                 }
 142
 143                                 $code = $before . $after;
 144
 145                                 if ( isset( $table[$code] ) ) {
 146                                         $breakdown[$code][$test] = $this->getTestStatusInfo( $test, $after );
 147                                 }
 148                         }
 149
 150                         # Write out results
 151                         foreach ( $table as $code => $label ) {
 152                                 if ( !empty( $breakdown[$code] ) ) {
 153                                         $count = count( $breakdown[$code] );
 154                                         printf( "\n%4d %s\n", $count, $label );
 155
 156                                         foreach ( $breakdown[$code] as $differing_test_name => $statusInfo ) {
 157                                                 print "      * $differing_test_name  [$statusInfo]\n";
 158                                         }
 159                                 }
 160                         }
 161                 } else {
 162                         print "No previous test runs to compare against.\n";
 163                 }
 164
 165                 print "\n";
 166                 parent::report();
 167         }
 168
 169         /**
 170          * Returns a string giving information about when a test last had a status change.
 171          * Could help to track down when regressions were introduced, as distinct from tests
 172          * which have never passed (which are more change requests than regressions).
 173          */
 174         private function getTestStatusInfo( $testname, $after ) {
 175                 // If we're looking at a test that has just been removed, then say when it first appeared.
 176                 if ( $after == 'n' ) {
 177                         $changedRun = $this->db->selectField ( 'testitem',
 178                                 'MIN(ti_run)',
 179                                 array( 'ti_name' => $testname ),
 180                                 __METHOD__ );
 181                         $appear = $this->db->selectRow ( 'testrun',
 182                                 array( 'tr_date', 'tr_mw_version' ),
 183                                 array( 'tr_id' => $changedRun ),
 184                                 __METHOD__ );
 185
 186                         return "First recorded appearance: "
 187                                    . date( "d-M-Y H:i:s",  strtotime ( $appear->tr_date ) )
 188                                    .  ", " . $appear->tr_mw_version;
 189                 }
 190
 191                 // Otherwise, this test has previous recorded results.
 192                 // See when this test last had a different result to what we're seeing now.
 193                 $conds = array(
 194                         'ti_name'    => $testname,
 195                         'ti_success' => ( $after == 'f' ? "1" : "0" ) );
 196
 197                 if ( $this->curRun ) {
 198                         $conds[] = "ti_run != " . $this->db->addQuotes ( $this->curRun );
 199                 }
 200
 201                 $changedRun = $this->db->selectField ( 'testitem', 'MAX(ti_run)', $conds, __METHOD__ );
 202
 203                 // If no record of ever having had a different result.
 204                 if ( is_null ( $changedRun ) ) {
 205                         if ( $after == "f" ) {
 206                                 return "Has never passed";
 207                         } else {
 208                                 return "Has never failed";
 209                         }
 210                 }
 211
 212                 // Otherwise, we're looking at a test whose status has changed.
 213                 // (i.e. it used to work, but now doesn't; or used to fail, but is now fixed.)
 214                 // In this situation, give as much info as we can as to when it changed status.
 215                 $pre  = $this->db->selectRow ( 'testrun',
 216                         array( 'tr_date', 'tr_mw_version' ),
 217                         array( 'tr_id' => $changedRun ),
 218                         __METHOD__ );
 219                 $post = $this->db->selectRow ( 'testrun',
 220                         array( 'tr_date', 'tr_mw_version' ),
 221                         array( "tr_id > " . $this->db->addQuotes ( $changedRun ) ),
 222                         __METHOD__,
 223                         array( "LIMIT" => 1, "ORDER BY" => 'tr_id' )
 224                 );
 225
 226                 if ( $post ) {
 227                         $postDate = date( "d-M-Y H:i:s",  strtotime ( $post->tr_date  ) ) . ", {$post->tr_mw_version}";
 228                 } else {
 229                         $postDate = 'now';
 230                 }
 231
 232                 return ( $after == "f" ? "Introduced" : "Fixed" ) . " between "
 233                                 . date( "d-M-Y H:i:s",  strtotime ( $pre->tr_date ) ) .  ", " . $pre->tr_mw_version
 234                                 . " and $postDate";
 235
 236         }
 237
 238         /**
 239          * Commit transaction and clean up for result recording
 240          */
 241         function end() {
 242                 $this->lb->commitMasterChanges();
 243                 $this->lb->closeAll();
 244                 parent::end();
 245         }
 246
 247 }
 248
 249 class DbTestRecorder extends DbTestPreviewer  {
 250         var $version;
 251
 252         /**
 253          * Set up result recording; insert a record for the run with the date
 254          * and all that fun stuff
 255          */
 256         function start() {
 257                 $this->db->begin();
 258
 259                 if ( ! $this->db->tableExists( 'testrun' )
 260                         || ! $this->db->tableExists( 'testitem' ) )
 261                 {
 262                         print "WARNING> `testrun` table not found in database. Trying to create table.\n";
 263                         $this->db->sourceFile( $this->db->patchPath( 'patch-testrun.sql' ) );
 264                         echo "OK, resuming.\n";
 265                 }
 266
 267                 parent::start();
 268
 269                 $this->db->insert( 'testrun',
 270                         array(
 271                                 'tr_date'        => $this->db->timestamp(),
 272                                 'tr_mw_version'  => $this->version,
 273                                 'tr_php_version' => phpversion(),
 274                                 'tr_db_version'  => $this->db->getServerVersion(),
 275                                 'tr_uname'       => php_uname()
 276                         ),
 277                         __METHOD__ );
 278                         if ( $this->db->getType() === 'postgres' ) {
 279                                 $this->curRun = $this->db->currentSequenceValue( 'testrun_id_seq' );
 280                         } else {
 281                                 $this->curRun = $this->db->insertId();
 282                         }
 283         }
 284
 285         /**
 286          * Record an individual test item's success or failure to the db
 287          *
 288          * @param $test String
 289          * @param $result Boolean
 290          */
 291         function record( $test, $result ) {
 292                 parent::record( $test, $result );
 293
 294                 $this->db->insert( 'testitem',
 295                         array(
 296                                 'ti_run'     => $this->curRun,
 297                                 'ti_name'    => $test,
 298                                 'ti_success' => $result ? 1 : 0,
 299                         ),
 300                         __METHOD__ );
 301         }
 302 }
 303
 304 class TestFileIterator implements Iterator {
 305         private $file;
 306         private $fh;
 307         private $parserTest; /* An instance of ParserTest (parserTests.php) or MediaWikiParserTest (phpunit) */
 308         private $index = 0;
 309         private $test;
 310         private $section = null; /** String|null: current test section being analyzed */
 311         private $sectionData = array();
 312         private $lineNum;
 313         private $eof;
 314
 315         function __construct( $file, $parserTest ) {
 316                 global $IP;
 317
 318                 $this->file = $file;
 319                 $this->fh = fopen( $this->file, "rt" );
 320
 321                 if ( !$this->fh ) {
 322                         throw new MWException( "Couldn't open file '$file'\n" );
 323                 }
 324
 325                 $this->parserTest = $parserTest;
 326                 $this->parserTest->showRunFile( wfRelativePath( $this->file, $IP ) );
 327
 328                 $this->lineNum = $this->index = 0;
 329         }
 330
 331         function rewind() {
 332                 if ( fseek( $this->fh, 0 ) ) {
 333                         throw new MWException( "Couldn't fseek to the start of '$this->file'\n" );
 334                 }
 335
 336                 $this->index = -1;
 337                 $this->lineNum = 0;
 338                 $this->eof = false;
 339                 $this->next();
 340
 341                 return true;
 342         }
 343
 344         function current() {
 345                 return $this->test;
 346         }
 347
 348         function key() {
 349                 return $this->index;
 350         }
 351
 352         function next() {
 353                 if ( $this->readNextTest() ) {
 354                         $this->index++;
 355                         return true;
 356                 } else {
 357                         $this->eof = true;
 358                 }
 359         }
 360
 361         function valid() {
 362                 return $this->eof != true;
 363         }
 364
 365         function readNextTest() {
 366                 $this->clearSection();
 367
 368                 # Create a fake parser tests which never run anything unless
 369                 # asked to do so. This will avoid running hooks for a disabled test
 370                 $delayedParserTest = new DelayedParserTest();
 371
 372                 while ( false !== ( $line = fgets( $this->fh ) ) ) {
 373                         $this->lineNum++;
 374                         $matches = array();
 375
 376                         if ( preg_match( '/^!!\s*(\w+)/', $line, $matches ) ) {
 377                                 $this->section = strtolower( $matches[1] );
 378
 379                                 if ( $this->section == 'endarticle' ) {
 380                                         $this->checkSection( 'text'    );
 381                                         $this->checkSection( 'article' );
 382
 383                                         $this->parserTest->addArticle( ParserTest::chomp( $this->sectionData['article'] ), $this->sectionData['text'], $this->lineNum );
 384
 385                                         $this->clearSection();
 386
 387                                         continue;
 388                                 }
 389
 390                                 if ( $this->section == 'endhooks' ) {
 391                                         $this->checkSection( 'hooks' );
 392
 393                                         foreach ( explode( "\n", $this->sectionData['hooks'] ) as $line ) {
 394                                                 $line = trim( $line );
 395
 396                                                 if ( $line ) {
 397                                                         $delayedParserTest->requireHook( $line );
 398                                                 }
 399                                         }
 400
 401                                         $this->clearSection();
 402
 403                                         continue;
 404                                 }
 405
 406                                 if ( $this->section == 'endfunctionhooks' ) {
 407                                         $this->checkSection( 'functionhooks' );
 408
 409                                         foreach ( explode( "\n", $this->sectionData['functionhooks'] ) as $line ) {
 410                                                 $line = trim( $line );
 411
 412                                                 if ( $line ) {
 413                                                         $delayedParserTest->requireFunctionHook( $line );
 414                                                 }
 415                                         }
 416
 417                                         $this->clearSection();
 418
 419                                         continue;
 420                                 }
 421
 422                                 if ( $this->section == 'end' ) {
 423                                         $this->checkSection( 'test'   );
 424                                         $this->checkSection( 'input'  );
 425                                         $this->checkSection( 'result' );
 426
 427                                         if ( !isset( $this->sectionData['options'] ) ) {
 428                                                 $this->sectionData['options'] = '';
 429                                         }
 430
 431                                         if ( !isset( $this->sectionData['config'] ) ) {
 432                                                 $this->sectionData['config'] = '';
 433                                         }
 434
 435                                         if ( ( ( preg_match( '/\\bdisabled\\b/i', $this->sectionData['options'] ) && !$this->parserTest->runDisabled )
 436                                                          || !preg_match( "/" . $this->parserTest->regex . "/i", $this->sectionData['test'] ) )  ) {
 437                                                 # disabled test
 438                                                 $this->clearSection();
 439
 440                                                 # Forget any pending hooks call since test is disabled
 441                                                 $delayedParserTest->reset();
 442
 443                                                 continue;
 444                                         }
 445
 446                                         # We are really going to run the test, run pending hooks and hooks function
 447                                         wfDebug( __METHOD__ . " unleashing delayed test for: {$this->sectionData['test']}" );
 448                                         $hooksResult = $delayedParserTest->unleash( $this->parserTest );
 449                                         if( !$hooksResult ) {
 450                                                 # Some hook reported an issue. Abort.
 451                                                 return false;
 452                                         }
 453
 454                                         $this->test = array(
 455                                                 'test'    => ParserTest::chomp( $this->sectionData['test']    ),
 456                                                 'input'   => ParserTest::chomp( $this->sectionData['input']   ),
 457                                                 'result'  => ParserTest::chomp( $this->sectionData['result']  ),
 458                                                 'options' => ParserTest::chomp( $this->sectionData['options'] ),
 459                                                 'config'  => ParserTest::chomp( $this->sectionData['config']  ),
 460                                         );
 461
 462                                         return true;
 463                                 }
 464
 465                                 if ( isset ( $this->sectionData[$this->section] ) ) {
 466                                         throw new MWException( "duplicate section '$section' at line {$this->lineNum} of $this->file\n" );
 467                                 }
 468
 469                                 $this->sectionData[$this->section] = '';
 470
 471                                 continue;
 472                         }
 473
 474                         if ( $this->section ) {
 475                                 $this->sectionData[$this->section] .= $line;
 476                         }
 477                 }
 478
 479                 return false;
 480         }
 481
 482
 483         /**
 484          * Clear section name and its data
 485          */
 486         private function clearSection() {
 487                 $this->sectionData = array();
 488                 $this->section = null;
 489
 490         }
 491
 492         /**
 493          * Verify the current section data has some value for the given token
 494          * name (first parameter).
 495          * Throw an exception if it is not set, referencing current section
 496          * and adding the current file name and line number
 497          *
 498          * @param $token String: expected token that should have been mentionned before closing this section
 499          */
 500         private function checkSection( $token ) {
 501                 if( is_null( $this->section ) ) {
 502                         throw new MWException( __METHOD__ . " can not verify a null section!\n" );
 503                 }
 504
 505                 if( !isset($this->sectionData[$token]) ) {
 506                         throw new MWException( sprintf(
 507                                 "'%s' without '%s' at line %s of %s\n",
 508                                 $this->section,
 509                                 $token,
 510                                 $this->lineNum,
 511                                 $this->file
 512                         ));
 513                 }
 514                 return true;
 515         }
 516 }
 517
 518
 519 /**
 520  * A class to delay execution of a parser test hooks.
 521  *
 522  */
 523 class DelayedParserTest {
 524
 525         /** Initialized on construction */
 526         private $hooks;
 527         private $fnHooks;
 528
 529         public function __construct() {
 530                 $this->reset();
 531         }
 532
 533         /**
 534          * Init/reset or forgot about the current delayed test.
 535          * Call to this will erase any hooks function that were pending.
 536          */
 537         public function reset() {
 538                 $this->hooks   = array();
 539                 $this->fnHooks = array();
 540         }
 541
 542         /**
 543          * Called whenever we actually want to run the hook.
 544          * Should be the case if we found the parserTest is not disabled
 545          */
 546         public function unleash( &$parserTest ) {
 547                 if( !($parserTest instanceof ParserTest || $parserTest instanceof NewParserTest
 548                 ) ) {
 549                         throw new MWException( __METHOD__ . " must be passed an instance of ParserTest or NewParserTest classes\n" );
 550                 }
 551
 552                 # Trigger delayed hooks. Any failure will make us abort
 553                 foreach( $this->hooks as $hook ) {
 554                         $ret = $parserTest->requireHook( $hook );
 555                         if( !$ret ) {
 556                                 return false;
 557                         }
 558                 }
 559
 560                 # Trigger delayed function hooks. Any failure will make us abort
 561                 foreach( $this->fnHooks as $fnHook ) {
 562                         $ret = $parserTest->requireFunctionHook( $fnHook );
 563                         if( !$ret ) {
 564                                 return false;
 565                         }
 566                 }
 567
 568                 # Delayed execution was successful.
 569                 return true;
 570         }
 571
 572         /**
 573          * Similar to ParserTest object but does not run anything
 574          * Use unleash() to really execute the hook
 575          */
 576         public function requireHook( $hook ) {
 577                 $this->hooks[] = $hook;
 578         }
 579         /**
 580          * Similar to ParserTest object but does not run anything
 581          * Use unleash() to really execute the hook function
 582          */
 583         public function requireFunctionHook( $fnHook ) {
 584                 $this->fnHooks[] = $fnHook;
 585         }
 586
 587 }