3 * Recording for passing/failing tests.
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
25 * Interface to record parser test results.
27 * The ITestRecorder is a very simple interface to record the result of
28 * MediaWiki parser tests. One should call start() before running the
29 * full parser tests and end() once all the tests have been finished.
30 * After each test, you should use record() to keep track of your tests
31 * results. Finally, report() is used to generate a summary of your
32 * test run, one could dump it to the console for human consumption or
33 * register the result in a database for tracking purposes.
37 interface ITestRecorder {
40 * Called at beginning of the parser test run
42 public function start();
45 * Called after each test
47 * @param integer $subtest
50 public function record( $test, $subtest, $result );
53 * Called before finishing the test run
55 public function report();
58 * Called at the end of the parser test run
60 public function end();
64 class TestRecorder implements ITestRecorder {
68 function __construct( $parent ) {
69 $this->parent = $parent;
70 $this->term = $parent->term;
78 function record( $test, $subtest, $result ) {
80 $this->success += ( $result ? 1 : 0 );
88 if ( $this->total > 0 ) {
89 $this->reportPercentage( $this->success, $this->total );
91 throw new MWException( "No tests found.\n" );
95 function reportPercentage( $success, $total ) {
96 $ratio = wfPercent( 100 * $success / $total );
97 print $this->term->color( 1 ) . "Passed $success of $total tests ($ratio)... ";
99 if ( $success == $total ) {
100 print $this->term->color( 32 ) . "ALL TESTS PASSED!";
102 $failed = $total - $success;
103 print $this->term->color( 31 ) . "$failed tests failed!";
106 print $this->term->reset() . "\n";
108 return ( $success == $total );
112 class DbTestPreviewer extends TestRecorder {
113 protected $lb; // /< Database load balancer
114 protected $db; // /< Database connection to the main DB
115 protected $curRun; // /< run ID number for the current run
116 protected $prevRun; // /< run ID number for the previous run, if any
117 protected $results; // /< Result array
120 * This should be called before the table prefix is changed
121 * @param TestRecorder $parent
123 function __construct( $parent ) {
124 parent::__construct( $parent );
126 $this->lb = wfGetLBFactory()->newMainLB();
127 // This connection will have the wiki's table prefix, not parsertest_
128 $this->db = $this->lb->getConnection( DB_MASTER );
132 * Set up result recording; insert a record for the run with the date
133 * and all that fun stuff
138 if ( !$this->db->tableExists( 'testrun', __METHOD__ )
139 || !$this->db->tableExists( 'testitem', __METHOD__ )
141 print "WARNING> `testrun` table not found in database.\n";
142 $this->prevRun = false;
144 // We'll make comparisons against the previous run later...
145 $this->prevRun = $this->db->selectField( 'testrun', 'MAX(tr_id)' );
151 function getName( $test, $subtest ) {
153 return "$test subtest #$subtest";
159 function record( $test, $subtest, $result ) {
160 parent::record( $test, $subtest, $result );
161 $this->results[ $this->getName( $test, $subtest ) ] = $result;
165 if ( $this->prevRun ) {
166 // f = fail, p = pass, n = nonexistent
167 // codes show before then after
169 'fp' => 'previously failing test(s) now PASSING! :)',
170 'pn' => 'previously PASSING test(s) removed o_O',
171 'np' => 'new PASSING test(s) :)',
173 'pf' => 'previously passing test(s) now FAILING! :(',
174 'fn' => 'previously FAILING test(s) removed O_o',
175 'nf' => 'new FAILING test(s) :(',
176 'ff' => 'still FAILING test(s) :(',
181 $res = $this->db->select( 'testitem', [ 'ti_name', 'ti_success' ],
182 [ 'ti_run' => $this->prevRun ], __METHOD__ );
184 foreach ( $res as $row ) {
185 if ( !$this->parent->regex
186 || preg_match( "/{$this->parent->regex}/i", $row->ti_name )
188 $prevResults[$row->ti_name] = $row->ti_success;
192 $combined = array_keys( $this->results + $prevResults );
194 # Determine breakdown by change type
196 foreach ( $combined as $test ) {
197 if ( !isset( $prevResults[$test] ) ) {
199 } elseif ( $prevResults[$test] == 1 ) {
201 } else /* if ( $prevResults[$test] == 0 )*/ {
205 if ( !isset( $this->results[$test] ) ) {
207 } elseif ( $this->results[$test] == 1 ) {
209 } else /*if ( $this->results[$test] == 0 ) */ {
213 $code = $before . $after;
215 if ( isset( $table[$code] ) ) {
216 $breakdown[$code][$test] = $this->getTestStatusInfo( $test, $after );
221 foreach ( $table as $code => $label ) {
222 if ( !empty( $breakdown[$code] ) ) {
223 $count = count( $breakdown[$code] );
224 printf( "\n%4d %s\n", $count, $label );
226 foreach ( $breakdown[$code] as $differing_test_name => $statusInfo ) {
227 print " * $differing_test_name [$statusInfo]\n";
232 print "No previous test runs to compare against.\n";
240 * Returns a string giving information about when a test last had a status change.
241 * Could help to track down when regressions were introduced, as distinct from tests
242 * which have never passed (which are more change requests than regressions).
243 * @param string $testname
244 * @param string $after
247 private function getTestStatusInfo( $testname, $after ) {
248 // If we're looking at a test that has just been removed, then say when it first appeared.
249 if ( $after == 'n' ) {
250 $changedRun = $this->db->selectField( 'testitem',
252 [ 'ti_name' => $testname ],
254 $appear = $this->db->selectRow( 'testrun',
255 [ 'tr_date', 'tr_mw_version' ],
256 [ 'tr_id' => $changedRun ],
259 return "First recorded appearance: "
260 . date( "d-M-Y H:i:s", strtotime( $appear->tr_date ) )
261 . ", " . $appear->tr_mw_version;
264 // Otherwise, this test has previous recorded results.
265 // See when this test last had a different result to what we're seeing now.
267 'ti_name' => $testname,
268 'ti_success' => ( $after == 'f' ? "1" : "0" ) ];
270 if ( $this->curRun ) {
271 $conds[] = "ti_run != " . $this->db->addQuotes( $this->curRun );
274 $changedRun = $this->db->selectField( 'testitem', 'MAX(ti_run)', $conds, __METHOD__ );
276 // If no record of ever having had a different result.
277 if ( is_null( $changedRun ) ) {
278 if ( $after == "f" ) {
279 return "Has never passed";
281 return "Has never failed";
285 // Otherwise, we're looking at a test whose status has changed.
286 // (i.e. it used to work, but now doesn't; or used to fail, but is now fixed.)
287 // In this situation, give as much info as we can as to when it changed status.
288 $pre = $this->db->selectRow( 'testrun',
289 [ 'tr_date', 'tr_mw_version' ],
290 [ 'tr_id' => $changedRun ],
292 $post = $this->db->selectRow( 'testrun',
293 [ 'tr_date', 'tr_mw_version' ],
294 [ "tr_id > " . $this->db->addQuotes( $changedRun ) ],
296 [ "LIMIT" => 1, "ORDER BY" => 'tr_id' ]
300 $postDate = date( "d-M-Y H:i:s", strtotime( $post->tr_date ) ) . ", {$post->tr_mw_version}";
305 return ( $after == "f" ? "Introduced" : "Fixed" ) . " between "
306 . date( "d-M-Y H:i:s", strtotime( $pre->tr_date ) ) . ", " . $pre->tr_mw_version
311 * Close the DB connection
314 $this->lb->closeAll();
319 class DbTestRecorder extends DbTestPreviewer {
323 * Set up result recording; insert a record for the run with the date
324 * and all that fun stuff
327 $this->db->begin( __METHOD__ );
329 if ( !$this->db->tableExists( 'testrun' )
330 || !$this->db->tableExists( 'testitem' )
332 print "WARNING> `testrun` table not found in database. Trying to create table.\n";
333 $this->db->sourceFile( $this->db->patchPath( 'patch-testrun.sql' ) );
334 echo "OK, resuming.\n";
339 $this->db->insert( 'testrun',
341 'tr_date' => $this->db->timestamp(),
342 'tr_mw_version' => $this->version,
343 'tr_php_version' => PHP_VERSION,
344 'tr_db_version' => $this->db->getServerVersion(),
345 'tr_uname' => php_uname()
348 if ( $this->db->getType() === 'postgres' ) {
349 $this->curRun = $this->db->currentSequenceValue( 'testrun_id_seq' );
351 $this->curRun = $this->db->insertId();
356 * Record an individual test item's success or failure to the db
358 * @param string $test
359 * @param bool $result
361 function record( $test, $subtest, $result ) {
362 parent::record( $test, $subtest, $result );
364 $this->db->insert( 'testitem',
366 'ti_run' => $this->curRun,
367 'ti_name' => $this->getName( $test, $subtest ),
368 'ti_success' => $result ? 1 : 0,
374 * Commit transaction and clean up for result recording
377 $this->db->commit( __METHOD__ );
382 class TestFileIterator implements Iterator {
386 * @var ParserTest|MediaWikiParserTest An instance of ParserTest (parserTests.php)
387 * or MediaWikiParserTest (phpunit)
392 private $section = null;
393 /** String|null: current test section being analyzed */
394 private $sectionData = [];
397 # Create a fake parser tests which never run anything unless
398 # asked to do so. This will avoid running hooks for a disabled test
399 private $delayedParserTest;
400 private $nextSubTest = 0;
402 function __construct( $file, $parserTest ) {
404 $this->fh = fopen( $this->file, "rt" );
407 throw new MWException( "Couldn't open file '$file'\n" );
410 $this->parserTest = $parserTest;
411 $this->delayedParserTest = new DelayedParserTest();
413 $this->lineNum = $this->index = 0;
417 if ( fseek( $this->fh, 0 ) ) {
418 throw new MWException( "Couldn't fseek to the start of '$this->file'\n" );
438 if ( $this->readNextTest() ) {
447 return $this->eof != true;
450 function setupCurrentTest() {
451 // "input" and "result" are old section names allowed
452 // for backwards-compatibility.
453 $input = $this->checkSection( [ 'wikitext', 'input' ], false );
454 $result = $this->checkSection( [ 'html/php', 'html/*', 'html', 'result' ], false );
455 // some tests have "with tidy" and "without tidy" variants
456 $tidy = $this->checkSection( [ 'html/php+tidy', 'html+tidy' ], false );
457 if ( $tidy != false ) {
458 if ( $this->nextSubTest == 0 ) {
459 if ( $result != false ) {
460 $this->nextSubTest = 1; // rerun non-tidy variant later
464 $this->nextSubTest = 0; // go on to next test after this
469 if ( !isset( $this->sectionData['options'] ) ) {
470 $this->sectionData['options'] = '';
473 if ( !isset( $this->sectionData['config'] ) ) {
474 $this->sectionData['config'] = '';
477 $isDisabled = preg_match( '/\\bdisabled\\b/i', $this->sectionData['options'] ) &&
478 !$this->parserTest->runDisabled;
479 $isParsoidOnly = preg_match( '/\\bparsoid\\b/i', $this->sectionData['options'] ) &&
481 !$this->parserTest->runParsoid;
482 $isFiltered = !preg_match( "/" . $this->parserTest->regex . "/i", $this->sectionData['test'] );
483 if ( $input == false || $result == false || $isDisabled || $isParsoidOnly || $isFiltered ) {
488 # We are really going to run the test, run pending hooks and hooks function
489 wfDebug( __METHOD__ . " unleashing delayed test for: {$this->sectionData['test']}" );
490 $hooksResult = $this->delayedParserTest->unleash( $this->parserTest );
491 if ( !$hooksResult ) {
492 # Some hook reported an issue. Abort.
493 throw new MWException( "Problem running requested parser hook from the test file" );
497 'test' => ParserTest::chomp( $this->sectionData['test'] ),
498 'subtest' => $this->nextSubTest,
499 'input' => ParserTest::chomp( $this->sectionData[$input] ),
500 'result' => ParserTest::chomp( $this->sectionData[$result] ),
501 'options' => ParserTest::chomp( $this->sectionData['options'] ),
502 'config' => ParserTest::chomp( $this->sectionData['config'] ),
504 if ( $tidy != false ) {
505 $this->test['options'] .= " tidy";
510 function readNextTest() {
511 # Run additional subtests of previous test
512 while ( $this->nextSubTest > 0 ) {
513 if ( $this->setupCurrentTest() ) {
518 $this->clearSection();
519 # Reset hooks for the delayed test object
520 $this->delayedParserTest->reset();
522 while ( false !== ( $line = fgets( $this->fh ) ) ) {
526 if ( preg_match( '/^!!\s*(\S+)/', $line, $matches ) ) {
527 $this->section = strtolower( $matches[1] );
529 if ( $this->section == 'endarticle' ) {
530 $this->checkSection( 'text' );
531 $this->checkSection( 'article' );
533 $this->parserTest->addArticle(
534 ParserTest::chomp( $this->sectionData['article'] ),
535 $this->sectionData['text'], $this->lineNum );
537 $this->clearSection();
542 if ( $this->section == 'endhooks' ) {
543 $this->checkSection( 'hooks' );
545 foreach ( explode( "\n", $this->sectionData['hooks'] ) as $line ) {
546 $line = trim( $line );
549 $this->delayedParserTest->requireHook( $line );
553 $this->clearSection();
558 if ( $this->section == 'endfunctionhooks' ) {
559 $this->checkSection( 'functionhooks' );
561 foreach ( explode( "\n", $this->sectionData['functionhooks'] ) as $line ) {
562 $line = trim( $line );
565 $this->delayedParserTest->requireFunctionHook( $line );
569 $this->clearSection();
574 if ( $this->section == 'endtransparenthooks' ) {
575 $this->checkSection( 'transparenthooks' );
577 foreach ( explode( "\n", $this->sectionData['transparenthooks'] ) as $line ) {
578 $line = trim( $line );
581 $this->delayedParserTest->requireTransparentHook( $line );
585 $this->clearSection();
590 if ( $this->section == 'end' ) {
591 $this->checkSection( 'test' );
593 if ( $this->setupCurrentTest() ) {
596 } while ( $this->nextSubTest > 0 );
597 # go on to next test (since this was disabled)
598 $this->clearSection();
599 $this->delayedParserTest->reset();
603 if ( isset( $this->sectionData[$this->section] ) ) {
604 throw new MWException( "duplicate section '$this->section' "
605 . "at line {$this->lineNum} of $this->file\n" );
608 $this->sectionData[$this->section] = '';
613 if ( $this->section ) {
614 $this->sectionData[$this->section] .= $line;
622 * Clear section name and its data
624 private function clearSection() {
625 $this->sectionData = [];
626 $this->section = null;
631 * Verify the current section data has some value for the given token
632 * name(s) (first parameter).
633 * Throw an exception if it is not set, referencing current section
634 * and adding the current file name and line number
636 * @param string|array $tokens Expected token(s) that should have been
637 * mentioned before closing this section
638 * @param bool $fatal True iff an exception should be thrown if
639 * the section is not found.
640 * @return bool|string
641 * @throws MWException
643 private function checkSection( $tokens, $fatal = true ) {
644 if ( is_null( $this->section ) ) {
645 throw new MWException( __METHOD__ . " can not verify a null section!\n" );
647 if ( !is_array( $tokens ) ) {
648 $tokens = [ $tokens ];
650 if ( count( $tokens ) == 0 ) {
651 throw new MWException( __METHOD__ . " can not verify zero sections!\n" );
654 $data = $this->sectionData;
655 $tokens = array_filter( $tokens, function ( $token ) use ( $data ) {
656 return isset( $data[$token] );
659 if ( count( $tokens ) == 0 ) {
663 throw new MWException( sprintf(
664 "'%s' without '%s' at line %s of %s\n",
666 implode( ',', $tokens ),
671 if ( count( $tokens ) > 1 ) {
672 throw new MWException( sprintf(
673 "'%s' with unexpected tokens '%s' at line %s of %s\n",
675 implode( ',', $tokens ),
681 return array_values( $tokens )[0];
686 * An iterator for use as a phpunit data provider. Provides the test arguments
687 * in the order expected by NewParserTest::testParserTest().
689 class TestFileDataProvider extends TestFileIterator {
691 $test = parent::current();
707 * A class to delay execution of a parser test hooks.
709 class DelayedParserTest {
711 /** Initialized on construction */
714 private $transparentHooks;
716 public function __construct() {
721 * Init/reset or forgot about the current delayed test.
722 * Call to this will erase any hooks function that were pending.
724 public function reset() {
727 $this->transparentHooks = [];
731 * Called whenever we actually want to run the hook.
732 * Should be the case if we found the parserTest is not disabled
733 * @param ParserTest|NewParserTest $parserTest
735 * @throws MWException
737 public function unleash( &$parserTest ) {
738 if ( !( $parserTest instanceof ParserTest || $parserTest instanceof NewParserTest ) ) {
739 throw new MWException( __METHOD__ . " must be passed an instance of ParserTest or "
740 . "NewParserTest classes\n" );
743 # Trigger delayed hooks. Any failure will make us abort
744 foreach ( $this->hooks as $hook ) {
745 $ret = $parserTest->requireHook( $hook );
751 # Trigger delayed function hooks. Any failure will make us abort
752 foreach ( $this->fnHooks as $fnHook ) {
753 $ret = $parserTest->requireFunctionHook( $fnHook );
759 # Trigger delayed transparent hooks. Any failure will make us abort
760 foreach ( $this->transparentHooks as $hook ) {
761 $ret = $parserTest->requireTransparentHook( $hook );
767 # Delayed execution was successful.
772 * Similar to ParserTest object but does not run anything
773 * Use unleash() to really execute the hook
774 * @param string $hook
776 public function requireHook( $hook ) {
777 $this->hooks[] = $hook;
781 * Similar to ParserTest object but does not run anything
782 * Use unleash() to really execute the hook function
783 * @param string $fnHook
785 public function requireFunctionHook( $fnHook ) {
786 $this->fnHooks[] = $fnHook;
790 * Similar to ParserTest object but does not run anything
791 * Use unleash() to really execute the hook function
792 * @param string $hook
794 public function requireTransparentHook( $hook ) {
795 $this->transparentHooks[] = $hook;
801 * Initialize and detect the DjVu files support
806 * Initialises DjVu tools global with default values
808 public function __construct() {
809 global $wgDjvuRenderer, $wgDjvuDump, $wgDjvuToXML, $wgFileExtensions, $wgDjvuTxt;
811 $wgDjvuRenderer = $wgDjvuRenderer ? $wgDjvuRenderer : '/usr/bin/ddjvu';
812 $wgDjvuDump = $wgDjvuDump ? $wgDjvuDump : '/usr/bin/djvudump';
813 $wgDjvuToXML = $wgDjvuToXML ? $wgDjvuToXML : '/usr/bin/djvutoxml';
814 $wgDjvuTxt = $wgDjvuTxt ? $wgDjvuTxt : '/usr/bin/djvutxt';
816 if ( !in_array( 'djvu', $wgFileExtensions ) ) {
817 $wgFileExtensions[] = 'djvu';
822 * Returns true if the DjVu tools are usable
826 public function isEnabled() {
827 global $wgDjvuRenderer, $wgDjvuDump, $wgDjvuToXML, $wgDjvuTxt;
829 return is_executable( $wgDjvuRenderer )
830 && is_executable( $wgDjvuDump )
831 && is_executable( $wgDjvuToXML )
832 && is_executable( $wgDjvuTxt );
837 * Initialize and detect the tidy support
840 private $internalTidy;
841 private $externalTidy;
844 * Determine if there is a usable tidy.
846 public function __construct() {
849 $this->internalTidy = extension_loaded( 'tidy' ) &&
850 class_exists( 'tidy' ) && !wfIsHHVM();
852 $this->externalTidy = is_executable( $wgTidyBin ) ||
853 Installer::locateExecutableInDefaultPaths( [ $wgTidyBin ] )
858 * Returns true if we should use internal tidy.
862 public function isInternal() {
863 return $this->internalTidy;
867 * Returns true if tidy is usable
871 public function isEnabled() {
872 return $this->internalTidy || $this->externalTidy;