tests/parser/DbTestPreviewer.php

   1 <?php
   2 /**
   3  * This program is free software; you can redistribute it and/or modify
   4  * it under the terms of the GNU General Public License as published by
   5  * the Free Software Foundation; either version 2 of the License, or
   6  * (at your option) any later version.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along
  14  * with this program; if not, write to the Free Software Foundation, Inc.,
  15  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  16  * http://www.gnu.org/copyleft/gpl.html
  17  *
  18  * @file
  19  * @ingroup Testing
  20  */
  21
  22 use Wikimedia\Rdbms\IMaintainableDatabase;
  23
  24 class DbTestPreviewer extends TestRecorder {
  25         /** @var callable|false Test name filter callback */
  26         protected $filter;
  27         /** @var IMaintainableDatabase Database connection to the main DB */
  28         protected $db;
  29         /** @var int run ID number for the current run */
  30         protected $curRun;
  31         /** @var int|false run ID number for the previous run, if any */
  32         protected $prevRun;
  33         /** @var array<string,int> Result array */
  34         protected $results;
  35
  36         /**
  37          * This should be called before the table prefix is changed
  38          * @param IMaintainableDatabase $db
  39          * @param callable|false $filter
  40          */
  41         public function __construct( $db, $filter = false ) {
  42                 $this->db = $db;
  43                 $this->filter = $filter;
  44         }
  45
  46         /**
  47          * Set up result recording; insert a record for the run with the date
  48          * and all that fun stuff
  49          */
  50         public function start() {
  51                 if ( !$this->db->tableExists( 'testrun', __METHOD__ )
  52                         || !$this->db->tableExists( 'testitem', __METHOD__ )
  53                 ) {
  54                         print "WARNING> `testrun` table not found in database.\n";
  55                         $this->prevRun = false;
  56                 } else {
  57                         // We'll make comparisons against the previous run later...
  58                         $this->prevRun = $this->db->newSelectQueryBuilder()
  59                                 ->select( 'MAX(tr_id)' )
  60                                 ->from( 'testrun' )
  61                                 ->fetchField();
  62                 }
  63
  64                 $this->results = [];
  65         }
  66
  67         public function record( ParserTestResult $result ) {
  68                 $desc = $result->getDescription();
  69                 $this->results[$desc] = $result->isSuccess() ? 1 : 0;
  70         }
  71
  72         public function report() {
  73                 if ( $this->prevRun ) {
  74                         // f = fail, p = pass, n = nonexistent
  75                         // codes show before then after
  76                         $table = [
  77                                 'fp' => 'previously failing test(s) now PASSING! :)',
  78                                 'pn' => 'previously PASSING test(s) removed o_O',
  79                                 'np' => 'new PASSING test(s) :)',
  80
  81                                 'pf' => 'previously passing test(s) now FAILING! :(',
  82                                 'fn' => 'previously FAILING test(s) removed O_o',
  83                                 'nf' => 'new FAILING test(s) :(',
  84                                 'ff' => 'still FAILING test(s) :(',
  85                         ];
  86
  87                         $prevResults = [];
  88
  89                         $res = $this->db->newSelectQueryBuilder()
  90                                 ->select( [ 'ti_name', 'ti_success' ] )
  91                                 ->from( 'testitem' )
  92                                 ->where( [ 'ti_run' => $this->prevRun ] )
  93                                 ->caller( __METHOD__ )->fetchResultSet();
  94                         $filter = $this->filter;
  95
  96                         foreach ( $res as $row ) {
  97                                 if ( !$filter || $filter( $row->ti_name ) ) {
  98                                         $prevResults[$row->ti_name] = $row->ti_success;
  99                                 }
 100                         }
 101
 102                         $combined = array_keys( $this->results + $prevResults );
 103
 104                         # Determine breakdown by change type
 105                         $breakdown = [];
 106                         foreach ( $combined as $test ) {
 107                                 if ( !isset( $prevResults[$test] ) ) {
 108                                         $before = 'n';
 109                                 } elseif ( $prevResults[$test] == 1 ) {
 110                                         $before = 'p';
 111                                 } else /* if ( $prevResults[$test] == 0 ) */ {
 112                                         $before = 'f';
 113                                 }
 114
 115                                 if ( !isset( $this->results[$test] ) ) {
 116                                         $after = 'n';
 117                                 } elseif ( $this->results[$test] == 1 ) {
 118                                         $after = 'p';
 119                                 } else /* if ( $this->results[$test] == 0 ) */ {
 120                                         $after = 'f';
 121                                 }
 122
 123                                 $code = $before . $after;
 124
 125                                 if ( isset( $table[$code] ) ) {
 126                                         $breakdown[$code][$test] = $this->getTestStatusInfo( $test, $after );
 127                                 }
 128                         }
 129
 130                         # Write out results
 131                         foreach ( $table as $code => $label ) {
 132                                 if ( !empty( $breakdown[$code] ) ) {
 133                                         $count = count( $breakdown[$code] );
 134                                         printf( "\n%4d %s\n", $count, $label );
 135
 136                                         foreach ( $breakdown[$code] as $differing_test_name => $statusInfo ) {
 137                                                 // @phan-suppress-next-line SecurityCheck-XSS CLI-only script
 138                                                 print "      * $differing_test_name  [$statusInfo]\n";
 139                                         }
 140                                 }
 141                         }
 142                 } else {
 143                         print "No previous test runs to compare against.\n";
 144                 }
 145
 146                 print "\n";
 147         }
 148
 149         /**
 150          * Returns a string giving information about when a test last had a status change.
 151          * Could help to track down when regressions were introduced, as distinct from tests
 152          * which have never passed (which are more change requests than regressions).
 153          * @param string $testname
 154          * @param string $after
 155          * @return string
 156          */
 157         private function getTestStatusInfo( $testname, $after ) {
 158                 // If we're looking at a test that has just been removed, then say when it first appeared.
 159                 if ( $after == 'n' ) {
 160                         $changedRun = $this->db->newSelectQueryBuilder()
 161                                 ->select( 'MIN(ti_run)' )
 162                                 ->from( 'testitem' )
 163                                 ->where( [ 'ti_name' => $testname ] )
 164                                 ->caller( __METHOD__ )->fetchField();
 165                         $appear = $this->db->newSelectQueryBuilder()
 166                                 ->select( [ 'tr_date', 'tr_mw_version' ] )
 167                                 ->from( 'testrun' )
 168                                 ->where( [ 'tr_id' => $changedRun ] )
 169                                 ->caller( __METHOD__ )->fetchRow();
 170
 171                         return "First recorded appearance: "
 172                                 . date( "d-M-Y H:i:s", strtotime( $appear->tr_date ) )
 173                                 . ", " . $appear->tr_mw_version;
 174                 }
 175
 176                 // Otherwise, this test has previous recorded results.
 177                 // See when this test last had a different result to what we're seeing now.
 178                 $conds = [
 179                         'ti_name' => $testname,
 180                         'ti_success' => ( $after == 'f' ? "1" : "0" ) ];
 181
 182                 if ( $this->curRun ) {
 183                         $conds[] = $this->db->expr( 'ti_run', '!=', $this->curRun );
 184                 }
 185
 186                 $changedRun = $this->db->newSelectQueryBuilder()
 187                         ->select( 'MAX(ti_run)' )
 188                         ->from( 'testitem' )
 189                         ->where( $conds )
 190                         ->caller( __METHOD__ )->fetchField();
 191
 192                 // If no record of ever having had a different result.
 193                 if ( $changedRun === null ) {
 194                         if ( $after == "f" ) {
 195                                 return "Has never passed";
 196                         } else {
 197                                 return "Has never failed";
 198                         }
 199                 }
 200
 201                 // Otherwise, we're looking at a test whose status has changed.
 202                 // (i.e. it used to work, but now doesn't; or used to fail, but is now fixed.)
 203                 // In this situation, give as much info as we can as to when it changed status.
 204                 $pre = $this->db->newSelectQueryBuilder()
 205                         ->select( [ 'tr_date', 'tr_mw_version' ] )
 206                         ->from( 'testrun' )
 207                         ->where( [ 'tr_id' => $changedRun ] )
 208                         ->caller( __METHOD__ )->fetchRow();
 209                 $post = $this->db->newSelectQueryBuilder()
 210                         ->select( [ 'tr_date', 'tr_mw_version' ] )
 211                         ->from( 'testrun' )
 212                         ->where( $this->db->expr( 'tr_id', '>', $changedRun ) )
 213                         ->orderBy( 'tr_id' )
 214                         ->limit( 1 )
 215                         ->caller( __METHOD__ )->fetchRow();
 216
 217                 if ( $post ) {
 218                         $postDate = date( "d-M-Y H:i:s", strtotime( $post->tr_date ) ) . ", {$post->tr_mw_version}";
 219                 } else {
 220                         $postDate = 'now';
 221                 }
 222
 223                 return ( $after == "f" ? "Introduced" : "Fixed" ) . " between "
 224                         . date( "d-M-Y H:i:s", strtotime( $pre->tr_date ) ) . ", " . $pre->tr_mw_version
 225                         . " and $postDate";
 226         }
 227 }