modules/test/libraries/Ninja_Reports_Test.php

   1 <?php
   2
   3 /**
   4  * A model that runs tests on the reports model,
   5  * based on a special test-DSL
   6  *
   7  * Inherits from Reports_Model for "dammit, it's protected!" reasons
   8  */
   9 class Ninja_Reports_Test extends Status_Reports_Model
  10 {
  11         public $test_file = false; /**< The file name we're testing */
  12         private $total = 0;
  13         public $description = false; /**< A string describing the purpose of this test */
  14         private $tests;
  15         private $results = array();
  16         private $config_files = false;
  17         public $passed = 0; /**< Number of passed tests */
  18         public $failed = 0; /**< Number of failed tests */
  19         private $logfiles = false;
  20         private $logfile = false;
  21         private $sqlfile = false;
  22         private $table_name = false;
  23         private $test_globals = array();
  24         private $interesting_prefixes = array();
  25         public $sub_reports = 0; /**< The number of sub reports */
  26         private $color_red   = '';
  27         private $color_green = '';
  28         private $color_reset = '';
  29         public $db_name; /**< Database name */
  30         public $db_user; /**< Database user */
  31         public $db_pass; /**< Database password */
  32         public $db_type; /**< Database type */
  33         public $db_host; /**< Database hostname */
  34         public $importer; /**< The command used to import logs into the database */
  35
  36         /**
  37          * Run new test file. Will parse the file, but not run it
  38          */
  39         public function __construct($test_file)
  40         {
  41                 if (PHP_SAPI === 'cli' && posix_isatty(STDOUT)) {
  42                         $this->color_red   =  "\033[31m";
  43                         $this->color_green =  "\033[32m";
  44                         $this->color_reset =  "\033[0m";
  45                 }
  46
  47                 if (!$test_file)
  48                         return false;
  49
  50                 $this->tests = $this->parse_test($test_file);
  51                 $this->test_file = $test_file;
  52         }
  53
  54         private function red($str)
  55         {
  56                 return $this->color_red.$str.$this->color_reset;
  57         }
  58
  59         private function green($str)
  60         {
  61                 return $this->color_green.$str.$this->color_reset;
  62         }
  63
  64         private function verify_correct($duration, $correct)
  65         {
  66                 $total = array();
  67                 $this->interesting_prefixes = array();
  68
  69                 foreach ($correct as $k => $v) {
  70                         if (!is_numeric($v))
  71                                 continue;
  72                         $prefix = explode('_', $k);
  73                         $prefix = $prefix[0];
  74                         if (!isset($total[$prefix]))
  75                                 $total[$prefix] = $v;
  76                         else
  77                                 $total[$prefix] += $v;
  78                         $this->interesting_prefixes[$prefix] = $prefix;
  79                 }
  80                 foreach ($total as $prefix => $tot) {
  81                         if ($tot == $duration || $prefix === 'TOTAL' || $prefix === 'PERCENT')
  82                                 continue;
  83                         echo "Wonky 'correct' for prefix $prefix: total != duration ($tot != $duration)\n";
  84                         print_r($correct);
  85                         return false;
  86                 }
  87                 return true;
  88         }
  89
  90         private function run_test($params)
  91         {
  92                 $timeperiods = array();
  93                 foreach ($this->test_globals as $k => $v) {
  94                         if ($k === 'timeperiod')
  95                                 $timeperiods[] = $v;
  96                         else if (!isset($params[$k]))
  97                                 $params[$k] = $v;
  98                 }
  99
 100                 if (!$params)
 101                         return false;
 102
 103                 if (!($correct = arr::search($params, 'correct'))) {
 104                         echo "No 'correct' block set for test. Bailing out\n";
 105                         return false;
 106                 }
 107                 $start_time = arr::search($params, 'start_time');
 108                 $end_time = arr::search($params, 'end_time');
 109                 $timeperiod = arr::search($params, 'timeperiod');
 110                 if ($timeperiod)
 111                         $timeperiods[] =& $timeperiod;
 112                 unset($params['correct']);
 113                 unset($params['timeperiod']);
 114
 115                 if (!$this->verify_correct($end_time - $start_time, $correct))
 116                         return -1;
 117
 118                 Old_Timeperiod_Model::$precreated = array();
 119                 foreach ($timeperiods as $idx => &$tp) {
 120                         if (!isset($tp['timeperiod_name']))
 121                                 $tp['timeperiod_name'] = 'the_timeperiod'.$idx;
 122
 123                         $tpobj = Old_Timeperiod_Model::instance(array('start_time' => $start_time, 'end_time' => $end_time, 'rpttimeperiod' => $tp['timeperiod_name']));
 124                         $tpobj->set_timeperiod_data($tp);
 125                         $tpobj->resolve_timeperiods();
 126                 }
 127
 128                 $this->sub_reports = 0;
 129                 $opts = new Test_report_options();
 130                 foreach ($params as $k => $v) {
 131                         if ($k == 'objects') {
 132                                 if ($params['report_type'] == 'hosts' || $params['report_type'] == 'services') {
 133                                         $this->sub_reports = count($v);
 134                                 }
 135                                 if ($params['report_type'] == 'hostgroups' || $params['report_type'] == 'servicegroups') {
 136                                         $opts->members = array_merge($opts->members, $v);
 137                                         $v = array_keys($v);
 138                                         $this->sub_reports = count($opts->members);
 139                                 }
 140                         }
 141                         if (!$opts->set($k, $v))
 142                                 echo "Failed to set option '$k' to '$v'\n";
 143                 }
 144                 $opts->properties_copy['rpttimeperiod']['options'][$timeperiod['timeperiod_name']] = $timeperiod['timeperiod_name'];
 145                 $opts['rpttimeperiod'] = $timeperiod['timeperiod_name'];
 146
 147                 # force logs to be kept so we can analyze them and make
 148                 # sure the durations add up
 149                 $opts['include_trends'] = true;
 150
 151                 $rpt = new Status_Reports_Model($opts, $this->table_name);
 152                 $return_arr = $rpt->get_uptime();
 153                 $this->result = $return_arr;
 154                 $this->report_objects[$this->cur_test] = $rpt;
 155
 156                 if (!$return_arr) {
 157                         return false;
 158                 }
 159
 160                 return $this->compare_test_result($return_arr, $correct, $rpt);
 161         }
 162
 163         private function parse_test($test_file = false)
 164         {
 165                 if (!$test_file)
 166                         return false;
 167
 168                 $req = array('description', 'logfiles');
 169                 $params = array();
 170
 171                 $buf = file_get_contents($test_file);
 172                 $lines = explode("\n", $buf);
 173                 $block = false;
 174                 $pushed_blocks = array();
 175                 $pushed_names = array();
 176                 $block_name = false;
 177                 $num_line = 0;
 178                 foreach ($lines as $raw_line) {
 179                         $num_line++;
 180                         $line = trim($raw_line);
 181                         if (!strlen($line) || $line{0} === '#')
 182                                 continue;
 183
 184                         if ($line{0} === '}') {
 185                                 if (!empty($pushed_blocks)) {
 186                                         $tmp = array_pop($pushed_blocks);
 187                                         $tmp[$block_name] = $block;
 188                                         $block = $tmp;
 189                                         $block_name = array_pop($pushed_names);
 190                                         $tmp = false;
 191                                 }
 192                                 else {
 193                                         if ($block_name === 'global_vars')
 194                                                 $this->test_globals = $block;
 195                                         elseif ($block_name === 'logfiles')
 196                                                 $this->logfiles = $block;
 197                                         else
 198                                                 $params[$block_name] = $block;
 199
 200                                         $block = $block_name = false;
 201                                 }
 202                                 continue;
 203                         }
 204
 205                         if ($line{strlen($line) - 1} === '{') {
 206                                 $ary = preg_split("/[\t ]*{[\t ]*/", $line);
 207                                 if ($block_name) {
 208                                         array_push($pushed_blocks, $block);
 209                                         array_push($pushed_names, $block_name);
 210                                 }
 211                                 $block_name = $ary[0];
 212                                 $block = array();
 213                                 continue;
 214                         }
 215
 216                         # regular variable, or possibly a single string
 217                         $ary = preg_split("/[\t ]*=[\t ]/", $line);
 218
 219                         if (count($ary) !== 2) {
 220                                 if ($block !== false) {
 221                                         $block[] = $line;
 222                                 }
 223                                 else {
 224                                         echo "Line $num_line in $test_file is malformed: $line\n";
 225                                 }
 226                                 continue;
 227                         }
 228                         $k = $ary[0];
 229                         $v = $ary[1];
 230                         if ($block !== false) {
 231                                 $block[$k] = $v;
 232                         }
 233                         else {
 234                                 switch ($k) {
 235                                  case 'description':
 236                                         $this->description = $v;
 237                                         break;
 238                                  case 'config_files':
 239                                         $this->config_files = $v;
 240                                         break;
 241                                  case 'logfile':
 242                                         $this->logfile = $v;
 243                                         break;
 244                                  case 'sqlfile':
 245                                         $this->sqlfile = $v;
 246                                         break;
 247                                  case 'db_table':
 248                                         $this->table_name = $v;
 249                                         break;
 250                                  default:
 251                                         if (!is_array($v)) {
 252                                                 $this->crash("Illegal variable: $k = $v\n");
 253                                                 exit(1);
 254                                         }
 255                                         $params[$k] = $v;
 256                                 }
 257                         }
 258                 }
 259
 260                 #       print_r($params);
 261                 //recurse_print($params);
 262                 $this->params = $params;
 263                 return $params;
 264         }
 265
 266         /**
 267          * Run the actual test file
 268          */
 269         public function run_test_series()
 270         {
 271                 echo "Preparing for test-series '" . $this->description . "'\n";
 272
 273                 $this->details = array();
 274                 if ($this->sqlfile) {
 275                         exec('mysql -u'.$this->db_user.' -p'.$this->db_pass.' '.$this->db_name.' < '.'test/unit_test/reports/'.$this->sqlfile);
 276                         $this->table_name = substr($this->sqlfile, 0, strpos($this->sqlfile, '.'));
 277                 }
 278                 else {
 279                         if ($this->logfile)
 280                                 $this->logfiles[] = "test/unit_test/reports/".$this->logfile;
 281
 282                         $result = $this->import_logs();
 283                         if ($result < 0)
 284                                 return $result;
 285                 }
 286
 287                 foreach ($this->tests as $test_name => $params) {
 288                         $this->cur_test = $test_name;
 289                         $result = $this->run_test($params);
 290                         printf("  %-7s $test_name\n", $result === true ? $this->green('OK') : $this->red('FAILED'));
 291                         if ($result === true)
 292                                 $this->passed++;
 293                         else {
 294                                 $this->details[$test_name] = $result;
 295                                 $this->failed++;
 296                         }
 297                 }
 298
 299                 foreach ($this->details as $test_name => $fail_desc) {
 300                         echo "$test_name: ";
 301                         print_r($fail_desc);
 302                         echo "\n";
 303                 }
 304                 echo "\n";
 305                 return $this->failed;
 306         }
 307
 308         private function import_logs()
 309         {
 310                 if (!$this->logfiles) {
 311                         echo "No logfiles to import\n";
 312                         return true;
 313                 }
 314                 $lfiles = join(" ", $this->logfiles);
 315
 316                 $line = exec("cat $lfiles | md5sum", $output, $retcode);
 317                 $ary = explode(" ", $line);
 318                 $checksum = $ary[0];
 319                 $table_name = substr($this->description, 0, 20) . substr($checksum, 0, 10);
 320                 $table_name = preg_replace("/[^A-Za-z0-9_]/", "_", $table_name);
 321                 $this->table_name = $table_name;
 322
 323                 echo "Using db table '".$this->table_name."'\n";
 324                 $cached = true;
 325                 $db = Database::instance();
 326                 try {
 327                     $db->query("SELECT * FROM ".$this->table_name." LIMIT 1");
 328                 }
 329                 catch (Kohana_Database_Exception $e) {
 330                     $cached = false;
 331                 }
 332
 333                 if ($cached) {
 334                         echo "Data is cached\n";
 335                 } else {
 336                         if ($this->db_type === 'oracle')
 337                                 $sql = "CREATE TABLE $table_name AS (SELECT * FROM report_data WHERE rownum < 0)";
 338                         else
 339                                 $sql = "CREATE TABLE $table_name AS SELECT * FROM report_data LIMIT 0";
 340                         echo "Building table [$table_name]. This might take a moment or three...\n";
 341                         if( ! $db->query($sql)) {
 342                                 $this->crash("Error creating table $table_name: ".$db->error_message());
 343                         }
 344                         echo "Importing $lfiles to '$table_name'\n";
 345                         $cmd = $this->importer .
 346                                 " --db-name=".$this->db_name .
 347                                 " --db-table=".$this->table_name .
 348                                 " --db-user=".$this->db_user .
 349                                 " --db-pass=".$this->db_pass." " .
 350                                 " --db-host=".$this->db_host." " .
 351                                 " --db-type=".$this->db_type." " .
 352                                 join(" ", $this->logfiles).' 2>&1';
 353                         $out = array();
 354                         exec($cmd, $out, $retval);
 355                         echo "$cmd\n".implode("\n", $out);
 356                         if ($retval) {
 357                                 echo "import failed. cleaning up and skipping test\n";
 358                                 echo $cmd."\n";
 359                                 $db->query("DROP TABLE ".$this->table_name);
 360                                 return -1;
 361                         }
 362                 }
 363
 364                 return true;
 365         }
 366
 367         private function count_sub_reports($top)
 368         {
 369                 $i = 0;
 370                 foreach ($top as $middle) {
 371                         if (!is_array($middle) || !isset($middle['states']))
 372                                 continue;
 373                         foreach ($middle as $bottom) {
 374                                 if (is_array($bottom) && isset($bottom['states']))
 375                                         $i++;
 376                         }
 377                 }
 378                 return $i;
 379         }
 380
 381         private function log_duration($st_log)
 382         {
 383                 if (!is_array($st_log))
 384                         return 0;
 385
 386                 $duration = 0;
 387                 foreach ($st_log as $le) {
 388                         $duration += $le['duration'];
 389                 }
 390                 return $duration;
 391         }
 392
 393         /**
 394         * compare_test_result
 395         *
 396         * Compare result from test with correct values
 397         * @return mixed true or array with diff
 398         *
 399         */
 400         private function compare_test_result($full_result, $correct, $rpt)
 401         {
 402                 if (empty($full_result) || empty($full_result['states']))
 403                         $this->crash("No test result\n");
 404                 $states = $full_result['states'];
 405
 406                 if (empty($correct))
 407                         $this->crash("No \$correct\n");
 408
 409                 $failed = false;
 410                 foreach ($correct as $k => $v) {
 411                         if ($k === 'subs') {
 412                                 foreach ($v as $sub_name => $sub_correct) {
 413                                         $sub = false;
 414                                         foreach ($full_result as $group) {
 415                                                 if (!isset($group['states']) || !$group['states'])
 416                                                         continue;
 417                                                 foreach ($group as $obj) {
 418                                                         $tmp_sub_name = '';
 419                                                         if (!isset($obj['states']) || !$obj['states'])
 420                                                                 continue;
 421                                                         $tmp_sub_name .= $obj['states']['HOST_NAME'];
 422                                                         if (isset($obj['states']['SERVICE_DESCRIPTION']))
 423                                                                 $tmp_sub_name .= ';'.$obj['states']['SERVICE_DESCRIPTION'];
 424                                                         if ($tmp_sub_name === $sub_name) {
 425                                                                 $sub = $obj;
 426                                                                 break;
 427                                                         }
 428                                                 }
 429                                         }
 430                                         if (!$sub) {
 431                                                 $failed[$sub_name] = "expected sub report $sub_name, but couldn't find it";
 432                                                 continue;
 433                                         }
 434                                         foreach ($sub_correct as $sk => $sv) {
 435                                                 if (!isset($sub['states']) || !isset($sub['states'][$sk])) {
 436                                                         $failed["$sub_name;$sk"] = "expected=$sv; lib_reports=(not set)";
 437                                                         continue;
 438                                                 }
 439                                                 if (strcmp($sub['states'][$sk], $sv)) {
 440                                                         $failed["$sub_name;$sk"] = "expected=$sv; lib_reports={$sub['states'][$sk]}";
 441                                                 }
 442                                         }
 443                                 }
 444                                 continue;
 445                         }
 446                         if (!isset($states[$k])) {
 447                                 $failed[$k] = "expected=$v; lib_reports=(not set)";
 448                                 continue;
 449                         }
 450                         if (strcmp($states[$k], $v)) {
 451                                 $failed[$k] = "expected=$v; lib_reports=$states[$k]";
 452                         }
 453                 }
 454
 455                 $subs = $this->count_sub_reports($full_result);
 456                 if ($this->sub_reports != $subs) {
 457                         if ($this->sub_reports === false) {
 458                                 $failed['sub-reports'] = "There are sub-reports, but shouldn't be";
 459                         }
 460                         else {
 461                                 $failed['sub-reports'] = "Expected $this->sub_reports sub reports. Got $subs";
 462                         }
 463                 }
 464
 465                 # check duration for all sub-reports individually
 466                 foreach ($full_result as $k => $l) {
 467                         if (!is_numeric($k))
 468                                 continue;
 469                         foreach ($l as $k2 => $obj) {
 470                                 if (!is_numeric($k2))
 471                                         continue;
 472                                 $duration = $this->log_duration($obj['log']);
 473                                 if ($duration != $rpt->options['end_time'] - $rpt->options['start_time']) {
 474                                         $failed['st_log ' . $k] = "Log duration doesn't match report period duration (expected ".($rpt->options['end_time'] - $rpt->options['start_time']).", was $duration)";
 475                                 }
 476                         }
 477                 }
 478
 479                 if (empty($failed)) {
 480                         return true;
 481                 }
 482                 foreach ($states as $k => $v) {
 483                         $prefix = explode('_', $k);
 484                         $prefix = $prefix[0];
 485                         if (!isset($this->interesting_prefixes[$prefix]))
 486                                 continue;
 487                         if ($v != 0 && empty($correct[$k])) {
 488                                 $failed[$k] = "expected=0; lib_reports=$v";
 489                         }
 490                 }
 491
 492                 return $failed;
 493         }
 494
 495         private function crash($msg)
 496         {
 497                 echo "test.php: $msg\n";
 498                 exit(1);
 499         }
 500 }