tools/testing/ktest/ktest.pl

   1 #!/usr/bin/perl -w
   2 #
   3 # Copyright 2010 - Steven Rostedt <srostedt@redhat.com>, Red Hat Inc.
   4 # Licensed under the terms of the GNU GPL License version 2
   5 #
   6
   7 use strict;
   8 use IPC::Open2;
   9 use Fcntl qw(F_GETFL F_SETFL O_NONBLOCK);
  10 use File::Path qw(mkpath);
  11 use File::Copy qw(cp);
  12 use FileHandle;
  13
  14 my $VERSION = "0.2";
  15
  16 $| = 1;
  17
  18 my %opt;
  19 my %repeat_tests;
  20 my %repeats;
  21 my %default;
  22
  23 #default opts
  24 $default{"NUM_TESTS"}           = 1;
  25 $default{"REBOOT_TYPE"}         = "grub";
  26 $default{"TEST_TYPE"}           = "test";
  27 $default{"BUILD_TYPE"}          = "randconfig";
  28 $default{"MAKE_CMD"}            = "make";
  29 $default{"TIMEOUT"}             = 120;
  30 $default{"TMP_DIR"}             = "/tmp/ktest/\${MACHINE}";
  31 $default{"SLEEP_TIME"}          = 60;   # sleep time between tests
  32 $default{"BUILD_NOCLEAN"}       = 0;
  33 $default{"REBOOT_ON_ERROR"}     = 0;
  34 $default{"POWEROFF_ON_ERROR"}   = 0;
  35 $default{"REBOOT_ON_SUCCESS"}   = 1;
  36 $default{"POWEROFF_ON_SUCCESS"} = 0;
  37 $default{"BUILD_OPTIONS"}       = "";
  38 $default{"BISECT_SLEEP_TIME"}   = 60;   # sleep time between bisects
  39 $default{"PATCHCHECK_SLEEP_TIME"} = 60; # sleep time between patch checks
  40 $default{"CLEAR_LOG"}           = 0;
  41 $default{"BISECT_MANUAL"}       = 0;
  42 $default{"BISECT_SKIP"}         = 1;
  43 $default{"SUCCESS_LINE"}        = "login:";
  44 $default{"DETECT_TRIPLE_FAULT"} = 1;
  45 $default{"NO_INSTALL"}          = 0;
  46 $default{"BOOTED_TIMEOUT"}      = 1;
  47 $default{"DIE_ON_FAILURE"}      = 1;
  48 $default{"SSH_EXEC"}            = "ssh \$SSH_USER\@\$MACHINE \$SSH_COMMAND";
  49 $default{"SCP_TO_TARGET"}       = "scp \$SRC_FILE \$SSH_USER\@\$MACHINE:\$DST_FILE";
  50 $default{"REBOOT"}              = "ssh \$SSH_USER\@\$MACHINE reboot";
  51 $default{"STOP_AFTER_SUCCESS"}  = 10;
  52 $default{"STOP_AFTER_FAILURE"}  = 60;
  53 $default{"STOP_TEST_AFTER"}     = 600;
  54 $default{"LOCALVERSION"}        = "-test";
  55
  56 my $ktest_config;
  57 my $version;
  58 my $machine;
  59 my $ssh_user;
  60 my $tmpdir;
  61 my $builddir;
  62 my $outputdir;
  63 my $output_config;
  64 my $test_type;
  65 my $build_type;
  66 my $build_options;
  67 my $pre_build;
  68 my $post_build;
  69 my $pre_build_die;
  70 my $post_build_die;
  71 my $reboot_type;
  72 my $reboot_script;
  73 my $power_cycle;
  74 my $reboot;
  75 my $reboot_on_error;
  76 my $poweroff_on_error;
  77 my $die_on_failure;
  78 my $powercycle_after_reboot;
  79 my $poweroff_after_halt;
  80 my $ssh_exec;
  81 my $scp_to_target;
  82 my $power_off;
  83 my $grub_menu;
  84 my $grub_number;
  85 my $target;
  86 my $make;
  87 my $post_install;
  88 my $no_install;
  89 my $noclean;
  90 my $minconfig;
  91 my $start_minconfig;
  92 my $start_minconfig_defined;
  93 my $output_minconfig;
  94 my $ignore_config;
  95 my $addconfig;
  96 my $in_bisect = 0;
  97 my $bisect_bad = "";
  98 my $reverse_bisect;
  99 my $bisect_manual;
 100 my $bisect_skip;
 101 my $config_bisect_good;
 102 my $in_patchcheck = 0;
 103 my $run_test;
 104 my $redirect;
 105 my $buildlog;
 106 my $dmesg;
 107 my $monitor_fp;
 108 my $monitor_pid;
 109 my $monitor_cnt = 0;
 110 my $sleep_time;
 111 my $bisect_sleep_time;
 112 my $patchcheck_sleep_time;
 113 my $ignore_warnings;
 114 my $store_failures;
 115 my $test_name;
 116 my $timeout;
 117 my $booted_timeout;
 118 my $detect_triplefault;
 119 my $console;
 120 my $reboot_success_line;
 121 my $success_line;
 122 my $stop_after_success;
 123 my $stop_after_failure;
 124 my $stop_test_after;
 125 my $build_target;
 126 my $target_image;
 127 my $localversion;
 128 my $iteration = 0;
 129 my $successes = 0;
 130
 131 my %entered_configs;
 132 my %config_help;
 133 my %variable;
 134 my %force_config;
 135
 136 # do not force reboots on config problems
 137 my $no_reboot = 1;
 138
 139 # default variables that can be used
 140 chomp ($variable{"PWD"} = `pwd`);
 141
 142 $config_help{"MACHINE"} = << "EOF"
 143  The machine hostname that you will test.
 144 EOF
 145     ;
 146 $config_help{"SSH_USER"} = << "EOF"
 147  The box is expected to have ssh on normal bootup, provide the user
 148   (most likely root, since you need privileged operations)
 149 EOF
 150     ;
 151 $config_help{"BUILD_DIR"} = << "EOF"
 152  The directory that contains the Linux source code (full path).
 153 EOF
 154     ;
 155 $config_help{"OUTPUT_DIR"} = << "EOF"
 156  The directory that the objects will be built (full path).
 157  (can not be same as BUILD_DIR)
 158 EOF
 159     ;
 160 $config_help{"BUILD_TARGET"} = << "EOF"
 161  The location of the compiled file to copy to the target.
 162  (relative to OUTPUT_DIR)
 163 EOF
 164     ;
 165 $config_help{"TARGET_IMAGE"} = << "EOF"
 166  The place to put your image on the test machine.
 167 EOF
 168     ;
 169 $config_help{"POWER_CYCLE"} = << "EOF"
 170  A script or command to reboot the box.
 171
 172  Here is a digital loggers power switch example
 173  POWER_CYCLE = wget --no-proxy -O /dev/null -q  --auth-no-challenge 'http://admin:admin\@power/outlet?5=CCL'
 174
 175  Here is an example to reboot a virtual box on the current host
 176  with the name "Guest".
 177  POWER_CYCLE = virsh destroy Guest; sleep 5; virsh start Guest
 178 EOF
 179     ;
 180 $config_help{"CONSOLE"} = << "EOF"
 181  The script or command that reads the console
 182
 183   If you use ttywatch server, something like the following would work.
 184 CONSOLE = nc -d localhost 3001
 185
 186  For a virtual machine with guest name "Guest".
 187 CONSOLE =  virsh console Guest
 188 EOF
 189     ;
 190 $config_help{"LOCALVERSION"} = << "EOF"
 191  Required version ending to differentiate the test
 192  from other linux builds on the system.
 193 EOF
 194     ;
 195 $config_help{"REBOOT_TYPE"} = << "EOF"
 196  Way to reboot the box to the test kernel.
 197  Only valid options so far are "grub" and "script".
 198
 199  If you specify grub, it will assume grub version 1
 200  and will search in /boot/grub/menu.lst for the title \$GRUB_MENU
 201  and select that target to reboot to the kernel. If this is not
 202  your setup, then specify "script" and have a command or script
 203  specified in REBOOT_SCRIPT to boot to the target.
 204
 205  The entry in /boot/grub/menu.lst must be entered in manually.
 206  The test will not modify that file.
 207 EOF
 208     ;
 209 $config_help{"GRUB_MENU"} = << "EOF"
 210  The grub title name for the test kernel to boot
 211  (Only mandatory if REBOOT_TYPE = grub)
 212
 213  Note, ktest.pl will not update the grub menu.lst, you need to
 214  manually add an option for the test. ktest.pl will search
 215  the grub menu.lst for this option to find what kernel to
 216  reboot into.
 217
 218  For example, if in the /boot/grub/menu.lst the test kernel title has:
 219  title Test Kernel
 220  kernel vmlinuz-test
 221  GRUB_MENU = Test Kernel
 222 EOF
 223     ;
 224 $config_help{"REBOOT_SCRIPT"} = << "EOF"
 225  A script to reboot the target into the test kernel
 226  (Only mandatory if REBOOT_TYPE = script)
 227 EOF
 228     ;
 229
 230 sub read_yn {
 231     my ($prompt) = @_;
 232
 233     my $ans;
 234
 235     for (;;) {
 236         print "$prompt [Y/n] ";
 237         $ans = <STDIN>;
 238         chomp $ans;
 239         if ($ans =~ /^\s*$/) {
 240             $ans = "y";
 241         }
 242         last if ($ans =~ /^y$/i || $ans =~ /^n$/i);
 243         print "Please answer either 'y' or 'n'.\n";
 244     }
 245     if ($ans !~ /^y$/i) {
 246         return 0;
 247     }
 248     return 1;
 249 }
 250
 251 sub get_ktest_config {
 252     my ($config) = @_;
 253     my $ans;
 254
 255     return if (defined($opt{$config}));
 256
 257     if (defined($config_help{$config})) {
 258         print "\n";
 259         print $config_help{$config};
 260     }
 261
 262     for (;;) {
 263         print "$config = ";
 264         if (defined($default{$config})) {
 265             print "\[$default{$config}\] ";
 266         }
 267         $ans = <STDIN>;
 268         $ans =~ s/^\s*(.*\S)\s*$/$1/;
 269         if ($ans =~ /^\s*$/) {
 270             if ($default{$config}) {
 271                 $ans = $default{$config};
 272             } else {
 273                 print "Your answer can not be blank\n";
 274                 next;
 275             }
 276         }
 277         $entered_configs{$config} = process_variables($ans);
 278         last;
 279     }
 280 }
 281
 282 sub get_ktest_configs {
 283     get_ktest_config("MACHINE");
 284     get_ktest_config("SSH_USER");
 285     get_ktest_config("BUILD_DIR");
 286     get_ktest_config("OUTPUT_DIR");
 287     get_ktest_config("BUILD_TARGET");
 288     get_ktest_config("TARGET_IMAGE");
 289     get_ktest_config("POWER_CYCLE");
 290     get_ktest_config("CONSOLE");
 291     get_ktest_config("LOCALVERSION");
 292
 293     my $rtype = $opt{"REBOOT_TYPE"};
 294
 295     if (!defined($rtype)) {
 296         if (!defined($opt{"GRUB_MENU"})) {
 297             get_ktest_config("REBOOT_TYPE");
 298             $rtype = $entered_configs{"REBOOT_TYPE"};
 299         } else {
 300             $rtype = "grub";
 301         }
 302     }
 303
 304     if ($rtype eq "grub") {
 305         get_ktest_config("GRUB_MENU");
 306     } else {
 307         get_ktest_config("REBOOT_SCRIPT");
 308     }
 309 }
 310
 311 sub process_variables {
 312     my ($value, $remove_undef) = @_;
 313     my $retval = "";
 314
 315     # We want to check for '\', and it is just easier
 316     # to check the previous characet of '$' and not need
 317     # to worry if '$' is the first character. By adding
 318     # a space to $value, we can just check [^\\]\$ and
 319     # it will still work.
 320     $value = " $value";
 321
 322     while ($value =~ /(.*?[^\\])\$\{(.*?)\}(.*)/) {
 323         my $begin = $1;
 324         my $var = $2;
 325         my $end = $3;
 326         # append beginning of value to retval
 327         $retval = "$retval$begin";
 328         if (defined($variable{$var})) {
 329             $retval = "$retval$variable{$var}";
 330         } elsif (defined($remove_undef) && $remove_undef) {
 331             # for if statements, any variable that is not defined,
 332             # we simple convert to 0
 333             $retval = "${retval}0";
 334         } else {
 335             # put back the origin piece.
 336             $retval = "$retval\$\{$var\}";
 337         }
 338         $value = $end;
 339     }
 340     $retval = "$retval$value";
 341
 342     # remove the space added in the beginning
 343     $retval =~ s/ //;
 344
 345     return "$retval"
 346 }
 347
 348 sub set_value {
 349     my ($lvalue, $rvalue, $override, $overrides, $name) = @_;
 350
 351     if (defined($opt{$lvalue})) {
 352         if (!$override || defined(${$overrides}{$lvalue})) {
 353             my $extra = "";
 354             if ($override) {
 355                 $extra = "In the same override section!\n";
 356             }
 357             die "$name: $.: Option $lvalue defined more than once!\n$extra";
 358         }
 359         ${$overrides}{$lvalue} = $rvalue;
 360     }
 361     if ($rvalue =~ /^\s*$/) {
 362         delete $opt{$lvalue};
 363     } else {
 364         $rvalue = process_variables($rvalue);
 365         $opt{$lvalue} = $rvalue;
 366     }
 367 }
 368
 369 sub set_variable {
 370     my ($lvalue, $rvalue) = @_;
 371
 372     if ($rvalue =~ /^\s*$/) {
 373         delete $variable{$lvalue};
 374     } else {
 375         $rvalue = process_variables($rvalue);
 376         $variable{$lvalue} = $rvalue;
 377     }
 378 }
 379
 380 sub process_compare {
 381     my ($lval, $cmp, $rval) = @_;
 382
 383     # remove whitespace
 384
 385     $lval =~ s/^\s*//;
 386     $lval =~ s/\s*$//;
 387
 388     $rval =~ s/^\s*//;
 389     $rval =~ s/\s*$//;
 390
 391     if ($cmp eq "==") {
 392         return $lval eq $rval;
 393     } elsif ($cmp eq "!=") {
 394         return $lval ne $rval;
 395     }
 396
 397     my $statement = "$lval $cmp $rval";
 398     my $ret = eval $statement;
 399
 400     # $@ stores error of eval
 401     if ($@) {
 402         return -1;
 403     }
 404
 405     return $ret;
 406 }
 407
 408 sub value_defined {
 409     my ($val) = @_;
 410
 411     return defined($variable{$2}) ||
 412         defined($opt{$2});
 413 }
 414
 415 my $d = 0;
 416 sub process_expression {
 417     my ($name, $val) = @_;
 418
 419     my $c = $d++;
 420
 421     while ($val =~ s/\(([^\(]*?)\)/\&\&\&\&VAL\&\&\&\&/) {
 422         my $express = $1;
 423
 424         if (process_expression($name, $express)) {
 425             $val =~ s/\&\&\&\&VAL\&\&\&\&/ 1 /;
 426         } else {
 427             $val =~ s/\&\&\&\&VAL\&\&\&\&/ 0 /;
 428         }
 429     }
 430
 431     $d--;
 432     my $OR = "\\|\\|";
 433     my $AND = "\\&\\&";
 434
 435     while ($val =~ s/^(.*?)($OR|$AND)//) {
 436         my $express = $1;
 437         my $op = $2;
 438
 439         if (process_expression($name, $express)) {
 440             if ($op eq "||") {
 441                 return 1;
 442             }
 443         } else {
 444             if ($op eq "&&") {
 445                 return 0;
 446             }
 447         }
 448     }
 449
 450     if ($val =~ /(.*)(==|\!=|>=|<=|>|<)(.*)/) {
 451         my $ret = process_compare($1, $2, $3);
 452         if ($ret < 0) {
 453             die "$name: $.: Unable to process comparison\n";
 454         }
 455         return $ret;
 456     }
 457
 458     if ($val =~ /^\s*(NOT\s*)?DEFINED\s+(\S+)\s*$/) {
 459         if (defined $1) {
 460             return !value_defined($2);
 461         } else {
 462             return value_defined($2);
 463         }
 464     }
 465
 466     if ($val =~ /^\s*0\s*$/) {
 467         return 0;
 468     } elsif ($val =~ /^\s*\d+\s*$/) {
 469         return 1;
 470     }
 471
 472     die ("$name: $.: Undefined content $val in if statement\n");
 473 }
 474
 475 sub process_if {
 476     my ($name, $value) = @_;
 477
 478     # Convert variables and replace undefined ones with 0
 479     my $val = process_variables($value, 1);
 480     my $ret = process_expression $name, $val;
 481
 482     return $ret;
 483 }
 484
 485 sub __read_config {
 486     my ($config, $current_test_num) = @_;
 487
 488     my $in;
 489     open($in, $config) || die "can't read file $config";
 490
 491     my $name = $config;
 492     $name =~ s,.*/(.*),$1,;
 493
 494     my $test_num = $$current_test_num;
 495     my $default = 1;
 496     my $repeat = 1;
 497     my $num_tests_set = 0;
 498     my $skip = 0;
 499     my $rest;
 500     my $line;
 501     my $test_case = 0;
 502     my $if = 0;
 503     my $if_set = 0;
 504     my $override = 0;
 505
 506     my %overrides;
 507
 508     while (<$in>) {
 509
 510         # ignore blank lines and comments
 511         next if (/^\s*$/ || /\s*\#/);
 512
 513         if (/^\s*(TEST_START|DEFAULTS)\b(.*)/) {
 514
 515             my $type = $1;
 516             $rest = $2;
 517             $line = $2;
 518
 519             my $old_test_num;
 520             my $old_repeat;
 521             $override = 0;
 522
 523             if ($type eq "TEST_START") {
 524
 525                 if ($num_tests_set) {
 526                     die "$name: $.: Can not specify both NUM_TESTS and TEST_START\n";
 527                 }
 528
 529                 $old_test_num = $test_num;
 530                 $old_repeat = $repeat;
 531
 532                 $test_num += $repeat;
 533                 $default = 0;
 534                 $repeat = 1;
 535             } else {
 536                 $default = 1;
 537             }
 538
 539             # If SKIP is anywhere in the line, the command will be skipped
 540             if ($rest =~ s/\s+SKIP\b//) {
 541                 $skip = 1;
 542             } else {
 543                 $test_case = 1;
 544                 $skip = 0;
 545             }
 546
 547             if ($rest =~ s/\sELSE\b//) {
 548                 if (!$if) {
 549                     die "$name: $.: ELSE found with out matching IF section\n$_";
 550                 }
 551                 $if = 0;
 552
 553                 if ($if_set) {
 554                     $skip = 1;
 555                 } else {
 556                     $skip = 0;
 557                 }
 558             }
 559
 560             if ($rest =~ s/\sIF\s+(.*)//) {
 561                 if (process_if($name, $1)) {
 562                     $if_set = 1;
 563                 } else {
 564                     $skip = 1;
 565                 }
 566                 $if = 1;
 567             } else {
 568                 $if = 0;
 569                 $if_set = 0;
 570             }
 571
 572             if (!$skip) {
 573                 if ($type eq "TEST_START") {
 574                     if ($rest =~ s/\s+ITERATE\s+(\d+)//) {
 575                         $repeat = $1;
 576                         $repeat_tests{"$test_num"} = $repeat;
 577                     }
 578                 } elsif ($rest =~ s/\sOVERRIDE\b//) {
 579                     # DEFAULT only
 580                     $override = 1;
 581                     # Clear previous overrides
 582                     %overrides = ();
 583                 }
 584             }
 585
 586             if (!$skip && $rest !~ /^\s*$/) {
 587                 die "$name: $.: Gargbage found after $type\n$_";
 588             }
 589
 590             if ($skip && $type eq "TEST_START") {
 591                 $test_num = $old_test_num;
 592                 $repeat = $old_repeat;
 593             }
 594
 595         } elsif (/^\s*ELSE\b(.*)$/) {
 596             if (!$if) {
 597                 die "$name: $.: ELSE found with out matching IF section\n$_";
 598             }
 599             $rest = $1;
 600             if ($if_set) {
 601                 $skip = 1;
 602                 $rest = "";
 603             } else {
 604                 $skip = 0;
 605
 606                 if ($rest =~ /\sIF\s+(.*)/) {
 607                     # May be a ELSE IF section.
 608                     if (!process_if($name, $1)) {
 609                         $skip = 1;
 610                     }
 611                     $rest = "";
 612                 } else {
 613                     $if = 0;
 614                 }
 615             }
 616
 617             if ($rest !~ /^\s*$/) {
 618                 die "$name: $.: Gargbage found after DEFAULTS\n$_";
 619             }
 620
 621         } elsif (/^\s*INCLUDE\s+(\S+)/) {
 622
 623             next if ($skip);
 624
 625             if (!$default) {
 626                 die "$name: $.: INCLUDE can only be done in default sections\n$_";
 627             }
 628
 629             my $file = process_variables($1);
 630
 631             if ($file !~ m,^/,) {
 632                 # check the path of the config file first
 633                 if ($config =~ m,(.*)/,) {
 634                     if (-f "$1/$file") {
 635                         $file = "$1/$file";
 636                     }
 637                 }
 638             }
 639
 640             if ( ! -r $file ) {
 641                 die "$name: $.: Can't read file $file\n$_";
 642             }
 643
 644             if (__read_config($file, \$test_num)) {
 645                 $test_case = 1;
 646             }
 647
 648         } elsif (/^\s*([A-Z_\[\]\d]+)\s*=\s*(.*?)\s*$/) {
 649
 650             next if ($skip);
 651
 652             my $lvalue = $1;
 653             my $rvalue = $2;
 654
 655             if (!$default &&
 656                 ($lvalue eq "NUM_TESTS" ||
 657                  $lvalue eq "LOG_FILE" ||
 658                  $lvalue eq "CLEAR_LOG")) {
 659                 die "$name: $.: $lvalue must be set in DEFAULTS section\n";
 660             }
 661
 662             if ($lvalue eq "NUM_TESTS") {
 663                 if ($test_num) {
 664                     die "$name: $.: Can not specify both NUM_TESTS and TEST_START\n";
 665                 }
 666                 if (!$default) {
 667                     die "$name: $.: NUM_TESTS must be set in default section\n";
 668                 }
 669                 $num_tests_set = 1;
 670             }
 671
 672             if ($default || $lvalue =~ /\[\d+\]$/) {
 673                 set_value($lvalue, $rvalue, $override, \%overrides, $name);
 674             } else {
 675                 my $val = "$lvalue\[$test_num\]";
 676                 set_value($val, $rvalue, $override, \%overrides, $name);
 677
 678                 if ($repeat > 1) {
 679                     $repeats{$val} = $repeat;
 680                 }
 681             }
 682         } elsif (/^\s*([A-Z_\[\]\d]+)\s*:=\s*(.*?)\s*$/) {
 683             next if ($skip);
 684
 685             my $lvalue = $1;
 686             my $rvalue = $2;
 687
 688             # process config variables.
 689             # Config variables are only active while reading the
 690             # config and can be defined anywhere. They also ignore
 691             # TEST_START and DEFAULTS, but are skipped if they are in
 692             # on of these sections that have SKIP defined.
 693             # The save variable can be
 694             # defined multiple times and the new one simply overrides
 695             # the prevous one.
 696             set_variable($lvalue, $rvalue);
 697
 698         } else {
 699             die "$name: $.: Garbage found in config\n$_";
 700         }
 701     }
 702
 703     if ($test_num) {
 704         $test_num += $repeat - 1;
 705         $opt{"NUM_TESTS"} = $test_num;
 706     }
 707
 708     close($in);
 709
 710     $$current_test_num = $test_num;
 711
 712     return $test_case;
 713 }
 714
 715 sub read_config {
 716     my ($config) = @_;
 717
 718     my $test_case;
 719     my $test_num = 0;
 720
 721     $test_case = __read_config $config, \$test_num;
 722
 723     # make sure we have all mandatory configs
 724     get_ktest_configs;
 725
 726     # was a test specified?
 727     if (!$test_case) {
 728         print "No test case specified.\n";
 729         print "What test case would you like to run?\n";
 730         my $ans = <STDIN>;
 731         chomp $ans;
 732         $default{"TEST_TYPE"} = $ans;
 733     }
 734
 735     # set any defaults
 736
 737     foreach my $default (keys %default) {
 738         if (!defined($opt{$default})) {
 739             $opt{$default} = $default{$default};
 740         }
 741     }
 742 }
 743
 744 sub __eval_option {
 745     my ($option, $i) = @_;
 746
 747     # Add space to evaluate the character before $
 748     $option = " $option";
 749     my $retval = "";
 750     my $repeated = 0;
 751     my $parent = 0;
 752
 753     foreach my $test (keys %repeat_tests) {
 754         if ($i >= $test &&
 755             $i < $test + $repeat_tests{$test}) {
 756
 757             $repeated = 1;
 758             $parent = $test;
 759             last;
 760         }
 761     }
 762
 763     while ($option =~ /(.*?[^\\])\$\{(.*?)\}(.*)/) {
 764         my $start = $1;
 765         my $var = $2;
 766         my $end = $3;
 767
 768         # Append beginning of line
 769         $retval = "$retval$start";
 770
 771         # If the iteration option OPT[$i] exists, then use that.
 772         # otherwise see if the default OPT (without [$i]) exists.
 773
 774         my $o = "$var\[$i\]";
 775         my $parento = "$var\[$parent\]";
 776
 777         if (defined($opt{$o})) {
 778             $o = $opt{$o};
 779             $retval = "$retval$o";
 780         } elsif ($repeated && defined($opt{$parento})) {
 781             $o = $opt{$parento};
 782             $retval = "$retval$o";
 783         } elsif (defined($opt{$var})) {
 784             $o = $opt{$var};
 785             $retval = "$retval$o";
 786         } else {
 787             $retval = "$retval\$\{$var\}";
 788         }
 789
 790         $option = $end;
 791     }
 792
 793     $retval = "$retval$option";
 794
 795     $retval =~ s/^ //;
 796
 797     return $retval;
 798 }
 799
 800 sub eval_option {
 801     my ($option, $i) = @_;
 802
 803     my $prev = "";
 804
 805     # Since an option can evaluate to another option,
 806     # keep iterating until we do not evaluate any more
 807     # options.
 808     my $r = 0;
 809     while ($prev ne $option) {
 810         # Check for recursive evaluations.
 811         # 100 deep should be more than enough.
 812         if ($r++ > 100) {
 813             die "Over 100 evaluations accurred with $option\n" .
 814                 "Check for recursive variables\n";
 815         }
 816         $prev = $option;
 817         $option = __eval_option($option, $i);
 818     }
 819
 820     return $option;
 821 }
 822
 823 sub _logit {
 824     if (defined($opt{"LOG_FILE"})) {
 825         open(OUT, ">> $opt{LOG_FILE}") or die "Can't write to $opt{LOG_FILE}";
 826         print OUT @_;
 827         close(OUT);
 828     }
 829 }
 830
 831 sub logit {
 832     if (defined($opt{"LOG_FILE"})) {
 833         _logit @_;
 834     } else {
 835         print @_;
 836     }
 837 }
 838
 839 sub doprint {
 840     print @_;
 841     _logit @_;
 842 }
 843
 844 sub run_command;
 845 sub start_monitor;
 846 sub end_monitor;
 847 sub wait_for_monitor;
 848
 849 sub reboot {
 850     my ($time) = @_;
 851
 852     if (defined($time)) {
 853         start_monitor;
 854         # flush out current monitor
 855         # May contain the reboot success line
 856         wait_for_monitor 1;
 857     }
 858
 859     # try to reboot normally
 860     if (run_command $reboot) {
 861         if (defined($powercycle_after_reboot)) {
 862             sleep $powercycle_after_reboot;
 863             run_command "$power_cycle";
 864         }
 865     } else {
 866         # nope? power cycle it.
 867         run_command "$power_cycle";
 868     }
 869
 870     if (defined($time)) {
 871         wait_for_monitor($time, $reboot_success_line);
 872         end_monitor;
 873     }
 874 }
 875
 876 sub do_not_reboot {
 877     my $i = $iteration;
 878
 879     return $test_type eq "build" || $no_reboot ||
 880         ($test_type eq "patchcheck" && $opt{"PATCHCHECK_TYPE[$i]"} eq "build") ||
 881         ($test_type eq "bisect" && $opt{"BISECT_TYPE[$i]"} eq "build");
 882 }
 883
 884 sub dodie {
 885     doprint "CRITICAL FAILURE... ", @_, "\n";
 886
 887     my $i = $iteration;
 888
 889     if ($reboot_on_error && !do_not_reboot) {
 890
 891         doprint "REBOOTING\n";
 892         reboot;
 893
 894     } elsif ($poweroff_on_error && defined($power_off)) {
 895         doprint "POWERING OFF\n";
 896         `$power_off`;
 897     }
 898
 899     if (defined($opt{"LOG_FILE"})) {
 900         print " See $opt{LOG_FILE} for more info.\n";
 901     }
 902
 903     die @_, "\n";
 904 }
 905
 906 sub open_console {
 907     my ($fp) = @_;
 908
 909     my $flags;
 910
 911     my $pid = open($fp, "$console|") or
 912         dodie "Can't open console $console";
 913
 914     $flags = fcntl($fp, F_GETFL, 0) or
 915         dodie "Can't get flags for the socket: $!";
 916     $flags = fcntl($fp, F_SETFL, $flags | O_NONBLOCK) or
 917         dodie "Can't set flags for the socket: $!";
 918
 919     return $pid;
 920 }
 921
 922 sub close_console {
 923     my ($fp, $pid) = @_;
 924
 925     doprint "kill child process $pid\n";
 926     kill 2, $pid;
 927
 928     print "closing!\n";
 929     close($fp);
 930 }
 931
 932 sub start_monitor {
 933     if ($monitor_cnt++) {
 934         return;
 935     }
 936     $monitor_fp = \*MONFD;
 937     $monitor_pid = open_console $monitor_fp;
 938
 939     return;
 940
 941     open(MONFD, "Stop perl from warning about single use of MONFD");
 942 }
 943
 944 sub end_monitor {
 945     if (--$monitor_cnt) {
 946         return;
 947     }
 948     close_console($monitor_fp, $monitor_pid);
 949 }
 950
 951 sub wait_for_monitor {
 952     my ($time, $stop) = @_;
 953     my $full_line = "";
 954     my $line;
 955     my $booted = 0;
 956
 957     doprint "** Wait for monitor to settle down **\n";
 958
 959     # read the monitor and wait for the system to calm down
 960     while (!$booted) {
 961         $line = wait_for_input($monitor_fp, $time);
 962         last if (!defined($line));
 963         print "$line";
 964         $full_line .= $line;
 965
 966         if (defined($stop) && $full_line =~ /$stop/) {
 967             doprint "wait for monitor detected $stop\n";
 968             $booted = 1;
 969         }
 970
 971         if ($line =~ /\n/) {
 972             $full_line = "";
 973         }
 974     }
 975     print "** Monitor flushed **\n";
 976 }
 977
 978 sub fail {
 979
 980         if ($die_on_failure) {
 981                 dodie @_;
 982         }
 983
 984         doprint "FAILED\n";
 985
 986         my $i = $iteration;
 987
 988         # no need to reboot for just building.
 989         if (!do_not_reboot) {
 990             doprint "REBOOTING\n";
 991             reboot $sleep_time;
 992         }
 993
 994         my $name = "";
 995
 996         if (defined($test_name)) {
 997             $name = " ($test_name)";
 998         }
 999
1000         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
1001         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
1002         doprint "KTEST RESULT: TEST $i$name Failed: ", @_, "\n";
1003         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
1004         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
1005
1006         return 1 if (!defined($store_failures));
1007
1008         my @t = localtime;
1009         my $date = sprintf "%04d%02d%02d%02d%02d%02d",
1010                 1900+$t[5],$t[4],$t[3],$t[2],$t[1],$t[0];
1011
1012         my $type = $build_type;
1013         if ($type =~ /useconfig/) {
1014             $type = "useconfig";
1015         }
1016
1017         my $dir = "$machine-$test_type-$type-fail-$date";
1018         my $faildir = "$store_failures/$dir";
1019
1020         if (!-d $faildir) {
1021             mkpath($faildir) or
1022                 die "can't create $faildir";
1023         }
1024         if (-f "$output_config") {
1025             cp "$output_config", "$faildir/config" or
1026                 die "failed to copy .config";
1027         }
1028         if (-f $buildlog) {
1029             cp $buildlog, "$faildir/buildlog" or
1030                 die "failed to move $buildlog";
1031         }
1032         if (-f $dmesg) {
1033             cp $dmesg, "$faildir/dmesg" or
1034                 die "failed to move $dmesg";
1035         }
1036
1037         doprint "*** Saved info to $faildir ***\n";
1038
1039         return 1;
1040 }
1041
1042 sub run_command {
1043     my ($command) = @_;
1044     my $dolog = 0;
1045     my $dord = 0;
1046     my $pid;
1047
1048     $command =~ s/\$SSH_USER/$ssh_user/g;
1049     $command =~ s/\$MACHINE/$machine/g;
1050
1051     doprint("$command ... ");
1052
1053     $pid = open(CMD, "$command 2>&1 |") or
1054         (fail "unable to exec $command" and return 0);
1055
1056     if (defined($opt{"LOG_FILE"})) {
1057         open(LOG, ">>$opt{LOG_FILE}") or
1058             dodie "failed to write to log";
1059         $dolog = 1;
1060     }
1061
1062     if (defined($redirect)) {
1063         open (RD, ">$redirect") or
1064             dodie "failed to write to redirect $redirect";
1065         $dord = 1;
1066     }
1067
1068     while (<CMD>) {
1069         print LOG if ($dolog);
1070         print RD  if ($dord);
1071     }
1072
1073     waitpid($pid, 0);
1074     my $failed = $?;
1075
1076     close(CMD);
1077     close(LOG) if ($dolog);
1078     close(RD)  if ($dord);
1079
1080     if ($failed) {
1081         doprint "FAILED!\n";
1082     } else {
1083         doprint "SUCCESS\n";
1084     }
1085
1086     return !$failed;
1087 }
1088
1089 sub run_ssh {
1090     my ($cmd) = @_;
1091     my $cp_exec = $ssh_exec;
1092
1093     $cp_exec =~ s/\$SSH_COMMAND/$cmd/g;
1094     return run_command "$cp_exec";
1095 }
1096
1097 sub run_scp {
1098     my ($src, $dst) = @_;
1099     my $cp_scp = $scp_to_target;
1100
1101     $cp_scp =~ s/\$SRC_FILE/$src/g;
1102     $cp_scp =~ s/\$DST_FILE/$dst/g;
1103
1104     return run_command "$cp_scp";
1105 }
1106
1107 sub get_grub_index {
1108
1109     if ($reboot_type ne "grub") {
1110         return;
1111     }
1112     return if (defined($grub_number));
1113
1114     doprint "Find grub menu ... ";
1115     $grub_number = -1;
1116
1117     my $ssh_grub = $ssh_exec;
1118     $ssh_grub =~ s,\$SSH_COMMAND,cat /boot/grub/menu.lst,g;
1119
1120     open(IN, "$ssh_grub |")
1121         or die "unable to get menu.lst";
1122
1123     my $found = 0;
1124
1125     while (<IN>) {
1126         if (/^\s*title\s+$grub_menu\s*$/) {
1127             $grub_number++;
1128             $found = 1;
1129             last;
1130         } elsif (/^\s*title\s/) {
1131             $grub_number++;
1132         }
1133     }
1134     close(IN);
1135
1136     die "Could not find '$grub_menu' in /boot/grub/menu on $machine"
1137         if (!$found);
1138     doprint "$grub_number\n";
1139 }
1140
1141 sub wait_for_input
1142 {
1143     my ($fp, $time) = @_;
1144     my $rin;
1145     my $ready;
1146     my $line;
1147     my $ch;
1148
1149     if (!defined($time)) {
1150         $time = $timeout;
1151     }
1152
1153     $rin = '';
1154     vec($rin, fileno($fp), 1) = 1;
1155     $ready = select($rin, undef, undef, $time);
1156
1157     $line = "";
1158
1159     # try to read one char at a time
1160     while (sysread $fp, $ch, 1) {
1161         $line .= $ch;
1162         last if ($ch eq "\n");
1163     }
1164
1165     if (!length($line)) {
1166         return undef;
1167     }
1168
1169     return $line;
1170 }
1171
1172 sub reboot_to {
1173     if ($reboot_type eq "grub") {
1174         run_ssh "'(echo \"savedefault --default=$grub_number --once\" | grub --batch)'";
1175         reboot;
1176         return;
1177     }
1178
1179     run_command "$reboot_script";
1180 }
1181
1182 sub get_sha1 {
1183     my ($commit) = @_;
1184
1185     doprint "git rev-list --max-count=1 $commit ... ";
1186     my $sha1 = `git rev-list --max-count=1 $commit`;
1187     my $ret = $?;
1188
1189     logit $sha1;
1190
1191     if ($ret) {
1192         doprint "FAILED\n";
1193         dodie "Failed to get git $commit";
1194     }
1195
1196     print "SUCCESS\n";
1197
1198     chomp $sha1;
1199
1200     return $sha1;
1201 }
1202
1203 sub monitor {
1204     my $booted = 0;
1205     my $bug = 0;
1206     my $skip_call_trace = 0;
1207     my $loops;
1208
1209     wait_for_monitor 5;
1210
1211     my $line;
1212     my $full_line = "";
1213
1214     open(DMESG, "> $dmesg") or
1215         die "unable to write to $dmesg";
1216
1217     reboot_to;
1218
1219     my $success_start;
1220     my $failure_start;
1221     my $monitor_start = time;
1222     my $done = 0;
1223     my $version_found = 0;
1224
1225     while (!$done) {
1226
1227         if ($bug && defined($stop_after_failure) &&
1228             $stop_after_failure >= 0) {
1229             my $time = $stop_after_failure - (time - $failure_start);
1230             $line = wait_for_input($monitor_fp, $time);
1231             if (!defined($line)) {
1232                 doprint "bug timed out after $booted_timeout seconds\n";
1233                 doprint "Test forced to stop after $stop_after_failure seconds after failure\n";
1234                 last;
1235             }
1236         } elsif ($booted) {
1237             $line = wait_for_input($monitor_fp, $booted_timeout);
1238             if (!defined($line)) {
1239                 my $s = $booted_timeout == 1 ? "" : "s";
1240                 doprint "Successful boot found: break after $booted_timeout second$s\n";
1241                 last;
1242             }
1243         } else {
1244             $line = wait_for_input($monitor_fp);
1245             if (!defined($line)) {
1246                 my $s = $timeout == 1 ? "" : "s";
1247                 doprint "Timed out after $timeout second$s\n";
1248                 last;
1249             }
1250         }
1251
1252         doprint $line;
1253         print DMESG $line;
1254
1255         # we are not guaranteed to get a full line
1256         $full_line .= $line;
1257
1258         if ($full_line =~ /$success_line/) {
1259             $booted = 1;
1260             $success_start = time;
1261         }
1262
1263         if ($booted && defined($stop_after_success) &&
1264             $stop_after_success >= 0) {
1265             my $now = time;
1266             if ($now - $success_start >= $stop_after_success) {
1267                 doprint "Test forced to stop after $stop_after_success seconds after success\n";
1268                 last;
1269             }
1270         }
1271
1272         if ($full_line =~ /\[ backtrace testing \]/) {
1273             $skip_call_trace = 1;
1274         }
1275
1276         if ($full_line =~ /call trace:/i) {
1277             if (!$bug && !$skip_call_trace) {
1278                 $bug = 1;
1279                 $failure_start = time;
1280             }
1281         }
1282
1283         if ($bug && defined($stop_after_failure) &&
1284             $stop_after_failure >= 0) {
1285             my $now = time;
1286             if ($now - $failure_start >= $stop_after_failure) {
1287                 doprint "Test forced to stop after $stop_after_failure seconds after failure\n";
1288                 last;
1289             }
1290         }
1291
1292         if ($full_line =~ /\[ end of backtrace testing \]/) {
1293             $skip_call_trace = 0;
1294         }
1295
1296         if ($full_line =~ /Kernel panic -/) {
1297             $failure_start = time;
1298             $bug = 1;
1299         }
1300
1301         # Detect triple faults by testing the banner
1302         if ($full_line =~ /\bLinux version (\S+).*\n/) {
1303             if ($1 eq $version) {
1304                 $version_found = 1;
1305             } elsif ($version_found && $detect_triplefault) {
1306                 # We already booted into the kernel we are testing,
1307                 # but now we booted into another kernel?
1308                 # Consider this a triple fault.
1309                 doprint "Aleady booted in Linux kernel $version, but now\n";
1310                 doprint "we booted into Linux kernel $1.\n";
1311                 doprint "Assuming that this is a triple fault.\n";
1312                 doprint "To disable this: set DETECT_TRIPLE_FAULT to 0\n";
1313                 last;
1314             }
1315         }
1316
1317         if ($line =~ /\n/) {
1318             $full_line = "";
1319         }
1320
1321         if ($stop_test_after > 0 && !$booted && !$bug) {
1322             if (time - $monitor_start > $stop_test_after) {
1323                 doprint "STOP_TEST_AFTER ($stop_test_after seconds) timed out\n";
1324                 $done = 1;
1325             }
1326         }
1327     }
1328
1329     close(DMESG);
1330
1331     if ($bug) {
1332         return 0 if ($in_bisect);
1333         fail "failed - got a bug report" and return 0;
1334     }
1335
1336     if (!$booted) {
1337         return 0 if ($in_bisect);
1338         fail "failed - never got a boot prompt." and return 0;
1339     }
1340
1341     return 1;
1342 }
1343
1344 sub do_post_install {
1345
1346     return if (!defined($post_install));
1347
1348     my $cp_post_install = $post_install;
1349     $cp_post_install =~ s/\$KERNEL_VERSION/$version/g;
1350     run_command "$cp_post_install" or
1351         dodie "Failed to run post install";
1352 }
1353
1354 sub install {
1355
1356     return if ($no_install);
1357
1358     run_scp "$outputdir/$build_target", "$target_image" or
1359         dodie "failed to copy image";
1360
1361     my $install_mods = 0;
1362
1363     # should we process modules?
1364     $install_mods = 0;
1365     open(IN, "$output_config") or dodie("Can't read config file");
1366     while (<IN>) {
1367         if (/CONFIG_MODULES(=y)?/) {
1368             $install_mods = 1 if (defined($1));
1369             last;
1370         }
1371     }
1372     close(IN);
1373
1374     if (!$install_mods) {
1375         do_post_install;
1376         doprint "No modules needed\n";
1377         return;
1378     }
1379
1380     run_command "$make INSTALL_MOD_PATH=$tmpdir modules_install" or
1381         dodie "Failed to install modules";
1382
1383     my $modlib = "/lib/modules/$version";
1384     my $modtar = "ktest-mods.tar.bz2";
1385
1386     run_ssh "rm -rf $modlib" or
1387         dodie "failed to remove old mods: $modlib";
1388
1389     # would be nice if scp -r did not follow symbolic links
1390     run_command "cd $tmpdir && tar -cjf $modtar lib/modules/$version" or
1391         dodie "making tarball";
1392
1393     run_scp "$tmpdir/$modtar", "/tmp" or
1394         dodie "failed to copy modules";
1395
1396     unlink "$tmpdir/$modtar";
1397
1398     run_ssh "'(cd / && tar xjf /tmp/$modtar)'" or
1399         dodie "failed to tar modules";
1400
1401     run_ssh "rm -f /tmp/$modtar";
1402
1403     do_post_install;
1404 }
1405
1406 sub get_version {
1407     # get the release name
1408     doprint "$make kernelrelease ... ";
1409     $version = `$make kernelrelease | tail -1`;
1410     chomp($version);
1411     doprint "$version\n";
1412 }
1413
1414 sub start_monitor_and_boot {
1415     # Make sure the stable kernel has finished booting
1416     start_monitor;
1417     wait_for_monitor 5;
1418     end_monitor;
1419
1420     get_grub_index;
1421     get_version;
1422     install;
1423
1424     start_monitor;
1425     return monitor;
1426 }
1427
1428 sub check_buildlog {
1429     my ($patch) = @_;
1430
1431     my @files = `git show $patch | diffstat -l`;
1432
1433     open(IN, "git show $patch |") or
1434         dodie "failed to show $patch";
1435     while (<IN>) {
1436         if (m,^--- a/(.*),) {
1437             chomp $1;
1438             $files[$#files] = $1;
1439         }
1440     }
1441     close(IN);
1442
1443     open(IN, $buildlog) or dodie "Can't open $buildlog";
1444     while (<IN>) {
1445         if (/^\s*(.*?):.*(warning|error)/) {
1446             my $err = $1;
1447             foreach my $file (@files) {
1448                 my $fullpath = "$builddir/$file";
1449                 if ($file eq $err || $fullpath eq $err) {
1450                     fail "$file built with warnings" and return 0;
1451                 }
1452             }
1453         }
1454     }
1455     close(IN);
1456
1457     return 1;
1458 }
1459
1460 sub apply_min_config {
1461     my $outconfig = "$output_config.new";
1462
1463     # Read the config file and remove anything that
1464     # is in the force_config hash (from minconfig and others)
1465     # then add the force config back.
1466
1467     doprint "Applying minimum configurations into $output_config.new\n";
1468
1469     open (OUT, ">$outconfig") or
1470         dodie "Can't create $outconfig";
1471
1472     if (-f $output_config) {
1473         open (IN, $output_config) or
1474             dodie "Failed to open $output_config";
1475         while (<IN>) {
1476             if (/^(# )?(CONFIG_[^\s=]*)/) {
1477                 next if (defined($force_config{$2}));
1478             }
1479             print OUT;
1480         }
1481         close IN;
1482     }
1483     foreach my $config (keys %force_config) {
1484         print OUT "$force_config{$config}\n";
1485     }
1486     close OUT;
1487
1488     run_command "mv $outconfig $output_config";
1489 }
1490
1491 sub make_oldconfig {
1492
1493     my @force_list = keys %force_config;
1494
1495     if ($#force_list >= 0) {
1496         apply_min_config;
1497     }
1498
1499     if (!run_command "$make oldnoconfig") {
1500         # Perhaps oldnoconfig doesn't exist in this version of the kernel
1501         # try a yes '' | oldconfig
1502         doprint "oldnoconfig failed, trying yes '' | make oldconfig\n";
1503         run_command "yes '' | $make oldconfig" or
1504             dodie "failed make config oldconfig";
1505     }
1506 }
1507
1508 # read a config file and use this to force new configs.
1509 sub load_force_config {
1510     my ($config) = @_;
1511
1512     open(IN, $config) or
1513         dodie "failed to read $config";
1514     while (<IN>) {
1515         chomp;
1516         if (/^(CONFIG[^\s=]*)(\s*=.*)/) {
1517             $force_config{$1} = $_;
1518         } elsif (/^# (CONFIG_\S*) is not set/) {
1519             $force_config{$1} = $_;
1520         }
1521     }
1522     close IN;
1523 }
1524
1525 sub build {
1526     my ($type) = @_;
1527
1528     unlink $buildlog;
1529
1530     # Failed builds should not reboot the target
1531     my $save_no_reboot = $no_reboot;
1532     $no_reboot = 1;
1533
1534     if (defined($pre_build)) {
1535         my $ret = run_command $pre_build;
1536         if (!$ret && defined($pre_build_die) &&
1537             $pre_build_die) {
1538             dodie "failed to pre_build\n";
1539         }
1540     }
1541
1542     if ($type =~ /^useconfig:(.*)/) {
1543         run_command "cp $1 $output_config" or
1544             dodie "could not copy $1 to .config";
1545
1546         $type = "oldconfig";
1547     }
1548
1549     # old config can ask questions
1550     if ($type eq "oldconfig") {
1551         $type = "oldnoconfig";
1552
1553         # allow for empty configs
1554         run_command "touch $output_config";
1555
1556         if (!$noclean) {
1557             run_command "mv $output_config $outputdir/config_temp" or
1558                 dodie "moving .config";
1559
1560             run_command "$make mrproper" or dodie "make mrproper";
1561
1562             run_command "mv $outputdir/config_temp $output_config" or
1563                 dodie "moving config_temp";
1564         }
1565
1566     } elsif (!$noclean) {
1567         unlink "$output_config";
1568         run_command "$make mrproper" or
1569             dodie "make mrproper";
1570     }
1571
1572     # add something to distinguish this build
1573     open(OUT, "> $outputdir/localversion") or dodie("Can't make localversion file");
1574     print OUT "$localversion\n";
1575     close(OUT);
1576
1577     if (defined($minconfig)) {
1578         load_force_config($minconfig);
1579     }
1580
1581     if ($type ne "oldnoconfig") {
1582         run_command "$make $type" or
1583             dodie "failed make config";
1584     }
1585     # Run old config regardless, to enforce min configurations
1586     make_oldconfig;
1587
1588     $redirect = "$buildlog";
1589     my $build_ret = run_command "$make $build_options";
1590     undef $redirect;
1591
1592     if (defined($post_build)) {
1593         my $ret = run_command $post_build;
1594         if (!$ret && defined($post_build_die) &&
1595             $post_build_die) {
1596             dodie "failed to post_build\n";
1597         }
1598     }
1599
1600     if (!$build_ret) {
1601         # bisect may need this to pass
1602         if ($in_bisect) {
1603             $no_reboot = $save_no_reboot;
1604             return 0;
1605         }
1606         fail "failed build" and return 0;
1607     }
1608
1609     $no_reboot = $save_no_reboot;
1610
1611     return 1;
1612 }
1613
1614 sub halt {
1615     if (!run_ssh "halt" or defined($power_off)) {
1616         if (defined($poweroff_after_halt)) {
1617             sleep $poweroff_after_halt;
1618             run_command "$power_off";
1619         }
1620     } else {
1621         # nope? the zap it!
1622         run_command "$power_off";
1623     }
1624 }
1625
1626 sub success {
1627     my ($i) = @_;
1628
1629     $successes++;
1630
1631     my $name = "";
1632
1633     if (defined($test_name)) {
1634         $name = " ($test_name)";
1635     }
1636
1637     doprint "\n\n*******************************************\n";
1638     doprint     "*******************************************\n";
1639     doprint     "KTEST RESULT: TEST $i$name SUCCESS!!!!         **\n";
1640     doprint     "*******************************************\n";
1641     doprint     "*******************************************\n";
1642
1643     if ($i != $opt{"NUM_TESTS"} && !do_not_reboot) {
1644         doprint "Reboot and wait $sleep_time seconds\n";
1645         reboot $sleep_time;
1646     }
1647 }
1648
1649 sub answer_bisect {
1650     for (;;) {
1651         doprint "Pass or fail? [p/f]";
1652         my $ans = <STDIN>;
1653         chomp $ans;
1654         if ($ans eq "p" || $ans eq "P") {
1655             return 1;
1656         } elsif ($ans eq "f" || $ans eq "F") {
1657             return 0;
1658         } else {
1659             print "Please answer 'P' or 'F'\n";
1660         }
1661     }
1662 }
1663
1664 sub child_run_test {
1665     my $failed = 0;
1666
1667     # child should have no power
1668     $reboot_on_error = 0;
1669     $poweroff_on_error = 0;
1670     $die_on_failure = 1;
1671
1672     run_command $run_test or $failed = 1;
1673     exit $failed;
1674 }
1675
1676 my $child_done;
1677
1678 sub child_finished {
1679     $child_done = 1;
1680 }
1681
1682 sub do_run_test {
1683     my $child_pid;
1684     my $child_exit;
1685     my $line;
1686     my $full_line;
1687     my $bug = 0;
1688
1689     wait_for_monitor 1;
1690
1691     doprint "run test $run_test\n";
1692
1693     $child_done = 0;
1694
1695     $SIG{CHLD} = qw(child_finished);
1696
1697     $child_pid = fork;
1698
1699     child_run_test if (!$child_pid);
1700
1701     $full_line = "";
1702
1703     do {
1704         $line = wait_for_input($monitor_fp, 1);
1705         if (defined($line)) {
1706
1707             # we are not guaranteed to get a full line
1708             $full_line .= $line;
1709             doprint $line;
1710
1711             if ($full_line =~ /call trace:/i) {
1712                 $bug = 1;
1713             }
1714
1715             if ($full_line =~ /Kernel panic -/) {
1716                 $bug = 1;
1717             }
1718
1719             if ($line =~ /\n/) {
1720                 $full_line = "";
1721             }
1722         }
1723     } while (!$child_done && !$bug);
1724
1725     if ($bug) {
1726         my $failure_start = time;
1727         my $now;
1728         do {
1729             $line = wait_for_input($monitor_fp, 1);
1730             if (defined($line)) {
1731                 doprint $line;
1732             }
1733             $now = time;
1734             if ($now - $failure_start >= $stop_after_failure) {
1735                 last;
1736             }
1737         } while (defined($line));
1738
1739         doprint "Detected kernel crash!\n";
1740         # kill the child with extreme prejudice
1741         kill 9, $child_pid;
1742     }
1743
1744     waitpid $child_pid, 0;
1745     $child_exit = $?;
1746
1747     if ($bug || $child_exit) {
1748         return 0 if $in_bisect;
1749         fail "test failed" and return 0;
1750     }
1751     return 1;
1752 }
1753
1754 sub run_git_bisect {
1755     my ($command) = @_;
1756
1757     doprint "$command ... ";
1758
1759     my $output = `$command 2>&1`;
1760     my $ret = $?;
1761
1762     logit $output;
1763
1764     if ($ret) {
1765         doprint "FAILED\n";
1766         dodie "Failed to git bisect";
1767     }
1768
1769     doprint "SUCCESS\n";
1770     if ($output =~ m/^(Bisecting: .*\(roughly \d+ steps?\))\s+\[([[:xdigit:]]+)\]/) {
1771         doprint "$1 [$2]\n";
1772     } elsif ($output =~ m/^([[:xdigit:]]+) is the first bad commit/) {
1773         $bisect_bad = $1;
1774         doprint "Found bad commit... $1\n";
1775         return 0;
1776     } else {
1777         # we already logged it, just print it now.
1778         print $output;
1779     }
1780
1781     return 1;
1782 }
1783
1784 sub bisect_reboot {
1785     doprint "Reboot and sleep $bisect_sleep_time seconds\n";
1786     reboot $bisect_sleep_time;
1787 }
1788
1789 # returns 1 on success, 0 on failure, -1 on skip
1790 sub run_bisect_test {
1791     my ($type, $buildtype) = @_;
1792
1793     my $failed = 0;
1794     my $result;
1795     my $output;
1796     my $ret;
1797
1798     $in_bisect = 1;
1799
1800     build $buildtype or $failed = 1;
1801
1802     if ($type ne "build") {
1803         if ($failed && $bisect_skip) {
1804             $in_bisect = 0;
1805             return -1;
1806         }
1807         dodie "Failed on build" if $failed;
1808
1809         # Now boot the box
1810         start_monitor_and_boot or $failed = 1;
1811
1812         if ($type ne "boot") {
1813             if ($failed && $bisect_skip) {
1814                 end_monitor;
1815                 bisect_reboot;
1816                 $in_bisect = 0;
1817                 return -1;
1818             }
1819             dodie "Failed on boot" if $failed;
1820
1821             do_run_test or $failed = 1;
1822         }
1823         end_monitor;
1824     }
1825
1826     if ($failed) {
1827         $result = 0;
1828     } else {
1829         $result = 1;
1830     }
1831
1832     # reboot the box to a kernel we can ssh to
1833     if ($type ne "build") {
1834         bisect_reboot;
1835     }
1836     $in_bisect = 0;
1837
1838     return $result;
1839 }
1840
1841 sub run_bisect {
1842     my ($type) = @_;
1843     my $buildtype = "oldconfig";
1844
1845     # We should have a minconfig to use?
1846     if (defined($minconfig)) {
1847         $buildtype = "useconfig:$minconfig";
1848     }
1849
1850     my $ret = run_bisect_test $type, $buildtype;
1851
1852     if ($bisect_manual) {
1853         $ret = answer_bisect;
1854     }
1855
1856     # Are we looking for where it worked, not failed?
1857     if ($reverse_bisect) {
1858         $ret = !$ret;
1859     }
1860
1861     if ($ret > 0) {
1862         return "good";
1863     } elsif ($ret == 0) {
1864         return  "bad";
1865     } elsif ($bisect_skip) {
1866         doprint "HIT A BAD COMMIT ... SKIPPING\n";
1867         return "skip";
1868     }
1869 }
1870
1871 sub bisect {
1872     my ($i) = @_;
1873
1874     my $result;
1875
1876     die "BISECT_GOOD[$i] not defined\n" if (!defined($opt{"BISECT_GOOD[$i]"}));
1877     die "BISECT_BAD[$i] not defined\n"  if (!defined($opt{"BISECT_BAD[$i]"}));
1878     die "BISECT_TYPE[$i] not defined\n" if (!defined($opt{"BISECT_TYPE[$i]"}));
1879
1880     my $good = $opt{"BISECT_GOOD[$i]"};
1881     my $bad = $opt{"BISECT_BAD[$i]"};
1882     my $type = $opt{"BISECT_TYPE[$i]"};
1883     my $start = $opt{"BISECT_START[$i]"};
1884     my $replay = $opt{"BISECT_REPLAY[$i]"};
1885     my $start_files = $opt{"BISECT_FILES[$i]"};
1886
1887     if (defined($start_files)) {
1888         $start_files = " -- " . $start_files;
1889     } else {
1890         $start_files = "";
1891     }
1892
1893     # convert to true sha1's
1894     $good = get_sha1($good);
1895     $bad = get_sha1($bad);
1896
1897     if (defined($opt{"BISECT_REVERSE[$i]"}) &&
1898         $opt{"BISECT_REVERSE[$i]"} == 1) {
1899         doprint "Performing a reverse bisect (bad is good, good is bad!)\n";
1900         $reverse_bisect = 1;
1901     } else {
1902         $reverse_bisect = 0;
1903     }
1904
1905     # Can't have a test without having a test to run
1906     if ($type eq "test" && !defined($run_test)) {
1907         $type = "boot";
1908     }
1909
1910     my $check = $opt{"BISECT_CHECK[$i]"};
1911     if (defined($check) && $check ne "0") {
1912
1913         # get current HEAD
1914         my $head = get_sha1("HEAD");
1915
1916         if ($check ne "good") {
1917             doprint "TESTING BISECT BAD [$bad]\n";
1918             run_command "git checkout $bad" or
1919                 die "Failed to checkout $bad";
1920
1921             $result = run_bisect $type;
1922
1923             if ($result ne "bad") {
1924                 fail "Tested BISECT_BAD [$bad] and it succeeded" and return 0;
1925             }
1926         }
1927
1928         if ($check ne "bad") {
1929             doprint "TESTING BISECT GOOD [$good]\n";
1930             run_command "git checkout $good" or
1931                 die "Failed to checkout $good";
1932
1933             $result = run_bisect $type;
1934
1935             if ($result ne "good") {
1936                 fail "Tested BISECT_GOOD [$good] and it failed" and return 0;
1937             }
1938         }
1939
1940         # checkout where we started
1941         run_command "git checkout $head" or
1942             die "Failed to checkout $head";
1943     }
1944
1945     run_command "git bisect start$start_files" or
1946         dodie "could not start bisect";
1947
1948     run_command "git bisect good $good" or
1949         dodie "could not set bisect good to $good";
1950
1951     run_git_bisect "git bisect bad $bad" or
1952         dodie "could not set bisect bad to $bad";
1953
1954     if (defined($replay)) {
1955         run_command "git bisect replay $replay" or
1956             dodie "failed to run replay";
1957     }
1958
1959     if (defined($start)) {
1960         run_command "git checkout $start" or
1961             dodie "failed to checkout $start";
1962     }
1963
1964     my $test;
1965     do {
1966         $result = run_bisect $type;
1967         $test = run_git_bisect "git bisect $result";
1968     } while ($test);
1969
1970     run_command "git bisect log" or
1971         dodie "could not capture git bisect log";
1972
1973     run_command "git bisect reset" or
1974         dodie "could not reset git bisect";
1975
1976     doprint "Bad commit was [$bisect_bad]\n";
1977
1978     success $i;
1979 }
1980
1981 my %config_ignore;
1982 my %config_set;
1983
1984 my %config_list;
1985 my %null_config;
1986
1987 my %dependency;
1988
1989 sub assign_configs {
1990     my ($hash, $config) = @_;
1991
1992     open (IN, $config)
1993         or dodie "Failed to read $config";
1994
1995     while (<IN>) {
1996         if (/^((CONFIG\S*)=.*)/) {
1997             ${$hash}{$2} = $1;
1998         }
1999     }
2000
2001     close(IN);
2002 }
2003
2004 sub process_config_ignore {
2005     my ($config) = @_;
2006
2007     assign_configs \%config_ignore, $config;
2008 }
2009
2010 sub read_current_config {
2011     my ($config_ref) = @_;
2012
2013     %{$config_ref} = ();
2014     undef %{$config_ref};
2015
2016     my @key = keys %{$config_ref};
2017     if ($#key >= 0) {
2018         print "did not delete!\n";
2019         exit;
2020     }
2021     open (IN, "$output_config");
2022
2023     while (<IN>) {
2024         if (/^(CONFIG\S+)=(.*)/) {
2025             ${$config_ref}{$1} = $2;
2026         }
2027     }
2028     close(IN);
2029 }
2030
2031 sub get_dependencies {
2032     my ($config) = @_;
2033
2034     my $arr = $dependency{$config};
2035     if (!defined($arr)) {
2036         return ();
2037     }
2038
2039     my @deps = @{$arr};
2040
2041     foreach my $dep (@{$arr}) {
2042         print "ADD DEP $dep\n";
2043         @deps = (@deps, get_dependencies $dep);
2044     }
2045
2046     return @deps;
2047 }
2048
2049 sub create_config {
2050     my @configs = @_;
2051
2052     open(OUT, ">$output_config") or dodie "Can not write to $output_config";
2053
2054     foreach my $config (@configs) {
2055         print OUT "$config_set{$config}\n";
2056         my @deps = get_dependencies $config;
2057         foreach my $dep (@deps) {
2058             print OUT "$config_set{$dep}\n";
2059         }
2060     }
2061
2062     foreach my $config (keys %config_ignore) {
2063         print OUT "$config_ignore{$config}\n";
2064     }
2065     close(OUT);
2066
2067 #    exit;
2068     make_oldconfig;
2069 }
2070
2071 sub compare_configs {
2072     my (%a, %b) = @_;
2073
2074     foreach my $item (keys %a) {
2075         if (!defined($b{$item})) {
2076             print "diff $item\n";
2077             return 1;
2078         }
2079         delete $b{$item};
2080     }
2081
2082     my @keys = keys %b;
2083     if ($#keys) {
2084         print "diff2 $keys[0]\n";
2085     }
2086     return -1 if ($#keys >= 0);
2087
2088     return 0;
2089 }
2090
2091 sub run_config_bisect_test {
2092     my ($type) = @_;
2093
2094     return run_bisect_test $type, "oldconfig";
2095 }
2096
2097 sub process_passed {
2098     my (%configs) = @_;
2099
2100     doprint "These configs had no failure: (Enabling them for further compiles)\n";
2101     # Passed! All these configs are part of a good compile.
2102     # Add them to the min options.
2103     foreach my $config (keys %configs) {
2104         if (defined($config_list{$config})) {
2105             doprint " removing $config\n";
2106             $config_ignore{$config} = $config_list{$config};
2107             delete $config_list{$config};
2108         }
2109     }
2110     doprint "config copied to $outputdir/config_good\n";
2111     run_command "cp -f $output_config $outputdir/config_good";
2112 }
2113
2114 sub process_failed {
2115     my ($config) = @_;
2116
2117     doprint "\n\n***************************************\n";
2118     doprint "Found bad config: $config\n";
2119     doprint "***************************************\n\n";
2120 }
2121
2122 sub run_config_bisect {
2123
2124     my @start_list = keys %config_list;
2125
2126     if ($#start_list < 0) {
2127         doprint "No more configs to test!!!\n";
2128         return -1;
2129     }
2130
2131     doprint "***** RUN TEST ***\n";
2132     my $type = $opt{"CONFIG_BISECT_TYPE[$iteration]"};
2133     my $ret;
2134     my %current_config;
2135
2136     my $count = $#start_list + 1;
2137     doprint "  $count configs to test\n";
2138
2139     my $half = int($#start_list / 2);
2140
2141     do {
2142         my @tophalf = @start_list[0 .. $half];
2143
2144         create_config @tophalf;
2145         read_current_config \%current_config;
2146
2147         $count = $#tophalf + 1;
2148         doprint "Testing $count configs\n";
2149         my $found = 0;
2150         # make sure we test something
2151         foreach my $config (@tophalf) {
2152             if (defined($current_config{$config})) {
2153                 logit " $config\n";
2154                 $found = 1;
2155             }
2156         }
2157         if (!$found) {
2158             # try the other half
2159             doprint "Top half produced no set configs, trying bottom half\n";
2160             @tophalf = @start_list[$half + 1 .. $#start_list];
2161             create_config @tophalf;
2162             read_current_config \%current_config;
2163             foreach my $config (@tophalf) {
2164                 if (defined($current_config{$config})) {
2165                     logit " $config\n";
2166                     $found = 1;
2167                 }
2168             }
2169             if (!$found) {
2170                 doprint "Failed: Can't make new config with current configs\n";
2171                 foreach my $config (@start_list) {
2172                     doprint "  CONFIG: $config\n";
2173                 }
2174                 return -1;
2175             }
2176             $count = $#tophalf + 1;
2177             doprint "Testing $count configs\n";
2178         }
2179
2180         $ret = run_config_bisect_test $type;
2181         if ($bisect_manual) {
2182             $ret = answer_bisect;
2183         }
2184         if ($ret) {
2185             process_passed %current_config;
2186             return 0;
2187         }
2188
2189         doprint "This config had a failure.\n";
2190         doprint "Removing these configs that were not set in this config:\n";
2191         doprint "config copied to $outputdir/config_bad\n";
2192         run_command "cp -f $output_config $outputdir/config_bad";
2193
2194         # A config exists in this group that was bad.
2195         foreach my $config (keys %config_list) {
2196             if (!defined($current_config{$config})) {
2197                 doprint " removing $config\n";
2198                 delete $config_list{$config};
2199             }
2200         }
2201
2202         @start_list = @tophalf;
2203
2204         if ($#start_list == 0) {
2205             process_failed $start_list[0];
2206             return 1;
2207         }
2208
2209         # remove half the configs we are looking at and see if
2210         # they are good.
2211         $half = int($#start_list / 2);
2212     } while ($#start_list > 0);
2213
2214     # we found a single config, try it again unless we are running manually
2215
2216     if ($bisect_manual) {
2217         process_failed $start_list[0];
2218         return 1;
2219     }
2220
2221     my @tophalf = @start_list[0 .. 0];
2222
2223     $ret = run_config_bisect_test $type;
2224     if ($ret) {
2225         process_passed %current_config;
2226         return 0;
2227     }
2228
2229     process_failed $start_list[0];
2230     return 1;
2231 }
2232
2233 sub config_bisect {
2234     my ($i) = @_;
2235
2236     my $start_config = $opt{"CONFIG_BISECT[$i]"};
2237
2238     my $tmpconfig = "$tmpdir/use_config";
2239
2240     if (defined($config_bisect_good)) {
2241         process_config_ignore $config_bisect_good;
2242     }
2243
2244     # Make the file with the bad config and the min config
2245     if (defined($minconfig)) {
2246         # read the min config for things to ignore
2247         run_command "cp $minconfig $tmpconfig" or
2248             dodie "failed to copy $minconfig to $tmpconfig";
2249     } else {
2250         unlink $tmpconfig;
2251     }
2252
2253     if (-f $tmpconfig) {
2254         load_force_config($tmpconfig);
2255         process_config_ignore $tmpconfig;
2256     }
2257
2258     # now process the start config
2259     run_command "cp $start_config $output_config" or
2260         dodie "failed to copy $start_config to $output_config";
2261
2262     # read directly what we want to check
2263     my %config_check;
2264     open (IN, $output_config)
2265         or dodie "faied to open $output_config";
2266
2267     while (<IN>) {
2268         if (/^((CONFIG\S*)=.*)/) {
2269             $config_check{$2} = $1;
2270         }
2271     }
2272     close(IN);
2273
2274     # Now run oldconfig with the minconfig
2275     make_oldconfig;
2276
2277     # check to see what we lost (or gained)
2278     open (IN, $output_config)
2279         or dodie "Failed to read $start_config";
2280
2281     my %removed_configs;
2282     my %added_configs;
2283
2284     while (<IN>) {
2285         if (/^((CONFIG\S*)=.*)/) {
2286             # save off all options
2287             $config_set{$2} = $1;
2288             if (defined($config_check{$2})) {
2289                 if (defined($config_ignore{$2})) {
2290                     $removed_configs{$2} = $1;
2291                 } else {
2292                     $config_list{$2} = $1;
2293                 }
2294             } elsif (!defined($config_ignore{$2})) {
2295                 $added_configs{$2} = $1;
2296                 $config_list{$2} = $1;
2297             }
2298         }
2299     }
2300     close(IN);
2301
2302     my @confs = keys %removed_configs;
2303     if ($#confs >= 0) {
2304         doprint "Configs overridden by default configs and removed from check:\n";
2305         foreach my $config (@confs) {
2306             doprint " $config\n";
2307         }
2308     }
2309     @confs = keys %added_configs;
2310     if ($#confs >= 0) {
2311         doprint "Configs appearing in make oldconfig and added:\n";
2312         foreach my $config (@confs) {
2313             doprint " $config\n";
2314         }
2315     }
2316
2317     my %config_test;
2318     my $once = 0;
2319
2320     # Sometimes kconfig does weird things. We must make sure
2321     # that the config we autocreate has everything we need
2322     # to test, otherwise we may miss testing configs, or
2323     # may not be able to create a new config.
2324     # Here we create a config with everything set.
2325     create_config (keys %config_list);
2326     read_current_config \%config_test;
2327     foreach my $config (keys %config_list) {
2328         if (!defined($config_test{$config})) {
2329             if (!$once) {
2330                 $once = 1;
2331                 doprint "Configs not produced by kconfig (will not be checked):\n";
2332             }
2333             doprint "  $config\n";
2334             delete $config_list{$config};
2335         }
2336     }
2337     my $ret;
2338     do {
2339         $ret = run_config_bisect;
2340     } while (!$ret);
2341
2342     return $ret if ($ret < 0);
2343
2344     success $i;
2345 }
2346
2347 sub patchcheck_reboot {
2348     doprint "Reboot and sleep $patchcheck_sleep_time seconds\n";
2349     reboot $patchcheck_sleep_time;
2350 }
2351
2352 sub patchcheck {
2353     my ($i) = @_;
2354
2355     die "PATCHCHECK_START[$i] not defined\n"
2356         if (!defined($opt{"PATCHCHECK_START[$i]"}));
2357     die "PATCHCHECK_TYPE[$i] not defined\n"
2358         if (!defined($opt{"PATCHCHECK_TYPE[$i]"}));
2359
2360     my $start = $opt{"PATCHCHECK_START[$i]"};
2361
2362     my $end = "HEAD";
2363     if (defined($opt{"PATCHCHECK_END[$i]"})) {
2364         $end = $opt{"PATCHCHECK_END[$i]"};
2365     }
2366
2367     # Get the true sha1's since we can use things like HEAD~3
2368     $start = get_sha1($start);
2369     $end = get_sha1($end);
2370
2371     my $type = $opt{"PATCHCHECK_TYPE[$i]"};
2372
2373     # Can't have a test without having a test to run
2374     if ($type eq "test" && !defined($run_test)) {
2375         $type = "boot";
2376     }
2377
2378     open (IN, "git log --pretty=oneline $end|") or
2379         dodie "could not get git list";
2380
2381     my @list;
2382
2383     while (<IN>) {
2384         chomp;
2385         $list[$#list+1] = $_;
2386         last if (/^$start/);
2387     }
2388     close(IN);
2389
2390     if ($list[$#list] !~ /^$start/) {
2391         fail "SHA1 $start not found";
2392     }
2393
2394     # go backwards in the list
2395     @list = reverse @list;
2396
2397     my $save_clean = $noclean;
2398     my %ignored_warnings;
2399
2400     if (defined($ignore_warnings)) {
2401         foreach my $sha1 (split /\s+/, $ignore_warnings) {
2402             $ignored_warnings{$sha1} = 1;
2403         }
2404     }
2405
2406     $in_patchcheck = 1;
2407     foreach my $item (@list) {
2408         my $sha1 = $item;
2409         $sha1 =~ s/^([[:xdigit:]]+).*/$1/;
2410
2411         doprint "\nProcessing commit $item\n\n";
2412
2413         run_command "git checkout $sha1" or
2414             die "Failed to checkout $sha1";
2415
2416         # only clean on the first and last patch
2417         if ($item eq $list[0] ||
2418             $item eq $list[$#list]) {
2419             $noclean = $save_clean;
2420         } else {
2421             $noclean = 1;
2422         }
2423
2424         if (defined($minconfig)) {
2425             build "useconfig:$minconfig" or return 0;
2426         } else {
2427             # ?? no config to use?
2428             build "oldconfig" or return 0;
2429         }
2430
2431
2432         if (!defined($ignored_warnings{$sha1})) {
2433             check_buildlog $sha1 or return 0;
2434         }
2435
2436         next if ($type eq "build");
2437
2438         my $failed = 0;
2439
2440         start_monitor_and_boot or $failed = 1;
2441
2442         if (!$failed && $type ne "boot"){
2443             do_run_test or $failed = 1;
2444         }
2445         end_monitor;
2446         return 0 if ($failed);
2447
2448         patchcheck_reboot;
2449
2450     }
2451     $in_patchcheck = 0;
2452     success $i;
2453
2454     return 1;
2455 }
2456
2457 my %depends;
2458 my %depcount;
2459 my $iflevel = 0;
2460 my @ifdeps;
2461
2462 # prevent recursion
2463 my %read_kconfigs;
2464
2465 sub add_dep {
2466     # $config depends on $dep
2467     my ($config, $dep) = @_;
2468
2469     if (defined($depends{$config})) {
2470         $depends{$config} .= " " . $dep;
2471     } else {
2472         $depends{$config} = $dep;
2473     }
2474
2475     # record the number of configs depending on $dep
2476     if (defined $depcount{$dep}) {
2477         $depcount{$dep}++;
2478     } else {
2479         $depcount{$dep} = 1;
2480     }
2481 }
2482
2483 # taken from streamline_config.pl
2484 sub read_kconfig {
2485     my ($kconfig) = @_;
2486
2487     my $state = "NONE";
2488     my $config;
2489     my @kconfigs;
2490
2491     my $cont = 0;
2492     my $line;
2493
2494
2495     if (! -f $kconfig) {
2496         doprint "file $kconfig does not exist, skipping\n";
2497         return;
2498     }
2499
2500     open(KIN, "$kconfig")
2501         or die "Can't open $kconfig";
2502     while (<KIN>) {
2503         chomp;
2504
2505         # Make sure that lines ending with \ continue
2506         if ($cont) {
2507             $_ = $line . " " . $_;
2508         }
2509
2510         if (s/\\$//) {
2511             $cont = 1;
2512             $line = $_;
2513             next;
2514         }
2515
2516         $cont = 0;
2517
2518         # collect any Kconfig sources
2519         if (/^source\s*"(.*)"/) {
2520             $kconfigs[$#kconfigs+1] = $1;
2521         }
2522
2523         # configs found
2524         if (/^\s*(menu)?config\s+(\S+)\s*$/) {
2525             $state = "NEW";
2526             $config = $2;
2527
2528             for (my $i = 0; $i < $iflevel; $i++) {
2529                 add_dep $config, $ifdeps[$i];
2530             }
2531
2532         # collect the depends for the config
2533         } elsif ($state eq "NEW" && /^\s*depends\s+on\s+(.*)$/) {
2534
2535             add_dep $config, $1;
2536
2537         # Get the configs that select this config
2538         } elsif ($state eq "NEW" && /^\s*select\s+(\S+)/) {
2539
2540             # selected by depends on config
2541             add_dep $1, $config;
2542
2543         # Check for if statements
2544         } elsif (/^if\s+(.*\S)\s*$/) {
2545             my $deps = $1;
2546             # remove beginning and ending non text
2547             $deps =~ s/^[^a-zA-Z0-9_]*//;
2548             $deps =~ s/[^a-zA-Z0-9_]*$//;
2549
2550             my @deps = split /[^a-zA-Z0-9_]+/, $deps;
2551
2552             $ifdeps[$iflevel++] = join ':', @deps;
2553
2554         } elsif (/^endif/) {
2555
2556             $iflevel-- if ($iflevel);
2557
2558         # stop on "help"
2559         } elsif (/^\s*help\s*$/) {
2560             $state = "NONE";
2561         }
2562     }
2563     close(KIN);
2564
2565     # read in any configs that were found.
2566     foreach $kconfig (@kconfigs) {
2567         if (!defined($read_kconfigs{$kconfig})) {
2568             $read_kconfigs{$kconfig} = 1;
2569             read_kconfig("$builddir/$kconfig");
2570         }
2571     }
2572 }
2573
2574 sub read_depends {
2575     # find out which arch this is by the kconfig file
2576     open (IN, $output_config)
2577         or dodie "Failed to read $output_config";
2578     my $arch;
2579     while (<IN>) {
2580         if (m,Linux/(\S+)\s+\S+\s+Kernel Configuration,) {
2581             $arch = $1;
2582             last;
2583         }
2584     }
2585     close IN;
2586
2587     if (!defined($arch)) {
2588         doprint "Could not find arch from config file\n";
2589         doprint "no dependencies used\n";
2590         return;
2591     }
2592
2593     # arch is really the subarch, we need to know
2594     # what directory to look at.
2595     if ($arch eq "i386" || $arch eq "x86_64") {
2596         $arch = "x86";
2597     } elsif ($arch =~ /^tile/) {
2598         $arch = "tile";
2599     }
2600
2601     my $kconfig = "$builddir/arch/$arch/Kconfig";
2602
2603     if (! -f $kconfig && $arch =~ /\d$/) {
2604         my $orig = $arch;
2605         # some subarchs have numbers, truncate them
2606         $arch =~ s/\d*$//;
2607         $kconfig = "$builddir/arch/$arch/Kconfig";
2608         if (! -f $kconfig) {
2609             doprint "No idea what arch dir $orig is for\n";
2610             doprint "no dependencies used\n";
2611             return;
2612         }
2613     }
2614
2615     read_kconfig($kconfig);
2616 }
2617
2618 sub read_config_list {
2619     my ($config) = @_;
2620
2621     open (IN, $config)
2622         or dodie "Failed to read $config";
2623
2624     while (<IN>) {
2625         if (/^((CONFIG\S*)=.*)/) {
2626             if (!defined($config_ignore{$2})) {
2627                 $config_list{$2} = $1;
2628             }
2629         }
2630     }
2631
2632     close(IN);
2633 }
2634
2635 sub read_output_config {
2636     my ($config) = @_;
2637
2638     assign_configs \%config_ignore, $config;
2639 }
2640
2641 sub make_new_config {
2642     my @configs = @_;
2643
2644     open (OUT, ">$output_config")
2645         or dodie "Failed to write $output_config";
2646
2647     foreach my $config (@configs) {
2648         print OUT "$config\n";
2649     }
2650     close OUT;
2651 }
2652
2653 sub chomp_config {
2654     my ($config) = @_;
2655
2656     $config =~ s/CONFIG_//;
2657
2658     return $config;
2659 }
2660
2661 sub get_depends {
2662     my ($dep) = @_;
2663
2664     my $kconfig = chomp_config $dep;
2665
2666     $dep = $depends{"$kconfig"};
2667
2668     # the dep string we have saves the dependencies as they
2669     # were found, including expressions like ! && ||. We
2670     # want to split this out into just an array of configs.
2671
2672     my $valid = "A-Za-z_0-9";
2673
2674     my @configs;
2675
2676     while ($dep =~ /[$valid]/) {
2677
2678         if ($dep =~ /^[^$valid]*([$valid]+)/) {
2679             my $conf = "CONFIG_" . $1;
2680
2681             $configs[$#configs + 1] = $conf;
2682
2683             $dep =~ s/^[^$valid]*[$valid]+//;
2684         } else {
2685             die "this should never happen";
2686         }
2687     }
2688
2689     return @configs;
2690 }
2691
2692 my %min_configs;
2693 my %keep_configs;
2694 my %save_configs;
2695 my %processed_configs;
2696 my %nochange_config;
2697
2698 sub test_this_config {
2699     my ($config) = @_;
2700
2701     my $found;
2702
2703     # if we already processed this config, skip it
2704     if (defined($processed_configs{$config})) {
2705         return undef;
2706     }
2707     $processed_configs{$config} = 1;
2708
2709     # if this config failed during this round, skip it
2710     if (defined($nochange_config{$config})) {
2711         return undef;
2712     }
2713
2714     my $kconfig = chomp_config $config;
2715
2716     # Test dependencies first
2717     if (defined($depends{"$kconfig"})) {
2718         my @parents = get_depends $config;
2719         foreach my $parent (@parents) {
2720             # if the parent is in the min config, check it first
2721             next if (!defined($min_configs{$parent}));
2722             $found = test_this_config($parent);
2723             if (defined($found)) {
2724                 return $found;
2725             }
2726         }
2727     }
2728
2729     # Remove this config from the list of configs
2730     # do a make oldnoconfig and then read the resulting
2731     # .config to make sure it is missing the config that
2732     # we had before
2733     my %configs = %min_configs;
2734     delete $configs{$config};
2735     make_new_config ((values %configs), (values %keep_configs));
2736     make_oldconfig;
2737     undef %configs;
2738     assign_configs \%configs, $output_config;
2739
2740     return $config if (!defined($configs{$config}));
2741
2742     doprint "disabling config $config did not change .config\n";
2743
2744     $nochange_config{$config} = 1;
2745
2746     return undef;
2747 }
2748
2749 sub make_min_config {
2750     my ($i) = @_;
2751
2752     if (!defined($output_minconfig)) {
2753         fail "OUTPUT_MIN_CONFIG not defined" and return;
2754     }
2755
2756     # If output_minconfig exists, and the start_minconfig
2757     # came from min_config, than ask if we should use
2758     # that instead.
2759     if (-f $output_minconfig && !$start_minconfig_defined) {
2760         print "$output_minconfig exists\n";
2761         if (read_yn " Use it as minconfig?") {
2762             $start_minconfig = $output_minconfig;
2763         }
2764     }
2765
2766     if (!defined($start_minconfig)) {
2767         fail "START_MIN_CONFIG or MIN_CONFIG not defined" and return;
2768     }
2769
2770     my $temp_config = "$tmpdir/temp_config";
2771
2772     # First things first. We build an allnoconfig to find
2773     # out what the defaults are that we can't touch.
2774     # Some are selections, but we really can't handle selections.
2775
2776     my $save_minconfig = $minconfig;
2777     undef $minconfig;
2778
2779     run_command "$make allnoconfig" or return 0;
2780
2781     read_depends;
2782
2783     process_config_ignore $output_config;
2784
2785     undef %save_configs;
2786     undef %min_configs;
2787
2788     if (defined($ignore_config)) {
2789         # make sure the file exists
2790         `touch $ignore_config`;
2791         assign_configs \%save_configs, $ignore_config;
2792     }
2793
2794     %keep_configs = %save_configs;
2795
2796     doprint "Load initial configs from $start_minconfig\n";
2797
2798     # Look at the current min configs, and save off all the
2799     # ones that were set via the allnoconfig
2800     assign_configs \%min_configs, $start_minconfig;
2801
2802     my @config_keys = keys %min_configs;
2803
2804     # All configs need a depcount
2805     foreach my $config (@config_keys) {
2806         my $kconfig = chomp_config $config;
2807         if (!defined $depcount{$kconfig}) {
2808                 $depcount{$kconfig} = 0;
2809         }
2810     }
2811
2812     # Remove anything that was set by the make allnoconfig
2813     # we shouldn't need them as they get set for us anyway.
2814     foreach my $config (@config_keys) {
2815         # Remove anything in the ignore_config
2816         if (defined($keep_configs{$config})) {
2817             my $file = $ignore_config;
2818             $file =~ s,.*/(.*?)$,$1,;
2819             doprint "$config set by $file ... ignored\n";
2820             delete $min_configs{$config};
2821             next;
2822         }
2823         # But make sure the settings are the same. If a min config
2824         # sets a selection, we do not want to get rid of it if
2825         # it is not the same as what we have. Just move it into
2826         # the keep configs.
2827         if (defined($config_ignore{$config})) {
2828             if ($config_ignore{$config} ne $min_configs{$config}) {
2829                 doprint "$config is in allnoconfig as '$config_ignore{$config}'";
2830                 doprint " but it is '$min_configs{$config}' in minconfig .. keeping\n";
2831                 $keep_configs{$config} = $min_configs{$config};
2832             } else {
2833                 doprint "$config set by allnoconfig ... ignored\n";
2834             }
2835             delete $min_configs{$config};
2836         }
2837     }
2838
2839     my $done = 0;
2840     my $take_two = 0;
2841
2842     while (!$done) {
2843
2844         my $config;
2845         my $found;
2846
2847         # Now disable each config one by one and do a make oldconfig
2848         # till we find a config that changes our list.
2849
2850         my @test_configs = keys %min_configs;
2851
2852         # Sort keys by who is most dependent on
2853         @test_configs = sort  { $depcount{chomp_config($b)} <=> $depcount{chomp_config($a)} }
2854                           @test_configs ;
2855
2856         # Put configs that did not modify the config at the end.
2857         my $reset = 1;
2858         for (my $i = 0; $i < $#test_configs; $i++) {
2859             if (!defined($nochange_config{$test_configs[0]})) {
2860                 $reset = 0;
2861                 last;
2862             }
2863             # This config didn't change the .config last time.
2864             # Place it at the end
2865             my $config = shift @test_configs;
2866             push @test_configs, $config;
2867         }
2868
2869         # if every test config has failed to modify the .config file
2870         # in the past, then reset and start over.
2871         if ($reset) {
2872             undef %nochange_config;
2873         }
2874
2875         undef %processed_configs;
2876
2877         foreach my $config (@test_configs) {
2878
2879             $found = test_this_config $config;
2880
2881             last if (defined($found));
2882
2883             # oh well, try another config
2884         }
2885
2886         if (!defined($found)) {
2887             # we could have failed due to the nochange_config hash
2888             # reset and try again
2889             if (!$take_two) {
2890                 undef %nochange_config;
2891                 $take_two = 1;
2892                 next;
2893             }
2894             doprint "No more configs found that we can disable\n";
2895             $done = 1;
2896             last;
2897         }
2898         $take_two = 0;
2899
2900         $config = $found;
2901
2902         doprint "Test with $config disabled\n";
2903
2904         # set in_bisect to keep build and monitor from dieing
2905         $in_bisect = 1;
2906
2907         my $failed = 0;
2908         build "oldconfig";
2909         start_monitor_and_boot or $failed = 1;
2910         end_monitor;
2911
2912         $in_bisect = 0;
2913
2914         if ($failed) {
2915             doprint "$min_configs{$config} is needed to boot the box... keeping\n";
2916             # this config is needed, add it to the ignore list.
2917             $keep_configs{$config} = $min_configs{$config};
2918             $save_configs{$config} = $min_configs{$config};
2919             delete $min_configs{$config};
2920
2921             # update new ignore configs
2922             if (defined($ignore_config)) {
2923                 open (OUT, ">$temp_config")
2924                     or die "Can't write to $temp_config";
2925                 foreach my $config (keys %save_configs) {
2926                     print OUT "$save_configs{$config}\n";
2927                 }
2928                 close OUT;
2929                 run_command "mv $temp_config $ignore_config" or
2930                     dodie "failed to copy update to $ignore_config";
2931             }
2932
2933         } else {
2934             # We booted without this config, remove it from the minconfigs.
2935             doprint "$config is not needed, disabling\n";
2936
2937             delete $min_configs{$config};
2938
2939             # Also disable anything that is not enabled in this config
2940             my %configs;
2941             assign_configs \%configs, $output_config;
2942             my @config_keys = keys %min_configs;
2943             foreach my $config (@config_keys) {
2944                 if (!defined($configs{$config})) {
2945                     doprint "$config is not set, disabling\n";
2946                     delete $min_configs{$config};
2947                 }
2948             }
2949
2950             # Save off all the current mandidory configs
2951             open (OUT, ">$temp_config")
2952                 or die "Can't write to $temp_config";
2953             foreach my $config (keys %keep_configs) {
2954                 print OUT "$keep_configs{$config}\n";
2955             }
2956             foreach my $config (keys %min_configs) {
2957                 print OUT "$min_configs{$config}\n";
2958             }
2959             close OUT;
2960
2961             run_command "mv $temp_config $output_minconfig" or
2962                 dodie "failed to copy update to $output_minconfig";
2963         }
2964
2965         doprint "Reboot and wait $sleep_time seconds\n";
2966         reboot $sleep_time;
2967     }
2968
2969     success $i;
2970     return 1;
2971 }
2972
2973 $#ARGV < 1 or die "ktest.pl version: $VERSION\n   usage: ktest.pl config-file\n";
2974
2975 if ($#ARGV == 0) {
2976     $ktest_config = $ARGV[0];
2977     if (! -f $ktest_config) {
2978         print "$ktest_config does not exist.\n";
2979         if (!read_yn "Create it?") {
2980             exit 0;
2981         }
2982     }
2983 } else {
2984     $ktest_config = "ktest.conf";
2985 }
2986
2987 if (! -f $ktest_config) {
2988     open(OUT, ">$ktest_config") or die "Can not create $ktest_config";
2989     print OUT << "EOF"
2990 # Generated by ktest.pl
2991 #
2992 # Define each test with TEST_START
2993 # The config options below it will override the defaults
2994 TEST_START
2995
2996 DEFAULTS
2997 EOF
2998 ;
2999     close(OUT);
3000 }
3001 read_config $ktest_config;
3002
3003 if (defined($opt{"LOG_FILE"})) {
3004     $opt{"LOG_FILE"} = eval_option($opt{"LOG_FILE"}, -1);
3005 }
3006
3007 # Append any configs entered in manually to the config file.
3008 my @new_configs = keys %entered_configs;
3009 if ($#new_configs >= 0) {
3010     print "\nAppending entered in configs to $ktest_config\n";
3011     open(OUT, ">>$ktest_config") or die "Can not append to $ktest_config";
3012     foreach my $config (@new_configs) {
3013         print OUT "$config = $entered_configs{$config}\n";
3014         $opt{$config} = $entered_configs{$config};
3015     }
3016 }
3017
3018 if ($opt{"CLEAR_LOG"} && defined($opt{"LOG_FILE"})) {
3019     unlink $opt{"LOG_FILE"};
3020 }
3021
3022 doprint "\n\nSTARTING AUTOMATED TESTS\n\n";
3023
3024 for (my $i = 0, my $repeat = 1; $i <= $opt{"NUM_TESTS"}; $i += $repeat) {
3025
3026     if (!$i) {
3027         doprint "DEFAULT OPTIONS:\n";
3028     } else {
3029         doprint "\nTEST $i OPTIONS";
3030         if (defined($repeat_tests{$i})) {
3031             $repeat = $repeat_tests{$i};
3032             doprint " ITERATE $repeat";
3033         }
3034         doprint "\n";
3035     }
3036
3037     foreach my $option (sort keys %opt) {
3038
3039         if ($option =~ /\[(\d+)\]$/) {
3040             next if ($i != $1);
3041         } else {
3042             next if ($i);
3043         }
3044
3045         doprint "$option = $opt{$option}\n";
3046     }
3047 }
3048
3049 sub __set_test_option {
3050     my ($name, $i) = @_;
3051
3052     my $option = "$name\[$i\]";
3053
3054     if (defined($opt{$option})) {
3055         return $opt{$option};
3056     }
3057
3058     foreach my $test (keys %repeat_tests) {
3059         if ($i >= $test &&
3060             $i < $test + $repeat_tests{$test}) {
3061             $option = "$name\[$test\]";
3062             if (defined($opt{$option})) {
3063                 return $opt{$option};
3064             }
3065         }
3066     }
3067
3068     if (defined($opt{$name})) {
3069         return $opt{$name};
3070     }
3071
3072     return undef;
3073 }
3074
3075 sub set_test_option {
3076     my ($name, $i) = @_;
3077
3078     my $option = __set_test_option($name, $i);
3079     return $option if (!defined($option));
3080
3081     return eval_option($option, $i);
3082 }
3083
3084 # First we need to do is the builds
3085 for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) {
3086
3087     # Do not reboot on failing test options
3088     $no_reboot = 1;
3089
3090     $iteration = $i;
3091
3092     my $makecmd = set_test_option("MAKE_CMD", $i);
3093
3094     $machine = set_test_option("MACHINE", $i);
3095     $ssh_user = set_test_option("SSH_USER", $i);
3096     $tmpdir = set_test_option("TMP_DIR", $i);
3097     $outputdir = set_test_option("OUTPUT_DIR", $i);
3098     $builddir = set_test_option("BUILD_DIR", $i);
3099     $test_type = set_test_option("TEST_TYPE", $i);
3100     $build_type = set_test_option("BUILD_TYPE", $i);
3101     $build_options = set_test_option("BUILD_OPTIONS", $i);
3102     $pre_build = set_test_option("PRE_BUILD", $i);
3103     $post_build = set_test_option("POST_BUILD", $i);
3104     $pre_build_die = set_test_option("PRE_BUILD_DIE", $i);
3105     $post_build_die = set_test_option("POST_BUILD_DIE", $i);
3106     $power_cycle = set_test_option("POWER_CYCLE", $i);
3107     $reboot = set_test_option("REBOOT", $i);
3108     $noclean = set_test_option("BUILD_NOCLEAN", $i);
3109     $minconfig = set_test_option("MIN_CONFIG", $i);
3110     $output_minconfig = set_test_option("OUTPUT_MIN_CONFIG", $i);
3111     $start_minconfig = set_test_option("START_MIN_CONFIG", $i);
3112     $ignore_config = set_test_option("IGNORE_CONFIG", $i);
3113     $run_test = set_test_option("TEST", $i);
3114     $addconfig = set_test_option("ADD_CONFIG", $i);
3115     $reboot_type = set_test_option("REBOOT_TYPE", $i);
3116     $grub_menu = set_test_option("GRUB_MENU", $i);
3117     $post_install = set_test_option("POST_INSTALL", $i);
3118     $no_install = set_test_option("NO_INSTALL", $i);
3119     $reboot_script = set_test_option("REBOOT_SCRIPT", $i);
3120     $reboot_on_error = set_test_option("REBOOT_ON_ERROR", $i);
3121     $poweroff_on_error = set_test_option("POWEROFF_ON_ERROR", $i);
3122     $die_on_failure = set_test_option("DIE_ON_FAILURE", $i);
3123     $power_off = set_test_option("POWER_OFF", $i);
3124     $powercycle_after_reboot = set_test_option("POWERCYCLE_AFTER_REBOOT", $i);
3125     $poweroff_after_halt = set_test_option("POWEROFF_AFTER_HALT", $i);
3126     $sleep_time = set_test_option("SLEEP_TIME", $i);
3127     $bisect_sleep_time = set_test_option("BISECT_SLEEP_TIME", $i);
3128     $patchcheck_sleep_time = set_test_option("PATCHCHECK_SLEEP_TIME", $i);
3129     $ignore_warnings = set_test_option("IGNORE_WARNINGS", $i);
3130     $bisect_manual = set_test_option("BISECT_MANUAL", $i);
3131     $bisect_skip = set_test_option("BISECT_SKIP", $i);
3132     $config_bisect_good = set_test_option("CONFIG_BISECT_GOOD", $i);
3133     $store_failures = set_test_option("STORE_FAILURES", $i);
3134     $test_name = set_test_option("TEST_NAME", $i);
3135     $timeout = set_test_option("TIMEOUT", $i);
3136     $booted_timeout = set_test_option("BOOTED_TIMEOUT", $i);
3137     $console = set_test_option("CONSOLE", $i);
3138     $detect_triplefault = set_test_option("DETECT_TRIPLE_FAULT", $i);
3139     $success_line = set_test_option("SUCCESS_LINE", $i);
3140     $reboot_success_line = set_test_option("REBOOT_SUCCESS_LINE", $i);
3141     $stop_after_success = set_test_option("STOP_AFTER_SUCCESS", $i);
3142     $stop_after_failure = set_test_option("STOP_AFTER_FAILURE", $i);
3143     $stop_test_after = set_test_option("STOP_TEST_AFTER", $i);
3144     $build_target = set_test_option("BUILD_TARGET", $i);
3145     $ssh_exec = set_test_option("SSH_EXEC", $i);
3146     $scp_to_target = set_test_option("SCP_TO_TARGET", $i);
3147     $target_image = set_test_option("TARGET_IMAGE", $i);
3148     $localversion = set_test_option("LOCALVERSION", $i);
3149
3150     $start_minconfig_defined = 1;
3151
3152     if (!defined($start_minconfig)) {
3153         $start_minconfig_defined = 0;
3154         $start_minconfig = $minconfig;
3155     }
3156
3157     chdir $builddir || die "can't change directory to $builddir";
3158
3159     foreach my $dir ($tmpdir, $outputdir) {
3160         if (!-d $dir) {
3161             mkpath($dir) or
3162                 die "can't create $dir";
3163         }
3164     }
3165
3166     $ENV{"SSH_USER"} = $ssh_user;
3167     $ENV{"MACHINE"} = $machine;
3168
3169     $target = "$ssh_user\@$machine";
3170
3171     $buildlog = "$tmpdir/buildlog-$machine";
3172     $dmesg = "$tmpdir/dmesg-$machine";
3173     $make = "$makecmd O=$outputdir";
3174     $output_config = "$outputdir/.config";
3175
3176     if ($reboot_type eq "grub") {
3177         dodie "GRUB_MENU not defined" if (!defined($grub_menu));
3178     } elsif (!defined($reboot_script)) {
3179         dodie "REBOOT_SCRIPT not defined"
3180     }
3181
3182     my $run_type = $build_type;
3183     if ($test_type eq "patchcheck") {
3184         $run_type = $opt{"PATCHCHECK_TYPE[$i]"};
3185     } elsif ($test_type eq "bisect") {
3186         $run_type = $opt{"BISECT_TYPE[$i]"};
3187     } elsif ($test_type eq "config_bisect") {
3188         $run_type = $opt{"CONFIG_BISECT_TYPE[$i]"};
3189     }
3190
3191     if ($test_type eq "make_min_config") {
3192         $run_type = "";
3193     }
3194
3195     # mistake in config file?
3196     if (!defined($run_type)) {
3197         $run_type = "ERROR";
3198     }
3199
3200     my $installme = "";
3201     $installme = " no_install" if ($no_install);
3202
3203     doprint "\n\n";
3204     doprint "RUNNING TEST $i of $opt{NUM_TESTS} with option $test_type $run_type$installme\n\n";
3205
3206     unlink $dmesg;
3207     unlink $buildlog;
3208
3209     if (defined($addconfig)) {
3210         my $min = $minconfig;
3211         if (!defined($minconfig)) {
3212             $min = "";
3213         }
3214         run_command "cat $addconfig $min > $tmpdir/add_config" or
3215             dodie "Failed to create temp config";
3216         $minconfig = "$tmpdir/add_config";
3217     }
3218
3219     my $checkout = $opt{"CHECKOUT[$i]"};
3220     if (defined($checkout)) {
3221         run_command "git checkout $checkout" or
3222             die "failed to checkout $checkout";
3223     }
3224
3225     $no_reboot = 0;
3226
3227
3228     if ($test_type eq "bisect") {
3229         bisect $i;
3230         next;
3231     } elsif ($test_type eq "config_bisect") {
3232         config_bisect $i;
3233         next;
3234     } elsif ($test_type eq "patchcheck") {
3235         patchcheck $i;
3236         next;
3237     } elsif ($test_type eq "make_min_config") {
3238         make_min_config $i;
3239         next;
3240     }
3241
3242     if ($build_type ne "nobuild") {
3243         build $build_type or next;
3244     }
3245
3246     if ($test_type eq "install") {
3247         get_version;
3248         install;
3249         success $i;
3250         next;
3251     }
3252
3253     if ($test_type ne "build") {
3254         my $failed = 0;
3255         start_monitor_and_boot or $failed = 1;
3256
3257         if (!$failed && $test_type ne "boot" && defined($run_test)) {
3258             do_run_test or $failed = 1;
3259         }
3260         end_monitor;
3261         next if ($failed);
3262     }
3263
3264     success $i;
3265 }
3266
3267 if ($opt{"POWEROFF_ON_SUCCESS"}) {
3268     halt;
3269 } elsif ($opt{"REBOOT_ON_SUCCESS"} && !do_not_reboot) {
3270     reboot;
3271 }
3272
3273 doprint "\n    $successes of $opt{NUM_TESTS} tests were successful\n\n";
3274
3275 exit 0;