tools/testing/ktest/ktest.pl

   1 #!/usr/bin/perl -w
   2 #
   3 # Copyright 2010 - Steven Rostedt <srostedt@redhat.com>, Red Hat Inc.
   4 # Licensed under the terms of the GNU GPL License version 2
   5 #
   6
   7 use strict;
   8 use IPC::Open2;
   9 use Fcntl qw(F_GETFL F_SETFL O_NONBLOCK);
  10 use File::Path qw(mkpath);
  11 use File::Copy qw(cp);
  12 use FileHandle;
  13
  14 my $VERSION = "0.2";
  15
  16 $| = 1;
  17
  18 my %opt;
  19 my %repeat_tests;
  20 my %repeats;
  21
  22 #default opts
  23 my %default = (
  24     "NUM_TESTS"                 => 1,
  25     "TEST_TYPE"                 => "build",
  26     "BUILD_TYPE"                => "randconfig",
  27     "MAKE_CMD"                  => "make",
  28     "TIMEOUT"                   => 120,
  29     "TMP_DIR"                   => "/tmp/ktest/\${MACHINE}",
  30     "SLEEP_TIME"                => 60,  # sleep time between tests
  31     "BUILD_NOCLEAN"             => 0,
  32     "REBOOT_ON_ERROR"           => 0,
  33     "POWEROFF_ON_ERROR"         => 0,
  34     "REBOOT_ON_SUCCESS"         => 1,
  35     "POWEROFF_ON_SUCCESS"       => 0,
  36     "BUILD_OPTIONS"             => "",
  37     "BISECT_SLEEP_TIME"         => 60,   # sleep time between bisects
  38     "PATCHCHECK_SLEEP_TIME"     => 60, # sleep time between patch checks
  39     "CLEAR_LOG"                 => 0,
  40     "BISECT_MANUAL"             => 0,
  41     "BISECT_SKIP"               => 1,
  42     "MIN_CONFIG_TYPE"           => "boot",
  43     "SUCCESS_LINE"              => "login:",
  44     "DETECT_TRIPLE_FAULT"       => 1,
  45     "NO_INSTALL"                => 0,
  46     "BOOTED_TIMEOUT"            => 1,
  47     "DIE_ON_FAILURE"            => 1,
  48     "SSH_EXEC"                  => "ssh \$SSH_USER\@\$MACHINE \$SSH_COMMAND",
  49     "SCP_TO_TARGET"             => "scp \$SRC_FILE \$SSH_USER\@\$MACHINE:\$DST_FILE",
  50     "SCP_TO_TARGET_INSTALL"     => "\${SCP_TO_TARGET}",
  51     "REBOOT"                    => "ssh \$SSH_USER\@\$MACHINE reboot",
  52     "STOP_AFTER_SUCCESS"        => 10,
  53     "STOP_AFTER_FAILURE"        => 60,
  54     "STOP_TEST_AFTER"           => 600,
  55     "MAX_MONITOR_WAIT"          => 1800,
  56     "GRUB_REBOOT"               => "grub2-reboot",
  57     "SYSLINUX"                  => "extlinux",
  58     "SYSLINUX_PATH"             => "/boot/extlinux",
  59
  60 # required, and we will ask users if they don't have them but we keep the default
  61 # value something that is common.
  62     "REBOOT_TYPE"               => "grub",
  63     "LOCALVERSION"              => "-test",
  64     "SSH_USER"                  => "root",
  65     "BUILD_TARGET"              => "arch/x86/boot/bzImage",
  66     "TARGET_IMAGE"              => "/boot/vmlinuz-test",
  67
  68     "LOG_FILE"                  => undef,
  69     "IGNORE_UNUSED"             => 0,
  70 );
  71
  72 my $ktest_config;
  73 my $version;
  74 my $have_version = 0;
  75 my $machine;
  76 my $last_machine;
  77 my $ssh_user;
  78 my $tmpdir;
  79 my $builddir;
  80 my $outputdir;
  81 my $output_config;
  82 my $test_type;
  83 my $build_type;
  84 my $build_options;
  85 my $final_post_ktest;
  86 my $pre_ktest;
  87 my $post_ktest;
  88 my $pre_test;
  89 my $post_test;
  90 my $pre_build;
  91 my $post_build;
  92 my $pre_build_die;
  93 my $post_build_die;
  94 my $reboot_type;
  95 my $reboot_script;
  96 my $power_cycle;
  97 my $reboot;
  98 my $reboot_on_error;
  99 my $switch_to_good;
 100 my $switch_to_test;
 101 my $poweroff_on_error;
 102 my $reboot_on_success;
 103 my $die_on_failure;
 104 my $powercycle_after_reboot;
 105 my $poweroff_after_halt;
 106 my $max_monitor_wait;
 107 my $ssh_exec;
 108 my $scp_to_target;
 109 my $scp_to_target_install;
 110 my $power_off;
 111 my $grub_menu;
 112 my $last_grub_menu;
 113 my $grub_file;
 114 my $grub_number;
 115 my $grub_reboot;
 116 my $syslinux;
 117 my $syslinux_path;
 118 my $syslinux_label;
 119 my $target;
 120 my $make;
 121 my $pre_install;
 122 my $post_install;
 123 my $no_install;
 124 my $noclean;
 125 my $minconfig;
 126 my $start_minconfig;
 127 my $start_minconfig_defined;
 128 my $output_minconfig;
 129 my $minconfig_type;
 130 my $use_output_minconfig;
 131 my $warnings_file;
 132 my $ignore_config;
 133 my $ignore_errors;
 134 my $addconfig;
 135 my $in_bisect = 0;
 136 my $bisect_bad_commit = "";
 137 my $reverse_bisect;
 138 my $bisect_manual;
 139 my $bisect_skip;
 140 my $config_bisect_good;
 141 my $bisect_ret_good;
 142 my $bisect_ret_bad;
 143 my $bisect_ret_skip;
 144 my $bisect_ret_abort;
 145 my $bisect_ret_default;
 146 my $in_patchcheck = 0;
 147 my $run_test;
 148 my $redirect;
 149 my $buildlog;
 150 my $testlog;
 151 my $dmesg;
 152 my $monitor_fp;
 153 my $monitor_pid;
 154 my $monitor_cnt = 0;
 155 my $sleep_time;
 156 my $bisect_sleep_time;
 157 my $patchcheck_sleep_time;
 158 my $ignore_warnings;
 159 my $store_failures;
 160 my $store_successes;
 161 my $test_name;
 162 my $timeout;
 163 my $booted_timeout;
 164 my $detect_triplefault;
 165 my $console;
 166 my $reboot_success_line;
 167 my $success_line;
 168 my $stop_after_success;
 169 my $stop_after_failure;
 170 my $stop_test_after;
 171 my $build_target;
 172 my $target_image;
 173 my $checkout;
 174 my $localversion;
 175 my $iteration = 0;
 176 my $successes = 0;
 177
 178 my $bisect_good;
 179 my $bisect_bad;
 180 my $bisect_type;
 181 my $bisect_start;
 182 my $bisect_replay;
 183 my $bisect_files;
 184 my $bisect_reverse;
 185 my $bisect_check;
 186
 187 my $config_bisect;
 188 my $config_bisect_type;
 189 my $config_bisect_check;
 190
 191 my $patchcheck_type;
 192 my $patchcheck_start;
 193 my $patchcheck_end;
 194
 195 # set when a test is something other that just building or install
 196 # which would require more options.
 197 my $buildonly = 1;
 198
 199 # tell build not to worry about warnings, even when WARNINGS_FILE is set
 200 my $warnings_ok = 0;
 201
 202 # set when creating a new config
 203 my $newconfig = 0;
 204
 205 my %entered_configs;
 206 my %config_help;
 207 my %variable;
 208
 209 # force_config is the list of configs that we force enabled (or disabled)
 210 # in a .config file. The MIN_CONFIG and ADD_CONFIG configs.
 211 my %force_config;
 212
 213 # do not force reboots on config problems
 214 my $no_reboot = 1;
 215
 216 # reboot on success
 217 my $reboot_success = 0;
 218
 219 my %option_map = (
 220     "MACHINE"                   => \$machine,
 221     "SSH_USER"                  => \$ssh_user,
 222     "TMP_DIR"                   => \$tmpdir,
 223     "OUTPUT_DIR"                => \$outputdir,
 224     "BUILD_DIR"                 => \$builddir,
 225     "TEST_TYPE"                 => \$test_type,
 226     "PRE_KTEST"                 => \$pre_ktest,
 227     "POST_KTEST"                => \$post_ktest,
 228     "PRE_TEST"                  => \$pre_test,
 229     "POST_TEST"                 => \$post_test,
 230     "BUILD_TYPE"                => \$build_type,
 231     "BUILD_OPTIONS"             => \$build_options,
 232     "PRE_BUILD"                 => \$pre_build,
 233     "POST_BUILD"                => \$post_build,
 234     "PRE_BUILD_DIE"             => \$pre_build_die,
 235     "POST_BUILD_DIE"            => \$post_build_die,
 236     "POWER_CYCLE"               => \$power_cycle,
 237     "REBOOT"                    => \$reboot,
 238     "BUILD_NOCLEAN"             => \$noclean,
 239     "MIN_CONFIG"                => \$minconfig,
 240     "OUTPUT_MIN_CONFIG"         => \$output_minconfig,
 241     "START_MIN_CONFIG"          => \$start_minconfig,
 242     "MIN_CONFIG_TYPE"           => \$minconfig_type,
 243     "USE_OUTPUT_MIN_CONFIG"     => \$use_output_minconfig,
 244     "WARNINGS_FILE"             => \$warnings_file,
 245     "IGNORE_CONFIG"             => \$ignore_config,
 246     "TEST"                      => \$run_test,
 247     "ADD_CONFIG"                => \$addconfig,
 248     "REBOOT_TYPE"               => \$reboot_type,
 249     "GRUB_MENU"                 => \$grub_menu,
 250     "GRUB_FILE"                 => \$grub_file,
 251     "GRUB_REBOOT"               => \$grub_reboot,
 252     "SYSLINUX"                  => \$syslinux,
 253     "SYSLINUX_PATH"             => \$syslinux_path,
 254     "SYSLINUX_LABEL"            => \$syslinux_label,
 255     "PRE_INSTALL"               => \$pre_install,
 256     "POST_INSTALL"              => \$post_install,
 257     "NO_INSTALL"                => \$no_install,
 258     "REBOOT_SCRIPT"             => \$reboot_script,
 259     "REBOOT_ON_ERROR"           => \$reboot_on_error,
 260     "SWITCH_TO_GOOD"            => \$switch_to_good,
 261     "SWITCH_TO_TEST"            => \$switch_to_test,
 262     "POWEROFF_ON_ERROR"         => \$poweroff_on_error,
 263     "REBOOT_ON_SUCCESS"         => \$reboot_on_success,
 264     "DIE_ON_FAILURE"            => \$die_on_failure,
 265     "POWER_OFF"                 => \$power_off,
 266     "POWERCYCLE_AFTER_REBOOT"   => \$powercycle_after_reboot,
 267     "POWEROFF_AFTER_HALT"       => \$poweroff_after_halt,
 268     "MAX_MONITOR_WAIT"          => \$max_monitor_wait,
 269     "SLEEP_TIME"                => \$sleep_time,
 270     "BISECT_SLEEP_TIME"         => \$bisect_sleep_time,
 271     "PATCHCHECK_SLEEP_TIME"     => \$patchcheck_sleep_time,
 272     "IGNORE_WARNINGS"           => \$ignore_warnings,
 273     "IGNORE_ERRORS"             => \$ignore_errors,
 274     "BISECT_MANUAL"             => \$bisect_manual,
 275     "BISECT_SKIP"               => \$bisect_skip,
 276     "CONFIG_BISECT_GOOD"        => \$config_bisect_good,
 277     "BISECT_RET_GOOD"           => \$bisect_ret_good,
 278     "BISECT_RET_BAD"            => \$bisect_ret_bad,
 279     "BISECT_RET_SKIP"           => \$bisect_ret_skip,
 280     "BISECT_RET_ABORT"          => \$bisect_ret_abort,
 281     "BISECT_RET_DEFAULT"        => \$bisect_ret_default,
 282     "STORE_FAILURES"            => \$store_failures,
 283     "STORE_SUCCESSES"           => \$store_successes,
 284     "TEST_NAME"                 => \$test_name,
 285     "TIMEOUT"                   => \$timeout,
 286     "BOOTED_TIMEOUT"            => \$booted_timeout,
 287     "CONSOLE"                   => \$console,
 288     "DETECT_TRIPLE_FAULT"       => \$detect_triplefault,
 289     "SUCCESS_LINE"              => \$success_line,
 290     "REBOOT_SUCCESS_LINE"       => \$reboot_success_line,
 291     "STOP_AFTER_SUCCESS"        => \$stop_after_success,
 292     "STOP_AFTER_FAILURE"        => \$stop_after_failure,
 293     "STOP_TEST_AFTER"           => \$stop_test_after,
 294     "BUILD_TARGET"              => \$build_target,
 295     "SSH_EXEC"                  => \$ssh_exec,
 296     "SCP_TO_TARGET"             => \$scp_to_target,
 297     "SCP_TO_TARGET_INSTALL"     => \$scp_to_target_install,
 298     "CHECKOUT"                  => \$checkout,
 299     "TARGET_IMAGE"              => \$target_image,
 300     "LOCALVERSION"              => \$localversion,
 301
 302     "BISECT_GOOD"               => \$bisect_good,
 303     "BISECT_BAD"                => \$bisect_bad,
 304     "BISECT_TYPE"               => \$bisect_type,
 305     "BISECT_START"              => \$bisect_start,
 306     "BISECT_REPLAY"             => \$bisect_replay,
 307     "BISECT_FILES"              => \$bisect_files,
 308     "BISECT_REVERSE"            => \$bisect_reverse,
 309     "BISECT_CHECK"              => \$bisect_check,
 310
 311     "CONFIG_BISECT"             => \$config_bisect,
 312     "CONFIG_BISECT_TYPE"        => \$config_bisect_type,
 313     "CONFIG_BISECT_CHECK"       => \$config_bisect_check,
 314
 315     "PATCHCHECK_TYPE"           => \$patchcheck_type,
 316     "PATCHCHECK_START"          => \$patchcheck_start,
 317     "PATCHCHECK_END"            => \$patchcheck_end,
 318 );
 319
 320 # Options may be used by other options, record them.
 321 my %used_options;
 322
 323 # default variables that can be used
 324 chomp ($variable{"PWD"} = `pwd`);
 325
 326 $config_help{"MACHINE"} = << "EOF"
 327  The machine hostname that you will test.
 328  For build only tests, it is still needed to differentiate log files.
 329 EOF
 330     ;
 331 $config_help{"SSH_USER"} = << "EOF"
 332  The box is expected to have ssh on normal bootup, provide the user
 333   (most likely root, since you need privileged operations)
 334 EOF
 335     ;
 336 $config_help{"BUILD_DIR"} = << "EOF"
 337  The directory that contains the Linux source code (full path).
 338  You can use \${PWD} that will be the path where ktest.pl is run, or use
 339  \${THIS_DIR} which is assigned \${PWD} but may be changed later.
 340 EOF
 341     ;
 342 $config_help{"OUTPUT_DIR"} = << "EOF"
 343  The directory that the objects will be built (full path).
 344  (can not be same as BUILD_DIR)
 345  You can use \${PWD} that will be the path where ktest.pl is run, or use
 346  \${THIS_DIR} which is assigned \${PWD} but may be changed later.
 347 EOF
 348     ;
 349 $config_help{"BUILD_TARGET"} = << "EOF"
 350  The location of the compiled file to copy to the target.
 351  (relative to OUTPUT_DIR)
 352 EOF
 353     ;
 354 $config_help{"BUILD_OPTIONS"} = << "EOF"
 355  Options to add to \"make\" when building.
 356  i.e.  -j20
 357 EOF
 358     ;
 359 $config_help{"TARGET_IMAGE"} = << "EOF"
 360  The place to put your image on the test machine.
 361 EOF
 362     ;
 363 $config_help{"POWER_CYCLE"} = << "EOF"
 364  A script or command to reboot the box.
 365
 366  Here is a digital loggers power switch example
 367  POWER_CYCLE = wget --no-proxy -O /dev/null -q  --auth-no-challenge 'http://admin:admin\@power/outlet?5=CCL'
 368
 369  Here is an example to reboot a virtual box on the current host
 370  with the name "Guest".
 371  POWER_CYCLE = virsh destroy Guest; sleep 5; virsh start Guest
 372 EOF
 373     ;
 374 $config_help{"CONSOLE"} = << "EOF"
 375  The script or command that reads the console
 376
 377   If you use ttywatch server, something like the following would work.
 378 CONSOLE = nc -d localhost 3001
 379
 380  For a virtual machine with guest name "Guest".
 381 CONSOLE =  virsh console Guest
 382 EOF
 383     ;
 384 $config_help{"LOCALVERSION"} = << "EOF"
 385  Required version ending to differentiate the test
 386  from other linux builds on the system.
 387 EOF
 388     ;
 389 $config_help{"REBOOT_TYPE"} = << "EOF"
 390  Way to reboot the box to the test kernel.
 391  Only valid options so far are "grub", "grub2", "syslinux", and "script".
 392
 393  If you specify grub, it will assume grub version 1
 394  and will search in /boot/grub/menu.lst for the title \$GRUB_MENU
 395  and select that target to reboot to the kernel. If this is not
 396  your setup, then specify "script" and have a command or script
 397  specified in REBOOT_SCRIPT to boot to the target.
 398
 399  The entry in /boot/grub/menu.lst must be entered in manually.
 400  The test will not modify that file.
 401
 402  If you specify grub2, then you also need to specify both \$GRUB_MENU
 403  and \$GRUB_FILE.
 404
 405  If you specify syslinux, then you may use SYSLINUX to define the syslinux
 406  command (defaults to extlinux), and SYSLINUX_PATH to specify the path to
 407  the syslinux install (defaults to /boot/extlinux). But you have to specify
 408  SYSLINUX_LABEL to define the label to boot to for the test kernel.
 409 EOF
 410     ;
 411 $config_help{"GRUB_MENU"} = << "EOF"
 412  The grub title name for the test kernel to boot
 413  (Only mandatory if REBOOT_TYPE = grub or grub2)
 414
 415  Note, ktest.pl will not update the grub menu.lst, you need to
 416  manually add an option for the test. ktest.pl will search
 417  the grub menu.lst for this option to find what kernel to
 418  reboot into.
 419
 420  For example, if in the /boot/grub/menu.lst the test kernel title has:
 421  title Test Kernel
 422  kernel vmlinuz-test
 423  GRUB_MENU = Test Kernel
 424
 425  For grub2, a search of \$GRUB_FILE is performed for the lines
 426  that begin with "menuentry". It will not detect submenus. The
 427  menu must be a non-nested menu. Add the quotes used in the menu
 428  to guarantee your selection, as the first menuentry with the content
 429  of \$GRUB_MENU that is found will be used.
 430 EOF
 431     ;
 432 $config_help{"GRUB_FILE"} = << "EOF"
 433  If grub2 is used, the full path for the grub.cfg file is placed
 434  here. Use something like /boot/grub2/grub.cfg to search.
 435 EOF
 436     ;
 437 $config_help{"SYSLINUX_LABEL"} = << "EOF"
 438  If syslinux is used, the label that boots the target kernel must
 439  be specified with SYSLINUX_LABEL.
 440 EOF
 441     ;
 442 $config_help{"REBOOT_SCRIPT"} = << "EOF"
 443  A script to reboot the target into the test kernel
 444  (Only mandatory if REBOOT_TYPE = script)
 445 EOF
 446     ;
 447
 448 sub read_prompt {
 449     my ($cancel, $prompt) = @_;
 450
 451     my $ans;
 452
 453     for (;;) {
 454         if ($cancel) {
 455             print "$prompt [y/n/C] ";
 456         } else {
 457             print "$prompt [Y/n] ";
 458         }
 459         $ans = <STDIN>;
 460         chomp $ans;
 461         if ($ans =~ /^\s*$/) {
 462             if ($cancel) {
 463                 $ans = "c";
 464             } else {
 465                 $ans = "y";
 466             }
 467         }
 468         last if ($ans =~ /^y$/i || $ans =~ /^n$/i);
 469         if ($cancel) {
 470             last if ($ans =~ /^c$/i);
 471             print "Please answer either 'y', 'n' or 'c'.\n";
 472         } else {
 473             print "Please answer either 'y' or 'n'.\n";
 474         }
 475     }
 476     if ($ans =~ /^c/i) {
 477         exit;
 478     }
 479     if ($ans !~ /^y$/i) {
 480         return 0;
 481     }
 482     return 1;
 483 }
 484
 485 sub read_yn {
 486     my ($prompt) = @_;
 487
 488     return read_prompt 0, $prompt;
 489 }
 490
 491 sub read_ync {
 492     my ($prompt) = @_;
 493
 494     return read_prompt 1, $prompt;
 495 }
 496
 497 sub get_ktest_config {
 498     my ($config) = @_;
 499     my $ans;
 500
 501     return if (defined($opt{$config}));
 502
 503     if (defined($config_help{$config})) {
 504         print "\n";
 505         print $config_help{$config};
 506     }
 507
 508     for (;;) {
 509         print "$config = ";
 510         if (defined($default{$config}) && length($default{$config})) {
 511             print "\[$default{$config}\] ";
 512         }
 513         $ans = <STDIN>;
 514         $ans =~ s/^\s*(.*\S)\s*$/$1/;
 515         if ($ans =~ /^\s*$/) {
 516             if ($default{$config}) {
 517                 $ans = $default{$config};
 518             } else {
 519                 print "Your answer can not be blank\n";
 520                 next;
 521             }
 522         }
 523         $entered_configs{$config} = ${ans};
 524         last;
 525     }
 526 }
 527
 528 sub get_ktest_configs {
 529     get_ktest_config("MACHINE");
 530     get_ktest_config("BUILD_DIR");
 531     get_ktest_config("OUTPUT_DIR");
 532
 533     if ($newconfig) {
 534         get_ktest_config("BUILD_OPTIONS");
 535     }
 536
 537     # options required for other than just building a kernel
 538     if (!$buildonly) {
 539         get_ktest_config("POWER_CYCLE");
 540         get_ktest_config("CONSOLE");
 541     }
 542
 543     # options required for install and more
 544     if ($buildonly != 1) {
 545         get_ktest_config("SSH_USER");
 546         get_ktest_config("BUILD_TARGET");
 547         get_ktest_config("TARGET_IMAGE");
 548     }
 549
 550     get_ktest_config("LOCALVERSION");
 551
 552     return if ($buildonly);
 553
 554     my $rtype = $opt{"REBOOT_TYPE"};
 555
 556     if (!defined($rtype)) {
 557         if (!defined($opt{"GRUB_MENU"})) {
 558             get_ktest_config("REBOOT_TYPE");
 559             $rtype = $entered_configs{"REBOOT_TYPE"};
 560         } else {
 561             $rtype = "grub";
 562         }
 563     }
 564
 565     if ($rtype eq "grub") {
 566         get_ktest_config("GRUB_MENU");
 567     }
 568
 569     if ($rtype eq "grub2") {
 570         get_ktest_config("GRUB_MENU");
 571         get_ktest_config("GRUB_FILE");
 572     }
 573
 574     if ($rtype eq "syslinux") {
 575         get_ktest_config("SYSLINUX_LABEL");
 576     }
 577 }
 578
 579 sub process_variables {
 580     my ($value, $remove_undef) = @_;
 581     my $retval = "";
 582
 583     # We want to check for '\', and it is just easier
 584     # to check the previous characet of '$' and not need
 585     # to worry if '$' is the first character. By adding
 586     # a space to $value, we can just check [^\\]\$ and
 587     # it will still work.
 588     $value = " $value";
 589
 590     while ($value =~ /(.*?[^\\])\$\{(.*?)\}(.*)/) {
 591         my $begin = $1;
 592         my $var = $2;
 593         my $end = $3;
 594         # append beginning of value to retval
 595         $retval = "$retval$begin";
 596         if (defined($variable{$var})) {
 597             $retval = "$retval$variable{$var}";
 598         } elsif (defined($remove_undef) && $remove_undef) {
 599             # for if statements, any variable that is not defined,
 600             # we simple convert to 0
 601             $retval = "${retval}0";
 602         } else {
 603             # put back the origin piece.
 604             $retval = "$retval\$\{$var\}";
 605             # This could be an option that is used later, save
 606             # it so we don't warn if this option is not one of
 607             # ktests options.
 608             $used_options{$var} = 1;
 609         }
 610         $value = $end;
 611     }
 612     $retval = "$retval$value";
 613
 614     # remove the space added in the beginning
 615     $retval =~ s/ //;
 616
 617     return "$retval"
 618 }
 619
 620 sub set_value {
 621     my ($lvalue, $rvalue, $override, $overrides, $name) = @_;
 622
 623     my $prvalue = process_variables($rvalue);
 624
 625     if ($buildonly && $lvalue =~ /^TEST_TYPE(\[.*\])?$/ && $prvalue ne "build") {
 626         # Note if a test is something other than build, then we
 627         # will need other manditory options.
 628         if ($prvalue ne "install") {
 629             # for bisect, we need to check BISECT_TYPE
 630             if ($prvalue ne "bisect") {
 631                 $buildonly = 0;
 632             }
 633         } else {
 634             # install still limits some manditory options.
 635             $buildonly = 2;
 636         }
 637     }
 638
 639     if ($buildonly && $lvalue =~ /^BISECT_TYPE(\[.*\])?$/ && $prvalue ne "build") {
 640         if ($prvalue ne "install") {
 641             $buildonly = 0;
 642         } else {
 643             # install still limits some manditory options.
 644             $buildonly = 2;
 645         }
 646     }
 647
 648     if (defined($opt{$lvalue})) {
 649         if (!$override || defined(${$overrides}{$lvalue})) {
 650             my $extra = "";
 651             if ($override) {
 652                 $extra = "In the same override section!\n";
 653             }
 654             die "$name: $.: Option $lvalue defined more than once!\n$extra";
 655         }
 656         ${$overrides}{$lvalue} = $prvalue;
 657     }
 658     if ($rvalue =~ /^\s*$/) {
 659         delete $opt{$lvalue};
 660     } else {
 661         $opt{$lvalue} = $prvalue;
 662     }
 663 }
 664
 665 sub set_variable {
 666     my ($lvalue, $rvalue) = @_;
 667
 668     if ($rvalue =~ /^\s*$/) {
 669         delete $variable{$lvalue};
 670     } else {
 671         $rvalue = process_variables($rvalue);
 672         $variable{$lvalue} = $rvalue;
 673     }
 674 }
 675
 676 sub process_compare {
 677     my ($lval, $cmp, $rval) = @_;
 678
 679     # remove whitespace
 680
 681     $lval =~ s/^\s*//;
 682     $lval =~ s/\s*$//;
 683
 684     $rval =~ s/^\s*//;
 685     $rval =~ s/\s*$//;
 686
 687     if ($cmp eq "==") {
 688         return $lval eq $rval;
 689     } elsif ($cmp eq "!=") {
 690         return $lval ne $rval;
 691     } elsif ($cmp eq "=~") {
 692         return $lval =~ m/$rval/;
 693     } elsif ($cmp eq "!~") {
 694         return $lval !~ m/$rval/;
 695     }
 696
 697     my $statement = "$lval $cmp $rval";
 698     my $ret = eval $statement;
 699
 700     # $@ stores error of eval
 701     if ($@) {
 702         return -1;
 703     }
 704
 705     return $ret;
 706 }
 707
 708 sub value_defined {
 709     my ($val) = @_;
 710
 711     return defined($variable{$2}) ||
 712         defined($opt{$2});
 713 }
 714
 715 my $d = 0;
 716 sub process_expression {
 717     my ($name, $val) = @_;
 718
 719     my $c = $d++;
 720
 721     while ($val =~ s/\(([^\(]*?)\)/\&\&\&\&VAL\&\&\&\&/) {
 722         my $express = $1;
 723
 724         if (process_expression($name, $express)) {
 725             $val =~ s/\&\&\&\&VAL\&\&\&\&/ 1 /;
 726         } else {
 727             $val =~ s/\&\&\&\&VAL\&\&\&\&/ 0 /;
 728         }
 729     }
 730
 731     $d--;
 732     my $OR = "\\|\\|";
 733     my $AND = "\\&\\&";
 734
 735     while ($val =~ s/^(.*?)($OR|$AND)//) {
 736         my $express = $1;
 737         my $op = $2;
 738
 739         if (process_expression($name, $express)) {
 740             if ($op eq "||") {
 741                 return 1;
 742             }
 743         } else {
 744             if ($op eq "&&") {
 745                 return 0;
 746             }
 747         }
 748     }
 749
 750     if ($val =~ /(.*)(==|\!=|>=|<=|>|<|=~|\!~)(.*)/) {
 751         my $ret = process_compare($1, $2, $3);
 752         if ($ret < 0) {
 753             die "$name: $.: Unable to process comparison\n";
 754         }
 755         return $ret;
 756     }
 757
 758     if ($val =~ /^\s*(NOT\s*)?DEFINED\s+(\S+)\s*$/) {
 759         if (defined $1) {
 760             return !value_defined($2);
 761         } else {
 762             return value_defined($2);
 763         }
 764     }
 765
 766     if ($val =~ /^\s*0\s*$/) {
 767         return 0;
 768     } elsif ($val =~ /^\s*\d+\s*$/) {
 769         return 1;
 770     }
 771
 772     die ("$name: $.: Undefined content $val in if statement\n");
 773 }
 774
 775 sub process_if {
 776     my ($name, $value) = @_;
 777
 778     # Convert variables and replace undefined ones with 0
 779     my $val = process_variables($value, 1);
 780     my $ret = process_expression $name, $val;
 781
 782     return $ret;
 783 }
 784
 785 sub __read_config {
 786     my ($config, $current_test_num) = @_;
 787
 788     my $in;
 789     open($in, $config) || die "can't read file $config";
 790
 791     my $name = $config;
 792     $name =~ s,.*/(.*),$1,;
 793
 794     my $test_num = $$current_test_num;
 795     my $default = 1;
 796     my $repeat = 1;
 797     my $num_tests_set = 0;
 798     my $skip = 0;
 799     my $rest;
 800     my $line;
 801     my $test_case = 0;
 802     my $if = 0;
 803     my $if_set = 0;
 804     my $override = 0;
 805
 806     my %overrides;
 807
 808     while (<$in>) {
 809
 810         # ignore blank lines and comments
 811         next if (/^\s*$/ || /\s*\#/);
 812
 813         if (/^\s*(TEST_START|DEFAULTS)\b(.*)/) {
 814
 815             my $type = $1;
 816             $rest = $2;
 817             $line = $2;
 818
 819             my $old_test_num;
 820             my $old_repeat;
 821             $override = 0;
 822
 823             if ($type eq "TEST_START") {
 824
 825                 if ($num_tests_set) {
 826                     die "$name: $.: Can not specify both NUM_TESTS and TEST_START\n";
 827                 }
 828
 829                 $old_test_num = $test_num;
 830                 $old_repeat = $repeat;
 831
 832                 $test_num += $repeat;
 833                 $default = 0;
 834                 $repeat = 1;
 835             } else {
 836                 $default = 1;
 837             }
 838
 839             # If SKIP is anywhere in the line, the command will be skipped
 840             if ($rest =~ s/\s+SKIP\b//) {
 841                 $skip = 1;
 842             } else {
 843                 $test_case = 1;
 844                 $skip = 0;
 845             }
 846
 847             if ($rest =~ s/\sELSE\b//) {
 848                 if (!$if) {
 849                     die "$name: $.: ELSE found with out matching IF section\n$_";
 850                 }
 851                 $if = 0;
 852
 853                 if ($if_set) {
 854                     $skip = 1;
 855                 } else {
 856                     $skip = 0;
 857                 }
 858             }
 859
 860             if ($rest =~ s/\sIF\s+(.*)//) {
 861                 if (process_if($name, $1)) {
 862                     $if_set = 1;
 863                 } else {
 864                     $skip = 1;
 865                 }
 866                 $if = 1;
 867             } else {
 868                 $if = 0;
 869                 $if_set = 0;
 870             }
 871
 872             if (!$skip) {
 873                 if ($type eq "TEST_START") {
 874                     if ($rest =~ s/\s+ITERATE\s+(\d+)//) {
 875                         $repeat = $1;
 876                         $repeat_tests{"$test_num"} = $repeat;
 877                     }
 878                 } elsif ($rest =~ s/\sOVERRIDE\b//) {
 879                     # DEFAULT only
 880                     $override = 1;
 881                     # Clear previous overrides
 882                     %overrides = ();
 883                 }
 884             }
 885
 886             if (!$skip && $rest !~ /^\s*$/) {
 887                 die "$name: $.: Gargbage found after $type\n$_";
 888             }
 889
 890             if ($skip && $type eq "TEST_START") {
 891                 $test_num = $old_test_num;
 892                 $repeat = $old_repeat;
 893             }
 894
 895         } elsif (/^\s*ELSE\b(.*)$/) {
 896             if (!$if) {
 897                 die "$name: $.: ELSE found with out matching IF section\n$_";
 898             }
 899             $rest = $1;
 900             if ($if_set) {
 901                 $skip = 1;
 902                 $rest = "";
 903             } else {
 904                 $skip = 0;
 905
 906                 if ($rest =~ /\sIF\s+(.*)/) {
 907                     # May be a ELSE IF section.
 908                     if (process_if($name, $1)) {
 909                         $if_set = 1;
 910                     } else {
 911                         $skip = 1;
 912                     }
 913                     $rest = "";
 914                 } else {
 915                     $if = 0;
 916                 }
 917             }
 918
 919             if ($rest !~ /^\s*$/) {
 920                 die "$name: $.: Gargbage found after DEFAULTS\n$_";
 921             }
 922
 923         } elsif (/^\s*INCLUDE\s+(\S+)/) {
 924
 925             next if ($skip);
 926
 927             if (!$default) {
 928                 die "$name: $.: INCLUDE can only be done in default sections\n$_";
 929             }
 930
 931             my $file = process_variables($1);
 932
 933             if ($file !~ m,^/,) {
 934                 # check the path of the config file first
 935                 if ($config =~ m,(.*)/,) {
 936                     if (-f "$1/$file") {
 937                         $file = "$1/$file";
 938                     }
 939                 }
 940             }
 941
 942             if ( ! -r $file ) {
 943                 die "$name: $.: Can't read file $file\n$_";
 944             }
 945
 946             if (__read_config($file, \$test_num)) {
 947                 $test_case = 1;
 948             }
 949
 950         } elsif (/^\s*([A-Z_\[\]\d]+)\s*=\s*(.*?)\s*$/) {
 951
 952             next if ($skip);
 953
 954             my $lvalue = $1;
 955             my $rvalue = $2;
 956
 957             if (!$default &&
 958                 ($lvalue eq "NUM_TESTS" ||
 959                  $lvalue eq "LOG_FILE" ||
 960                  $lvalue eq "CLEAR_LOG")) {
 961                 die "$name: $.: $lvalue must be set in DEFAULTS section\n";
 962             }
 963
 964             if ($lvalue eq "NUM_TESTS") {
 965                 if ($test_num) {
 966                     die "$name: $.: Can not specify both NUM_TESTS and TEST_START\n";
 967                 }
 968                 if (!$default) {
 969                     die "$name: $.: NUM_TESTS must be set in default section\n";
 970                 }
 971                 $num_tests_set = 1;
 972             }
 973
 974             if ($default || $lvalue =~ /\[\d+\]$/) {
 975                 set_value($lvalue, $rvalue, $override, \%overrides, $name);
 976             } else {
 977                 my $val = "$lvalue\[$test_num\]";
 978                 set_value($val, $rvalue, $override, \%overrides, $name);
 979
 980                 if ($repeat > 1) {
 981                     $repeats{$val} = $repeat;
 982                 }
 983             }
 984         } elsif (/^\s*([A-Z_\[\]\d]+)\s*:=\s*(.*?)\s*$/) {
 985             next if ($skip);
 986
 987             my $lvalue = $1;
 988             my $rvalue = $2;
 989
 990             # process config variables.
 991             # Config variables are only active while reading the
 992             # config and can be defined anywhere. They also ignore
 993             # TEST_START and DEFAULTS, but are skipped if they are in
 994             # on of these sections that have SKIP defined.
 995             # The save variable can be
 996             # defined multiple times and the new one simply overrides
 997             # the prevous one.
 998             set_variable($lvalue, $rvalue);
 999
1000         } else {
1001             die "$name: $.: Garbage found in config\n$_";
1002         }
1003     }
1004
1005     if ($test_num) {
1006         $test_num += $repeat - 1;
1007         $opt{"NUM_TESTS"} = $test_num;
1008     }
1009
1010     close($in);
1011
1012     $$current_test_num = $test_num;
1013
1014     return $test_case;
1015 }
1016
1017 sub get_test_case {
1018         print "What test case would you like to run?\n";
1019         print " (build, install or boot)\n";
1020         print " Other tests are available but require editing the config file\n";
1021         my $ans = <STDIN>;
1022         chomp $ans;
1023         $default{"TEST_TYPE"} = $ans;
1024 }
1025
1026 sub read_config {
1027     my ($config) = @_;
1028
1029     my $test_case;
1030     my $test_num = 0;
1031
1032     $test_case = __read_config $config, \$test_num;
1033
1034     # make sure we have all mandatory configs
1035     get_ktest_configs;
1036
1037     # was a test specified?
1038     if (!$test_case) {
1039         print "No test case specified.\n";
1040         get_test_case;
1041     }
1042
1043     # set any defaults
1044
1045     foreach my $default (keys %default) {
1046         if (!defined($opt{$default})) {
1047             $opt{$default} = $default{$default};
1048         }
1049     }
1050
1051     if ($opt{"IGNORE_UNUSED"} == 1) {
1052         return;
1053     }
1054
1055     my %not_used;
1056
1057     # check if there are any stragglers (typos?)
1058     foreach my $option (keys %opt) {
1059         my $op = $option;
1060         # remove per test labels.
1061         $op =~ s/\[.*\]//;
1062         if (!exists($option_map{$op}) &&
1063             !exists($default{$op}) &&
1064             !exists($used_options{$op})) {
1065             $not_used{$op} = 1;
1066         }
1067     }
1068
1069     if (%not_used) {
1070         my $s = "s are";
1071         $s = " is" if (keys %not_used == 1);
1072         print "The following option$s not used; could be a typo:\n";
1073         foreach my $option (keys %not_used) {
1074             print "$option\n";
1075         }
1076         print "Set IGRNORE_UNUSED = 1 to have ktest ignore unused variables\n";
1077         if (!read_yn "Do you want to continue?") {
1078             exit -1;
1079         }
1080     }
1081 }
1082
1083 sub __eval_option {
1084     my ($name, $option, $i) = @_;
1085
1086     # Add space to evaluate the character before $
1087     $option = " $option";
1088     my $retval = "";
1089     my $repeated = 0;
1090     my $parent = 0;
1091
1092     foreach my $test (keys %repeat_tests) {
1093         if ($i >= $test &&
1094             $i < $test + $repeat_tests{$test}) {
1095
1096             $repeated = 1;
1097             $parent = $test;
1098             last;
1099         }
1100     }
1101
1102     while ($option =~ /(.*?[^\\])\$\{(.*?)\}(.*)/) {
1103         my $start = $1;
1104         my $var = $2;
1105         my $end = $3;
1106
1107         # Append beginning of line
1108         $retval = "$retval$start";
1109
1110         # If the iteration option OPT[$i] exists, then use that.
1111         # otherwise see if the default OPT (without [$i]) exists.
1112
1113         my $o = "$var\[$i\]";
1114         my $parento = "$var\[$parent\]";
1115
1116         # If a variable contains itself, use the default var
1117         if (($var eq $name) && defined($opt{$var})) {
1118             $o = $opt{$var};
1119             $retval = "$retval$o";
1120         } elsif (defined($opt{$o})) {
1121             $o = $opt{$o};
1122             $retval = "$retval$o";
1123         } elsif ($repeated && defined($opt{$parento})) {
1124             $o = $opt{$parento};
1125             $retval = "$retval$o";
1126         } elsif (defined($opt{$var})) {
1127             $o = $opt{$var};
1128             $retval = "$retval$o";
1129         } else {
1130             $retval = "$retval\$\{$var\}";
1131         }
1132
1133         $option = $end;
1134     }
1135
1136     $retval = "$retval$option";
1137
1138     $retval =~ s/^ //;
1139
1140     return $retval;
1141 }
1142
1143 sub eval_option {
1144     my ($name, $option, $i) = @_;
1145
1146     my $prev = "";
1147
1148     # Since an option can evaluate to another option,
1149     # keep iterating until we do not evaluate any more
1150     # options.
1151     my $r = 0;
1152     while ($prev ne $option) {
1153         # Check for recursive evaluations.
1154         # 100 deep should be more than enough.
1155         if ($r++ > 100) {
1156             die "Over 100 evaluations accurred with $option\n" .
1157                 "Check for recursive variables\n";
1158         }
1159         $prev = $option;
1160         $option = __eval_option($name, $option, $i);
1161     }
1162
1163     return $option;
1164 }
1165
1166 sub _logit {
1167     if (defined($opt{"LOG_FILE"})) {
1168         open(OUT, ">> $opt{LOG_FILE}") or die "Can't write to $opt{LOG_FILE}";
1169         print OUT @_;
1170         close(OUT);
1171     }
1172 }
1173
1174 sub logit {
1175     if (defined($opt{"LOG_FILE"})) {
1176         _logit @_;
1177     } else {
1178         print @_;
1179     }
1180 }
1181
1182 sub doprint {
1183     print @_;
1184     _logit @_;
1185 }
1186
1187 sub run_command;
1188 sub start_monitor;
1189 sub end_monitor;
1190 sub wait_for_monitor;
1191
1192 sub reboot {
1193     my ($time) = @_;
1194
1195     # Make sure everything has been written to disk
1196     run_ssh("sync");
1197
1198     if (defined($time)) {
1199         start_monitor;
1200         # flush out current monitor
1201         # May contain the reboot success line
1202         wait_for_monitor 1;
1203     }
1204
1205     # try to reboot normally
1206     if (run_command $reboot) {
1207         if (defined($powercycle_after_reboot)) {
1208             sleep $powercycle_after_reboot;
1209             run_command "$power_cycle";
1210         }
1211     } else {
1212         # nope? power cycle it.
1213         run_command "$power_cycle";
1214     }
1215
1216     if (defined($time)) {
1217
1218         # We only want to get to the new kernel, don't fail
1219         # if we stumble over a call trace.
1220         my $save_ignore_errors = $ignore_errors;
1221         $ignore_errors = 1;
1222
1223         # Look for the good kernel to boot
1224         if (wait_for_monitor($time, "Linux version")) {
1225             # reboot got stuck?
1226             doprint "Reboot did not finish. Forcing power cycle\n";
1227             run_command "$power_cycle";
1228         }
1229
1230         $ignore_errors = $save_ignore_errors;
1231
1232         # Still need to wait for the reboot to finish
1233         wait_for_monitor($time, $reboot_success_line);
1234
1235         end_monitor;
1236     }
1237 }
1238
1239 sub reboot_to_good {
1240     my ($time) = @_;
1241
1242     if (defined($switch_to_good)) {
1243         run_command $switch_to_good;
1244     }
1245
1246     reboot $time;
1247 }
1248
1249 sub do_not_reboot {
1250     my $i = $iteration;
1251
1252     return $test_type eq "build" || $no_reboot ||
1253         ($test_type eq "patchcheck" && $opt{"PATCHCHECK_TYPE[$i]"} eq "build") ||
1254         ($test_type eq "bisect" && $opt{"BISECT_TYPE[$i]"} eq "build");
1255 }
1256
1257 sub dodie {
1258     doprint "CRITICAL FAILURE... ", @_, "\n";
1259
1260     my $i = $iteration;
1261
1262     if ($reboot_on_error && !do_not_reboot) {
1263
1264         doprint "REBOOTING\n";
1265         reboot_to_good;
1266
1267     } elsif ($poweroff_on_error && defined($power_off)) {
1268         doprint "POWERING OFF\n";
1269         `$power_off`;
1270     }
1271
1272     if (defined($opt{"LOG_FILE"})) {
1273         print " See $opt{LOG_FILE} for more info.\n";
1274     }
1275
1276     die @_, "\n";
1277 }
1278
1279 sub open_console {
1280     my ($fp) = @_;
1281
1282     my $flags;
1283
1284     my $pid = open($fp, "$console|") or
1285         dodie "Can't open console $console";
1286
1287     $flags = fcntl($fp, F_GETFL, 0) or
1288         dodie "Can't get flags for the socket: $!";
1289     $flags = fcntl($fp, F_SETFL, $flags | O_NONBLOCK) or
1290         dodie "Can't set flags for the socket: $!";
1291
1292     return $pid;
1293 }
1294
1295 sub close_console {
1296     my ($fp, $pid) = @_;
1297
1298     doprint "kill child process $pid\n";
1299     kill 2, $pid;
1300
1301     print "closing!\n";
1302     close($fp);
1303 }
1304
1305 sub start_monitor {
1306     if ($monitor_cnt++) {
1307         return;
1308     }
1309     $monitor_fp = \*MONFD;
1310     $monitor_pid = open_console $monitor_fp;
1311
1312     return;
1313
1314     open(MONFD, "Stop perl from warning about single use of MONFD");
1315 }
1316
1317 sub end_monitor {
1318     return if (!defined $console);
1319     if (--$monitor_cnt) {
1320         return;
1321     }
1322     close_console($monitor_fp, $monitor_pid);
1323 }
1324
1325 sub wait_for_monitor {
1326     my ($time, $stop) = @_;
1327     my $full_line = "";
1328     my $line;
1329     my $booted = 0;
1330     my $start_time = time;
1331     my $skip_call_trace = 0;
1332     my $bug = 0;
1333     my $bug_ignored = 0;
1334     my $now;
1335
1336     doprint "** Wait for monitor to settle down **\n";
1337
1338     # read the monitor and wait for the system to calm down
1339     while (!$booted) {
1340         $line = wait_for_input($monitor_fp, $time);
1341         last if (!defined($line));
1342         print "$line";
1343         $full_line .= $line;
1344
1345         if (defined($stop) && $full_line =~ /$stop/) {
1346             doprint "wait for monitor detected $stop\n";
1347             $booted = 1;
1348         }
1349
1350         if ($full_line =~ /\[ backtrace testing \]/) {
1351             $skip_call_trace = 1;
1352         }
1353
1354         if ($full_line =~ /call trace:/i) {
1355             if (!$bug && !$skip_call_trace) {
1356                 if ($ignore_errors) {
1357                     $bug_ignored = 1;
1358                 } else {
1359                     $bug = 1;
1360                 }
1361             }
1362         }
1363
1364         if ($full_line =~ /\[ end of backtrace testing \]/) {
1365             $skip_call_trace = 0;
1366         }
1367
1368         if ($full_line =~ /Kernel panic -/) {
1369             $bug = 1;
1370         }
1371
1372         if ($line =~ /\n/) {
1373             $full_line = "";
1374         }
1375         $now = time;
1376         if ($now - $start_time >= $max_monitor_wait) {
1377             doprint "Exiting monitor flush due to hitting MAX_MONITOR_WAIT\n";
1378             return 1;
1379         }
1380     }
1381     print "** Monitor flushed **\n";
1382     return $bug;
1383 }
1384
1385 sub save_logs {
1386         my ($result, $basedir) = @_;
1387         my @t = localtime;
1388         my $date = sprintf "%04d%02d%02d%02d%02d%02d",
1389                 1900+$t[5],$t[4],$t[3],$t[2],$t[1],$t[0];
1390
1391         my $type = $build_type;
1392         if ($type =~ /useconfig/) {
1393             $type = "useconfig";
1394         }
1395
1396         my $dir = "$machine-$test_type-$type-$result-$date";
1397
1398         $dir = "$basedir/$dir";
1399
1400         if (!-d $dir) {
1401             mkpath($dir) or
1402                 die "can't create $dir";
1403         }
1404
1405         my %files = (
1406                 "config" => $output_config,
1407                 "buildlog" => $buildlog,
1408                 "dmesg" => $dmesg,
1409                 "testlog" => $testlog,
1410         );
1411
1412         while (my ($name, $source) = each(%files)) {
1413                 if (-f "$source") {
1414                         cp "$source", "$dir/$name" or
1415                                 die "failed to copy $source";
1416                 }
1417         }
1418
1419         doprint "*** Saved info to $dir ***\n";
1420 }
1421
1422 sub fail {
1423
1424         if (defined($post_test)) {
1425                 run_command $post_test;
1426         }
1427
1428         if ($die_on_failure) {
1429                 dodie @_;
1430         }
1431
1432         doprint "FAILED\n";
1433
1434         my $i = $iteration;
1435
1436         # no need to reboot for just building.
1437         if (!do_not_reboot) {
1438             doprint "REBOOTING\n";
1439             reboot_to_good $sleep_time;
1440         }
1441
1442         my $name = "";
1443
1444         if (defined($test_name)) {
1445             $name = " ($test_name)";
1446         }
1447
1448         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
1449         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
1450         doprint "KTEST RESULT: TEST $i$name Failed: ", @_, "\n";
1451         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
1452         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
1453
1454         if (defined($store_failures)) {
1455             save_logs "fail", $store_failures;
1456         }
1457
1458         return 1;
1459 }
1460
1461 sub run_command {
1462     my ($command) = @_;
1463     my $dolog = 0;
1464     my $dord = 0;
1465     my $pid;
1466
1467     $command =~ s/\$SSH_USER/$ssh_user/g;
1468     $command =~ s/\$MACHINE/$machine/g;
1469
1470     doprint("$command ... ");
1471
1472     $pid = open(CMD, "$command 2>&1 |") or
1473         (fail "unable to exec $command" and return 0);
1474
1475     if (defined($opt{"LOG_FILE"})) {
1476         open(LOG, ">>$opt{LOG_FILE}") or
1477             dodie "failed to write to log";
1478         $dolog = 1;
1479     }
1480
1481     if (defined($redirect)) {
1482         open (RD, ">$redirect") or
1483             dodie "failed to write to redirect $redirect";
1484         $dord = 1;
1485     }
1486
1487     while (<CMD>) {
1488         print LOG if ($dolog);
1489         print RD  if ($dord);
1490     }
1491
1492     waitpid($pid, 0);
1493     my $failed = $?;
1494
1495     close(CMD);
1496     close(LOG) if ($dolog);
1497     close(RD)  if ($dord);
1498
1499     if ($failed) {
1500         doprint "FAILED!\n";
1501     } else {
1502         doprint "SUCCESS\n";
1503     }
1504
1505     return !$failed;
1506 }
1507
1508 sub run_ssh {
1509     my ($cmd) = @_;
1510     my $cp_exec = $ssh_exec;
1511
1512     $cp_exec =~ s/\$SSH_COMMAND/$cmd/g;
1513     return run_command "$cp_exec";
1514 }
1515
1516 sub run_scp {
1517     my ($src, $dst, $cp_scp) = @_;
1518
1519     $cp_scp =~ s/\$SRC_FILE/$src/g;
1520     $cp_scp =~ s/\$DST_FILE/$dst/g;
1521
1522     return run_command "$cp_scp";
1523 }
1524
1525 sub run_scp_install {
1526     my ($src, $dst) = @_;
1527
1528     my $cp_scp = $scp_to_target_install;
1529
1530     return run_scp($src, $dst, $cp_scp);
1531 }
1532
1533 sub run_scp_mod {
1534     my ($src, $dst) = @_;
1535
1536     my $cp_scp = $scp_to_target;
1537
1538     return run_scp($src, $dst, $cp_scp);
1539 }
1540
1541 sub get_grub2_index {
1542
1543     return if (defined($grub_number) && defined($last_grub_menu) &&
1544                $last_grub_menu eq $grub_menu && defined($last_machine) &&
1545                $last_machine eq $machine);
1546
1547     doprint "Find grub2 menu ... ";
1548     $grub_number = -1;
1549
1550     my $ssh_grub = $ssh_exec;
1551     $ssh_grub =~ s,\$SSH_COMMAND,cat $grub_file,g;
1552
1553     open(IN, "$ssh_grub |")
1554         or die "unable to get $grub_file";
1555
1556     my $found = 0;
1557
1558     while (<IN>) {
1559         if (/^menuentry.*$grub_menu/) {
1560             $grub_number++;
1561             $found = 1;
1562             last;
1563         } elsif (/^menuentry\s/) {
1564             $grub_number++;
1565         }
1566     }
1567     close(IN);
1568
1569     die "Could not find '$grub_menu' in $grub_file on $machine"
1570         if (!$found);
1571     doprint "$grub_number\n";
1572     $last_grub_menu = $grub_menu;
1573     $last_machine = $machine;
1574 }
1575
1576 sub get_grub_index {
1577
1578     if ($reboot_type eq "grub2") {
1579         get_grub2_index;
1580         return;
1581     }
1582
1583     if ($reboot_type ne "grub") {
1584         return;
1585     }
1586     return if (defined($grub_number) && defined($last_grub_menu) &&
1587                $last_grub_menu eq $grub_menu && defined($last_machine) &&
1588                $last_machine eq $machine);
1589
1590     doprint "Find grub menu ... ";
1591     $grub_number = -1;
1592
1593     my $ssh_grub = $ssh_exec;
1594     $ssh_grub =~ s,\$SSH_COMMAND,cat /boot/grub/menu.lst,g;
1595
1596     open(IN, "$ssh_grub |")
1597         or die "unable to get menu.lst";
1598
1599     my $found = 0;
1600
1601     while (<IN>) {
1602         if (/^\s*title\s+$grub_menu\s*$/) {
1603             $grub_number++;
1604             $found = 1;
1605             last;
1606         } elsif (/^\s*title\s/) {
1607             $grub_number++;
1608         }
1609     }
1610     close(IN);
1611
1612     die "Could not find '$grub_menu' in /boot/grub/menu on $machine"
1613         if (!$found);
1614     doprint "$grub_number\n";
1615     $last_grub_menu = $grub_menu;
1616     $last_machine = $machine;
1617 }
1618
1619 sub wait_for_input
1620 {
1621     my ($fp, $time) = @_;
1622     my $rin;
1623     my $ready;
1624     my $line;
1625     my $ch;
1626
1627     if (!defined($time)) {
1628         $time = $timeout;
1629     }
1630
1631     $rin = '';
1632     vec($rin, fileno($fp), 1) = 1;
1633     ($ready, $time) = select($rin, undef, undef, $time);
1634
1635     $line = "";
1636
1637     # try to read one char at a time
1638     while (sysread $fp, $ch, 1) {
1639         $line .= $ch;
1640         last if ($ch eq "\n");
1641     }
1642
1643     if (!length($line)) {
1644         return undef;
1645     }
1646
1647     return $line;
1648 }
1649
1650 sub reboot_to {
1651     if (defined($switch_to_test)) {
1652         run_command $switch_to_test;
1653     }
1654
1655     if ($reboot_type eq "grub") {
1656         run_ssh "'(echo \"savedefault --default=$grub_number --once\" | grub --batch)'";
1657     } elsif ($reboot_type eq "grub2") {
1658         run_ssh "$grub_reboot $grub_number";
1659     } elsif ($reboot_type eq "syslinux") {
1660         run_ssh "$syslinux --once \\\"$syslinux_label\\\" $syslinux_path";
1661     } elsif (defined $reboot_script) {
1662         run_command "$reboot_script";
1663     }
1664     reboot;
1665 }
1666
1667 sub get_sha1 {
1668     my ($commit) = @_;
1669
1670     doprint "git rev-list --max-count=1 $commit ... ";
1671     my $sha1 = `git rev-list --max-count=1 $commit`;
1672     my $ret = $?;
1673
1674     logit $sha1;
1675
1676     if ($ret) {
1677         doprint "FAILED\n";
1678         dodie "Failed to get git $commit";
1679     }
1680
1681     print "SUCCESS\n";
1682
1683     chomp $sha1;
1684
1685     return $sha1;
1686 }
1687
1688 sub monitor {
1689     my $booted = 0;
1690     my $bug = 0;
1691     my $bug_ignored = 0;
1692     my $skip_call_trace = 0;
1693     my $loops;
1694
1695     wait_for_monitor 5;
1696
1697     my $line;
1698     my $full_line = "";
1699
1700     open(DMESG, "> $dmesg") or
1701         die "unable to write to $dmesg";
1702
1703     reboot_to;
1704
1705     my $success_start;
1706     my $failure_start;
1707     my $monitor_start = time;
1708     my $done = 0;
1709     my $version_found = 0;
1710
1711     while (!$done) {
1712
1713         if ($bug && defined($stop_after_failure) &&
1714             $stop_after_failure >= 0) {
1715             my $time = $stop_after_failure - (time - $failure_start);
1716             $line = wait_for_input($monitor_fp, $time);
1717             if (!defined($line)) {
1718                 doprint "bug timed out after $booted_timeout seconds\n";
1719                 doprint "Test forced to stop after $stop_after_failure seconds after failure\n";
1720                 last;
1721             }
1722         } elsif ($booted) {
1723             $line = wait_for_input($monitor_fp, $booted_timeout);
1724             if (!defined($line)) {
1725                 my $s = $booted_timeout == 1 ? "" : "s";
1726                 doprint "Successful boot found: break after $booted_timeout second$s\n";
1727                 last;
1728             }
1729         } else {
1730             $line = wait_for_input($monitor_fp);
1731             if (!defined($line)) {
1732                 my $s = $timeout == 1 ? "" : "s";
1733                 doprint "Timed out after $timeout second$s\n";
1734                 last;
1735             }
1736         }
1737
1738         doprint $line;
1739         print DMESG $line;
1740
1741         # we are not guaranteed to get a full line
1742         $full_line .= $line;
1743
1744         if ($full_line =~ /$success_line/) {
1745             $booted = 1;
1746             $success_start = time;
1747         }
1748
1749         if ($booted && defined($stop_after_success) &&
1750             $stop_after_success >= 0) {
1751             my $now = time;
1752             if ($now - $success_start >= $stop_after_success) {
1753                 doprint "Test forced to stop after $stop_after_success seconds after success\n";
1754                 last;
1755             }
1756         }
1757
1758         if ($full_line =~ /\[ backtrace testing \]/) {
1759             $skip_call_trace = 1;
1760         }
1761
1762         if ($full_line =~ /call trace:/i) {
1763             if (!$bug && !$skip_call_trace) {
1764                 if ($ignore_errors) {
1765                     $bug_ignored = 1;
1766                 } else {
1767                     $bug = 1;
1768                     $failure_start = time;
1769                 }
1770             }
1771         }
1772
1773         if ($bug && defined($stop_after_failure) &&
1774             $stop_after_failure >= 0) {
1775             my $now = time;
1776             if ($now - $failure_start >= $stop_after_failure) {
1777                 doprint "Test forced to stop after $stop_after_failure seconds after failure\n";
1778                 last;
1779             }
1780         }
1781
1782         if ($full_line =~ /\[ end of backtrace testing \]/) {
1783             $skip_call_trace = 0;
1784         }
1785
1786         if ($full_line =~ /Kernel panic -/) {
1787             $failure_start = time;
1788             $bug = 1;
1789         }
1790
1791         # Detect triple faults by testing the banner
1792         if ($full_line =~ /\bLinux version (\S+).*\n/) {
1793             if ($1 eq $version) {
1794                 $version_found = 1;
1795             } elsif ($version_found && $detect_triplefault) {
1796                 # We already booted into the kernel we are testing,
1797                 # but now we booted into another kernel?
1798                 # Consider this a triple fault.
1799                 doprint "Already booted in Linux kernel $version, but now\n";
1800                 doprint "we booted into Linux kernel $1.\n";
1801                 doprint "Assuming that this is a triple fault.\n";
1802                 doprint "To disable this: set DETECT_TRIPLE_FAULT to 0\n";
1803                 last;
1804             }
1805         }
1806
1807         if ($line =~ /\n/) {
1808             $full_line = "";
1809         }
1810
1811         if ($stop_test_after > 0 && !$booted && !$bug) {
1812             if (time - $monitor_start > $stop_test_after) {
1813                 doprint "STOP_TEST_AFTER ($stop_test_after seconds) timed out\n";
1814                 $done = 1;
1815             }
1816         }
1817     }
1818
1819     close(DMESG);
1820
1821     if ($bug) {
1822         return 0 if ($in_bisect);
1823         fail "failed - got a bug report" and return 0;
1824     }
1825
1826     if (!$booted) {
1827         return 0 if ($in_bisect);
1828         fail "failed - never got a boot prompt." and return 0;
1829     }
1830
1831     if ($bug_ignored) {
1832         doprint "WARNING: Call Trace detected but ignored due to IGNORE_ERRORS=1\n";
1833     }
1834
1835     return 1;
1836 }
1837
1838 sub eval_kernel_version {
1839     my ($option) = @_;
1840
1841     $option =~ s/\$KERNEL_VERSION/$version/g;
1842
1843     return $option;
1844 }
1845
1846 sub do_post_install {
1847
1848     return if (!defined($post_install));
1849
1850     my $cp_post_install = eval_kernel_version $post_install;
1851     run_command "$cp_post_install" or
1852         dodie "Failed to run post install";
1853 }
1854
1855 # Sometimes the reboot fails, and will hang. We try to ssh to the box
1856 # and if we fail, we force another reboot, that should powercycle it.
1857 sub test_booted {
1858     if (!run_ssh "echo testing connection") {
1859         reboot $sleep_time;
1860     }
1861 }
1862
1863 sub install {
1864
1865     return if ($no_install);
1866
1867     if (defined($pre_install)) {
1868         my $cp_pre_install = eval_kernel_version $pre_install;
1869         run_command "$cp_pre_install" or
1870             dodie "Failed to run pre install";
1871     }
1872
1873     my $cp_target = eval_kernel_version $target_image;
1874
1875     test_booted;
1876
1877     run_scp_install "$outputdir/$build_target", "$cp_target" or
1878         dodie "failed to copy image";
1879
1880     my $install_mods = 0;
1881
1882     # should we process modules?
1883     $install_mods = 0;
1884     open(IN, "$output_config") or dodie("Can't read config file");
1885     while (<IN>) {
1886         if (/CONFIG_MODULES(=y)?/) {
1887             if (defined($1)) {
1888                 $install_mods = 1;
1889                 last;
1890             }
1891         }
1892     }
1893     close(IN);
1894
1895     if (!$install_mods) {
1896         do_post_install;
1897         doprint "No modules needed\n";
1898         return;
1899     }
1900
1901     run_command "$make INSTALL_MOD_STRIP=1 INSTALL_MOD_PATH=$tmpdir modules_install" or
1902         dodie "Failed to install modules";
1903
1904     my $modlib = "/lib/modules/$version";
1905     my $modtar = "ktest-mods.tar.bz2";
1906
1907     run_ssh "rm -rf $modlib" or
1908         dodie "failed to remove old mods: $modlib";
1909
1910     # would be nice if scp -r did not follow symbolic links
1911     run_command "cd $tmpdir && tar -cjf $modtar lib/modules/$version" or
1912         dodie "making tarball";
1913
1914     run_scp_mod "$tmpdir/$modtar", "/tmp" or
1915         dodie "failed to copy modules";
1916
1917     unlink "$tmpdir/$modtar";
1918
1919     run_ssh "'(cd / && tar xjf /tmp/$modtar)'" or
1920         dodie "failed to tar modules";
1921
1922     run_ssh "rm -f /tmp/$modtar";
1923
1924     do_post_install;
1925 }
1926
1927 sub get_version {
1928     # get the release name
1929     return if ($have_version);
1930     doprint "$make kernelrelease ... ";
1931     $version = `$make kernelrelease | tail -1`;
1932     chomp($version);
1933     doprint "$version\n";
1934     $have_version = 1;
1935 }
1936
1937 sub start_monitor_and_boot {
1938     # Make sure the stable kernel has finished booting
1939
1940     # Install bisects, don't need console
1941     if (defined $console) {
1942         start_monitor;
1943         wait_for_monitor 5;
1944         end_monitor;
1945     }
1946
1947     get_grub_index;
1948     get_version;
1949     install;
1950
1951     start_monitor if (defined $console);
1952     return monitor;
1953 }
1954
1955 my $check_build_re = ".*:.*(warning|error|Error):.*";
1956 my $utf8_quote = "\\x{e2}\\x{80}(\\x{98}|\\x{99})";
1957
1958 sub process_warning_line {
1959     my ($line) = @_;
1960
1961     chomp $line;
1962
1963     # for distcc heterogeneous systems, some compilers
1964     # do things differently causing warning lines
1965     # to be slightly different. This makes an attempt
1966     # to fixe those issues.
1967
1968     # chop off the index into the line
1969     # using distcc, some compilers give different indexes
1970     # depending on white space
1971     $line =~ s/^(\s*\S+:\d+:)\d+/$1/;
1972
1973     # Some compilers use UTF-8 extended for quotes and some don't.
1974     $line =~ s/$utf8_quote/'/g;
1975
1976     return $line;
1977 }
1978
1979 # Read buildlog and check against warnings file for any
1980 # new warnings.
1981 #
1982 # Returns 1 if OK
1983 #         0 otherwise
1984 sub check_buildlog {
1985     return 1 if (!defined $warnings_file);
1986
1987     my %warnings_list;
1988
1989     # Failed builds should not reboot the target
1990     my $save_no_reboot = $no_reboot;
1991     $no_reboot = 1;
1992
1993     if (-f $warnings_file) {
1994         open(IN, $warnings_file) or
1995             dodie "Error opening $warnings_file";
1996
1997         while (<IN>) {
1998             if (/$check_build_re/) {
1999                 my $warning = process_warning_line $_;
2000
2001                 $warnings_list{$warning} = 1;
2002             }
2003         }
2004         close(IN);
2005     }
2006
2007     # If warnings file didn't exist, and WARNINGS_FILE exist,
2008     # then we fail on any warning!
2009
2010     open(IN, $buildlog) or dodie "Can't open $buildlog";
2011     while (<IN>) {
2012         if (/$check_build_re/) {
2013             my $warning = process_warning_line $_;
2014
2015             if (!defined $warnings_list{$warning}) {
2016                 fail "New warning found (not in $warnings_file)\n$_\n";
2017                 $no_reboot = $save_no_reboot;
2018                 return 0;
2019             }
2020         }
2021     }
2022     $no_reboot = $save_no_reboot;
2023     close(IN);
2024 }
2025
2026 sub check_patch_buildlog {
2027     my ($patch) = @_;
2028
2029     my @files = `git show $patch | diffstat -l`;
2030
2031     foreach my $file (@files) {
2032         chomp $file;
2033     }
2034
2035     open(IN, "git show $patch |") or
2036         dodie "failed to show $patch";
2037     while (<IN>) {
2038         if (m,^--- a/(.*),) {
2039             chomp $1;
2040             $files[$#files] = $1;
2041         }
2042     }
2043     close(IN);
2044
2045     open(IN, $buildlog) or dodie "Can't open $buildlog";
2046     while (<IN>) {
2047         if (/^\s*(.*?):.*(warning|error)/) {
2048             my $err = $1;
2049             foreach my $file (@files) {
2050                 my $fullpath = "$builddir/$file";
2051                 if ($file eq $err || $fullpath eq $err) {
2052                     fail "$file built with warnings" and return 0;
2053                 }
2054             }
2055         }
2056     }
2057     close(IN);
2058
2059     return 1;
2060 }
2061
2062 sub apply_min_config {
2063     my $outconfig = "$output_config.new";
2064
2065     # Read the config file and remove anything that
2066     # is in the force_config hash (from minconfig and others)
2067     # then add the force config back.
2068
2069     doprint "Applying minimum configurations into $output_config.new\n";
2070
2071     open (OUT, ">$outconfig") or
2072         dodie "Can't create $outconfig";
2073
2074     if (-f $output_config) {
2075         open (IN, $output_config) or
2076             dodie "Failed to open $output_config";
2077         while (<IN>) {
2078             if (/^(# )?(CONFIG_[^\s=]*)/) {
2079                 next if (defined($force_config{$2}));
2080             }
2081             print OUT;
2082         }
2083         close IN;
2084     }
2085     foreach my $config (keys %force_config) {
2086         print OUT "$force_config{$config}\n";
2087     }
2088     close OUT;
2089
2090     run_command "mv $outconfig $output_config";
2091 }
2092
2093 sub make_oldconfig {
2094
2095     my @force_list = keys %force_config;
2096
2097     if ($#force_list >= 0) {
2098         apply_min_config;
2099     }
2100
2101     if (!run_command "$make olddefconfig") {
2102         # Perhaps olddefconfig doesn't exist in this version of the kernel
2103         # try oldnoconfig
2104         doprint "olddefconfig failed, trying make oldnoconfig\n";
2105         if (!run_command "$make oldnoconfig") {
2106             doprint "oldnoconfig failed, trying yes '' | make oldconfig\n";
2107             # try a yes '' | oldconfig
2108             run_command "yes '' | $make oldconfig" or
2109                 dodie "failed make config oldconfig";
2110         }
2111     }
2112 }
2113
2114 # read a config file and use this to force new configs.
2115 sub load_force_config {
2116     my ($config) = @_;
2117
2118     doprint "Loading force configs from $config\n";
2119     open(IN, $config) or
2120         dodie "failed to read $config";
2121     while (<IN>) {
2122         chomp;
2123         if (/^(CONFIG[^\s=]*)(\s*=.*)/) {
2124             $force_config{$1} = $_;
2125         } elsif (/^# (CONFIG_\S*) is not set/) {
2126             $force_config{$1} = $_;
2127         }
2128     }
2129     close IN;
2130 }
2131
2132 sub build {
2133     my ($type) = @_;
2134
2135     unlink $buildlog;
2136
2137     # Failed builds should not reboot the target
2138     my $save_no_reboot = $no_reboot;
2139     $no_reboot = 1;
2140
2141     # Calculate a new version from here.
2142     $have_version = 0;
2143
2144     if (defined($pre_build)) {
2145         my $ret = run_command $pre_build;
2146         if (!$ret && defined($pre_build_die) &&
2147             $pre_build_die) {
2148             dodie "failed to pre_build\n";
2149         }
2150     }
2151
2152     if ($type =~ /^useconfig:(.*)/) {
2153         run_command "cp $1 $output_config" or
2154             dodie "could not copy $1 to .config";
2155
2156         $type = "oldconfig";
2157     }
2158
2159     # old config can ask questions
2160     if ($type eq "oldconfig") {
2161         $type = "olddefconfig";
2162
2163         # allow for empty configs
2164         run_command "touch $output_config";
2165
2166         if (!$noclean) {
2167             run_command "mv $output_config $outputdir/config_temp" or
2168                 dodie "moving .config";
2169
2170             run_command "$make mrproper" or dodie "make mrproper";
2171
2172             run_command "mv $outputdir/config_temp $output_config" or
2173                 dodie "moving config_temp";
2174         }
2175
2176     } elsif (!$noclean) {
2177         unlink "$output_config";
2178         run_command "$make mrproper" or
2179             dodie "make mrproper";
2180     }
2181
2182     # add something to distinguish this build
2183     open(OUT, "> $outputdir/localversion") or dodie("Can't make localversion file");
2184     print OUT "$localversion\n";
2185     close(OUT);
2186
2187     if (defined($minconfig)) {
2188         load_force_config($minconfig);
2189     }
2190
2191     if ($type ne "olddefconfig") {
2192         run_command "$make $type" or
2193             dodie "failed make config";
2194     }
2195     # Run old config regardless, to enforce min configurations
2196     make_oldconfig;
2197
2198     $redirect = "$buildlog";
2199     my $build_ret = run_command "$make $build_options";
2200     undef $redirect;
2201
2202     if (defined($post_build)) {
2203         # Because a post build may change the kernel version
2204         # do it now.
2205         get_version;
2206         my $ret = run_command $post_build;
2207         if (!$ret && defined($post_build_die) &&
2208             $post_build_die) {
2209             dodie "failed to post_build\n";
2210         }
2211     }
2212
2213     if (!$build_ret) {
2214         # bisect may need this to pass
2215         if ($in_bisect) {
2216             $no_reboot = $save_no_reboot;
2217             return 0;
2218         }
2219         fail "failed build" and return 0;
2220     }
2221
2222     $no_reboot = $save_no_reboot;
2223
2224     return 1;
2225 }
2226
2227 sub halt {
2228     if (!run_ssh "halt" or defined($power_off)) {
2229         if (defined($poweroff_after_halt)) {
2230             sleep $poweroff_after_halt;
2231             run_command "$power_off";
2232         }
2233     } else {
2234         # nope? the zap it!
2235         run_command "$power_off";
2236     }
2237 }
2238
2239 sub success {
2240     my ($i) = @_;
2241
2242     if (defined($post_test)) {
2243         run_command $post_test;
2244     }
2245
2246     $successes++;
2247
2248     my $name = "";
2249
2250     if (defined($test_name)) {
2251         $name = " ($test_name)";
2252     }
2253
2254     doprint "\n\n*******************************************\n";
2255     doprint     "*******************************************\n";
2256     doprint     "KTEST RESULT: TEST $i$name SUCCESS!!!!         **\n";
2257     doprint     "*******************************************\n";
2258     doprint     "*******************************************\n";
2259
2260     if (defined($store_successes)) {
2261         save_logs "success", $store_successes;
2262     }
2263
2264     if ($i != $opt{"NUM_TESTS"} && !do_not_reboot) {
2265         doprint "Reboot and wait $sleep_time seconds\n";
2266         reboot_to_good $sleep_time;
2267     }
2268 }
2269
2270 sub answer_bisect {
2271     for (;;) {
2272         doprint "Pass or fail? [p/f]";
2273         my $ans = <STDIN>;
2274         chomp $ans;
2275         if ($ans eq "p" || $ans eq "P") {
2276             return 1;
2277         } elsif ($ans eq "f" || $ans eq "F") {
2278             return 0;
2279         } else {
2280             print "Please answer 'P' or 'F'\n";
2281         }
2282     }
2283 }
2284
2285 sub child_run_test {
2286     my $failed = 0;
2287
2288     # child should have no power
2289     $reboot_on_error = 0;
2290     $poweroff_on_error = 0;
2291     $die_on_failure = 1;
2292
2293     $redirect = "$testlog";
2294     run_command $run_test or $failed = 1;
2295     undef $redirect;
2296
2297     exit $failed;
2298 }
2299
2300 my $child_done;
2301
2302 sub child_finished {
2303     $child_done = 1;
2304 }
2305
2306 sub do_run_test {
2307     my $child_pid;
2308     my $child_exit;
2309     my $line;
2310     my $full_line;
2311     my $bug = 0;
2312     my $bug_ignored = 0;
2313
2314     wait_for_monitor 1;
2315
2316     doprint "run test $run_test\n";
2317
2318     $child_done = 0;
2319
2320     $SIG{CHLD} = qw(child_finished);
2321
2322     $child_pid = fork;
2323
2324     child_run_test if (!$child_pid);
2325
2326     $full_line = "";
2327
2328     do {
2329         $line = wait_for_input($monitor_fp, 1);
2330         if (defined($line)) {
2331
2332             # we are not guaranteed to get a full line
2333             $full_line .= $line;
2334             doprint $line;
2335
2336             if ($full_line =~ /call trace:/i) {
2337                 if ($ignore_errors) {
2338                     $bug_ignored = 1;
2339                 } else {
2340                     $bug = 1;
2341                 }
2342             }
2343
2344             if ($full_line =~ /Kernel panic -/) {
2345                 $bug = 1;
2346             }
2347
2348             if ($line =~ /\n/) {
2349                 $full_line = "";
2350             }
2351         }
2352     } while (!$child_done && !$bug);
2353
2354     if (!$bug && $bug_ignored) {
2355         doprint "WARNING: Call Trace detected but ignored due to IGNORE_ERRORS=1\n";
2356     }
2357
2358     if ($bug) {
2359         my $failure_start = time;
2360         my $now;
2361         do {
2362             $line = wait_for_input($monitor_fp, 1);
2363             if (defined($line)) {
2364                 doprint $line;
2365             }
2366             $now = time;
2367             if ($now - $failure_start >= $stop_after_failure) {
2368                 last;
2369             }
2370         } while (defined($line));
2371
2372         doprint "Detected kernel crash!\n";
2373         # kill the child with extreme prejudice
2374         kill 9, $child_pid;
2375     }
2376
2377     waitpid $child_pid, 0;
2378     $child_exit = $?;
2379
2380     if (!$bug && $in_bisect) {
2381         if (defined($bisect_ret_good)) {
2382             if ($child_exit == $bisect_ret_good) {
2383                 return 1;
2384             }
2385         }
2386         if (defined($bisect_ret_skip)) {
2387             if ($child_exit == $bisect_ret_skip) {
2388                 return -1;
2389             }
2390         }
2391         if (defined($bisect_ret_abort)) {
2392             if ($child_exit == $bisect_ret_abort) {
2393                 fail "test abort" and return -2;
2394             }
2395         }
2396         if (defined($bisect_ret_bad)) {
2397             if ($child_exit == $bisect_ret_skip) {
2398                 return 0;
2399             }
2400         }
2401         if (defined($bisect_ret_default)) {
2402             if ($bisect_ret_default eq "good") {
2403                 return 1;
2404             } elsif ($bisect_ret_default eq "bad") {
2405                 return 0;
2406             } elsif ($bisect_ret_default eq "skip") {
2407                 return -1;
2408             } elsif ($bisect_ret_default eq "abort") {
2409                 return -2;
2410             } else {
2411                 fail "unknown default action: $bisect_ret_default"
2412                     and return -2;
2413             }
2414         }
2415     }
2416
2417     if ($bug || $child_exit) {
2418         return 0 if $in_bisect;
2419         fail "test failed" and return 0;
2420     }
2421     return 1;
2422 }
2423
2424 sub run_git_bisect {
2425     my ($command) = @_;
2426
2427     doprint "$command ... ";
2428
2429     my $output = `$command 2>&1`;
2430     my $ret = $?;
2431
2432     logit $output;
2433
2434     if ($ret) {
2435         doprint "FAILED\n";
2436         dodie "Failed to git bisect";
2437     }
2438
2439     doprint "SUCCESS\n";
2440     if ($output =~ m/^(Bisecting: .*\(roughly \d+ steps?\))\s+\[([[:xdigit:]]+)\]/) {
2441         doprint "$1 [$2]\n";
2442     } elsif ($output =~ m/^([[:xdigit:]]+) is the first bad commit/) {
2443         $bisect_bad_commit = $1;
2444         doprint "Found bad commit... $1\n";
2445         return 0;
2446     } else {
2447         # we already logged it, just print it now.
2448         print $output;
2449     }
2450
2451     return 1;
2452 }
2453
2454 sub bisect_reboot {
2455     doprint "Reboot and sleep $bisect_sleep_time seconds\n";
2456     reboot_to_good $bisect_sleep_time;
2457 }
2458
2459 # returns 1 on success, 0 on failure, -1 on skip
2460 sub run_bisect_test {
2461     my ($type, $buildtype) = @_;
2462
2463     my $failed = 0;
2464     my $result;
2465     my $output;
2466     my $ret;
2467
2468     $in_bisect = 1;
2469
2470     build $buildtype or $failed = 1;
2471
2472     if ($type ne "build") {
2473         if ($failed && $bisect_skip) {
2474             $in_bisect = 0;
2475             return -1;
2476         }
2477         dodie "Failed on build" if $failed;
2478
2479         # Now boot the box
2480         start_monitor_and_boot or $failed = 1;
2481
2482         if ($type ne "boot") {
2483             if ($failed && $bisect_skip) {
2484                 end_monitor;
2485                 bisect_reboot;
2486                 $in_bisect = 0;
2487                 return -1;
2488             }
2489             dodie "Failed on boot" if $failed;
2490
2491             do_run_test or $failed = 1;
2492         }
2493         end_monitor;
2494     }
2495
2496     if ($failed) {
2497         $result = 0;
2498     } else {
2499         $result = 1;
2500     }
2501
2502     # reboot the box to a kernel we can ssh to
2503     if ($type ne "build") {
2504         bisect_reboot;
2505     }
2506     $in_bisect = 0;
2507
2508     return $result;
2509 }
2510
2511 sub run_bisect {
2512     my ($type) = @_;
2513     my $buildtype = "oldconfig";
2514
2515     # We should have a minconfig to use?
2516     if (defined($minconfig)) {
2517         $buildtype = "useconfig:$minconfig";
2518     }
2519
2520     my $ret = run_bisect_test $type, $buildtype;
2521
2522     if ($bisect_manual) {
2523         $ret = answer_bisect;
2524     }
2525
2526     # Are we looking for where it worked, not failed?
2527     if ($reverse_bisect && $ret >= 0) {
2528         $ret = !$ret;
2529     }
2530
2531     if ($ret > 0) {
2532         return "good";
2533     } elsif ($ret == 0) {
2534         return  "bad";
2535     } elsif ($bisect_skip) {
2536         doprint "HIT A BAD COMMIT ... SKIPPING\n";
2537         return "skip";
2538     }
2539 }
2540
2541 sub update_bisect_replay {
2542     my $tmp_log = "$tmpdir/ktest_bisect_log";
2543     run_command "git bisect log > $tmp_log" or
2544         die "can't create bisect log";
2545     return $tmp_log;
2546 }
2547
2548 sub bisect {
2549     my ($i) = @_;
2550
2551     my $result;
2552
2553     die "BISECT_GOOD[$i] not defined\n" if (!defined($bisect_good));
2554     die "BISECT_BAD[$i] not defined\n"  if (!defined($bisect_bad));
2555     die "BISECT_TYPE[$i] not defined\n" if (!defined($bisect_type));
2556
2557     my $good = $bisect_good;
2558     my $bad = $bisect_bad;
2559     my $type = $bisect_type;
2560     my $start = $bisect_start;
2561     my $replay = $bisect_replay;
2562     my $start_files = $bisect_files;
2563
2564     if (defined($start_files)) {
2565         $start_files = " -- " . $start_files;
2566     } else {
2567         $start_files = "";
2568     }
2569
2570     # convert to true sha1's
2571     $good = get_sha1($good);
2572     $bad = get_sha1($bad);
2573
2574     if (defined($bisect_reverse) && $bisect_reverse == 1) {
2575         doprint "Performing a reverse bisect (bad is good, good is bad!)\n";
2576         $reverse_bisect = 1;
2577     } else {
2578         $reverse_bisect = 0;
2579     }
2580
2581     # Can't have a test without having a test to run
2582     if ($type eq "test" && !defined($run_test)) {
2583         $type = "boot";
2584     }
2585
2586     # Check if a bisect was running
2587     my $bisect_start_file = "$builddir/.git/BISECT_START";
2588
2589     my $check = $bisect_check;
2590     my $do_check = defined($check) && $check ne "0";
2591
2592     if ( -f $bisect_start_file ) {
2593         print "Bisect in progress found\n";
2594         if ($do_check) {
2595             print " If you say yes, then no checks of good or bad will be done\n";
2596         }
2597         if (defined($replay)) {
2598             print "** BISECT_REPLAY is defined in config file **";
2599             print " Ignore config option and perform new git bisect log?\n";
2600             if (read_ync " (yes, no, or cancel) ") {
2601                 $replay = update_bisect_replay;
2602                 $do_check = 0;
2603             }
2604         } elsif (read_yn "read git log and continue?") {
2605             $replay = update_bisect_replay;
2606             $do_check = 0;
2607         }
2608     }
2609
2610     if ($do_check) {
2611
2612         # get current HEAD
2613         my $head = get_sha1("HEAD");
2614
2615         if ($check ne "good") {
2616             doprint "TESTING BISECT BAD [$bad]\n";
2617             run_command "git checkout $bad" or
2618                 die "Failed to checkout $bad";
2619
2620             $result = run_bisect $type;
2621
2622             if ($result ne "bad") {
2623                 fail "Tested BISECT_BAD [$bad] and it succeeded" and return 0;
2624             }
2625         }
2626
2627         if ($check ne "bad") {
2628             doprint "TESTING BISECT GOOD [$good]\n";
2629             run_command "git checkout $good" or
2630                 die "Failed to checkout $good";
2631
2632             $result = run_bisect $type;
2633
2634             if ($result ne "good") {
2635                 fail "Tested BISECT_GOOD [$good] and it failed" and return 0;
2636             }
2637         }
2638
2639         # checkout where we started
2640         run_command "git checkout $head" or
2641             die "Failed to checkout $head";
2642     }
2643
2644     run_command "git bisect start$start_files" or
2645         dodie "could not start bisect";
2646
2647     run_command "git bisect good $good" or
2648         dodie "could not set bisect good to $good";
2649
2650     run_git_bisect "git bisect bad $bad" or
2651         dodie "could not set bisect bad to $bad";
2652
2653     if (defined($replay)) {
2654         run_command "git bisect replay $replay" or
2655             dodie "failed to run replay";
2656     }
2657
2658     if (defined($start)) {
2659         run_command "git checkout $start" or
2660             dodie "failed to checkout $start";
2661     }
2662
2663     my $test;
2664     do {
2665         $result = run_bisect $type;
2666         $test = run_git_bisect "git bisect $result";
2667     } while ($test);
2668
2669     run_command "git bisect log" or
2670         dodie "could not capture git bisect log";
2671
2672     run_command "git bisect reset" or
2673         dodie "could not reset git bisect";
2674
2675     doprint "Bad commit was [$bisect_bad_commit]\n";
2676
2677     success $i;
2678 }
2679
2680 # config_ignore holds the configs that were set (or unset) for
2681 # a good config and we will ignore these configs for the rest
2682 # of a config bisect. These configs stay as they were.
2683 my %config_ignore;
2684
2685 # config_set holds what all configs were set as.
2686 my %config_set;
2687
2688 # config_off holds the set of configs that the bad config had disabled.
2689 # We need to record them and set them in the .config when running
2690 # olddefconfig, because olddefconfig keeps the defaults.
2691 my %config_off;
2692
2693 # config_off_tmp holds a set of configs to turn off for now
2694 my @config_off_tmp;
2695
2696 # config_list is the set of configs that are being tested
2697 my %config_list;
2698 my %null_config;
2699
2700 my %dependency;
2701
2702 sub assign_configs {
2703     my ($hash, $config) = @_;
2704
2705     open (IN, $config)
2706         or dodie "Failed to read $config";
2707
2708     while (<IN>) {
2709         if (/^((CONFIG\S*)=.*)/) {
2710             ${$hash}{$2} = $1;
2711         }
2712     }
2713
2714     close(IN);
2715 }
2716
2717 sub process_config_ignore {
2718     my ($config) = @_;
2719
2720     assign_configs \%config_ignore, $config;
2721 }
2722
2723 sub read_current_config {
2724     my ($config_ref) = @_;
2725
2726     %{$config_ref} = ();
2727     undef %{$config_ref};
2728
2729     my @key = keys %{$config_ref};
2730     if ($#key >= 0) {
2731         print "did not delete!\n";
2732         exit;
2733     }
2734     open (IN, "$output_config");
2735
2736     while (<IN>) {
2737         if (/^(CONFIG\S+)=(.*)/) {
2738             ${$config_ref}{$1} = $2;
2739         }
2740     }
2741     close(IN);
2742 }
2743
2744 sub get_dependencies {
2745     my ($config) = @_;
2746
2747     my $arr = $dependency{$config};
2748     if (!defined($arr)) {
2749         return ();
2750     }
2751
2752     my @deps = @{$arr};
2753
2754     foreach my $dep (@{$arr}) {
2755         print "ADD DEP $dep\n";
2756         @deps = (@deps, get_dependencies $dep);
2757     }
2758
2759     return @deps;
2760 }
2761
2762 sub create_config {
2763     my @configs = @_;
2764
2765     open(OUT, ">$output_config") or dodie "Can not write to $output_config";
2766
2767     foreach my $config (@configs) {
2768         print OUT "$config_set{$config}\n";
2769         my @deps = get_dependencies $config;
2770         foreach my $dep (@deps) {
2771             print OUT "$config_set{$dep}\n";
2772         }
2773     }
2774
2775     # turn off configs to keep off
2776     foreach my $config (keys %config_off) {
2777         print OUT "# $config is not set\n";
2778     }
2779
2780     # turn off configs that should be off for now
2781     foreach my $config (@config_off_tmp) {
2782         print OUT "# $config is not set\n";
2783     }
2784
2785     foreach my $config (keys %config_ignore) {
2786         print OUT "$config_ignore{$config}\n";
2787     }
2788     close(OUT);
2789
2790     make_oldconfig;
2791 }
2792
2793 sub compare_configs {
2794     my (%a, %b) = @_;
2795
2796     foreach my $item (keys %a) {
2797         if (!defined($b{$item})) {
2798             print "diff $item\n";
2799             return 1;
2800         }
2801         delete $b{$item};
2802     }
2803
2804     my @keys = keys %b;
2805     if ($#keys) {
2806         print "diff2 $keys[0]\n";
2807     }
2808     return -1 if ($#keys >= 0);
2809
2810     return 0;
2811 }
2812
2813 sub run_config_bisect_test {
2814     my ($type) = @_;
2815
2816     return run_bisect_test $type, "oldconfig";
2817 }
2818
2819 sub process_passed {
2820     my (%configs) = @_;
2821
2822     doprint "These configs had no failure: (Enabling them for further compiles)\n";
2823     # Passed! All these configs are part of a good compile.
2824     # Add them to the min options.
2825     foreach my $config (keys %configs) {
2826         if (defined($config_list{$config})) {
2827             doprint " removing $config\n";
2828             $config_ignore{$config} = $config_list{$config};
2829             delete $config_list{$config};
2830         }
2831     }
2832     doprint "config copied to $outputdir/config_good\n";
2833     run_command "cp -f $output_config $outputdir/config_good";
2834 }
2835
2836 sub process_failed {
2837     my ($config) = @_;
2838
2839     doprint "\n\n***************************************\n";
2840     doprint "Found bad config: $config\n";
2841     doprint "***************************************\n\n";
2842 }
2843
2844 sub run_config_bisect {
2845
2846     my @start_list = keys %config_list;
2847
2848     if ($#start_list < 0) {
2849         doprint "No more configs to test!!!\n";
2850         return -1;
2851     }
2852
2853     doprint "***** RUN TEST ***\n";
2854     my $type = $config_bisect_type;
2855     my $ret;
2856     my %current_config;
2857
2858     my $count = $#start_list + 1;
2859     doprint "  $count configs to test\n";
2860
2861     my $half = int($#start_list / 2);
2862
2863     do {
2864         my @tophalf = @start_list[0 .. $half];
2865
2866         # keep the bottom half off
2867         if ($half < $#start_list) {
2868             @config_off_tmp = @start_list[$half + 1 .. $#start_list];
2869         } else {
2870             @config_off_tmp = ();
2871         }
2872
2873         create_config @tophalf;
2874         read_current_config \%current_config;
2875
2876         $count = $#tophalf + 1;
2877         doprint "Testing $count configs\n";
2878         my $found = 0;
2879         # make sure we test something
2880         foreach my $config (@tophalf) {
2881             if (defined($current_config{$config})) {
2882                 logit " $config\n";
2883                 $found = 1;
2884             }
2885         }
2886         if (!$found) {
2887             # try the other half
2888             doprint "Top half produced no set configs, trying bottom half\n";
2889
2890             # keep the top half off
2891             @config_off_tmp = @tophalf;
2892             @tophalf = @start_list[$half + 1 .. $#start_list];
2893
2894             create_config @tophalf;
2895             read_current_config \%current_config;
2896             foreach my $config (@tophalf) {
2897                 if (defined($current_config{$config})) {
2898                     logit " $config\n";
2899                     $found = 1;
2900                 }
2901             }
2902             if (!$found) {
2903                 doprint "Failed: Can't make new config with current configs\n";
2904                 foreach my $config (@start_list) {
2905                     doprint "  CONFIG: $config\n";
2906                 }
2907                 return -1;
2908             }
2909             $count = $#tophalf + 1;
2910             doprint "Testing $count configs\n";
2911         }
2912
2913         $ret = run_config_bisect_test $type;
2914         if ($bisect_manual) {
2915             $ret = answer_bisect;
2916         }
2917         if ($ret) {
2918             process_passed %current_config;
2919             return 0;
2920         }
2921
2922         doprint "This config had a failure.\n";
2923         doprint "Removing these configs that were not set in this config:\n";
2924         doprint "config copied to $outputdir/config_bad\n";
2925         run_command "cp -f $output_config $outputdir/config_bad";
2926
2927         # A config exists in this group that was bad.
2928         foreach my $config (keys %config_list) {
2929             if (!defined($current_config{$config})) {
2930                 doprint " removing $config\n";
2931                 delete $config_list{$config};
2932             }
2933         }
2934
2935         @start_list = @tophalf;
2936
2937         if ($#start_list == 0) {
2938             process_failed $start_list[0];
2939             return 1;
2940         }
2941
2942         # remove half the configs we are looking at and see if
2943         # they are good.
2944         $half = int($#start_list / 2);
2945     } while ($#start_list > 0);
2946
2947     # we found a single config, try it again unless we are running manually
2948
2949     if ($bisect_manual) {
2950         process_failed $start_list[0];
2951         return 1;
2952     }
2953
2954     my @tophalf = @start_list[0 .. 0];
2955
2956     $ret = run_config_bisect_test $type;
2957     if ($ret) {
2958         process_passed %current_config;
2959         return 0;
2960     }
2961
2962     process_failed $start_list[0];
2963     return 1;
2964 }
2965
2966 sub config_bisect {
2967     my ($i) = @_;
2968
2969     my $start_config = $config_bisect;
2970
2971     my $tmpconfig = "$tmpdir/use_config";
2972
2973     if (defined($config_bisect_good)) {
2974         process_config_ignore $config_bisect_good;
2975     }
2976
2977     # Make the file with the bad config and the min config
2978     if (defined($minconfig)) {
2979         # read the min config for things to ignore
2980         run_command "cp $minconfig $tmpconfig" or
2981             dodie "failed to copy $minconfig to $tmpconfig";
2982     } else {
2983         unlink $tmpconfig;
2984     }
2985
2986     if (-f $tmpconfig) {
2987         load_force_config($tmpconfig);
2988         process_config_ignore $tmpconfig;
2989     }
2990
2991     # now process the start config
2992     run_command "cp $start_config $output_config" or
2993         dodie "failed to copy $start_config to $output_config";
2994
2995     # read directly what we want to check
2996     my %config_check;
2997     open (IN, $output_config)
2998         or dodie "failed to open $output_config";
2999
3000     while (<IN>) {
3001         if (/^((CONFIG\S*)=.*)/) {
3002             $config_check{$2} = $1;
3003         }
3004     }
3005     close(IN);
3006
3007     # Now run oldconfig with the minconfig
3008     make_oldconfig;
3009
3010     # check to see what we lost (or gained)
3011     open (IN, $output_config)
3012         or dodie "Failed to read $start_config";
3013
3014     my %removed_configs;
3015     my %added_configs;
3016
3017     while (<IN>) {
3018         if (/^((CONFIG\S*)=.*)/) {
3019             # save off all options
3020             $config_set{$2} = $1;
3021             if (defined($config_check{$2})) {
3022                 if (defined($config_ignore{$2})) {
3023                     $removed_configs{$2} = $1;
3024                 } else {
3025                     $config_list{$2} = $1;
3026                 }
3027             } elsif (!defined($config_ignore{$2})) {
3028                 $added_configs{$2} = $1;
3029                 $config_list{$2} = $1;
3030             }
3031         } elsif (/^# ((CONFIG\S*).*)/) {
3032             # Keep these configs disabled
3033             $config_set{$2} = $1;
3034             $config_off{$2} = $1;
3035         }
3036     }
3037     close(IN);
3038
3039     my @confs = keys %removed_configs;
3040     if ($#confs >= 0) {
3041         doprint "Configs overridden by default configs and removed from check:\n";
3042         foreach my $config (@confs) {
3043             doprint " $config\n";
3044         }
3045     }
3046     @confs = keys %added_configs;
3047     if ($#confs >= 0) {
3048         doprint "Configs appearing in make oldconfig and added:\n";
3049         foreach my $config (@confs) {
3050             doprint " $config\n";
3051         }
3052     }
3053
3054     my %config_test;
3055     my $once = 0;
3056
3057     @config_off_tmp = ();
3058
3059     # Sometimes kconfig does weird things. We must make sure
3060     # that the config we autocreate has everything we need
3061     # to test, otherwise we may miss testing configs, or
3062     # may not be able to create a new config.
3063     # Here we create a config with everything set.
3064     create_config (keys %config_list);
3065     read_current_config \%config_test;
3066     foreach my $config (keys %config_list) {
3067         if (!defined($config_test{$config})) {
3068             if (!$once) {
3069                 $once = 1;
3070                 doprint "Configs not produced by kconfig (will not be checked):\n";
3071             }
3072             doprint "  $config\n";
3073             delete $config_list{$config};
3074         }
3075     }
3076     my $ret;
3077
3078     if (defined($config_bisect_check) && $config_bisect_check) {
3079         doprint " Checking to make sure bad config with min config fails\n";
3080         create_config keys %config_list;
3081         $ret = run_config_bisect_test $config_bisect_type;
3082         if ($ret) {
3083             doprint " FAILED! Bad config with min config boots fine\n";
3084             return -1;
3085         }
3086         doprint " Bad config with min config fails as expected\n";
3087     }
3088
3089     do {
3090         $ret = run_config_bisect;
3091     } while (!$ret);
3092
3093     return $ret if ($ret < 0);
3094
3095     success $i;
3096 }
3097
3098 sub patchcheck_reboot {
3099     doprint "Reboot and sleep $patchcheck_sleep_time seconds\n";
3100     reboot_to_good $patchcheck_sleep_time;
3101 }
3102
3103 sub patchcheck {
3104     my ($i) = @_;
3105
3106     die "PATCHCHECK_START[$i] not defined\n"
3107         if (!defined($patchcheck_start));
3108     die "PATCHCHECK_TYPE[$i] not defined\n"
3109         if (!defined($patchcheck_type));
3110
3111     my $start = $patchcheck_start;
3112
3113     my $end = "HEAD";
3114     if (defined($patchcheck_end)) {
3115         $end = $patchcheck_end;
3116     }
3117
3118     # Get the true sha1's since we can use things like HEAD~3
3119     $start = get_sha1($start);
3120     $end = get_sha1($end);
3121
3122     my $type = $patchcheck_type;
3123
3124     # Can't have a test without having a test to run
3125     if ($type eq "test" && !defined($run_test)) {
3126         $type = "boot";
3127     }
3128
3129     open (IN, "git log --pretty=oneline $end|") or
3130         dodie "could not get git list";
3131
3132     my @list;
3133
3134     while (<IN>) {
3135         chomp;
3136         $list[$#list+1] = $_;
3137         last if (/^$start/);
3138     }
3139     close(IN);
3140
3141     if ($list[$#list] !~ /^$start/) {
3142         fail "SHA1 $start not found";
3143     }
3144
3145     # go backwards in the list
3146     @list = reverse @list;
3147
3148     my $save_clean = $noclean;
3149     my %ignored_warnings;
3150
3151     if (defined($ignore_warnings)) {
3152         foreach my $sha1 (split /\s+/, $ignore_warnings) {
3153             $ignored_warnings{$sha1} = 1;
3154         }
3155     }
3156
3157     $in_patchcheck = 1;
3158     foreach my $item (@list) {
3159         my $sha1 = $item;
3160         $sha1 =~ s/^([[:xdigit:]]+).*/$1/;
3161
3162         doprint "\nProcessing commit $item\n\n";
3163
3164         run_command "git checkout $sha1" or
3165             die "Failed to checkout $sha1";
3166
3167         # only clean on the first and last patch
3168         if ($item eq $list[0] ||
3169             $item eq $list[$#list]) {
3170             $noclean = $save_clean;
3171         } else {
3172             $noclean = 1;
3173         }
3174
3175         if (defined($minconfig)) {
3176             build "useconfig:$minconfig" or return 0;
3177         } else {
3178             # ?? no config to use?
3179             build "oldconfig" or return 0;
3180         }
3181
3182         # No need to do per patch checking if warnings file exists
3183         if (!defined($warnings_file) && !defined($ignored_warnings{$sha1})) {
3184             check_patch_buildlog $sha1 or return 0;
3185         }
3186
3187         check_buildlog or return 0;
3188
3189         next if ($type eq "build");
3190
3191         my $failed = 0;
3192
3193         start_monitor_and_boot or $failed = 1;
3194
3195         if (!$failed && $type ne "boot"){
3196             do_run_test or $failed = 1;
3197         }
3198         end_monitor;
3199         return 0 if ($failed);
3200
3201         patchcheck_reboot;
3202
3203     }
3204     $in_patchcheck = 0;
3205     success $i;
3206
3207     return 1;
3208 }
3209
3210 my %depends;
3211 my %depcount;
3212 my $iflevel = 0;
3213 my @ifdeps;
3214
3215 # prevent recursion
3216 my %read_kconfigs;
3217
3218 sub add_dep {
3219     # $config depends on $dep
3220     my ($config, $dep) = @_;
3221
3222     if (defined($depends{$config})) {
3223         $depends{$config} .= " " . $dep;
3224     } else {
3225         $depends{$config} = $dep;
3226     }
3227
3228     # record the number of configs depending on $dep
3229     if (defined $depcount{$dep}) {
3230         $depcount{$dep}++;
3231     } else {
3232         $depcount{$dep} = 1;
3233     }
3234 }
3235
3236 # taken from streamline_config.pl
3237 sub read_kconfig {
3238     my ($kconfig) = @_;
3239
3240     my $state = "NONE";
3241     my $config;
3242     my @kconfigs;
3243
3244     my $cont = 0;
3245     my $line;
3246
3247
3248     if (! -f $kconfig) {
3249         doprint "file $kconfig does not exist, skipping\n";
3250         return;
3251     }
3252
3253     open(KIN, "$kconfig")
3254         or die "Can't open $kconfig";
3255     while (<KIN>) {
3256         chomp;
3257
3258         # Make sure that lines ending with \ continue
3259         if ($cont) {
3260             $_ = $line . " " . $_;
3261         }
3262
3263         if (s/\\$//) {
3264             $cont = 1;
3265             $line = $_;
3266             next;
3267         }
3268
3269         $cont = 0;
3270
3271         # collect any Kconfig sources
3272         if (/^source\s*"(.*)"/) {
3273             $kconfigs[$#kconfigs+1] = $1;
3274         }
3275
3276         # configs found
3277         if (/^\s*(menu)?config\s+(\S+)\s*$/) {
3278             $state = "NEW";
3279             $config = $2;
3280
3281             for (my $i = 0; $i < $iflevel; $i++) {
3282                 add_dep $config, $ifdeps[$i];
3283             }
3284
3285         # collect the depends for the config
3286         } elsif ($state eq "NEW" && /^\s*depends\s+on\s+(.*)$/) {
3287
3288             add_dep $config, $1;
3289
3290         # Get the configs that select this config
3291         } elsif ($state eq "NEW" && /^\s*select\s+(\S+)/) {
3292
3293             # selected by depends on config
3294             add_dep $1, $config;
3295
3296         # Check for if statements
3297         } elsif (/^if\s+(.*\S)\s*$/) {
3298             my $deps = $1;
3299             # remove beginning and ending non text
3300             $deps =~ s/^[^a-zA-Z0-9_]*//;
3301             $deps =~ s/[^a-zA-Z0-9_]*$//;
3302
3303             my @deps = split /[^a-zA-Z0-9_]+/, $deps;
3304
3305             $ifdeps[$iflevel++] = join ':', @deps;
3306
3307         } elsif (/^endif/) {
3308
3309             $iflevel-- if ($iflevel);
3310
3311         # stop on "help"
3312         } elsif (/^\s*help\s*$/) {
3313             $state = "NONE";
3314         }
3315     }
3316     close(KIN);
3317
3318     # read in any configs that were found.
3319     foreach $kconfig (@kconfigs) {
3320         if (!defined($read_kconfigs{$kconfig})) {
3321             $read_kconfigs{$kconfig} = 1;
3322             read_kconfig("$builddir/$kconfig");
3323         }
3324     }
3325 }
3326
3327 sub read_depends {
3328     # find out which arch this is by the kconfig file
3329     open (IN, $output_config)
3330         or dodie "Failed to read $output_config";
3331     my $arch;
3332     while (<IN>) {
3333         if (m,Linux/(\S+)\s+\S+\s+Kernel Configuration,) {
3334             $arch = $1;
3335             last;
3336         }
3337     }
3338     close IN;
3339
3340     if (!defined($arch)) {
3341         doprint "Could not find arch from config file\n";
3342         doprint "no dependencies used\n";
3343         return;
3344     }
3345
3346     # arch is really the subarch, we need to know
3347     # what directory to look at.
3348     if ($arch eq "i386" || $arch eq "x86_64") {
3349         $arch = "x86";
3350     } elsif ($arch =~ /^tile/) {
3351         $arch = "tile";
3352     }
3353
3354     my $kconfig = "$builddir/arch/$arch/Kconfig";
3355
3356     if (! -f $kconfig && $arch =~ /\d$/) {
3357         my $orig = $arch;
3358         # some subarchs have numbers, truncate them
3359         $arch =~ s/\d*$//;
3360         $kconfig = "$builddir/arch/$arch/Kconfig";
3361         if (! -f $kconfig) {
3362             doprint "No idea what arch dir $orig is for\n";
3363             doprint "no dependencies used\n";
3364             return;
3365         }
3366     }
3367
3368     read_kconfig($kconfig);
3369 }
3370
3371 sub read_config_list {
3372     my ($config) = @_;
3373
3374     open (IN, $config)
3375         or dodie "Failed to read $config";
3376
3377     while (<IN>) {
3378         if (/^((CONFIG\S*)=.*)/) {
3379             if (!defined($config_ignore{$2})) {
3380                 $config_list{$2} = $1;
3381             }
3382         }
3383     }
3384
3385     close(IN);
3386 }
3387
3388 sub read_output_config {
3389     my ($config) = @_;
3390
3391     assign_configs \%config_ignore, $config;
3392 }
3393
3394 sub make_new_config {
3395     my @configs = @_;
3396
3397     open (OUT, ">$output_config")
3398         or dodie "Failed to write $output_config";
3399
3400     foreach my $config (@configs) {
3401         print OUT "$config\n";
3402     }
3403     close OUT;
3404 }
3405
3406 sub chomp_config {
3407     my ($config) = @_;
3408
3409     $config =~ s/CONFIG_//;
3410
3411     return $config;
3412 }
3413
3414 sub get_depends {
3415     my ($dep) = @_;
3416
3417     my $kconfig = chomp_config $dep;
3418
3419     $dep = $depends{"$kconfig"};
3420
3421     # the dep string we have saves the dependencies as they
3422     # were found, including expressions like ! && ||. We
3423     # want to split this out into just an array of configs.
3424
3425     my $valid = "A-Za-z_0-9";
3426
3427     my @configs;
3428
3429     while ($dep =~ /[$valid]/) {
3430
3431         if ($dep =~ /^[^$valid]*([$valid]+)/) {
3432             my $conf = "CONFIG_" . $1;
3433
3434             $configs[$#configs + 1] = $conf;
3435
3436             $dep =~ s/^[^$valid]*[$valid]+//;
3437         } else {
3438             die "this should never happen";
3439         }
3440     }
3441
3442     return @configs;
3443 }
3444
3445 my %min_configs;
3446 my %keep_configs;
3447 my %save_configs;
3448 my %processed_configs;
3449 my %nochange_config;
3450
3451 sub test_this_config {
3452     my ($config) = @_;
3453
3454     my $found;
3455
3456     # if we already processed this config, skip it
3457     if (defined($processed_configs{$config})) {
3458         return undef;
3459     }
3460     $processed_configs{$config} = 1;
3461
3462     # if this config failed during this round, skip it
3463     if (defined($nochange_config{$config})) {
3464         return undef;
3465     }
3466
3467     my $kconfig = chomp_config $config;
3468
3469     # Test dependencies first
3470     if (defined($depends{"$kconfig"})) {
3471         my @parents = get_depends $config;
3472         foreach my $parent (@parents) {
3473             # if the parent is in the min config, check it first
3474             next if (!defined($min_configs{$parent}));
3475             $found = test_this_config($parent);
3476             if (defined($found)) {
3477                 return $found;
3478             }
3479         }
3480     }
3481
3482     # Remove this config from the list of configs
3483     # do a make olddefconfig and then read the resulting
3484     # .config to make sure it is missing the config that
3485     # we had before
3486     my %configs = %min_configs;
3487     delete $configs{$config};
3488     make_new_config ((values %configs), (values %keep_configs));
3489     make_oldconfig;
3490     undef %configs;
3491     assign_configs \%configs, $output_config;
3492
3493     return $config if (!defined($configs{$config}));
3494
3495     doprint "disabling config $config did not change .config\n";
3496
3497     $nochange_config{$config} = 1;
3498
3499     return undef;
3500 }
3501
3502 sub make_min_config {
3503     my ($i) = @_;
3504
3505     my $type = $minconfig_type;
3506     if ($type ne "boot" && $type ne "test") {
3507         fail "Invalid MIN_CONFIG_TYPE '$minconfig_type'\n" .
3508             " make_min_config works only with 'boot' and 'test'\n" and return;
3509     }
3510
3511     if (!defined($output_minconfig)) {
3512         fail "OUTPUT_MIN_CONFIG not defined" and return;
3513     }
3514
3515     # If output_minconfig exists, and the start_minconfig
3516     # came from min_config, than ask if we should use
3517     # that instead.
3518     if (-f $output_minconfig && !$start_minconfig_defined) {
3519         print "$output_minconfig exists\n";
3520         if (!defined($use_output_minconfig)) {
3521             if (read_yn " Use it as minconfig?") {
3522                 $start_minconfig = $output_minconfig;
3523             }
3524         } elsif ($use_output_minconfig > 0) {
3525             doprint "Using $output_minconfig as MIN_CONFIG\n";
3526             $start_minconfig = $output_minconfig;
3527         } else {
3528             doprint "Set to still use MIN_CONFIG as starting point\n";
3529         }
3530     }
3531
3532     if (!defined($start_minconfig)) {
3533         fail "START_MIN_CONFIG or MIN_CONFIG not defined" and return;
3534     }
3535
3536     my $temp_config = "$tmpdir/temp_config";
3537
3538     # First things first. We build an allnoconfig to find
3539     # out what the defaults are that we can't touch.
3540     # Some are selections, but we really can't handle selections.
3541
3542     my $save_minconfig = $minconfig;
3543     undef $minconfig;
3544
3545     run_command "$make allnoconfig" or return 0;
3546
3547     read_depends;
3548
3549     process_config_ignore $output_config;
3550
3551     undef %save_configs;
3552     undef %min_configs;
3553
3554     if (defined($ignore_config)) {
3555         # make sure the file exists
3556         `touch $ignore_config`;
3557         assign_configs \%save_configs, $ignore_config;
3558     }
3559
3560     %keep_configs = %save_configs;
3561
3562     doprint "Load initial configs from $start_minconfig\n";
3563
3564     # Look at the current min configs, and save off all the
3565     # ones that were set via the allnoconfig
3566     assign_configs \%min_configs, $start_minconfig;
3567
3568     my @config_keys = keys %min_configs;
3569
3570     # All configs need a depcount
3571     foreach my $config (@config_keys) {
3572         my $kconfig = chomp_config $config;
3573         if (!defined $depcount{$kconfig}) {
3574                 $depcount{$kconfig} = 0;
3575         }
3576     }
3577
3578     # Remove anything that was set by the make allnoconfig
3579     # we shouldn't need them as they get set for us anyway.
3580     foreach my $config (@config_keys) {
3581         # Remove anything in the ignore_config
3582         if (defined($keep_configs{$config})) {
3583             my $file = $ignore_config;
3584             $file =~ s,.*/(.*?)$,$1,;
3585             doprint "$config set by $file ... ignored\n";
3586             delete $min_configs{$config};
3587             next;
3588         }
3589         # But make sure the settings are the same. If a min config
3590         # sets a selection, we do not want to get rid of it if
3591         # it is not the same as what we have. Just move it into
3592         # the keep configs.
3593         if (defined($config_ignore{$config})) {
3594             if ($config_ignore{$config} ne $min_configs{$config}) {
3595                 doprint "$config is in allnoconfig as '$config_ignore{$config}'";
3596                 doprint " but it is '$min_configs{$config}' in minconfig .. keeping\n";
3597                 $keep_configs{$config} = $min_configs{$config};
3598             } else {
3599                 doprint "$config set by allnoconfig ... ignored\n";
3600             }
3601             delete $min_configs{$config};
3602         }
3603     }
3604
3605     my $done = 0;
3606     my $take_two = 0;
3607
3608     while (!$done) {
3609
3610         my $config;
3611         my $found;
3612
3613         # Now disable each config one by one and do a make oldconfig
3614         # till we find a config that changes our list.
3615
3616         my @test_configs = keys %min_configs;
3617
3618         # Sort keys by who is most dependent on
3619         @test_configs = sort  { $depcount{chomp_config($b)} <=> $depcount{chomp_config($a)} }
3620                           @test_configs ;
3621
3622         # Put configs that did not modify the config at the end.
3623         my $reset = 1;
3624         for (my $i = 0; $i < $#test_configs; $i++) {
3625             if (!defined($nochange_config{$test_configs[0]})) {
3626                 $reset = 0;
3627                 last;
3628             }
3629             # This config didn't change the .config last time.
3630             # Place it at the end
3631             my $config = shift @test_configs;
3632             push @test_configs, $config;
3633         }
3634
3635         # if every test config has failed to modify the .config file
3636         # in the past, then reset and start over.
3637         if ($reset) {
3638             undef %nochange_config;
3639         }
3640
3641         undef %processed_configs;
3642
3643         foreach my $config (@test_configs) {
3644
3645             $found = test_this_config $config;
3646
3647             last if (defined($found));
3648
3649             # oh well, try another config
3650         }
3651
3652         if (!defined($found)) {
3653             # we could have failed due to the nochange_config hash
3654             # reset and try again
3655             if (!$take_two) {
3656                 undef %nochange_config;
3657                 $take_two = 1;
3658                 next;
3659             }
3660             doprint "No more configs found that we can disable\n";
3661             $done = 1;
3662             last;
3663         }
3664         $take_two = 0;
3665
3666         $config = $found;
3667
3668         doprint "Test with $config disabled\n";
3669
3670         # set in_bisect to keep build and monitor from dieing
3671         $in_bisect = 1;
3672
3673         my $failed = 0;
3674         build "oldconfig" or $failed = 1;
3675         if (!$failed) {
3676                 start_monitor_and_boot or $failed = 1;
3677
3678                 if ($type eq "test" && !$failed) {
3679                     do_run_test or $failed = 1;
3680                 }
3681
3682                 end_monitor;
3683         }
3684
3685         $in_bisect = 0;
3686
3687         if ($failed) {
3688             doprint "$min_configs{$config} is needed to boot the box... keeping\n";
3689             # this config is needed, add it to the ignore list.
3690             $keep_configs{$config} = $min_configs{$config};
3691             $save_configs{$config} = $min_configs{$config};
3692             delete $min_configs{$config};
3693
3694             # update new ignore configs
3695             if (defined($ignore_config)) {
3696                 open (OUT, ">$temp_config")
3697                     or die "Can't write to $temp_config";
3698                 foreach my $config (keys %save_configs) {
3699                     print OUT "$save_configs{$config}\n";
3700                 }
3701                 close OUT;
3702                 run_command "mv $temp_config $ignore_config" or
3703                     dodie "failed to copy update to $ignore_config";
3704             }
3705
3706         } else {
3707             # We booted without this config, remove it from the minconfigs.
3708             doprint "$config is not needed, disabling\n";
3709
3710             delete $min_configs{$config};
3711
3712             # Also disable anything that is not enabled in this config
3713             my %configs;
3714             assign_configs \%configs, $output_config;
3715             my @config_keys = keys %min_configs;
3716             foreach my $config (@config_keys) {
3717                 if (!defined($configs{$config})) {
3718                     doprint "$config is not set, disabling\n";
3719                     delete $min_configs{$config};
3720                 }
3721             }
3722
3723             # Save off all the current mandidory configs
3724             open (OUT, ">$temp_config")
3725                 or die "Can't write to $temp_config";
3726             foreach my $config (keys %keep_configs) {
3727                 print OUT "$keep_configs{$config}\n";
3728             }
3729             foreach my $config (keys %min_configs) {
3730                 print OUT "$min_configs{$config}\n";
3731             }
3732             close OUT;
3733
3734             run_command "mv $temp_config $output_minconfig" or
3735                 dodie "failed to copy update to $output_minconfig";
3736         }
3737
3738         doprint "Reboot and wait $sleep_time seconds\n";
3739         reboot_to_good $sleep_time;
3740     }
3741
3742     success $i;
3743     return 1;
3744 }
3745
3746 sub make_warnings_file {
3747     my ($i) = @_;
3748
3749     if (!defined($warnings_file)) {
3750         dodie "Must define WARNINGS_FILE for make_warnings_file test";
3751     }
3752
3753     if ($build_type eq "nobuild") {
3754         dodie "BUILD_TYPE can not be 'nobuild' for make_warnings_file test";
3755     }
3756
3757     build $build_type or dodie "Failed to build";
3758
3759     open(OUT, ">$warnings_file") or dodie "Can't create $warnings_file";
3760
3761     open(IN, $buildlog) or dodie "Can't open $buildlog";
3762     while (<IN>) {
3763
3764         # Some compilers use UTF-8 extended for quotes
3765         # for distcc heterogeneous systems, this causes issues
3766         s/$utf8_quote/'/g;
3767
3768         if (/$check_build_re/) {
3769             print OUT;
3770         }
3771     }
3772     close(IN);
3773
3774     close(OUT);
3775
3776     success $i;
3777 }
3778
3779 $#ARGV < 1 or die "ktest.pl version: $VERSION\n   usage: ktest.pl config-file\n";
3780
3781 if ($#ARGV == 0) {
3782     $ktest_config = $ARGV[0];
3783     if (! -f $ktest_config) {
3784         print "$ktest_config does not exist.\n";
3785         if (!read_yn "Create it?") {
3786             exit 0;
3787         }
3788     }
3789 } else {
3790     $ktest_config = "ktest.conf";
3791 }
3792
3793 if (! -f $ktest_config) {
3794     $newconfig = 1;
3795     get_test_case;
3796     open(OUT, ">$ktest_config") or die "Can not create $ktest_config";
3797     print OUT << "EOF"
3798 # Generated by ktest.pl
3799 #
3800
3801 # PWD is a ktest.pl variable that will result in the process working
3802 # directory that ktest.pl is executed in.
3803
3804 # THIS_DIR is automatically assigned the PWD of the path that generated
3805 # the config file. It is best to use this variable when assigning other
3806 # directory paths within this directory. This allows you to easily
3807 # move the test cases to other locations or to other machines.
3808 #
3809 THIS_DIR := $variable{"PWD"}
3810
3811 # Define each test with TEST_START
3812 # The config options below it will override the defaults
3813 TEST_START
3814 TEST_TYPE = $default{"TEST_TYPE"}
3815
3816 DEFAULTS
3817 EOF
3818 ;
3819     close(OUT);
3820 }
3821 read_config $ktest_config;
3822
3823 if (defined($opt{"LOG_FILE"})) {
3824     $opt{"LOG_FILE"} = eval_option("LOG_FILE", $opt{"LOG_FILE"}, -1);
3825 }
3826
3827 # Append any configs entered in manually to the config file.
3828 my @new_configs = keys %entered_configs;
3829 if ($#new_configs >= 0) {
3830     print "\nAppending entered in configs to $ktest_config\n";
3831     open(OUT, ">>$ktest_config") or die "Can not append to $ktest_config";
3832     foreach my $config (@new_configs) {
3833         print OUT "$config = $entered_configs{$config}\n";
3834         $opt{$config} = process_variables($entered_configs{$config});
3835     }
3836 }
3837
3838 if ($opt{"CLEAR_LOG"} && defined($opt{"LOG_FILE"})) {
3839     unlink $opt{"LOG_FILE"};
3840 }
3841
3842 doprint "\n\nSTARTING AUTOMATED TESTS\n\n";
3843
3844 for (my $i = 0, my $repeat = 1; $i <= $opt{"NUM_TESTS"}; $i += $repeat) {
3845
3846     if (!$i) {
3847         doprint "DEFAULT OPTIONS:\n";
3848     } else {
3849         doprint "\nTEST $i OPTIONS";
3850         if (defined($repeat_tests{$i})) {
3851             $repeat = $repeat_tests{$i};
3852             doprint " ITERATE $repeat";
3853         }
3854         doprint "\n";
3855     }
3856
3857     foreach my $option (sort keys %opt) {
3858
3859         if ($option =~ /\[(\d+)\]$/) {
3860             next if ($i != $1);
3861         } else {
3862             next if ($i);
3863         }
3864
3865         doprint "$option = $opt{$option}\n";
3866     }
3867 }
3868
3869 sub __set_test_option {
3870     my ($name, $i) = @_;
3871
3872     my $option = "$name\[$i\]";
3873
3874     if (defined($opt{$option})) {
3875         return $opt{$option};
3876     }
3877
3878     foreach my $test (keys %repeat_tests) {
3879         if ($i >= $test &&
3880             $i < $test + $repeat_tests{$test}) {
3881             $option = "$name\[$test\]";
3882             if (defined($opt{$option})) {
3883                 return $opt{$option};
3884             }
3885         }
3886     }
3887
3888     if (defined($opt{$name})) {
3889         return $opt{$name};
3890     }
3891
3892     return undef;
3893 }
3894
3895 sub set_test_option {
3896     my ($name, $i) = @_;
3897
3898     my $option = __set_test_option($name, $i);
3899     return $option if (!defined($option));
3900
3901     return eval_option($name, $option, $i);
3902 }
3903
3904 # First we need to do is the builds
3905 for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) {
3906
3907     # Do not reboot on failing test options
3908     $no_reboot = 1;
3909     $reboot_success = 0;
3910
3911     $have_version = 0;
3912
3913     $iteration = $i;
3914
3915     undef %force_config;
3916
3917     my $makecmd = set_test_option("MAKE_CMD", $i);
3918
3919     # Load all the options into their mapped variable names
3920     foreach my $opt (keys %option_map) {
3921         ${$option_map{$opt}} = set_test_option($opt, $i);
3922     }
3923
3924     $start_minconfig_defined = 1;
3925
3926     # The first test may override the PRE_KTEST option
3927     if (defined($pre_ktest) && $i == 1) {
3928         doprint "\n";
3929         run_command $pre_ktest;
3930     }
3931
3932     # Any test can override the POST_KTEST option
3933     # The last test takes precedence.
3934     if (defined($post_ktest)) {
3935         $final_post_ktest = $post_ktest;
3936     }
3937
3938     if (!defined($start_minconfig)) {
3939         $start_minconfig_defined = 0;
3940         $start_minconfig = $minconfig;
3941     }
3942
3943     chdir $builddir || die "can't change directory to $builddir";
3944
3945     foreach my $dir ($tmpdir, $outputdir) {
3946         if (!-d $dir) {
3947             mkpath($dir) or
3948                 die "can't create $dir";
3949         }
3950     }
3951
3952     $ENV{"SSH_USER"} = $ssh_user;
3953     $ENV{"MACHINE"} = $machine;
3954
3955     $buildlog = "$tmpdir/buildlog-$machine";
3956     $testlog = "$tmpdir/testlog-$machine";
3957     $dmesg = "$tmpdir/dmesg-$machine";
3958     $make = "$makecmd O=$outputdir";
3959     $output_config = "$outputdir/.config";
3960
3961     if (!$buildonly) {
3962         $target = "$ssh_user\@$machine";
3963         if ($reboot_type eq "grub") {
3964             dodie "GRUB_MENU not defined" if (!defined($grub_menu));
3965         } elsif ($reboot_type eq "grub2") {
3966             dodie "GRUB_MENU not defined" if (!defined($grub_menu));
3967             dodie "GRUB_FILE not defined" if (!defined($grub_file));
3968         } elsif ($reboot_type eq "syslinux") {
3969             dodie "SYSLINUX_LABEL not defined" if (!defined($syslinux_label));
3970         }
3971     }
3972
3973     my $run_type = $build_type;
3974     if ($test_type eq "patchcheck") {
3975         $run_type = $patchcheck_type;
3976     } elsif ($test_type eq "bisect") {
3977         $run_type = $bisect_type;
3978     } elsif ($test_type eq "config_bisect") {
3979         $run_type = $config_bisect_type;
3980     } elsif ($test_type eq "make_min_config") {
3981         $run_type = "";
3982     } elsif ($test_type eq "make_warnings_file") {
3983         $run_type = "";
3984     }
3985
3986     # mistake in config file?
3987     if (!defined($run_type)) {
3988         $run_type = "ERROR";
3989     }
3990
3991     my $installme = "";
3992     $installme = " no_install" if ($no_install);
3993
3994     doprint "\n\n";
3995     doprint "RUNNING TEST $i of $opt{NUM_TESTS} with option $test_type $run_type$installme\n\n";
3996
3997     if (defined($pre_test)) {
3998         run_command $pre_test;
3999     }
4000
4001     unlink $dmesg;
4002     unlink $buildlog;
4003     unlink $testlog;
4004
4005     if (defined($addconfig)) {
4006         my $min = $minconfig;
4007         if (!defined($minconfig)) {
4008             $min = "";
4009         }
4010         run_command "cat $addconfig $min > $tmpdir/add_config" or
4011             dodie "Failed to create temp config";
4012         $minconfig = "$tmpdir/add_config";
4013     }
4014
4015     if (defined($checkout)) {
4016         run_command "git checkout $checkout" or
4017             die "failed to checkout $checkout";
4018     }
4019
4020     $no_reboot = 0;
4021
4022     # A test may opt to not reboot the box
4023     if ($reboot_on_success) {
4024         $reboot_success = 1;
4025     }
4026
4027     if ($test_type eq "bisect") {
4028         bisect $i;
4029         next;
4030     } elsif ($test_type eq "config_bisect") {
4031         config_bisect $i;
4032         next;
4033     } elsif ($test_type eq "patchcheck") {
4034         patchcheck $i;
4035         next;
4036     } elsif ($test_type eq "make_min_config") {
4037         make_min_config $i;
4038         next;
4039     } elsif ($test_type eq "make_warnings_file") {
4040         $no_reboot = 1;
4041         make_warnings_file $i;
4042         next;
4043     }
4044
4045     if ($build_type ne "nobuild") {
4046         build $build_type or next;
4047         check_buildlog or next;
4048     }
4049
4050     if ($test_type eq "install") {
4051         get_version;
4052         install;
4053         success $i;
4054         next;
4055     }
4056
4057     if ($test_type ne "build") {
4058         my $failed = 0;
4059         start_monitor_and_boot or $failed = 1;
4060
4061         if (!$failed && $test_type ne "boot" && defined($run_test)) {
4062             do_run_test or $failed = 1;
4063         }
4064         end_monitor;
4065         next if ($failed);
4066     }
4067
4068     success $i;
4069 }
4070
4071 if (defined($final_post_ktest)) {
4072     run_command $final_post_ktest;
4073 }
4074
4075 if ($opt{"POWEROFF_ON_SUCCESS"}) {
4076     halt;
4077 } elsif ($opt{"REBOOT_ON_SUCCESS"} && !do_not_reboot && $reboot_success) {
4078     reboot_to_good;
4079 } elsif (defined($switch_to_good)) {
4080     # still need to get to the good kernel
4081     run_command $switch_to_good;
4082 }
4083
4084
4085 doprint "\n    $successes of $opt{NUM_TESTS} tests were successful\n\n";
4086
4087 exit 0;