tools/testing/ktest/ktest.pl

   1 #!/usr/bin/perl -w
   2 # SPDX-License-Identifier: GPL-2.0-only
   3 #
   4 # Copyright 2010 - Steven Rostedt <srostedt@redhat.com>, Red Hat Inc.
   5 #
   6
   7 use strict;
   8 use IPC::Open2;
   9 use Fcntl qw(F_GETFL F_SETFL O_NONBLOCK);
  10 use File::Path qw(mkpath);
  11 use File::Copy qw(cp);
  12 use FileHandle;
  13 use FindBin;
  14 use IO::Handle;
  15
  16 my $VERSION = "0.2";
  17
  18 $| = 1;
  19
  20 my %opt;
  21 my %repeat_tests;
  22 my %repeats;
  23 my %evals;
  24
  25 #default opts
  26 my %default = (
  27     "MAILER"                    => "sendmail",  # default mailer
  28     "EMAIL_ON_ERROR"            => 1,
  29     "EMAIL_WHEN_FINISHED"       => 1,
  30     "EMAIL_WHEN_CANCELED"       => 0,
  31     "EMAIL_WHEN_STARTED"        => 0,
  32     "NUM_TESTS"                 => 1,
  33     "TEST_TYPE"                 => "build",
  34     "BUILD_TYPE"                => "oldconfig",
  35     "MAKE_CMD"                  => "make",
  36     "CLOSE_CONSOLE_SIGNAL"      => "INT",
  37     "TIMEOUT"                   => 120,
  38     "TMP_DIR"                   => "/tmp/ktest/\${MACHINE}",
  39     "SLEEP_TIME"                => 60,  # sleep time between tests
  40     "BUILD_NOCLEAN"             => 0,
  41     "REBOOT_ON_ERROR"           => 0,
  42     "POWEROFF_ON_ERROR"         => 0,
  43     "REBOOT_ON_SUCCESS"         => 1,
  44     "POWEROFF_ON_SUCCESS"       => 0,
  45     "BUILD_OPTIONS"             => "",
  46     "BISECT_SLEEP_TIME"         => 60,   # sleep time between bisects
  47     "PATCHCHECK_SLEEP_TIME"     => 60, # sleep time between patch checks
  48     "CLEAR_LOG"                 => 0,
  49     "BISECT_MANUAL"             => 0,
  50     "BISECT_SKIP"               => 1,
  51     "BISECT_TRIES"              => 1,
  52     "MIN_CONFIG_TYPE"           => "boot",
  53     "SUCCESS_LINE"              => "login:",
  54     "DETECT_TRIPLE_FAULT"       => 1,
  55     "NO_INSTALL"                => 0,
  56     "BOOTED_TIMEOUT"            => 1,
  57     "DIE_ON_FAILURE"            => 1,
  58     "SSH_EXEC"                  => "ssh \$SSH_USER\@\$MACHINE \$SSH_COMMAND",
  59     "SCP_TO_TARGET"             => "scp \$SRC_FILE \$SSH_USER\@\$MACHINE:\$DST_FILE",
  60     "SCP_TO_TARGET_INSTALL"     => "\${SCP_TO_TARGET}",
  61     "REBOOT"                    => "ssh \$SSH_USER\@\$MACHINE reboot",
  62     "REBOOT_RETURN_CODE"        => 255,
  63     "STOP_AFTER_SUCCESS"        => 10,
  64     "STOP_AFTER_FAILURE"        => 60,
  65     "STOP_TEST_AFTER"           => 600,
  66     "MAX_MONITOR_WAIT"          => 1800,
  67     "GRUB_REBOOT"               => "grub2-reboot",
  68     "GRUB_BLS_GET"              => "grubby --info=ALL",
  69     "SYSLINUX"                  => "extlinux",
  70     "SYSLINUX_PATH"             => "/boot/extlinux",
  71     "CONNECT_TIMEOUT"           => 25,
  72
  73 # required, and we will ask users if they don't have them but we keep the default
  74 # value something that is common.
  75     "REBOOT_TYPE"               => "grub",
  76     "LOCALVERSION"              => "-test",
  77     "SSH_USER"                  => "root",
  78     "BUILD_TARGET"              => "arch/x86/boot/bzImage",
  79     "TARGET_IMAGE"              => "/boot/vmlinuz-test",
  80
  81     "LOG_FILE"                  => undef,
  82     "IGNORE_UNUSED"             => 0,
  83 );
  84
  85 my $test_log_start = 0;
  86
  87 my $ktest_config = "ktest.conf";
  88 my $version;
  89 my $have_version = 0;
  90 my $machine;
  91 my $last_machine;
  92 my $ssh_user;
  93 my $tmpdir;
  94 my $builddir;
  95 my $outputdir;
  96 my $output_config;
  97 my $test_type;
  98 my $build_type;
  99 my $build_options;
 100 my $final_post_ktest;
 101 my $pre_ktest;
 102 my $post_ktest;
 103 my $pre_test;
 104 my $pre_test_die;
 105 my $post_test;
 106 my $pre_build;
 107 my $post_build;
 108 my $pre_build_die;
 109 my $post_build_die;
 110 my $reboot_type;
 111 my $reboot_script;
 112 my $power_cycle;
 113 my $reboot;
 114 my $reboot_return_code;
 115 my $reboot_on_error;
 116 my $switch_to_good;
 117 my $switch_to_test;
 118 my $poweroff_on_error;
 119 my $reboot_on_success;
 120 my $die_on_failure;
 121 my $powercycle_after_reboot;
 122 my $poweroff_after_halt;
 123 my $max_monitor_wait;
 124 my $ssh_exec;
 125 my $scp_to_target;
 126 my $scp_to_target_install;
 127 my $power_off;
 128 my $grub_menu;
 129 my $last_grub_menu;
 130 my $grub_file;
 131 my $grub_number;
 132 my $grub_reboot;
 133 my $grub_bls_get;
 134 my $syslinux;
 135 my $syslinux_path;
 136 my $syslinux_label;
 137 my $target;
 138 my $make;
 139 my $pre_install;
 140 my $post_install;
 141 my $no_install;
 142 my $noclean;
 143 my $minconfig;
 144 my $start_minconfig;
 145 my $start_minconfig_defined;
 146 my $output_minconfig;
 147 my $minconfig_type;
 148 my $use_output_minconfig;
 149 my $warnings_file;
 150 my $ignore_config;
 151 my $ignore_errors;
 152 my $addconfig;
 153 my $in_bisect = 0;
 154 my $bisect_bad_commit = "";
 155 my $reverse_bisect;
 156 my $bisect_manual;
 157 my $bisect_skip;
 158 my $bisect_tries;
 159 my $config_bisect_good;
 160 my $bisect_ret_good;
 161 my $bisect_ret_bad;
 162 my $bisect_ret_skip;
 163 my $bisect_ret_abort;
 164 my $bisect_ret_default;
 165 my $in_patchcheck = 0;
 166 my $run_test;
 167 my $buildlog;
 168 my $testlog;
 169 my $dmesg;
 170 my $monitor_fp;
 171 my $monitor_pid;
 172 my $monitor_cnt = 0;
 173 my $sleep_time;
 174 my $bisect_sleep_time;
 175 my $patchcheck_sleep_time;
 176 my $ignore_warnings;
 177 my $store_failures;
 178 my $store_successes;
 179 my $test_name;
 180 my $timeout;
 181 my $connect_timeout;
 182 my $config_bisect_exec;
 183 my $booted_timeout;
 184 my $detect_triplefault;
 185 my $console;
 186 my $close_console_signal;
 187 my $reboot_success_line;
 188 my $success_line;
 189 my $stop_after_success;
 190 my $stop_after_failure;
 191 my $stop_test_after;
 192 my $build_target;
 193 my $target_image;
 194 my $checkout;
 195 my $localversion;
 196 my $iteration = 0;
 197 my $successes = 0;
 198 my $stty_orig;
 199 my $run_command_status = 0;
 200
 201 my $bisect_good;
 202 my $bisect_bad;
 203 my $bisect_type;
 204 my $bisect_start;
 205 my $bisect_replay;
 206 my $bisect_files;
 207 my $bisect_reverse;
 208 my $bisect_check;
 209
 210 my $config_bisect;
 211 my $config_bisect_type;
 212 my $config_bisect_check;
 213
 214 my $patchcheck_type;
 215 my $patchcheck_start;
 216 my $patchcheck_cherry;
 217 my $patchcheck_end;
 218
 219 my $build_time;
 220 my $install_time;
 221 my $reboot_time;
 222 my $test_time;
 223
 224 my $pwd;
 225 my $dirname = $FindBin::Bin;
 226
 227 my $mailto;
 228 my $mailer;
 229 my $mail_path;
 230 my $mail_max_size;
 231 my $mail_command;
 232 my $email_on_error;
 233 my $email_when_finished;
 234 my $email_when_started;
 235 my $email_when_canceled;
 236
 237 my $script_start_time = localtime();
 238
 239 # set when a test is something other that just building or install
 240 # which would require more options.
 241 my $buildonly = 1;
 242
 243 # tell build not to worry about warnings, even when WARNINGS_FILE is set
 244 my $warnings_ok = 0;
 245
 246 # set when creating a new config
 247 my $newconfig = 0;
 248
 249 my %entered_configs;
 250 my %config_help;
 251 my %variable;
 252
 253 # force_config is the list of configs that we force enabled (or disabled)
 254 # in a .config file. The MIN_CONFIG and ADD_CONFIG configs.
 255 my %force_config;
 256
 257 # do not force reboots on config problems
 258 my $no_reboot = 1;
 259
 260 # reboot on success
 261 my $reboot_success = 0;
 262
 263 my %option_map = (
 264     "MAILTO"                    => \$mailto,
 265     "MAILER"                    => \$mailer,
 266     "MAIL_PATH"                 => \$mail_path,
 267     "MAIL_MAX_SIZE"             => \$mail_max_size,
 268     "MAIL_COMMAND"              => \$mail_command,
 269     "EMAIL_ON_ERROR"            => \$email_on_error,
 270     "EMAIL_WHEN_FINISHED"       => \$email_when_finished,
 271     "EMAIL_WHEN_STARTED"        => \$email_when_started,
 272     "EMAIL_WHEN_CANCELED"       => \$email_when_canceled,
 273     "MACHINE"                   => \$machine,
 274     "SSH_USER"                  => \$ssh_user,
 275     "TMP_DIR"                   => \$tmpdir,
 276     "OUTPUT_DIR"                => \$outputdir,
 277     "BUILD_DIR"                 => \$builddir,
 278     "TEST_TYPE"                 => \$test_type,
 279     "PRE_KTEST"                 => \$pre_ktest,
 280     "POST_KTEST"                => \$post_ktest,
 281     "PRE_TEST"                  => \$pre_test,
 282     "PRE_TEST_DIE"              => \$pre_test_die,
 283     "POST_TEST"                 => \$post_test,
 284     "BUILD_TYPE"                => \$build_type,
 285     "BUILD_OPTIONS"             => \$build_options,
 286     "PRE_BUILD"                 => \$pre_build,
 287     "POST_BUILD"                => \$post_build,
 288     "PRE_BUILD_DIE"             => \$pre_build_die,
 289     "POST_BUILD_DIE"            => \$post_build_die,
 290     "POWER_CYCLE"               => \$power_cycle,
 291     "REBOOT"                    => \$reboot,
 292     "REBOOT_RETURN_CODE"        => \$reboot_return_code,
 293     "BUILD_NOCLEAN"             => \$noclean,
 294     "MIN_CONFIG"                => \$minconfig,
 295     "OUTPUT_MIN_CONFIG"         => \$output_minconfig,
 296     "START_MIN_CONFIG"          => \$start_minconfig,
 297     "MIN_CONFIG_TYPE"           => \$minconfig_type,
 298     "USE_OUTPUT_MIN_CONFIG"     => \$use_output_minconfig,
 299     "WARNINGS_FILE"             => \$warnings_file,
 300     "IGNORE_CONFIG"             => \$ignore_config,
 301     "TEST"                      => \$run_test,
 302     "ADD_CONFIG"                => \$addconfig,
 303     "REBOOT_TYPE"               => \$reboot_type,
 304     "GRUB_MENU"                 => \$grub_menu,
 305     "GRUB_FILE"                 => \$grub_file,
 306     "GRUB_REBOOT"               => \$grub_reboot,
 307     "GRUB_BLS_GET"              => \$grub_bls_get,
 308     "SYSLINUX"                  => \$syslinux,
 309     "SYSLINUX_PATH"             => \$syslinux_path,
 310     "SYSLINUX_LABEL"            => \$syslinux_label,
 311     "PRE_INSTALL"               => \$pre_install,
 312     "POST_INSTALL"              => \$post_install,
 313     "NO_INSTALL"                => \$no_install,
 314     "REBOOT_SCRIPT"             => \$reboot_script,
 315     "REBOOT_ON_ERROR"           => \$reboot_on_error,
 316     "SWITCH_TO_GOOD"            => \$switch_to_good,
 317     "SWITCH_TO_TEST"            => \$switch_to_test,
 318     "POWEROFF_ON_ERROR"         => \$poweroff_on_error,
 319     "REBOOT_ON_SUCCESS"         => \$reboot_on_success,
 320     "DIE_ON_FAILURE"            => \$die_on_failure,
 321     "POWER_OFF"                 => \$power_off,
 322     "POWERCYCLE_AFTER_REBOOT"   => \$powercycle_after_reboot,
 323     "POWEROFF_AFTER_HALT"       => \$poweroff_after_halt,
 324     "MAX_MONITOR_WAIT"          => \$max_monitor_wait,
 325     "SLEEP_TIME"                => \$sleep_time,
 326     "BISECT_SLEEP_TIME"         => \$bisect_sleep_time,
 327     "PATCHCHECK_SLEEP_TIME"     => \$patchcheck_sleep_time,
 328     "IGNORE_WARNINGS"           => \$ignore_warnings,
 329     "IGNORE_ERRORS"             => \$ignore_errors,
 330     "BISECT_MANUAL"             => \$bisect_manual,
 331     "BISECT_SKIP"               => \$bisect_skip,
 332     "BISECT_TRIES"              => \$bisect_tries,
 333     "CONFIG_BISECT_GOOD"        => \$config_bisect_good,
 334     "BISECT_RET_GOOD"           => \$bisect_ret_good,
 335     "BISECT_RET_BAD"            => \$bisect_ret_bad,
 336     "BISECT_RET_SKIP"           => \$bisect_ret_skip,
 337     "BISECT_RET_ABORT"          => \$bisect_ret_abort,
 338     "BISECT_RET_DEFAULT"        => \$bisect_ret_default,
 339     "STORE_FAILURES"            => \$store_failures,
 340     "STORE_SUCCESSES"           => \$store_successes,
 341     "TEST_NAME"                 => \$test_name,
 342     "TIMEOUT"                   => \$timeout,
 343     "CONNECT_TIMEOUT"           => \$connect_timeout,
 344     "CONFIG_BISECT_EXEC"        => \$config_bisect_exec,
 345     "BOOTED_TIMEOUT"            => \$booted_timeout,
 346     "CONSOLE"                   => \$console,
 347     "CLOSE_CONSOLE_SIGNAL"      => \$close_console_signal,
 348     "DETECT_TRIPLE_FAULT"       => \$detect_triplefault,
 349     "SUCCESS_LINE"              => \$success_line,
 350     "REBOOT_SUCCESS_LINE"       => \$reboot_success_line,
 351     "STOP_AFTER_SUCCESS"        => \$stop_after_success,
 352     "STOP_AFTER_FAILURE"        => \$stop_after_failure,
 353     "STOP_TEST_AFTER"           => \$stop_test_after,
 354     "BUILD_TARGET"              => \$build_target,
 355     "SSH_EXEC"                  => \$ssh_exec,
 356     "SCP_TO_TARGET"             => \$scp_to_target,
 357     "SCP_TO_TARGET_INSTALL"     => \$scp_to_target_install,
 358     "CHECKOUT"                  => \$checkout,
 359     "TARGET_IMAGE"              => \$target_image,
 360     "LOCALVERSION"              => \$localversion,
 361
 362     "BISECT_GOOD"               => \$bisect_good,
 363     "BISECT_BAD"                => \$bisect_bad,
 364     "BISECT_TYPE"               => \$bisect_type,
 365     "BISECT_START"              => \$bisect_start,
 366     "BISECT_REPLAY"             => \$bisect_replay,
 367     "BISECT_FILES"              => \$bisect_files,
 368     "BISECT_REVERSE"            => \$bisect_reverse,
 369     "BISECT_CHECK"              => \$bisect_check,
 370
 371     "CONFIG_BISECT"             => \$config_bisect,
 372     "CONFIG_BISECT_TYPE"        => \$config_bisect_type,
 373     "CONFIG_BISECT_CHECK"       => \$config_bisect_check,
 374
 375     "PATCHCHECK_TYPE"           => \$patchcheck_type,
 376     "PATCHCHECK_START"          => \$patchcheck_start,
 377     "PATCHCHECK_CHERRY"         => \$patchcheck_cherry,
 378     "PATCHCHECK_END"            => \$patchcheck_end,
 379 );
 380
 381 # Options may be used by other options, record them.
 382 my %used_options;
 383
 384 # default variables that can be used
 385 chomp ($variable{"PWD"} = `pwd`);
 386 $pwd = $variable{"PWD"};
 387
 388 $config_help{"MACHINE"} = << "EOF"
 389  The machine hostname that you will test.
 390  For build only tests, it is still needed to differentiate log files.
 391 EOF
 392     ;
 393 $config_help{"SSH_USER"} = << "EOF"
 394  The box is expected to have ssh on normal bootup, provide the user
 395   (most likely root, since you need privileged operations)
 396 EOF
 397     ;
 398 $config_help{"BUILD_DIR"} = << "EOF"
 399  The directory that contains the Linux source code (full path).
 400  You can use \${PWD} that will be the path where ktest.pl is run, or use
 401  \${THIS_DIR} which is assigned \${PWD} but may be changed later.
 402 EOF
 403     ;
 404 $config_help{"OUTPUT_DIR"} = << "EOF"
 405  The directory that the objects will be built (full path).
 406  (can not be same as BUILD_DIR)
 407  You can use \${PWD} that will be the path where ktest.pl is run, or use
 408  \${THIS_DIR} which is assigned \${PWD} but may be changed later.
 409 EOF
 410     ;
 411 $config_help{"BUILD_TARGET"} = << "EOF"
 412  The location of the compiled file to copy to the target.
 413  (relative to OUTPUT_DIR)
 414 EOF
 415     ;
 416 $config_help{"BUILD_OPTIONS"} = << "EOF"
 417  Options to add to \"make\" when building.
 418  i.e.  -j20
 419 EOF
 420     ;
 421 $config_help{"TARGET_IMAGE"} = << "EOF"
 422  The place to put your image on the test machine.
 423 EOF
 424     ;
 425 $config_help{"POWER_CYCLE"} = << "EOF"
 426  A script or command to reboot the box.
 427
 428  Here is a digital loggers power switch example
 429  POWER_CYCLE = wget --no-proxy -O /dev/null -q  --auth-no-challenge 'http://admin:admin\@power/outlet?5=CCL'
 430
 431  Here is an example to reboot a virtual box on the current host
 432  with the name "Guest".
 433  POWER_CYCLE = virsh destroy Guest; sleep 5; virsh start Guest
 434 EOF
 435     ;
 436 $config_help{"CONSOLE"} = << "EOF"
 437  The script or command that reads the console
 438
 439   If you use ttywatch server, something like the following would work.
 440 CONSOLE = nc -d localhost 3001
 441
 442  For a virtual machine with guest name "Guest".
 443 CONSOLE =  virsh console Guest
 444 EOF
 445     ;
 446 $config_help{"LOCALVERSION"} = << "EOF"
 447  Required version ending to differentiate the test
 448  from other linux builds on the system.
 449 EOF
 450     ;
 451 $config_help{"REBOOT_TYPE"} = << "EOF"
 452  Way to reboot the box to the test kernel.
 453  Only valid options so far are "grub", "grub2", "grub2bls", "syslinux", and "script".
 454
 455  If you specify grub, it will assume grub version 1
 456  and will search in /boot/grub/menu.lst for the title \$GRUB_MENU
 457  and select that target to reboot to the kernel. If this is not
 458  your setup, then specify "script" and have a command or script
 459  specified in REBOOT_SCRIPT to boot to the target.
 460
 461  The entry in /boot/grub/menu.lst must be entered in manually.
 462  The test will not modify that file.
 463
 464  If you specify grub2, then you also need to specify both \$GRUB_MENU
 465  and \$GRUB_FILE.
 466
 467  If you specify grub2bls, then you also need to specify \$GRUB_MENU.
 468
 469  If you specify syslinux, then you may use SYSLINUX to define the syslinux
 470  command (defaults to extlinux), and SYSLINUX_PATH to specify the path to
 471  the syslinux install (defaults to /boot/extlinux). But you have to specify
 472  SYSLINUX_LABEL to define the label to boot to for the test kernel.
 473 EOF
 474     ;
 475 $config_help{"GRUB_MENU"} = << "EOF"
 476  The grub title name for the test kernel to boot
 477  (Only mandatory if REBOOT_TYPE = grub or grub2)
 478
 479  Note, ktest.pl will not update the grub menu.lst, you need to
 480  manually add an option for the test. ktest.pl will search
 481  the grub menu.lst for this option to find what kernel to
 482  reboot into.
 483
 484  For example, if in the /boot/grub/menu.lst the test kernel title has:
 485  title Test Kernel
 486  kernel vmlinuz-test
 487  GRUB_MENU = Test Kernel
 488
 489  For grub2, a search of \$GRUB_FILE is performed for the lines
 490  that begin with "menuentry". It will not detect submenus. The
 491  menu must be a non-nested menu. Add the quotes used in the menu
 492  to guarantee your selection, as the first menuentry with the content
 493  of \$GRUB_MENU that is found will be used.
 494
 495  For grub2bls, \$GRUB_MENU is searched on the result of \$GRUB_BLS_GET
 496  command for the lines that begin with "title".
 497 EOF
 498     ;
 499 $config_help{"GRUB_FILE"} = << "EOF"
 500  If grub2 is used, the full path for the grub.cfg file is placed
 501  here. Use something like /boot/grub2/grub.cfg to search.
 502 EOF
 503     ;
 504 $config_help{"SYSLINUX_LABEL"} = << "EOF"
 505  If syslinux is used, the label that boots the target kernel must
 506  be specified with SYSLINUX_LABEL.
 507 EOF
 508     ;
 509 $config_help{"REBOOT_SCRIPT"} = << "EOF"
 510  A script to reboot the target into the test kernel
 511  (Only mandatory if REBOOT_TYPE = script)
 512 EOF
 513     ;
 514
 515 sub _logit {
 516     if (defined($opt{"LOG_FILE"})) {
 517         print LOG @_;
 518     }
 519 }
 520
 521 sub logit {
 522     if (defined($opt{"LOG_FILE"})) {
 523         _logit @_;
 524     } else {
 525         print @_;
 526     }
 527 }
 528
 529 sub doprint {
 530     print @_;
 531     _logit @_;
 532 }
 533
 534 sub read_prompt {
 535     my ($cancel, $prompt) = @_;
 536
 537     my $ans;
 538
 539     for (;;) {
 540         if ($cancel) {
 541             print "$prompt [y/n/C] ";
 542         } else {
 543             print "$prompt [Y/n] ";
 544         }
 545         $ans = <STDIN>;
 546         chomp $ans;
 547         if ($ans =~ /^\s*$/) {
 548             if ($cancel) {
 549                 $ans = "c";
 550             } else {
 551                 $ans = "y";
 552             }
 553         }
 554         last if ($ans =~ /^y$/i || $ans =~ /^n$/i);
 555         if ($cancel) {
 556             last if ($ans =~ /^c$/i);
 557             print "Please answer either 'y', 'n' or 'c'.\n";
 558         } else {
 559             print "Please answer either 'y' or 'n'.\n";
 560         }
 561     }
 562     if ($ans =~ /^c/i) {
 563         exit;
 564     }
 565     if ($ans !~ /^y$/i) {
 566         return 0;
 567     }
 568     return 1;
 569 }
 570
 571 sub read_yn {
 572     my ($prompt) = @_;
 573
 574     return read_prompt 0, $prompt;
 575 }
 576
 577 sub read_ync {
 578     my ($prompt) = @_;
 579
 580     return read_prompt 1, $prompt;
 581 }
 582
 583 sub get_mandatory_config {
 584     my ($config) = @_;
 585     my $ans;
 586
 587     return if (defined($opt{$config}));
 588
 589     if (defined($config_help{$config})) {
 590         print "\n";
 591         print $config_help{$config};
 592     }
 593
 594     for (;;) {
 595         print "$config = ";
 596         if (defined($default{$config}) && length($default{$config})) {
 597             print "\[$default{$config}\] ";
 598         }
 599         $ans = <STDIN>;
 600         $ans =~ s/^\s*(.*\S)\s*$/$1/;
 601         if ($ans =~ /^\s*$/) {
 602             if ($default{$config}) {
 603                 $ans = $default{$config};
 604             } else {
 605                 print "Your answer can not be blank\n";
 606                 next;
 607             }
 608         }
 609         $entered_configs{$config} = ${ans};
 610         last;
 611     }
 612 }
 613
 614 sub show_time {
 615     my ($time) = @_;
 616
 617     my $hours = 0;
 618     my $minutes = 0;
 619
 620     if ($time > 3600) {
 621         $hours = int($time / 3600);
 622         $time -= $hours * 3600;
 623     }
 624     if ($time > 60) {
 625         $minutes = int($time / 60);
 626         $time -= $minutes * 60;
 627     }
 628
 629     if ($hours > 0) {
 630         doprint "$hours hour";
 631         doprint "s" if ($hours > 1);
 632         doprint " ";
 633     }
 634
 635     if ($minutes > 0) {
 636         doprint "$minutes minute";
 637         doprint "s" if ($minutes > 1);
 638         doprint " ";
 639     }
 640
 641     doprint "$time second";
 642     doprint "s" if ($time != 1);
 643 }
 644
 645 sub print_times {
 646     doprint "\n";
 647     if ($build_time) {
 648         doprint "Build time:   ";
 649         show_time($build_time);
 650         doprint "\n";
 651     }
 652     if ($install_time) {
 653         doprint "Install time: ";
 654         show_time($install_time);
 655         doprint "\n";
 656     }
 657     if ($reboot_time) {
 658         doprint "Reboot time:  ";
 659         show_time($reboot_time);
 660         doprint "\n";
 661     }
 662     if ($test_time) {
 663         doprint "Test time:    ";
 664         show_time($test_time);
 665         doprint "\n";
 666     }
 667     # reset for iterations like bisect
 668     $build_time = 0;
 669     $install_time = 0;
 670     $reboot_time = 0;
 671     $test_time = 0;
 672 }
 673
 674 sub get_mandatory_configs {
 675     get_mandatory_config("MACHINE");
 676     get_mandatory_config("BUILD_DIR");
 677     get_mandatory_config("OUTPUT_DIR");
 678
 679     if ($newconfig) {
 680         get_mandatory_config("BUILD_OPTIONS");
 681     }
 682
 683     # options required for other than just building a kernel
 684     if (!$buildonly) {
 685         get_mandatory_config("POWER_CYCLE");
 686         get_mandatory_config("CONSOLE");
 687     }
 688
 689     # options required for install and more
 690     if ($buildonly != 1) {
 691         get_mandatory_config("SSH_USER");
 692         get_mandatory_config("BUILD_TARGET");
 693         get_mandatory_config("TARGET_IMAGE");
 694     }
 695
 696     get_mandatory_config("LOCALVERSION");
 697
 698     return if ($buildonly);
 699
 700     my $rtype = $opt{"REBOOT_TYPE"};
 701
 702     if (!defined($rtype)) {
 703         if (!defined($opt{"GRUB_MENU"})) {
 704             get_mandatory_config("REBOOT_TYPE");
 705             $rtype = $entered_configs{"REBOOT_TYPE"};
 706         } else {
 707             $rtype = "grub";
 708         }
 709     }
 710
 711     if (($rtype eq "grub") or ($rtype eq "grub2bls")) {
 712         get_mandatory_config("GRUB_MENU");
 713     }
 714
 715     if ($rtype eq "grub2") {
 716         get_mandatory_config("GRUB_MENU");
 717         get_mandatory_config("GRUB_FILE");
 718     }
 719
 720     if ($rtype eq "syslinux") {
 721         get_mandatory_config("SYSLINUX_LABEL");
 722     }
 723 }
 724
 725 sub process_variables {
 726     my ($value, $remove_undef) = @_;
 727     my $retval = "";
 728
 729     # We want to check for '\', and it is just easier
 730     # to check the previous characet of '$' and not need
 731     # to worry if '$' is the first character. By adding
 732     # a space to $value, we can just check [^\\]\$ and
 733     # it will still work.
 734     $value = " $value";
 735
 736     while ($value =~ /(.*?[^\\])\$\{(.*?)\}(.*)/) {
 737         my $begin = $1;
 738         my $var = $2;
 739         my $end = $3;
 740         # append beginning of value to retval
 741         $retval = "$retval$begin";
 742         if (defined($variable{$var})) {
 743             $retval = "$retval$variable{$var}";
 744         } elsif (defined($remove_undef) && $remove_undef) {
 745             # for if statements, any variable that is not defined,
 746             # we simple convert to 0
 747             $retval = "${retval}0";
 748         } else {
 749             # put back the origin piece.
 750             $retval = "$retval\$\{$var\}";
 751             # This could be an option that is used later, save
 752             # it so we don't warn if this option is not one of
 753             # ktests options.
 754             $used_options{$var} = 1;
 755         }
 756         $value = $end;
 757     }
 758     $retval = "$retval$value";
 759
 760     # remove the space added in the beginning
 761     $retval =~ s/ //;
 762
 763     return "$retval"
 764 }
 765
 766 sub set_value {
 767     my ($lvalue, $rvalue, $override, $overrides, $name) = @_;
 768
 769     my $prvalue = process_variables($rvalue);
 770
 771     if ($lvalue =~ /^(TEST|BISECT|CONFIG_BISECT)_TYPE(\[.*\])?$/ &&
 772         $prvalue !~ /^(config_|)bisect$/ &&
 773         $prvalue !~ /^build$/ &&
 774         $buildonly) {
 775
 776         # Note if a test is something other than build, then we
 777         # will need other mandatory options.
 778         if ($prvalue ne "install") {
 779             $buildonly = 0;
 780         } else {
 781             # install still limits some mandatory options.
 782             $buildonly = 2;
 783         }
 784     }
 785
 786     if (defined($opt{$lvalue})) {
 787         if (!$override || defined(${$overrides}{$lvalue})) {
 788             my $extra = "";
 789             if ($override) {
 790                 $extra = "In the same override section!\n";
 791             }
 792             die "$name: $.: Option $lvalue defined more than once!\n$extra";
 793         }
 794         ${$overrides}{$lvalue} = $prvalue;
 795     }
 796
 797     $opt{$lvalue} = $prvalue;
 798 }
 799
 800 sub set_eval {
 801     my ($lvalue, $rvalue, $name) = @_;
 802
 803     my $prvalue = process_variables($rvalue);
 804     my $arr;
 805
 806     if (defined($evals{$lvalue})) {
 807         $arr = $evals{$lvalue};
 808     } else {
 809         $arr = [];
 810         $evals{$lvalue} = $arr;
 811     }
 812
 813     push @{$arr}, $rvalue;
 814 }
 815
 816 sub set_variable {
 817     my ($lvalue, $rvalue) = @_;
 818
 819     if ($rvalue =~ /^\s*$/) {
 820         delete $variable{$lvalue};
 821     } else {
 822         $rvalue = process_variables($rvalue);
 823         $variable{$lvalue} = $rvalue;
 824     }
 825 }
 826
 827 sub process_compare {
 828     my ($lval, $cmp, $rval) = @_;
 829
 830     # remove whitespace
 831
 832     $lval =~ s/^\s*//;
 833     $lval =~ s/\s*$//;
 834
 835     $rval =~ s/^\s*//;
 836     $rval =~ s/\s*$//;
 837
 838     if ($cmp eq "==") {
 839         return $lval eq $rval;
 840     } elsif ($cmp eq "!=") {
 841         return $lval ne $rval;
 842     } elsif ($cmp eq "=~") {
 843         return $lval =~ m/$rval/;
 844     } elsif ($cmp eq "!~") {
 845         return $lval !~ m/$rval/;
 846     }
 847
 848     my $statement = "$lval $cmp $rval";
 849     my $ret = eval $statement;
 850
 851     # $@ stores error of eval
 852     if ($@) {
 853         return -1;
 854     }
 855
 856     return $ret;
 857 }
 858
 859 sub value_defined {
 860     my ($val) = @_;
 861
 862     return defined($variable{$2}) ||
 863         defined($opt{$2});
 864 }
 865
 866 my $d = 0;
 867 sub process_expression {
 868     my ($name, $val) = @_;
 869
 870     my $c = $d++;
 871
 872     while ($val =~ s/\(([^\(]*?)\)/\&\&\&\&VAL\&\&\&\&/) {
 873         my $express = $1;
 874
 875         if (process_expression($name, $express)) {
 876             $val =~ s/\&\&\&\&VAL\&\&\&\&/ 1 /;
 877         } else {
 878             $val =~ s/\&\&\&\&VAL\&\&\&\&/ 0 /;
 879         }
 880     }
 881
 882     $d--;
 883     my $OR = "\\|\\|";
 884     my $AND = "\\&\\&";
 885
 886     while ($val =~ s/^(.*?)($OR|$AND)//) {
 887         my $express = $1;
 888         my $op = $2;
 889
 890         if (process_expression($name, $express)) {
 891             if ($op eq "||") {
 892                 return 1;
 893             }
 894         } else {
 895             if ($op eq "&&") {
 896                 return 0;
 897             }
 898         }
 899     }
 900
 901     if ($val =~ /(.*)(==|\!=|>=|<=|>|<|=~|\!~)(.*)/) {
 902         my $ret = process_compare($1, $2, $3);
 903         if ($ret < 0) {
 904             die "$name: $.: Unable to process comparison\n";
 905         }
 906         return $ret;
 907     }
 908
 909     if ($val =~ /^\s*(NOT\s*)?DEFINED\s+(\S+)\s*$/) {
 910         if (defined $1) {
 911             return !value_defined($2);
 912         } else {
 913             return value_defined($2);
 914         }
 915     }
 916
 917     if ($val =~ s/^\s*NOT\s+(.*)//) {
 918         my $express = $1;
 919         my $ret = process_expression($name, $express);
 920         return !$ret;
 921     }
 922
 923     if ($val =~ /^\s*0\s*$/) {
 924         return 0;
 925     } elsif ($val =~ /^\s*\d+\s*$/) {
 926         return 1;
 927     }
 928
 929     die ("$name: $.: Undefined content $val in if statement\n");
 930 }
 931
 932 sub process_if {
 933     my ($name, $value) = @_;
 934
 935     # Convert variables and replace undefined ones with 0
 936     my $val = process_variables($value, 1);
 937     my $ret = process_expression $name, $val;
 938
 939     return $ret;
 940 }
 941
 942 sub __read_config {
 943     my ($config, $current_test_num) = @_;
 944
 945     my $in;
 946     open($in, $config) || die "can't read file $config";
 947
 948     my $name = $config;
 949     $name =~ s,.*/(.*),$1,;
 950
 951     my $test_num = $$current_test_num;
 952     my $default = 1;
 953     my $repeat = 1;
 954     my $num_tests_set = 0;
 955     my $skip = 0;
 956     my $rest;
 957     my $line;
 958     my $test_case = 0;
 959     my $if = 0;
 960     my $if_set = 0;
 961     my $override = 0;
 962
 963     my %overrides;
 964
 965     while (<$in>) {
 966
 967         # ignore blank lines and comments
 968         next if (/^\s*$/ || /\s*\#/);
 969
 970         if (/^\s*(TEST_START|DEFAULTS)\b(.*)/) {
 971
 972             my $type = $1;
 973             $rest = $2;
 974             $line = $2;
 975
 976             my $old_test_num;
 977             my $old_repeat;
 978             $override = 0;
 979
 980             if ($type eq "TEST_START") {
 981
 982                 if ($num_tests_set) {
 983                     die "$name: $.: Can not specify both NUM_TESTS and TEST_START\n";
 984                 }
 985
 986                 $old_test_num = $test_num;
 987                 $old_repeat = $repeat;
 988
 989                 $test_num += $repeat;
 990                 $default = 0;
 991                 $repeat = 1;
 992             } else {
 993                 $default = 1;
 994             }
 995
 996             # If SKIP is anywhere in the line, the command will be skipped
 997             if ($rest =~ s/\s+SKIP\b//) {
 998                 $skip = 1;
 999             } else {
1000                 $test_case = 1;
1001                 $skip = 0;
1002             }
1003
1004             if ($rest =~ s/\sELSE\b//) {
1005                 if (!$if) {
1006                     die "$name: $.: ELSE found with out matching IF section\n$_";
1007                 }
1008                 $if = 0;
1009
1010                 if ($if_set) {
1011                     $skip = 1;
1012                 } else {
1013                     $skip = 0;
1014                 }
1015             }
1016
1017             if ($rest =~ s/\sIF\s+(.*)//) {
1018                 if (process_if($name, $1)) {
1019                     $if_set = 1;
1020                 } else {
1021                     $skip = 1;
1022                 }
1023                 $if = 1;
1024             } else {
1025                 $if = 0;
1026                 $if_set = 0;
1027             }
1028
1029             if (!$skip) {
1030                 if ($type eq "TEST_START") {
1031                     if ($rest =~ s/\s+ITERATE\s+(\d+)//) {
1032                         $repeat = $1;
1033                         $repeat_tests{"$test_num"} = $repeat;
1034                     }
1035                 } elsif ($rest =~ s/\sOVERRIDE\b//) {
1036                     # DEFAULT only
1037                     $override = 1;
1038                     # Clear previous overrides
1039                     %overrides = ();
1040                 }
1041             }
1042
1043             if (!$skip && $rest !~ /^\s*$/) {
1044                 die "$name: $.: Garbage found after $type\n$_";
1045             }
1046
1047             if ($skip && $type eq "TEST_START") {
1048                 $test_num = $old_test_num;
1049                 $repeat = $old_repeat;
1050             }
1051
1052         } elsif (/^\s*ELSE\b(.*)$/) {
1053             if (!$if) {
1054                 die "$name: $.: ELSE found with out matching IF section\n$_";
1055             }
1056             $rest = $1;
1057             if ($if_set) {
1058                 $skip = 1;
1059                 $rest = "";
1060             } else {
1061                 $skip = 0;
1062
1063                 if ($rest =~ /\sIF\s+(.*)/) {
1064                     # May be a ELSE IF section.
1065                     if (process_if($name, $1)) {
1066                         $if_set = 1;
1067                     } else {
1068                         $skip = 1;
1069                     }
1070                     $rest = "";
1071                 } else {
1072                     $if = 0;
1073                 }
1074             }
1075
1076             if ($rest !~ /^\s*$/) {
1077                 die "$name: $.: Garbage found after DEFAULTS\n$_";
1078             }
1079
1080         } elsif (/^\s*INCLUDE\s+(\S+)/) {
1081
1082             next if ($skip);
1083
1084             if (!$default) {
1085                 die "$name: $.: INCLUDE can only be done in default sections\n$_";
1086             }
1087
1088             my $file = process_variables($1);
1089
1090             if ($file !~ m,^/,) {
1091                 # check the path of the config file first
1092                 if ($config =~ m,(.*)/,) {
1093                     if (-f "$1/$file") {
1094                         $file = "$1/$file";
1095                     }
1096                 }
1097             }
1098
1099             if ( ! -r $file ) {
1100                 die "$name: $.: Can't read file $file\n$_";
1101             }
1102
1103             if (__read_config($file, \$test_num)) {
1104                 $test_case = 1;
1105             }
1106
1107         } elsif (/^\s*([A-Z_\[\]\d]+)\s*=~\s*(.*?)\s*$/) {
1108
1109             next if ($skip);
1110
1111             my $lvalue = $1;
1112             my $rvalue = $2;
1113
1114             if ($default || $lvalue =~ /\[\d+\]$/) {
1115                 set_eval($lvalue, $rvalue, $name);
1116             } else {
1117                 my $val = "$lvalue\[$test_num\]";
1118                 set_eval($val, $rvalue, $name);
1119             }
1120
1121         } elsif (/^\s*([A-Z_\[\]\d]+)\s*=\s*(.*?)\s*$/) {
1122
1123             next if ($skip);
1124
1125             my $lvalue = $1;
1126             my $rvalue = $2;
1127
1128             if (!$default &&
1129                 ($lvalue eq "NUM_TESTS" ||
1130                  $lvalue eq "LOG_FILE" ||
1131                  $lvalue eq "CLEAR_LOG")) {
1132                 die "$name: $.: $lvalue must be set in DEFAULTS section\n";
1133             }
1134
1135             if ($lvalue eq "NUM_TESTS") {
1136                 if ($test_num) {
1137                     die "$name: $.: Can not specify both NUM_TESTS and TEST_START\n";
1138                 }
1139                 if (!$default) {
1140                     die "$name: $.: NUM_TESTS must be set in default section\n";
1141                 }
1142                 $num_tests_set = 1;
1143             }
1144
1145             if ($default || $lvalue =~ /\[\d+\]$/) {
1146                 set_value($lvalue, $rvalue, $override, \%overrides, $name);
1147             } else {
1148                 my $val = "$lvalue\[$test_num\]";
1149                 set_value($val, $rvalue, $override, \%overrides, $name);
1150
1151                 if ($repeat > 1) {
1152                     $repeats{$val} = $repeat;
1153                 }
1154             }
1155         } elsif (/^\s*([A-Z_\[\]\d]+)\s*:=\s*(.*?)\s*$/) {
1156             next if ($skip);
1157
1158             my $lvalue = $1;
1159             my $rvalue = $2;
1160
1161             # process config variables.
1162             # Config variables are only active while reading the
1163             # config and can be defined anywhere. They also ignore
1164             # TEST_START and DEFAULTS, but are skipped if they are in
1165             # on of these sections that have SKIP defined.
1166             # The save variable can be
1167             # defined multiple times and the new one simply overrides
1168             # the previous one.
1169             set_variable($lvalue, $rvalue);
1170
1171         } else {
1172             die "$name: $.: Garbage found in config\n$_";
1173         }
1174     }
1175
1176     if ($test_num) {
1177         $test_num += $repeat - 1;
1178         $opt{"NUM_TESTS"} = $test_num;
1179     }
1180
1181     close($in);
1182
1183     $$current_test_num = $test_num;
1184
1185     return $test_case;
1186 }
1187
1188 sub get_test_case {
1189         print "What test case would you like to run?\n";
1190         print " (build, install or boot)\n";
1191         print " Other tests are available but require editing ktest.conf\n";
1192         print " (see tools/testing/ktest/sample.conf)\n";
1193         my $ans = <STDIN>;
1194         chomp $ans;
1195         $default{"TEST_TYPE"} = $ans;
1196 }
1197
1198 sub read_config {
1199     my ($config) = @_;
1200
1201     my $test_case;
1202     my $test_num = 0;
1203
1204     $test_case = __read_config $config, \$test_num;
1205
1206     # make sure we have all mandatory configs
1207     get_mandatory_configs;
1208
1209     # was a test specified?
1210     if (!$test_case) {
1211         print "No test case specified.\n";
1212         get_test_case;
1213     }
1214
1215     # set any defaults
1216
1217     foreach my $default (keys %default) {
1218         if (!defined($opt{$default})) {
1219             $opt{$default} = $default{$default};
1220         }
1221     }
1222
1223     if ($opt{"IGNORE_UNUSED"} == 1) {
1224         return;
1225     }
1226
1227     my %not_used;
1228
1229     # check if there are any stragglers (typos?)
1230     foreach my $option (keys %opt) {
1231         my $op = $option;
1232         # remove per test labels.
1233         $op =~ s/\[.*\]//;
1234         if (!exists($option_map{$op}) &&
1235             !exists($default{$op}) &&
1236             !exists($used_options{$op})) {
1237             $not_used{$op} = 1;
1238         }
1239     }
1240
1241     if (%not_used) {
1242         my $s = "s are";
1243         $s = " is" if (keys %not_used == 1);
1244         print "The following option$s not used; could be a typo:\n";
1245         foreach my $option (keys %not_used) {
1246             print "$option\n";
1247         }
1248         print "Set IGNORE_UNUSED = 1 to have ktest ignore unused variables\n";
1249         if (!read_yn "Do you want to continue?") {
1250             exit -1;
1251         }
1252     }
1253 }
1254
1255 sub __eval_option {
1256     my ($name, $option, $i) = @_;
1257
1258     # Add space to evaluate the character before $
1259     $option = " $option";
1260     my $retval = "";
1261     my $repeated = 0;
1262     my $parent = 0;
1263
1264     foreach my $test (keys %repeat_tests) {
1265         if ($i >= $test &&
1266             $i < $test + $repeat_tests{$test}) {
1267
1268             $repeated = 1;
1269             $parent = $test;
1270             last;
1271         }
1272     }
1273
1274     while ($option =~ /(.*?[^\\])\$\{(.*?)\}(.*)/) {
1275         my $start = $1;
1276         my $var = $2;
1277         my $end = $3;
1278
1279         # Append beginning of line
1280         $retval = "$retval$start";
1281
1282         # If the iteration option OPT[$i] exists, then use that.
1283         # otherwise see if the default OPT (without [$i]) exists.
1284
1285         my $o = "$var\[$i\]";
1286         my $parento = "$var\[$parent\]";
1287
1288         # If a variable contains itself, use the default var
1289         if (($var eq $name) && defined($opt{$var})) {
1290             $o = $opt{$var};
1291             $retval = "$retval$o";
1292         } elsif (defined($opt{$o})) {
1293             $o = $opt{$o};
1294             $retval = "$retval$o";
1295         } elsif ($repeated && defined($opt{$parento})) {
1296             $o = $opt{$parento};
1297             $retval = "$retval$o";
1298         } elsif (defined($opt{$var})) {
1299             $o = $opt{$var};
1300             $retval = "$retval$o";
1301         } elsif ($var eq "KERNEL_VERSION" && defined($make)) {
1302             # special option KERNEL_VERSION uses kernel version
1303             get_version();
1304             $retval = "$retval$version";
1305         } else {
1306             $retval = "$retval\$\{$var\}";
1307         }
1308
1309         $option = $end;
1310     }
1311
1312     $retval = "$retval$option";
1313
1314     $retval =~ s/^ //;
1315
1316     return $retval;
1317 }
1318
1319 sub process_evals {
1320     my ($name, $option, $i) = @_;
1321
1322     my $option_name = "$name\[$i\]";
1323     my $ev;
1324
1325     my $old_option = $option;
1326
1327     if (defined($evals{$option_name})) {
1328         $ev = $evals{$option_name};
1329     } elsif (defined($evals{$name})) {
1330         $ev = $evals{$name};
1331     } else {
1332         return $option;
1333     }
1334
1335     for my $e (@{$ev}) {
1336         eval "\$option =~ $e";
1337     }
1338
1339     if ($option ne $old_option) {
1340         doprint("$name changed from '$old_option' to '$option'\n");
1341     }
1342
1343     return $option;
1344 }
1345
1346 sub eval_option {
1347     my ($name, $option, $i) = @_;
1348
1349     my $prev = "";
1350
1351     # Since an option can evaluate to another option,
1352     # keep iterating until we do not evaluate any more
1353     # options.
1354     my $r = 0;
1355     while ($prev ne $option) {
1356         # Check for recursive evaluations.
1357         # 100 deep should be more than enough.
1358         if ($r++ > 100) {
1359             die "Over 100 evaluations occurred with $option\n" .
1360                 "Check for recursive variables\n";
1361         }
1362         $prev = $option;
1363         $option = __eval_option($name, $option, $i);
1364     }
1365
1366     $option = process_evals($name, $option, $i);
1367
1368     return $option;
1369 }
1370
1371 sub run_command;
1372 sub start_monitor;
1373 sub end_monitor;
1374 sub wait_for_monitor;
1375
1376 sub reboot {
1377     my ($time) = @_;
1378     my $powercycle = 0;
1379
1380     # test if the machine can be connected to within a few seconds
1381     my $stat = run_ssh("echo check machine status", $connect_timeout);
1382     if (!$stat) {
1383         doprint("power cycle\n");
1384         $powercycle = 1;
1385     }
1386
1387     if ($powercycle) {
1388         run_command "$power_cycle";
1389
1390         start_monitor;
1391         # flush out current monitor
1392         # May contain the reboot success line
1393         wait_for_monitor 1;
1394
1395     } else {
1396         # Make sure everything has been written to disk
1397         run_ssh("sync", 10);
1398
1399         if (defined($time)) {
1400             start_monitor;
1401             # flush out current monitor
1402             # May contain the reboot success line
1403             wait_for_monitor 1;
1404         }
1405
1406         # try to reboot normally
1407         if (run_command $reboot) {
1408             if (defined($powercycle_after_reboot)) {
1409                 sleep $powercycle_after_reboot;
1410                 run_command "$power_cycle";
1411             }
1412         } else {
1413             # nope? power cycle it.
1414             run_command "$power_cycle";
1415         }
1416     }
1417
1418     if (defined($time)) {
1419
1420         # We only want to get to the new kernel, don't fail
1421         # if we stumble over a call trace.
1422         my $save_ignore_errors = $ignore_errors;
1423         $ignore_errors = 1;
1424
1425         # Look for the good kernel to boot
1426         if (wait_for_monitor($time, "Linux version")) {
1427             # reboot got stuck?
1428             doprint "Reboot did not finish. Forcing power cycle\n";
1429             run_command "$power_cycle";
1430         }
1431
1432         $ignore_errors = $save_ignore_errors;
1433
1434         # Still need to wait for the reboot to finish
1435         wait_for_monitor($time, $reboot_success_line);
1436
1437         end_monitor;
1438     }
1439 }
1440
1441 sub reboot_to_good {
1442     my ($time) = @_;
1443
1444     if (defined($switch_to_good)) {
1445         run_command $switch_to_good;
1446     }
1447
1448     reboot $time;
1449 }
1450
1451 sub do_not_reboot {
1452     my $i = $iteration;
1453
1454     return $test_type eq "build" || $no_reboot ||
1455         ($test_type eq "patchcheck" && $opt{"PATCHCHECK_TYPE[$i]"} eq "build") ||
1456         ($test_type eq "bisect" && $opt{"BISECT_TYPE[$i]"} eq "build") ||
1457         ($test_type eq "config_bisect" && $opt{"CONFIG_BISECT_TYPE[$i]"} eq "build");
1458 }
1459
1460 my $in_die = 0;
1461
1462 sub get_test_name() {
1463     my $name;
1464
1465     if (defined($test_name)) {
1466         $name = "$test_name:$test_type";
1467     } else {
1468         $name = $test_type;
1469     }
1470     return $name;
1471 }
1472
1473 sub dodie {
1474
1475     # avoid recursion
1476     return if ($in_die);
1477     $in_die = 1;
1478
1479     my $i = $iteration;
1480
1481     doprint "CRITICAL FAILURE... [TEST $i] ", @_, "\n";
1482
1483     if ($reboot_on_error && !do_not_reboot) {
1484
1485         doprint "REBOOTING\n";
1486         reboot_to_good;
1487
1488     } elsif ($poweroff_on_error && defined($power_off)) {
1489         doprint "POWERING OFF\n";
1490         `$power_off`;
1491     }
1492
1493     if (defined($opt{"LOG_FILE"})) {
1494         print " See $opt{LOG_FILE} for more info.\n";
1495     }
1496
1497     if ($email_on_error) {
1498         my $name = get_test_name;
1499         my $log_file;
1500
1501         if (defined($opt{"LOG_FILE"})) {
1502             my $whence = 2; # End of file
1503             my $log_size = tell LOG;
1504             my $size = $log_size - $test_log_start;
1505
1506             if (defined($mail_max_size)) {
1507                 if ($size > $mail_max_size) {
1508                     $size = $mail_max_size;
1509                 }
1510             }
1511             my $pos = - $size;
1512             $log_file = "$tmpdir/log";
1513             open (L, "$opt{LOG_FILE}") or die "Can't open $opt{LOG_FILE} to read)";
1514             open (O, "> $tmpdir/log") or die "Can't open $tmpdir/log\n";
1515             seek(L, $pos, $whence);
1516             while (<L>) {
1517                 print O;
1518             }
1519             close O;
1520             close L;
1521         }
1522         send_email("KTEST: critical failure for test $i [$name]",
1523                 "Your test started at $script_start_time has failed with:\n@_\n", $log_file);
1524     }
1525
1526     if ($monitor_cnt) {
1527             # restore terminal settings
1528             system("stty $stty_orig");
1529     }
1530
1531     if (defined($post_test)) {
1532         run_command $post_test;
1533     }
1534
1535     die @_, "\n";
1536 }
1537
1538 sub create_pty {
1539     my ($ptm, $pts) = @_;
1540     my $tmp;
1541     my $TIOCSPTLCK = 0x40045431;
1542     my $TIOCGPTN = 0x80045430;
1543
1544     sysopen($ptm, "/dev/ptmx", O_RDWR | O_NONBLOCK) or
1545         dodie "Can't open /dev/ptmx";
1546
1547     # unlockpt()
1548     $tmp = pack("i", 0);
1549     ioctl($ptm, $TIOCSPTLCK, $tmp) or
1550         dodie "ioctl TIOCSPTLCK for /dev/ptmx failed";
1551
1552     # ptsname()
1553     ioctl($ptm, $TIOCGPTN, $tmp) or
1554         dodie "ioctl TIOCGPTN for /dev/ptmx failed";
1555     $tmp = unpack("i", $tmp);
1556
1557     sysopen($pts, "/dev/pts/$tmp", O_RDWR | O_NONBLOCK) or
1558         dodie "Can't open /dev/pts/$tmp";
1559 }
1560
1561 sub exec_console {
1562     my ($ptm, $pts) = @_;
1563
1564     close($ptm);
1565
1566     close(\*STDIN);
1567     close(\*STDOUT);
1568     close(\*STDERR);
1569
1570     open(\*STDIN, '<&', $pts);
1571     open(\*STDOUT, '>&', $pts);
1572     open(\*STDERR, '>&', $pts);
1573
1574     close($pts);
1575
1576     exec $console or
1577         dodie "Can't open console $console";
1578 }
1579
1580 sub open_console {
1581     my ($ptm) = @_;
1582     my $pts = \*PTSFD;
1583     my $pid;
1584
1585     # save terminal settings
1586     $stty_orig = `stty -g`;
1587
1588     # place terminal in cbreak mode so that stdin can be read one character at
1589     # a time without having to wait for a newline
1590     system("stty -icanon -echo -icrnl");
1591
1592     create_pty($ptm, $pts);
1593
1594     $pid = fork;
1595
1596     if (!$pid) {
1597         # child
1598         exec_console($ptm, $pts)
1599     }
1600
1601     # parent
1602     close($pts);
1603
1604     return $pid;
1605
1606     open(PTSFD, "Stop perl from warning about single use of PTSFD");
1607 }
1608
1609 sub close_console {
1610     my ($fp, $pid) = @_;
1611
1612     doprint "kill child process $pid\n";
1613     kill $close_console_signal, $pid;
1614
1615     doprint "wait for child process $pid to exit\n";
1616     waitpid($pid, 0);
1617
1618     print "closing!\n";
1619     close($fp);
1620
1621     # restore terminal settings
1622     system("stty $stty_orig");
1623 }
1624
1625 sub start_monitor {
1626     if ($monitor_cnt++) {
1627         return;
1628     }
1629     $monitor_fp = \*MONFD;
1630     $monitor_pid = open_console $monitor_fp;
1631
1632     return;
1633
1634     open(MONFD, "Stop perl from warning about single use of MONFD");
1635 }
1636
1637 sub end_monitor {
1638     return if (!defined $console);
1639     if (--$monitor_cnt) {
1640         return;
1641     }
1642     close_console($monitor_fp, $monitor_pid);
1643 }
1644
1645 sub wait_for_monitor {
1646     my ($time, $stop) = @_;
1647     my $full_line = "";
1648     my $line;
1649     my $booted = 0;
1650     my $start_time = time;
1651     my $skip_call_trace = 0;
1652     my $bug = 0;
1653     my $bug_ignored = 0;
1654     my $now;
1655
1656     doprint "** Wait for monitor to settle down **\n";
1657
1658     # read the monitor and wait for the system to calm down
1659     while (!$booted) {
1660         $line = wait_for_input($monitor_fp, $time);
1661         last if (!defined($line));
1662         print "$line";
1663         $full_line .= $line;
1664
1665         if (defined($stop) && $full_line =~ /$stop/) {
1666             doprint "wait for monitor detected $stop\n";
1667             $booted = 1;
1668         }
1669
1670         if ($full_line =~ /\[ backtrace testing \]/) {
1671             $skip_call_trace = 1;
1672         }
1673
1674         if ($full_line =~ /call trace:/i) {
1675             if (!$bug && !$skip_call_trace) {
1676                 if ($ignore_errors) {
1677                     $bug_ignored = 1;
1678                 } else {
1679                     $bug = 1;
1680                 }
1681             }
1682         }
1683
1684         if ($full_line =~ /\[ end of backtrace testing \]/) {
1685             $skip_call_trace = 0;
1686         }
1687
1688         if ($full_line =~ /Kernel panic -/) {
1689             $bug = 1;
1690         }
1691
1692         if ($line =~ /\n/) {
1693             $full_line = "";
1694         }
1695         $now = time;
1696         if ($now - $start_time >= $max_monitor_wait) {
1697             doprint "Exiting monitor flush due to hitting MAX_MONITOR_WAIT\n";
1698             return 1;
1699         }
1700     }
1701     print "** Monitor flushed **\n";
1702
1703     # if stop is defined but wasn't hit, return error
1704     # used by reboot (which wants to see a reboot)
1705     if (defined($stop) && !$booted) {
1706         $bug = 1;
1707     }
1708     return $bug;
1709 }
1710
1711 sub save_logs {
1712         my ($result, $basedir) = @_;
1713         my @t = localtime;
1714         my $date = sprintf "%04d%02d%02d%02d%02d%02d",
1715                 1900+$t[5],$t[4],$t[3],$t[2],$t[1],$t[0];
1716
1717         my $type = $build_type;
1718         if ($type =~ /useconfig/) {
1719             $type = "useconfig";
1720         }
1721
1722         my $dir = "$machine-$test_type-$type-$result-$date";
1723
1724         $dir = "$basedir/$dir";
1725
1726         if (!-d $dir) {
1727             mkpath($dir) or
1728                 dodie "can't create $dir";
1729         }
1730
1731         my %files = (
1732                 "config" => $output_config,
1733                 "buildlog" => $buildlog,
1734                 "dmesg" => $dmesg,
1735                 "testlog" => $testlog,
1736         );
1737
1738         while (my ($name, $source) = each(%files)) {
1739                 if (-f "$source") {
1740                         cp "$source", "$dir/$name" or
1741                                 dodie "failed to copy $source";
1742                 }
1743         }
1744
1745         doprint "*** Saved info to $dir ***\n";
1746 }
1747
1748 sub fail {
1749
1750         if ($die_on_failure) {
1751                 dodie @_;
1752         }
1753
1754         doprint "FAILED\n";
1755
1756         my $i = $iteration;
1757
1758         # no need to reboot for just building.
1759         if (!do_not_reboot) {
1760             doprint "REBOOTING\n";
1761             reboot_to_good $sleep_time;
1762         }
1763
1764         my $name = "";
1765
1766         if (defined($test_name)) {
1767             $name = " ($test_name)";
1768         }
1769
1770         print_times;
1771
1772         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
1773         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
1774         doprint "KTEST RESULT: TEST $i$name Failed: ", @_, "\n";
1775         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
1776         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
1777
1778         if (defined($store_failures)) {
1779             save_logs "fail", $store_failures;
1780         }
1781
1782         if (defined($post_test)) {
1783                 run_command $post_test;
1784         }
1785
1786         return 1;
1787 }
1788
1789 sub run_command {
1790     my ($command, $redirect, $timeout) = @_;
1791     my $start_time;
1792     my $end_time;
1793     my $dolog = 0;
1794     my $dord = 0;
1795     my $dostdout = 0;
1796     my $pid;
1797     my $command_orig = $command;
1798
1799     $command =~ s/\$SSH_USER/$ssh_user/g;
1800     $command =~ s/\$MACHINE/$machine/g;
1801
1802     doprint("$command ... ");
1803     $start_time = time;
1804
1805     $pid = open(CMD, "$command 2>&1 |") or
1806         (fail "unable to exec $command" and return 0);
1807
1808     if (defined($opt{"LOG_FILE"})) {
1809         $dolog = 1;
1810     }
1811
1812     if (defined($redirect)) {
1813         if ($redirect eq 1) {
1814             $dostdout = 1;
1815             # Have the output of the command on its own line
1816             doprint "\n";
1817         } else {
1818             open (RD, ">$redirect") or
1819                 dodie "failed to write to redirect $redirect";
1820             $dord = 1;
1821         }
1822     }
1823
1824     my $hit_timeout = 0;
1825
1826     while (1) {
1827         my $fp = \*CMD;
1828         if (defined($timeout)) {
1829             doprint "timeout = $timeout\n";
1830         }
1831         my $line = wait_for_input($fp, $timeout);
1832         if (!defined($line)) {
1833             my $now = time;
1834             if (defined($timeout) && (($now - $start_time) >= $timeout)) {
1835                 doprint "Hit timeout of $timeout, killing process\n";
1836                 $hit_timeout = 1;
1837                 kill 9, $pid;
1838             }
1839             last;
1840         }
1841         print LOG $line if ($dolog);
1842         print RD $line if ($dord);
1843         print $line if ($dostdout);
1844     }
1845
1846     waitpid($pid, 0);
1847     # shift 8 for real exit status
1848     $run_command_status = $? >> 8;
1849
1850     if ($command_orig eq $default{REBOOT} &&
1851         $run_command_status == $reboot_return_code) {
1852         $run_command_status = 0;
1853     }
1854
1855     close(CMD);
1856     close(RD)  if ($dord);
1857
1858     $end_time = time;
1859     my $delta = $end_time - $start_time;
1860
1861     if ($delta == 1) {
1862         doprint "[1 second] ";
1863     } else {
1864         doprint "[$delta seconds] ";
1865     }
1866
1867     if ($hit_timeout) {
1868         $run_command_status = 1;
1869     }
1870
1871     if ($run_command_status) {
1872         doprint "FAILED!\n";
1873     } else {
1874         doprint "SUCCESS\n";
1875     }
1876
1877     return !$run_command_status;
1878 }
1879
1880 sub run_ssh {
1881     my ($cmd, $timeout) = @_;
1882     my $cp_exec = $ssh_exec;
1883
1884     $cp_exec =~ s/\$SSH_COMMAND/$cmd/g;
1885     return run_command "$cp_exec", undef , $timeout;
1886 }
1887
1888 sub run_scp {
1889     my ($src, $dst, $cp_scp) = @_;
1890
1891     $cp_scp =~ s/\$SRC_FILE/$src/g;
1892     $cp_scp =~ s/\$DST_FILE/$dst/g;
1893
1894     return run_command "$cp_scp";
1895 }
1896
1897 sub run_scp_install {
1898     my ($src, $dst) = @_;
1899
1900     my $cp_scp = $scp_to_target_install;
1901
1902     return run_scp($src, $dst, $cp_scp);
1903 }
1904
1905 sub run_scp_mod {
1906     my ($src, $dst) = @_;
1907
1908     my $cp_scp = $scp_to_target;
1909
1910     return run_scp($src, $dst, $cp_scp);
1911 }
1912
1913 sub _get_grub_index {
1914
1915     my ($command, $target, $skip) = @_;
1916
1917     return if (defined($grub_number) && defined($last_grub_menu) &&
1918                $last_grub_menu eq $grub_menu && defined($last_machine) &&
1919                $last_machine eq $machine);
1920
1921     doprint "Find $reboot_type menu ... ";
1922     $grub_number = -1;
1923
1924     my $ssh_grub = $ssh_exec;
1925     $ssh_grub =~ s,\$SSH_COMMAND,$command,g;
1926
1927     open(IN, "$ssh_grub |")
1928         or dodie "unable to execute $command";
1929
1930     my $found = 0;
1931
1932     while (<IN>) {
1933         if (/$target/) {
1934             $grub_number++;
1935             $found = 1;
1936             last;
1937         } elsif (/$skip/) {
1938             $grub_number++;
1939         }
1940     }
1941     close(IN);
1942
1943     dodie "Could not find '$grub_menu' through $command on $machine"
1944         if (!$found);
1945     doprint "$grub_number\n";
1946     $last_grub_menu = $grub_menu;
1947     $last_machine = $machine;
1948 }
1949
1950 sub get_grub_index {
1951
1952     my $command;
1953     my $target;
1954     my $skip;
1955     my $grub_menu_qt;
1956
1957     if ($reboot_type !~ /^grub/) {
1958         return;
1959     }
1960
1961     $grub_menu_qt = quotemeta($grub_menu);
1962
1963     if ($reboot_type eq "grub") {
1964         $command = "cat /boot/grub/menu.lst";
1965         $target = '^\s*title\s+' . $grub_menu_qt . '\s*$';
1966         $skip = '^\s*title\s';
1967     } elsif ($reboot_type eq "grub2") {
1968         $command = "cat $grub_file";
1969         $target = '^menuentry.*' . $grub_menu_qt;
1970         $skip = '^menuentry\s|^submenu\s';
1971     } elsif ($reboot_type eq "grub2bls") {
1972         $command = $grub_bls_get;
1973         $target = '^title=.*' . $grub_menu_qt;
1974         $skip = '^title=';
1975     } else {
1976         return;
1977     }
1978
1979     _get_grub_index($command, $target, $skip);
1980 }
1981
1982 sub wait_for_input
1983 {
1984     my ($fp, $time) = @_;
1985     my $start_time;
1986     my $rin;
1987     my $rout;
1988     my $nr;
1989     my $buf;
1990     my $line;
1991     my $ch;
1992
1993     if (!defined($time)) {
1994         $time = $timeout;
1995     }
1996
1997     $rin = '';
1998     vec($rin, fileno($fp), 1) = 1;
1999     vec($rin, fileno(\*STDIN), 1) = 1;
2000
2001     $start_time = time;
2002
2003     while (1) {
2004         $nr = select($rout=$rin, undef, undef, $time);
2005
2006         last if ($nr <= 0);
2007
2008         # copy data from stdin to the console
2009         if (vec($rout, fileno(\*STDIN), 1) == 1) {
2010             $nr = sysread(\*STDIN, $buf, 1000);
2011             syswrite($fp, $buf, $nr) if ($nr > 0);
2012         }
2013
2014         # The timeout is based on time waiting for the fp data
2015         if (vec($rout, fileno($fp), 1) != 1) {
2016             last if (defined($time) && (time - $start_time > $time));
2017             next;
2018         }
2019
2020         $line = "";
2021
2022         # try to read one char at a time
2023         while (sysread $fp, $ch, 1) {
2024             $line .= $ch;
2025             last if ($ch eq "\n");
2026         }
2027
2028         last if (!length($line));
2029
2030         return $line;
2031     }
2032     return undef;
2033 }
2034
2035 sub reboot_to {
2036     if (defined($switch_to_test)) {
2037         run_command $switch_to_test;
2038     }
2039
2040     if ($reboot_type eq "grub") {
2041         run_ssh "'(echo \"savedefault --default=$grub_number --once\" | grub --batch)'";
2042     } elsif (($reboot_type eq "grub2") or ($reboot_type eq "grub2bls")) {
2043         run_ssh "$grub_reboot $grub_number";
2044     } elsif ($reboot_type eq "syslinux") {
2045         run_ssh "$syslinux --once \\\"$syslinux_label\\\" $syslinux_path";
2046     } elsif (defined $reboot_script) {
2047         run_command "$reboot_script";
2048     }
2049     reboot;
2050 }
2051
2052 sub get_sha1 {
2053     my ($commit) = @_;
2054
2055     doprint "git rev-list --max-count=1 $commit ... ";
2056     my $sha1 = `git rev-list --max-count=1 $commit`;
2057     my $ret = $?;
2058
2059     logit $sha1;
2060
2061     if ($ret) {
2062         doprint "FAILED\n";
2063         dodie "Failed to get git $commit";
2064     }
2065
2066     print "SUCCESS\n";
2067
2068     chomp $sha1;
2069
2070     return $sha1;
2071 }
2072
2073 sub monitor {
2074     my $booted = 0;
2075     my $bug = 0;
2076     my $bug_ignored = 0;
2077     my $skip_call_trace = 0;
2078     my $loops;
2079
2080     my $start_time = time;
2081
2082     wait_for_monitor 5;
2083
2084     my $line;
2085     my $full_line = "";
2086
2087     open(DMESG, "> $dmesg") or
2088         dodie "unable to write to $dmesg";
2089
2090     reboot_to;
2091
2092     my $success_start;
2093     my $failure_start;
2094     my $monitor_start = time;
2095     my $done = 0;
2096     my $version_found = 0;
2097
2098     while (!$done) {
2099
2100         if ($bug && defined($stop_after_failure) &&
2101             $stop_after_failure >= 0) {
2102             my $time = $stop_after_failure - (time - $failure_start);
2103             $line = wait_for_input($monitor_fp, $time);
2104             if (!defined($line)) {
2105                 doprint "bug timed out after $booted_timeout seconds\n";
2106                 doprint "Test forced to stop after $stop_after_failure seconds after failure\n";
2107                 last;
2108             }
2109         } elsif ($booted) {
2110             $line = wait_for_input($monitor_fp, $booted_timeout);
2111             if (!defined($line)) {
2112                 my $s = $booted_timeout == 1 ? "" : "s";
2113                 doprint "Successful boot found: break after $booted_timeout second$s\n";
2114                 last;
2115             }
2116         } else {
2117             $line = wait_for_input($monitor_fp);
2118             if (!defined($line)) {
2119                 my $s = $timeout == 1 ? "" : "s";
2120                 doprint "Timed out after $timeout second$s\n";
2121                 last;
2122             }
2123         }
2124
2125         doprint $line;
2126         print DMESG $line;
2127
2128         # we are not guaranteed to get a full line
2129         $full_line .= $line;
2130
2131         if ($full_line =~ /$success_line/) {
2132             $booted = 1;
2133             $success_start = time;
2134         }
2135
2136         if ($booted && defined($stop_after_success) &&
2137             $stop_after_success >= 0) {
2138             my $now = time;
2139             if ($now - $success_start >= $stop_after_success) {
2140                 doprint "Test forced to stop after $stop_after_success seconds after success\n";
2141                 last;
2142             }
2143         }
2144
2145         if ($full_line =~ /\[ backtrace testing \]/) {
2146             $skip_call_trace = 1;
2147         }
2148
2149         if ($full_line =~ /call trace:/i) {
2150             if (!$bug && !$skip_call_trace) {
2151                 if ($ignore_errors) {
2152                     $bug_ignored = 1;
2153                 } else {
2154                     $bug = 1;
2155                     $failure_start = time;
2156                 }
2157             }
2158         }
2159
2160         if ($bug && defined($stop_after_failure) &&
2161             $stop_after_failure >= 0) {
2162             my $now = time;
2163             if ($now - $failure_start >= $stop_after_failure) {
2164                 doprint "Test forced to stop after $stop_after_failure seconds after failure\n";
2165                 last;
2166             }
2167         }
2168
2169         if ($full_line =~ /\[ end of backtrace testing \]/) {
2170             $skip_call_trace = 0;
2171         }
2172
2173         if ($full_line =~ /Kernel panic -/) {
2174             $failure_start = time;
2175             $bug = 1;
2176         }
2177
2178         # Detect triple faults by testing the banner
2179         if ($full_line =~ /\bLinux version (\S+).*\n/) {
2180             if ($1 eq $version) {
2181                 $version_found = 1;
2182             } elsif ($version_found && $detect_triplefault) {
2183                 # We already booted into the kernel we are testing,
2184                 # but now we booted into another kernel?
2185                 # Consider this a triple fault.
2186                 doprint "Already booted in Linux kernel $version, but now\n";
2187                 doprint "we booted into Linux kernel $1.\n";
2188                 doprint "Assuming that this is a triple fault.\n";
2189                 doprint "To disable this: set DETECT_TRIPLE_FAULT to 0\n";
2190                 last;
2191             }
2192         }
2193
2194         if ($line =~ /\n/) {
2195             $full_line = "";
2196         }
2197
2198         if ($stop_test_after > 0 && !$booted && !$bug) {
2199             if (time - $monitor_start > $stop_test_after) {
2200                 doprint "STOP_TEST_AFTER ($stop_test_after seconds) timed out\n";
2201                 $done = 1;
2202             }
2203         }
2204     }
2205
2206     my $end_time = time;
2207     $reboot_time = $end_time - $start_time;
2208
2209     close(DMESG);
2210
2211     if ($bug) {
2212         return 0 if ($in_bisect);
2213         fail "failed - got a bug report" and return 0;
2214     }
2215
2216     if (!$booted) {
2217         return 0 if ($in_bisect);
2218         fail "failed - never got a boot prompt." and return 0;
2219     }
2220
2221     if ($bug_ignored) {
2222         doprint "WARNING: Call Trace detected but ignored due to IGNORE_ERRORS=1\n";
2223     }
2224
2225     return 1;
2226 }
2227
2228 sub eval_kernel_version {
2229     my ($option) = @_;
2230
2231     $option =~ s/\$KERNEL_VERSION/$version/g;
2232
2233     return $option;
2234 }
2235
2236 sub do_post_install {
2237
2238     return if (!defined($post_install));
2239
2240     my $cp_post_install = eval_kernel_version $post_install;
2241     run_command "$cp_post_install" or
2242         dodie "Failed to run post install";
2243 }
2244
2245 # Sometimes the reboot fails, and will hang. We try to ssh to the box
2246 # and if we fail, we force another reboot, that should powercycle it.
2247 sub test_booted {
2248     if (!run_ssh "echo testing connection") {
2249         reboot $sleep_time;
2250     }
2251 }
2252
2253 sub install {
2254
2255     return if ($no_install);
2256
2257     my $start_time = time;
2258
2259     if (defined($pre_install)) {
2260         my $cp_pre_install = eval_kernel_version $pre_install;
2261         run_command "$cp_pre_install" or
2262             dodie "Failed to run pre install";
2263     }
2264
2265     my $cp_target = eval_kernel_version $target_image;
2266
2267     test_booted;
2268
2269     run_scp_install "$outputdir/$build_target", "$cp_target" or
2270         dodie "failed to copy image";
2271
2272     my $install_mods = 0;
2273
2274     # should we process modules?
2275     $install_mods = 0;
2276     open(IN, "$output_config") or dodie("Can't read config file");
2277     while (<IN>) {
2278         if (/CONFIG_MODULES(=y)?/) {
2279             if (defined($1)) {
2280                 $install_mods = 1;
2281                 last;
2282             }
2283         }
2284     }
2285     close(IN);
2286
2287     if (!$install_mods) {
2288         do_post_install;
2289         doprint "No modules needed\n";
2290         my $end_time = time;
2291         $install_time = $end_time - $start_time;
2292         return;
2293     }
2294
2295     run_command "$make INSTALL_MOD_STRIP=1 INSTALL_MOD_PATH=$tmpdir modules_install" or
2296         dodie "Failed to install modules";
2297
2298     my $modlib = "/lib/modules/$version";
2299     my $modtar = "ktest-mods.tar.bz2";
2300
2301     run_ssh "rm -rf $modlib" or
2302         dodie "failed to remove old mods: $modlib";
2303
2304     # would be nice if scp -r did not follow symbolic links
2305     run_command "cd $tmpdir && tar -cjf $modtar lib/modules/$version" or
2306         dodie "making tarball";
2307
2308     run_scp_mod "$tmpdir/$modtar", "/tmp" or
2309         dodie "failed to copy modules";
2310
2311     unlink "$tmpdir/$modtar";
2312
2313     run_ssh "'(cd / && tar xjf /tmp/$modtar)'" or
2314         dodie "failed to tar modules";
2315
2316     run_ssh "rm -f /tmp/$modtar";
2317
2318     do_post_install;
2319
2320     my $end_time = time;
2321     $install_time = $end_time - $start_time;
2322 }
2323
2324 sub get_version {
2325     # get the release name
2326     return if ($have_version);
2327     doprint "$make kernelrelease ... ";
2328     $version = `$make -s kernelrelease | tail -1`;
2329     chomp($version);
2330     doprint "$version\n";
2331     $have_version = 1;
2332 }
2333
2334 sub start_monitor_and_install {
2335     # Make sure the stable kernel has finished booting
2336
2337     # Install bisects, don't need console
2338     if (defined $console) {
2339         start_monitor;
2340         wait_for_monitor 5;
2341         end_monitor;
2342     }
2343
2344     get_grub_index;
2345     get_version;
2346     install;
2347
2348     start_monitor if (defined $console);
2349     return monitor;
2350 }
2351
2352 my $check_build_re = ".*:.*(warning|error|Error):.*";
2353 my $utf8_quote = "\\x{e2}\\x{80}(\\x{98}|\\x{99})";
2354
2355 sub process_warning_line {
2356     my ($line) = @_;
2357
2358     chomp $line;
2359
2360     # for distcc heterogeneous systems, some compilers
2361     # do things differently causing warning lines
2362     # to be slightly different. This makes an attempt
2363     # to fixe those issues.
2364
2365     # chop off the index into the line
2366     # using distcc, some compilers give different indexes
2367     # depending on white space
2368     $line =~ s/^(\s*\S+:\d+:)\d+/$1/;
2369
2370     # Some compilers use UTF-8 extended for quotes and some don't.
2371     $line =~ s/$utf8_quote/'/g;
2372
2373     return $line;
2374 }
2375
2376 # Read buildlog and check against warnings file for any
2377 # new warnings.
2378 #
2379 # Returns 1 if OK
2380 #         0 otherwise
2381 sub check_buildlog {
2382     return 1 if (!defined $warnings_file);
2383
2384     my %warnings_list;
2385
2386     # Failed builds should not reboot the target
2387     my $save_no_reboot = $no_reboot;
2388     $no_reboot = 1;
2389
2390     if (-f $warnings_file) {
2391         open(IN, $warnings_file) or
2392             dodie "Error opening $warnings_file";
2393
2394         while (<IN>) {
2395             if (/$check_build_re/) {
2396                 my $warning = process_warning_line $_;
2397
2398                 $warnings_list{$warning} = 1;
2399             }
2400         }
2401         close(IN);
2402     }
2403
2404     # If warnings file didn't exist, and WARNINGS_FILE exist,
2405     # then we fail on any warning!
2406
2407     open(IN, $buildlog) or dodie "Can't open $buildlog";
2408     while (<IN>) {
2409         if (/$check_build_re/) {
2410             my $warning = process_warning_line $_;
2411
2412             if (!defined $warnings_list{$warning}) {
2413                 fail "New warning found (not in $warnings_file)\n$_\n";
2414                 $no_reboot = $save_no_reboot;
2415                 return 0;
2416             }
2417         }
2418     }
2419     $no_reboot = $save_no_reboot;
2420     close(IN);
2421 }
2422
2423 sub check_patch_buildlog {
2424     my ($patch) = @_;
2425
2426     my @files = `git show $patch | diffstat -l`;
2427
2428     foreach my $file (@files) {
2429         chomp $file;
2430     }
2431
2432     open(IN, "git show $patch |") or
2433         dodie "failed to show $patch";
2434     while (<IN>) {
2435         if (m,^--- a/(.*),) {
2436             chomp $1;
2437             $files[$#files] = $1;
2438         }
2439     }
2440     close(IN);
2441
2442     open(IN, $buildlog) or dodie "Can't open $buildlog";
2443     while (<IN>) {
2444         if (/^\s*(.*?):.*(warning|error)/) {
2445             my $err = $1;
2446             foreach my $file (@files) {
2447                 my $fullpath = "$builddir/$file";
2448                 if ($file eq $err || $fullpath eq $err) {
2449                     fail "$file built with warnings" and return 0;
2450                 }
2451             }
2452         }
2453     }
2454     close(IN);
2455
2456     return 1;
2457 }
2458
2459 sub apply_min_config {
2460     my $outconfig = "$output_config.new";
2461
2462     # Read the config file and remove anything that
2463     # is in the force_config hash (from minconfig and others)
2464     # then add the force config back.
2465
2466     doprint "Applying minimum configurations into $output_config.new\n";
2467
2468     open (OUT, ">$outconfig") or
2469         dodie "Can't create $outconfig";
2470
2471     if (-f $output_config) {
2472         open (IN, $output_config) or
2473             dodie "Failed to open $output_config";
2474         while (<IN>) {
2475             if (/^(# )?(CONFIG_[^\s=]*)/) {
2476                 next if (defined($force_config{$2}));
2477             }
2478             print OUT;
2479         }
2480         close IN;
2481     }
2482     foreach my $config (keys %force_config) {
2483         print OUT "$force_config{$config}\n";
2484     }
2485     close OUT;
2486
2487     run_command "mv $outconfig $output_config";
2488 }
2489
2490 sub make_oldconfig {
2491
2492     my @force_list = keys %force_config;
2493
2494     if ($#force_list >= 0) {
2495         apply_min_config;
2496     }
2497
2498     if (!run_command "$make olddefconfig") {
2499         # Perhaps olddefconfig doesn't exist in this version of the kernel
2500         # try oldnoconfig
2501         doprint "olddefconfig failed, trying make oldnoconfig\n";
2502         if (!run_command "$make oldnoconfig") {
2503             doprint "oldnoconfig failed, trying yes '' | make oldconfig\n";
2504             # try a yes '' | oldconfig
2505             run_command "yes '' | $make oldconfig" or
2506                 dodie "failed make config oldconfig";
2507         }
2508     }
2509 }
2510
2511 # read a config file and use this to force new configs.
2512 sub load_force_config {
2513     my ($config) = @_;
2514
2515     doprint "Loading force configs from $config\n";
2516     open(IN, $config) or
2517         dodie "failed to read $config";
2518     while (<IN>) {
2519         chomp;
2520         if (/^(CONFIG[^\s=]*)(\s*=.*)/) {
2521             $force_config{$1} = $_;
2522         } elsif (/^# (CONFIG_\S*) is not set/) {
2523             $force_config{$1} = $_;
2524         }
2525     }
2526     close IN;
2527 }
2528
2529 sub build {
2530     my ($type) = @_;
2531
2532     unlink $buildlog;
2533
2534     my $start_time = time;
2535
2536     # Failed builds should not reboot the target
2537     my $save_no_reboot = $no_reboot;
2538     $no_reboot = 1;
2539
2540     # Calculate a new version from here.
2541     $have_version = 0;
2542
2543     if (defined($pre_build)) {
2544         my $ret = run_command $pre_build;
2545         if (!$ret && defined($pre_build_die) &&
2546             $pre_build_die) {
2547             dodie "failed to pre_build\n";
2548         }
2549     }
2550
2551     if ($type =~ /^useconfig:(.*)/) {
2552         run_command "cp $1 $output_config" or
2553             dodie "could not copy $1 to .config";
2554
2555         $type = "oldconfig";
2556     }
2557
2558     # old config can ask questions
2559     if ($type eq "oldconfig") {
2560         $type = "olddefconfig";
2561
2562         # allow for empty configs
2563         run_command "touch $output_config";
2564
2565         if (!$noclean) {
2566             run_command "mv $output_config $outputdir/config_temp" or
2567                 dodie "moving .config";
2568
2569             run_command "$make mrproper" or dodie "make mrproper";
2570
2571             run_command "mv $outputdir/config_temp $output_config" or
2572                 dodie "moving config_temp";
2573         }
2574
2575     } elsif (!$noclean) {
2576         unlink "$output_config";
2577         run_command "$make mrproper" or
2578             dodie "make mrproper";
2579     }
2580
2581     # add something to distinguish this build
2582     open(OUT, "> $outputdir/localversion") or dodie("Can't make localversion file");
2583     print OUT "$localversion\n";
2584     close(OUT);
2585
2586     if (defined($minconfig)) {
2587         load_force_config($minconfig);
2588     }
2589
2590     if ($type ne "olddefconfig") {
2591         run_command "$make $type" or
2592             dodie "failed make config";
2593     }
2594     # Run old config regardless, to enforce min configurations
2595     make_oldconfig;
2596
2597     my $build_ret = run_command "$make $build_options", $buildlog;
2598
2599     if (defined($post_build)) {
2600         # Because a post build may change the kernel version
2601         # do it now.
2602         get_version;
2603         my $ret = run_command $post_build;
2604         if (!$ret && defined($post_build_die) &&
2605             $post_build_die) {
2606             dodie "failed to post_build\n";
2607         }
2608     }
2609
2610     if (!$build_ret) {
2611         # bisect may need this to pass
2612         if ($in_bisect) {
2613             $no_reboot = $save_no_reboot;
2614             return 0;
2615         }
2616         fail "failed build" and return 0;
2617     }
2618
2619     $no_reboot = $save_no_reboot;
2620
2621     my $end_time = time;
2622     $build_time = $end_time - $start_time;
2623
2624     return 1;
2625 }
2626
2627 sub halt {
2628     if (!run_ssh "halt" or defined($power_off)) {
2629         if (defined($poweroff_after_halt)) {
2630             sleep $poweroff_after_halt;
2631             run_command "$power_off";
2632         }
2633     } else {
2634         # nope? the zap it!
2635         run_command "$power_off";
2636     }
2637 }
2638
2639 sub success {
2640     my ($i) = @_;
2641
2642     $successes++;
2643
2644     my $name = "";
2645
2646     if (defined($test_name)) {
2647         $name = " ($test_name)";
2648     }
2649
2650     print_times;
2651
2652     doprint "\n\n*******************************************\n";
2653     doprint     "*******************************************\n";
2654     doprint     "KTEST RESULT: TEST $i$name SUCCESS!!!!         **\n";
2655     doprint     "*******************************************\n";
2656     doprint     "*******************************************\n";
2657
2658     if (defined($store_successes)) {
2659         save_logs "success", $store_successes;
2660     }
2661
2662     if ($i != $opt{"NUM_TESTS"} && !do_not_reboot) {
2663         doprint "Reboot and wait $sleep_time seconds\n";
2664         reboot_to_good $sleep_time;
2665     }
2666
2667     if (defined($post_test)) {
2668         run_command $post_test;
2669     }
2670 }
2671
2672 sub answer_bisect {
2673     for (;;) {
2674         doprint "Pass, fail, or skip? [p/f/s]";
2675         my $ans = <STDIN>;
2676         chomp $ans;
2677         if ($ans eq "p" || $ans eq "P") {
2678             return 1;
2679         } elsif ($ans eq "f" || $ans eq "F") {
2680             return 0;
2681         } elsif ($ans eq "s" || $ans eq "S") {
2682             return -1;
2683         } else {
2684             print "Please answer 'p', 'f', or 's'\n";
2685         }
2686     }
2687 }
2688
2689 sub child_run_test {
2690
2691     # child should have no power
2692     $reboot_on_error = 0;
2693     $poweroff_on_error = 0;
2694     $die_on_failure = 1;
2695
2696     run_command $run_test, $testlog;
2697
2698     exit $run_command_status;
2699 }
2700
2701 my $child_done;
2702
2703 sub child_finished {
2704     $child_done = 1;
2705 }
2706
2707 sub do_run_test {
2708     my $child_pid;
2709     my $child_exit;
2710     my $line;
2711     my $full_line;
2712     my $bug = 0;
2713     my $bug_ignored = 0;
2714
2715     my $start_time = time;
2716
2717     wait_for_monitor 1;
2718
2719     doprint "run test $run_test\n";
2720
2721     $child_done = 0;
2722
2723     $SIG{CHLD} = qw(child_finished);
2724
2725     $child_pid = fork;
2726
2727     child_run_test if (!$child_pid);
2728
2729     $full_line = "";
2730
2731     do {
2732         $line = wait_for_input($monitor_fp, 1);
2733         if (defined($line)) {
2734
2735             # we are not guaranteed to get a full line
2736             $full_line .= $line;
2737             doprint $line;
2738
2739             if ($full_line =~ /call trace:/i) {
2740                 if ($ignore_errors) {
2741                     $bug_ignored = 1;
2742                 } else {
2743                     $bug = 1;
2744                 }
2745             }
2746
2747             if ($full_line =~ /Kernel panic -/) {
2748                 $bug = 1;
2749             }
2750
2751             if ($line =~ /\n/) {
2752                 $full_line = "";
2753             }
2754         }
2755     } while (!$child_done && !$bug);
2756
2757     if (!$bug && $bug_ignored) {
2758         doprint "WARNING: Call Trace detected but ignored due to IGNORE_ERRORS=1\n";
2759     }
2760
2761     if ($bug) {
2762         my $failure_start = time;
2763         my $now;
2764         do {
2765             $line = wait_for_input($monitor_fp, 1);
2766             if (defined($line)) {
2767                 doprint $line;
2768             }
2769             $now = time;
2770             if ($now - $failure_start >= $stop_after_failure) {
2771                 last;
2772             }
2773         } while (defined($line));
2774
2775         doprint "Detected kernel crash!\n";
2776         # kill the child with extreme prejudice
2777         kill 9, $child_pid;
2778     }
2779
2780     waitpid $child_pid, 0;
2781     $child_exit = $? >> 8;
2782
2783     my $end_time = time;
2784     $test_time = $end_time - $start_time;
2785
2786     if (!$bug && $in_bisect) {
2787         if (defined($bisect_ret_good)) {
2788             if ($child_exit == $bisect_ret_good) {
2789                 return 1;
2790             }
2791         }
2792         if (defined($bisect_ret_skip)) {
2793             if ($child_exit == $bisect_ret_skip) {
2794                 return -1;
2795             }
2796         }
2797         if (defined($bisect_ret_abort)) {
2798             if ($child_exit == $bisect_ret_abort) {
2799                 fail "test abort" and return -2;
2800             }
2801         }
2802         if (defined($bisect_ret_bad)) {
2803             if ($child_exit == $bisect_ret_skip) {
2804                 return 0;
2805             }
2806         }
2807         if (defined($bisect_ret_default)) {
2808             if ($bisect_ret_default eq "good") {
2809                 return 1;
2810             } elsif ($bisect_ret_default eq "bad") {
2811                 return 0;
2812             } elsif ($bisect_ret_default eq "skip") {
2813                 return -1;
2814             } elsif ($bisect_ret_default eq "abort") {
2815                 return -2;
2816             } else {
2817                 fail "unknown default action: $bisect_ret_default"
2818                     and return -2;
2819             }
2820         }
2821     }
2822
2823     if ($bug || $child_exit) {
2824         return 0 if $in_bisect;
2825         fail "test failed" and return 0;
2826     }
2827     return 1;
2828 }
2829
2830 sub run_git_bisect {
2831     my ($command) = @_;
2832
2833     doprint "$command ... ";
2834
2835     my $output = `$command 2>&1`;
2836     my $ret = $?;
2837
2838     logit $output;
2839
2840     if ($ret) {
2841         doprint "FAILED\n";
2842         dodie "Failed to git bisect";
2843     }
2844
2845     doprint "SUCCESS\n";
2846     if ($output =~ m/^(Bisecting: .*\(roughly \d+ steps?\))\s+\[([[:xdigit:]]+)\]/) {
2847         doprint "$1 [$2]\n";
2848     } elsif ($output =~ m/^([[:xdigit:]]+) is the first bad commit/) {
2849         $bisect_bad_commit = $1;
2850         doprint "Found bad commit... $1\n";
2851         return 0;
2852     } else {
2853         # we already logged it, just print it now.
2854         print $output;
2855     }
2856
2857     return 1;
2858 }
2859
2860 sub bisect_reboot {
2861     doprint "Reboot and sleep $bisect_sleep_time seconds\n";
2862     reboot_to_good $bisect_sleep_time;
2863 }
2864
2865 # returns 1 on success, 0 on failure, -1 on skip
2866 sub run_bisect_test {
2867     my ($type, $buildtype) = @_;
2868
2869     my $failed = 0;
2870     my $result;
2871     my $output;
2872     my $ret;
2873
2874     $in_bisect = 1;
2875
2876     build $buildtype or $failed = 1;
2877
2878     if ($type ne "build") {
2879         if ($failed && $bisect_skip) {
2880             $in_bisect = 0;
2881             return -1;
2882         }
2883         dodie "Failed on build" if $failed;
2884
2885         # Now boot the box
2886         start_monitor_and_install or $failed = 1;
2887
2888         if ($type ne "boot") {
2889             if ($failed && $bisect_skip) {
2890                 end_monitor;
2891                 bisect_reboot;
2892                 $in_bisect = 0;
2893                 return -1;
2894             }
2895             dodie "Failed on boot" if $failed;
2896
2897             do_run_test or $failed = 1;
2898         }
2899         end_monitor;
2900     }
2901
2902     if ($failed) {
2903         $result = 0;
2904     } else {
2905         $result = 1;
2906     }
2907
2908     # reboot the box to a kernel we can ssh to
2909     if ($type ne "build") {
2910         bisect_reboot;
2911     }
2912     $in_bisect = 0;
2913
2914     return $result;
2915 }
2916
2917 sub run_bisect {
2918     my ($type) = @_;
2919     my $buildtype = "oldconfig";
2920
2921     # We should have a minconfig to use?
2922     if (defined($minconfig)) {
2923         $buildtype = "useconfig:$minconfig";
2924     }
2925
2926     # If the user sets bisect_tries to less than 1, then no tries
2927     # is a success.
2928     my $ret = 1;
2929
2930     # Still let the user manually decide that though.
2931     if ($bisect_tries < 1 && $bisect_manual) {
2932         $ret = answer_bisect;
2933     }
2934
2935     for (my $i = 0; $i < $bisect_tries; $i++) {
2936         if ($bisect_tries > 1) {
2937             my $t = $i + 1;
2938             doprint("Running bisect trial $t of $bisect_tries:\n");
2939         }
2940         $ret = run_bisect_test $type, $buildtype;
2941
2942         if ($bisect_manual) {
2943             $ret = answer_bisect;
2944         }
2945
2946         last if (!$ret);
2947     }
2948
2949     # Are we looking for where it worked, not failed?
2950     if ($reverse_bisect && $ret >= 0) {
2951         $ret = !$ret;
2952     }
2953
2954     if ($ret > 0) {
2955         return "good";
2956     } elsif ($ret == 0) {
2957         return  "bad";
2958     } elsif ($bisect_skip) {
2959         doprint "HIT A BAD COMMIT ... SKIPPING\n";
2960         return "skip";
2961     }
2962 }
2963
2964 sub update_bisect_replay {
2965     my $tmp_log = "$tmpdir/ktest_bisect_log";
2966     run_command "git bisect log > $tmp_log" or
2967         dodie "can't create bisect log";
2968     return $tmp_log;
2969 }
2970
2971 sub bisect {
2972     my ($i) = @_;
2973
2974     my $result;
2975
2976     dodie "BISECT_GOOD[$i] not defined\n"       if (!defined($bisect_good));
2977     dodie "BISECT_BAD[$i] not defined\n"        if (!defined($bisect_bad));
2978     dodie "BISECT_TYPE[$i] not defined\n"       if (!defined($bisect_type));
2979
2980     my $good = $bisect_good;
2981     my $bad = $bisect_bad;
2982     my $type = $bisect_type;
2983     my $start = $bisect_start;
2984     my $replay = $bisect_replay;
2985     my $start_files = $bisect_files;
2986
2987     if (defined($start_files)) {
2988         $start_files = " -- " . $start_files;
2989     } else {
2990         $start_files = "";
2991     }
2992
2993     # convert to true sha1's
2994     $good = get_sha1($good);
2995     $bad = get_sha1($bad);
2996
2997     if (defined($bisect_reverse) && $bisect_reverse == 1) {
2998         doprint "Performing a reverse bisect (bad is good, good is bad!)\n";
2999         $reverse_bisect = 1;
3000     } else {
3001         $reverse_bisect = 0;
3002     }
3003
3004     # Can't have a test without having a test to run
3005     if ($type eq "test" && !defined($run_test)) {
3006         $type = "boot";
3007     }
3008
3009     # Check if a bisect was running
3010     my $bisect_start_file = "$builddir/.git/BISECT_START";
3011
3012     my $check = $bisect_check;
3013     my $do_check = defined($check) && $check ne "0";
3014
3015     if ( -f $bisect_start_file ) {
3016         print "Bisect in progress found\n";
3017         if ($do_check) {
3018             print " If you say yes, then no checks of good or bad will be done\n";
3019         }
3020         if (defined($replay)) {
3021             print "** BISECT_REPLAY is defined in config file **";
3022             print " Ignore config option and perform new git bisect log?\n";
3023             if (read_ync " (yes, no, or cancel) ") {
3024                 $replay = update_bisect_replay;
3025                 $do_check = 0;
3026             }
3027         } elsif (read_yn "read git log and continue?") {
3028             $replay = update_bisect_replay;
3029             $do_check = 0;
3030         }
3031     }
3032
3033     if ($do_check) {
3034
3035         # get current HEAD
3036         my $head = get_sha1("HEAD");
3037
3038         if ($check ne "good") {
3039             doprint "TESTING BISECT BAD [$bad]\n";
3040             run_command "git checkout $bad" or
3041                 dodie "Failed to checkout $bad";
3042
3043             $result = run_bisect $type;
3044
3045             if ($result ne "bad") {
3046                 fail "Tested BISECT_BAD [$bad] and it succeeded" and return 0;
3047             }
3048         }
3049
3050         if ($check ne "bad") {
3051             doprint "TESTING BISECT GOOD [$good]\n";
3052             run_command "git checkout $good" or
3053                 dodie "Failed to checkout $good";
3054
3055             $result = run_bisect $type;
3056
3057             if ($result ne "good") {
3058                 fail "Tested BISECT_GOOD [$good] and it failed" and return 0;
3059             }
3060         }
3061
3062         # checkout where we started
3063         run_command "git checkout $head" or
3064             dodie "Failed to checkout $head";
3065     }
3066
3067     run_command "git bisect start$start_files" or
3068         dodie "could not start bisect";
3069
3070     if (defined($replay)) {
3071         run_command "git bisect replay $replay" or
3072             dodie "failed to run replay";
3073     } else {
3074
3075         run_command "git bisect good $good" or
3076             dodie "could not set bisect good to $good";
3077
3078         run_git_bisect "git bisect bad $bad" or
3079             dodie "could not set bisect bad to $bad";
3080
3081     }
3082
3083     if (defined($start)) {
3084         run_command "git checkout $start" or
3085             dodie "failed to checkout $start";
3086     }
3087
3088     my $test;
3089     do {
3090         $result = run_bisect $type;
3091         $test = run_git_bisect "git bisect $result";
3092         print_times;
3093     } while ($test);
3094
3095     run_command "git bisect log" or
3096         dodie "could not capture git bisect log";
3097
3098     run_command "git bisect reset" or
3099         dodie "could not reset git bisect";
3100
3101     doprint "Bad commit was [$bisect_bad_commit]\n";
3102
3103     success $i;
3104 }
3105
3106 # config_ignore holds the configs that were set (or unset) for
3107 # a good config and we will ignore these configs for the rest
3108 # of a config bisect. These configs stay as they were.
3109 my %config_ignore;
3110
3111 # config_set holds what all configs were set as.
3112 my %config_set;
3113
3114 # config_off holds the set of configs that the bad config had disabled.
3115 # We need to record them and set them in the .config when running
3116 # olddefconfig, because olddefconfig keeps the defaults.
3117 my %config_off;
3118
3119 # config_off_tmp holds a set of configs to turn off for now
3120 my @config_off_tmp;
3121
3122 # config_list is the set of configs that are being tested
3123 my %config_list;
3124 my %null_config;
3125
3126 my %dependency;
3127
3128 sub assign_configs {
3129     my ($hash, $config) = @_;
3130
3131     doprint "Reading configs from $config\n";
3132
3133     open (IN, $config)
3134         or dodie "Failed to read $config";
3135
3136     while (<IN>) {
3137         chomp;
3138         if (/^((CONFIG\S*)=.*)/) {
3139             ${$hash}{$2} = $1;
3140         } elsif (/^(# (CONFIG\S*) is not set)/) {
3141             ${$hash}{$2} = $1;
3142         }
3143     }
3144
3145     close(IN);
3146 }
3147
3148 sub process_config_ignore {
3149     my ($config) = @_;
3150
3151     assign_configs \%config_ignore, $config;
3152 }
3153
3154 sub get_dependencies {
3155     my ($config) = @_;
3156
3157     my $arr = $dependency{$config};
3158     if (!defined($arr)) {
3159         return ();
3160     }
3161
3162     my @deps = @{$arr};
3163
3164     foreach my $dep (@{$arr}) {
3165         print "ADD DEP $dep\n";
3166         @deps = (@deps, get_dependencies $dep);
3167     }
3168
3169     return @deps;
3170 }
3171
3172 sub save_config {
3173     my ($pc, $file) = @_;
3174
3175     my %configs = %{$pc};
3176
3177     doprint "Saving configs into $file\n";
3178
3179     open(OUT, ">$file") or dodie "Can not write to $file";
3180
3181     foreach my $config (keys %configs) {
3182         print OUT "$configs{$config}\n";
3183     }
3184     close(OUT);
3185 }
3186
3187 sub create_config {
3188     my ($name, $pc) = @_;
3189
3190     doprint "Creating old config from $name configs\n";
3191
3192     save_config $pc, $output_config;
3193
3194     make_oldconfig;
3195 }
3196
3197 sub run_config_bisect_test {
3198     my ($type) = @_;
3199
3200     my $ret = run_bisect_test $type, "oldconfig";
3201
3202     if ($bisect_manual) {
3203         $ret = answer_bisect;
3204     }
3205
3206     return $ret;
3207 }
3208
3209 sub config_bisect_end {
3210     my ($good, $bad) = @_;
3211     my $diffexec = "diff -u";
3212
3213     if (-f "$builddir/scripts/diffconfig") {
3214         $diffexec = "$builddir/scripts/diffconfig";
3215     }
3216     doprint "\n\n***************************************\n";
3217     doprint "No more config bisecting possible.\n";
3218     run_command "$diffexec $good $bad", 1;
3219     doprint "***************************************\n\n";
3220 }
3221
3222 my $pass = 1;
3223
3224 sub run_config_bisect {
3225     my ($good, $bad, $last_result) = @_;
3226     my $reset = "";
3227     my $cmd;
3228     my $ret;
3229
3230     if (!length($last_result)) {
3231         $reset = "-r";
3232     }
3233     run_command "$config_bisect_exec $reset -b $outputdir $good $bad $last_result", 1;
3234
3235     # config-bisect returns:
3236     #   0 if there is more to bisect
3237     #   1 for finding a good config
3238     #   2 if it can not find any more configs
3239     #  -1 (255) on error
3240     if ($run_command_status) {
3241         return $run_command_status;
3242     }
3243
3244     $ret = run_config_bisect_test $config_bisect_type;
3245     if ($ret) {
3246         doprint "NEW GOOD CONFIG ($pass)\n";
3247         system("cp $output_config $tmpdir/good_config.tmp.$pass");
3248         $pass++;
3249         # Return 3 for good config
3250         return 3;
3251     } else {
3252         doprint "NEW BAD CONFIG ($pass)\n";
3253         system("cp $output_config $tmpdir/bad_config.tmp.$pass");
3254         $pass++;
3255         # Return 4 for bad config
3256         return 4;
3257     }
3258 }
3259
3260 sub config_bisect {
3261     my ($i) = @_;
3262
3263     my $good_config;
3264     my $bad_config;
3265
3266     my $type = $config_bisect_type;
3267     my $ret;
3268
3269     $bad_config = $config_bisect;
3270
3271     if (defined($config_bisect_good)) {
3272         $good_config = $config_bisect_good;
3273     } elsif (defined($minconfig)) {
3274         $good_config = $minconfig;
3275     } else {
3276         doprint "No config specified, checking if defconfig works";
3277         $ret = run_bisect_test $type, "defconfig";
3278         if (!$ret) {
3279             fail "Have no good config to compare with, please set CONFIG_BISECT_GOOD";
3280             return 1;
3281         }
3282         $good_config = $output_config;
3283     }
3284
3285     if (!defined($config_bisect_exec)) {
3286         # First check the location that ktest.pl ran
3287         my @locations = ( "$pwd/config-bisect.pl",
3288                           "$dirname/config-bisect.pl",
3289                           "$builddir/tools/testing/ktest/config-bisect.pl",
3290                           undef );
3291         foreach my $loc (@locations) {
3292             doprint "loc = $loc\n";
3293             $config_bisect_exec = $loc;
3294             last if (defined($config_bisect_exec && -x $config_bisect_exec));
3295         }
3296         if (!defined($config_bisect_exec)) {
3297             fail "Could not find an executable config-bisect.pl\n",
3298                 "  Set CONFIG_BISECT_EXEC to point to config-bisect.pl";
3299             return 1;
3300         }
3301     }
3302
3303     # we don't want min configs to cause issues here.
3304     doprint "Disabling 'MIN_CONFIG' for this test\n";
3305     undef $minconfig;
3306
3307     my %good_configs;
3308     my %bad_configs;
3309     my %tmp_configs;
3310
3311     if (-f "$tmpdir/good_config.tmp" || -f "$tmpdir/bad_config.tmp") {
3312         if (read_yn "Interrupted config-bisect. Continue (n - will start new)?") {
3313             if (-f "$tmpdir/good_config.tmp") {
3314                 $good_config = "$tmpdir/good_config.tmp";
3315             } else {
3316                 $good_config = "$tmpdir/good_config";
3317             }
3318             if (-f "$tmpdir/bad_config.tmp") {
3319                 $bad_config = "$tmpdir/bad_config.tmp";
3320             } else {
3321                 $bad_config = "$tmpdir/bad_config";
3322             }
3323         }
3324     }
3325     doprint "Run good configs through make oldconfig\n";
3326     assign_configs \%tmp_configs, $good_config;
3327     create_config "$good_config", \%tmp_configs;
3328     $good_config = "$tmpdir/good_config";
3329     system("cp $output_config $good_config") == 0 or dodie "cp good config";
3330
3331     doprint "Run bad configs through make oldconfig\n";
3332     assign_configs \%tmp_configs, $bad_config;
3333     create_config "$bad_config", \%tmp_configs;
3334     $bad_config = "$tmpdir/bad_config";
3335     system("cp $output_config $bad_config") == 0 or dodie "cp bad config";
3336
3337     if (defined($config_bisect_check) && $config_bisect_check ne "0") {
3338         if ($config_bisect_check ne "good") {
3339             doprint "Testing bad config\n";
3340
3341             $ret = run_bisect_test $type, "useconfig:$bad_config";
3342             if ($ret) {
3343                 fail "Bad config succeeded when expected to fail!";
3344                 return 0;
3345             }
3346         }
3347         if ($config_bisect_check ne "bad") {
3348             doprint "Testing good config\n";
3349
3350             $ret = run_bisect_test $type, "useconfig:$good_config";
3351             if (!$ret) {
3352                 fail "Good config failed when expected to succeed!";
3353                 return 0;
3354             }
3355         }
3356     }
3357
3358     my $last_run = "";
3359
3360     do {
3361         $ret = run_config_bisect $good_config, $bad_config, $last_run;
3362         if ($ret == 3) {
3363             $last_run = "good";
3364         } elsif ($ret == 4) {
3365             $last_run = "bad";
3366         }
3367         print_times;
3368     } while ($ret == 3 || $ret == 4);
3369
3370     if ($ret == 2) {
3371         config_bisect_end "$good_config.tmp", "$bad_config.tmp";
3372     }
3373
3374     return $ret if ($ret < 0);
3375
3376     success $i;
3377 }
3378
3379 sub patchcheck_reboot {
3380     doprint "Reboot and sleep $patchcheck_sleep_time seconds\n";
3381     reboot_to_good $patchcheck_sleep_time;
3382 }
3383
3384 sub patchcheck {
3385     my ($i) = @_;
3386
3387     dodie "PATCHCHECK_START[$i] not defined\n"
3388         if (!defined($patchcheck_start));
3389     dodie "PATCHCHECK_TYPE[$i] not defined\n"
3390         if (!defined($patchcheck_type));
3391
3392     my $start = $patchcheck_start;
3393
3394     my $cherry = $patchcheck_cherry;
3395     if (!defined($cherry)) {
3396         $cherry = 0;
3397     }
3398
3399     my $end = "HEAD";
3400     if (defined($patchcheck_end)) {
3401         $end = $patchcheck_end;
3402     } elsif ($cherry) {
3403         dodie "PATCHCHECK_END must be defined with PATCHCHECK_CHERRY\n";
3404     }
3405
3406     # Get the true sha1's since we can use things like HEAD~3
3407     $start = get_sha1($start);
3408     $end = get_sha1($end);
3409
3410     my $type = $patchcheck_type;
3411
3412     # Can't have a test without having a test to run
3413     if ($type eq "test" && !defined($run_test)) {
3414         $type = "boot";
3415     }
3416
3417     if ($cherry) {
3418         open (IN, "git cherry -v $start $end|") or
3419             dodie "could not get git list";
3420     } else {
3421         open (IN, "git log --pretty=oneline $end|") or
3422             dodie "could not get git list";
3423     }
3424
3425     my @list;
3426
3427     while (<IN>) {
3428         chomp;
3429         # git cherry adds a '+' we want to remove
3430         s/^\+ //;
3431         $list[$#list+1] = $_;
3432         last if (/^$start/);
3433     }
3434     close(IN);
3435
3436     if (!$cherry) {
3437         if ($list[$#list] !~ /^$start/) {
3438             fail "SHA1 $start not found";
3439         }
3440
3441         # go backwards in the list
3442         @list = reverse @list;
3443     }
3444
3445     doprint("Going to test the following commits:\n");
3446     foreach my $l (@list) {
3447         doprint "$l\n";
3448     }
3449
3450     my $save_clean = $noclean;
3451     my %ignored_warnings;
3452
3453     if (defined($ignore_warnings)) {
3454         foreach my $sha1 (split /\s+/, $ignore_warnings) {
3455             $ignored_warnings{$sha1} = 1;
3456         }
3457     }
3458
3459     $in_patchcheck = 1;
3460     foreach my $item (@list) {
3461         my $sha1 = $item;
3462         $sha1 =~ s/^([[:xdigit:]]+).*/$1/;
3463
3464         doprint "\nProcessing commit \"$item\"\n\n";
3465
3466         run_command "git checkout $sha1" or
3467             dodie "Failed to checkout $sha1";
3468
3469         # only clean on the first and last patch
3470         if ($item eq $list[0] ||
3471             $item eq $list[$#list]) {
3472             $noclean = $save_clean;
3473         } else {
3474             $noclean = 1;
3475         }
3476
3477         if (defined($minconfig)) {
3478             build "useconfig:$minconfig" or return 0;
3479         } else {
3480             # ?? no config to use?
3481             build "oldconfig" or return 0;
3482         }
3483
3484         # No need to do per patch checking if warnings file exists
3485         if (!defined($warnings_file) && !defined($ignored_warnings{$sha1})) {
3486             check_patch_buildlog $sha1 or return 0;
3487         }
3488
3489         check_buildlog or return 0;
3490
3491         next if ($type eq "build");
3492
3493         my $failed = 0;
3494
3495         start_monitor_and_install or $failed = 1;
3496
3497         if (!$failed && $type ne "boot"){
3498             do_run_test or $failed = 1;
3499         }
3500         end_monitor;
3501         if ($failed) {
3502             print_times;
3503             return 0;
3504         }
3505         patchcheck_reboot;
3506         print_times;
3507     }
3508     $in_patchcheck = 0;
3509     success $i;
3510
3511     return 1;
3512 }
3513
3514 my %depends;
3515 my %depcount;
3516 my $iflevel = 0;
3517 my @ifdeps;
3518
3519 # prevent recursion
3520 my %read_kconfigs;
3521
3522 sub add_dep {
3523     # $config depends on $dep
3524     my ($config, $dep) = @_;
3525
3526     if (defined($depends{$config})) {
3527         $depends{$config} .= " " . $dep;
3528     } else {
3529         $depends{$config} = $dep;
3530     }
3531
3532     # record the number of configs depending on $dep
3533     if (defined $depcount{$dep}) {
3534         $depcount{$dep}++;
3535     } else {
3536         $depcount{$dep} = 1;
3537     }
3538 }
3539
3540 # taken from streamline_config.pl
3541 sub read_kconfig {
3542     my ($kconfig) = @_;
3543
3544     my $state = "NONE";
3545     my $config;
3546     my @kconfigs;
3547
3548     my $cont = 0;
3549     my $line;
3550
3551
3552     if (! -f $kconfig) {
3553         doprint "file $kconfig does not exist, skipping\n";
3554         return;
3555     }
3556
3557     open(KIN, "$kconfig")
3558         or dodie "Can't open $kconfig";
3559     while (<KIN>) {
3560         chomp;
3561
3562         # Make sure that lines ending with \ continue
3563         if ($cont) {
3564             $_ = $line . " " . $_;
3565         }
3566
3567         if (s/\\$//) {
3568             $cont = 1;
3569             $line = $_;
3570             next;
3571         }
3572
3573         $cont = 0;
3574
3575         # collect any Kconfig sources
3576         if (/^source\s*"(.*)"/) {
3577             $kconfigs[$#kconfigs+1] = $1;
3578         }
3579
3580         # configs found
3581         if (/^\s*(menu)?config\s+(\S+)\s*$/) {
3582             $state = "NEW";
3583             $config = $2;
3584
3585             for (my $i = 0; $i < $iflevel; $i++) {
3586                 add_dep $config, $ifdeps[$i];
3587             }
3588
3589         # collect the depends for the config
3590         } elsif ($state eq "NEW" && /^\s*depends\s+on\s+(.*)$/) {
3591
3592             add_dep $config, $1;
3593
3594         # Get the configs that select this config
3595         } elsif ($state eq "NEW" && /^\s*select\s+(\S+)/) {
3596
3597             # selected by depends on config
3598             add_dep $1, $config;
3599
3600         # Check for if statements
3601         } elsif (/^if\s+(.*\S)\s*$/) {
3602             my $deps = $1;
3603             # remove beginning and ending non text
3604             $deps =~ s/^[^a-zA-Z0-9_]*//;
3605             $deps =~ s/[^a-zA-Z0-9_]*$//;
3606
3607             my @deps = split /[^a-zA-Z0-9_]+/, $deps;
3608
3609             $ifdeps[$iflevel++] = join ':', @deps;
3610
3611         } elsif (/^endif/) {
3612
3613             $iflevel-- if ($iflevel);
3614
3615         # stop on "help"
3616         } elsif (/^\s*help\s*$/) {
3617             $state = "NONE";
3618         }
3619     }
3620     close(KIN);
3621
3622     # read in any configs that were found.
3623     foreach $kconfig (@kconfigs) {
3624         if (!defined($read_kconfigs{$kconfig})) {
3625             $read_kconfigs{$kconfig} = 1;
3626             read_kconfig("$builddir/$kconfig");
3627         }
3628     }
3629 }
3630
3631 sub read_depends {
3632     # find out which arch this is by the kconfig file
3633     open (IN, $output_config)
3634         or dodie "Failed to read $output_config";
3635     my $arch;
3636     while (<IN>) {
3637         if (m,Linux/(\S+)\s+\S+\s+Kernel Configuration,) {
3638             $arch = $1;
3639             last;
3640         }
3641     }
3642     close IN;
3643
3644     if (!defined($arch)) {
3645         doprint "Could not find arch from config file\n";
3646         doprint "no dependencies used\n";
3647         return;
3648     }
3649
3650     # arch is really the subarch, we need to know
3651     # what directory to look at.
3652     if ($arch eq "i386" || $arch eq "x86_64") {
3653         $arch = "x86";
3654     }
3655
3656     my $kconfig = "$builddir/arch/$arch/Kconfig";
3657
3658     if (! -f $kconfig && $arch =~ /\d$/) {
3659         my $orig = $arch;
3660         # some subarchs have numbers, truncate them
3661         $arch =~ s/\d*$//;
3662         $kconfig = "$builddir/arch/$arch/Kconfig";
3663         if (! -f $kconfig) {
3664             doprint "No idea what arch dir $orig is for\n";
3665             doprint "no dependencies used\n";
3666             return;
3667         }
3668     }
3669
3670     read_kconfig($kconfig);
3671 }
3672
3673 sub make_new_config {
3674     my @configs = @_;
3675
3676     open (OUT, ">$output_config")
3677         or dodie "Failed to write $output_config";
3678
3679     foreach my $config (@configs) {
3680         print OUT "$config\n";
3681     }
3682     close OUT;
3683 }
3684
3685 sub chomp_config {
3686     my ($config) = @_;
3687
3688     $config =~ s/CONFIG_//;
3689
3690     return $config;
3691 }
3692
3693 sub get_depends {
3694     my ($dep) = @_;
3695
3696     my $kconfig = chomp_config $dep;
3697
3698     $dep = $depends{"$kconfig"};
3699
3700     # the dep string we have saves the dependencies as they
3701     # were found, including expressions like ! && ||. We
3702     # want to split this out into just an array of configs.
3703
3704     my $valid = "A-Za-z_0-9";
3705
3706     my @configs;
3707
3708     while ($dep =~ /[$valid]/) {
3709
3710         if ($dep =~ /^[^$valid]*([$valid]+)/) {
3711             my $conf = "CONFIG_" . $1;
3712
3713             $configs[$#configs + 1] = $conf;
3714
3715             $dep =~ s/^[^$valid]*[$valid]+//;
3716         } else {
3717             dodie "this should never happen";
3718         }
3719     }
3720
3721     return @configs;
3722 }
3723
3724 my %min_configs;
3725 my %keep_configs;
3726 my %save_configs;
3727 my %processed_configs;
3728 my %nochange_config;
3729
3730 sub test_this_config {
3731     my ($config) = @_;
3732
3733     my $found;
3734
3735     # if we already processed this config, skip it
3736     if (defined($processed_configs{$config})) {
3737         return undef;
3738     }
3739     $processed_configs{$config} = 1;
3740
3741     # if this config failed during this round, skip it
3742     if (defined($nochange_config{$config})) {
3743         return undef;
3744     }
3745
3746     my $kconfig = chomp_config $config;
3747
3748     # Test dependencies first
3749     if (defined($depends{"$kconfig"})) {
3750         my @parents = get_depends $config;
3751         foreach my $parent (@parents) {
3752             # if the parent is in the min config, check it first
3753             next if (!defined($min_configs{$parent}));
3754             $found = test_this_config($parent);
3755             if (defined($found)) {
3756                 return $found;
3757             }
3758         }
3759     }
3760
3761     # Remove this config from the list of configs
3762     # do a make olddefconfig and then read the resulting
3763     # .config to make sure it is missing the config that
3764     # we had before
3765     my %configs = %min_configs;
3766     delete $configs{$config};
3767     make_new_config ((values %configs), (values %keep_configs));
3768     make_oldconfig;
3769     undef %configs;
3770     assign_configs \%configs, $output_config;
3771
3772     if (!defined($configs{$config}) || $configs{$config} =~ /^#/) {
3773         return $config;
3774     }
3775
3776     doprint "disabling config $config did not change .config\n";
3777
3778     $nochange_config{$config} = 1;
3779
3780     return undef;
3781 }
3782
3783 sub make_min_config {
3784     my ($i) = @_;
3785
3786     my $type = $minconfig_type;
3787     if ($type ne "boot" && $type ne "test") {
3788         fail "Invalid MIN_CONFIG_TYPE '$minconfig_type'\n" .
3789             " make_min_config works only with 'boot' and 'test'\n" and return;
3790     }
3791
3792     if (!defined($output_minconfig)) {
3793         fail "OUTPUT_MIN_CONFIG not defined" and return;
3794     }
3795
3796     # If output_minconfig exists, and the start_minconfig
3797     # came from min_config, than ask if we should use
3798     # that instead.
3799     if (-f $output_minconfig && !$start_minconfig_defined) {
3800         print "$output_minconfig exists\n";
3801         if (!defined($use_output_minconfig)) {
3802             if (read_yn " Use it as minconfig?") {
3803                 $start_minconfig = $output_minconfig;
3804             }
3805         } elsif ($use_output_minconfig > 0) {
3806             doprint "Using $output_minconfig as MIN_CONFIG\n";
3807             $start_minconfig = $output_minconfig;
3808         } else {
3809             doprint "Set to still use MIN_CONFIG as starting point\n";
3810         }
3811     }
3812
3813     if (!defined($start_minconfig)) {
3814         fail "START_MIN_CONFIG or MIN_CONFIG not defined" and return;
3815     }
3816
3817     my $temp_config = "$tmpdir/temp_config";
3818
3819     # First things first. We build an allnoconfig to find
3820     # out what the defaults are that we can't touch.
3821     # Some are selections, but we really can't handle selections.
3822
3823     my $save_minconfig = $minconfig;
3824     undef $minconfig;
3825
3826     run_command "$make allnoconfig" or return 0;
3827
3828     read_depends;
3829
3830     process_config_ignore $output_config;
3831
3832     undef %save_configs;
3833     undef %min_configs;
3834
3835     if (defined($ignore_config)) {
3836         # make sure the file exists
3837         `touch $ignore_config`;
3838         assign_configs \%save_configs, $ignore_config;
3839     }
3840
3841     %keep_configs = %save_configs;
3842
3843     doprint "Load initial configs from $start_minconfig\n";
3844
3845     # Look at the current min configs, and save off all the
3846     # ones that were set via the allnoconfig
3847     assign_configs \%min_configs, $start_minconfig;
3848
3849     my @config_keys = keys %min_configs;
3850
3851     # All configs need a depcount
3852     foreach my $config (@config_keys) {
3853         my $kconfig = chomp_config $config;
3854         if (!defined $depcount{$kconfig}) {
3855                 $depcount{$kconfig} = 0;
3856         }
3857     }
3858
3859     # Remove anything that was set by the make allnoconfig
3860     # we shouldn't need them as they get set for us anyway.
3861     foreach my $config (@config_keys) {
3862         # Remove anything in the ignore_config
3863         if (defined($keep_configs{$config})) {
3864             my $file = $ignore_config;
3865             $file =~ s,.*/(.*?)$,$1,;
3866             doprint "$config set by $file ... ignored\n";
3867             delete $min_configs{$config};
3868             next;
3869         }
3870         # But make sure the settings are the same. If a min config
3871         # sets a selection, we do not want to get rid of it if
3872         # it is not the same as what we have. Just move it into
3873         # the keep configs.
3874         if (defined($config_ignore{$config})) {
3875             if ($config_ignore{$config} ne $min_configs{$config}) {
3876                 doprint "$config is in allnoconfig as '$config_ignore{$config}'";
3877                 doprint " but it is '$min_configs{$config}' in minconfig .. keeping\n";
3878                 $keep_configs{$config} = $min_configs{$config};
3879             } else {
3880                 doprint "$config set by allnoconfig ... ignored\n";
3881             }
3882             delete $min_configs{$config};
3883         }
3884     }
3885
3886     my $done = 0;
3887     my $take_two = 0;
3888
3889     while (!$done) {
3890
3891         my $config;
3892         my $found;
3893
3894         # Now disable each config one by one and do a make oldconfig
3895         # till we find a config that changes our list.
3896
3897         my @test_configs = keys %min_configs;
3898
3899         # Sort keys by who is most dependent on
3900         @test_configs = sort  { $depcount{chomp_config($b)} <=> $depcount{chomp_config($a)} }
3901                           @test_configs ;
3902
3903         # Put configs that did not modify the config at the end.
3904         my $reset = 1;
3905         for (my $i = 0; $i < $#test_configs; $i++) {
3906             if (!defined($nochange_config{$test_configs[0]})) {
3907                 $reset = 0;
3908                 last;
3909             }
3910             # This config didn't change the .config last time.
3911             # Place it at the end
3912             my $config = shift @test_configs;
3913             push @test_configs, $config;
3914         }
3915
3916         # if every test config has failed to modify the .config file
3917         # in the past, then reset and start over.
3918         if ($reset) {
3919             undef %nochange_config;
3920         }
3921
3922         undef %processed_configs;
3923
3924         foreach my $config (@test_configs) {
3925
3926             $found = test_this_config $config;
3927
3928             last if (defined($found));
3929
3930             # oh well, try another config
3931         }
3932
3933         if (!defined($found)) {
3934             # we could have failed due to the nochange_config hash
3935             # reset and try again
3936             if (!$take_two) {
3937                 undef %nochange_config;
3938                 $take_two = 1;
3939                 next;
3940             }
3941             doprint "No more configs found that we can disable\n";
3942             $done = 1;
3943             last;
3944         }
3945         $take_two = 0;
3946
3947         $config = $found;
3948
3949         doprint "Test with $config disabled\n";
3950
3951         # set in_bisect to keep build and monitor from dieing
3952         $in_bisect = 1;
3953
3954         my $failed = 0;
3955         build "oldconfig" or $failed = 1;
3956         if (!$failed) {
3957                 start_monitor_and_install or $failed = 1;
3958
3959                 if ($type eq "test" && !$failed) {
3960                     do_run_test or $failed = 1;
3961                 }
3962
3963                 end_monitor;
3964         }
3965
3966         $in_bisect = 0;
3967
3968         if ($failed) {
3969             doprint "$min_configs{$config} is needed to boot the box... keeping\n";
3970             # this config is needed, add it to the ignore list.
3971             $keep_configs{$config} = $min_configs{$config};
3972             $save_configs{$config} = $min_configs{$config};
3973             delete $min_configs{$config};
3974
3975             # update new ignore configs
3976             if (defined($ignore_config)) {
3977                 open (OUT, ">$temp_config")
3978                     or dodie "Can't write to $temp_config";
3979                 foreach my $config (keys %save_configs) {
3980                     print OUT "$save_configs{$config}\n";
3981                 }
3982                 close OUT;
3983                 run_command "mv $temp_config $ignore_config" or
3984                     dodie "failed to copy update to $ignore_config";
3985             }
3986
3987         } else {
3988             # We booted without this config, remove it from the minconfigs.
3989             doprint "$config is not needed, disabling\n";
3990
3991             delete $min_configs{$config};
3992
3993             # Also disable anything that is not enabled in this config
3994             my %configs;
3995             assign_configs \%configs, $output_config;
3996             my @config_keys = keys %min_configs;
3997             foreach my $config (@config_keys) {
3998                 if (!defined($configs{$config})) {
3999                     doprint "$config is not set, disabling\n";
4000                     delete $min_configs{$config};
4001                 }
4002             }
4003
4004             # Save off all the current mandatory configs
4005             open (OUT, ">$temp_config")
4006                 or dodie "Can't write to $temp_config";
4007             foreach my $config (keys %keep_configs) {
4008                 print OUT "$keep_configs{$config}\n";
4009             }
4010             foreach my $config (keys %min_configs) {
4011                 print OUT "$min_configs{$config}\n";
4012             }
4013             close OUT;
4014
4015             run_command "mv $temp_config $output_minconfig" or
4016                 dodie "failed to copy update to $output_minconfig";
4017         }
4018
4019         doprint "Reboot and wait $sleep_time seconds\n";
4020         reboot_to_good $sleep_time;
4021     }
4022
4023     success $i;
4024     return 1;
4025 }
4026
4027 sub make_warnings_file {
4028     my ($i) = @_;
4029
4030     if (!defined($warnings_file)) {
4031         dodie "Must define WARNINGS_FILE for make_warnings_file test";
4032     }
4033
4034     if ($build_type eq "nobuild") {
4035         dodie "BUILD_TYPE can not be 'nobuild' for make_warnings_file test";
4036     }
4037
4038     build $build_type or dodie "Failed to build";
4039
4040     open(OUT, ">$warnings_file") or dodie "Can't create $warnings_file";
4041
4042     open(IN, $buildlog) or dodie "Can't open $buildlog";
4043     while (<IN>) {
4044
4045         # Some compilers use UTF-8 extended for quotes
4046         # for distcc heterogeneous systems, this causes issues
4047         s/$utf8_quote/'/g;
4048
4049         if (/$check_build_re/) {
4050             print OUT;
4051         }
4052     }
4053     close(IN);
4054
4055     close(OUT);
4056
4057     success $i;
4058 }
4059
4060 $#ARGV < 1 or die "ktest.pl version: $VERSION\n   usage: ktest.pl [config-file]\n";
4061
4062 if ($#ARGV == 0) {
4063     $ktest_config = $ARGV[0];
4064     if (! -f $ktest_config) {
4065         print "$ktest_config does not exist.\n";
4066         if (!read_yn "Create it?") {
4067             exit 0;
4068         }
4069     }
4070 }
4071
4072 if (! -f $ktest_config) {
4073     $newconfig = 1;
4074     get_test_case;
4075     open(OUT, ">$ktest_config") or die "Can not create $ktest_config";
4076     print OUT << "EOF"
4077 # Generated by ktest.pl
4078 #
4079
4080 # PWD is a ktest.pl variable that will result in the process working
4081 # directory that ktest.pl is executed in.
4082
4083 # THIS_DIR is automatically assigned the PWD of the path that generated
4084 # the config file. It is best to use this variable when assigning other
4085 # directory paths within this directory. This allows you to easily
4086 # move the test cases to other locations or to other machines.
4087 #
4088 THIS_DIR := $variable{"PWD"}
4089
4090 # Define each test with TEST_START
4091 # The config options below it will override the defaults
4092 TEST_START
4093 TEST_TYPE = $default{"TEST_TYPE"}
4094
4095 DEFAULTS
4096 EOF
4097 ;
4098     close(OUT);
4099 }
4100 read_config $ktest_config;
4101
4102 if (defined($opt{"LOG_FILE"})) {
4103     $opt{"LOG_FILE"} = eval_option("LOG_FILE", $opt{"LOG_FILE"}, -1);
4104 }
4105
4106 # Append any configs entered in manually to the config file.
4107 my @new_configs = keys %entered_configs;
4108 if ($#new_configs >= 0) {
4109     print "\nAppending entered in configs to $ktest_config\n";
4110     open(OUT, ">>$ktest_config") or die "Can not append to $ktest_config";
4111     foreach my $config (@new_configs) {
4112         print OUT "$config = $entered_configs{$config}\n";
4113         $opt{$config} = process_variables($entered_configs{$config});
4114     }
4115 }
4116
4117 if (defined($opt{"LOG_FILE"})) {
4118     if ($opt{"CLEAR_LOG"}) {
4119         unlink $opt{"LOG_FILE"};
4120     }
4121     open(LOG, ">> $opt{LOG_FILE}") or die "Can't write to $opt{LOG_FILE}";
4122     LOG->autoflush(1);
4123 }
4124
4125 doprint "\n\nSTARTING AUTOMATED TESTS\n\n";
4126
4127 for (my $i = 0, my $repeat = 1; $i <= $opt{"NUM_TESTS"}; $i += $repeat) {
4128
4129     if (!$i) {
4130         doprint "DEFAULT OPTIONS:\n";
4131     } else {
4132         doprint "\nTEST $i OPTIONS";
4133         if (defined($repeat_tests{$i})) {
4134             $repeat = $repeat_tests{$i};
4135             doprint " ITERATE $repeat";
4136         }
4137         doprint "\n";
4138     }
4139
4140     foreach my $option (sort keys %opt) {
4141
4142         if ($option =~ /\[(\d+)\]$/) {
4143             next if ($i != $1);
4144         } else {
4145             next if ($i);
4146         }
4147
4148         doprint "$option = $opt{$option}\n";
4149     }
4150 }
4151
4152 sub option_defined {
4153     my ($option) = @_;
4154
4155     if (defined($opt{$option}) && $opt{$option} !~ /^\s*$/) {
4156         return 1;
4157     }
4158
4159     return 0;
4160 }
4161
4162 sub __set_test_option {
4163     my ($name, $i) = @_;
4164
4165     my $option = "$name\[$i\]";
4166
4167     if (option_defined($option)) {
4168         return $opt{$option};
4169     }
4170
4171     foreach my $test (keys %repeat_tests) {
4172         if ($i >= $test &&
4173             $i < $test + $repeat_tests{$test}) {
4174             $option = "$name\[$test\]";
4175             if (option_defined($option)) {
4176                 return $opt{$option};
4177             }
4178         }
4179     }
4180
4181     if (option_defined($name)) {
4182         return $opt{$name};
4183     }
4184
4185     return undef;
4186 }
4187
4188 sub set_test_option {
4189     my ($name, $i) = @_;
4190
4191     my $option = __set_test_option($name, $i);
4192     return $option if (!defined($option));
4193
4194     return eval_option($name, $option, $i);
4195 }
4196
4197 sub find_mailer {
4198     my ($mailer) = @_;
4199
4200     my @paths = split /:/, $ENV{PATH};
4201
4202     # sendmail is usually in /usr/sbin
4203     $paths[$#paths + 1] = "/usr/sbin";
4204
4205     foreach my $path (@paths) {
4206         if (-x "$path/$mailer") {
4207             return $path;
4208         }
4209     }
4210
4211     return undef;
4212 }
4213
4214 sub do_send_mail {
4215     my ($subject, $message, $file) = @_;
4216
4217     if (!defined($mail_path)) {
4218         # find the mailer
4219         $mail_path = find_mailer $mailer;
4220         if (!defined($mail_path)) {
4221             die "\nCan not find $mailer in PATH\n";
4222         }
4223     }
4224
4225     my $header_file = "$tmpdir/header";
4226     open (HEAD, ">$header_file") or die "Can not create $header_file\n";
4227     print HEAD "To: $mailto\n";
4228     print HEAD "Subject: $subject\n\n";
4229     print HEAD "$message\n";
4230     close HEAD;
4231
4232     if (!defined($mail_command)) {
4233         if ($mailer eq "mail" || $mailer eq "mailx") {
4234             $mail_command = "cat \$HEADER_FILE \$BODY_FILE | \$MAIL_PATH/\$MAILER -s \'\$SUBJECT\' \$MAILTO";
4235         } elsif ($mailer eq "sendmail" ) {
4236             $mail_command =  "cat \$HEADER_FILE \$BODY_FILE | \$MAIL_PATH/\$MAILER -t \$MAILTO";
4237         } else {
4238             die "\nYour mailer: $mailer is not supported.\n";
4239         }
4240     }
4241
4242     if (defined($file)) {
4243         $mail_command =~ s/\$BODY_FILE/$file/g;
4244     } else {
4245         $mail_command =~ s/\$BODY_FILE//g;
4246     }
4247
4248     $mail_command =~ s/\$HEADER_FILE/$header_file/g;
4249     $mail_command =~ s/\$MAILER/$mailer/g;
4250     $mail_command =~ s/\$MAIL_PATH/$mail_path/g;
4251     $mail_command =~ s/\$MAILTO/$mailto/g;
4252     $mail_command =~ s/\$SUBJECT/$subject/g;
4253     $mail_command =~ s/\$MESSAGE/$message/g;
4254
4255     my $ret = run_command $mail_command;
4256     if (!$ret && defined($file)) {
4257         # try again without the file
4258         $message .= "\n\n*** FAILED TO SEND LOG ***\n\n";
4259         do_send_email($subject, $message);
4260     }
4261 }
4262
4263 sub send_email {
4264
4265     if (defined($mailto)) {
4266         if (!defined($mailer)) {
4267             doprint "No email sent: email or mailer not specified in config.\n";
4268             return;
4269         }
4270         do_send_mail @_;
4271     }
4272 }
4273
4274 sub cancel_test {
4275     if ($email_when_canceled) {
4276         my $name = get_test_name;
4277         send_email("KTEST: Your [$name] test was cancelled",
4278                 "Your test started at $script_start_time was cancelled: sig int");
4279     }
4280     die "\nCaught Sig Int, test interrupted: $!\n"
4281 }
4282
4283 $SIG{INT} = qw(cancel_test);
4284
4285 # First we need to do is the builds
4286 for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) {
4287
4288     # Do not reboot on failing test options
4289     $no_reboot = 1;
4290     $reboot_success = 0;
4291
4292     $have_version = 0;
4293
4294     $iteration = $i;
4295
4296     $build_time = 0;
4297     $install_time = 0;
4298     $reboot_time = 0;
4299     $test_time = 0;
4300
4301     undef %force_config;
4302
4303     my $makecmd = set_test_option("MAKE_CMD", $i);
4304
4305     $outputdir = set_test_option("OUTPUT_DIR", $i);
4306     $builddir = set_test_option("BUILD_DIR", $i);
4307
4308     chdir $builddir || dodie "can't change directory to $builddir";
4309
4310     if (!-d $outputdir) {
4311         mkpath($outputdir) or
4312             dodie "can't create $outputdir";
4313     }
4314
4315     $make = "$makecmd O=$outputdir";
4316
4317     # Load all the options into their mapped variable names
4318     foreach my $opt (keys %option_map) {
4319         ${$option_map{$opt}} = set_test_option($opt, $i);
4320     }
4321
4322     $start_minconfig_defined = 1;
4323
4324     # The first test may override the PRE_KTEST option
4325     if ($i == 1) {
4326         if (defined($pre_ktest)) {
4327             doprint "\n";
4328             run_command $pre_ktest;
4329         }
4330         if ($email_when_started) {
4331             my $name = get_test_name;
4332             send_email("KTEST: Your [$name] test was started",
4333                 "Your test was started on $script_start_time");
4334         }
4335     }
4336
4337     # Any test can override the POST_KTEST option
4338     # The last test takes precedence.
4339     if (defined($post_ktest)) {
4340         $final_post_ktest = $post_ktest;
4341     }
4342
4343     if (!defined($start_minconfig)) {
4344         $start_minconfig_defined = 0;
4345         $start_minconfig = $minconfig;
4346     }
4347
4348     if (!-d $tmpdir) {
4349         mkpath($tmpdir) or
4350             dodie "can't create $tmpdir";
4351     }
4352
4353     $ENV{"SSH_USER"} = $ssh_user;
4354     $ENV{"MACHINE"} = $machine;
4355
4356     $buildlog = "$tmpdir/buildlog-$machine";
4357     $testlog = "$tmpdir/testlog-$machine";
4358     $dmesg = "$tmpdir/dmesg-$machine";
4359     $output_config = "$outputdir/.config";
4360
4361     if (!$buildonly) {
4362         $target = "$ssh_user\@$machine";
4363         if (($reboot_type eq "grub") or ($reboot_type eq "grub2bls")) {
4364             dodie "GRUB_MENU not defined" if (!defined($grub_menu));
4365         } elsif ($reboot_type eq "grub2") {
4366             dodie "GRUB_MENU not defined" if (!defined($grub_menu));
4367             dodie "GRUB_FILE not defined" if (!defined($grub_file));
4368         } elsif ($reboot_type eq "syslinux") {
4369             dodie "SYSLINUX_LABEL not defined" if (!defined($syslinux_label));
4370         }
4371     }
4372
4373     my $run_type = $build_type;
4374     if ($test_type eq "patchcheck") {
4375         $run_type = $patchcheck_type;
4376     } elsif ($test_type eq "bisect") {
4377         $run_type = $bisect_type;
4378     } elsif ($test_type eq "config_bisect") {
4379         $run_type = $config_bisect_type;
4380     } elsif ($test_type eq "make_min_config") {
4381         $run_type = "";
4382     } elsif ($test_type eq "make_warnings_file") {
4383         $run_type = "";
4384     }
4385
4386     # mistake in config file?
4387     if (!defined($run_type)) {
4388         $run_type = "ERROR";
4389     }
4390
4391     my $installme = "";
4392     $installme = " no_install" if ($no_install);
4393
4394     my $name = "";
4395
4396     if (defined($test_name)) {
4397         $name = " ($test_name)";
4398     }
4399
4400     doprint "\n\n";
4401
4402     if (defined($opt{"LOG_FILE"})) {
4403         $test_log_start = tell(LOG);
4404     }
4405
4406     doprint "RUNNING TEST $i of $opt{NUM_TESTS}$name with option $test_type $run_type$installme\n\n";
4407
4408     if (defined($pre_test)) {
4409         my $ret = run_command $pre_test;
4410         if (!$ret && defined($pre_test_die) &&
4411             $pre_test_die) {
4412             dodie "failed to pre_test\n";
4413         }
4414     }
4415
4416     unlink $dmesg;
4417     unlink $buildlog;
4418     unlink $testlog;
4419
4420     if (defined($addconfig)) {
4421         my $min = $minconfig;
4422         if (!defined($minconfig)) {
4423             $min = "";
4424         }
4425         run_command "cat $addconfig $min > $tmpdir/add_config" or
4426             dodie "Failed to create temp config";
4427         $minconfig = "$tmpdir/add_config";
4428     }
4429
4430     if (defined($checkout)) {
4431         run_command "git checkout $checkout" or
4432             dodie "failed to checkout $checkout";
4433     }
4434
4435     $no_reboot = 0;
4436
4437     # A test may opt to not reboot the box
4438     if ($reboot_on_success) {
4439         $reboot_success = 1;
4440     }
4441
4442     if ($test_type eq "bisect") {
4443         bisect $i;
4444         next;
4445     } elsif ($test_type eq "config_bisect") {
4446         config_bisect $i;
4447         next;
4448     } elsif ($test_type eq "patchcheck") {
4449         patchcheck $i;
4450         next;
4451     } elsif ($test_type eq "make_min_config") {
4452         make_min_config $i;
4453         next;
4454     } elsif ($test_type eq "make_warnings_file") {
4455         $no_reboot = 1;
4456         make_warnings_file $i;
4457         next;
4458     }
4459
4460     if ($build_type ne "nobuild") {
4461         build $build_type or next;
4462         check_buildlog or next;
4463     }
4464
4465     if ($test_type eq "install") {
4466         get_version;
4467         install;
4468         success $i;
4469         next;
4470     }
4471
4472     if ($test_type ne "build") {
4473         my $failed = 0;
4474         start_monitor_and_install or $failed = 1;
4475
4476         if (!$failed && $test_type ne "boot" && defined($run_test)) {
4477             do_run_test or $failed = 1;
4478         }
4479         end_monitor;
4480         if ($failed) {
4481             print_times;
4482             next;
4483         }
4484     }
4485
4486     print_times;
4487
4488     success $i;
4489 }
4490
4491 if (defined($final_post_ktest)) {
4492
4493     my $cp_final_post_ktest = eval_kernel_version $final_post_ktest;
4494     run_command $cp_final_post_ktest;
4495 }
4496
4497 if ($opt{"POWEROFF_ON_SUCCESS"}) {
4498     halt;
4499 } elsif ($opt{"REBOOT_ON_SUCCESS"} && !do_not_reboot && $reboot_success) {
4500     reboot_to_good;
4501 } elsif (defined($switch_to_good)) {
4502     # still need to get to the good kernel
4503     run_command $switch_to_good;
4504 }
4505
4506
4507 doprint "\n    $successes of $opt{NUM_TESTS} tests were successful\n\n";
4508
4509 if ($email_when_finished) {
4510     send_email("KTEST: Your test has finished!",
4511             "$successes of $opt{NUM_TESTS} tests started at $script_start_time were successful!");
4512 }
4513
4514 if (defined($opt{"LOG_FILE"})) {
4515     print "\n See $opt{LOG_FILE} for the record of results.\n\n";
4516     close LOG;
4517 }
4518
4519 exit 0;