tools/testing/ktest/ktest.pl

   1 #!/usr/bin/perl -w
   2 # SPDX-License-Identifier: GPL-2.0-only
   3 #
   4 # Copyright 2010 - Steven Rostedt <srostedt@redhat.com>, Red Hat Inc.
   5 #
   6
   7 use strict;
   8 use IPC::Open2;
   9 use Fcntl qw(F_GETFL F_SETFL O_NONBLOCK);
  10 use File::Path qw(mkpath);
  11 use File::Copy qw(cp);
  12 use FileHandle;
  13 use FindBin;
  14 use IO::Handle;
  15
  16 my $VERSION = "0.2";
  17
  18 $| = 1;
  19
  20 my %opt;
  21 my %repeat_tests;
  22 my %repeats;
  23 my %evals;
  24
  25 #default opts
  26 my %default = (
  27     "MAILER"                    => "sendmail",  # default mailer
  28     "EMAIL_ON_ERROR"            => 1,
  29     "EMAIL_WHEN_FINISHED"       => 1,
  30     "EMAIL_WHEN_CANCELED"       => 0,
  31     "EMAIL_WHEN_STARTED"        => 0,
  32     "NUM_TESTS"                 => 1,
  33     "TEST_TYPE"                 => "build",
  34     "BUILD_TYPE"                => "oldconfig",
  35     "MAKE_CMD"                  => "make",
  36     "CLOSE_CONSOLE_SIGNAL"      => "INT",
  37     "TIMEOUT"                   => 120,
  38     "TMP_DIR"                   => "/tmp/ktest/\${MACHINE}",
  39     "SLEEP_TIME"                => 60,          # sleep time between tests
  40     "BUILD_NOCLEAN"             => 0,
  41     "REBOOT_ON_ERROR"           => 0,
  42     "POWEROFF_ON_ERROR"         => 0,
  43     "REBOOT_ON_SUCCESS"         => 1,
  44     "POWEROFF_ON_SUCCESS"       => 0,
  45     "BUILD_OPTIONS"             => "",
  46     "BISECT_SLEEP_TIME"         => 60,          # sleep time between bisects
  47     "PATCHCHECK_SLEEP_TIME"     => 60,          # sleep time between patch checks
  48     "CLEAR_LOG"                 => 0,
  49     "BISECT_MANUAL"             => 0,
  50     "BISECT_SKIP"               => 1,
  51     "BISECT_TRIES"              => 1,
  52     "MIN_CONFIG_TYPE"           => "boot",
  53     "SUCCESS_LINE"              => "login:",
  54     "DETECT_TRIPLE_FAULT"       => 1,
  55     "NO_INSTALL"                => 0,
  56     "BOOTED_TIMEOUT"            => 1,
  57     "DIE_ON_FAILURE"            => 1,
  58     "SSH_EXEC"                  => "ssh \$SSH_USER\@\$MACHINE \$SSH_COMMAND",
  59     "SCP_TO_TARGET"             => "scp \$SRC_FILE \$SSH_USER\@\$MACHINE:\$DST_FILE",
  60     "SCP_TO_TARGET_INSTALL"     => "\${SCP_TO_TARGET}",
  61     "REBOOT"                    => "ssh \$SSH_USER\@\$MACHINE reboot",
  62     "REBOOT_RETURN_CODE"        => 255,
  63     "STOP_AFTER_SUCCESS"        => 10,
  64     "STOP_AFTER_FAILURE"        => 60,
  65     "STOP_TEST_AFTER"           => 600,
  66     "MAX_MONITOR_WAIT"          => 1800,
  67     "GRUB_REBOOT"               => "grub2-reboot",
  68     "GRUB_BLS_GET"              => "grubby --info=ALL",
  69     "SYSLINUX"                  => "extlinux",
  70     "SYSLINUX_PATH"             => "/boot/extlinux",
  71     "CONNECT_TIMEOUT"           => 25,
  72
  73 # required, and we will ask users if they don't have them but we keep the default
  74 # value something that is common.
  75     "REBOOT_TYPE"               => "grub",
  76     "LOCALVERSION"              => "-test",
  77     "SSH_USER"                  => "root",
  78     "BUILD_TARGET"              => "arch/x86/boot/bzImage",
  79     "TARGET_IMAGE"              => "/boot/vmlinuz-test",
  80
  81     "LOG_FILE"                  => undef,
  82     "IGNORE_UNUSED"             => 0,
  83 );
  84
  85 my $test_log_start = 0;
  86
  87 my $ktest_config = "ktest.conf";
  88 my $version;
  89 my $have_version = 0;
  90 my $machine;
  91 my $last_machine;
  92 my $ssh_user;
  93 my $tmpdir;
  94 my $builddir;
  95 my $outputdir;
  96 my $output_config;
  97 my $test_type;
  98 my $build_type;
  99 my $build_options;
 100 my $final_post_ktest;
 101 my $pre_ktest;
 102 my $post_ktest;
 103 my $pre_test;
 104 my $pre_test_die;
 105 my $post_test;
 106 my $pre_build;
 107 my $post_build;
 108 my $pre_build_die;
 109 my $post_build_die;
 110 my $reboot_type;
 111 my $reboot_script;
 112 my $power_cycle;
 113 my $reboot;
 114 my $reboot_return_code;
 115 my $reboot_on_error;
 116 my $switch_to_good;
 117 my $switch_to_test;
 118 my $poweroff_on_error;
 119 my $reboot_on_success;
 120 my $die_on_failure;
 121 my $powercycle_after_reboot;
 122 my $poweroff_after_halt;
 123 my $max_monitor_wait;
 124 my $ssh_exec;
 125 my $scp_to_target;
 126 my $scp_to_target_install;
 127 my $power_off;
 128 my $grub_menu;
 129 my $last_grub_menu;
 130 my $grub_file;
 131 my $grub_number;
 132 my $grub_reboot;
 133 my $grub_bls_get;
 134 my $syslinux;
 135 my $syslinux_path;
 136 my $syslinux_label;
 137 my $target;
 138 my $make;
 139 my $pre_install;
 140 my $post_install;
 141 my $no_install;
 142 my $noclean;
 143 my $minconfig;
 144 my $start_minconfig;
 145 my $start_minconfig_defined;
 146 my $output_minconfig;
 147 my $minconfig_type;
 148 my $use_output_minconfig;
 149 my $warnings_file;
 150 my $ignore_config;
 151 my $ignore_errors;
 152 my $addconfig;
 153 my $in_bisect = 0;
 154 my $bisect_bad_commit = "";
 155 my $reverse_bisect;
 156 my $bisect_manual;
 157 my $bisect_skip;
 158 my $bisect_tries;
 159 my $config_bisect_good;
 160 my $bisect_ret_good;
 161 my $bisect_ret_bad;
 162 my $bisect_ret_skip;
 163 my $bisect_ret_abort;
 164 my $bisect_ret_default;
 165 my $in_patchcheck = 0;
 166 my $run_test;
 167 my $buildlog;
 168 my $testlog;
 169 my $dmesg;
 170 my $monitor_fp;
 171 my $monitor_pid;
 172 my $monitor_cnt = 0;
 173 my $sleep_time;
 174 my $bisect_sleep_time;
 175 my $patchcheck_sleep_time;
 176 my $ignore_warnings;
 177 my $store_failures;
 178 my $store_successes;
 179 my $test_name;
 180 my $timeout;
 181 my $run_timeout;
 182 my $connect_timeout;
 183 my $config_bisect_exec;
 184 my $booted_timeout;
 185 my $detect_triplefault;
 186 my $console;
 187 my $close_console_signal;
 188 my $reboot_success_line;
 189 my $success_line;
 190 my $stop_after_success;
 191 my $stop_after_failure;
 192 my $stop_test_after;
 193 my $build_target;
 194 my $target_image;
 195 my $checkout;
 196 my $localversion;
 197 my $iteration = 0;
 198 my $successes = 0;
 199 my $stty_orig;
 200 my $run_command_status = 0;
 201
 202 my $bisect_good;
 203 my $bisect_bad;
 204 my $bisect_type;
 205 my $bisect_start;
 206 my $bisect_replay;
 207 my $bisect_files;
 208 my $bisect_reverse;
 209 my $bisect_check;
 210
 211 my $config_bisect;
 212 my $config_bisect_type;
 213 my $config_bisect_check;
 214
 215 my $patchcheck_type;
 216 my $patchcheck_start;
 217 my $patchcheck_cherry;
 218 my $patchcheck_end;
 219
 220 my $build_time;
 221 my $install_time;
 222 my $reboot_time;
 223 my $test_time;
 224
 225 my $warning_found = 0;
 226
 227 my $pwd;
 228 my $dirname = $FindBin::Bin;
 229
 230 my $mailto;
 231 my $mailer;
 232 my $mail_path;
 233 my $mail_max_size;
 234 my $mail_command;
 235 my $email_on_error;
 236 my $email_when_finished;
 237 my $email_when_started;
 238 my $email_when_canceled;
 239
 240 my $script_start_time = localtime();
 241
 242 # set when a test is something other that just building or install
 243 # which would require more options.
 244 my $buildonly = 1;
 245
 246 # tell build not to worry about warnings, even when WARNINGS_FILE is set
 247 my $warnings_ok = 0;
 248
 249 # set when creating a new config
 250 my $newconfig = 0;
 251
 252 my %entered_configs;
 253 my %config_help;
 254 my %variable;
 255
 256 # force_config is the list of configs that we force enabled (or disabled)
 257 # in a .config file. The MIN_CONFIG and ADD_CONFIG configs.
 258 my %force_config;
 259
 260 # do not force reboots on config problems
 261 my $no_reboot = 1;
 262
 263 # reboot on success
 264 my $reboot_success = 0;
 265
 266 my %option_map = (
 267     "MAILTO"                    => \$mailto,
 268     "MAILER"                    => \$mailer,
 269     "MAIL_PATH"                 => \$mail_path,
 270     "MAIL_MAX_SIZE"             => \$mail_max_size,
 271     "MAIL_COMMAND"              => \$mail_command,
 272     "EMAIL_ON_ERROR"            => \$email_on_error,
 273     "EMAIL_WHEN_FINISHED"       => \$email_when_finished,
 274     "EMAIL_WHEN_STARTED"        => \$email_when_started,
 275     "EMAIL_WHEN_CANCELED"       => \$email_when_canceled,
 276     "MACHINE"                   => \$machine,
 277     "SSH_USER"                  => \$ssh_user,
 278     "TMP_DIR"                   => \$tmpdir,
 279     "OUTPUT_DIR"                => \$outputdir,
 280     "BUILD_DIR"                 => \$builddir,
 281     "TEST_TYPE"                 => \$test_type,
 282     "PRE_KTEST"                 => \$pre_ktest,
 283     "POST_KTEST"                => \$post_ktest,
 284     "PRE_TEST"                  => \$pre_test,
 285     "PRE_TEST_DIE"              => \$pre_test_die,
 286     "POST_TEST"                 => \$post_test,
 287     "BUILD_TYPE"                => \$build_type,
 288     "BUILD_OPTIONS"             => \$build_options,
 289     "PRE_BUILD"                 => \$pre_build,
 290     "POST_BUILD"                => \$post_build,
 291     "PRE_BUILD_DIE"             => \$pre_build_die,
 292     "POST_BUILD_DIE"            => \$post_build_die,
 293     "POWER_CYCLE"               => \$power_cycle,
 294     "REBOOT"                    => \$reboot,
 295     "REBOOT_RETURN_CODE"        => \$reboot_return_code,
 296     "BUILD_NOCLEAN"             => \$noclean,
 297     "MIN_CONFIG"                => \$minconfig,
 298     "OUTPUT_MIN_CONFIG"         => \$output_minconfig,
 299     "START_MIN_CONFIG"          => \$start_minconfig,
 300     "MIN_CONFIG_TYPE"           => \$minconfig_type,
 301     "USE_OUTPUT_MIN_CONFIG"     => \$use_output_minconfig,
 302     "WARNINGS_FILE"             => \$warnings_file,
 303     "IGNORE_CONFIG"             => \$ignore_config,
 304     "TEST"                      => \$run_test,
 305     "ADD_CONFIG"                => \$addconfig,
 306     "REBOOT_TYPE"               => \$reboot_type,
 307     "GRUB_MENU"                 => \$grub_menu,
 308     "GRUB_FILE"                 => \$grub_file,
 309     "GRUB_REBOOT"               => \$grub_reboot,
 310     "GRUB_BLS_GET"              => \$grub_bls_get,
 311     "SYSLINUX"                  => \$syslinux,
 312     "SYSLINUX_PATH"             => \$syslinux_path,
 313     "SYSLINUX_LABEL"            => \$syslinux_label,
 314     "PRE_INSTALL"               => \$pre_install,
 315     "POST_INSTALL"              => \$post_install,
 316     "NO_INSTALL"                => \$no_install,
 317     "REBOOT_SCRIPT"             => \$reboot_script,
 318     "REBOOT_ON_ERROR"           => \$reboot_on_error,
 319     "SWITCH_TO_GOOD"            => \$switch_to_good,
 320     "SWITCH_TO_TEST"            => \$switch_to_test,
 321     "POWEROFF_ON_ERROR"         => \$poweroff_on_error,
 322     "REBOOT_ON_SUCCESS"         => \$reboot_on_success,
 323     "DIE_ON_FAILURE"            => \$die_on_failure,
 324     "POWER_OFF"                 => \$power_off,
 325     "POWERCYCLE_AFTER_REBOOT"   => \$powercycle_after_reboot,
 326     "POWEROFF_AFTER_HALT"       => \$poweroff_after_halt,
 327     "MAX_MONITOR_WAIT"          => \$max_monitor_wait,
 328     "SLEEP_TIME"                => \$sleep_time,
 329     "BISECT_SLEEP_TIME"         => \$bisect_sleep_time,
 330     "PATCHCHECK_SLEEP_TIME"     => \$patchcheck_sleep_time,
 331     "IGNORE_WARNINGS"           => \$ignore_warnings,
 332     "IGNORE_ERRORS"             => \$ignore_errors,
 333     "BISECT_MANUAL"             => \$bisect_manual,
 334     "BISECT_SKIP"               => \$bisect_skip,
 335     "BISECT_TRIES"              => \$bisect_tries,
 336     "CONFIG_BISECT_GOOD"        => \$config_bisect_good,
 337     "BISECT_RET_GOOD"           => \$bisect_ret_good,
 338     "BISECT_RET_BAD"            => \$bisect_ret_bad,
 339     "BISECT_RET_SKIP"           => \$bisect_ret_skip,
 340     "BISECT_RET_ABORT"          => \$bisect_ret_abort,
 341     "BISECT_RET_DEFAULT"        => \$bisect_ret_default,
 342     "STORE_FAILURES"            => \$store_failures,
 343     "STORE_SUCCESSES"           => \$store_successes,
 344     "TEST_NAME"                 => \$test_name,
 345     "TIMEOUT"                   => \$timeout,
 346     "RUN_TIMEOUT"               => \$run_timeout,
 347     "CONNECT_TIMEOUT"           => \$connect_timeout,
 348     "CONFIG_BISECT_EXEC"        => \$config_bisect_exec,
 349     "BOOTED_TIMEOUT"            => \$booted_timeout,
 350     "CONSOLE"                   => \$console,
 351     "CLOSE_CONSOLE_SIGNAL"      => \$close_console_signal,
 352     "DETECT_TRIPLE_FAULT"       => \$detect_triplefault,
 353     "SUCCESS_LINE"              => \$success_line,
 354     "REBOOT_SUCCESS_LINE"       => \$reboot_success_line,
 355     "STOP_AFTER_SUCCESS"        => \$stop_after_success,
 356     "STOP_AFTER_FAILURE"        => \$stop_after_failure,
 357     "STOP_TEST_AFTER"           => \$stop_test_after,
 358     "BUILD_TARGET"              => \$build_target,
 359     "SSH_EXEC"                  => \$ssh_exec,
 360     "SCP_TO_TARGET"             => \$scp_to_target,
 361     "SCP_TO_TARGET_INSTALL"     => \$scp_to_target_install,
 362     "CHECKOUT"                  => \$checkout,
 363     "TARGET_IMAGE"              => \$target_image,
 364     "LOCALVERSION"              => \$localversion,
 365
 366     "BISECT_GOOD"               => \$bisect_good,
 367     "BISECT_BAD"                => \$bisect_bad,
 368     "BISECT_TYPE"               => \$bisect_type,
 369     "BISECT_START"              => \$bisect_start,
 370     "BISECT_REPLAY"             => \$bisect_replay,
 371     "BISECT_FILES"              => \$bisect_files,
 372     "BISECT_REVERSE"            => \$bisect_reverse,
 373     "BISECT_CHECK"              => \$bisect_check,
 374
 375     "CONFIG_BISECT"             => \$config_bisect,
 376     "CONFIG_BISECT_TYPE"        => \$config_bisect_type,
 377     "CONFIG_BISECT_CHECK"       => \$config_bisect_check,
 378
 379     "PATCHCHECK_TYPE"           => \$patchcheck_type,
 380     "PATCHCHECK_START"          => \$patchcheck_start,
 381     "PATCHCHECK_CHERRY"         => \$patchcheck_cherry,
 382     "PATCHCHECK_END"            => \$patchcheck_end,
 383 );
 384
 385 # Options may be used by other options, record them.
 386 my %used_options;
 387
 388 # default variables that can be used
 389 chomp ($variable{"PWD"} = `pwd`);
 390 $pwd = $variable{"PWD"};
 391
 392 $config_help{"MACHINE"} = << "EOF"
 393  The machine hostname that you will test.
 394  For build only tests, it is still needed to differentiate log files.
 395 EOF
 396     ;
 397 $config_help{"SSH_USER"} = << "EOF"
 398  The box is expected to have ssh on normal bootup, provide the user
 399   (most likely root, since you need privileged operations)
 400 EOF
 401     ;
 402 $config_help{"BUILD_DIR"} = << "EOF"
 403  The directory that contains the Linux source code (full path).
 404  You can use \${PWD} that will be the path where ktest.pl is run, or use
 405  \${THIS_DIR} which is assigned \${PWD} but may be changed later.
 406 EOF
 407     ;
 408 $config_help{"OUTPUT_DIR"} = << "EOF"
 409  The directory that the objects will be built (full path).
 410  (can not be same as BUILD_DIR)
 411  You can use \${PWD} that will be the path where ktest.pl is run, or use
 412  \${THIS_DIR} which is assigned \${PWD} but may be changed later.
 413 EOF
 414     ;
 415 $config_help{"BUILD_TARGET"} = << "EOF"
 416  The location of the compiled file to copy to the target.
 417  (relative to OUTPUT_DIR)
 418 EOF
 419     ;
 420 $config_help{"BUILD_OPTIONS"} = << "EOF"
 421  Options to add to \"make\" when building.
 422  i.e.  -j20
 423 EOF
 424     ;
 425 $config_help{"TARGET_IMAGE"} = << "EOF"
 426  The place to put your image on the test machine.
 427 EOF
 428     ;
 429 $config_help{"POWER_CYCLE"} = << "EOF"
 430  A script or command to reboot the box.
 431
 432  Here is a digital loggers power switch example
 433  POWER_CYCLE = wget --no-proxy -O /dev/null -q  --auth-no-challenge 'http://admin:admin\@power/outlet?5=CCL'
 434
 435  Here is an example to reboot a virtual box on the current host
 436  with the name "Guest".
 437  POWER_CYCLE = virsh destroy Guest; sleep 5; virsh start Guest
 438 EOF
 439     ;
 440 $config_help{"CONSOLE"} = << "EOF"
 441  The script or command that reads the console
 442
 443   If you use ttywatch server, something like the following would work.
 444 CONSOLE = nc -d localhost 3001
 445
 446  For a virtual machine with guest name "Guest".
 447 CONSOLE =  virsh console Guest
 448 EOF
 449     ;
 450 $config_help{"LOCALVERSION"} = << "EOF"
 451  Required version ending to differentiate the test
 452  from other linux builds on the system.
 453 EOF
 454     ;
 455 $config_help{"REBOOT_TYPE"} = << "EOF"
 456  Way to reboot the box to the test kernel.
 457  Only valid options so far are "grub", "grub2", "grub2bls", "syslinux", and "script".
 458
 459  If you specify grub, it will assume grub version 1
 460  and will search in /boot/grub/menu.lst for the title \$GRUB_MENU
 461  and select that target to reboot to the kernel. If this is not
 462  your setup, then specify "script" and have a command or script
 463  specified in REBOOT_SCRIPT to boot to the target.
 464
 465  The entry in /boot/grub/menu.lst must be entered in manually.
 466  The test will not modify that file.
 467
 468  If you specify grub2, then you also need to specify both \$GRUB_MENU
 469  and \$GRUB_FILE.
 470
 471  If you specify grub2bls, then you also need to specify \$GRUB_MENU.
 472
 473  If you specify syslinux, then you may use SYSLINUX to define the syslinux
 474  command (defaults to extlinux), and SYSLINUX_PATH to specify the path to
 475  the syslinux install (defaults to /boot/extlinux). But you have to specify
 476  SYSLINUX_LABEL to define the label to boot to for the test kernel.
 477 EOF
 478     ;
 479 $config_help{"GRUB_MENU"} = << "EOF"
 480  The grub title name for the test kernel to boot
 481  (Only mandatory if REBOOT_TYPE = grub or grub2)
 482
 483  Note, ktest.pl will not update the grub menu.lst, you need to
 484  manually add an option for the test. ktest.pl will search
 485  the grub menu.lst for this option to find what kernel to
 486  reboot into.
 487
 488  For example, if in the /boot/grub/menu.lst the test kernel title has:
 489  title Test Kernel
 490  kernel vmlinuz-test
 491  GRUB_MENU = Test Kernel
 492
 493  For grub2, a search of \$GRUB_FILE is performed for the lines
 494  that begin with "menuentry". It will not detect submenus. The
 495  menu must be a non-nested menu. Add the quotes used in the menu
 496  to guarantee your selection, as the first menuentry with the content
 497  of \$GRUB_MENU that is found will be used.
 498
 499  For grub2bls, \$GRUB_MENU is searched on the result of \$GRUB_BLS_GET
 500  command for the lines that begin with "title".
 501 EOF
 502     ;
 503 $config_help{"GRUB_FILE"} = << "EOF"
 504  If grub2 is used, the full path for the grub.cfg file is placed
 505  here. Use something like /boot/grub2/grub.cfg to search.
 506 EOF
 507     ;
 508 $config_help{"SYSLINUX_LABEL"} = << "EOF"
 509  If syslinux is used, the label that boots the target kernel must
 510  be specified with SYSLINUX_LABEL.
 511 EOF
 512     ;
 513 $config_help{"REBOOT_SCRIPT"} = << "EOF"
 514  A script to reboot the target into the test kernel
 515  (Only mandatory if REBOOT_TYPE = script)
 516 EOF
 517     ;
 518
 519 # used with process_expression()
 520 my $d = 0;
 521
 522 # defined before get_test_name()
 523 my $in_die = 0;
 524
 525 # defined before process_warning_line()
 526 my $check_build_re = ".*:.*(warning|error|Error):.*";
 527 my $utf8_quote = "\\x{e2}\\x{80}(\\x{98}|\\x{99})";
 528
 529 # defined before child_finished()
 530 my $child_done;
 531
 532 # config_ignore holds the configs that were set (or unset) for
 533 # a good config and we will ignore these configs for the rest
 534 # of a config bisect. These configs stay as they were.
 535 my %config_ignore;
 536
 537 # config_set holds what all configs were set as.
 538 my %config_set;
 539
 540 # config_off holds the set of configs that the bad config had disabled.
 541 # We need to record them and set them in the .config when running
 542 # olddefconfig, because olddefconfig keeps the defaults.
 543 my %config_off;
 544
 545 # config_off_tmp holds a set of configs to turn off for now
 546 my @config_off_tmp;
 547
 548 # config_list is the set of configs that are being tested
 549 my %config_list;
 550 my %null_config;
 551
 552 my %dependency;
 553
 554 # found above run_config_bisect()
 555 my $pass = 1;
 556
 557 # found above add_dep()
 558
 559 my %depends;
 560 my %depcount;
 561 my $iflevel = 0;
 562 my @ifdeps;
 563
 564 # prevent recursion
 565 my %read_kconfigs;
 566
 567 # found above test_this_config()
 568 my %min_configs;
 569 my %keep_configs;
 570 my %save_configs;
 571 my %processed_configs;
 572 my %nochange_config;
 573
 574 #
 575 # These are first defined here, main function later on
 576 #
 577 sub run_command;
 578 sub start_monitor;
 579 sub end_monitor;
 580 sub wait_for_monitor;
 581
 582 sub _logit {
 583     if (defined($opt{"LOG_FILE"})) {
 584         print LOG @_;
 585     }
 586 }
 587
 588 sub logit {
 589     if (defined($opt{"LOG_FILE"})) {
 590         _logit @_;
 591     } else {
 592         print @_;
 593     }
 594 }
 595
 596 sub doprint {
 597     print @_;
 598     _logit @_;
 599 }
 600
 601 sub read_prompt {
 602     my ($cancel, $prompt) = @_;
 603
 604     my $ans;
 605
 606     for (;;) {
 607         if ($cancel) {
 608             print "$prompt [y/n/C] ";
 609         } else {
 610             print "$prompt [Y/n] ";
 611         }
 612         $ans = <STDIN>;
 613         chomp $ans;
 614         if ($ans =~ /^\s*$/) {
 615             if ($cancel) {
 616                 $ans = "c";
 617             } else {
 618                 $ans = "y";
 619             }
 620         }
 621         last if ($ans =~ /^y$/i || $ans =~ /^n$/i);
 622         if ($cancel) {
 623             last if ($ans =~ /^c$/i);
 624             print "Please answer either 'y', 'n' or 'c'.\n";
 625         } else {
 626             print "Please answer either 'y' or 'n'.\n";
 627         }
 628     }
 629     if ($ans =~ /^c/i) {
 630         exit;
 631     }
 632     if ($ans !~ /^y$/i) {
 633         return 0;
 634     }
 635     return 1;
 636 }
 637
 638 sub read_yn {
 639     my ($prompt) = @_;
 640
 641     return read_prompt 0, $prompt;
 642 }
 643
 644 sub read_ync {
 645     my ($prompt) = @_;
 646
 647     return read_prompt 1, $prompt;
 648 }
 649
 650 sub get_mandatory_config {
 651     my ($config) = @_;
 652     my $ans;
 653
 654     return if (defined($opt{$config}));
 655
 656     if (defined($config_help{$config})) {
 657         print "\n";
 658         print $config_help{$config};
 659     }
 660
 661     for (;;) {
 662         print "$config = ";
 663         if (defined($default{$config}) && length($default{$config})) {
 664             print "\[$default{$config}\] ";
 665         }
 666         $ans = <STDIN>;
 667         $ans =~ s/^\s*(.*\S)\s*$/$1/;
 668         if ($ans =~ /^\s*$/) {
 669             if ($default{$config}) {
 670                 $ans = $default{$config};
 671             } else {
 672                 print "Your answer can not be blank\n";
 673                 next;
 674             }
 675         }
 676         $entered_configs{$config} = ${ans};
 677         last;
 678     }
 679 }
 680
 681 sub show_time {
 682     my ($time) = @_;
 683
 684     my $hours = 0;
 685     my $minutes = 0;
 686
 687     if ($time > 3600) {
 688         $hours = int($time / 3600);
 689         $time -= $hours * 3600;
 690     }
 691     if ($time > 60) {
 692         $minutes = int($time / 60);
 693         $time -= $minutes * 60;
 694     }
 695
 696     if ($hours > 0) {
 697         doprint "$hours hour";
 698         doprint "s" if ($hours > 1);
 699         doprint " ";
 700     }
 701
 702     if ($minutes > 0) {
 703         doprint "$minutes minute";
 704         doprint "s" if ($minutes > 1);
 705         doprint " ";
 706     }
 707
 708     doprint "$time second";
 709     doprint "s" if ($time != 1);
 710 }
 711
 712 sub print_times {
 713     doprint "\n";
 714     if ($build_time) {
 715         doprint "Build time:   ";
 716         show_time($build_time);
 717         doprint "\n";
 718     }
 719     if ($install_time) {
 720         doprint "Install time: ";
 721         show_time($install_time);
 722         doprint "\n";
 723     }
 724     if ($reboot_time) {
 725         doprint "Reboot time:  ";
 726         show_time($reboot_time);
 727         doprint "\n";
 728     }
 729     if ($test_time) {
 730         doprint "Test time:    ";
 731         show_time($test_time);
 732         doprint "\n";
 733     }
 734     if ($warning_found) {
 735         doprint "\n*** WARNING";
 736         doprint "S" if ($warning_found > 1);
 737         doprint " found in build: $warning_found ***\n\n";
 738     }
 739
 740     # reset for iterations like bisect
 741     $build_time = 0;
 742     $install_time = 0;
 743     $reboot_time = 0;
 744     $test_time = 0;
 745     $warning_found = 0;
 746 }
 747
 748 sub get_mandatory_configs {
 749     get_mandatory_config("MACHINE");
 750     get_mandatory_config("BUILD_DIR");
 751     get_mandatory_config("OUTPUT_DIR");
 752
 753     if ($newconfig) {
 754         get_mandatory_config("BUILD_OPTIONS");
 755     }
 756
 757     # options required for other than just building a kernel
 758     if (!$buildonly) {
 759         get_mandatory_config("POWER_CYCLE");
 760         get_mandatory_config("CONSOLE");
 761     }
 762
 763     # options required for install and more
 764     if ($buildonly != 1) {
 765         get_mandatory_config("SSH_USER");
 766         get_mandatory_config("BUILD_TARGET");
 767         get_mandatory_config("TARGET_IMAGE");
 768     }
 769
 770     get_mandatory_config("LOCALVERSION");
 771
 772     return if ($buildonly);
 773
 774     my $rtype = $opt{"REBOOT_TYPE"};
 775
 776     if (!defined($rtype)) {
 777         if (!defined($opt{"GRUB_MENU"})) {
 778             get_mandatory_config("REBOOT_TYPE");
 779             $rtype = $entered_configs{"REBOOT_TYPE"};
 780         } else {
 781             $rtype = "grub";
 782         }
 783     }
 784
 785     if (($rtype eq "grub") or ($rtype eq "grub2bls")) {
 786         get_mandatory_config("GRUB_MENU");
 787     }
 788
 789     if ($rtype eq "grub2") {
 790         get_mandatory_config("GRUB_MENU");
 791         get_mandatory_config("GRUB_FILE");
 792     }
 793
 794     if ($rtype eq "syslinux") {
 795         get_mandatory_config("SYSLINUX_LABEL");
 796     }
 797 }
 798
 799 sub process_variables {
 800     my ($value, $remove_undef) = @_;
 801     my $retval = "";
 802
 803     # We want to check for '\', and it is just easier
 804     # to check the previous character of '$' and not need
 805     # to worry if '$' is the first character. By adding
 806     # a space to $value, we can just check [^\\]\$ and
 807     # it will still work.
 808     $value = " $value";
 809
 810     while ($value =~ /(.*?[^\\])\$\{([^\{]*?)\}(.*)/) {
 811         my $begin = $1;
 812         my $var = $2;
 813         my $end = $3;
 814         # append beginning of value to retval
 815         $retval = "$retval$begin";
 816         if ($var =~ s/^shell\s+//) {
 817             $retval = `$var`;
 818             if ($?) {
 819                 doprint "WARNING: $var returned an error\n";
 820             } else {
 821                 chomp $retval;
 822             }
 823         } elsif (defined($variable{$var})) {
 824             $retval = "$retval$variable{$var}";
 825         } elsif (defined($remove_undef) && $remove_undef) {
 826             # for if statements, any variable that is not defined,
 827             # we simple convert to 0
 828             $retval = "${retval}0";
 829         } else {
 830             # put back the origin piece, but with $#### to not reprocess it
 831             $retval = "$retval\$####\{$var\}";
 832             # This could be an option that is used later, save
 833             # it so we don't warn if this option is not one of
 834             # ktests options.
 835             $used_options{$var} = 1;
 836         }
 837         $value = "$retval$end";
 838         $retval = "";
 839     }
 840     $retval = $value;
 841
 842     # Convert the saved variables with $####{var} back to ${var}
 843     $retval =~ s/\$####/\$/g;
 844
 845     # remove the space added in the beginning
 846     $retval =~ s/ //;
 847
 848     return "$retval";
 849 }
 850
 851 sub set_value {
 852     my ($lvalue, $rvalue, $override, $overrides, $name) = @_;
 853
 854     my $prvalue = process_variables($rvalue);
 855
 856     if ($lvalue =~ /^(TEST|BISECT|CONFIG_BISECT)_TYPE(\[.*\])?$/ &&
 857         $prvalue !~ /^(config_|)bisect$/ &&
 858         $prvalue !~ /^build$/ &&
 859         $prvalue !~ /^make_warnings_file$/ &&
 860         $buildonly) {
 861
 862         # Note if a test is something other than build, then we
 863         # will need other mandatory options.
 864         if ($prvalue ne "install") {
 865             $buildonly = 0;
 866         } else {
 867             # install still limits some mandatory options.
 868             $buildonly = 2;
 869         }
 870     }
 871
 872     if (defined($opt{$lvalue})) {
 873         if (!$override || defined(${$overrides}{$lvalue})) {
 874             my $extra = "";
 875             if ($override) {
 876                 $extra = "In the same override section!\n";
 877             }
 878             die "$name: $.: Option $lvalue defined more than once!\n$extra";
 879         }
 880         ${$overrides}{$lvalue} = $prvalue;
 881     }
 882
 883     $opt{$lvalue} = $prvalue;
 884 }
 885
 886 sub set_eval {
 887     my ($lvalue, $rvalue, $name) = @_;
 888
 889     my $prvalue = process_variables($rvalue);
 890     my $arr;
 891
 892     if (defined($evals{$lvalue})) {
 893         $arr = $evals{$lvalue};
 894     } else {
 895         $arr = [];
 896         $evals{$lvalue} = $arr;
 897     }
 898
 899     push @{$arr}, $rvalue;
 900 }
 901
 902 sub set_variable {
 903     my ($lvalue, $rvalue) = @_;
 904
 905     if ($rvalue =~ /^\s*$/) {
 906         delete $variable{$lvalue};
 907     } else {
 908         $rvalue = process_variables($rvalue);
 909         $variable{$lvalue} = $rvalue;
 910     }
 911 }
 912
 913 sub process_compare {
 914     my ($lval, $cmp, $rval) = @_;
 915
 916     # remove whitespace
 917
 918     $lval =~ s/^\s*//;
 919     $lval =~ s/\s*$//;
 920
 921     $rval =~ s/^\s*//;
 922     $rval =~ s/\s*$//;
 923
 924     if ($cmp eq "==") {
 925         return $lval eq $rval;
 926     } elsif ($cmp eq "!=") {
 927         return $lval ne $rval;
 928     } elsif ($cmp eq "=~") {
 929         return $lval =~ m/$rval/;
 930     } elsif ($cmp eq "!~") {
 931         return $lval !~ m/$rval/;
 932     }
 933
 934     my $statement = "$lval $cmp $rval";
 935     my $ret = eval $statement;
 936
 937     # $@ stores error of eval
 938     if ($@) {
 939         return -1;
 940     }
 941
 942     return $ret;
 943 }
 944
 945 sub value_defined {
 946     my ($val) = @_;
 947
 948     return defined($variable{$2}) ||
 949         defined($opt{$2});
 950 }
 951
 952 sub process_expression {
 953     my ($name, $val) = @_;
 954
 955     my $c = $d++;
 956
 957     while ($val =~ s/\(([^\(]*?)\)/\&\&\&\&VAL\&\&\&\&/) {
 958         my $express = $1;
 959
 960         if (process_expression($name, $express)) {
 961             $val =~ s/\&\&\&\&VAL\&\&\&\&/ 1 /;
 962         } else {
 963             $val =~ s/\&\&\&\&VAL\&\&\&\&/ 0 /;
 964         }
 965     }
 966
 967     $d--;
 968     my $OR = "\\|\\|";
 969     my $AND = "\\&\\&";
 970
 971     while ($val =~ s/^(.*?)($OR|$AND)//) {
 972         my $express = $1;
 973         my $op = $2;
 974
 975         if (process_expression($name, $express)) {
 976             if ($op eq "||") {
 977                 return 1;
 978             }
 979         } else {
 980             if ($op eq "&&") {
 981                 return 0;
 982             }
 983         }
 984     }
 985
 986     if ($val =~ /(.*)(==|\!=|>=|<=|>|<|=~|\!~)(.*)/) {
 987         my $ret = process_compare($1, $2, $3);
 988         if ($ret < 0) {
 989             die "$name: $.: Unable to process comparison\n";
 990         }
 991         return $ret;
 992     }
 993
 994     if ($val =~ /^\s*(NOT\s*)?DEFINED\s+(\S+)\s*$/) {
 995         if (defined $1) {
 996             return !value_defined($2);
 997         } else {
 998             return value_defined($2);
 999         }
1000     }
1001
1002     if ($val =~ s/^\s*NOT\s+(.*)//) {
1003         my $express = $1;
1004         my $ret = process_expression($name, $express);
1005         return !$ret;
1006     }
1007
1008     if ($val =~ /^\s*0\s*$/) {
1009         return 0;
1010     } elsif ($val =~ /^\s*\d+\s*$/) {
1011         return 1;
1012     }
1013
1014     die ("$name: $.: Undefined content $val in if statement\n");
1015 }
1016
1017 sub process_if {
1018     my ($name, $value) = @_;
1019
1020     # Convert variables and replace undefined ones with 0
1021     my $val = process_variables($value, 1);
1022     my $ret = process_expression $name, $val;
1023
1024     return $ret;
1025 }
1026
1027 sub __read_config {
1028     my ($config, $current_test_num) = @_;
1029
1030     my $in;
1031     open($in, $config) || die "can't read file $config";
1032
1033     my $name = $config;
1034     $name =~ s,.*/(.*),$1,;
1035
1036     my $test_num = $$current_test_num;
1037     my $default = 1;
1038     my $repeat = 1;
1039     my $num_tests_set = 0;
1040     my $skip = 0;
1041     my $rest;
1042     my $line;
1043     my $test_case = 0;
1044     my $if = 0;
1045     my $if_set = 0;
1046     my $override = 0;
1047
1048     my %overrides;
1049
1050     while (<$in>) {
1051
1052         # ignore blank lines and comments
1053         next if (/^\s*$/ || /\s*\#/);
1054
1055         if (/^\s*(TEST_START|DEFAULTS)\b(.*)/) {
1056
1057             my $type = $1;
1058             $rest = $2;
1059             $line = $2;
1060
1061             my $old_test_num;
1062             my $old_repeat;
1063             $override = 0;
1064
1065             if ($type eq "TEST_START") {
1066                 if ($num_tests_set) {
1067                     die "$name: $.: Can not specify both NUM_TESTS and TEST_START\n";
1068                 }
1069
1070                 $old_test_num = $test_num;
1071                 $old_repeat = $repeat;
1072
1073                 $test_num += $repeat;
1074                 $default = 0;
1075                 $repeat = 1;
1076             } else {
1077                 $default = 1;
1078             }
1079
1080             # If SKIP is anywhere in the line, the command will be skipped
1081             if ($rest =~ s/\s+SKIP\b//) {
1082                 $skip = 1;
1083             } else {
1084                 $test_case = 1;
1085                 $skip = 0;
1086             }
1087
1088             if ($rest =~ s/\sELSE\b//) {
1089                 if (!$if) {
1090                     die "$name: $.: ELSE found with out matching IF section\n$_";
1091                 }
1092                 $if = 0;
1093
1094                 if ($if_set) {
1095                     $skip = 1;
1096                 } else {
1097                     $skip = 0;
1098                 }
1099             }
1100
1101             if ($rest =~ s/\sIF\s+(.*)//) {
1102                 if (process_if($name, $1)) {
1103                     $if_set = 1;
1104                 } else {
1105                     $skip = 1;
1106                 }
1107                 $if = 1;
1108             } else {
1109                 $if = 0;
1110                 $if_set = 0;
1111             }
1112
1113             if (!$skip) {
1114                 if ($type eq "TEST_START") {
1115                     if ($rest =~ s/\s+ITERATE\s+(\d+)//) {
1116                         $repeat = $1;
1117                         $repeat_tests{"$test_num"} = $repeat;
1118                     }
1119                 } elsif ($rest =~ s/\sOVERRIDE\b//) {
1120                     # DEFAULT only
1121                     $override = 1;
1122                     # Clear previous overrides
1123                     %overrides = ();
1124                 }
1125             }
1126
1127             if (!$skip && $rest !~ /^\s*$/) {
1128                 die "$name: $.: Garbage found after $type\n$_";
1129             }
1130
1131             if ($skip && $type eq "TEST_START") {
1132                 $test_num = $old_test_num;
1133                 $repeat = $old_repeat;
1134             }
1135         } elsif (/^\s*ELSE\b(.*)$/) {
1136             if (!$if) {
1137                 die "$name: $.: ELSE found with out matching IF section\n$_";
1138             }
1139             $rest = $1;
1140             if ($if_set) {
1141                 $skip = 1;
1142                 $rest = "";
1143             } else {
1144                 $skip = 0;
1145
1146                 if ($rest =~ /\sIF\s+(.*)/) {
1147                     # May be a ELSE IF section.
1148                     if (process_if($name, $1)) {
1149                         $if_set = 1;
1150                     } else {
1151                         $skip = 1;
1152                     }
1153                     $rest = "";
1154                 } else {
1155                     $if = 0;
1156                 }
1157             }
1158
1159             if ($rest !~ /^\s*$/) {
1160                 die "$name: $.: Garbage found after DEFAULTS\n$_";
1161             }
1162
1163         } elsif (/^\s*INCLUDE\s+(\S+)/) {
1164
1165             next if ($skip);
1166
1167             if (!$default) {
1168                 die "$name: $.: INCLUDE can only be done in default sections\n$_";
1169             }
1170
1171             my $file = process_variables($1);
1172
1173             if ($file !~ m,^/,) {
1174                 # check the path of the config file first
1175                 if ($config =~ m,(.*)/,) {
1176                     if (-f "$1/$file") {
1177                         $file = "$1/$file";
1178                     }
1179                 }
1180             }
1181
1182             if ( ! -r $file ) {
1183                 die "$name: $.: Can't read file $file\n$_";
1184             }
1185
1186             if (__read_config($file, \$test_num)) {
1187                 $test_case = 1;
1188             }
1189
1190         } elsif (/^\s*([A-Z_\[\]\d]+)\s*=~\s*(.*?)\s*$/) {
1191
1192             next if ($skip);
1193
1194             my $lvalue = $1;
1195             my $rvalue = $2;
1196
1197             if ($default || $lvalue =~ /\[\d+\]$/) {
1198                 set_eval($lvalue, $rvalue, $name);
1199             } else {
1200                 my $val = "$lvalue\[$test_num\]";
1201                 set_eval($val, $rvalue, $name);
1202             }
1203
1204         } elsif (/^\s*([A-Z_\[\]\d]+)\s*=\s*(.*?)\s*$/) {
1205
1206             next if ($skip);
1207
1208             my $lvalue = $1;
1209             my $rvalue = $2;
1210
1211             if (!$default &&
1212                 ($lvalue eq "NUM_TESTS" ||
1213                  $lvalue eq "LOG_FILE" ||
1214                  $lvalue eq "CLEAR_LOG")) {
1215                 die "$name: $.: $lvalue must be set in DEFAULTS section\n";
1216             }
1217
1218             if ($lvalue eq "NUM_TESTS") {
1219                 if ($test_num) {
1220                     die "$name: $.: Can not specify both NUM_TESTS and TEST_START\n";
1221                 }
1222                 if (!$default) {
1223                     die "$name: $.: NUM_TESTS must be set in default section\n";
1224                 }
1225                 $num_tests_set = 1;
1226             }
1227
1228             if ($default || $lvalue =~ /\[\d+\]$/) {
1229                 set_value($lvalue, $rvalue, $override, \%overrides, $name);
1230             } else {
1231                 my $val = "$lvalue\[$test_num\]";
1232                 set_value($val, $rvalue, $override, \%overrides, $name);
1233
1234                 if ($repeat > 1) {
1235                     $repeats{$val} = $repeat;
1236                 }
1237             }
1238         } elsif (/^\s*([A-Z_\[\]\d]+)\s*:=\s*(.*?)\s*$/) {
1239             next if ($skip);
1240
1241             my $lvalue = $1;
1242             my $rvalue = $2;
1243
1244             # process config variables.
1245             # Config variables are only active while reading the
1246             # config and can be defined anywhere. They also ignore
1247             # TEST_START and DEFAULTS, but are skipped if they are in
1248             # on of these sections that have SKIP defined.
1249             # The save variable can be
1250             # defined multiple times and the new one simply overrides
1251             # the previous one.
1252             set_variable($lvalue, $rvalue);
1253
1254         } else {
1255             die "$name: $.: Garbage found in config\n$_";
1256         }
1257     }
1258
1259     if ($test_num) {
1260         $test_num += $repeat - 1;
1261         $opt{"NUM_TESTS"} = $test_num;
1262     }
1263
1264     close($in);
1265
1266     $$current_test_num = $test_num;
1267
1268     return $test_case;
1269 }
1270
1271 sub get_test_case {
1272     print "What test case would you like to run?\n";
1273     print " (build, install or boot)\n";
1274     print " Other tests are available but require editing ktest.conf\n";
1275     print " (see tools/testing/ktest/sample.conf)\n";
1276     my $ans = <STDIN>;
1277     chomp $ans;
1278     $default{"TEST_TYPE"} = $ans;
1279 }
1280
1281 sub read_config {
1282     my ($config) = @_;
1283
1284     my $test_case;
1285     my $test_num = 0;
1286
1287     $test_case = __read_config $config, \$test_num;
1288
1289     # make sure we have all mandatory configs
1290     get_mandatory_configs;
1291
1292     # was a test specified?
1293     if (!$test_case) {
1294         print "No test case specified.\n";
1295         get_test_case;
1296     }
1297
1298     # set any defaults
1299
1300     foreach my $default (keys %default) {
1301         if (!defined($opt{$default})) {
1302             $opt{$default} = $default{$default};
1303         }
1304     }
1305
1306     if ($opt{"IGNORE_UNUSED"} == 1) {
1307         return;
1308     }
1309
1310     my %not_used;
1311
1312     # check if there are any stragglers (typos?)
1313     foreach my $option (keys %opt) {
1314         my $op = $option;
1315         # remove per test labels.
1316         $op =~ s/\[.*\]//;
1317         if (!exists($option_map{$op}) &&
1318             !exists($default{$op}) &&
1319             !exists($used_options{$op})) {
1320             $not_used{$op} = 1;
1321         }
1322     }
1323
1324     if (%not_used) {
1325         my $s = "s are";
1326         $s = " is" if (keys %not_used == 1);
1327         print "The following option$s not used; could be a typo:\n";
1328         foreach my $option (keys %not_used) {
1329             print "$option\n";
1330         }
1331         print "Set IGNORE_UNUSED = 1 to have ktest ignore unused variables\n";
1332         if (!read_yn "Do you want to continue?") {
1333             exit -1;
1334         }
1335     }
1336 }
1337
1338 sub __eval_option {
1339     my ($name, $option, $i) = @_;
1340
1341     # Add space to evaluate the character before $
1342     $option = " $option";
1343     my $retval = "";
1344     my $repeated = 0;
1345     my $parent = 0;
1346
1347     foreach my $test (keys %repeat_tests) {
1348         if ($i >= $test &&
1349             $i < $test + $repeat_tests{$test}) {
1350
1351             $repeated = 1;
1352             $parent = $test;
1353             last;
1354         }
1355     }
1356
1357     while ($option =~ /(.*?[^\\])\$\{(.*?)\}(.*)/) {
1358         my $start = $1;
1359         my $var = $2;
1360         my $end = $3;
1361
1362         # Append beginning of line
1363         $retval = "$retval$start";
1364
1365         # If the iteration option OPT[$i] exists, then use that.
1366         # otherwise see if the default OPT (without [$i]) exists.
1367
1368         my $o = "$var\[$i\]";
1369         my $parento = "$var\[$parent\]";
1370
1371         # If a variable contains itself, use the default var
1372         if (($var eq $name) && defined($opt{$var})) {
1373             $o = $opt{$var};
1374             $retval = "$retval$o";
1375         } elsif (defined($opt{$o})) {
1376             $o = $opt{$o};
1377             $retval = "$retval$o";
1378         } elsif ($repeated && defined($opt{$parento})) {
1379             $o = $opt{$parento};
1380             $retval = "$retval$o";
1381         } elsif (defined($opt{$var})) {
1382             $o = $opt{$var};
1383             $retval = "$retval$o";
1384         } elsif ($var eq "KERNEL_VERSION" && defined($make)) {
1385             # special option KERNEL_VERSION uses kernel version
1386             get_version();
1387             $retval = "$retval$version";
1388         } else {
1389             $retval = "$retval\$\{$var\}";
1390         }
1391
1392         $option = $end;
1393     }
1394
1395     $retval = "$retval$option";
1396
1397     $retval =~ s/^ //;
1398
1399     return $retval;
1400 }
1401
1402 sub process_evals {
1403     my ($name, $option, $i) = @_;
1404
1405     my $option_name = "$name\[$i\]";
1406     my $ev;
1407
1408     my $old_option = $option;
1409
1410     if (defined($evals{$option_name})) {
1411         $ev = $evals{$option_name};
1412     } elsif (defined($evals{$name})) {
1413         $ev = $evals{$name};
1414     } else {
1415         return $option;
1416     }
1417
1418     for my $e (@{$ev}) {
1419         eval "\$option =~ $e";
1420     }
1421
1422     if ($option ne $old_option) {
1423         doprint("$name changed from '$old_option' to '$option'\n");
1424     }
1425
1426     return $option;
1427 }
1428
1429 sub eval_option {
1430     my ($name, $option, $i) = @_;
1431
1432     my $prev = "";
1433
1434     # Since an option can evaluate to another option,
1435     # keep iterating until we do not evaluate any more
1436     # options.
1437     my $r = 0;
1438     while ($prev ne $option) {
1439         # Check for recursive evaluations.
1440         # 100 deep should be more than enough.
1441         if ($r++ > 100) {
1442             die "Over 100 evaluations occurred with $option\n" .
1443                 "Check for recursive variables\n";
1444         }
1445         $prev = $option;
1446         $option = __eval_option($name, $option, $i);
1447     }
1448
1449     $option = process_evals($name, $option, $i);
1450
1451     return $option;
1452 }
1453
1454 sub reboot {
1455     my ($time) = @_;
1456     my $powercycle = 0;
1457
1458     # test if the machine can be connected to within a few seconds
1459     my $stat = run_ssh("echo check machine status", $connect_timeout);
1460     if (!$stat) {
1461         doprint("power cycle\n");
1462         $powercycle = 1;
1463     }
1464
1465     if ($powercycle) {
1466         run_command "$power_cycle";
1467
1468         start_monitor;
1469         # flush out current monitor
1470         # May contain the reboot success line
1471         wait_for_monitor 1;
1472
1473     } else {
1474         # Make sure everything has been written to disk
1475         run_ssh("sync", 10);
1476
1477         if (defined($time)) {
1478             start_monitor;
1479             # flush out current monitor
1480             # May contain the reboot success line
1481             wait_for_monitor 1;
1482         }
1483
1484         # try to reboot normally
1485         if (run_command $reboot) {
1486             if (defined($powercycle_after_reboot)) {
1487                 sleep $powercycle_after_reboot;
1488                 run_command "$power_cycle";
1489             }
1490         } else {
1491             # nope? power cycle it.
1492             run_command "$power_cycle";
1493         }
1494     }
1495
1496     if (defined($time)) {
1497
1498         # We only want to get to the new kernel, don't fail
1499         # if we stumble over a call trace.
1500         my $save_ignore_errors = $ignore_errors;
1501         $ignore_errors = 1;
1502
1503         # Look for the good kernel to boot
1504         if (wait_for_monitor($time, "Linux version")) {
1505             # reboot got stuck?
1506             doprint "Reboot did not finish. Forcing power cycle\n";
1507             run_command "$power_cycle";
1508         }
1509
1510         $ignore_errors = $save_ignore_errors;
1511
1512         # Still need to wait for the reboot to finish
1513         wait_for_monitor($time, $reboot_success_line);
1514     }
1515     if ($powercycle || $time) {
1516         end_monitor;
1517     }
1518 }
1519
1520 sub reboot_to_good {
1521     my ($time) = @_;
1522
1523     if (defined($switch_to_good)) {
1524         run_command $switch_to_good;
1525     }
1526
1527     reboot $time;
1528 }
1529
1530 sub do_not_reboot {
1531     my $i = $iteration;
1532
1533     return $test_type eq "build" || $no_reboot ||
1534         ($test_type eq "patchcheck" && $opt{"PATCHCHECK_TYPE[$i]"} eq "build") ||
1535         ($test_type eq "bisect" && $opt{"BISECT_TYPE[$i]"} eq "build") ||
1536         ($test_type eq "config_bisect" && $opt{"CONFIG_BISECT_TYPE[$i]"} eq "build");
1537 }
1538
1539 sub get_test_name() {
1540     my $name;
1541
1542     if (defined($test_name)) {
1543         $name = "$test_name:$test_type";
1544     } else {
1545         $name = $test_type;
1546     }
1547     return $name;
1548 }
1549
1550 sub dodie {
1551     # avoid recursion
1552     return if ($in_die);
1553     $in_die = 1;
1554
1555     if ($monitor_cnt) {
1556         # restore terminal settings
1557         system("stty $stty_orig");
1558     }
1559
1560     my $i = $iteration;
1561
1562     doprint "CRITICAL FAILURE... [TEST $i] ", @_, "\n";
1563
1564     if ($reboot_on_error && !do_not_reboot) {
1565         doprint "REBOOTING\n";
1566         reboot_to_good;
1567     } elsif ($poweroff_on_error && defined($power_off)) {
1568         doprint "POWERING OFF\n";
1569         `$power_off`;
1570     }
1571
1572     if (defined($opt{"LOG_FILE"})) {
1573         print " See $opt{LOG_FILE} for more info.\n";
1574     }
1575
1576     if ($email_on_error) {
1577         my $name = get_test_name;
1578         my $log_file;
1579
1580         if (defined($opt{"LOG_FILE"})) {
1581             my $whence = 2; # End of file
1582             my $log_size = tell LOG;
1583             my $size = $log_size - $test_log_start;
1584
1585             if (defined($mail_max_size)) {
1586                 if ($size > $mail_max_size) {
1587                     $size = $mail_max_size;
1588                 }
1589             }
1590             my $pos = - $size;
1591             $log_file = "$tmpdir/log";
1592             open (L, "$opt{LOG_FILE}") or die "Can't open $opt{LOG_FILE} to read)";
1593             open (O, "> $tmpdir/log") or die "Can't open $tmpdir/log\n";
1594             seek(L, $pos, $whence);
1595             while (<L>) {
1596                 print O;
1597             }
1598             close O;
1599             close L;
1600         }
1601
1602         send_email("KTEST: critical failure for test $i [$name]",
1603                 "Your test started at $script_start_time has failed with:\n@_\n", $log_file);
1604     }
1605
1606     if (defined($post_test)) {
1607         run_command $post_test;
1608     }
1609
1610     die @_, "\n";
1611 }
1612
1613 sub create_pty {
1614     my ($ptm, $pts) = @_;
1615     my $tmp;
1616     my $TIOCSPTLCK = 0x40045431;
1617     my $TIOCGPTN = 0x80045430;
1618
1619     sysopen($ptm, "/dev/ptmx", O_RDWR | O_NONBLOCK) or
1620         dodie "Can't open /dev/ptmx";
1621
1622     # unlockpt()
1623     $tmp = pack("i", 0);
1624     ioctl($ptm, $TIOCSPTLCK, $tmp) or
1625         dodie "ioctl TIOCSPTLCK for /dev/ptmx failed";
1626
1627     # ptsname()
1628     ioctl($ptm, $TIOCGPTN, $tmp) or
1629         dodie "ioctl TIOCGPTN for /dev/ptmx failed";
1630     $tmp = unpack("i", $tmp);
1631
1632     sysopen($pts, "/dev/pts/$tmp", O_RDWR | O_NONBLOCK) or
1633         dodie "Can't open /dev/pts/$tmp";
1634 }
1635
1636 sub exec_console {
1637     my ($ptm, $pts) = @_;
1638
1639     close($ptm);
1640
1641     close(\*STDIN);
1642     close(\*STDOUT);
1643     close(\*STDERR);
1644
1645     open(\*STDIN, '<&', $pts);
1646     open(\*STDOUT, '>&', $pts);
1647     open(\*STDERR, '>&', $pts);
1648
1649     close($pts);
1650
1651     exec $console or
1652         dodie "Can't open console $console";
1653 }
1654
1655 sub open_console {
1656     my ($ptm) = @_;
1657     my $pts = \*PTSFD;
1658     my $pid;
1659
1660     # save terminal settings
1661     $stty_orig = `stty -g`;
1662
1663     # place terminal in cbreak mode so that stdin can be read one character at
1664     # a time without having to wait for a newline
1665     system("stty -icanon -echo -icrnl");
1666
1667     create_pty($ptm, $pts);
1668
1669     $pid = fork;
1670
1671     if (!$pid) {
1672         # child
1673         exec_console($ptm, $pts)
1674     }
1675
1676     # parent
1677     close($pts);
1678
1679     return $pid;
1680
1681     open(PTSFD, "Stop perl from warning about single use of PTSFD");
1682 }
1683
1684 sub close_console {
1685     my ($fp, $pid) = @_;
1686
1687     doprint "kill child process $pid\n";
1688     kill $close_console_signal, $pid;
1689
1690     doprint "wait for child process $pid to exit\n";
1691     waitpid($pid, 0);
1692
1693     print "closing!\n";
1694     close($fp);
1695
1696     # restore terminal settings
1697     system("stty $stty_orig");
1698 }
1699
1700 sub start_monitor {
1701     if ($monitor_cnt++) {
1702         return;
1703     }
1704     $monitor_fp = \*MONFD;
1705     $monitor_pid = open_console $monitor_fp;
1706
1707     return;
1708
1709     open(MONFD, "Stop perl from warning about single use of MONFD");
1710 }
1711
1712 sub end_monitor {
1713     return if (!defined $console);
1714     if (--$monitor_cnt) {
1715         return;
1716     }
1717     close_console($monitor_fp, $monitor_pid);
1718 }
1719
1720 sub wait_for_monitor {
1721     my ($time, $stop) = @_;
1722     my $full_line = "";
1723     my $line;
1724     my $booted = 0;
1725     my $start_time = time;
1726     my $skip_call_trace = 0;
1727     my $bug = 0;
1728     my $bug_ignored = 0;
1729     my $now;
1730
1731     doprint "** Wait for monitor to settle down **\n";
1732
1733     # read the monitor and wait for the system to calm down
1734     while (!$booted) {
1735         $line = wait_for_input($monitor_fp, $time);
1736         last if (!defined($line));
1737         print "$line";
1738         $full_line .= $line;
1739
1740         if (defined($stop) && $full_line =~ /$stop/) {
1741             doprint "wait for monitor detected $stop\n";
1742             $booted = 1;
1743         }
1744
1745         if ($full_line =~ /\[ backtrace testing \]/) {
1746             $skip_call_trace = 1;
1747         }
1748
1749         if ($full_line =~ /call trace:/i) {
1750             if (!$bug && !$skip_call_trace) {
1751                 if ($ignore_errors) {
1752                     $bug_ignored = 1;
1753                 } else {
1754                     $bug = 1;
1755                 }
1756             }
1757         }
1758
1759         if ($full_line =~ /\[ end of backtrace testing \]/) {
1760             $skip_call_trace = 0;
1761         }
1762
1763         if ($full_line =~ /Kernel panic -/) {
1764             $bug = 1;
1765         }
1766
1767         if ($line =~ /\n/) {
1768             $full_line = "";
1769         }
1770         $now = time;
1771         if ($now - $start_time >= $max_monitor_wait) {
1772             doprint "Exiting monitor flush due to hitting MAX_MONITOR_WAIT\n";
1773             return 1;
1774         }
1775     }
1776     print "** Monitor flushed **\n";
1777
1778     # if stop is defined but wasn't hit, return error
1779     # used by reboot (which wants to see a reboot)
1780     if (defined($stop) && !$booted) {
1781         $bug = 1;
1782     }
1783     return $bug;
1784 }
1785
1786 sub save_logs {
1787     my ($result, $basedir) = @_;
1788     my @t = localtime;
1789     my $date = sprintf "%04d%02d%02d%02d%02d%02d",
1790         1900+$t[5],$t[4],$t[3],$t[2],$t[1],$t[0];
1791
1792     my $type = $build_type;
1793     if ($type =~ /useconfig/) {
1794         $type = "useconfig";
1795     }
1796
1797     my $dir = "$machine-$test_type-$type-$result-$date";
1798
1799     $dir = "$basedir/$dir";
1800
1801     if (!-d $dir) {
1802         mkpath($dir) or
1803             dodie "can't create $dir";
1804     }
1805
1806     my %files = (
1807         "config" => $output_config,
1808         "buildlog" => $buildlog,
1809         "dmesg" => $dmesg,
1810         "testlog" => $testlog,
1811     );
1812
1813     while (my ($name, $source) = each(%files)) {
1814         if (-f "$source") {
1815             cp "$source", "$dir/$name" or
1816                 dodie "failed to copy $source";
1817         }
1818     }
1819
1820     doprint "*** Saved info to $dir ***\n";
1821 }
1822
1823 sub fail {
1824
1825     if ($die_on_failure) {
1826         dodie @_;
1827     }
1828
1829     doprint "FAILED\n";
1830
1831     my $i = $iteration;
1832
1833     # no need to reboot for just building.
1834     if (!do_not_reboot) {
1835         doprint "REBOOTING\n";
1836         reboot_to_good $sleep_time;
1837     }
1838
1839     my $name = "";
1840
1841     if (defined($test_name)) {
1842         $name = " ($test_name)";
1843     }
1844
1845     print_times;
1846
1847     doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
1848     doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
1849     doprint "KTEST RESULT: TEST $i$name Failed: ", @_, "\n";
1850     doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
1851     doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
1852
1853     if (defined($store_failures)) {
1854         save_logs "fail", $store_failures;
1855     }
1856
1857     if (defined($post_test)) {
1858         run_command $post_test;
1859     }
1860
1861     return 1;
1862 }
1863
1864 sub run_command {
1865     my ($command, $redirect, $timeout) = @_;
1866     my $start_time;
1867     my $end_time;
1868     my $dolog = 0;
1869     my $dord = 0;
1870     my $dostdout = 0;
1871     my $pid;
1872     my $command_orig = $command;
1873
1874     $command =~ s/\$SSH_USER/$ssh_user/g;
1875     $command =~ s/\$MACHINE/$machine/g;
1876
1877     if (!defined($timeout)) {
1878         $timeout = $run_timeout;
1879     }
1880
1881     if (!defined($timeout)) {
1882         $timeout = -1; # tell wait_for_input to wait indefinitely
1883     }
1884
1885     doprint("$command ... ");
1886     $start_time = time;
1887
1888     $pid = open(CMD, "$command 2>&1 |") or
1889         (fail "unable to exec $command" and return 0);
1890
1891     if (defined($opt{"LOG_FILE"})) {
1892         $dolog = 1;
1893     }
1894
1895     if (defined($redirect)) {
1896         if ($redirect eq 1) {
1897             $dostdout = 1;
1898             # Have the output of the command on its own line
1899             doprint "\n";
1900         } else {
1901             open (RD, ">$redirect") or
1902                 dodie "failed to write to redirect $redirect";
1903             $dord = 1;
1904         }
1905     }
1906
1907     my $hit_timeout = 0;
1908
1909     while (1) {
1910         my $fp = \*CMD;
1911         my $line = wait_for_input($fp, $timeout);
1912         if (!defined($line)) {
1913             my $now = time;
1914             if ($timeout >= 0 && (($now - $start_time) >= $timeout)) {
1915                 doprint "Hit timeout of $timeout, killing process\n";
1916                 $hit_timeout = 1;
1917                 kill 9, $pid;
1918             }
1919             last;
1920         }
1921         print LOG $line if ($dolog);
1922         print RD $line if ($dord);
1923         print $line if ($dostdout);
1924     }
1925
1926     waitpid($pid, 0);
1927     # shift 8 for real exit status
1928     $run_command_status = $? >> 8;
1929
1930     if ($command_orig eq $default{REBOOT} &&
1931         $run_command_status == $reboot_return_code) {
1932         $run_command_status = 0;
1933     }
1934
1935     close(CMD);
1936     close(RD)  if ($dord);
1937
1938     $end_time = time;
1939     my $delta = $end_time - $start_time;
1940
1941     if ($delta == 1) {
1942         doprint "[1 second] ";
1943     } else {
1944         doprint "[$delta seconds] ";
1945     }
1946
1947     if ($hit_timeout) {
1948         $run_command_status = 1;
1949     }
1950
1951     if ($run_command_status) {
1952         doprint "FAILED!\n";
1953     } else {
1954         doprint "SUCCESS\n";
1955     }
1956
1957     return !$run_command_status;
1958 }
1959
1960 sub run_ssh {
1961     my ($cmd, $timeout) = @_;
1962     my $cp_exec = $ssh_exec;
1963
1964     $cp_exec =~ s/\$SSH_COMMAND/$cmd/g;
1965     return run_command "$cp_exec", undef , $timeout;
1966 }
1967
1968 sub run_scp {
1969     my ($src, $dst, $cp_scp) = @_;
1970
1971     $cp_scp =~ s/\$SRC_FILE/$src/g;
1972     $cp_scp =~ s/\$DST_FILE/$dst/g;
1973
1974     return run_command "$cp_scp";
1975 }
1976
1977 sub run_scp_install {
1978     my ($src, $dst) = @_;
1979
1980     my $cp_scp = $scp_to_target_install;
1981
1982     return run_scp($src, $dst, $cp_scp);
1983 }
1984
1985 sub run_scp_mod {
1986     my ($src, $dst) = @_;
1987
1988     my $cp_scp = $scp_to_target;
1989
1990     return run_scp($src, $dst, $cp_scp);
1991 }
1992
1993 sub _get_grub_index {
1994
1995     my ($command, $target, $skip, $submenu) = @_;
1996
1997     return if (defined($grub_number) && defined($last_grub_menu) &&
1998         $last_grub_menu eq $grub_menu && defined($last_machine) &&
1999         $last_machine eq $machine);
2000
2001     doprint "Find $reboot_type menu ... ";
2002     $grub_number = -1;
2003
2004     my $ssh_grub = $ssh_exec;
2005     $ssh_grub =~ s,\$SSH_COMMAND,$command,g;
2006
2007     open(IN, "$ssh_grub |") or
2008         dodie "unable to execute $command";
2009
2010     my $found = 0;
2011
2012     my $submenu_number = 0;
2013
2014     while (<IN>) {
2015         if (/$target/) {
2016             $grub_number++;
2017             $found = 1;
2018             last;
2019         } elsif (defined($submenu) && /$submenu/) {
2020                 $submenu_number++;
2021                 $grub_number = -1;
2022         } elsif (/$skip/) {
2023             $grub_number++;
2024         }
2025     }
2026     close(IN);
2027
2028     dodie "Could not find '$grub_menu' through $command on $machine"
2029         if (!$found);
2030     if ($submenu_number > 0) {
2031         $grub_number = "$submenu_number>$grub_number";
2032     }
2033     doprint "$grub_number\n";
2034     $last_grub_menu = $grub_menu;
2035     $last_machine = $machine;
2036 }
2037
2038 sub get_grub_index {
2039
2040     my $command;
2041     my $target;
2042     my $skip;
2043     my $submenu;
2044     my $grub_menu_qt;
2045
2046     if ($reboot_type !~ /^grub/) {
2047         return;
2048     }
2049
2050     $grub_menu_qt = quotemeta($grub_menu);
2051
2052     if ($reboot_type eq "grub") {
2053         $command = "cat /boot/grub/menu.lst";
2054         $target = '^\s*title\s+' . $grub_menu_qt . '\s*$';
2055         $skip = '^\s*title\s';
2056     } elsif ($reboot_type eq "grub2") {
2057         $command = "cat $grub_file";
2058         $target = '^\s*menuentry.*' . $grub_menu_qt;
2059         $skip = '^\s*menuentry\s';
2060         $submenu = '^\s*submenu\s';
2061     } elsif ($reboot_type eq "grub2bls") {
2062         $command = $grub_bls_get;
2063         $target = '^title=.*' . $grub_menu_qt;
2064         $skip = '^title=';
2065     } else {
2066         return;
2067     }
2068
2069     _get_grub_index($command, $target, $skip, $submenu);
2070 }
2071
2072 sub wait_for_input {
2073     my ($fp, $time) = @_;
2074     my $start_time;
2075     my $rin;
2076     my $rout;
2077     my $nr;
2078     my $buf;
2079     my $line;
2080     my $ch;
2081
2082     if (!defined($time)) {
2083         $time = $timeout;
2084     }
2085
2086     if ($time < 0) {
2087         # Negative number means wait indefinitely
2088         undef $time;
2089     }
2090
2091     $rin = '';
2092     vec($rin, fileno($fp), 1) = 1;
2093     vec($rin, fileno(\*STDIN), 1) = 1;
2094
2095     $start_time = time;
2096
2097     while (1) {
2098         $nr = select($rout=$rin, undef, undef, $time);
2099
2100         last if ($nr <= 0);
2101
2102         # copy data from stdin to the console
2103         if (vec($rout, fileno(\*STDIN), 1) == 1) {
2104             $nr = sysread(\*STDIN, $buf, 1000);
2105             syswrite($fp, $buf, $nr) if ($nr > 0);
2106         }
2107
2108         # The timeout is based on time waiting for the fp data
2109         if (vec($rout, fileno($fp), 1) != 1) {
2110             last if (defined($time) && (time - $start_time > $time));
2111             next;
2112         }
2113
2114         $line = "";
2115
2116         # try to read one char at a time
2117         while (sysread $fp, $ch, 1) {
2118             $line .= $ch;
2119             last if ($ch eq "\n");
2120         }
2121
2122         last if (!length($line));
2123
2124         return $line;
2125     }
2126     return undef;
2127 }
2128
2129 sub reboot_to {
2130     if (defined($switch_to_test)) {
2131         run_command $switch_to_test;
2132     }
2133
2134     if ($reboot_type eq "grub") {
2135         run_ssh "'(echo \"savedefault --default=$grub_number --once\" | grub --batch)'";
2136     } elsif (($reboot_type eq "grub2") or ($reboot_type eq "grub2bls")) {
2137         run_ssh "$grub_reboot \"'$grub_number'\"";
2138     } elsif ($reboot_type eq "syslinux") {
2139         run_ssh "$syslinux --once \\\"$syslinux_label\\\" $syslinux_path";
2140     } elsif (defined $reboot_script) {
2141         run_command "$reboot_script";
2142     }
2143     reboot;
2144 }
2145
2146 sub get_sha1 {
2147     my ($commit) = @_;
2148
2149     doprint "git rev-list --max-count=1 $commit ... ";
2150     my $sha1 = `git rev-list --max-count=1 $commit`;
2151     my $ret = $?;
2152
2153     logit $sha1;
2154
2155     if ($ret) {
2156         doprint "FAILED\n";
2157         dodie "Failed to get git $commit";
2158     }
2159
2160     print "SUCCESS\n";
2161
2162     chomp $sha1;
2163
2164     return $sha1;
2165 }
2166
2167 sub monitor {
2168     my $booted = 0;
2169     my $bug = 0;
2170     my $bug_ignored = 0;
2171     my $skip_call_trace = 0;
2172     my $loops;
2173
2174     my $start_time = time;
2175
2176     wait_for_monitor 5;
2177
2178     my $line;
2179     my $full_line = "";
2180
2181     open(DMESG, "> $dmesg") or
2182         dodie "unable to write to $dmesg";
2183
2184     reboot_to;
2185
2186     my $success_start;
2187     my $failure_start;
2188     my $monitor_start = time;
2189     my $done = 0;
2190     my $version_found = 0;
2191
2192     while (!$done) {
2193         if ($bug && defined($stop_after_failure) &&
2194             $stop_after_failure >= 0) {
2195             my $time = $stop_after_failure - (time - $failure_start);
2196             $line = wait_for_input($monitor_fp, $time);
2197             if (!defined($line)) {
2198                 doprint "bug timed out after $booted_timeout seconds\n";
2199                 doprint "Test forced to stop after $stop_after_failure seconds after failure\n";
2200                 last;
2201             }
2202         } elsif ($booted) {
2203             $line = wait_for_input($monitor_fp, $booted_timeout);
2204             if (!defined($line)) {
2205                 my $s = $booted_timeout == 1 ? "" : "s";
2206                 doprint "Successful boot found: break after $booted_timeout second$s\n";
2207                 last;
2208             }
2209         } else {
2210             $line = wait_for_input($monitor_fp);
2211             if (!defined($line)) {
2212                 my $s = $timeout == 1 ? "" : "s";
2213                 doprint "Timed out after $timeout second$s\n";
2214                 last;
2215             }
2216         }
2217
2218         doprint $line;
2219         print DMESG $line;
2220
2221         # we are not guaranteed to get a full line
2222         $full_line .= $line;
2223
2224         if ($full_line =~ /$success_line/) {
2225             $booted = 1;
2226             $success_start = time;
2227         }
2228
2229         if ($booted && defined($stop_after_success) &&
2230             $stop_after_success >= 0) {
2231             my $now = time;
2232             if ($now - $success_start >= $stop_after_success) {
2233                 doprint "Test forced to stop after $stop_after_success seconds after success\n";
2234                 last;
2235             }
2236         }
2237
2238         if ($full_line =~ /\[ backtrace testing \]/) {
2239             $skip_call_trace = 1;
2240         }
2241
2242         if ($full_line =~ /call trace:/i) {
2243             if (!$bug && !$skip_call_trace) {
2244                 if ($ignore_errors) {
2245                     $bug_ignored = 1;
2246                 } else {
2247                     $bug = 1;
2248                     $failure_start = time;
2249                 }
2250             }
2251         }
2252
2253         if ($bug && defined($stop_after_failure) &&
2254             $stop_after_failure >= 0) {
2255             my $now = time;
2256             if ($now - $failure_start >= $stop_after_failure) {
2257                 doprint "Test forced to stop after $stop_after_failure seconds after failure\n";
2258                 last;
2259             }
2260         }
2261
2262         if ($full_line =~ /\[ end of backtrace testing \]/) {
2263             $skip_call_trace = 0;
2264         }
2265
2266         if ($full_line =~ /Kernel panic -/) {
2267             $failure_start = time;
2268             $bug = 1;
2269         }
2270
2271         # Detect triple faults by testing the banner
2272         if ($full_line =~ /\bLinux version (\S+).*\n/) {
2273             if ($1 eq $version) {
2274                 $version_found = 1;
2275             } elsif ($version_found && $detect_triplefault) {
2276                 # We already booted into the kernel we are testing,
2277                 # but now we booted into another kernel?
2278                 # Consider this a triple fault.
2279                 doprint "Already booted in Linux kernel $version, but now\n";
2280                 doprint "we booted into Linux kernel $1.\n";
2281                 doprint "Assuming that this is a triple fault.\n";
2282                 doprint "To disable this: set DETECT_TRIPLE_FAULT to 0\n";
2283                 last;
2284             }
2285         }
2286
2287         if ($line =~ /\n/) {
2288             $full_line = "";
2289         }
2290
2291         if ($stop_test_after > 0 && !$booted && !$bug) {
2292             if (time - $monitor_start > $stop_test_after) {
2293                 doprint "STOP_TEST_AFTER ($stop_test_after seconds) timed out\n";
2294                 $done = 1;
2295             }
2296         }
2297     }
2298
2299     my $end_time = time;
2300     $reboot_time = $end_time - $start_time;
2301
2302     close(DMESG);
2303
2304     if ($bug) {
2305         return 0 if ($in_bisect);
2306         fail "failed - got a bug report" and return 0;
2307     }
2308
2309     if (!$booted) {
2310         return 0 if ($in_bisect);
2311         fail "failed - never got a boot prompt." and return 0;
2312     }
2313
2314     if ($bug_ignored) {
2315         doprint "WARNING: Call Trace detected but ignored due to IGNORE_ERRORS=1\n";
2316     }
2317
2318     return 1;
2319 }
2320
2321 sub eval_kernel_version {
2322     my ($option) = @_;
2323
2324     $option =~ s/\$KERNEL_VERSION/$version/g;
2325
2326     return $option;
2327 }
2328
2329 sub do_post_install {
2330
2331     return if (!defined($post_install));
2332
2333     my $cp_post_install = eval_kernel_version $post_install;
2334     run_command "$cp_post_install" or
2335         dodie "Failed to run post install";
2336 }
2337
2338 # Sometimes the reboot fails, and will hang. We try to ssh to the box
2339 # and if we fail, we force another reboot, that should powercycle it.
2340 sub test_booted {
2341     if (!run_ssh "echo testing connection") {
2342         reboot $sleep_time;
2343     }
2344 }
2345
2346 sub install {
2347
2348     return if ($no_install);
2349
2350     my $start_time = time;
2351
2352     if (defined($pre_install)) {
2353         my $cp_pre_install = eval_kernel_version $pre_install;
2354         run_command "$cp_pre_install" or
2355             dodie "Failed to run pre install";
2356     }
2357
2358     my $cp_target = eval_kernel_version $target_image;
2359
2360     test_booted;
2361
2362     run_scp_install "$outputdir/$build_target", "$cp_target" or
2363         dodie "failed to copy image";
2364
2365     my $install_mods = 0;
2366
2367     # should we process modules?
2368     $install_mods = 0;
2369     open(IN, "$output_config") or dodie("Can't read config file");
2370     while (<IN>) {
2371         if (/CONFIG_MODULES(=y)?/) {
2372             if (defined($1)) {
2373                 $install_mods = 1;
2374                 last;
2375             }
2376         }
2377     }
2378     close(IN);
2379
2380     if (!$install_mods) {
2381         do_post_install;
2382         doprint "No modules needed\n";
2383         my $end_time = time;
2384         $install_time = $end_time - $start_time;
2385         return;
2386     }
2387
2388     run_command "$make INSTALL_MOD_STRIP=1 INSTALL_MOD_PATH=$tmpdir modules_install" or
2389         dodie "Failed to install modules";
2390
2391     my $modlib = "/lib/modules/$version";
2392     my $modtar = "ktest-mods.tar.bz2";
2393
2394     run_ssh "rm -rf $modlib" or
2395         dodie "failed to remove old mods: $modlib";
2396
2397     # would be nice if scp -r did not follow symbolic links
2398     run_command "cd $tmpdir && tar -cjf $modtar lib/modules/$version" or
2399         dodie "making tarball";
2400
2401     run_scp_mod "$tmpdir/$modtar", "/tmp" or
2402         dodie "failed to copy modules";
2403
2404     unlink "$tmpdir/$modtar";
2405
2406     run_ssh "'(cd / && tar xjf /tmp/$modtar)'" or
2407         dodie "failed to tar modules";
2408
2409     run_ssh "rm -f /tmp/$modtar";
2410
2411     do_post_install;
2412
2413     my $end_time = time;
2414     $install_time = $end_time - $start_time;
2415 }
2416
2417 sub get_version {
2418     # get the release name
2419     return if ($have_version);
2420     doprint "$make kernelrelease ... ";
2421     $version = `$make -s kernelrelease | tail -1`;
2422     chomp($version);
2423     doprint "$version\n";
2424     $have_version = 1;
2425 }
2426
2427 sub start_monitor_and_install {
2428     # Make sure the stable kernel has finished booting
2429
2430     # Install bisects, don't need console
2431     if (defined $console) {
2432         start_monitor;
2433         wait_for_monitor 5;
2434         end_monitor;
2435     }
2436
2437     get_grub_index;
2438     get_version;
2439     install;
2440
2441     start_monitor if (defined $console);
2442     return monitor;
2443 }
2444
2445 sub process_warning_line {
2446     my ($line) = @_;
2447
2448     chomp $line;
2449
2450     # for distcc heterogeneous systems, some compilers
2451     # do things differently causing warning lines
2452     # to be slightly different. This makes an attempt
2453     # to fixe those issues.
2454
2455     # chop off the index into the line
2456     # using distcc, some compilers give different indexes
2457     # depending on white space
2458     $line =~ s/^(\s*\S+:\d+:)\d+/$1/;
2459
2460     # Some compilers use UTF-8 extended for quotes and some don't.
2461     $line =~ s/$utf8_quote/'/g;
2462
2463     return $line;
2464 }
2465
2466 # Read buildlog and check against warnings file for any
2467 # new warnings.
2468 #
2469 # Returns 1 if OK
2470 #         0 otherwise
2471 sub check_buildlog {
2472     my %warnings_list;
2473
2474     # Failed builds should not reboot the target
2475     my $save_no_reboot = $no_reboot;
2476     $no_reboot = 1;
2477
2478     if (-f $warnings_file) {
2479         open(IN, $warnings_file) or
2480             dodie "Error opening $warnings_file";
2481
2482         while (<IN>) {
2483             if (/$check_build_re/) {
2484                 my $warning = process_warning_line $_;
2485
2486                 $warnings_list{$warning} = 1;
2487             }
2488         }
2489         close(IN);
2490     }
2491
2492     open(IN, $buildlog) or dodie "Can't open $buildlog";
2493     while (<IN>) {
2494         if (/$check_build_re/) {
2495             my $warning = process_warning_line $_;
2496
2497             if (!defined $warnings_list{$warning}) {
2498                 $warning_found++;
2499
2500                 # If warnings file didn't exist, and WARNINGS_FILE exist,
2501                 # then we fail on any warning!
2502                 if (defined $warnings_file) {
2503                     fail "New warning found (not in $warnings_file)\n$_\n";
2504                     $no_reboot = $save_no_reboot;
2505                     return 0;
2506                 }
2507             }
2508         }
2509     }
2510     $no_reboot = $save_no_reboot;
2511     close(IN);
2512 }
2513
2514 sub check_patch_buildlog {
2515     my ($patch) = @_;
2516
2517     my @files = `git show $patch | diffstat -l`;
2518
2519     foreach my $file (@files) {
2520         chomp $file;
2521     }
2522
2523     open(IN, "git show $patch |") or
2524         dodie "failed to show $patch";
2525     while (<IN>) {
2526         if (m,^--- a/(.*),) {
2527             chomp $1;
2528             $files[$#files] = $1;
2529         }
2530     }
2531     close(IN);
2532
2533     open(IN, $buildlog) or dodie "Can't open $buildlog";
2534     while (<IN>) {
2535         if (/^\s*(.*?):.*(warning|error)/) {
2536             my $err = $1;
2537             foreach my $file (@files) {
2538                 my $fullpath = "$builddir/$file";
2539                 if ($file eq $err || $fullpath eq $err) {
2540                     fail "$file built with warnings" and return 0;
2541                 }
2542             }
2543         }
2544     }
2545     close(IN);
2546
2547     return 1;
2548 }
2549
2550 sub apply_min_config {
2551     my $outconfig = "$output_config.new";
2552
2553     # Read the config file and remove anything that
2554     # is in the force_config hash (from minconfig and others)
2555     # then add the force config back.
2556
2557     doprint "Applying minimum configurations into $output_config.new\n";
2558
2559     open (OUT, ">$outconfig") or
2560         dodie "Can't create $outconfig";
2561
2562     if (-f $output_config) {
2563         open (IN, $output_config) or
2564             dodie "Failed to open $output_config";
2565         while (<IN>) {
2566             if (/^(# )?(CONFIG_[^\s=]*)/) {
2567                 next if (defined($force_config{$2}));
2568             }
2569             print OUT;
2570         }
2571         close IN;
2572     }
2573     foreach my $config (keys %force_config) {
2574         print OUT "$force_config{$config}\n";
2575     }
2576     close OUT;
2577
2578     run_command "mv $outconfig $output_config";
2579 }
2580
2581 sub make_oldconfig {
2582
2583     my @force_list = keys %force_config;
2584
2585     if ($#force_list >= 0) {
2586         apply_min_config;
2587     }
2588
2589     if (!run_command "$make olddefconfig") {
2590         # Perhaps olddefconfig doesn't exist in this version of the kernel
2591         # try oldnoconfig
2592         doprint "olddefconfig failed, trying make oldnoconfig\n";
2593         if (!run_command "$make oldnoconfig") {
2594             doprint "oldnoconfig failed, trying yes '' | make oldconfig\n";
2595             # try a yes '' | oldconfig
2596             run_command "yes '' | $make oldconfig" or
2597                 dodie "failed make config oldconfig";
2598         }
2599     }
2600 }
2601
2602 # read a config file and use this to force new configs.
2603 sub load_force_config {
2604     my ($config) = @_;
2605
2606     doprint "Loading force configs from $config\n";
2607     open(IN, $config) or
2608         dodie "failed to read $config";
2609     while (<IN>) {
2610         chomp;
2611         if (/^(CONFIG[^\s=]*)(\s*=.*)/) {
2612             $force_config{$1} = $_;
2613         } elsif (/^# (CONFIG_\S*) is not set/) {
2614             $force_config{$1} = $_;
2615         }
2616     }
2617     close IN;
2618 }
2619
2620 sub build {
2621     my ($type) = @_;
2622
2623     unlink $buildlog;
2624
2625     my $start_time = time;
2626
2627     # Failed builds should not reboot the target
2628     my $save_no_reboot = $no_reboot;
2629     $no_reboot = 1;
2630
2631     # Calculate a new version from here.
2632     $have_version = 0;
2633
2634     if (defined($pre_build)) {
2635         my $ret = run_command $pre_build;
2636         if (!$ret && defined($pre_build_die) &&
2637             $pre_build_die) {
2638             dodie "failed to pre_build\n";
2639         }
2640     }
2641
2642     if ($type =~ /^useconfig:(.*)/) {
2643         run_command "cp $1 $output_config" or
2644             dodie "could not copy $1 to .config";
2645
2646         $type = "oldconfig";
2647     }
2648
2649     # old config can ask questions
2650     if ($type eq "oldconfig") {
2651         $type = "olddefconfig";
2652
2653         # allow for empty configs
2654         run_command "touch $output_config";
2655
2656         if (!$noclean) {
2657             run_command "mv $output_config $outputdir/config_temp" or
2658                 dodie "moving .config";
2659
2660             run_command "$make mrproper" or dodie "make mrproper";
2661
2662             run_command "mv $outputdir/config_temp $output_config" or
2663                 dodie "moving config_temp";
2664         }
2665     } elsif (!$noclean) {
2666         unlink "$output_config";
2667         run_command "$make mrproper" or
2668             dodie "make mrproper";
2669     }
2670
2671     # add something to distinguish this build
2672     open(OUT, "> $outputdir/localversion") or dodie("Can't make localversion file");
2673     print OUT "$localversion\n";
2674     close(OUT);
2675
2676     if (defined($minconfig)) {
2677         load_force_config($minconfig);
2678     }
2679
2680     if ($type ne "olddefconfig") {
2681         run_command "$make $type" or
2682             dodie "failed make config";
2683     }
2684     # Run old config regardless, to enforce min configurations
2685     make_oldconfig;
2686
2687     if (not defined($build_options)){
2688         $build_options = "";
2689     }
2690     my $build_ret = run_command "$make $build_options", $buildlog;
2691
2692     if (defined($post_build)) {
2693         # Because a post build may change the kernel version
2694         # do it now.
2695         get_version;
2696         my $ret = run_command $post_build;
2697         if (!$ret && defined($post_build_die) &&
2698             $post_build_die) {
2699             dodie "failed to post_build\n";
2700         }
2701     }
2702
2703     if (!$build_ret) {
2704         # bisect may need this to pass
2705         if ($in_bisect) {
2706             $no_reboot = $save_no_reboot;
2707             return 0;
2708         }
2709         fail "failed build" and return 0;
2710     }
2711
2712     $no_reboot = $save_no_reboot;
2713
2714     my $end_time = time;
2715     $build_time = $end_time - $start_time;
2716
2717     return 1;
2718 }
2719
2720 sub halt {
2721     if (!run_ssh "halt" or defined($power_off)) {
2722         if (defined($poweroff_after_halt)) {
2723             sleep $poweroff_after_halt;
2724             run_command "$power_off";
2725         }
2726     } else {
2727         # nope? the zap it!
2728         run_command "$power_off";
2729     }
2730 }
2731
2732 sub success {
2733     my ($i) = @_;
2734
2735     $successes++;
2736
2737     my $name = "";
2738
2739     if (defined($test_name)) {
2740         $name = " ($test_name)";
2741     }
2742
2743     print_times;
2744
2745     doprint "\n\n";
2746     doprint "*******************************************\n";
2747     doprint "*******************************************\n";
2748     doprint "KTEST RESULT: TEST $i$name SUCCESS!!!!   **\n";
2749     doprint "*******************************************\n";
2750     doprint "*******************************************\n";
2751
2752     if (defined($store_successes)) {
2753         save_logs "success", $store_successes;
2754     }
2755
2756     if ($i != $opt{"NUM_TESTS"} && !do_not_reboot) {
2757         doprint "Reboot and wait $sleep_time seconds\n";
2758         reboot_to_good $sleep_time;
2759     }
2760
2761     if (defined($post_test)) {
2762         run_command $post_test;
2763     }
2764 }
2765
2766 sub answer_bisect {
2767     for (;;) {
2768         doprint "Pass, fail, or skip? [p/f/s]";
2769         my $ans = <STDIN>;
2770         chomp $ans;
2771         if ($ans eq "p" || $ans eq "P") {
2772             return 1;
2773         } elsif ($ans eq "f" || $ans eq "F") {
2774             return 0;
2775         } elsif ($ans eq "s" || $ans eq "S") {
2776             return -1;
2777         } else {
2778             print "Please answer 'p', 'f', or 's'\n";
2779         }
2780     }
2781 }
2782
2783 sub child_run_test {
2784
2785     # child should have no power
2786     $reboot_on_error = 0;
2787     $poweroff_on_error = 0;
2788     $die_on_failure = 1;
2789
2790     run_command $run_test, $testlog;
2791
2792     exit $run_command_status;
2793 }
2794
2795 sub child_finished {
2796     $child_done = 1;
2797 }
2798
2799 sub do_run_test {
2800     my $child_pid;
2801     my $child_exit;
2802     my $line;
2803     my $full_line;
2804     my $bug = 0;
2805     my $bug_ignored = 0;
2806
2807     my $start_time = time;
2808
2809     wait_for_monitor 1;
2810
2811     doprint "run test $run_test\n";
2812
2813     $child_done = 0;
2814
2815     $SIG{CHLD} = qw(child_finished);
2816
2817     $child_pid = fork;
2818
2819     child_run_test if (!$child_pid);
2820
2821     $full_line = "";
2822
2823     do {
2824         $line = wait_for_input($monitor_fp, 1);
2825         if (defined($line)) {
2826
2827             # we are not guaranteed to get a full line
2828             $full_line .= $line;
2829             doprint $line;
2830
2831             if ($full_line =~ /call trace:/i) {
2832                 if ($ignore_errors) {
2833                     $bug_ignored = 1;
2834                 } else {
2835                     $bug = 1;
2836                 }
2837             }
2838
2839             if ($full_line =~ /Kernel panic -/) {
2840                 $bug = 1;
2841             }
2842
2843             if ($line =~ /\n/) {
2844                 $full_line = "";
2845             }
2846         }
2847     } while (!$child_done && !$bug);
2848
2849     if (!$bug && $bug_ignored) {
2850         doprint "WARNING: Call Trace detected but ignored due to IGNORE_ERRORS=1\n";
2851     }
2852
2853     if ($bug) {
2854         my $failure_start = time;
2855         my $now;
2856         do {
2857             $line = wait_for_input($monitor_fp, 1);
2858             if (defined($line)) {
2859                 doprint $line;
2860             }
2861             $now = time;
2862             if ($now - $failure_start >= $stop_after_failure) {
2863                 last;
2864             }
2865         } while (defined($line));
2866
2867         doprint "Detected kernel crash!\n";
2868         # kill the child with extreme prejudice
2869         kill 9, $child_pid;
2870     }
2871
2872     waitpid $child_pid, 0;
2873     $child_exit = $? >> 8;
2874
2875     my $end_time = time;
2876     $test_time = $end_time - $start_time;
2877
2878     if (!$bug && $in_bisect) {
2879         if (defined($bisect_ret_good)) {
2880             if ($child_exit == $bisect_ret_good) {
2881                 return 1;
2882             }
2883         }
2884         if (defined($bisect_ret_skip)) {
2885             if ($child_exit == $bisect_ret_skip) {
2886                 return -1;
2887             }
2888         }
2889         if (defined($bisect_ret_abort)) {
2890             if ($child_exit == $bisect_ret_abort) {
2891                 fail "test abort" and return -2;
2892             }
2893         }
2894         if (defined($bisect_ret_bad)) {
2895             if ($child_exit == $bisect_ret_skip) {
2896                 return 0;
2897             }
2898         }
2899         if (defined($bisect_ret_default)) {
2900             if ($bisect_ret_default eq "good") {
2901                 return 1;
2902             } elsif ($bisect_ret_default eq "bad") {
2903                 return 0;
2904             } elsif ($bisect_ret_default eq "skip") {
2905                 return -1;
2906             } elsif ($bisect_ret_default eq "abort") {
2907                 return -2;
2908             } else {
2909                 fail "unknown default action: $bisect_ret_default"
2910                     and return -2;
2911             }
2912         }
2913     }
2914
2915     if ($bug || $child_exit) {
2916         return 0 if $in_bisect;
2917         fail "test failed" and return 0;
2918     }
2919     return 1;
2920 }
2921
2922 sub run_git_bisect {
2923     my ($command) = @_;
2924
2925     doprint "$command ... ";
2926
2927     my $output = `$command 2>&1`;
2928     my $ret = $?;
2929
2930     logit $output;
2931
2932     if ($ret) {
2933         doprint "FAILED\n";
2934         dodie "Failed to git bisect";
2935     }
2936
2937     doprint "SUCCESS\n";
2938     if ($output =~ m/^(Bisecting: .*\(roughly \d+ steps?\))\s+\[([[:xdigit:]]+)\]/) {
2939         doprint "$1 [$2]\n";
2940     } elsif ($output =~ m/^([[:xdigit:]]+) is the first bad commit/) {
2941         $bisect_bad_commit = $1;
2942         doprint "Found bad commit... $1\n";
2943         return 0;
2944     } else {
2945         # we already logged it, just print it now.
2946         print $output;
2947     }
2948
2949     return 1;
2950 }
2951
2952 sub bisect_reboot {
2953     doprint "Reboot and sleep $bisect_sleep_time seconds\n";
2954     reboot_to_good $bisect_sleep_time;
2955 }
2956
2957 # returns 1 on success, 0 on failure, -1 on skip
2958 sub run_bisect_test {
2959     my ($type, $buildtype) = @_;
2960
2961     my $failed = 0;
2962     my $result;
2963     my $output;
2964     my $ret;
2965
2966     $in_bisect = 1;
2967
2968     build $buildtype or $failed = 1;
2969
2970     if ($type ne "build") {
2971         if ($failed && $bisect_skip) {
2972             $in_bisect = 0;
2973             return -1;
2974         }
2975         dodie "Failed on build" if $failed;
2976
2977         # Now boot the box
2978         start_monitor_and_install or $failed = 1;
2979
2980         if ($type ne "boot") {
2981             if ($failed && $bisect_skip) {
2982                 end_monitor;
2983                 bisect_reboot;
2984                 $in_bisect = 0;
2985                 return -1;
2986             }
2987             dodie "Failed on boot" if $failed;
2988
2989             do_run_test or $failed = 1;
2990         }
2991         end_monitor;
2992     }
2993
2994     if ($failed) {
2995         $result = 0;
2996     } else {
2997         $result = 1;
2998     }
2999
3000     # reboot the box to a kernel we can ssh to
3001     if ($type ne "build") {
3002         bisect_reboot;
3003     }
3004     $in_bisect = 0;
3005
3006     return $result;
3007 }
3008
3009 sub run_bisect {
3010     my ($type) = @_;
3011     my $buildtype = "oldconfig";
3012
3013     # We should have a minconfig to use?
3014     if (defined($minconfig)) {
3015         $buildtype = "useconfig:$minconfig";
3016     }
3017
3018     # If the user sets bisect_tries to less than 1, then no tries
3019     # is a success.
3020     my $ret = 1;
3021
3022     # Still let the user manually decide that though.
3023     if ($bisect_tries < 1 && $bisect_manual) {
3024         $ret = answer_bisect;
3025     }
3026
3027     for (my $i = 0; $i < $bisect_tries; $i++) {
3028         if ($bisect_tries > 1) {
3029             my $t = $i + 1;
3030             doprint("Running bisect trial $t of $bisect_tries:\n");
3031         }
3032         $ret = run_bisect_test $type, $buildtype;
3033
3034         if ($bisect_manual) {
3035             $ret = answer_bisect;
3036         }
3037
3038         last if (!$ret);
3039     }
3040
3041     # Are we looking for where it worked, not failed?
3042     if ($reverse_bisect && $ret >= 0) {
3043         $ret = !$ret;
3044     }
3045
3046     if ($ret > 0) {
3047         return "good";
3048     } elsif ($ret == 0) {
3049         return  "bad";
3050     } elsif ($bisect_skip) {
3051         doprint "HIT A BAD COMMIT ... SKIPPING\n";
3052         return "skip";
3053     }
3054 }
3055
3056 sub update_bisect_replay {
3057     my $tmp_log = "$tmpdir/ktest_bisect_log";
3058     run_command "git bisect log > $tmp_log" or
3059         dodie "can't create bisect log";
3060     return $tmp_log;
3061 }
3062
3063 sub bisect {
3064     my ($i) = @_;
3065
3066     my $result;
3067
3068     dodie "BISECT_GOOD[$i] not defined\n"       if (!defined($bisect_good));
3069     dodie "BISECT_BAD[$i] not defined\n"        if (!defined($bisect_bad));
3070     dodie "BISECT_TYPE[$i] not defined\n"       if (!defined($bisect_type));
3071
3072     my $good = $bisect_good;
3073     my $bad = $bisect_bad;
3074     my $type = $bisect_type;
3075     my $start = $bisect_start;
3076     my $replay = $bisect_replay;
3077     my $start_files = $bisect_files;
3078
3079     if (defined($start_files)) {
3080         $start_files = " -- " . $start_files;
3081     } else {
3082         $start_files = "";
3083     }
3084
3085     # convert to true sha1's
3086     $good = get_sha1($good);
3087     $bad = get_sha1($bad);
3088
3089     if (defined($bisect_reverse) && $bisect_reverse == 1) {
3090         doprint "Performing a reverse bisect (bad is good, good is bad!)\n";
3091         $reverse_bisect = 1;
3092     } else {
3093         $reverse_bisect = 0;
3094     }
3095
3096     # Can't have a test without having a test to run
3097     if ($type eq "test" && !defined($run_test)) {
3098         $type = "boot";
3099     }
3100
3101     # Check if a bisect was running
3102     my $bisect_start_file = "$builddir/.git/BISECT_START";
3103
3104     my $check = $bisect_check;
3105     my $do_check = defined($check) && $check ne "0";
3106
3107     if ( -f $bisect_start_file ) {
3108         print "Bisect in progress found\n";
3109         if ($do_check) {
3110             print " If you say yes, then no checks of good or bad will be done\n";
3111         }
3112         if (defined($replay)) {
3113             print "** BISECT_REPLAY is defined in config file **";
3114             print " Ignore config option and perform new git bisect log?\n";
3115             if (read_ync " (yes, no, or cancel) ") {
3116                 $replay = update_bisect_replay;
3117                 $do_check = 0;
3118             }
3119         } elsif (read_yn "read git log and continue?") {
3120             $replay = update_bisect_replay;
3121             $do_check = 0;
3122         }
3123     }
3124
3125     if ($do_check) {
3126         # get current HEAD
3127         my $head = get_sha1("HEAD");
3128
3129         if ($check ne "good") {
3130             doprint "TESTING BISECT BAD [$bad]\n";
3131             run_command "git checkout $bad" or
3132                 dodie "Failed to checkout $bad";
3133
3134             $result = run_bisect $type;
3135
3136             if ($result ne "bad") {
3137                 fail "Tested BISECT_BAD [$bad] and it succeeded" and return 0;
3138             }
3139         }
3140
3141         if ($check ne "bad") {
3142             doprint "TESTING BISECT GOOD [$good]\n";
3143             run_command "git checkout $good" or
3144                 dodie "Failed to checkout $good";
3145
3146             $result = run_bisect $type;
3147
3148             if ($result ne "good") {
3149                 fail "Tested BISECT_GOOD [$good] and it failed" and return 0;
3150             }
3151         }
3152
3153         # checkout where we started
3154         run_command "git checkout $head" or
3155             dodie "Failed to checkout $head";
3156     }
3157
3158     run_command "git bisect start$start_files" or
3159         dodie "could not start bisect";
3160
3161     if (defined($replay)) {
3162         run_command "git bisect replay $replay" or
3163             dodie "failed to run replay";
3164     } else {
3165         run_command "git bisect good $good" or
3166             dodie "could not set bisect good to $good";
3167
3168         run_git_bisect "git bisect bad $bad" or
3169             dodie "could not set bisect bad to $bad";
3170     }
3171
3172     if (defined($start)) {
3173         run_command "git checkout $start" or
3174             dodie "failed to checkout $start";
3175     }
3176
3177     my $test;
3178     do {
3179         $result = run_bisect $type;
3180         $test = run_git_bisect "git bisect $result";
3181         print_times;
3182     } while ($test);
3183
3184     run_command "git bisect log" or
3185         dodie "could not capture git bisect log";
3186
3187     run_command "git bisect reset" or
3188         dodie "could not reset git bisect";
3189
3190     doprint "Bad commit was [$bisect_bad_commit]\n";
3191
3192     success $i;
3193 }
3194
3195 sub assign_configs {
3196     my ($hash, $config) = @_;
3197
3198     doprint "Reading configs from $config\n";
3199
3200     open (IN, $config) or
3201         dodie "Failed to read $config";
3202
3203     while (<IN>) {
3204         chomp;
3205         if (/^((CONFIG\S*)=.*)/) {
3206             ${$hash}{$2} = $1;
3207         } elsif (/^(# (CONFIG\S*) is not set)/) {
3208             ${$hash}{$2} = $1;
3209         }
3210     }
3211
3212     close(IN);
3213 }
3214
3215 sub process_config_ignore {
3216     my ($config) = @_;
3217
3218     assign_configs \%config_ignore, $config;
3219 }
3220
3221 sub get_dependencies {
3222     my ($config) = @_;
3223
3224     my $arr = $dependency{$config};
3225     if (!defined($arr)) {
3226         return ();
3227     }
3228
3229     my @deps = @{$arr};
3230
3231     foreach my $dep (@{$arr}) {
3232         print "ADD DEP $dep\n";
3233         @deps = (@deps, get_dependencies $dep);
3234     }
3235
3236     return @deps;
3237 }
3238
3239 sub save_config {
3240     my ($pc, $file) = @_;
3241
3242     my %configs = %{$pc};
3243
3244     doprint "Saving configs into $file\n";
3245
3246     open(OUT, ">$file") or dodie "Can not write to $file";
3247
3248     foreach my $config (keys %configs) {
3249         print OUT "$configs{$config}\n";
3250     }
3251     close(OUT);
3252 }
3253
3254 sub create_config {
3255     my ($name, $pc) = @_;
3256
3257     doprint "Creating old config from $name configs\n";
3258
3259     save_config $pc, $output_config;
3260
3261     make_oldconfig;
3262 }
3263
3264 sub run_config_bisect_test {
3265     my ($type) = @_;
3266
3267     my $ret = run_bisect_test $type, "oldconfig";
3268
3269     if ($bisect_manual) {
3270         $ret = answer_bisect;
3271     }
3272
3273     return $ret;
3274 }
3275
3276 sub config_bisect_end {
3277     my ($good, $bad) = @_;
3278     my $diffexec = "diff -u";
3279
3280     if (-f "$builddir/scripts/diffconfig") {
3281         $diffexec = "$builddir/scripts/diffconfig";
3282     }
3283     doprint "\n\n***************************************\n";
3284     doprint "No more config bisecting possible.\n";
3285     run_command "$diffexec $good $bad", 1;
3286     doprint "***************************************\n\n";
3287 }
3288
3289 sub run_config_bisect {
3290     my ($good, $bad, $last_result) = @_;
3291     my $reset = "";
3292     my $cmd;
3293     my $ret;
3294
3295     if (!length($last_result)) {
3296         $reset = "-r";
3297     }
3298     run_command "$config_bisect_exec $reset -b $outputdir $good $bad $last_result", 1;
3299
3300     # config-bisect returns:
3301     #   0 if there is more to bisect
3302     #   1 for finding a good config
3303     #   2 if it can not find any more configs
3304     #  -1 (255) on error
3305     if ($run_command_status) {
3306         return $run_command_status;
3307     }
3308
3309     $ret = run_config_bisect_test $config_bisect_type;
3310     if ($ret) {
3311         doprint "NEW GOOD CONFIG ($pass)\n";
3312         system("cp $output_config $tmpdir/good_config.tmp.$pass");
3313         $pass++;
3314         # Return 3 for good config
3315         return 3;
3316     } else {
3317         doprint "NEW BAD CONFIG ($pass)\n";
3318         system("cp $output_config $tmpdir/bad_config.tmp.$pass");
3319         $pass++;
3320         # Return 4 for bad config
3321         return 4;
3322     }
3323 }
3324
3325 sub config_bisect {
3326     my ($i) = @_;
3327
3328     my $good_config;
3329     my $bad_config;
3330
3331     my $type = $config_bisect_type;
3332     my $ret;
3333
3334     $bad_config = $config_bisect;
3335
3336     if (defined($config_bisect_good)) {
3337         $good_config = $config_bisect_good;
3338     } elsif (defined($minconfig)) {
3339         $good_config = $minconfig;
3340     } else {
3341         doprint "No config specified, checking if defconfig works";
3342         $ret = run_bisect_test $type, "defconfig";
3343         if (!$ret) {
3344             fail "Have no good config to compare with, please set CONFIG_BISECT_GOOD";
3345             return 1;
3346         }
3347         $good_config = $output_config;
3348     }
3349
3350     if (!defined($config_bisect_exec)) {
3351         # First check the location that ktest.pl ran
3352         my @locations = (
3353                 "$pwd/config-bisect.pl",
3354                 "$dirname/config-bisect.pl",
3355                 "$builddir/tools/testing/ktest/config-bisect.pl",
3356                 undef );
3357         foreach my $loc (@locations) {
3358             doprint "loc = $loc\n";
3359             $config_bisect_exec = $loc;
3360             last if (defined($config_bisect_exec && -x $config_bisect_exec));
3361         }
3362         if (!defined($config_bisect_exec)) {
3363             fail "Could not find an executable config-bisect.pl\n",
3364                 "  Set CONFIG_BISECT_EXEC to point to config-bisect.pl";
3365             return 1;
3366         }
3367     }
3368
3369     # we don't want min configs to cause issues here.
3370     doprint "Disabling 'MIN_CONFIG' for this test\n";
3371     undef $minconfig;
3372
3373     my %good_configs;
3374     my %bad_configs;
3375     my %tmp_configs;
3376
3377     if (-f "$tmpdir/good_config.tmp" || -f "$tmpdir/bad_config.tmp") {
3378         if (read_yn "Interrupted config-bisect. Continue (n - will start new)?") {
3379             if (-f "$tmpdir/good_config.tmp") {
3380                 $good_config = "$tmpdir/good_config.tmp";
3381             } else {
3382                 $good_config = "$tmpdir/good_config";
3383             }
3384             if (-f "$tmpdir/bad_config.tmp") {
3385                 $bad_config = "$tmpdir/bad_config.tmp";
3386             } else {
3387                 $bad_config = "$tmpdir/bad_config";
3388             }
3389         }
3390     }
3391     doprint "Run good configs through make oldconfig\n";
3392     assign_configs \%tmp_configs, $good_config;
3393     create_config "$good_config", \%tmp_configs;
3394     $good_config = "$tmpdir/good_config";
3395     system("cp $output_config $good_config") == 0 or dodie "cp good config";
3396
3397     doprint "Run bad configs through make oldconfig\n";
3398     assign_configs \%tmp_configs, $bad_config;
3399     create_config "$bad_config", \%tmp_configs;
3400     $bad_config = "$tmpdir/bad_config";
3401     system("cp $output_config $bad_config") == 0 or dodie "cp bad config";
3402
3403     if (defined($config_bisect_check) && $config_bisect_check ne "0") {
3404         if ($config_bisect_check ne "good") {
3405             doprint "Testing bad config\n";
3406
3407             $ret = run_bisect_test $type, "useconfig:$bad_config";
3408             if ($ret) {
3409                 fail "Bad config succeeded when expected to fail!";
3410                 return 0;
3411             }
3412         }
3413         if ($config_bisect_check ne "bad") {
3414             doprint "Testing good config\n";
3415
3416             $ret = run_bisect_test $type, "useconfig:$good_config";
3417             if (!$ret) {
3418                 fail "Good config failed when expected to succeed!";
3419                 return 0;
3420             }
3421         }
3422     }
3423
3424     my $last_run = "";
3425
3426     do {
3427         $ret = run_config_bisect $good_config, $bad_config, $last_run;
3428         if ($ret == 3) {
3429             $last_run = "good";
3430         } elsif ($ret == 4) {
3431             $last_run = "bad";
3432         }
3433         print_times;
3434     } while ($ret == 3 || $ret == 4);
3435
3436     if ($ret == 2) {
3437         config_bisect_end "$good_config.tmp", "$bad_config.tmp";
3438     }
3439
3440     return $ret if ($ret < 0);
3441
3442     success $i;
3443 }
3444
3445 sub patchcheck_reboot {
3446     doprint "Reboot and sleep $patchcheck_sleep_time seconds\n";
3447     reboot_to_good $patchcheck_sleep_time;
3448 }
3449
3450 sub patchcheck {
3451     my ($i) = @_;
3452
3453     dodie "PATCHCHECK_START[$i] not defined\n"
3454         if (!defined($patchcheck_start));
3455     dodie "PATCHCHECK_TYPE[$i] not defined\n"
3456         if (!defined($patchcheck_type));
3457
3458     my $start = $patchcheck_start;
3459
3460     my $cherry = $patchcheck_cherry;
3461     if (!defined($cherry)) {
3462         $cherry = 0;
3463     }
3464
3465     my $end = "HEAD";
3466     if (defined($patchcheck_end)) {
3467         $end = $patchcheck_end;
3468     } elsif ($cherry) {
3469         dodie "PATCHCHECK_END must be defined with PATCHCHECK_CHERRY\n";
3470     }
3471
3472     # Get the true sha1's since we can use things like HEAD~3
3473     $start = get_sha1($start);
3474     $end = get_sha1($end);
3475
3476     my $type = $patchcheck_type;
3477
3478     # Can't have a test without having a test to run
3479     if ($type eq "test" && !defined($run_test)) {
3480         $type = "boot";
3481     }
3482
3483     if ($cherry) {
3484         open (IN, "git cherry -v $start $end|") or
3485             dodie "could not get git list";
3486     } else {
3487         open (IN, "git log --pretty=oneline $end|") or
3488             dodie "could not get git list";
3489     }
3490
3491     my @list;
3492
3493     while (<IN>) {
3494         chomp;
3495         # git cherry adds a '+' we want to remove
3496         s/^\+ //;
3497         $list[$#list+1] = $_;
3498         last if (/^$start/);
3499     }
3500     close(IN);
3501
3502     if (!$cherry) {
3503         if ($list[$#list] !~ /^$start/) {
3504             fail "SHA1 $start not found";
3505         }
3506
3507         # go backwards in the list
3508         @list = reverse @list;
3509     }
3510
3511     doprint("Going to test the following commits:\n");
3512     foreach my $l (@list) {
3513         doprint "$l\n";
3514     }
3515
3516     my $save_clean = $noclean;
3517     my %ignored_warnings;
3518
3519     if (defined($ignore_warnings)) {
3520         foreach my $sha1 (split /\s+/, $ignore_warnings) {
3521             $ignored_warnings{$sha1} = 1;
3522         }
3523     }
3524
3525     $in_patchcheck = 1;
3526     foreach my $item (@list) {
3527         my $sha1 = $item;
3528         $sha1 =~ s/^([[:xdigit:]]+).*/$1/;
3529
3530         doprint "\nProcessing commit \"$item\"\n\n";
3531
3532         run_command "git checkout $sha1" or
3533             dodie "Failed to checkout $sha1";
3534
3535         # only clean on the first and last patch
3536         if ($item eq $list[0] ||
3537             $item eq $list[$#list]) {
3538             $noclean = $save_clean;
3539         } else {
3540             $noclean = 1;
3541         }
3542
3543         if (defined($minconfig)) {
3544             build "useconfig:$minconfig" or return 0;
3545         } else {
3546             # ?? no config to use?
3547             build "oldconfig" or return 0;
3548         }
3549
3550         # No need to do per patch checking if warnings file exists
3551         if (!defined($warnings_file) && !defined($ignored_warnings{$sha1})) {
3552             check_patch_buildlog $sha1 or return 0;
3553         }
3554
3555         check_buildlog or return 0;
3556
3557         next if ($type eq "build");
3558
3559         my $failed = 0;
3560
3561         start_monitor_and_install or $failed = 1;
3562
3563         if (!$failed && $type ne "boot"){
3564             do_run_test or $failed = 1;
3565         }
3566         end_monitor;
3567         if ($failed) {
3568             print_times;
3569             return 0;
3570         }
3571         patchcheck_reboot;
3572         print_times;
3573     }
3574     $in_patchcheck = 0;
3575     success $i;
3576
3577     return 1;
3578 }
3579
3580 sub add_dep {
3581     # $config depends on $dep
3582     my ($config, $dep) = @_;
3583
3584     if (defined($depends{$config})) {
3585         $depends{$config} .= " " . $dep;
3586     } else {
3587         $depends{$config} = $dep;
3588     }
3589
3590     # record the number of configs depending on $dep
3591     if (defined $depcount{$dep}) {
3592         $depcount{$dep}++;
3593     } else {
3594         $depcount{$dep} = 1;
3595     }
3596 }
3597
3598 # taken from streamline_config.pl
3599 sub read_kconfig {
3600     my ($kconfig) = @_;
3601
3602     my $state = "NONE";
3603     my $config;
3604     my @kconfigs;
3605
3606     my $cont = 0;
3607     my $line;
3608
3609     if (! -f $kconfig) {
3610         doprint "file $kconfig does not exist, skipping\n";
3611         return;
3612     }
3613
3614     open(KIN, "$kconfig")
3615         or dodie "Can't open $kconfig";
3616     while (<KIN>) {
3617         chomp;
3618
3619         # Make sure that lines ending with \ continue
3620         if ($cont) {
3621             $_ = $line . " " . $_;
3622         }
3623
3624         if (s/\\$//) {
3625             $cont = 1;
3626             $line = $_;
3627             next;
3628         }
3629
3630         $cont = 0;
3631
3632         # collect any Kconfig sources
3633         if (/^source\s*"(.*)"/) {
3634             $kconfigs[$#kconfigs+1] = $1;
3635         }
3636
3637         # configs found
3638         if (/^\s*(menu)?config\s+(\S+)\s*$/) {
3639             $state = "NEW";
3640             $config = $2;
3641
3642             for (my $i = 0; $i < $iflevel; $i++) {
3643                 add_dep $config, $ifdeps[$i];
3644             }
3645
3646         # collect the depends for the config
3647         } elsif ($state eq "NEW" && /^\s*depends\s+on\s+(.*)$/) {
3648
3649             add_dep $config, $1;
3650
3651         # Get the configs that select this config
3652         } elsif ($state eq "NEW" && /^\s*select\s+(\S+)/) {
3653
3654             # selected by depends on config
3655             add_dep $1, $config;
3656
3657         # Check for if statements
3658         } elsif (/^if\s+(.*\S)\s*$/) {
3659             my $deps = $1;
3660             # remove beginning and ending non text
3661             $deps =~ s/^[^a-zA-Z0-9_]*//;
3662             $deps =~ s/[^a-zA-Z0-9_]*$//;
3663
3664             my @deps = split /[^a-zA-Z0-9_]+/, $deps;
3665
3666             $ifdeps[$iflevel++] = join ':', @deps;
3667
3668         } elsif (/^endif/) {
3669
3670             $iflevel-- if ($iflevel);
3671
3672         # stop on "help"
3673         } elsif (/^\s*help\s*$/) {
3674             $state = "NONE";
3675         }
3676     }
3677     close(KIN);
3678
3679     # read in any configs that were found.
3680     foreach $kconfig (@kconfigs) {
3681         if (!defined($read_kconfigs{$kconfig})) {
3682             $read_kconfigs{$kconfig} = 1;
3683             read_kconfig("$builddir/$kconfig");
3684         }
3685     }
3686 }
3687
3688 sub read_depends {
3689     # find out which arch this is by the kconfig file
3690     open (IN, $output_config) or
3691         dodie "Failed to read $output_config";
3692     my $arch;
3693     while (<IN>) {
3694         if (m,Linux/(\S+)\s+\S+\s+Kernel Configuration,) {
3695             $arch = $1;
3696             last;
3697         }
3698     }
3699     close IN;
3700
3701     if (!defined($arch)) {
3702         doprint "Could not find arch from config file\n";
3703         doprint "no dependencies used\n";
3704         return;
3705     }
3706
3707     # arch is really the subarch, we need to know
3708     # what directory to look at.
3709     if ($arch eq "i386" || $arch eq "x86_64") {
3710         $arch = "x86";
3711     }
3712
3713     my $kconfig = "$builddir/arch/$arch/Kconfig";
3714
3715     if (! -f $kconfig && $arch =~ /\d$/) {
3716         my $orig = $arch;
3717         # some subarchs have numbers, truncate them
3718         $arch =~ s/\d*$//;
3719         $kconfig = "$builddir/arch/$arch/Kconfig";
3720         if (! -f $kconfig) {
3721             doprint "No idea what arch dir $orig is for\n";
3722             doprint "no dependencies used\n";
3723             return;
3724         }
3725     }
3726
3727     read_kconfig($kconfig);
3728 }
3729
3730 sub make_new_config {
3731     my @configs = @_;
3732
3733     open (OUT, ">$output_config")
3734         or dodie "Failed to write $output_config";
3735
3736     foreach my $config (@configs) {
3737         print OUT "$config\n";
3738     }
3739     close OUT;
3740 }
3741
3742 sub chomp_config {
3743     my ($config) = @_;
3744
3745     $config =~ s/CONFIG_//;
3746
3747     return $config;
3748 }
3749
3750 sub get_depends {
3751     my ($dep) = @_;
3752
3753     my $kconfig = chomp_config $dep;
3754
3755     $dep = $depends{"$kconfig"};
3756
3757     # the dep string we have saves the dependencies as they
3758     # were found, including expressions like ! && ||. We
3759     # want to split this out into just an array of configs.
3760
3761     my $valid = "A-Za-z_0-9";
3762
3763     my @configs;
3764
3765     while ($dep =~ /[$valid]/) {
3766         if ($dep =~ /^[^$valid]*([$valid]+)/) {
3767             my $conf = "CONFIG_" . $1;
3768
3769             $configs[$#configs + 1] = $conf;
3770
3771             $dep =~ s/^[^$valid]*[$valid]+//;
3772         } else {
3773             dodie "this should never happen";
3774         }
3775     }
3776
3777     return @configs;
3778 }
3779
3780 sub test_this_config {
3781     my ($config) = @_;
3782
3783     my $found;
3784
3785     # if we already processed this config, skip it
3786     if (defined($processed_configs{$config})) {
3787         return undef;
3788     }
3789     $processed_configs{$config} = 1;
3790
3791     # if this config failed during this round, skip it
3792     if (defined($nochange_config{$config})) {
3793         return undef;
3794     }
3795
3796     my $kconfig = chomp_config $config;
3797
3798     # Test dependencies first
3799     if (defined($depends{"$kconfig"})) {
3800         my @parents = get_depends $config;
3801         foreach my $parent (@parents) {
3802             # if the parent is in the min config, check it first
3803             next if (!defined($min_configs{$parent}));
3804             $found = test_this_config($parent);
3805             if (defined($found)) {
3806                 return $found;
3807             }
3808         }
3809     }
3810
3811     # Remove this config from the list of configs
3812     # do a make olddefconfig and then read the resulting
3813     # .config to make sure it is missing the config that
3814     # we had before
3815     my %configs = %min_configs;
3816     $configs{$config} = "# $config is not set";
3817     make_new_config ((values %configs), (values %keep_configs));
3818     make_oldconfig;
3819     delete $configs{$config};
3820     undef %configs;
3821     assign_configs \%configs, $output_config;
3822
3823     if (!defined($configs{$config}) || $configs{$config} =~ /^#/) {
3824         return $config;
3825     }
3826
3827     doprint "disabling config $config did not change .config\n";
3828
3829     $nochange_config{$config} = 1;
3830
3831     return undef;
3832 }
3833
3834 sub make_min_config {
3835     my ($i) = @_;
3836
3837     my $type = $minconfig_type;
3838     if ($type ne "boot" && $type ne "test") {
3839         fail "Invalid MIN_CONFIG_TYPE '$minconfig_type'\n" .
3840             " make_min_config works only with 'boot' and 'test'\n" and return;
3841     }
3842
3843     if (!defined($output_minconfig)) {
3844         fail "OUTPUT_MIN_CONFIG not defined" and return;
3845     }
3846
3847     # If output_minconfig exists, and the start_minconfig
3848     # came from min_config, than ask if we should use
3849     # that instead.
3850     if (-f $output_minconfig && !$start_minconfig_defined) {
3851         print "$output_minconfig exists\n";
3852         if (!defined($use_output_minconfig)) {
3853             if (read_yn " Use it as minconfig?") {
3854                 $start_minconfig = $output_minconfig;
3855             }
3856         } elsif ($use_output_minconfig > 0) {
3857             doprint "Using $output_minconfig as MIN_CONFIG\n";
3858             $start_minconfig = $output_minconfig;
3859         } else {
3860             doprint "Set to still use MIN_CONFIG as starting point\n";
3861         }
3862     }
3863
3864     if (!defined($start_minconfig)) {
3865         fail "START_MIN_CONFIG or MIN_CONFIG not defined" and return;
3866     }
3867
3868     my $temp_config = "$tmpdir/temp_config";
3869
3870     # First things first. We build an allnoconfig to find
3871     # out what the defaults are that we can't touch.
3872     # Some are selections, but we really can't handle selections.
3873
3874     my $save_minconfig = $minconfig;
3875     undef $minconfig;
3876
3877     run_command "$make allnoconfig" or return 0;
3878
3879     read_depends;
3880
3881     process_config_ignore $output_config;
3882
3883     undef %save_configs;
3884     undef %min_configs;
3885
3886     if (defined($ignore_config)) {
3887         # make sure the file exists
3888         `touch $ignore_config`;
3889         assign_configs \%save_configs, $ignore_config;
3890     }
3891
3892     %keep_configs = %save_configs;
3893
3894     doprint "Load initial configs from $start_minconfig\n";
3895
3896     # Look at the current min configs, and save off all the
3897     # ones that were set via the allnoconfig
3898     assign_configs \%min_configs, $start_minconfig;
3899
3900     my @config_keys = keys %min_configs;
3901
3902     # All configs need a depcount
3903     foreach my $config (@config_keys) {
3904         my $kconfig = chomp_config $config;
3905         if (!defined $depcount{$kconfig}) {
3906             $depcount{$kconfig} = 0;
3907         }
3908     }
3909
3910     # Remove anything that was set by the make allnoconfig
3911     # we shouldn't need them as they get set for us anyway.
3912     foreach my $config (@config_keys) {
3913         # Remove anything in the ignore_config
3914         if (defined($keep_configs{$config})) {
3915             my $file = $ignore_config;
3916             $file =~ s,.*/(.*?)$,$1,;
3917             doprint "$config set by $file ... ignored\n";
3918             delete $min_configs{$config};
3919             next;
3920         }
3921         # But make sure the settings are the same. If a min config
3922         # sets a selection, we do not want to get rid of it if
3923         # it is not the same as what we have. Just move it into
3924         # the keep configs.
3925         if (defined($config_ignore{$config})) {
3926             if ($config_ignore{$config} ne $min_configs{$config}) {
3927                 doprint "$config is in allnoconfig as '$config_ignore{$config}'";
3928                 doprint " but it is '$min_configs{$config}' in minconfig .. keeping\n";
3929                 $keep_configs{$config} = $min_configs{$config};
3930             } else {
3931                 doprint "$config set by allnoconfig ... ignored\n";
3932             }
3933             delete $min_configs{$config};
3934         }
3935     }
3936
3937     my $done = 0;
3938     my $take_two = 0;
3939
3940     while (!$done) {
3941         my $config;
3942         my $found;
3943
3944         # Now disable each config one by one and do a make oldconfig
3945         # till we find a config that changes our list.
3946
3947         my @test_configs = keys %min_configs;
3948
3949         # Sort keys by who is most dependent on
3950         @test_configs = sort  { $depcount{chomp_config($b)} <=> $depcount{chomp_config($a)} }
3951             @test_configs ;
3952
3953         # Put configs that did not modify the config at the end.
3954         my $reset = 1;
3955         for (my $i = 0; $i < $#test_configs; $i++) {
3956             if (!defined($nochange_config{$test_configs[0]})) {
3957                 $reset = 0;
3958                 last;
3959             }
3960             # This config didn't change the .config last time.
3961             # Place it at the end
3962             my $config = shift @test_configs;
3963             push @test_configs, $config;
3964         }
3965
3966         # if every test config has failed to modify the .config file
3967         # in the past, then reset and start over.
3968         if ($reset) {
3969             undef %nochange_config;
3970         }
3971
3972         undef %processed_configs;
3973
3974         foreach my $config (@test_configs) {
3975
3976             $found = test_this_config $config;
3977
3978             last if (defined($found));
3979
3980             # oh well, try another config
3981         }
3982
3983         if (!defined($found)) {
3984             # we could have failed due to the nochange_config hash
3985             # reset and try again
3986             if (!$take_two) {
3987                 undef %nochange_config;
3988                 $take_two = 1;
3989                 next;
3990             }
3991             doprint "No more configs found that we can disable\n";
3992             $done = 1;
3993             last;
3994         }
3995         $take_two = 0;
3996
3997         $config = $found;
3998
3999         doprint "Test with $config disabled\n";
4000
4001         # set in_bisect to keep build and monitor from dieing
4002         $in_bisect = 1;
4003
4004         my $failed = 0;
4005         build "oldconfig" or $failed = 1;
4006         if (!$failed) {
4007             start_monitor_and_install or $failed = 1;
4008
4009             if ($type eq "test" && !$failed) {
4010                 do_run_test or $failed = 1;
4011             }
4012
4013             end_monitor;
4014         }
4015
4016         $in_bisect = 0;
4017
4018         if ($failed) {
4019             doprint "$min_configs{$config} is needed to boot the box... keeping\n";
4020             # this config is needed, add it to the ignore list.
4021             $keep_configs{$config} = $min_configs{$config};
4022             $save_configs{$config} = $min_configs{$config};
4023             delete $min_configs{$config};
4024
4025             # update new ignore configs
4026             if (defined($ignore_config)) {
4027                 open (OUT, ">$temp_config") or
4028                     dodie "Can't write to $temp_config";
4029                 foreach my $config (keys %save_configs) {
4030                     print OUT "$save_configs{$config}\n";
4031                 }
4032                 close OUT;
4033                 run_command "mv $temp_config $ignore_config" or
4034                     dodie "failed to copy update to $ignore_config";
4035             }
4036
4037         } else {
4038             # We booted without this config, remove it from the minconfigs.
4039             doprint "$config is not needed, disabling\n";
4040
4041             delete $min_configs{$config};
4042
4043             # Also disable anything that is not enabled in this config
4044             my %configs;
4045             assign_configs \%configs, $output_config;
4046             my @config_keys = keys %min_configs;
4047             foreach my $config (@config_keys) {
4048                 if (!defined($configs{$config})) {
4049                     doprint "$config is not set, disabling\n";
4050                     delete $min_configs{$config};
4051                 }
4052             }
4053
4054             # Save off all the current mandatory configs
4055             open (OUT, ">$temp_config") or
4056                 dodie "Can't write to $temp_config";
4057             foreach my $config (keys %keep_configs) {
4058                 print OUT "$keep_configs{$config}\n";
4059             }
4060             foreach my $config (keys %min_configs) {
4061                 print OUT "$min_configs{$config}\n";
4062             }
4063             close OUT;
4064
4065             run_command "mv $temp_config $output_minconfig" or
4066                 dodie "failed to copy update to $output_minconfig";
4067         }
4068
4069         doprint "Reboot and wait $sleep_time seconds\n";
4070         reboot_to_good $sleep_time;
4071     }
4072
4073     success $i;
4074     return 1;
4075 }
4076
4077 sub make_warnings_file {
4078     my ($i) = @_;
4079
4080     if (!defined($warnings_file)) {
4081         dodie "Must define WARNINGS_FILE for make_warnings_file test";
4082     }
4083
4084     if ($build_type eq "nobuild") {
4085         dodie "BUILD_TYPE can not be 'nobuild' for make_warnings_file test";
4086     }
4087
4088     build $build_type or dodie "Failed to build";
4089
4090     open(OUT, ">$warnings_file") or dodie "Can't create $warnings_file";
4091
4092     open(IN, $buildlog) or dodie "Can't open $buildlog";
4093     while (<IN>) {
4094         # Some compilers use UTF-8 extended for quotes
4095         # for distcc heterogeneous systems, this causes issues
4096         s/$utf8_quote/'/g;
4097
4098         if (/$check_build_re/) {
4099             print OUT;
4100         }
4101     }
4102     close(IN);
4103
4104     close(OUT);
4105
4106     success $i;
4107 }
4108
4109 sub option_defined {
4110     my ($option) = @_;
4111
4112     if (defined($opt{$option}) && $opt{$option} !~ /^\s*$/) {
4113         return 1;
4114     }
4115
4116     return 0;
4117 }
4118
4119 sub __set_test_option {
4120     my ($name, $i) = @_;
4121
4122     my $option = "$name\[$i\]";
4123
4124     if (option_defined($option)) {
4125         return $opt{$option};
4126     }
4127
4128     foreach my $test (keys %repeat_tests) {
4129         if ($i >= $test &&
4130             $i < $test + $repeat_tests{$test}) {
4131             $option = "$name\[$test\]";
4132             if (option_defined($option)) {
4133                 return $opt{$option};
4134             }
4135         }
4136     }
4137
4138     if (option_defined($name)) {
4139         return $opt{$name};
4140     }
4141
4142     return undef;
4143 }
4144
4145 sub set_test_option {
4146     my ($name, $i) = @_;
4147
4148     my $option = __set_test_option($name, $i);
4149     return $option if (!defined($option));
4150
4151     return eval_option($name, $option, $i);
4152 }
4153
4154 sub find_mailer {
4155     my ($mailer) = @_;
4156
4157     my @paths = split /:/, $ENV{PATH};
4158
4159     # sendmail is usually in /usr/sbin
4160     $paths[$#paths + 1] = "/usr/sbin";
4161
4162     foreach my $path (@paths) {
4163         if (-x "$path/$mailer") {
4164             return $path;
4165         }
4166     }
4167
4168     return undef;
4169 }
4170
4171 sub do_send_mail {
4172     my ($subject, $message, $file) = @_;
4173
4174     if (!defined($mail_path)) {
4175         # find the mailer
4176         $mail_path = find_mailer $mailer;
4177         if (!defined($mail_path)) {
4178             die "\nCan not find $mailer in PATH\n";
4179         }
4180     }
4181
4182     my $header_file = "$tmpdir/header";
4183     open (HEAD, ">$header_file") or die "Can not create $header_file\n";
4184     print HEAD "To: $mailto\n";
4185     print HEAD "Subject: $subject\n\n";
4186     print HEAD "$message\n";
4187     close HEAD;
4188
4189     if (!defined($mail_command)) {
4190         if ($mailer eq "mail" || $mailer eq "mailx") {
4191             $mail_command = "cat \$HEADER_FILE \$BODY_FILE | \$MAIL_PATH/\$MAILER -s \'\$SUBJECT\' \$MAILTO";
4192         } elsif ($mailer eq "sendmail" ) {
4193             $mail_command =  "cat \$HEADER_FILE \$BODY_FILE | \$MAIL_PATH/\$MAILER -t \$MAILTO";
4194         } else {
4195             die "\nYour mailer: $mailer is not supported.\n";
4196         }
4197     }
4198
4199     if (defined($file)) {
4200         $mail_command =~ s/\$BODY_FILE/$file/g;
4201     } else {
4202         $mail_command =~ s/\$BODY_FILE//g;
4203     }
4204
4205     $mail_command =~ s/\$HEADER_FILE/$header_file/g;
4206     $mail_command =~ s/\$MAILER/$mailer/g;
4207     $mail_command =~ s/\$MAIL_PATH/$mail_path/g;
4208     $mail_command =~ s/\$MAILTO/$mailto/g;
4209     $mail_command =~ s/\$SUBJECT/$subject/g;
4210     $mail_command =~ s/\$MESSAGE/$message/g;
4211
4212     my $ret = run_command $mail_command;
4213     if (!$ret && defined($file)) {
4214         # try again without the file
4215         $message .= "\n\n*** FAILED TO SEND LOG ***\n\n";
4216         do_send_email($subject, $message);
4217     }
4218 }
4219
4220 sub send_email {
4221     if (defined($mailto)) {
4222         if (!defined($mailer)) {
4223             doprint "No email sent: email or mailer not specified in config.\n";
4224             return;
4225         }
4226         do_send_mail @_;
4227     }
4228 }
4229
4230 sub cancel_test {
4231     if ($monitor_cnt) {
4232         end_monitor;
4233     }
4234     if ($email_when_canceled) {
4235         my $name = get_test_name;
4236         send_email("KTEST: Your [$name] test was cancelled",
4237             "Your test started at $script_start_time was cancelled: sig int");
4238     }
4239     die "\nCaught Sig Int, test interrupted: $!\n"
4240 }
4241
4242 $#ARGV < 1 or die "ktest.pl version: $VERSION\n   usage: ktest.pl [config-file]\n";
4243
4244 if ($#ARGV == 0) {
4245     $ktest_config = $ARGV[0];
4246     if (! -f $ktest_config) {
4247         print "$ktest_config does not exist.\n";
4248         if (!read_yn "Create it?") {
4249             exit 0;
4250         }
4251     }
4252 }
4253
4254 if (! -f $ktest_config) {
4255     $newconfig = 1;
4256     get_test_case;
4257     open(OUT, ">$ktest_config") or die "Can not create $ktest_config";
4258     print OUT << "EOF"
4259 # Generated by ktest.pl
4260 #
4261
4262 # PWD is a ktest.pl variable that will result in the process working
4263 # directory that ktest.pl is executed in.
4264
4265 # THIS_DIR is automatically assigned the PWD of the path that generated
4266 # the config file. It is best to use this variable when assigning other
4267 # directory paths within this directory. This allows you to easily
4268 # move the test cases to other locations or to other machines.
4269 #
4270 THIS_DIR := $variable{"PWD"}
4271
4272 # Define each test with TEST_START
4273 # The config options below it will override the defaults
4274 TEST_START
4275 TEST_TYPE = $default{"TEST_TYPE"}
4276
4277 DEFAULTS
4278 EOF
4279 ;
4280     close(OUT);
4281 }
4282 read_config $ktest_config;
4283
4284 if (defined($opt{"LOG_FILE"})) {
4285     $opt{"LOG_FILE"} = eval_option("LOG_FILE", $opt{"LOG_FILE"}, -1);
4286 }
4287
4288 # Append any configs entered in manually to the config file.
4289 my @new_configs = keys %entered_configs;
4290 if ($#new_configs >= 0) {
4291     print "\nAppending entered in configs to $ktest_config\n";
4292     open(OUT, ">>$ktest_config") or die "Can not append to $ktest_config";
4293     foreach my $config (@new_configs) {
4294         print OUT "$config = $entered_configs{$config}\n";
4295         $opt{$config} = process_variables($entered_configs{$config});
4296     }
4297 }
4298
4299 if (defined($opt{"LOG_FILE"})) {
4300     if ($opt{"CLEAR_LOG"}) {
4301         unlink $opt{"LOG_FILE"};
4302     }
4303     open(LOG, ">> $opt{LOG_FILE}") or die "Can't write to $opt{LOG_FILE}";
4304     LOG->autoflush(1);
4305 }
4306
4307 doprint "\n\nSTARTING AUTOMATED TESTS\n\n";
4308
4309 for (my $i = 0, my $repeat = 1; $i <= $opt{"NUM_TESTS"}; $i += $repeat) {
4310
4311     if (!$i) {
4312         doprint "DEFAULT OPTIONS:\n";
4313     } else {
4314         doprint "\nTEST $i OPTIONS";
4315         if (defined($repeat_tests{$i})) {
4316             $repeat = $repeat_tests{$i};
4317             doprint " ITERATE $repeat";
4318         }
4319         doprint "\n";
4320     }
4321
4322     foreach my $option (sort keys %opt) {
4323         if ($option =~ /\[(\d+)\]$/) {
4324             next if ($i != $1);
4325         } else {
4326             next if ($i);
4327         }
4328
4329         doprint "$option = $opt{$option}\n";
4330     }
4331 }
4332
4333 $SIG{INT} = qw(cancel_test);
4334
4335 # First we need to do is the builds
4336 for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) {
4337
4338     # Do not reboot on failing test options
4339     $no_reboot = 1;
4340     $reboot_success = 0;
4341
4342     $have_version = 0;
4343
4344     $iteration = $i;
4345
4346     $build_time = 0;
4347     $install_time = 0;
4348     $reboot_time = 0;
4349     $test_time = 0;
4350
4351     undef %force_config;
4352
4353     my $makecmd = set_test_option("MAKE_CMD", $i);
4354
4355     $outputdir = set_test_option("OUTPUT_DIR", $i);
4356     $builddir = set_test_option("BUILD_DIR", $i);
4357
4358     chdir $builddir || dodie "can't change directory to $builddir";
4359
4360     if (!-d $outputdir) {
4361         mkpath($outputdir) or
4362             dodie "can't create $outputdir";
4363     }
4364
4365     $make = "$makecmd O=$outputdir";
4366
4367     # Load all the options into their mapped variable names
4368     foreach my $opt (keys %option_map) {
4369         ${$option_map{$opt}} = set_test_option($opt, $i);
4370     }
4371
4372     $start_minconfig_defined = 1;
4373
4374     # The first test may override the PRE_KTEST option
4375     if ($i == 1) {
4376         if (defined($pre_ktest)) {
4377             doprint "\n";
4378             run_command $pre_ktest;
4379         }
4380         if ($email_when_started) {
4381             my $name = get_test_name;
4382             send_email("KTEST: Your [$name] test was started",
4383                 "Your test was started on $script_start_time");
4384         }
4385     }
4386
4387     # Any test can override the POST_KTEST option
4388     # The last test takes precedence.
4389     if (defined($post_ktest)) {
4390         $final_post_ktest = $post_ktest;
4391     }
4392
4393     if (!defined($start_minconfig)) {
4394         $start_minconfig_defined = 0;
4395         $start_minconfig = $minconfig;
4396     }
4397
4398     if (!-d $tmpdir) {
4399         mkpath($tmpdir) or
4400             dodie "can't create $tmpdir";
4401     }
4402
4403     $ENV{"SSH_USER"} = $ssh_user;
4404     $ENV{"MACHINE"} = $machine;
4405
4406     $buildlog = "$tmpdir/buildlog-$machine";
4407     $testlog = "$tmpdir/testlog-$machine";
4408     $dmesg = "$tmpdir/dmesg-$machine";
4409     $output_config = "$outputdir/.config";
4410
4411     if (!$buildonly) {
4412         $target = "$ssh_user\@$machine";
4413         if (($reboot_type eq "grub") or ($reboot_type eq "grub2bls")) {
4414             dodie "GRUB_MENU not defined" if (!defined($grub_menu));
4415         } elsif ($reboot_type eq "grub2") {
4416             dodie "GRUB_MENU not defined" if (!defined($grub_menu));
4417             dodie "GRUB_FILE not defined" if (!defined($grub_file));
4418         } elsif ($reboot_type eq "syslinux") {
4419             dodie "SYSLINUX_LABEL not defined" if (!defined($syslinux_label));
4420         }
4421     }
4422
4423     my $run_type = $build_type;
4424     if ($test_type eq "patchcheck") {
4425         $run_type = $patchcheck_type;
4426     } elsif ($test_type eq "bisect") {
4427         $run_type = $bisect_type;
4428     } elsif ($test_type eq "config_bisect") {
4429         $run_type = $config_bisect_type;
4430     } elsif ($test_type eq "make_min_config") {
4431         $run_type = "";
4432     } elsif ($test_type eq "make_warnings_file") {
4433         $run_type = "";
4434     }
4435
4436     # mistake in config file?
4437     if (!defined($run_type)) {
4438         $run_type = "ERROR";
4439     }
4440
4441     my $installme = "";
4442     $installme = " no_install" if ($no_install);
4443
4444     my $name = "";
4445
4446     if (defined($test_name)) {
4447         $name = " ($test_name)";
4448     }
4449
4450     doprint "\n\n";
4451
4452     if (defined($opt{"LOG_FILE"})) {
4453         $test_log_start = tell(LOG);
4454     }
4455
4456     doprint "RUNNING TEST $i of $opt{NUM_TESTS}$name with option $test_type $run_type$installme\n\n";
4457
4458     if (defined($pre_test)) {
4459         my $ret = run_command $pre_test;
4460         if (!$ret && defined($pre_test_die) &&
4461             $pre_test_die) {
4462                 dodie "failed to pre_test\n";
4463         }
4464     }
4465
4466     unlink $dmesg;
4467     unlink $buildlog;
4468     unlink $testlog;
4469
4470     if (defined($addconfig)) {
4471         my $min = $minconfig;
4472         if (!defined($minconfig)) {
4473             $min = "";
4474         }
4475         run_command "cat $addconfig $min > $tmpdir/add_config" or
4476             dodie "Failed to create temp config";
4477         $minconfig = "$tmpdir/add_config";
4478     }
4479
4480     if (defined($checkout)) {
4481         run_command "git checkout $checkout" or
4482             dodie "failed to checkout $checkout";
4483     }
4484
4485     $no_reboot = 0;
4486
4487     # A test may opt to not reboot the box
4488     if ($reboot_on_success) {
4489         $reboot_success = 1;
4490     }
4491
4492     if ($test_type eq "bisect") {
4493         bisect $i;
4494         next;
4495     } elsif ($test_type eq "config_bisect") {
4496         config_bisect $i;
4497         next;
4498     } elsif ($test_type eq "patchcheck") {
4499         patchcheck $i;
4500         next;
4501     } elsif ($test_type eq "make_min_config") {
4502         make_min_config $i;
4503         next;
4504     } elsif ($test_type eq "make_warnings_file") {
4505         $no_reboot = 1;
4506         make_warnings_file $i;
4507         next;
4508     }
4509
4510     if ($build_type ne "nobuild") {
4511         build $build_type or next;
4512         check_buildlog or next;
4513     }
4514
4515     if ($test_type eq "install") {
4516         get_version;
4517         install;
4518         success $i;
4519         next;
4520     }
4521
4522     if ($test_type ne "build") {
4523         my $failed = 0;
4524         start_monitor_and_install or $failed = 1;
4525
4526         if (!$failed && $test_type ne "boot" && defined($run_test)) {
4527             do_run_test or $failed = 1;
4528         }
4529         end_monitor;
4530         if ($failed) {
4531             print_times;
4532             next;
4533         }
4534     }
4535
4536     print_times;
4537
4538     success $i;
4539 }
4540
4541 if (defined($final_post_ktest)) {
4542
4543     my $cp_final_post_ktest = eval_kernel_version $final_post_ktest;
4544     run_command $cp_final_post_ktest;
4545 }
4546
4547 if ($opt{"POWEROFF_ON_SUCCESS"}) {
4548     halt;
4549 } elsif ($opt{"REBOOT_ON_SUCCESS"} && !do_not_reboot && $reboot_success) {
4550     reboot_to_good;
4551 } elsif (defined($switch_to_good)) {
4552     # still need to get to the good kernel
4553     run_command $switch_to_good;
4554 }
4555
4556 doprint "\n    $successes of $opt{NUM_TESTS} tests were successful\n\n";
4557
4558 if ($email_when_finished) {
4559     send_email("KTEST: Your test has finished!",
4560         "$successes of $opt{NUM_TESTS} tests started at $script_start_time were successful!");
4561 }
4562
4563 if (defined($opt{"LOG_FILE"})) {
4564     print "\n See $opt{LOG_FILE} for the record of results.\n\n";
4565     close LOG;
4566 }
4567
4568 exit 0;
4569
4570 ##
4571 # The following are here to standardize tabs/spaces/etc across the most likely editors
4572 ###
4573
4574 # Local Variables:
4575 # mode: perl
4576 # End:
4577 # vim: softtabstop=4