tools/testing/ktest/ktest.pl

   1 #!/usr/bin/perl -w
   2 #
   3 # Copyright 2010 - Steven Rostedt <srostedt@redhat.com>, Red Hat Inc.
   4 # Licensed under the terms of the GNU GPL License version 2
   5 #
   6
   7 use strict;
   8 use IPC::Open2;
   9 use Fcntl qw(F_GETFL F_SETFL O_NONBLOCK);
  10 use File::Path qw(mkpath);
  11 use File::Copy qw(cp);
  12 use FileHandle;
  13
  14 my $VERSION = "0.2";
  15
  16 $| = 1;
  17
  18 my %opt;
  19 my %repeat_tests;
  20 my %repeats;
  21 my %default;
  22
  23 #default opts
  24 $default{"NUM_TESTS"}           = 1;
  25 $default{"REBOOT_TYPE"}         = "grub";
  26 $default{"TEST_TYPE"}           = "test";
  27 $default{"BUILD_TYPE"}          = "randconfig";
  28 $default{"MAKE_CMD"}            = "make";
  29 $default{"TIMEOUT"}             = 120;
  30 $default{"TMP_DIR"}             = "/tmp/ktest";
  31 $default{"SLEEP_TIME"}          = 60;   # sleep time between tests
  32 $default{"BUILD_NOCLEAN"}       = 0;
  33 $default{"REBOOT_ON_ERROR"}     = 0;
  34 $default{"POWEROFF_ON_ERROR"}   = 0;
  35 $default{"REBOOT_ON_SUCCESS"}   = 1;
  36 $default{"POWEROFF_ON_SUCCESS"} = 0;
  37 $default{"BUILD_OPTIONS"}       = "";
  38 $default{"BISECT_SLEEP_TIME"}   = 60;   # sleep time between bisects
  39 $default{"CLEAR_LOG"}           = 0;
  40 $default{"SUCCESS_LINE"}        = "login:";
  41 $default{"BOOTED_TIMEOUT"}      = 1;
  42 $default{"DIE_ON_FAILURE"}      = 1;
  43 $default{"SSH_EXEC"}            = "ssh \$SSH_USER\@\$MACHINE \$SSH_COMMAND";
  44 $default{"SCP_TO_TARGET"}       = "scp \$SRC_FILE \$SSH_USER\@\$MACHINE:\$DST_FILE";
  45 $default{"REBOOT"}              = "ssh \$SSH_USER\@\$MACHINE reboot";
  46 $default{"STOP_AFTER_SUCCESS"}  = 10;
  47 $default{"STOP_AFTER_FAILURE"}  = 60;
  48 $default{"LOCALVERSION"}        = "-test";
  49
  50 my $ktest_config;
  51 my $version;
  52 my $machine;
  53 my $ssh_user;
  54 my $tmpdir;
  55 my $builddir;
  56 my $outputdir;
  57 my $output_config;
  58 my $test_type;
  59 my $build_type;
  60 my $build_options;
  61 my $reboot_type;
  62 my $reboot_script;
  63 my $power_cycle;
  64 my $reboot;
  65 my $reboot_on_error;
  66 my $poweroff_on_error;
  67 my $die_on_failure;
  68 my $powercycle_after_reboot;
  69 my $poweroff_after_halt;
  70 my $ssh_exec;
  71 my $scp_to_target;
  72 my $power_off;
  73 my $grub_menu;
  74 my $grub_number;
  75 my $target;
  76 my $make;
  77 my $post_install;
  78 my $noclean;
  79 my $minconfig;
  80 my $addconfig;
  81 my $in_bisect = 0;
  82 my $bisect_bad = "";
  83 my $reverse_bisect;
  84 my $in_patchcheck = 0;
  85 my $run_test;
  86 my $redirect;
  87 my $buildlog;
  88 my $dmesg;
  89 my $monitor_fp;
  90 my $monitor_pid;
  91 my $monitor_cnt = 0;
  92 my $sleep_time;
  93 my $bisect_sleep_time;
  94 my $store_failures;
  95 my $timeout;
  96 my $booted_timeout;
  97 my $console;
  98 my $success_line;
  99 my $stop_after_success;
 100 my $stop_after_failure;
 101 my $build_target;
 102 my $target_image;
 103 my $localversion;
 104 my $iteration = 0;
 105 my $successes = 0;
 106
 107 my %entered_configs;
 108 my %config_help;
 109
 110 $config_help{"MACHINE"} = << "EOF"
 111  The machine hostname that you will test.
 112 EOF
 113     ;
 114 $config_help{"SSH_USER"} = << "EOF"
 115  The box is expected to have ssh on normal bootup, provide the user
 116   (most likely root, since you need privileged operations)
 117 EOF
 118     ;
 119 $config_help{"BUILD_DIR"} = << "EOF"
 120  The directory that contains the Linux source code (full path).
 121 EOF
 122     ;
 123 $config_help{"OUTPUT_DIR"} = << "EOF"
 124  The directory that the objects will be built (full path).
 125  (can not be same as BUILD_DIR)
 126 EOF
 127     ;
 128 $config_help{"BUILD_TARGET"} = << "EOF"
 129  The location of the compiled file to copy to the target.
 130  (relative to OUTPUT_DIR)
 131 EOF
 132     ;
 133 $config_help{"TARGET_IMAGE"} = << "EOF"
 134  The place to put your image on the test machine.
 135 EOF
 136     ;
 137 $config_help{"POWER_CYCLE"} = << "EOF"
 138  A script or command to reboot the box.
 139
 140  Here is a digital loggers power switch example
 141  POWER_CYCLE = wget --no-proxy -O /dev/null -q  --auth-no-challenge 'http://admin:admin\@power/outlet?5=CCL'
 142
 143  Here is an example to reboot a virtual box on the current host
 144  with the name "Guest".
 145  POWER_CYCLE = virsh destroy Guest; sleep 5; virsh start Guest
 146 EOF
 147     ;
 148 $config_help{"CONSOLE"} = << "EOF"
 149  The script or command that reads the console
 150
 151   If you use ttywatch server, something like the following would work.
 152 CONSOLE = nc -d localhost 3001
 153
 154  For a virtual machine with guest name "Guest".
 155 CONSOLE =  virsh console Guest
 156 EOF
 157     ;
 158 $config_help{"LOCALVERSION"} = << "EOF"
 159  Required version ending to differentiate the test
 160  from other linux builds on the system.
 161 EOF
 162     ;
 163 $config_help{"REBOOT_TYPE"} = << "EOF"
 164  Way to reboot the box to the test kernel.
 165  Only valid options so far are "grub" and "script".
 166
 167  If you specify grub, it will assume grub version 1
 168  and will search in /boot/grub/menu.lst for the title \$GRUB_MENU
 169  and select that target to reboot to the kernel. If this is not
 170  your setup, then specify "script" and have a command or script
 171  specified in REBOOT_SCRIPT to boot to the target.
 172
 173  The entry in /boot/grub/menu.lst must be entered in manually.
 174  The test will not modify that file.
 175 EOF
 176     ;
 177 $config_help{"GRUB_MENU"} = << "EOF"
 178  The grub title name for the test kernel to boot
 179  (Only mandatory if REBOOT_TYPE = grub)
 180
 181  Note, ktest.pl will not update the grub menu.lst, you need to
 182  manually add an option for the test. ktest.pl will search
 183  the grub menu.lst for this option to find what kernel to
 184  reboot into.
 185
 186  For example, if in the /boot/grub/menu.lst the test kernel title has:
 187  title Test Kernel
 188  kernel vmlinuz-test
 189  GRUB_MENU = Test Kernel
 190 EOF
 191     ;
 192 $config_help{"REBOOT_SCRIPT"} = << "EOF"
 193  A script to reboot the target into the test kernel
 194  (Only mandatory if REBOOT_TYPE = script)
 195 EOF
 196     ;
 197
 198
 199 sub get_ktest_config {
 200     my ($config) = @_;
 201
 202     return if (defined($opt{$config}));
 203
 204     if (defined($config_help{$config})) {
 205         print "\n";
 206         print $config_help{$config};
 207     }
 208
 209     for (;;) {
 210         print "$config = ";
 211         if (defined($default{$config})) {
 212             print "\[$default{$config}\] ";
 213         }
 214         $entered_configs{$config} = <STDIN>;
 215         $entered_configs{$config} =~ s/^\s*(.*\S)\s*$/$1/;
 216         if ($entered_configs{$config} =~ /^\s*$/) {
 217             if ($default{$config}) {
 218                 $entered_configs{$config} = $default{$config};
 219             } else {
 220                 print "Your answer can not be blank\n";
 221                 next;
 222             }
 223         }
 224         last;
 225     }
 226 }
 227
 228 sub get_ktest_configs {
 229     get_ktest_config("MACHINE");
 230     get_ktest_config("SSH_USER");
 231     get_ktest_config("BUILD_DIR");
 232     get_ktest_config("OUTPUT_DIR");
 233     get_ktest_config("BUILD_TARGET");
 234     get_ktest_config("TARGET_IMAGE");
 235     get_ktest_config("POWER_CYCLE");
 236     get_ktest_config("CONSOLE");
 237     get_ktest_config("LOCALVERSION");
 238
 239     my $rtype = $opt{"REBOOT_TYPE"};
 240
 241     if (!defined($rtype)) {
 242         if (!defined($opt{"GRUB_MENU"})) {
 243             get_ktest_config("REBOOT_TYPE");
 244             $rtype = $entered_configs{"REBOOT_TYPE"};
 245         } else {
 246             $rtype = "grub";
 247         }
 248     }
 249
 250     if ($rtype eq "grub") {
 251         get_ktest_config("GRUB_MENU");
 252     } else {
 253         get_ktest_config("REBOOT_SCRIPT");
 254     }
 255 }
 256
 257 sub set_value {
 258     my ($lvalue, $rvalue) = @_;
 259
 260     if (defined($opt{$lvalue})) {
 261         die "Error: Option $lvalue defined more than once!\n";
 262     }
 263     if ($rvalue =~ /^\s*$/) {
 264         delete $opt{$lvalue};
 265     } else {
 266         $opt{$lvalue} = $rvalue;
 267     }
 268 }
 269
 270 sub read_config {
 271     my ($config) = @_;
 272
 273     open(IN, $config) || die "can't read file $config";
 274
 275     my $name = $config;
 276     $name =~ s,.*/(.*),$1,;
 277
 278     my $test_num = 0;
 279     my $default = 1;
 280     my $repeat = 1;
 281     my $num_tests_set = 0;
 282     my $skip = 0;
 283     my $rest;
 284
 285     while (<IN>) {
 286
 287         # ignore blank lines and comments
 288         next if (/^\s*$/ || /\s*\#/);
 289
 290         if (/^\s*TEST_START(.*)/) {
 291
 292             $rest = $1;
 293
 294             if ($num_tests_set) {
 295                 die "$name: $.: Can not specify both NUM_TESTS and TEST_START\n";
 296             }
 297
 298             my $old_test_num = $test_num;
 299             my $old_repeat = $repeat;
 300
 301             $test_num += $repeat;
 302             $default = 0;
 303             $repeat = 1;
 304
 305             if ($rest =~ /\s+SKIP(.*)/) {
 306                 $rest = $1;
 307                 $skip = 1;
 308             } else {
 309                 $skip = 0;
 310             }
 311
 312             if ($rest =~ /\s+ITERATE\s+(\d+)(.*)$/) {
 313                 $repeat = $1;
 314                 $rest = $2;
 315                 $repeat_tests{"$test_num"} = $repeat;
 316             }
 317
 318             if ($rest =~ /\s+SKIP(.*)/) {
 319                 $rest = $1;
 320                 $skip = 1;
 321             }
 322
 323             if ($rest !~ /^\s*$/) {
 324                 die "$name: $.: Gargbage found after TEST_START\n$_";
 325             }
 326
 327             if ($skip) {
 328                 $test_num = $old_test_num;
 329                 $repeat = $old_repeat;
 330             }
 331
 332         } elsif (/^\s*DEFAULTS(.*)$/) {
 333             $default = 1;
 334
 335             $rest = $1;
 336
 337             if ($rest =~ /\s+SKIP(.*)/) {
 338                 $rest = $1;
 339                 $skip = 1;
 340             } else {
 341                 $skip = 0;
 342             }
 343
 344             if ($rest !~ /^\s*$/) {
 345                 die "$name: $.: Gargbage found after DEFAULTS\n$_";
 346             }
 347
 348         } elsif (/^\s*([A-Z_\[\]\d]+)\s*=\s*(.*?)\s*$/) {
 349
 350             next if ($skip);
 351
 352             my $lvalue = $1;
 353             my $rvalue = $2;
 354
 355             if (!$default &&
 356                 ($lvalue eq "NUM_TESTS" ||
 357                  $lvalue eq "LOG_FILE" ||
 358                  $lvalue eq "CLEAR_LOG")) {
 359                 die "$name: $.: $lvalue must be set in DEFAULTS section\n";
 360             }
 361
 362             if ($lvalue eq "NUM_TESTS") {
 363                 if ($test_num) {
 364                     die "$name: $.: Can not specify both NUM_TESTS and TEST_START\n";
 365                 }
 366                 if (!$default) {
 367                     die "$name: $.: NUM_TESTS must be set in default section\n";
 368                 }
 369                 $num_tests_set = 1;
 370             }
 371
 372             if ($default || $lvalue =~ /\[\d+\]$/) {
 373                 set_value($lvalue, $rvalue);
 374             } else {
 375                 my $val = "$lvalue\[$test_num\]";
 376                 set_value($val, $rvalue);
 377
 378                 if ($repeat > 1) {
 379                     $repeats{$val} = $repeat;
 380                 }
 381             }
 382         } else {
 383             die "$name: $.: Garbage found in config\n$_";
 384         }
 385     }
 386
 387     close(IN);
 388
 389     if ($test_num) {
 390         $test_num += $repeat - 1;
 391         $opt{"NUM_TESTS"} = $test_num;
 392     }
 393
 394     # make sure we have all mandatory configs
 395     get_ktest_configs;
 396
 397     # set any defaults
 398
 399     foreach my $default (keys %default) {
 400         if (!defined($opt{$default})) {
 401             $opt{$default} = $default{$default};
 402         }
 403     }
 404 }
 405
 406 sub _logit {
 407     if (defined($opt{"LOG_FILE"})) {
 408         open(OUT, ">> $opt{LOG_FILE}") or die "Can't write to $opt{LOG_FILE}";
 409         print OUT @_;
 410         close(OUT);
 411     }
 412 }
 413
 414 sub logit {
 415     if (defined($opt{"LOG_FILE"})) {
 416         _logit @_;
 417     } else {
 418         print @_;
 419     }
 420 }
 421
 422 sub doprint {
 423     print @_;
 424     _logit @_;
 425 }
 426
 427 sub run_command;
 428
 429 sub reboot {
 430     # try to reboot normally
 431     if (run_command $reboot) {
 432         if (defined($powercycle_after_reboot)) {
 433             sleep $powercycle_after_reboot;
 434             run_command "$power_cycle";
 435         }
 436     } else {
 437         # nope? power cycle it.
 438         run_command "$power_cycle";
 439     }
 440 }
 441
 442 sub do_not_reboot {
 443     my $i = $iteration;
 444
 445     return $test_type eq "build" ||
 446         ($test_type eq "patchcheck" && $opt{"PATCHCHECK_TYPE[$i]"} eq "build") ||
 447         ($test_type eq "bisect" && $opt{"BISECT_TYPE[$i]"} eq "build");
 448 }
 449
 450 sub dodie {
 451     doprint "CRITICAL FAILURE... ", @_, "\n";
 452
 453     my $i = $iteration;
 454
 455     if ($reboot_on_error && !do_not_reboot) {
 456
 457         doprint "REBOOTING\n";
 458         reboot;
 459
 460     } elsif ($poweroff_on_error && defined($power_off)) {
 461         doprint "POWERING OFF\n";
 462         `$power_off`;
 463     }
 464
 465     die @_, "\n";
 466 }
 467
 468 sub open_console {
 469     my ($fp) = @_;
 470
 471     my $flags;
 472
 473     my $pid = open($fp, "$console|") or
 474         dodie "Can't open console $console";
 475
 476     $flags = fcntl($fp, F_GETFL, 0) or
 477         dodie "Can't get flags for the socket: $!";
 478     $flags = fcntl($fp, F_SETFL, $flags | O_NONBLOCK) or
 479         dodie "Can't set flags for the socket: $!";
 480
 481     return $pid;
 482 }
 483
 484 sub close_console {
 485     my ($fp, $pid) = @_;
 486
 487     doprint "kill child process $pid\n";
 488     kill 2, $pid;
 489
 490     print "closing!\n";
 491     close($fp);
 492 }
 493
 494 sub start_monitor {
 495     if ($monitor_cnt++) {
 496         return;
 497     }
 498     $monitor_fp = \*MONFD;
 499     $monitor_pid = open_console $monitor_fp;
 500
 501     return;
 502
 503     open(MONFD, "Stop perl from warning about single use of MONFD");
 504 }
 505
 506 sub end_monitor {
 507     if (--$monitor_cnt) {
 508         return;
 509     }
 510     close_console($monitor_fp, $monitor_pid);
 511 }
 512
 513 sub wait_for_monitor {
 514     my ($time) = @_;
 515     my $line;
 516
 517     doprint "** Wait for monitor to settle down **\n";
 518
 519     # read the monitor and wait for the system to calm down
 520     do {
 521         $line = wait_for_input($monitor_fp, $time);
 522         print "$line" if (defined($line));
 523     } while (defined($line));
 524     print "** Monitor flushed **\n";
 525 }
 526
 527 sub fail {
 528
 529         if ($die_on_failure) {
 530                 dodie @_;
 531         }
 532
 533         doprint "FAILED\n";
 534
 535         my $i = $iteration;
 536
 537         # no need to reboot for just building.
 538         if (!do_not_reboot) {
 539             doprint "REBOOTING\n";
 540             reboot;
 541             start_monitor;
 542             wait_for_monitor $sleep_time;
 543             end_monitor;
 544         }
 545
 546         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 547         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 548         doprint "KTEST RESULT: TEST $i Failed: ", @_, "\n";
 549         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 550         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 551
 552         return 1 if (!defined($store_failures));
 553
 554         my @t = localtime;
 555         my $date = sprintf "%04d%02d%02d%02d%02d%02d",
 556                 1900+$t[5],$t[4],$t[3],$t[2],$t[1],$t[0];
 557
 558         my $type = $build_type;
 559         if ($type =~ /useconfig/) {
 560             $type = "useconfig";
 561         }
 562
 563         my $dir = "$machine-$test_type-$type-fail-$date";
 564         my $faildir = "$store_failures/$dir";
 565
 566         if (!-d $faildir) {
 567             mkpath($faildir) or
 568                 die "can't create $faildir";
 569         }
 570         if (-f "$output_config") {
 571             cp "$output_config", "$faildir/config" or
 572                 die "failed to copy .config";
 573         }
 574         if (-f $buildlog) {
 575             cp $buildlog, "$faildir/buildlog" or
 576                 die "failed to move $buildlog";
 577         }
 578         if (-f $dmesg) {
 579             cp $dmesg, "$faildir/dmesg" or
 580                 die "failed to move $dmesg";
 581         }
 582
 583         doprint "*** Saved info to $faildir ***\n";
 584
 585         return 1;
 586 }
 587
 588 sub run_command {
 589     my ($command) = @_;
 590     my $dolog = 0;
 591     my $dord = 0;
 592     my $pid;
 593
 594     $command =~ s/\$SSH_USER/$ssh_user/g;
 595     $command =~ s/\$MACHINE/$machine/g;
 596
 597     doprint("$command ... ");
 598
 599     $pid = open(CMD, "$command 2>&1 |") or
 600         (fail "unable to exec $command" and return 0);
 601
 602     if (defined($opt{"LOG_FILE"})) {
 603         open(LOG, ">>$opt{LOG_FILE}") or
 604             dodie "failed to write to log";
 605         $dolog = 1;
 606     }
 607
 608     if (defined($redirect)) {
 609         open (RD, ">$redirect") or
 610             dodie "failed to write to redirect $redirect";
 611         $dord = 1;
 612     }
 613
 614     while (<CMD>) {
 615         print LOG if ($dolog);
 616         print RD  if ($dord);
 617     }
 618
 619     waitpid($pid, 0);
 620     my $failed = $?;
 621
 622     close(CMD);
 623     close(LOG) if ($dolog);
 624     close(RD)  if ($dord);
 625
 626     if ($failed) {
 627         doprint "FAILED!\n";
 628     } else {
 629         doprint "SUCCESS\n";
 630     }
 631
 632     return !$failed;
 633 }
 634
 635 sub run_ssh {
 636     my ($cmd) = @_;
 637     my $cp_exec = $ssh_exec;
 638
 639     $cp_exec =~ s/\$SSH_COMMAND/$cmd/g;
 640     return run_command "$cp_exec";
 641 }
 642
 643 sub run_scp {
 644     my ($src, $dst) = @_;
 645     my $cp_scp = $scp_to_target;
 646
 647     $cp_scp =~ s/\$SRC_FILE/$src/g;
 648     $cp_scp =~ s/\$DST_FILE/$dst/g;
 649
 650     return run_command "$cp_scp";
 651 }
 652
 653 sub get_grub_index {
 654
 655     if ($reboot_type ne "grub") {
 656         return;
 657     }
 658     return if (defined($grub_number));
 659
 660     doprint "Find grub menu ... ";
 661     $grub_number = -1;
 662
 663     my $ssh_grub = $ssh_exec;
 664     $ssh_grub =~ s,\$SSH_COMMAND,cat /boot/grub/menu.lst,g;
 665
 666     open(IN, "$ssh_grub |")
 667         or die "unable to get menu.lst";
 668
 669     while (<IN>) {
 670         if (/^\s*title\s+$grub_menu\s*$/) {
 671             $grub_number++;
 672             last;
 673         } elsif (/^\s*title\s/) {
 674             $grub_number++;
 675         }
 676     }
 677     close(IN);
 678
 679     die "Could not find '$grub_menu' in /boot/grub/menu on $machine"
 680         if ($grub_number < 0);
 681     doprint "$grub_number\n";
 682 }
 683
 684 sub wait_for_input
 685 {
 686     my ($fp, $time) = @_;
 687     my $rin;
 688     my $ready;
 689     my $line;
 690     my $ch;
 691
 692     if (!defined($time)) {
 693         $time = $timeout;
 694     }
 695
 696     $rin = '';
 697     vec($rin, fileno($fp), 1) = 1;
 698     $ready = select($rin, undef, undef, $time);
 699
 700     $line = "";
 701
 702     # try to read one char at a time
 703     while (sysread $fp, $ch, 1) {
 704         $line .= $ch;
 705         last if ($ch eq "\n");
 706     }
 707
 708     if (!length($line)) {
 709         return undef;
 710     }
 711
 712     return $line;
 713 }
 714
 715 sub reboot_to {
 716     if ($reboot_type eq "grub") {
 717         run_ssh "'(echo \"savedefault --default=$grub_number --once\" | grub --batch; reboot)'";
 718         return;
 719     }
 720
 721     run_command "$reboot_script";
 722 }
 723
 724 sub get_sha1 {
 725     my ($commit) = @_;
 726
 727     doprint "git rev-list --max-count=1 $commit ... ";
 728     my $sha1 = `git rev-list --max-count=1 $commit`;
 729     my $ret = $?;
 730
 731     logit $sha1;
 732
 733     if ($ret) {
 734         doprint "FAILED\n";
 735         dodie "Failed to get git $commit";
 736     }
 737
 738     print "SUCCESS\n";
 739
 740     chomp $sha1;
 741
 742     return $sha1;
 743 }
 744
 745 sub monitor {
 746     my $booted = 0;
 747     my $bug = 0;
 748     my $skip_call_trace = 0;
 749     my $loops;
 750
 751     wait_for_monitor 5;
 752
 753     my $line;
 754     my $full_line = "";
 755
 756     open(DMESG, "> $dmesg") or
 757         die "unable to write to $dmesg";
 758
 759     reboot_to;
 760
 761     my $success_start;
 762     my $failure_start;
 763
 764     for (;;) {
 765
 766         if ($booted) {
 767             $line = wait_for_input($monitor_fp, $booted_timeout);
 768         } else {
 769             $line = wait_for_input($monitor_fp);
 770         }
 771
 772         last if (!defined($line));
 773
 774         doprint $line;
 775         print DMESG $line;
 776
 777         # we are not guaranteed to get a full line
 778         $full_line .= $line;
 779
 780         if ($full_line =~ /$success_line/) {
 781             $booted = 1;
 782             $success_start = time;
 783         }
 784
 785         if ($booted && defined($stop_after_success) &&
 786             $stop_after_success >= 0) {
 787             my $now = time;
 788             if ($now - $success_start >= $stop_after_success) {
 789                 doprint "Test forced to stop after $stop_after_success seconds after success\n";
 790                 last;
 791             }
 792         }
 793
 794         if ($full_line =~ /\[ backtrace testing \]/) {
 795             $skip_call_trace = 1;
 796         }
 797
 798         if ($full_line =~ /call trace:/i) {
 799             if (!$skip_call_trace) {
 800                 $bug = 1;
 801                 $failure_start = time;
 802             }
 803         }
 804
 805         if ($bug && defined($stop_after_failure) &&
 806             $stop_after_failure >= 0) {
 807             my $now = time;
 808             if ($now - $failure_start >= $stop_after_failure) {
 809                 doprint "Test forced to stop after $stop_after_failure seconds after failure\n";
 810                 last;
 811             }
 812         }
 813
 814         if ($full_line =~ /\[ end of backtrace testing \]/) {
 815             $skip_call_trace = 0;
 816         }
 817
 818         if ($full_line =~ /Kernel panic -/) {
 819             $bug = 1;
 820         }
 821
 822         if ($line =~ /\n/) {
 823             $full_line = "";
 824         }
 825     }
 826
 827     close(DMESG);
 828
 829     if ($bug) {
 830         return 0 if ($in_bisect);
 831         fail "failed - got a bug report" and return 0;
 832     }
 833
 834     if (!$booted) {
 835         return 0 if ($in_bisect);
 836         fail "failed - never got a boot prompt." and return 0;
 837     }
 838
 839     return 1;
 840 }
 841
 842 sub install {
 843
 844     run_scp "$outputdir/$build_target", "$target_image" or
 845         dodie "failed to copy image";
 846
 847     my $install_mods = 0;
 848
 849     # should we process modules?
 850     $install_mods = 0;
 851     open(IN, "$output_config") or dodie("Can't read config file");
 852     while (<IN>) {
 853         if (/CONFIG_MODULES(=y)?/) {
 854             $install_mods = 1 if (defined($1));
 855             last;
 856         }
 857     }
 858     close(IN);
 859
 860     if (!$install_mods) {
 861         doprint "No modules needed\n";
 862         return;
 863     }
 864
 865     run_command "$make INSTALL_MOD_PATH=$tmpdir modules_install" or
 866         dodie "Failed to install modules";
 867
 868     my $modlib = "/lib/modules/$version";
 869     my $modtar = "ktest-mods.tar.bz2";
 870
 871     run_ssh "rm -rf $modlib" or
 872         dodie "failed to remove old mods: $modlib";
 873
 874     # would be nice if scp -r did not follow symbolic links
 875     run_command "cd $tmpdir && tar -cjf $modtar lib/modules/$version" or
 876         dodie "making tarball";
 877
 878     run_scp "$tmpdir/$modtar", "/tmp" or
 879         dodie "failed to copy modules";
 880
 881     unlink "$tmpdir/$modtar";
 882
 883     run_ssh "'(cd / && tar xf /tmp/$modtar)'" or
 884         dodie "failed to tar modules";
 885
 886     run_ssh "rm -f /tmp/$modtar";
 887
 888     return if (!defined($post_install));
 889
 890     my $cp_post_install = $post_install;
 891     $cp_post_install = s/\$KERNEL_VERSION/$version/g;
 892     run_command "$cp_post_install" or
 893         dodie "Failed to run post install";
 894 }
 895
 896 sub check_buildlog {
 897     my ($patch) = @_;
 898
 899     my @files = `git show $patch | diffstat -l`;
 900
 901     open(IN, "git show $patch |") or
 902         dodie "failed to show $patch";
 903     while (<IN>) {
 904         if (m,^--- a/(.*),) {
 905             chomp $1;
 906             $files[$#files] = $1;
 907         }
 908     }
 909     close(IN);
 910
 911     open(IN, $buildlog) or dodie "Can't open $buildlog";
 912     while (<IN>) {
 913         if (/^\s*(.*?):.*(warning|error)/) {
 914             my $err = $1;
 915             foreach my $file (@files) {
 916                 my $fullpath = "$builddir/$file";
 917                 if ($file eq $err || $fullpath eq $err) {
 918                     fail "$file built with warnings" and return 0;
 919                 }
 920             }
 921         }
 922     }
 923     close(IN);
 924
 925     return 1;
 926 }
 927
 928 sub build {
 929     my ($type) = @_;
 930     my $defconfig = "";
 931
 932     unlink $buildlog;
 933
 934     if ($type =~ /^useconfig:(.*)/) {
 935         run_command "cp $1 $output_config" or
 936             dodie "could not copy $1 to .config";
 937
 938         $type = "oldconfig";
 939     }
 940
 941     # old config can ask questions
 942     if ($type eq "oldconfig") {
 943         $type = "oldnoconfig";
 944
 945         # allow for empty configs
 946         run_command "touch $output_config";
 947
 948         run_command "mv $output_config $outputdir/config_temp" or
 949             dodie "moving .config";
 950
 951         if (!$noclean && !run_command "$make mrproper") {
 952             dodie "make mrproper";
 953         }
 954
 955         run_command "mv $outputdir/config_temp $output_config" or
 956             dodie "moving config_temp";
 957
 958     } elsif (!$noclean) {
 959         unlink "$output_config";
 960         run_command "$make mrproper" or
 961             dodie "make mrproper";
 962     }
 963
 964     # add something to distinguish this build
 965     open(OUT, "> $outputdir/localversion") or dodie("Can't make localversion file");
 966     print OUT "$localversion\n";
 967     close(OUT);
 968
 969     if (defined($minconfig)) {
 970         $defconfig = "KCONFIG_ALLCONFIG=$minconfig";
 971     }
 972
 973     run_command "$defconfig $make $type" or
 974         dodie "failed make config";
 975
 976     $redirect = "$buildlog";
 977     if (!run_command "$make $build_options") {
 978         undef $redirect;
 979         # bisect may need this to pass
 980         return 0 if ($in_bisect);
 981         fail "failed build" and return 0;
 982     }
 983     undef $redirect;
 984
 985     return 1;
 986 }
 987
 988 sub halt {
 989     if (!run_ssh "halt" or defined($power_off)) {
 990         if (defined($poweroff_after_halt)) {
 991             sleep $poweroff_after_halt;
 992             run_command "$power_off";
 993         }
 994     } else {
 995         # nope? the zap it!
 996         run_command "$power_off";
 997     }
 998 }
 999
1000 sub success {
1001     my ($i) = @_;
1002
1003     $successes++;
1004
1005     doprint "\n\n*******************************************\n";
1006     doprint     "*******************************************\n";
1007     doprint     "KTEST RESULT: TEST $i SUCCESS!!!!         **\n";
1008     doprint     "*******************************************\n";
1009     doprint     "*******************************************\n";
1010
1011     if ($i != $opt{"NUM_TESTS"} && !do_not_reboot) {
1012         doprint "Reboot and wait $sleep_time seconds\n";
1013         reboot;
1014         start_monitor;
1015         wait_for_monitor $sleep_time;
1016         end_monitor;
1017     }
1018 }
1019
1020 sub get_version {
1021     # get the release name
1022     doprint "$make kernelrelease ... ";
1023     $version = `$make kernelrelease | tail -1`;
1024     chomp($version);
1025     doprint "$version\n";
1026 }
1027
1028 sub child_run_test {
1029     my $failed = 0;
1030
1031     # child should have no power
1032     $reboot_on_error = 0;
1033     $poweroff_on_error = 0;
1034     $die_on_failure = 1;
1035
1036     run_command $run_test or $failed = 1;
1037     exit $failed;
1038 }
1039
1040 my $child_done;
1041
1042 sub child_finished {
1043     $child_done = 1;
1044 }
1045
1046 sub do_run_test {
1047     my $child_pid;
1048     my $child_exit;
1049     my $line;
1050     my $full_line;
1051     my $bug = 0;
1052
1053     wait_for_monitor 1;
1054
1055     doprint "run test $run_test\n";
1056
1057     $child_done = 0;
1058
1059     $SIG{CHLD} = qw(child_finished);
1060
1061     $child_pid = fork;
1062
1063     child_run_test if (!$child_pid);
1064
1065     $full_line = "";
1066
1067     do {
1068         $line = wait_for_input($monitor_fp, 1);
1069         if (defined($line)) {
1070
1071             # we are not guaranteed to get a full line
1072             $full_line .= $line;
1073
1074             if ($full_line =~ /call trace:/i) {
1075                 $bug = 1;
1076             }
1077
1078             if ($full_line =~ /Kernel panic -/) {
1079                 $bug = 1;
1080             }
1081
1082             if ($line =~ /\n/) {
1083                 $full_line = "";
1084             }
1085         }
1086     } while (!$child_done && !$bug);
1087
1088     if ($bug) {
1089         doprint "Detected kernel crash!\n";
1090         # kill the child with extreme prejudice
1091         kill 9, $child_pid;
1092     }
1093
1094     waitpid $child_pid, 0;
1095     $child_exit = $?;
1096
1097     if ($bug || $child_exit) {
1098         return 0 if $in_bisect;
1099         fail "test failed" and return 0;
1100     }
1101     return 1;
1102 }
1103
1104 sub run_git_bisect {
1105     my ($command) = @_;
1106
1107     doprint "$command ... ";
1108
1109     my $output = `$command 2>&1`;
1110     my $ret = $?;
1111
1112     logit $output;
1113
1114     if ($ret) {
1115         doprint "FAILED\n";
1116         dodie "Failed to git bisect";
1117     }
1118
1119     doprint "SUCCESS\n";
1120     if ($output =~ m/^(Bisecting: .*\(roughly \d+ steps?\))\s+\[([[:xdigit:]]+)\]/) {
1121         doprint "$1 [$2]\n";
1122     } elsif ($output =~ m/^([[:xdigit:]]+) is the first bad commit/) {
1123         $bisect_bad = $1;
1124         doprint "Found bad commit... $1\n";
1125         return 0;
1126     } else {
1127         # we already logged it, just print it now.
1128         print $output;
1129     }
1130
1131     return 1;
1132 }
1133
1134 # returns 1 on success, 0 on failure
1135 sub run_bisect_test {
1136     my ($type, $buildtype) = @_;
1137
1138     my $failed = 0;
1139     my $result;
1140     my $output;
1141     my $ret;
1142
1143     $in_bisect = 1;
1144
1145     build $buildtype or $failed = 1;
1146
1147     if ($type ne "build") {
1148         dodie "Failed on build" if $failed;
1149
1150         # Now boot the box
1151         get_grub_index;
1152         get_version;
1153         install;
1154
1155         start_monitor;
1156         monitor or $failed = 1;
1157
1158         if ($type ne "boot") {
1159             dodie "Failed on boot" if $failed;
1160
1161             do_run_test or $failed = 1;
1162         }
1163         end_monitor;
1164     }
1165
1166     if ($failed) {
1167         $result = 0;
1168
1169         # reboot the box to a good kernel
1170         if ($type ne "build") {
1171             doprint "Reboot and sleep $bisect_sleep_time seconds\n";
1172             reboot;
1173             start_monitor;
1174             wait_for_monitor $bisect_sleep_time;
1175             end_monitor;
1176         }
1177     } else {
1178         $result = 1;
1179     }
1180     $in_bisect = 0;
1181
1182     return $result;
1183 }
1184
1185 sub run_bisect {
1186     my ($type) = @_;
1187     my $buildtype = "oldconfig";
1188
1189     # We should have a minconfig to use?
1190     if (defined($minconfig)) {
1191         $buildtype = "useconfig:$minconfig";
1192     }
1193
1194     my $ret = run_bisect_test $type, $buildtype;
1195
1196
1197     # Are we looking for where it worked, not failed?
1198     if ($reverse_bisect) {
1199         $ret = !$ret;
1200     }
1201
1202     if ($ret) {
1203         return "good";
1204     } else {
1205         return  "bad";
1206     }
1207 }
1208
1209 sub bisect {
1210     my ($i) = @_;
1211
1212     my $result;
1213
1214     die "BISECT_GOOD[$i] not defined\n" if (!defined($opt{"BISECT_GOOD[$i]"}));
1215     die "BISECT_BAD[$i] not defined\n"  if (!defined($opt{"BISECT_BAD[$i]"}));
1216     die "BISECT_TYPE[$i] not defined\n" if (!defined($opt{"BISECT_TYPE[$i]"}));
1217
1218     my $good = $opt{"BISECT_GOOD[$i]"};
1219     my $bad = $opt{"BISECT_BAD[$i]"};
1220     my $type = $opt{"BISECT_TYPE[$i]"};
1221     my $start = $opt{"BISECT_START[$i]"};
1222     my $replay = $opt{"BISECT_REPLAY[$i]"};
1223
1224     # convert to true sha1's
1225     $good = get_sha1($good);
1226     $bad = get_sha1($bad);
1227
1228     if (defined($opt{"BISECT_REVERSE[$i]"}) &&
1229         $opt{"BISECT_REVERSE[$i]"} == 1) {
1230         doprint "Performing a reverse bisect (bad is good, good is bad!)\n";
1231         $reverse_bisect = 1;
1232     } else {
1233         $reverse_bisect = 0;
1234     }
1235
1236     # Can't have a test without having a test to run
1237     if ($type eq "test" && !defined($run_test)) {
1238         $type = "boot";
1239     }
1240
1241     my $check = $opt{"BISECT_CHECK[$i]"};
1242     if (defined($check) && $check ne "0") {
1243
1244         # get current HEAD
1245         my $head = get_sha1("HEAD");
1246
1247         if ($check ne "good") {
1248             doprint "TESTING BISECT BAD [$bad]\n";
1249             run_command "git checkout $bad" or
1250                 die "Failed to checkout $bad";
1251
1252             $result = run_bisect $type;
1253
1254             if ($result ne "bad") {
1255                 fail "Tested BISECT_BAD [$bad] and it succeeded" and return 0;
1256             }
1257         }
1258
1259         if ($check ne "bad") {
1260             doprint "TESTING BISECT GOOD [$good]\n";
1261             run_command "git checkout $good" or
1262                 die "Failed to checkout $good";
1263
1264             $result = run_bisect $type;
1265
1266             if ($result ne "good") {
1267                 fail "Tested BISECT_GOOD [$good] and it failed" and return 0;
1268             }
1269         }
1270
1271         # checkout where we started
1272         run_command "git checkout $head" or
1273             die "Failed to checkout $head";
1274     }
1275
1276     run_command "git bisect start" or
1277         dodie "could not start bisect";
1278
1279     run_command "git bisect good $good" or
1280         dodie "could not set bisect good to $good";
1281
1282     run_git_bisect "git bisect bad $bad" or
1283         dodie "could not set bisect bad to $bad";
1284
1285     if (defined($replay)) {
1286         run_command "git bisect replay $replay" or
1287             dodie "failed to run replay";
1288     }
1289
1290     if (defined($start)) {
1291         run_command "git checkout $start" or
1292             dodie "failed to checkout $start";
1293     }
1294
1295     my $test;
1296     do {
1297         $result = run_bisect $type;
1298         $test = run_git_bisect "git bisect $result";
1299     } while ($test);
1300
1301     run_command "git bisect log" or
1302         dodie "could not capture git bisect log";
1303
1304     run_command "git bisect reset" or
1305         dodie "could not reset git bisect";
1306
1307     doprint "Bad commit was [$bisect_bad]\n";
1308
1309     success $i;
1310 }
1311
1312 my %config_ignore;
1313 my %config_set;
1314
1315 my %config_list;
1316 my %null_config;
1317
1318 my %dependency;
1319
1320 sub process_config_ignore {
1321     my ($config) = @_;
1322
1323     open (IN, $config)
1324         or dodie "Failed to read $config";
1325
1326     while (<IN>) {
1327         if (/^(.*?(CONFIG\S*)(=.*| is not set))/) {
1328             $config_ignore{$2} = $1;
1329         }
1330     }
1331
1332     close(IN);
1333 }
1334
1335 sub read_current_config {
1336     my ($config_ref) = @_;
1337
1338     %{$config_ref} = ();
1339     undef %{$config_ref};
1340
1341     my @key = keys %{$config_ref};
1342     if ($#key >= 0) {
1343         print "did not delete!\n";
1344         exit;
1345     }
1346     open (IN, "$output_config");
1347
1348     while (<IN>) {
1349         if (/^(CONFIG\S+)=(.*)/) {
1350             ${$config_ref}{$1} = $2;
1351         }
1352     }
1353     close(IN);
1354 }
1355
1356 sub get_dependencies {
1357     my ($config) = @_;
1358
1359     my $arr = $dependency{$config};
1360     if (!defined($arr)) {
1361         return ();
1362     }
1363
1364     my @deps = @{$arr};
1365
1366     foreach my $dep (@{$arr}) {
1367         print "ADD DEP $dep\n";
1368         @deps = (@deps, get_dependencies $dep);
1369     }
1370
1371     return @deps;
1372 }
1373
1374 sub create_config {
1375     my @configs = @_;
1376
1377     open(OUT, ">$output_config") or dodie "Can not write to $output_config";
1378
1379     foreach my $config (@configs) {
1380         print OUT "$config_set{$config}\n";
1381         my @deps = get_dependencies $config;
1382         foreach my $dep (@deps) {
1383             print OUT "$config_set{$dep}\n";
1384         }
1385     }
1386
1387     foreach my $config (keys %config_ignore) {
1388         print OUT "$config_ignore{$config}\n";
1389     }
1390     close(OUT);
1391
1392 #    exit;
1393     run_command "$make oldnoconfig" or
1394         dodie "failed make config oldconfig";
1395
1396 }
1397
1398 sub compare_configs {
1399     my (%a, %b) = @_;
1400
1401     foreach my $item (keys %a) {
1402         if (!defined($b{$item})) {
1403             print "diff $item\n";
1404             return 1;
1405         }
1406         delete $b{$item};
1407     }
1408
1409     my @keys = keys %b;
1410     if ($#keys) {
1411         print "diff2 $keys[0]\n";
1412     }
1413     return -1 if ($#keys >= 0);
1414
1415     return 0;
1416 }
1417
1418 sub run_config_bisect_test {
1419     my ($type) = @_;
1420
1421     return run_bisect_test $type, "oldconfig";
1422 }
1423
1424 sub process_passed {
1425     my (%configs) = @_;
1426
1427     doprint "These configs had no failure: (Enabling them for further compiles)\n";
1428     # Passed! All these configs are part of a good compile.
1429     # Add them to the min options.
1430     foreach my $config (keys %configs) {
1431         if (defined($config_list{$config})) {
1432             doprint " removing $config\n";
1433             $config_ignore{$config} = $config_list{$config};
1434             delete $config_list{$config};
1435         }
1436     }
1437     doprint "config copied to $outputdir/config_good\n";
1438     run_command "cp -f $output_config $outputdir/config_good";
1439 }
1440
1441 sub process_failed {
1442     my ($config) = @_;
1443
1444     doprint "\n\n***************************************\n";
1445     doprint "Found bad config: $config\n";
1446     doprint "***************************************\n\n";
1447 }
1448
1449 sub run_config_bisect {
1450
1451     my @start_list = keys %config_list;
1452
1453     if ($#start_list < 0) {
1454         doprint "No more configs to test!!!\n";
1455         return -1;
1456     }
1457
1458     doprint "***** RUN TEST ***\n";
1459     my $type = $opt{"CONFIG_BISECT_TYPE[$iteration]"};
1460     my $ret;
1461     my %current_config;
1462
1463     my $count = $#start_list + 1;
1464     doprint "  $count configs to test\n";
1465
1466     my $half = int($#start_list / 2);
1467
1468     do {
1469         my @tophalf = @start_list[0 .. $half];
1470
1471         create_config @tophalf;
1472         read_current_config \%current_config;
1473
1474         $count = $#tophalf + 1;
1475         doprint "Testing $count configs\n";
1476         my $found = 0;
1477         # make sure we test something
1478         foreach my $config (@tophalf) {
1479             if (defined($current_config{$config})) {
1480                 logit " $config\n";
1481                 $found = 1;
1482             }
1483         }
1484         if (!$found) {
1485             # try the other half
1486             doprint "Top half produced no set configs, trying bottom half\n";
1487             @tophalf = @start_list[$half .. $#start_list];
1488             create_config @tophalf;
1489             read_current_config \%current_config;
1490             foreach my $config (@tophalf) {
1491                 if (defined($current_config{$config})) {
1492                     logit " $config\n";
1493                     $found = 1;
1494                 }
1495             }
1496             if (!$found) {
1497                 doprint "Failed: Can't make new config with current configs\n";
1498                 foreach my $config (@start_list) {
1499                     doprint "  CONFIG: $config\n";
1500                 }
1501                 return -1;
1502             }
1503             $count = $#tophalf + 1;
1504             doprint "Testing $count configs\n";
1505         }
1506
1507         $ret = run_config_bisect_test $type;
1508
1509         if ($ret) {
1510             process_passed %current_config;
1511             return 0;
1512         }
1513
1514         doprint "This config had a failure.\n";
1515         doprint "Removing these configs that were not set in this config:\n";
1516         doprint "config copied to $outputdir/config_bad\n";
1517         run_command "cp -f $output_config $outputdir/config_bad";
1518
1519         # A config exists in this group that was bad.
1520         foreach my $config (keys %config_list) {
1521             if (!defined($current_config{$config})) {
1522                 doprint " removing $config\n";
1523                 delete $config_list{$config};
1524             }
1525         }
1526
1527         @start_list = @tophalf;
1528
1529         if ($#start_list == 0) {
1530             process_failed $start_list[0];
1531             return 1;
1532         }
1533
1534         # remove half the configs we are looking at and see if
1535         # they are good.
1536         $half = int($#start_list / 2);
1537     } while ($half > 0);
1538
1539     # we found a single config, try it again
1540     my @tophalf = @start_list[0 .. 0];
1541
1542     $ret = run_config_bisect_test $type;
1543     if ($ret) {
1544         process_passed %current_config;
1545         return 0;
1546     }
1547
1548     process_failed $start_list[0];
1549     return 1;
1550 }
1551
1552 sub config_bisect {
1553     my ($i) = @_;
1554
1555     my $start_config = $opt{"CONFIG_BISECT[$i]"};
1556
1557     my $tmpconfig = "$tmpdir/use_config";
1558
1559     # Make the file with the bad config and the min config
1560     if (defined($minconfig)) {
1561         # read the min config for things to ignore
1562         run_command "cp $minconfig $tmpconfig" or
1563             dodie "failed to copy $minconfig to $tmpconfig";
1564     } else {
1565         unlink $tmpconfig;
1566     }
1567
1568     # Add other configs
1569     if (defined($addconfig)) {
1570         run_command "cat $addconfig >> $tmpconfig" or
1571             dodie "failed to append $addconfig";
1572     }
1573
1574     my $defconfig = "";
1575     if (-f $tmpconfig) {
1576         $defconfig = "KCONFIG_ALLCONFIG=$tmpconfig";
1577         process_config_ignore $tmpconfig;
1578     }
1579
1580     # now process the start config
1581     run_command "cp $start_config $output_config" or
1582         dodie "failed to copy $start_config to $output_config";
1583
1584     # read directly what we want to check
1585     my %config_check;
1586     open (IN, $output_config)
1587         or dodie "faied to open $output_config";
1588
1589     while (<IN>) {
1590         if (/^((CONFIG\S*)=.*)/) {
1591             $config_check{$2} = $1;
1592         }
1593     }
1594     close(IN);
1595
1596     # Now run oldconfig with the minconfig (and addconfigs)
1597     run_command "$defconfig $make oldnoconfig" or
1598         dodie "failed make config oldconfig";
1599
1600     # check to see what we lost (or gained)
1601     open (IN, $output_config)
1602         or dodie "Failed to read $start_config";
1603
1604     my %removed_configs;
1605     my %added_configs;
1606
1607     while (<IN>) {
1608         if (/^((CONFIG\S*)=.*)/) {
1609             # save off all options
1610             $config_set{$2} = $1;
1611             if (defined($config_check{$2})) {
1612                 if (defined($config_ignore{$2})) {
1613                     $removed_configs{$2} = $1;
1614                 } else {
1615                     $config_list{$2} = $1;
1616                 }
1617             } elsif (!defined($config_ignore{$2})) {
1618                 $added_configs{$2} = $1;
1619                 $config_list{$2} = $1;
1620             }
1621         }
1622     }
1623     close(IN);
1624
1625     my @confs = keys %removed_configs;
1626     if ($#confs >= 0) {
1627         doprint "Configs overridden by default configs and removed from check:\n";
1628         foreach my $config (@confs) {
1629             doprint " $config\n";
1630         }
1631     }
1632     @confs = keys %added_configs;
1633     if ($#confs >= 0) {
1634         doprint "Configs appearing in make oldconfig and added:\n";
1635         foreach my $config (@confs) {
1636             doprint " $config\n";
1637         }
1638     }
1639
1640     my %config_test;
1641     my $once = 0;
1642
1643     # Sometimes kconfig does weird things. We must make sure
1644     # that the config we autocreate has everything we need
1645     # to test, otherwise we may miss testing configs, or
1646     # may not be able to create a new config.
1647     # Here we create a config with everything set.
1648     create_config (keys %config_list);
1649     read_current_config \%config_test;
1650     foreach my $config (keys %config_list) {
1651         if (!defined($config_test{$config})) {
1652             if (!$once) {
1653                 $once = 1;
1654                 doprint "Configs not produced by kconfig (will not be checked):\n";
1655             }
1656             doprint "  $config\n";
1657             delete $config_list{$config};
1658         }
1659     }
1660     my $ret;
1661     do {
1662         $ret = run_config_bisect;
1663     } while (!$ret);
1664
1665     return $ret if ($ret < 0);
1666
1667     success $i;
1668 }
1669
1670 sub patchcheck {
1671     my ($i) = @_;
1672
1673     die "PATCHCHECK_START[$i] not defined\n"
1674         if (!defined($opt{"PATCHCHECK_START[$i]"}));
1675     die "PATCHCHECK_TYPE[$i] not defined\n"
1676         if (!defined($opt{"PATCHCHECK_TYPE[$i]"}));
1677
1678     my $start = $opt{"PATCHCHECK_START[$i]"};
1679
1680     my $end = "HEAD";
1681     if (defined($opt{"PATCHCHECK_END[$i]"})) {
1682         $end = $opt{"PATCHCHECK_END[$i]"};
1683     }
1684
1685     # Get the true sha1's since we can use things like HEAD~3
1686     $start = get_sha1($start);
1687     $end = get_sha1($end);
1688
1689     my $type = $opt{"PATCHCHECK_TYPE[$i]"};
1690
1691     # Can't have a test without having a test to run
1692     if ($type eq "test" && !defined($run_test)) {
1693         $type = "boot";
1694     }
1695
1696     open (IN, "git log --pretty=oneline $end|") or
1697         dodie "could not get git list";
1698
1699     my @list;
1700
1701     while (<IN>) {
1702         chomp;
1703         $list[$#list+1] = $_;
1704         last if (/^$start/);
1705     }
1706     close(IN);
1707
1708     if ($list[$#list] !~ /^$start/) {
1709         fail "SHA1 $start not found";
1710     }
1711
1712     # go backwards in the list
1713     @list = reverse @list;
1714
1715     my $save_clean = $noclean;
1716
1717     $in_patchcheck = 1;
1718     foreach my $item (@list) {
1719         my $sha1 = $item;
1720         $sha1 =~ s/^([[:xdigit:]]+).*/$1/;
1721
1722         doprint "\nProcessing commit $item\n\n";
1723
1724         run_command "git checkout $sha1" or
1725             die "Failed to checkout $sha1";
1726
1727         # only clean on the first and last patch
1728         if ($item eq $list[0] ||
1729             $item eq $list[$#list]) {
1730             $noclean = $save_clean;
1731         } else {
1732             $noclean = 1;
1733         }
1734
1735         if (defined($minconfig)) {
1736             build "useconfig:$minconfig" or return 0;
1737         } else {
1738             # ?? no config to use?
1739             build "oldconfig" or return 0;
1740         }
1741
1742         check_buildlog $sha1 or return 0;
1743
1744         next if ($type eq "build");
1745
1746         get_grub_index;
1747         get_version;
1748         install;
1749
1750         my $failed = 0;
1751
1752         start_monitor;
1753         monitor or $failed = 1;
1754
1755         if (!$failed && $type ne "boot"){
1756             do_run_test or $failed = 1;
1757         }
1758         end_monitor;
1759         return 0 if ($failed);
1760
1761     }
1762     $in_patchcheck = 0;
1763     success $i;
1764
1765     return 1;
1766 }
1767
1768 $#ARGV < 1 or die "ktest.pl version: $VERSION\n   usage: ktest.pl config-file\n";
1769
1770 if ($#ARGV == 0) {
1771     $ktest_config = $ARGV[0];
1772     if (! -f $ktest_config) {
1773         print "$ktest_config does not exist.\n";
1774         my $ans;
1775         for (;;) {
1776             print "Create it? [Y/n] ";
1777             $ans = <STDIN>;
1778             chomp $ans;
1779             if ($ans =~ /^\s*$/) {
1780                 $ans = "y";
1781             }
1782             last if ($ans =~ /^y$/i || $ans =~ /^n$/i);
1783             print "Please answer either 'y' or 'n'.\n";
1784         }
1785         if ($ans !~ /^y$/i) {
1786             exit 0;
1787         }
1788     }
1789 } else {
1790     $ktest_config = "ktest.conf";
1791 }
1792
1793 if (! -f $ktest_config) {
1794     open(OUT, ">$ktest_config") or die "Can not create $ktest_config";
1795     print OUT << "EOF"
1796 # Generated by ktest.pl
1797 #
1798 # Define each test with TEST_START
1799 # The config options below it will override the defaults
1800 TEST_START
1801
1802 DEFAULTS
1803 EOF
1804 ;
1805     close(OUT);
1806 }
1807 read_config $ktest_config;
1808
1809 # Append any configs entered in manually to the config file.
1810 my @new_configs = keys %entered_configs;
1811 if ($#new_configs >= 0) {
1812     print "\nAppending entered in configs to $ktest_config\n";
1813     open(OUT, ">>$ktest_config") or die "Can not append to $ktest_config";
1814     foreach my $config (@new_configs) {
1815         print OUT "$config = $entered_configs{$config}\n";
1816         $opt{$config} = $entered_configs{$config};
1817     }
1818 }
1819
1820 if ($opt{"CLEAR_LOG"} && defined($opt{"LOG_FILE"})) {
1821     unlink $opt{"LOG_FILE"};
1822 }
1823
1824 doprint "\n\nSTARTING AUTOMATED TESTS\n\n";
1825
1826 for (my $i = 0, my $repeat = 1; $i <= $opt{"NUM_TESTS"}; $i += $repeat) {
1827
1828     if (!$i) {
1829         doprint "DEFAULT OPTIONS:\n";
1830     } else {
1831         doprint "\nTEST $i OPTIONS";
1832         if (defined($repeat_tests{$i})) {
1833             $repeat = $repeat_tests{$i};
1834             doprint " ITERATE $repeat";
1835         }
1836         doprint "\n";
1837     }
1838
1839     foreach my $option (sort keys %opt) {
1840
1841         if ($option =~ /\[(\d+)\]$/) {
1842             next if ($i != $1);
1843         } else {
1844             next if ($i);
1845         }
1846
1847         doprint "$option = $opt{$option}\n";
1848     }
1849 }
1850
1851 sub set_test_option {
1852     my ($name, $i) = @_;
1853
1854     my $option = "$name\[$i\]";
1855
1856     if (defined($opt{$option})) {
1857         return $opt{$option};
1858     }
1859
1860     foreach my $test (keys %repeat_tests) {
1861         if ($i >= $test &&
1862             $i < $test + $repeat_tests{$test}) {
1863             $option = "$name\[$test\]";
1864             if (defined($opt{$option})) {
1865                 return $opt{$option};
1866             }
1867         }
1868     }
1869
1870     if (defined($opt{$name})) {
1871         return $opt{$name};
1872     }
1873
1874     return undef;
1875 }
1876
1877 # First we need to do is the builds
1878 for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) {
1879
1880     $iteration = $i;
1881
1882     my $makecmd = set_test_option("MAKE_CMD", $i);
1883
1884     $machine = set_test_option("MACHINE", $i);
1885     $ssh_user = set_test_option("SSH_USER", $i);
1886     $tmpdir = set_test_option("TMP_DIR", $i);
1887     $outputdir = set_test_option("OUTPUT_DIR", $i);
1888     $builddir = set_test_option("BUILD_DIR", $i);
1889     $test_type = set_test_option("TEST_TYPE", $i);
1890     $build_type = set_test_option("BUILD_TYPE", $i);
1891     $build_options = set_test_option("BUILD_OPTIONS", $i);
1892     $power_cycle = set_test_option("POWER_CYCLE", $i);
1893     $reboot = set_test_option("REBOOT", $i);
1894     $noclean = set_test_option("BUILD_NOCLEAN", $i);
1895     $minconfig = set_test_option("MIN_CONFIG", $i);
1896     $run_test = set_test_option("TEST", $i);
1897     $addconfig = set_test_option("ADD_CONFIG", $i);
1898     $reboot_type = set_test_option("REBOOT_TYPE", $i);
1899     $grub_menu = set_test_option("GRUB_MENU", $i);
1900     $post_install = set_test_option("POST_INSTALL", $i);
1901     $reboot_script = set_test_option("REBOOT_SCRIPT", $i);
1902     $reboot_on_error = set_test_option("REBOOT_ON_ERROR", $i);
1903     $poweroff_on_error = set_test_option("POWEROFF_ON_ERROR", $i);
1904     $die_on_failure = set_test_option("DIE_ON_FAILURE", $i);
1905     $power_off = set_test_option("POWER_OFF", $i);
1906     $powercycle_after_reboot = set_test_option("POWERCYCLE_AFTER_REBOOT", $i);
1907     $poweroff_after_halt = set_test_option("POWEROFF_AFTER_HALT", $i);
1908     $sleep_time = set_test_option("SLEEP_TIME", $i);
1909     $bisect_sleep_time = set_test_option("BISECT_SLEEP_TIME", $i);
1910     $store_failures = set_test_option("STORE_FAILURES", $i);
1911     $timeout = set_test_option("TIMEOUT", $i);
1912     $booted_timeout = set_test_option("BOOTED_TIMEOUT", $i);
1913     $console = set_test_option("CONSOLE", $i);
1914     $success_line = set_test_option("SUCCESS_LINE", $i);
1915     $stop_after_success = set_test_option("STOP_AFTER_SUCCESS", $i);
1916     $stop_after_failure = set_test_option("STOP_AFTER_FAILURE", $i);
1917     $build_target = set_test_option("BUILD_TARGET", $i);
1918     $ssh_exec = set_test_option("SSH_EXEC", $i);
1919     $scp_to_target = set_test_option("SCP_TO_TARGET", $i);
1920     $target_image = set_test_option("TARGET_IMAGE", $i);
1921     $localversion = set_test_option("LOCALVERSION", $i);
1922
1923     chdir $builddir || die "can't change directory to $builddir";
1924
1925     if (!-d $tmpdir) {
1926         mkpath($tmpdir) or
1927             die "can't create $tmpdir";
1928     }
1929
1930     $ENV{"SSH_USER"} = $ssh_user;
1931     $ENV{"MACHINE"} = $machine;
1932
1933     $target = "$ssh_user\@$machine";
1934
1935     $buildlog = "$tmpdir/buildlog-$machine";
1936     $dmesg = "$tmpdir/dmesg-$machine";
1937     $make = "$makecmd O=$outputdir";
1938     $output_config = "$outputdir/.config";
1939
1940     if ($reboot_type eq "grub") {
1941         dodie "GRUB_MENU not defined" if (!defined($grub_menu));
1942     } elsif (!defined($reboot_script)) {
1943         dodie "REBOOT_SCRIPT not defined"
1944     }
1945
1946     my $run_type = $build_type;
1947     if ($test_type eq "patchcheck") {
1948         $run_type = $opt{"PATCHCHECK_TYPE[$i]"};
1949     } elsif ($test_type eq "bisect") {
1950         $run_type = $opt{"BISECT_TYPE[$i]"};
1951     } elsif ($test_type eq "config_bisect") {
1952         $run_type = $opt{"CONFIG_BISECT_TYPE[$i]"};
1953     }
1954
1955     # mistake in config file?
1956     if (!defined($run_type)) {
1957         $run_type = "ERROR";
1958     }
1959
1960     doprint "\n\n";
1961     doprint "RUNNING TEST $i of $opt{NUM_TESTS} with option $test_type $run_type\n\n";
1962
1963     unlink $dmesg;
1964     unlink $buildlog;
1965
1966     if (!defined($minconfig)) {
1967         $minconfig = $addconfig;
1968
1969     } elsif (defined($addconfig)) {
1970         run_command "cat $addconfig $minconfig > $tmpdir/add_config" or
1971             dodie "Failed to create temp config";
1972         $minconfig = "$tmpdir/add_config";
1973     }
1974
1975     my $checkout = $opt{"CHECKOUT[$i]"};
1976     if (defined($checkout)) {
1977         run_command "git checkout $checkout" or
1978             die "failed to checkout $checkout";
1979     }
1980
1981     if ($test_type eq "bisect") {
1982         bisect $i;
1983         next;
1984     } elsif ($test_type eq "config_bisect") {
1985         config_bisect $i;
1986         next;
1987     } elsif ($test_type eq "patchcheck") {
1988         patchcheck $i;
1989         next;
1990     }
1991
1992     if ($build_type ne "nobuild") {
1993         build $build_type or next;
1994     }
1995
1996     if ($test_type ne "build") {
1997         get_grub_index;
1998         get_version;
1999         install;
2000
2001         my $failed = 0;
2002         start_monitor;
2003         monitor or $failed = 1;;
2004
2005         if (!$failed && $test_type ne "boot" && defined($run_test)) {
2006             do_run_test or $failed = 1;
2007         }
2008         end_monitor;
2009         next if ($failed);
2010     }
2011
2012     success $i;
2013 }
2014
2015 if ($opt{"POWEROFF_ON_SUCCESS"}) {
2016     halt;
2017 } elsif ($opt{"REBOOT_ON_SUCCESS"} && !do_not_reboot) {
2018     reboot;
2019 }
2020
2021 doprint "\n    $successes of $opt{NUM_TESTS} tests were successful\n\n";
2022
2023 exit 0;