test-in-vm.sh

   1 #!/bin/bash
   2
   3 #
   4 # Copyright (c) 2016 Vojtech Horky
   5 # All rights reserved.
   6 #
   7 # Redistribution and use in source and binary forms, with or without
   8 # modification, are permitted provided that the following conditions
   9 # are met:
  10 #
  11 # - Redistributions of source code must retain the above copyright
  12 #   notice, this list of conditions and the following disclaimer.
  13 # - Redistributions in binary form must reproduce the above copyright
  14 #   notice, this list of conditions and the following disclaimer in the
  15 #   documentation and/or other materials provided with the distribution.
  16 # - The name of the author may not be used to endorse or promote products
  17 #   derived from this software without specific prior written permission.
  18 #
  19 # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  20 # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  21 # OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  22 # IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  23 # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  24 # NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  28 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29 #
  30
  31 #
  32 # This code was originally inspired by "virtual testing lab" scripts created
  33 # by Jan Buchar for testing his firewall implementation in HelenOS:
  34 # https://code.launchpad.net/~teyras/hpf-virtlab/trunk
  35 #
  36
  37 xx_echo() {
  38     echo ">>" "$@"
  39 }
  40
  41 xx_echo2() {
  42     echo "   -" "$@"
  43 }
  44
  45 xx_debug() {
  46     $XX_DEBUG_ECHO "   :" "$@"
  47 }
  48
  49 xx_run_debug() {
  50     xx_debug "$@"
  51     "$@"
  52 }
  53
  54 xx_activity() {
  55     $XX_ACTIVITY_ECHO "     +" "$@"
  56 }
  57
  58 xx_fatal() {
  59     echo "!!" "$@"
  60     xx_shutdown
  61 }
  62
  63 ## Prepares pipe to QEMU monitor.
  64 # Feed QEMU commands to stdin of this function.
  65 #
  66 # Do not forget to check for return code as checking it inside pipe
  67 # is not very useful (we cannot terminate the script from that).
  68 #
  69 # @param 1 Machine id.
  70 xx_do_qemu_command_pipe() {
  71     socat STDIN "UNIX-CONNECT:$XX_TEMP/$1.monitor"
  72 }
  73
  74 ## Sends command to QEMU monitor.
  75 # This function terminates the scenario on failure of the monitoring pipe.
  76 #
  77 # @param 1 Machine id.
  78 # @param 2 Command to send.
  79 xx_do_qemu_command() {
  80     xx_debug "echo '$2' | socat STDIN 'UNIX-CONNECT:$XX_TEMP/$1.monitor'"
  81     echo "$2" | socat STDIN "UNIX-CONNECT:$XX_TEMP/$1.monitor"
  82     res=$?
  83     if [ $res -ne 0 ]; then
  84         xx_shutdown
  85     fi
  86 }
  87
  88 ## Types the text to a specific machine.
  89 #
  90 # @param 1 Machine id.
  91 # @param 2 Text to type.
  92 xx_do_type() {
  93     ( echo "$2" | fold -w 1 | sed \
  94         -e 's/ /spc/' \
  95         -e 's/\./dot/' \
  96         -e 's/-/minus/' \
  97         -e 's/+/shift-equal/' \
  98         -e 's/_/shift-minus/' \
  99         -e 's/@/shift-2/' \
 100         -e 's/:/shift-semicolon/' \
 101         -e 's/\//slash/' \
 102         -e 's/=/equal/' \
 103         -e 's/|/shift-backslash/' \
 104         -e 's/\([[:upper:]]\)/shift-\L\1/' \
 105         -e 's/\\/backslash/' | \
 106         while read code; do
 107             xx_do_qemu_command "$1" "sendkey $code"
 108         done
 109     )
 110     res=$?
 111     if [ $res -ne 0 ]; then
 112         xx_shutdown
 113     fi
 114 }
 115
 116 xx_get_var() {
 117     local varname="$1"
 118     local default="$2"
 119     shift 2
 120     local i
 121     for i in "$@"; do
 122         case $i in
 123             $varname=*)
 124                 echo "$i" | cut '-d=' -f 2-
 125                 return
 126                 ;;
 127             *)
 128                 ;;
 129         esac
 130     done
 131     echo "$default"
 132 }
 133
 134 xx_get_def_var() {
 135     local i
 136     for i in "$@"; do
 137         case $i in
 138             *=*)
 139                 ;;
 140             *)
 141                 echo "$i"
 142                 ;;
 143         esac
 144     done
 145     echo
 146 }
 147
 148 xx_get_boolean_var() {
 149     value=`xx_get_var "$@" | tr 'A-Z' 'a-z'`
 150     if [ "$value" = "yes" -o "$value" = "true" ]; then
 151         echo "true"
 152     elif [ "$value" = "no" -o "$value" = "false" ]; then
 153         echo "false"
 154     else
 155         echo
 156     fi
 157 }
 158
 159
 160 xx_shutdown() {
 161     local i
 162     for i in $XX_KNOWN_MACHINES; do
 163         xx_stop_machine name="$i"
 164     done
 165     exit 1
 166 }
 167
 168 xx_assert_var() {
 169     if [ -z "$2" ]; then
 170         xx_echo "Option $1 not specified or invalid, terminating."
 171         xx_shutdown
 172     fi
 173 }
 174
 175 xx_do_check_var() {
 176     if [ -z "$2" ]; then
 177         xx_fatal "Option $1 not specified with no suitable default."
 178     fi
 179     if [ -n "$3" ]; then
 180         if ! echo "$2" | grep -q "$3"; then
 181             xx_fatal "Option $1 does not match '$3' (got '$2')."
 182         fi
 183     fi
 184 }
 185
 186 xx_do_compute_timeout() {
 187     echo $(( `date +%s` + $1 ))
 188 }
 189
 190 #
 191 xx_start_machine() {
 192     local cdrom=`xx_get_var cdrom "$XX_CDROM_FILE" "$@"`
 193     local name=`xx_get_var name default "$@"`
 194     local wait_for_vterm=`xx_get_boolean_var vterm true "$@"`
 195
 196     xx_do_check_var "xx_start_machine/name" "$name" '^[a-zA-Z][0-9a-zA-Z]*$'
 197     xx_do_check_var "xx_start_machine/cdrom" "$cdrom"
 198
 199     local extra_opts=""
 200     if $XX_HEADLESS; then
 201         extra_opts="-display none"
 202     fi
 203     if $XX_USE_KVM; then
 204         extra_opts="$extra_opts -enable-kvm"
 205     fi
 206
 207     xx_echo "Starting machine $name from $cdrom."
 208
 209     local qemu_command=""
 210     if [ $XX_ARCH == "ia32" ]; then
 211         qemu_command=qemu-system-i386
 212     elif [ $XX_ARCH == "amd64" ]; then
 213         qemu_command=qemu-system-x86_64
 214     fi
 215     if [ -z "$qemu_command" ]; then
 216         xx_fatal "Unable to find proper emulator."
 217     fi
 218
 219     xx_run_debug $qemu_command \
 220         $extra_opts \
 221         -device e1000,vlan=0 -net user \
 222         -redir udp:8080::8080 -redir udp:8081::8081 \
 223         -redir tcp:8080::8080 -redir tcp:8081::8081 \
 224         -redir tcp:2223::2223 \
 225         -usb \
 226         -daemonize -pidfile "$XX_TEMP/$name.pid" \
 227         -monitor "unix:$XX_TEMP/$name.monitor,server,nowait" \
 228         -boot d \
 229         -cdrom "$cdrom"
 230     sleep 1
 231     xx_do_qemu_command "$name" "sendkey ret"
 232
 233     XX_LAST_MACHINE=$name
 234
 235     XX_KNOWN_MACHINES="$XX_KNOWN_MACHINES $name"
 236
 237     if $wait_for_vterm; then
 238         xx_echo2 "Waiting for OS to boot into GUI..."
 239         if ! xx_do_wait_for_text "$name" `xx_do_compute_timeout 15` "to see a few survival tips"; then
 240             xx_fatal "OS have not booted into a known state."
 241         fi
 242     fi
 243 }
 244
 245
 246
 247 #
 248 xx_stop_machine() {
 249     local name=`xx_get_var name $XX_LAST_MACHINE "$@"`
 250
 251     xx_echo "Forcefully killing machine $name."
 252     xx_do_qemu_command "$name" "quit"
 253     sleep 1
 254
 255     if [ "$name" = "$XX_LAST_MACHINE" ]; then
 256         XX_LAST_MACHINE=""
 257     fi
 258 }
 259
 260
 261 ## Wait for text to appear on machine console.
 262 #
 263 # @param 1 Machine id.
 264 # @param 2 UNIX end-time (date %s + TIME_OUT).
 265 # @param 3 Text to match.
 266 xx_do_wait_for_text() {
 267     while true; do
 268         xx_activity "Taking screenshot, looking for '$3'."
 269
 270         xx_do_screenshot "$1" "$XX_TEMP/$1-full.ppm" \
 271             "$XX_TEMP/$1-term.png" "$XX_TEMP/$1-term.txt"
 272
 273         if grep -q "$3" <"$XX_TEMP/$1-term.txt"; then
 274             return 0
 275         fi
 276
 277         if [ `date +%s` -gt $2 ]; then
 278             return 1
 279         fi
 280
 281         sleep 1
 282     done
 283 }
 284
 285 xx_assert() {
 286     local timeout=`xx_get_var timeout $XX_DEFAULT_TIMEOUT "$@"`
 287     local machine=`xx_get_var machine $XX_LAST_MACHINE "$@"`
 288     local error_msg=`xx_get_var error "" "$@"`
 289     local text=`xx_get_def_var "$@"`
 290
 291     xx_echo "Checking that '$text' will appear on $machine within ${timeout}s."
 292
 293     if ! xx_do_wait_for_text "$machine" `xx_do_compute_timeout $timeout` "$text"; then
 294         xx_fatal "Failed to recognize '$text' on $machine."
 295     fi
 296 }
 297
 298 xx_die_on() {
 299     local timeout=`xx_get_var timeout $XX_DEFAULT_TIMEOUT "$@"`
 300     local machine=`xx_get_var machine $XX_LAST_MACHINE "$@"`
 301     local error_msg=`xx_get_var message "" "$@"`
 302     local text=`xx_get_def_var "$@"`
 303
 304     xx_echo "Checking that '$text' will not appear on $machine within ${timeout}s."
 305
 306     if xx_do_wait_for_text "$machine" `xx_do_compute_timeout $timeout` "$text"; then
 307         xx_fatal "Prohibited text '$text' spotted on $machine."
 308     fi
 309 }
 310
 311 xx_sleep() {
 312     local amount=`xx_get_def_var "$@"`
 313
 314     xx_echo "Waiting for ${amount}s."
 315     sleep $amount
 316 }
 317
 318
 319 xx_cls() {
 320     local machine=`xx_get_var machine $XX_LAST_MACHINE "$@"`
 321
 322     xx_echo "Clearing the screen on $machine."
 323     for i in `seq 1 35`; do
 324         xx_do_qemu_command "$machine" "sendkey ret"
 325     done
 326     sleep 1
 327 }
 328
 329 xx_do_ocr_prepare() {
 330     convert "$1" -crop "8x16" +repage +adjoin txt:- \
 331         | sed -e 's|[0-9]*,[0-9]*: ([^)]*)[ ]*#\([0-9A-Fa-f]\{6\}\).*|\1|' -e 's:^#.*:@:' -e 's#000000#0#g' -e 's#FFFFFF#F#' \
 332         | sed -e ':a' -e 'N;s#\n##;s#^@##;/@$/{s#@$##p;d}' -e 't a'
 333 }
 334
 335
 336 xx_do_ocr() {
 337     xx_do_ocr_prepare "$1" | sed -f ocr.sed
 338 }
 339
 340 xx_do_screenshot() {
 341     xx_do_qemu_command "$1" "screendump $2"
 342     if [ -n "$3" ]; then
 343         convert "$2" -crop 640x480+4+24 +repage -colors 2 -monochrome "$3"
 344         if [ -n "$4" ]; then
 345             xx_do_ocr "$3" | paste -sd '' | fold -w 80 >"$4"
 346         fi
 347     fi
 348 }
 349
 350 ## Wait for text to appear on machine console.
 351 #
 352 # @param 1 Machine id.
 353 # @param 2 UNIX end-time (date %s + TIME_OUT).
 354 # @param 3 Text that shall not be matched.
 355 # @param 4 Text that shall be matched before timeout.
 356 # @param 5 Consider prompt as a success.
 357 # @param 6 Check for standard Bdsh error messages.
 358 # @retval 0 Expected text was matched.
 359 # @retval 1 Prompt text was matched.
 360 # @retval 2 Time-out, nothing matched.
 361 # @retval 3 Unexpected text was matched.
 362 # @retval 4 Standard Bdsh error message detected.
 363 xx_do_complex_text_waiting() {
 364     xx_debug "waiting: $1/$2 match='$4', no-match='$3'"
 365     xx_debug "waiting: $1/$2 prompt_is_success=$5  check_for_bdsh_error=$6"
 366
 367     while true; do
 368         xx_activity "Taking screenshot, checking for specific output."
 369
 370         xx_do_screenshot "$1" "$XX_TEMP/$1-full.ppm" \
 371             "$XX_TEMP/$1-term.png" "$XX_TEMP/$1-term.txt"
 372
 373         if [ -n "$3" ] && grep -q "$3" <"$XX_TEMP/$1-term.txt"; then
 374             return 3
 375         fi
 376
 377         if [ -n "$4" ] && grep -q "$4" <"$XX_TEMP/$1-term.txt" ; then
 378                 return 0
 379             fi
 380
 381             if $5 && grep -q '^.*/ # _[ ]*$' <"$XX_TEMP/$1-term.txt" ; then
 382             return 0
 383         fi
 384
 385         if $6 && grep -q -e 'Cannot spawn' -e 'Command failed' <"$XX_TEMP/$1-term.txt"; then
 386             return 4
 387         fi
 388
 389         if [ `date +%s` -gt $2 ]; then
 390             return 2
 391         fi
 392
 393         sleep 1
 394     done
 395 }
 396
 397 xx_cmd() {
 398     local cmd=`xx_get_def_var "$@"`
 399     local timeout=`xx_get_var timeout $XX_DEFAULT_TIMEOUT "$@"`
 400     local machine=`xx_get_var machine $XX_LAST_MACHINE "$@"`
 401     local error_msg=`xx_get_var error "" "$@"`
 402     local text=`xx_get_var assert "" "$@"`
 403     local negtext=`xx_get_var die_on "" "$@"`
 404     local text_is_empty=`if [ -n "$text" ]; then echo false; else echo true; fi`
 405
 406     xx_echo "Sending '$cmd' to $machine."
 407     xx_do_type "$machine" "$cmd"
 408     xx_do_qemu_command "$machine" "sendkey ret"
 409
 410
 411     xx_do_complex_text_waiting "$machine" `xx_do_compute_timeout $timeout` \
 412         "$negtext" "$text" $text_is_empty true
 413     local res=$?
 414
 415     xx_debug "xx_do_complex_text_waiting = $res"
 416
 417     case $res in
 418         0|1)
 419             return 0
 420             ;;
 421         2)
 422             if $text_is_empty; then
 423                 xx_fatal "Command timed-out."
 424             else
 425                 xx_fatal "Failed to match '$text'."
 426             fi
 427             ;;
 428         3|4)
 429             if [ -n "$error_msg" ]; then
 430                 xx_fatal "$error_msg"
 431             else
 432                 xx_fatal "Command failed."
 433             fi
 434             ;;
 435         *)
 436             xx_fatal "Internal error, we shall never reach this line."
 437             ;;
 438     esac
 439 }
 440
 441
 442
 443 xx_do_print_help() {
 444     echo "Usage: $1 [options] scenario-file [scenarios-file ...]"
 445     cat <<'EOF_USAGE'
 446 where [options] are:
 447
 448 --help          Print this help and exit.
 449 --headless      Hide the screen of the virtual machine.
 450 --root=DIR      Find HelenOS image in the source directory.
 451 --image=FILE    File with main HelenOS image (specify --arch).
 452 --arch=ARCH     Architecture of the image file (see --image).
 453 --no-kvm        Do not try to run QEMU with KVM enabled.
 454 --fail-fast      Exit with first error.
 455 --debug         Print (a lot of) debugging messages.
 456
 457 EOF_USAGE
 458 }
 459
 460
 461 XX_DEBUG_ECHO=:
 462 XX_ACTIVITY_ECHO=:
 463 XX_TRAIN_OCR=false
 464 XX_HEADLESS=false
 465 XX_USE_KVM=true
 466 XX_TEMP="$PWD/tmp-vm/"
 467 XX_HELENOS_ROOT="."
 468 XX_AUTODETECT_HELENOS=false
 469 XX_ARCH=""
 470 XX_CDROM_FILE=""
 471 XX_FAIL_FAST=false
 472
 473 XX_KNOWN_MACHINES=""
 474 XX_DEFAULT_TIMEOUT=5
 475 XX_LAST_MACHINE=default
 476
 477
 478 # Replace with getopt eventually.
 479 while [ $# -gt 0 ]; do
 480     case "$1" in
 481         --headless)
 482             XX_HEADLESS=true
 483             ;;
 484         --train-ocr)
 485             XX_TRAIN_OCR=true
 486             ;;
 487         --train-ocr=*)
 488             XX_TRAIN_OCR=true
 489             XX_TRAIN_OCR_FILE=`echo "$1" | cut '-d=' -f 2-`
 490             ;;
 491         --debug)
 492             XX_DEBUG_ECHO=echo
 493             ;;
 494         --activity)
 495             XX_ACTIVITY_ECHO=echo
 496             ;;
 497         --root=*)
 498             XX_HELENOS_ROOT=`echo "$1" | cut '-d=' -f 2-`
 499             XX_AUTODETECT_HELENOS=true
 500             ;;
 501         --image=*)
 502             XX_CDROM_FILE=`echo "$1" | cut '-d=' -f 2-`
 503             XX_AUTODETECT_HELENOS=false
 504             ;;
 505         --arch=*)
 506             XX_ARCH=`echo "$1" | cut '-d=' -f 2-`
 507             XX_AUTODETECT_HELENOS=false
 508             ;;
 509         --temp=*)
 510             XX_TEMP=`echo "$1" | cut '-d=' -f 2-`
 511             ;;
 512         --no-kvm)
 513             XX_USE_KVM=false
 514             ;;
 515         --fail-fast)
 516             XX_FAIL_FAST=true
 517             ;;
 518         --help|-h|-?)
 519             xx_do_print_help "$0"
 520             exit 0
 521             ;;
 522         --*)
 523             xx_fatal "Unknown option $1."
 524             ;;
 525         *)
 526             break
 527             ;;
 528     esac
 529     shift
 530 done
 531
 532 mkdir -p "$XX_TEMP"
 533
 534 if $XX_AUTODETECT_HELENOS; then
 535     if ! [ -r "$XX_HELENOS_ROOT/Makefile.config" ]; then
 536         xx_fatal "Cannot open $XX_HELENOS_ROOT/Makefile.config."
 537     fi
 538     XX_ARCH=`grep '^PLATFORM = ' "$XX_HELENOS_ROOT/Makefile.config" | cut '-d=' -f 2- | tr -d ' '`
 539     XX_CDROM_FILE=$XX_HELENOS_ROOT/`cat "$XX_HELENOS_ROOT/defaults/$XX_ARCH/output"`
 540 fi
 541
 542
 543 if $XX_TRAIN_OCR; then
 544     if [ -z "$XX_TRAIN_OCR_FILE" ]; then
 545         echo "Usage notes for --train-ocr command-line switch."
 546         echo
 547         echo "Prepare HelenOS image.iso with train-ocr.txt in its root."
 548         echo
 549         echo "Run this script with the ISO and with --train-ocr=out.sed"
 550         echo "where out.sed is output file with OCR commands."
 551         echo
 552         echo "If the script finishes with  no error you can replace the"
 553         echo "old OCR scheme (ocr.sed) with the new one in out.sed."
 554         echo
 555         exit
 556     fi
 557
 558     # Set to false to debug the script below without running QEMU
 559     # all the time
 560     if true; then
 561             xx_start_machine name=ocr vterm=false
 562             xx_echo "Will display the training set on the VM screen"
 563             xx_echo "(this will take a while)."
 564             sleep 20
 565             xx_do_type ocr "cat train-ocr.txt"
 566             sleep 1
 567             xx_do_qemu_command ocr "sendkey ret"
 568             sleep 5
 569             xx_do_screenshot ocr "$XX_TEMP/ocr-full.ppm" "$XX_TEMP/ocr-term.png"
 570             xx_stop_machine
 571     fi
 572
 573     xx_echo "Doing OCR..."
 574     xx_do_ocr_prepare "$XX_TEMP/ocr-term.png" \
 575         | (
 576         prev1_line=""
 577         prev2_line=""
 578         prev3_line=""
 579         counter=0
 580         while read current_line; do
 581             if [ $counter -gt 5 ]; then
 582                 while read current_line; do
 583                     if ! [ "$current_line" = "$prev1_line" -o "$current_line" = "$prev2_line" ]; then
 584                         break
 585                     fi
 586                 done
 587                 alphabet=`sed -n 's#^\(.\)\(.\)\(\1\2\)\{4,\}##p' train-ocr.txt`
 588                 alphabet_size=`echo "$alphabet" | wc -c`
 589                 for i in `seq 1 $(( $alphabet_size - 1))`; do
 590                     symbol="`echo \"$alphabet\" | fold -w 1 | sed -n \"${i}p\" | sed 's#[*\.\&\\:\/\[]\|\]#\\\\&#g'`"
 591                     echo "s:$current_line:$symbol:"
 592                     read current_line
 593                 done
 594                 echo "$current_line" | tr 'F' '0' | sed 's#.*#s:&:_:#'
 595                 echo "$current_line" | tr '0F' '.' | sed 's#.*#s:&:?:#'
 596                 break
 597             fi
 598             if [ "$current_line" = "$prev2_line" -a "$prev1_line" = "$prev3_line" -a "$prev1_line" != "$prev2_line" ]; then
 599                 counter=$(( $counter + 1 ))
 600             else
 601                 counter=0
 602             fi
 603             prev3_line="$prev2_line"
 604             prev2_line="$prev1_line"
 605             prev1_line="$current_line"
 606         done >"$XX_TRAIN_OCR_FILE"
 607         )
 608     xx_echo "New OCR scheme is in $XX_TRAIN_OCR_FILE."
 609     exit
 610 fi
 611
 612
 613 XX_RESULT_SUMMARY="$XX_TEMP/summary.$$.txt"
 614
 615 date '+# Execution started on %Y-%m-%d %H:%M.' >"$XX_RESULT_SUMMARY"
 616 echo '# =======================================' >>"$XX_RESULT_SUMMARY"
 617
 618
 619 # Run it
 620 for i in "$@"; do
 621     echo "# Starting scenario $i..."
 622     (
 623         . $i
 624     )
 625     if [ $? -eq 0 ]; then
 626         res="OK"
 627     else
 628         res="FAIL"
 629     fi
 630     echo "# Scenario $i terminated, $res."
 631     printf '%-35s  %4s\n' "$i" "$res" >>"$XX_RESULT_SUMMARY"
 632     if $FAIL_FAST && [ "$res" = "FAIL" ]; then
 633         exit 1
 634     fi
 635 done
 636
 637
 638 date '+# Execution finished on %Y-%m-%d %H:%M.' >>"$XX_RESULT_SUMMARY"
 639
 640 # Display the results.
 641 echo
 642
 643 cat "$XX_RESULT_SUMMARY"