tests/test.sh

   1 #!/usr/bin/env bash
   2 #  Copyright (C)              Lumiera.org
   3 #    2007, 2008, 2009, 2010,  Christian Thaeter <ct@pipapo.org>
   4 #
   5 #  This program is free software; you can redistribute it and/or
   6 #  modify it under the terms of the GNU General Public License as
   7 #  published by the Free Software Foundation; either version 2 of the
   8 #  License, or (at your option) any later version.
   9 #
  10 #  This program is distributed in the hope that it will be useful,
  11 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 #  GNU General Public License for more details.
  14 #
  15 #  You should have received a copy of the GNU General Public License
  16 #  along with this program; if not, write to the Free Software
  17 #  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  18
  19 # TESTMODE=FULL yet unimplemented
  20 #   run all tests, PLANNED which fail count as error
  21 #
  22 # TESTMODE=FAST
  23 #   run only tests which recently failed
  24 #
  25 # TESTMODE=FIRSTFAIL
  26 #   stop testing on the first failure
  27
  28 #intro Test.sh
  29 #intro =======
  30 #intro Christian Thäter
  31 #intro
  32 #intro A shell script driving software tests.
  33 #intro
  34
  35 # doc generation disabled here (temporary) for inclusion in the nobug manual
  36 # =intro
  37 # =tests Writing tests
  38 #  =run
  39 # =config
  40 # =configf
  41 #  =make
  42 #  =control
  43 # =valgrind
  44 # =libtool
  45 # _
  46 # _ Index
  47 # _ -----
  48 # _
  49 # =index
  50 # _
  51
  52 LC_ALL=C
  53
  54 #config HEAD~ Test Configuration; configuration, tests; configure tests
  55 #config
  56 #config Runtime behaviour of the testsuite can be configured by setting certain
  57 #config envirinment variables.
  58 #config
  59 #config PARA LOGSUPPRESS; LOGSUPPRESS; suppress certain lines from stderr
  60 #config
  61 #config  LOGSUPPRESS='^[0-9]\{10,\}: \(TRACE\|INFO\|NOTICE\):'
  62 #config
  63 #config Programms sometimes emit additional diagnostics on stderr which is volatile and not necessary for
  64 #config validating the output the `LOGSUPRESS` variable can be set to a regex to filter this things out.
  65 #config The default as shown above filters some NoBug annotations and non fatal logging out.
  66 #config
  67 LOGSUPPRESS='^\(\*\*[0-9]*\*\* \)\?[0-9]\{10,\}[:!] \(TRACE\|INFO\|NOTICE\|WARNING\|ERR\):'
  68
  69 #config HEAD^ Resource Limits; ulimit, tests; constrain test resource limits
  70 #config
  71 #config It is possible to set some limits for tests to protect the system against really broken cases.
  72 #config Since running under valgrind takes consinderable more resources there are separate variants for
  73 #config limits when running under valgrind.
  74 #config INDEX LIMIT_CPU; LIMIT_CPU; limit the cpu time
  75 #config
  76 #config  LIMIT_CPU=5
  77 #config
  78 #config Maximal CPU time the test may take after it will be killed with SIGXCPU. This protects agaist Lifelocks.
  79 #config INDEX LIMIT_TIME; LIMIT_TIME; limit the wall time
  80 #config
  81 #config  LIMIT_TIME=10
  82 #config
  83 #config Maximal wall-time a test may take after this it will be killed with SIGKILL. Protects against Deadlocks.
  84 #config INDEX LIMIT_VSZ; LIMIT_VSZ; limit virtual memory size
  85 #config
  86 #config  LIMIT_VSZ=524288
  87 #config
  88 #config Maximal virtual memory size the process may map, allocations/mappings will fail when this limit is reached.
  89 #config Protects against memory leaks.
  90 #config INDEX LIMIT_VG_*; LIMIT_VG_*; lmitation when running tests under valgrind
  91 #config
  92 #config  LIMIT_VG_CPU=20
  93 #config  LIMIT_VG_TIME=30
  94 #config  LIMIT_VG_VSZ=524288
  95 #config
  96 #config Same variables again with limits when running under valgrind.
  97 #config
  98 LIMIT_CPU=5
  99 LIMIT_TIME=10
 100 LIMIT_VSZ=524288
 101 LIMIT_VG_CPU=20
 102 LIMIT_VG_TIME=30
 103 LIMIT_VG_VSZ=524288
 104
 105
 106 #configf HEAD~ Configuration Files; configuration files; define variables to configure the test
 107 #configf
 108 #configf `test.sh` reads config files from the following location if they are exist
 109 #configf
 110 #configf  . 'test.conf' from the current directory
 111 #configf  . '$srcdir/test.conf' `$srcdir` is set by autotools
 112 #configf  . '$srcdir/tests/test.conf' `tests/` is suspected as default directory for tests
 113 #configf  . '$TEST_CONF' a user defineable variable to point to a config file
 114 #configf
 115 test -f 'test.conf' && source test.conf
 116 test -n "$srcdir" -a -e "$srcdir/test.conf" && source "$srcdir/test.conf"
 117 test -n "$srcdir" -a -e "$srcdir/tests/test.conf" && source "$srcdir/tests/test.conf"
 118 test -n "$TEST_CONF" -a -e "$TEST_CONF" && source "$TEST_CONF"
 119
 120
 121
 122 arg0="$0"
 123 TESTDIR="$(dirname "$arg0")"
 124
 125
 126 #libtool HEAD Libtool; libtool; support for libtool
 127 #libtool
 128 #libtool When test.sh detects the presence of './libtool' it runs all tests with
 129 #libtool `./libtool --mode=execute`.
 130 #libtool
 131 LIBTOOL_EX=
 132 if test -x ./libtool; then
 133     LIBTOOL_EX="./libtool --mode=execute"
 134 fi
 135
 136 #valgrind HEAD~ Valgrind; valgrind; valgrind support
 137 #valgrind
 138 #valgrind Test are run under valgrind supervision by default, if not disabled.
 139 #valgrind
 140 #valgrind PARA VALGRINDFLAGS; VALGRINDFLAGS; control valgrind options
 141 #valgrind
 142 #valgrind  VALGRINDFLAGS="--leak-check=yes --show-reachable=yes"
 143 #valgrind
 144 #valgrind `VALGRINDFLAGS` define the options which are passed to valgrind. This can be used to override
 145 #valgrind the defaults or switching the valgrind tool. The special case `VALGRINDFLAGS=DISABLE` will disable
 146 #valgrind valgrind for the tests.
 147 #valgrind
 148 #valgrind HEAD^ Generating Valgrind Suppression Files; vgsuppression; ignore false positives
 149 #valgrind
 150 #valgrind When there is a 'vgsuppression' executable in the current dir (build by something external) then
 151 #valgrind test.sh uses this to generate a local 'vgsuppression.supp' file and uses that to suppress
 152 #valgrind all errors generated by 'vgsuppression'. The Idea here is that one adds code which triggers known
 153 #valgrind false positives in 'vgsuppression'. Care must be taken that this file is simple and does
 154 #valgrind not generate true positives.
 155 #valgrind
 156 ulimit -S -t ${LIMIT_CPU:-5} -v ${LIMIT_VSZ:-524288}
 157 valgrind=""
 158 LIMIT_TIME_REAL="$LIMIT_TIME"
 159 if [ "$VALGRINDFLAGS" = 'DISABLE' ]; then
 160     echo "valgrind explicit disabled"
 161 else
 162     if [ "$(which valgrind)" ]; then
 163         ulimit -S -t ${LIMIT_VG_CPU:-20} -v ${LIMIT_VG_VSZ:-524288}
 164         LIMIT_TIME_REAL="$LIMIT_VG_TIME"
 165         if [[ -x 'vgsuppression' ]]; then
 166             if [[ 'vgsuppression' -nt 'vgsuppression.supp' ]]; then
 167                 echo 'generating valgrind supression file'
 168
 169                 $LIBTOOL_EX $(which valgrind) ${VALGRINDFLAGS:---leak-check=yes --show-reachable=yes} -q --gen-suppressions=all vgsuppression 2>&1 \
 170                     | awk '/^{/ {i = 1;} /^}/ {i = 0; print $0;} {if (i == 1) print $0;}' >vgsuppression.supp
 171             fi
 172             valgrind="$(which valgrind) ${VALGRINDFLAGS:---leak-check=yes --show-reachable=no} --suppressions=vgsuppression.supp -q"
 173         else
 174             valgrind="$(which valgrind) ${VALGRINDFLAGS:---leak-check=yes --show-reachable=no -q}"
 175         fi
 176     else
 177         echo "no valgrind found, go without it"
 178     fi
 179 fi
 180
 181 echo
 182 echo "================ ${0##*/} ================"
 183
 184 TESTCNT=0
 185 SKIPCNT=0
 186 FAILCNT=0
 187
 188
 189 # the old testlog if existing will be used to check for previous test states
 190 if test -f ,testlog; then
 191     mv ,testlog ,testlog.pre
 192 else
 193     touch ,testlog.pre
 194 fi
 195
 196 date >,testlog
 197
 198 function compare_template() # template plainfile
 199 {
 200     local template
 201     local line
 202     local miss
 203     local lineno=1
 204     local templateno=1
 205     {
 206         IFS='' read -u 3 -r template || return 0
 207         IFS='' read -u 4 -r line || { echo "no output"; return 1; }
 208         while true; do
 209             local cmd="${template%%:*}:"
 210             local arg="${template#*: }"
 211
 212             case $cmd in
 213             'regex_cont:')
 214                 if [[ $line =~ $arg ]]; then
 215                     IFS='' read -u 4 -r line ||
 216                     if IFS='' read -u 3 -r template; then
 217                         echo "premature end in output, expecting $template:$templateno"
 218                         return 1
 219                     else
 220                         return 0
 221                     fi
 222                     : $((++lineno))
 223                     miss=0
 224                 else
 225                     if [[ $((++miss)) -gt 1 ]]; then
 226                         echo -e "'$line':$lineno\ndoes not match\n$template:$templateno"
 227                         return 1
 228                     fi
 229                     IFS='' read -u 3 -r template || { echo "more output than expected: '$line':$lineno"; return 1; }
 230                     : $((++templateno))
 231                 fi
 232                 ;;
 233             'literal:')
 234                 if [[ "$line" = "$arg" ]]; then
 235                     IFS='' read -u 3 -r template && IFS='' read -u 4 -r line || {
 236                         return 0
 237                     }
 238                 else
 239                     echo -e "'$line':$lineno\ndoes not match\n$template:$templateno"
 240                     return 1
 241                 fi
 242                 ;;
 243             *)
 244                 echo "UNKOWN MATCH COMMAND '$cmd'" 1>&2
 245                 exit
 246                 ;;
 247             esac
 248         done
 249     } 3<"$1" 4<"$2"
 250 }
 251
 252 #tests HEAD- Testing; testing; how to write testsuites
 253 #tests
 254 #tests Tests are nothing more than bash scripts with some functions from the test.sh
 255 #tests framework defined. Test.sh looks in the current directory for all files which ending in .test
 256 #tests and runs them in alphabetical order. The selection of this tests can be constrained with the
 257 #tests `TESTSUITES` environment variable.
 258 #tests
 259 #tests HEAD~ Testsuites; test files; writing tests
 260 #tests
 261 #tests It is common to start the name of the '.test' files with a 2 digit number to give them a proper
 262 #tests order: '10foo.test', '20bar.test' and so on. Each such test should only test a certain aspect of
 263 #tests the system. You have to select the testing binary with the `TESTING` function and then write
 264 #tests certain TEST's defining how the test should react. Since tests are shell scripts it is possible
 265 #tests to add some supplemental commands there to set and clean up the given test environment.
 266 #tests
 267 #tests PARA Select Test Executable; TESTING; set the test binary
 268 #tests
 269 #tests  TESTING "message" test_program
 270 #tests
 271 #tests Selects the test binary for the follwing tests, prints an informal message.
 272 #tests
 273 #tests  `message`::
 274 #tests    message to be printed
 275 #tests  `test_program`::
 276 #tests    an existing program to drive the tests or a shell function
 277 #tests
 278 function TESTING()
 279 {
 280     echo
 281     echo "$1"
 282     echo -e "\n#### $1, $TESTFILE, $2" >>,testlog
 283
 284     TESTBIN="$2"
 285 }
 286
 287 #tests PARA Define a Test; TEST; single test
 288 #tests
 289 #tests  TEST "title" arguments.. <<END
 290 #tests
 291 #tests Defines a single test
 292 #tests
 293 #tests  `title`::
 294 #tests    describes this test and is also used as identifier for this test,
 295 #tests    must be unique for all your tests
 296 #tests  `arguments`::
 297 #tests    the following arguments are passed to the test program
 298 #tests  `<<END .. END`::
 299 #tests    a list of control commands expected in and outputs is given as 'heredoc'.
 300 #tests
 301 #tests Each line of the test specification in the heredoc starts with an arbitary number of spaces
 302 #tests followed by a command, followed by a colon and a space, followed by additional arguments or
 303 #tests being an empty or comment line.
 304 #tests
 305 #testcmds HEAD^ Test Commands; commands; define expected in and outputs
 306 #testcmds
 307 #testcmds
 308 #testcmds
 309 #testcmds
 310 #testcmds PARA in; in; stdin data for a test
 311 #testcmds
 312 #testcmds  in: text
 313 #testcmds
 314 #testcmds Send `text` to stdin of the test binary. If no `in:` commands are given, nothing is send to the
 315 #testcmds tests input.
 316 #testcmds
 317 #testcmds PARA out; out; expected stdout (regex) from a test
 318 #testcmds
 319 #testcmds  out: regex
 320 #testcmds
 321 #testcmds Expect `regex` on stdout. This regexes have a 'triggering' semantic. That means it is tried to match
 322 #testcmds a given regex on as much lines as possible (`.*` will match any remaining output), if the match fails,
 323 #testcmds the next expected output line is tried. When that fails too the test is aborted and counted as failure.
 324 #testcmds
 325 #testcmds When no `out:` or `out-lit:` commands are given, then stdout is not checked, any output is ignored.
 326 #testcmds
 327 #testcmds PARA err; err; expected stderr (regex) from a test
 328 #testcmds
 329 #testcmds  out: regex
 330 #testcmds
 331 #testcmds Same as 'out:' but expects data on stderr. When no `err:` or `err-lit:` commands are given, then stdout is
 332 #testcmds not checked, any output there is ignored.
 333 #testcmds
 334 #testcmds PARA out-lit; out-lit; expected stdout (literal) from a test
 335 #testcmds
 336 #testcmds  out-lit: text
 337 #testcmds
 338 #testcmds Expect `text` on stdout, must match exactly or will fail.
 339 #testcmds
 340 #testcmds PARA err-lit; err-lit; expected stderr (literal) from a test
 341 #testcmds
 342 #testcmds  err-lit: text
 343 #testcmds
 344 #testcmds Same as 'out-lit:' but expects data on stderr.
 345 #testcmds
 346 #testcmds PARA return; return; expected exit value of a test
 347 #testcmds
 348 #testcmds  return: value
 349 #testcmds
 350 #testcmds Expects `value` as exit code of the tested program. The check can be negated by prepending the value with
 351 #testcmds an exclamation mark, `return: !0` expects any exist code except zero.
 352 #testcmds
 353 #testcmds If no `return:` command is given then a zero (success) return from the test program is expected.
 354 #testcmds
 355 #testcmds HEAD^ Conditional Tests; conditional tests; switch tests on conditions
 356 #testcmds
 357 #testcmds Sometimes tests need to be adapted to the environment/platform they are running on. This can be archived
 358 #testcmds with common if-else-elseif-endif statements. This statements can be nested.
 359 #testcmds
 360 #testcmds PARA if; if; conditional test
 361 #testcmds
 362 #testcmds  if: check
 363 #testcmds
 364 #testcmds Executes `check` as shell command, if its return is zero (success) then the following test parts are used.
 365 #testcmds
 366 #testcmds PARA else; else; conditional alternative
 367 #testcmds
 368 #testcmds  else:
 369 #testcmds
 370 #testcmds If the previous `if` failed then the following test parts are included in the test, otherwise they
 371 #testcmds are excluded.
 372 #testcmds
 373 #testcmds PARA elseif; elseif; conditional alternative with test
 374 #testcmds
 375 #testcmds  elseif: check
 376 #testcmds
 377 #testcmds Composition of else and if, only includes the following test parts if the if's and elseif's before failed
 378 #testcmds and `check` succeeded.
 379 #testcmds
 380 #testcmds PARA endif; endif; end of conditonal test part
 381 #testcmds
 382 #testcmds  endif:
 383 #testcmds
 384 #testcmds Ends an `if` statement.
 385 #testcmds
 386 #testcmds HEAD^ Other Elements;;
 387 #testcmds
 388 #testcmds PARA msg; msg; print a diagnostic message
 389 #testcmds
 390 #testcmds  msg: message..
 391 #testcmds
 392 #testcmds Prints `message` while processing the test suite.
 393 #testcmds
 394 #testcmds PARA comments; comments; adding comments to tests
 395 #testcmds
 396 #testcmds  # anything
 397 #testcmds
 398 #testcmds Lines starting with the hash mark and empty lines count as comment and are not used.
 399 #testcmds
 400 function TEST()
 401 {
 402         name="$1"
 403         shift
 404         rm -f ,send_stdin
 405         rm -f ,expect_stdout
 406         rm -f ,expect_stderr
 407         expect_return=0
 408
 409         local valgrind="$valgrind"
 410         if [ "$VALGRINDFLAGS" = 'DISABLE' ]; then
 411             valgrind=
 412         fi
 413
 414         local condstack="1"
 415         while read -r line; do
 416             local cmd="${line%%:*}:"
 417             local arg="${line#*: }"
 418
 419             if [[ ! "$arg" ]]; then
 420                 arg='^$'
 421             fi
 422
 423             case $cmd in
 424             'if:')
 425                 if $arg; then
 426                     condstack="1$condstack"
 427                 else
 428                     condstack="0$condstack"
 429                 fi
 430                 ;;
 431             'elseif:')
 432                 if [[ "${condstack:0:1}" = "0" ]]; then
 433                     if $arg; then
 434                         condstack="1${condstack:1}"
 435                     else
 436                         condstack="0${condstack:1}"
 437                     fi
 438                 else
 439                     condstack="2${condstack:1}"
 440                 fi
 441                 ;;
 442             'else:')
 443                 if [[ "${condstack:0:1}" != "0" ]]; then
 444                     condstack="0${condstack:1}"
 445                 else
 446                     condstack="1${condstack:1}"
 447                 fi
 448                 ;;
 449             'endif:')
 450                 condstack="${condstack:1}"
 451                 ;;
 452             *)
 453                 if [[ "${condstack:0:1}" = "1" ]]; then
 454                     case $cmd in
 455                     'msg:')
 456                         echo "MSG $arg"
 457                         ;;
 458                     'in:')
 459                         echo "$arg" >>,send_stdin
 460                         ;;
 461                     'out:')
 462                         echo "regex_cont: $arg" >>,expect_stdout
 463                         ;;
 464                     'err:')
 465                         echo "regex_cont: $arg" >>,expect_stderr
 466                         ;;
 467                     'out-lit:')
 468                         echo "literal: $arg" >>,expect_stdout
 469                         ;;
 470                     'err-lit:')
 471                         echo "literal: $arg" >>,expect_stderr
 472                         ;;
 473                     'return:')
 474                         expect_return="$arg"
 475                         ;;
 476                     '#'*|':')
 477                         :
 478                         ;;
 479                     *)
 480                         echo "UNKOWN TEST COMMAND '$cmd'" 1>&2
 481                         exit
 482                         ;;
 483                     esac
 484                 fi
 485                 ;;
 486             esac
 487         done
 488         echo -n "TEST $name: "
 489         echo -en "\nTEST $name: $* " >>,testlog
 490
 491         case "$TESTMODE" in
 492         *FAST*)
 493             if grep "^TEST $name: .* FAILED" ,testlog.pre >&/dev/null; then
 494                 MSGOK=" (fixed)"
 495                 MSGFAIL=" (still broken)"
 496             elif grep "^TEST $name: .* \\(SKIPPED (ok)\\|OK\\)" ,testlog.pre >&/dev/null; then
 497                 echo ".. SKIPPED (ok)"
 498                 echo ".. SKIPPED (ok)" >>,testlog
 499                 SKIPCNT=$(($SKIPCNT + 1))
 500                 TESTCNT=$(($TESTCNT + 1))
 501                 return
 502             else
 503                 MSGOK=" (new)"
 504                 MSGFAIL=" (new)"
 505             fi
 506             ;;
 507         *)
 508             MSGOK=""
 509             MSGFAIL=""
 510             ;;
 511         esac
 512
 513         TESTCNT=$(($TESTCNT + 1))
 514
 515         fails=0
 516
 517         echo -n >,testtmp
 518
 519         local CALL
 520         if declare -F | grep $TESTBIN >&/dev/null; then
 521             CALL=
 522         elif test -x $TESTBIN; then
 523             CALL="env $LIBTOOL_EX $valgrind"
 524         else
 525             CALL='-'
 526             echo -n >,stdout
 527             echo "test binary '$TESTBIN' not found" >,stderr
 528             ((fails+=1))
 529         fi
 530
 531         if test "$CALL" != '-'; then
 532             if test -f ,send_stdin; then
 533                 (
 534                     $CALL $TESTBIN "$@" <,send_stdin 2>,stderr >,stdout
 535                     echo $? >,return
 536                 ) &
 537             else
 538                 (
 539                     $CALL $TESTBIN "$@" 2>,stderr >,stdout
 540                     echo $? >,return
 541                 ) &
 542             fi &>/dev/null
 543             pid=$!
 544
 545             # watchdog
 546             ( sleep $LIMIT_TIME_REAL && kill -KILL $pid ) &>/dev/null &
 547             wpid=$!
 548             wait $pid
 549             return=$(<,return)
 550             if [[ "$return" -le 128 ]]; then
 551                 kill -INT $wpid >&/dev/null
 552             fi
 553
 554             if test -f ,expect_stdout; then
 555                 grep -v "$LOGSUPPRESS" <,stdout >,tmp
 556                 if ! compare_template ,expect_stdout ,tmp >>,cmptmp; then
 557                     echo "unexpected data on stdout" >>,testtmp
 558                     cat ,cmptmp >>,testtmp
 559                     ((fails+=1))
 560                 fi
 561                 rm ,tmp ,cmptmp
 562             fi
 563
 564             if test -f ,expect_stderr; then
 565                 grep -v "$LOGSUPPRESS" <,stderr >,tmp
 566                     cat ,tmp >>,testtmp
 567                 if ! compare_template ,expect_stderr ,tmp >>,cmptmp; then
 568                     echo "unexpected data on stderr" >>,testtmp
 569                     cat ,cmptmp >>,testtmp
 570                     ((fails+=1))
 571                 fi
 572                 rm ,tmp ,cmptmp
 573             fi
 574
 575             if [[ "${expect_return:0:1}" = '!' ]]; then
 576                 if [[ "${expect_return#\!}" = "$return" ]]; then
 577                     echo "unexpected return value $return, expected $expect_return" >>,testtmp
 578                     ((fails+=1))
 579                 fi
 580             else
 581                 if [[ "${expect_return}" != "$return" ]]; then
 582                     echo "unexpected return value $return, expected $expect_return" >>,testtmp
 583                     ((fails+=1))
 584                 fi
 585             fi
 586         fi
 587
 588         if test $fails -eq 0; then
 589             echo ".. OK$MSGOK"
 590             echo ".. OK$MSGOK" >>,testlog
 591         else
 592             echo ".. FAILED$MSGFAIL";
 593             echo ".. FAILED$MSGFAIL" >>,testlog
 594             cat ,testtmp >>,testlog
 595             rm ,testtmp
 596             echo "stderr was:" >>,testlog
 597             cat ,stderr >>,testlog
 598             echo END >>,testlog
 599             FAILCNT=$(($FAILCNT + 1))
 600             case $TESTMODE in
 601             *FIRSTFAIL*)
 602                 break 2
 603                 ;;
 604             esac
 605         fi
 606 }
 607
 608 #tests PARA Planned Tests; PLANNED; deactivated test
 609 #tests
 610 #tests  PLANNED "title" arguments.. <<END
 611 #tests
 612 #tests Skip a single test.
 613 #tests
 614 #tests  `title`::
 615 #tests    describes this test and is also used as identifier for this test,
 616 #tests    must be unique for all your tests
 617 #tests  `arguments`::
 618 #tests    the following arguments are passed to the test program
 619 #tests  `<<END .. END`::
 620 #tests    a list of control commands expected in and outputs is given as 'heredoc'.
 621 #tests
 622 #tests `PLANNED` acts as dropin replacement for `TEST`. Each such test is skipped (and counted as skipped)
 623 #tests This can be used to specify tests in advance and activate them as soon development goes on or
 624 #tests deactivate intentional broken tests to be fixed later.
 625 #tests
 626 function PLANNED()
 627 {
 628         echo -n "PLANNED $1: "
 629         echo -en "\nPLANNED $* " >>,testlog
 630         echo ".. SKIPPED (planned)"
 631         echo ".. SKIPPED (planned)" >>,testlog
 632         SKIPCNT=$(($SKIPCNT + 1))
 633         TESTCNT=$(($TESTCNT + 1))
 634 }
 635
 636 function RUNTESTS()
 637 {
 638     if test \( ! "${TESTSUITES/*,*/}" \) -a "$TESTSUITES"; then
 639         TESTSUITES="{$TESTSUITES}"
 640     fi
 641     for t in $(eval echo "$TESTDIR/*$TESTSUITES*.tests"); do
 642         echo "$t"
 643     done | sort | uniq | {
 644         while read TESTFILE; do
 645             echo
 646             echo "### $TESTFILE" >&2
 647             if test -f $TESTFILE; then
 648                 source $TESTFILE
 649             fi
 650         done
 651         echo
 652         if [ $FAILCNT = 0 ]; then
 653             echo " ... PASSED $(($TESTCNT - $SKIPCNT)) TESTS, $SKIPCNT SKIPPED"
 654             #rm ,testlog
 655         else
 656             echo " ... SUCCEEDED $(($TESTCNT - $FAILCNT - $SKIPCNT)) TESTS"
 657             echo " ... FAILED $FAILCNT TESTS"
 658             echo " ... SKIPPED $SKIPCNT TESTS"
 659             echo " see ',testlog' for details"
 660             exit 1
 661         fi
 662     }
 663 }
 664
 665 TESTSUITES="${TESTSUITES}${1:+${TESTSUITES:+,}$1}"
 666
 667 RUNTESTS