| blob | 8525b220c7ca85a97ca3424afd89eed00c338419 |
1 #
2 # Automated Testing Framework (atf)
3 #
4 # Copyright (c) 2007 The NetBSD Foundation, Inc.
5 # All rights reserved.
6 #
7 # Redistribution and use in source and binary forms, with or without
8 # modification, are permitted provided that the following conditions
9 # are met:
10 # 1. Redistributions of source code must retain the above copyright
11 # notice, this list of conditions and the following disclaimer.
12 # 2. Redistributions in binary form must reproduce the above copyright
13 # notice, this list of conditions and the following disclaimer in the
14 # documentation and/or other materials provided with the distribution.
15 #
16 # THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND
17 # CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
18 # INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
19 # MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 # IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY
21 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
23 # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24 # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
25 # IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
26 # OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
27 # IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 #
30 set -e
32 # ------------------------------------------------------------------------
33 # GLOBAL VARIABLES
34 # ------------------------------------------------------------------------
36 # Values for the expect property.
37 Expect=pass
38 Expect_Reason=
40 # A boolean variable that indicates whether we are parsing a test case's
41 # head or not.
42 Parsing_Head=false
44 # The program name.
45 Prog_Name=${0##*/}
47 # The file to which the test case will print its result.
48 Results_File=
50 # The test program's source directory: i.e. where its auxiliary data files
51 # and helper utilities can be found. Can be overriden through the '-s' flag.
52 Source_Dir="$(dirname ${0})"
54 # Indicates the test case we are currently processing.
55 Test_Case=
57 # List of meta-data variables for the current test case.
58 Test_Case_Vars=
60 # The list of all test cases provided by the test program.
61 Test_Cases=
63 # ------------------------------------------------------------------------
64 # PUBLIC INTERFACE
65 # ------------------------------------------------------------------------
67 #
68 # atf_add_test_case tc-name
69 #
70 # Adds the given test case to the list of test cases that form the test
71 # program. The name provided here must be accompanied by two functions
72 # named after it: <tc-name>_head and <tc-name>_body, and optionally by
73 # a <tc-name>_cleanup function.
74 #
75 atf_add_test_case()
76 {
77 Test_Cases="${Test_Cases} ${1}"
78 }
80 #
81 # atf_check cmd expcode expout experr
82 #
83 # Executes atf-check with given arguments and automatically calls
84 # atf_fail in case of failure.
85 #
86 atf_check()
87 {
88 ${Atf_Check} "${@}" || \
89 atf_fail "atf-check failed; see the output of the test for details"
90 }
92 #
93 # atf_check_equal expr1 expr2
94 #
95 # Checks that expr1's value matches expr2's and, if not, raises an
96 # error. Ideally expr1 and expr2 should be provided quoted (not
97 # expanded) so that the error message is helpful; otherwise it will
98 # only show the values, not the expressions themselves.
99 #
100 atf_check_equal()
101 {
102 eval _val1=\"${1}\"
103 eval _val2=\"${2}\"
104 test "${_val1}" = "${_val2}" || \
105 atf_fail "${1} != ${2} (${_val1} != ${_val2})"
106 }
108 #
109 # atf_config_get varname [defvalue]
110 #
111 # Prints the value of a configuration variable. If it is not
112 # defined, prints the given default value.
113 #
114 atf_config_get()
115 {
116 _varname="__tc_config_var_$(_atf_normalize ${1})"
117 if [ ${#} -eq 1 ]; then
118 eval _value=\"\${${_varname}-__unset__}\"
119 [ "${_value}" = __unset__ ] && \
120 _atf_error 1 "Could not find configuration variable \`${1}'"
121 echo ${_value}
122 elif [ ${#} -eq 2 ]; then
123 eval echo \${${_varname}-${2}}
124 else
125 _atf_error 1 "Incorrect number of parameters for atf_config_get"
126 fi
127 }
129 #
130 # atf_config_has varname
131 #
132 # Returns a boolean indicating if the given configuration variable is
133 # defined or not.
134 #
135 atf_config_has()
136 {
137 _varname="__tc_config_var_$(_atf_normalize ${1})"
138 eval _value=\"\${${_varname}-__unset__}\"
139 [ "${_value}" != __unset__ ]
140 }
142 #
143 # atf_expect_death reason
144 #
145 # Sets the expectations to 'death'.
146 #
147 atf_expect_death()
148 {
149 _atf_validate_expect
151 Expect=death
152 _atf_create_resfile "expected_death: ${*}"
153 }
155 #
156 # atf_expect_timeout reason
157 #
158 # Sets the expectations to 'timeout'.
159 #
160 atf_expect_timeout()
161 {
162 _atf_validate_expect
164 Expect=timeout
165 _atf_create_resfile "expected_timeout: ${*}"
166 }
168 #
169 # atf_expect_exit exitcode reason
170 #
171 # Sets the expectations to 'exit'.
172 #
173 atf_expect_exit()
174 {
175 _exitcode="${1}"; shift
177 _atf_validate_expect
179 Expect=exit
180 if [ "${_exitcode}" = "-1" ]; then
181 _atf_create_resfile "expected_exit: ${*}"
182 else
183 _atf_create_resfile "expected_exit(${_exitcode}): ${*}"
184 fi
185 }
187 #
188 # atf_expect_fail reason
189 #
190 # Sets the expectations to 'fail'.
191 #
192 atf_expect_fail()
193 {
194 _atf_validate_expect
196 Expect=fail
197 Expect_Reason="${*}"
198 }
200 #
201 # atf_expect_pass
202 #
203 # Sets the expectations to 'pass'.
204 #
205 atf_expect_pass()
206 {
207 _atf_validate_expect
209 Expect=pass
210 Expect_Reason=
211 }
213 #
214 # atf_expect_signal signo reason
215 #
216 # Sets the expectations to 'signal'.
217 #
218 atf_expect_signal()
219 {
220 _signo="${1}"; shift
222 _atf_validate_expect
224 Expect=signal
225 if [ "${_signo}" = "-1" ]; then
226 _atf_create_resfile "expected_signal: ${*}"
227 else
228 _atf_create_resfile "expected_signal(${_signo}): ${*}"
229 fi
230 }
232 #
233 # atf_expected_failure msg1 [.. msgN]
234 #
235 # Makes the test case report an expected failure with the given error
236 # message. Multiple words can be provided, which are concatenated with
237 # a single blank space.
238 #
239 atf_expected_failure()
240 {
241 _atf_create_resfile "expected_failure: ${Expect_Reason}: ${*}"
242 exit 0
243 }
245 #
246 # atf_fail msg1 [.. msgN]
247 #
248 # Makes the test case fail with the given error message. Multiple
249 # words can be provided, in which case they are joined by a single
250 # blank space.
251 #
252 atf_fail()
253 {
254 case "${Expect}" in
255 fail)
256 atf_expected_failure "${@}"
257 ;;
258 pass)
259 _atf_create_resfile "failed: ${*}"
260 exit 1
261 ;;
262 *)
263 _atf_error 128 "Unreachable"
264 ;;
265 esac
266 }
268 #
269 # atf_get varname
270 #
271 # Prints the value of a test case-specific variable. Given that one
272 # should not get the value of non-existent variables, it is fine to
273 # always use this function as 'val=$(atf_get var)'.
274 #
275 atf_get()
276 {
277 eval echo \${__tc_var_${Test_Case}_$(_atf_normalize ${1})}
278 }
280 #
281 # atf_get_srcdir
282 #
283 # Prints the value of the test case's source directory.
284 #
285 atf_get_srcdir()
286 {
287 echo ${Source_Dir}
288 }
290 #
291 # atf_pass
292 #
293 # Makes the test case pass. Shouldn't be used in general, as a test
294 # case that does not explicitly fail is assumed to pass.
295 #
296 atf_pass()
297 {
298 case "${Expect}" in
299 fail)
300 Expect=pass
301 atf_fail "Test case was expecting a failure but got a pass instead"
302 ;;
303 pass)
304 _atf_create_resfile passed
305 exit 0
306 ;;
307 *)
308 _atf_error 128 "Unreachable"
309 ;;
310 esac
311 }
313 #
314 # atf_require_prog prog
315 #
316 # Checks that the given program name (either provided as an absolute
317 # path or as a plain file name) can be found. If it is not available,
318 # automatically skips the test case with an appropriate message.
319 #
320 # Relative paths are not allowed because the test case cannot predict
321 # where it will be executed from.
322 #
323 atf_require_prog()
324 {
325 _prog=
326 case ${1} in
327 /*)
328 _prog="${1}"
329 [ -x ${_prog} ] || \
330 atf_skip "The required program ${1} could not be found"
331 ;;
332 */*)
333 atf_fail "atf_require_prog does not accept relative path names \`${1}'"
334 ;;
335 *)
336 _prog=$(_atf_find_in_path "${1}")
337 [ -n "${_prog}" ] || \
338 atf_skip "The required program ${1} could not be found" \
339 "in the PATH"
340 ;;
341 esac
342 }
344 #
345 # atf_set varname val1 [.. valN]
346 #
347 # Sets the test case's variable 'varname' to the specified values
348 # which are concatenated using a single blank space. This function
349 # is supposed to be called form the test case's head only.
350 #
351 atf_set()
352 {
353 ${Parsing_Head} || \
354 _atf_error 128 "atf_set called from the test case's body"
356 Test_Case_Vars="${Test_Case_Vars} ${1}"
357 _var=$(_atf_normalize ${1}); shift
358 eval __tc_var_${Test_Case}_${_var}=\"\${*}\"
359 }
361 #
362 # atf_skip msg1 [.. msgN]
363 #
364 # Skips the test case because of the reason provided. Multiple words
365 # can be given, in which case they are joined by a single blank space.
366 #
367 atf_skip()
368 {
369 _atf_create_resfile "skipped: ${*}"
370 exit 0
371 }
373 #
374 # atf_test_case tc-name cleanup
375 #
376 # Defines a new test case named tc-name. The name provided here must be
377 # accompanied by two functions named after it: <tc-name>_head and
378 # <tc-name>_body. If cleanup is set to 'cleanup', then this also expects
379 # a <tc-name>_cleanup function to be defined.
380 #
381 atf_test_case()
382 {
383 eval "${1}_head() { :; }"
384 eval "${1}_body() { atf_fail 'Test case not implemented'; }"
385 if [ "${2}" = cleanup ]; then
386 eval __has_cleanup_${1}=true
387 eval "${1}_cleanup() { :; }"
388 else
389 eval "${1}_cleanup() {
390 _atf_error 1 'Test case ${1} declared without a cleanup routine'; }"
391 fi
392 }
394 # ------------------------------------------------------------------------
395 # PRIVATE INTERFACE
396 # ------------------------------------------------------------------------
398 #
399 # _atf_config_set varname val1 [.. valN]
400 #
401 # Sets the test case's private variable 'varname' to the specified
402 # values which are concatenated using a single blank space.
403 #
404 _atf_config_set()
405 {
406 _var=$(_atf_normalize ${1}); shift
407 eval __tc_config_var_${_var}=\"\${*}\"
408 Config_Vars="${Config_Vars} __tc_config_var_${_var}"
409 }
411 #
412 # _atf_config_set_str varname=val
413 #
414 # Sets the test case's private variable 'varname' to the specified
415 # value. The parameter is of the form 'varname=val'.
416 #
417 _atf_config_set_from_str()
418 {
419 _oldifs=${IFS}
420 IFS='='
421 set -- ${*}
422 _var=${1}
423 shift
424 _val="${@}"
425 IFS=${_oldifs}
426 _atf_config_set "${_var}" "${_val}"
427 }
429 #
430 # _atf_create_resfile contents
431 #
432 # Creates the results file.
433 #
434 _atf_create_resfile()
435 {
436 if [ -n "${Results_File}" ]; then
437 echo "${*}" >"${Results_File}" || \
438 _atf_error 128 "Cannot create results file '${Results_File}'"
439 else
440 echo "${*}"
441 fi
442 }
444 #
445 # _atf_error error_code [msg1 [.. msgN]]
446 #
447 # Prints the given error message (which can be composed of multiple
448 # arguments, in which case are joined by a single space) and exits
449 # with the specified error code.
450 #
451 # This must not be used by test programs themselves (hence making
452 # the function private) to indicate a test case's failure. They
453 # have to use the atf_fail function.
454 #
455 _atf_error()
456 {
457 _error_code="${1}"; shift
459 echo "${Prog_Name}: ERROR:" "$@" 1>&2
460 exit ${_error_code}
461 }
463 #
464 # _atf_warning msg1 [.. msgN]
465 #
466 # Prints the given warning message (which can be composed of multiple
467 # arguments, in which case are joined by a single space).
468 #
469 _atf_warning()
470 {
471 echo "${Prog_Name}: WARNING:" "$@" 1>&2
472 }
474 #
475 # _atf_find_in_path program
476 #
477 # Looks for a program in the path and prints the full path to it or
478 # nothing if it could not be found. It also returns true in case of
479 # success.
480 #
481 _atf_find_in_path()
482 {
483 _prog="${1}"
485 _oldifs=${IFS}
486 IFS=:
487 for _dir in ${PATH}
488 do
489 if [ -x ${_dir}/${_prog} ]; then
490 IFS=${_oldifs}
491 echo ${_dir}/${_prog}
492 return 0
493 fi
494 done
495 IFS=${_oldifs}
497 return 1
498 }
500 #
501 # _atf_has_tc name
502 #
503 # Returns true if the given test case exists.
504 #
505 _atf_has_tc()
506 {
507 for _tc in ${Test_Cases}; do
508 [ "${_tc}" != "${1}" ] || return 0
509 done
510 return 1
511 }
513 #
514 # _atf_list_tcs
515 #
516 # Describes all test cases and prints the list to the standard output.
517 #
518 _atf_list_tcs()
519 {
520 echo 'Content-Type: application/X-atf-tp; version="1"'
521 echo
523 set -- ${Test_Cases}
524 while [ ${#} -gt 0 ]; do
525 _atf_parse_head ${1}
527 echo "ident: $(atf_get ident)"
528 for _var in ${Test_Case_Vars}; do
529 [ "${_var}" != "ident" ] && echo "${_var}: $(atf_get ${_var})"
530 done
532 [ ${#} -gt 1 ] && echo
533 shift
534 done
535 }
537 #
538 # _atf_normalize str
539 #
540 # Normalizes a string so that it is a valid shell variable name.
541 #
542 _atf_normalize()
543 {
544 echo ${1} | tr .- __
545 }
547 #
548 # _atf_parse_head tcname
549 #
550 # Evaluates a test case's head to gather its variables and prepares the
551 # test program to run it.
552 #
553 _atf_parse_head()
554 {
555 Parsing_Head=true
557 Test_Case="${1}"
558 Test_Case_Vars=
560 if _atf_has_cleanup "${1}"; then
561 atf_set has.cleanup "true"
562 fi
564 ${1}_head
565 atf_set ident "${1}"
567 Parsing_Head=false
568 }
570 #
571 # _atf_run_tc tc
572 #
573 # Runs the specified test case. Prints its exit status to the
574 # standard output and returns a boolean indicating if the test was
575 # successful or not.
576 #
577 _atf_run_tc()
578 {
579 case ${1} in
580 *:*)
581 _tcname=${1%%:*}
582 _tcpart=${1#*:}
584 if [ "${_tcpart}" != body -a "${_tcpart}" != cleanup ]; then
585 _atf_syntax_error "Unknown test case part \`${_tcpart}'"
586 fi
587 ;;
589 *)
590 _tcname=${1}
591 _tcpart=body
592 ;;
593 esac
595 _atf_has_tc "${_tcname}" || _atf_syntax_error "Unknown test case \`${1}'"
597 if [ "${__RUNNING_INSIDE_ATF_RUN}" != "internal-yes-value" ]; then
598 _atf_warning "Running test cases without atf-run(1) is unsupported"
599 _atf_warning "No isolation nor timeout control is being applied;" \
600 "you may get unexpected failures; see atf-test-case(4)"
601 fi
603 _atf_parse_head ${_tcname}
605 case ${_tcpart} in
606 body)
607 if ${_tcname}_body; then
608 _atf_validate_expect
609 _atf_create_resfile passed
610 else
611 Expect=pass
612 atf_fail "Test case body returned a non-ok exit code, but" \
613 "this is not allowed"
614 fi
615 ;;
616 cleanup)
617 if _atf_has_cleanup "${_tcname}"; then
618 ${_tcname}_cleanup || _atf_error 128 "The test case cleanup" \
619 "returned a non-ok exit code, but this is not allowed"
620 fi
621 ;;
622 *)
623 _atf_error 128 "Unknown test case part"
624 ;;
625 esac
626 }
628 #
629 # _atf_syntax_error msg1 [.. msgN]
630 #
631 # Formats and prints a syntax error message and terminates the
632 # program prematurely.
633 #
634 _atf_syntax_error()
635 {
636 echo "${Prog_Name}: ERROR: ${@}" 1>&2
637 echo "${Prog_Name}: See atf-test-program(1) for usage details." 1>&2
638 exit 1
639 }
641 #
642 # _atf_has_cleanup tc-name
643 #
644 # Returns a boolean indicating if the given test case has a cleanup
645 # routine or not.
646 #
647 _atf_has_cleanup()
648 {
649 _found=true
650 eval "[ x\"\${__has_cleanup_${1}}\" = xtrue ] || _found=false"
651 [ "${_found}" = true ]
652 }
654 #
655 # _atf_validate_expect
656 #
657 # Ensures that the current test case state is correct regarding the expect
658 # status.
659 #
660 _atf_validate_expect()
661 {
662 case "${Expect}" in
663 death)
664 Expect=pass
665 atf_fail "Test case was expected to terminate abruptly but it" \
666 "continued execution"
667 ;;
668 exit)
669 Expect=pass
670 atf_fail "Test case was expected to exit cleanly but it continued" \
671 "execution"
672 ;;
673 fail)
674 Expect=pass
675 atf_fail "Test case was expecting a failure but none were raised"
676 ;;
677 pass)
678 ;;
679 signal)
680 Expect=pass
681 atf_fail "Test case was expected to receive a termination signal" \
682 "but it continued execution"
683 ;;
684 timeout)
685 Expect=pass
686 atf_fail "Test case was expected to hang but it continued execution"
687 ;;
688 *)
689 _atf_error 128 "Unreachable"
690 ;;
691 esac
692 }
694 #
695 # _atf_warning [msg1 [.. msgN]]
696 #
697 # Prints the given warning message (which can be composed of multiple
698 # arguments, in which case are joined by a single space).
699 #
700 # This must not be used by test programs themselves (hence making
701 # the function private).
702 #
703 _atf_warning()
704 {
705 echo "${Prog_Name}: WARNING:" "$@" 1>&2
706 }
708 #
709 # main [options] test_case
710 #
711 # Test program's entry point.
712 #
713 main()
714 {
715 # Process command-line options first.
716 _numargs=${#}
717 _lflag=false
718 while getopts :lr:s:v: arg; do
719 case ${arg} in
720 l)
721 _lflag=true
722 ;;
724 r)
725 Results_File=${OPTARG}
726 ;;
728 s)
729 Source_Dir=${OPTARG}
730 ;;
732 v)
733 _atf_config_set_from_str "${OPTARG}"
734 ;;
736 \?)
737 _atf_syntax_error "Unknown option -${OPTARG}."
738 # NOTREACHED
739 ;;
740 esac
741 done
742 shift `expr ${OPTIND} - 1`
744 # First of all, make sure that the source directory is correct. It
745 # doesn't matter if the user did not change it, because the default
746 # value may not work. (TODO: It possibly should, even though it is
747 # not a big deal because atf-run deals with this.)
748 case ${Source_Dir} in
749 /*)
750 ;;
751 *)
752 Source_Dir=$(pwd)/${Source_Dir}
753 ;;
754 esac
755 [ -f ${Source_Dir}/${Prog_Name} ] || \
756 _atf_error 1 "Cannot find the test program in the source" \
757 "directory \`${Source_Dir}'"
759 # Call the test program's hook to register all available test cases.
760 atf_init_test_cases
762 # Run or list test cases.
763 if `${_lflag}`; then
764 if [ ${#} -gt 0 ]; then
765 _atf_syntax_error "Cannot provide test case names with -l"
766 fi
767 _atf_list_tcs
768 else
769 if [ ${#} -eq 0 ]; then
770 _atf_syntax_error "Must provide a test case name"
771 elif [ ${#} -gt 1 ]; then
772 _atf_syntax_error "Cannot provide more than one test case name"
773 else
774 _atf_run_tc "${1}"
775 fi
776 fi
777 }
779 # vim: syntax=sh:expandtab:shiftwidth=4:softtabstop=4