2 .\" Automated Testing Framework (atf)
4 .\" Copyright (c) 2008 The NetBSD Foundation, Inc.
5 .\" All rights reserved.
7 .\" Redistribution and use in source and binary forms, with or without
8 .\" modification, are permitted provided that the following conditions
10 .\" 1. Redistributions of source code must retain the above copyright
11 .\" notice, this list of conditions and the following disclaimer.
12 .\" 2. Redistributions in binary form must reproduce the above copyright
13 .\" notice, this list of conditions and the following disclaimer in the
14 .\" documentation and/or other materials provided with the distribution.
16 .\" THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND
17 .\" CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
18 .\" INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
19 .\" MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 .\" IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY
21 .\" DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 .\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
23 .\" GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24 .\" INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
25 .\" IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
26 .\" OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
27 .\" IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37 .Nm ATF_CHECK_EQ_MSG ,
39 .Nm ATF_CHECK_MATCH_MSG ,
41 .Nm ATF_CHECK_STREQ_MSG ,
46 .Nm ATF_REQUIRE_EQ_MSG ,
47 .Nm ATF_REQUIRE_MATCH ,
48 .Nm ATF_REQUIRE_MATCH_MSG ,
49 .Nm ATF_REQUIRE_STREQ ,
50 .Nm ATF_REQUIRE_STREQ_MSG ,
51 .Nm ATF_REQUIRE_ERRNO ,
54 .Nm ATF_TC_BODY_NAME ,
56 .Nm ATF_TC_CLEANUP_NAME ,
58 .Nm ATF_TC_HEAD_NAME ,
60 .Nm ATF_TC_WITH_CLEANUP ,
61 .Nm ATF_TC_WITHOUT_HEAD ,
64 .Nm atf_tc_get_config_var ,
65 .Nm atf_tc_get_config_var_wd ,
66 .Nm atf_tc_get_config_var_as_bool ,
67 .Nm atf_tc_get_config_var_as_bool_wd ,
68 .Nm atf_tc_get_config_var_as_long ,
69 .Nm atf_tc_get_config_var_as_long_wd ,
71 .Nm atf_tc_expect_death ,
72 .Nm atf_tc_expect_exit ,
73 .Nm atf_tc_expect_fail ,
74 .Nm atf_tc_expect_pass ,
75 .Nm atf_tc_expect_signal ,
76 .Nm atf_tc_expect_timeout ,
78 .Nm atf_tc_fail_nonfatal ,
81 .Nm atf_utils_cat_file ,
82 .Nm atf_utils_compare_file ,
83 .Nm atf_utils_copy_file ,
84 .Nm atf_utils_create_file ,
85 .Nm atf_utils_file_exists ,
87 .Nm atf_utils_free_charpp ,
88 .Nm atf_utils_grep_file ,
89 .Nm atf_utils_grep_string ,
90 .Nm atf_utils_readline ,
91 .Nm atf_utils_redirect ,
93 .Nd C API to write ATF-based test programs
96 .Fn ATF_CHECK "expression"
97 .Fn ATF_CHECK_MSG "expression" "fail_msg_fmt" ...
98 .Fn ATF_CHECK_EQ "expression_1" "expression_2"
99 .Fn ATF_CHECK_EQ_MSG "expression_1" "expression_2" "fail_msg_fmt" ...
100 .Fn ATF_CHECK_MATCH "regexp" "string"
101 .Fn ATF_CHECK_MATCH_MSG "regexp" "string" "fail_msg_fmt" ...
102 .Fn ATF_CHECK_STREQ "string_1" "string_2"
103 .Fn ATF_CHECK_STREQ_MSG "string_1" "string_2" "fail_msg_fmt" ...
104 .Fn ATF_CHECK_ERRNO "exp_errno" "bool_expression"
105 .Fn ATF_REQUIRE "expression"
106 .Fn ATF_REQUIRE_MSG "expression" "fail_msg_fmt" ...
107 .Fn ATF_REQUIRE_EQ "expression_1" "expression_2"
108 .Fn ATF_REQUIRE_EQ_MSG "expression_1" "expression_2" "fail_msg_fmt" ...
109 .Fn ATF_REQUIRE_MATCH "regexp" "string"
110 .Fn ATF_REQUIRE_MATCH_MSG "regexp" "string" "fail_msg_fmt" ...
111 .Fn ATF_REQUIRE_STREQ "string_1" "string_2"
112 .Fn ATF_REQUIRE_STREQ_MSG "string_1" "string_2" "fail_msg_fmt" ...
113 .Fn ATF_REQUIRE_ERRNO "exp_errno" "bool_expression"
115 .Fn ATF_TC_BODY "name" "tc"
116 .Fn ATF_TC_BODY_NAME "name"
117 .Fn ATF_TC_CLEANUP "name" "tc"
118 .Fn ATF_TC_CLEANUP_NAME "name"
119 .Fn ATF_TC_HEAD "name" "tc"
120 .Fn ATF_TC_HEAD_NAME "name"
121 .Fn ATF_TC_NAME "name"
122 .Fn ATF_TC_WITH_CLEANUP "name"
123 .Fn ATF_TC_WITHOUT_HEAD "name"
124 .Fn ATF_TP_ADD_TC "tp_name" "tc_name"
125 .Fn ATF_TP_ADD_TCS "tp_name"
126 .Fn atf_tc_get_config_var "tc" "varname"
127 .Fn atf_tc_get_config_var_wd "tc" "variable_name" "default_value"
128 .Fn atf_tc_get_config_var_as_bool "tc" "variable_name"
129 .Fn atf_tc_get_config_var_as_bool_wd "tc" "variable_name" "default_value"
130 .Fn atf_tc_get_config_var_as_long "tc" "variable_name"
131 .Fn atf_tc_get_config_var_as_long_wd "tc" "variable_name" "default_value"
133 .Fn atf_tc_expect_death "reason" "..."
134 .Fn atf_tc_expect_exit "exitcode" "reason" "..."
135 .Fn atf_tc_expect_fail "reason" "..."
136 .Fn atf_tc_expect_pass
137 .Fn atf_tc_expect_signal "signo" "reason" "..."
138 .Fn atf_tc_expect_timeout "reason" "..."
139 .Fn atf_tc_fail "reason"
140 .Fn atf_tc_fail_nonfatal "reason"
142 .Fn atf_tc_skip "reason"
144 .Fo atf_utils_cat_file
145 .Fa "const char *file"
146 .Fa "const char *prefix"
149 .Fo atf_utils_compare_file
150 .Fa "const char *file"
151 .Fa "const char *contents"
154 .Fo atf_utils_copy_file
155 .Fa "const char *source"
156 .Fa "const char *destination"
159 .Fo atf_utils_create_file
160 .Fa "const char *file"
161 .Fa "const char *contents"
165 .Fo atf_utils_file_exists
166 .Fa "const char *file"
173 .Fo atf_utils_free_charpp
177 .Fo atf_utils_grep_file
178 .Fa "const char *regexp"
179 .Fa "const char *file"
183 .Fo atf_utils_grep_string
184 .Fa "const char *regexp"
185 .Fa "const char *str"
189 .Fo atf_utils_readline
193 .Fo atf_utils_redirect
195 .Fa "const char *file"
199 .Fa "const pid_t pid"
200 .Fa "const int expected_exit_status"
201 .Fa "const char *expected_stdout"
202 .Fa "const char *expected_stderr"
207 C-based test programs always follow this template:
208 .Bd -literal -offset indent
209 .Ns ... C-specific includes go here ...
216 ... first test case's header ...
220 ... first test case's body ...
223 ATF_TC_WITH_CLEANUP(tc2);
226 ... second test case's header ...
230 ... second test case's body ...
232 ATF_TC_CLEANUP(tc2, tc)
234 ... second test case's cleanup ...
237 ATF_TC_WITHOUT_HEAD(tc3);
240 ... third test case's body ...
243 .Ns ... additional test cases ...
247 ATF_TP_ADD_TC(tcs, tc1);
248 ATF_TP_ADD_TC(tcs, tc2);
249 ATF_TP_ADD_TC(tcs, tc3);
250 ... add additional test cases ...
252 return atf_no_error();
255 .Ss Definition of test cases
256 Test cases have an identifier and are composed of three different parts:
257 the header, the body and an optional cleanup routine, all of which are
259 .Xr atf-test-case 4 .
260 To define test cases, one can use the
262 .Fn ATF_TC_WITH_CLEANUP
264 .Fn ATF_TC_WITHOUT_HEAD
265 macros, which take a single parameter specifiying the test case's name.
267 requires to define a head and a body for the test case,
268 .Fn ATF_TC_WITH_CLEANUP
269 requires to define a head, a body and a cleanup for the test case and
270 .Fn ATF_TC_WITHOUT_HEAD
271 requires only a body for the test case.
272 It is important to note that these
274 set the test case up for execution when the program is run.
275 In order to do so, a later registration is needed with the
278 .Sx Program initialization .
280 Later on, one must define the three parts of the body by means of three
282 Their headers are given by the
287 macros, all of which take the test case name provided to the
289 .Fn ATF_TC_WITH_CLEANUP ,
291 .Fn ATF_TC_WITHOUT_HEAD
292 macros and the name of the variable that will hold a pointer to the
294 Following each of these, a block of code is expected, surrounded by the
295 opening and closing brackets.
296 .Ss Program initialization
297 The library provides a way to easily define the test program's
300 You should never define one on your own, but rely on the
301 library to do it for you.
302 This is done by using the
304 macro, which is passed the name of the object that will hold the test
305 cases; i.e. the test program instance.
306 This name can be whatever you want as long as it is a valid variable
309 After the macro, you are supposed to provide the body of a function, which
312 macro to register the test cases the test program will execute and return
313 a success error code.
314 The first parameter of this macro matches the name you provided in the
316 The success status can be returned using the
319 .Ss Header definitions
320 The test case's header can define the meta-data by using the
321 .Fn atf_tc_set_md_var
322 method, which takes three parameters: the first one points to the test
323 case data, the second one specifies the meta-data variable to be set
324 and the third one specifies its value.
325 Both of them are strings.
326 .Ss Configuration variables
327 The test case has read-only access to the current configuration variables
330 .Fn atf_tc_has_config_var ,
332 .Fn atf_tc_get_config_var ,
334 .Fn atf_tc_get_config_var_wd ,
336 .Fn atf_tc_get_config_var_as_bool ,
338 .Fn atf_tc_get_config_var_as_bool_wd ,
340 .Fn atf_tc_get_config_var_as_long ,
343 .Fn atf_tc_get_config_var_as_long_wd
344 functions, which can be called in any of the three parts of a test case.
348 variants take a default value for the variable which is returned if the
349 variable is not defined.
350 The other functions without the
354 the variable to be defined.
355 .Ss Access to the source directory
356 It is possible to get the path to the test case's source directory from any
357 of its three components by querying the
359 configuration variable.
360 .Ss Requiring programs
363 meta-data variable available in the header only, one can also check for
364 additional programs in the test case's body by using the
365 .Fn atf_tc_require_prog
366 function, which takes the base name or full path of a single binary.
367 Relative paths are forbidden.
368 If it is not found, the test case will be automatically skipped.
369 .Ss Test case finalization
370 The test case finalizes either when the body reaches its end, at which
371 point the test is assumed to have
373 unless any non-fatal errors were raised using
374 .Fn atf_tc_fail_nonfatal ,
375 or at any explicit call to
380 These three functions terminate the execution of the test case immediately.
381 The cleanup routine will be processed afterwards in a completely automated
382 way, regardless of the test case's termination reason.
385 does not take any parameters.
387 .Fn atf_tc_fail_nonfatal
390 take a format string and a variable list of parameters, which describe, in
391 a user-friendly manner, why the test case failed or was skipped,
393 It is very important to provide a clear error message in both cases so that
394 the user can quickly know why the test did not pass.
396 Everything explained in the previous section changes when the test case
397 expectations are redefined by the programmer.
399 Each test case has an internal state called
401 that describes what the test case expectations are at any point in time.
402 The value of this property can change during execution by any of:
403 .Bl -tag -width indent
404 .It Fn atf_tc_expect_death "reason" "..."
405 Expects the test case to exit prematurely regardless of the nature of the
407 .It Fn atf_tc_expect_exit "exitcode" "reason" "..."
408 Expects the test case to exit cleanly.
414 will validate that the exit code of the test case matches the one provided
416 Otherwise, the exact value will be ignored.
417 .It Fn atf_tc_expect_fail "reason" "..."
418 Any failure (be it fatal or non-fatal) raised in this mode is recorded.
419 However, such failures do not report the test case as failed; instead, the
420 test case finalizes cleanly and is reported as
421 .Sq expected failure ;
422 this report includes the provided
425 If no error is raised while running in this mode, then the test case is
429 This mode is useful to reproduce actual known bugs in tests.
430 Whenever the developer fixes the bug later on, the test case will start
431 reporting a failure, signaling the developer that the test case must be
432 adjusted to the new conditions.
433 In this situation, it is useful, for example, to set
435 as the bug number for tracking purposes.
436 .It Fn atf_tc_expect_pass
437 This is the normal mode of execution.
438 In this mode, any failure is reported as such to the user and the test case
441 .It Fn atf_tc_expect_signal "signo" "reason" "..."
442 Expects the test case to terminate due to the reception of a signal.
448 will validate that the signal that terminated the test case matches the one
449 provided in this call.
450 Otherwise, the exact value will be ignored.
451 .It Fn atf_tc_expect_timeout "reason" "..."
452 Expects the test case to execute for longer than its timeout.
454 .Ss Helper macros for common checks
455 The library provides several macros that are very handy in multiple
457 These basically check some condition after executing a given statement or
458 processing a given expression and, if the condition is not met, they
459 report the test case as failed.
463 variant of the macros immediately abort the test case as soon as an error
464 condition is detected by calling the
467 Use this variant whenever it makes no sense to continue the execution of a
468 test case when the checked condition is not met.
471 variant, on the other hand, reports a failure as soon as it is encountered
473 .Fn atf_tc_fail_nonfatal
474 function, but the execution of the test case continues as if nothing had
476 Use this variant whenever the checked condition is important as a result of
477 the test case, but there are other conditions that can be subsequently
478 checked on the same run without aborting.
482 variants take an extra set of parameters to explicitly specify the failure
484 This failure message is formatted according to the
493 take an expression and fail if the expression evaluates to false.
496 .Fn ATF_CHECK_EQ_MSG ,
499 .Fn ATF_REQUIRE_EQ_MSG
500 take two expressions and fail if the two evaluated values are not equal.
502 .Fn ATF_CHECK_MATCH ,
503 .Fn ATF_CHECK_MATCH_MSG ,
504 .Fn ATF_REQUIRE_MATCH
506 .Fn ATF_REQUIRE_MATCH_MSG
507 take a regular expression and a string and fail if the regular expression does
508 not match the given string.
509 Note that the regular expression is not anchored, so it will match anywhere in
512 .Fn ATF_CHECK_STREQ ,
513 .Fn ATF_CHECK_STREQ_MSG ,
514 .Fn ATF_REQUIRE_STREQ
516 .Fn ATF_REQUIRE_STREQ_MSG
517 take two strings and fail if the two are not equal character by character.
521 .Fn ATF_REQUIRE_ERRNO
522 take, first, the error code that the check is expecting to find in the
524 variable and, second, a boolean expression that, if evaluates to true,
525 means that a call failed and
527 has to be checked against the first value.
528 .Ss Utility functions
529 The following functions are provided as part of the
531 API to simplify the creation of a variety of tests.
532 In particular, these are useful to write tests for command-line interfaces.
535 .Fo atf_utils_cat_file
536 .Fa "const char *file"
537 .Fa "const char *prefix"
540 Prints the contents of
542 to the standard output, prefixing every line with the string in
547 .Fo atf_utils_compare_file
548 .Fa "const char *file"
549 .Fa "const char *contents"
552 Returns true if the given
554 matches exactly the expected inlined
559 .Fo atf_utils_copy_file
560 .Fa "const char *source"
561 .Fa "const char *destination"
568 The permissions of the file are preserved during the code.
572 .Fo atf_utils_create_file
573 .Fa "const char *file"
574 .Fa "const char *contents"
580 with the text given in
582 which is a formatting string that uses the rest of the variable arguments.
586 .Fo atf_utils_file_exists
587 .Fa "const char *file"
600 Forks a process and redirects the standard output and standard error of the
601 child to files for later validation with
603 Fails the test case if the fork fails, so this does not return an error.
607 .Fo atf_utils_free_charpp
611 Frees a dynamically-allocated array of dynamically-allocated strings.
615 .Fo atf_utils_grep_file
616 .Fa "const char *regexp"
617 .Fa "const char *file"
623 which is a formatting string representing the regular expression,
626 The variable arguments are used to construct the regular expression.
630 .Fo atf_utils_grep_string
631 .Fa "const char *regexp"
632 .Fa "const char *str"
638 which is a formatting string representing the regular expression,
639 in the literal string
641 The variable arguments are used to construct the regular expression.
645 .Fo atf_utils_readline
649 Reads a line from the file descriptor
651 The line, if any, is returned as a dynamically-allocated buffer that must be
654 If there was nothing to read, returns
659 .Fo atf_utils_redirect
661 .Fa "const char *file"
664 Redirects the given file descriptor
668 This function exits the process in case of an error and does not properly mark
669 the test case as failed.
670 As a result, it should only be used in subprocesses of the test case; specially
677 .Fa "const pid_t pid"
678 .Fa "const int expected_exit_status"
679 .Fa "const char *expected_stdout"
680 .Fa "const char *expected_stderr"
683 Waits and validates the result of a subprocess spawned with
685 The validation involves checking that the subprocess exited cleanly and returned
686 the code specified in
687 .Fa expected_exit_status
688 and that its standard output and standard error match the strings given in
691 .Fa expected_stderr .
697 strings are prefixed with
699 then they specify the name of the file into which to store the stdout or stderr
700 of the subprocess, and no comparison is performed.
703 The following shows a complete test program with a single test case that
704 validates the addition operator:
705 .Bd -literal -offset indent
709 ATF_TC_HEAD(addition, tc)
711 atf_tc_set_md_var(tc, "descr",
712 "Sample tests for the addition operator");
714 ATF_TC_BODY(addition, tc)
716 ATF_CHECK_EQ(0 + 0, 0);
717 ATF_CHECK_EQ(0 + 1, 1);
718 ATF_CHECK_EQ(1 + 0, 1);
720 ATF_CHECK_EQ(1 + 1, 2);
722 ATF_CHECK_EQ(100 + 200, 300);
725 ATF_TC(string_formatting);
726 ATF_TC_HEAD(string_formatting, tc)
728 atf_tc_set_md_var(tc, "descr",
729 "Sample tests for the snprintf");
731 ATF_TC_BODY(string_formatting, tc)
734 snprintf(buf, sizeof(buf), "a %s", "string");
735 ATF_CHECK_STREQ_MSG("a string", buf, "%s is not working");
738 ATF_TC(open_failure);
739 ATF_TC_HEAD(open_failure, tc)
741 atf_tc_set_md_var(tc, "descr",
742 "Sample tests for the open function");
744 ATF_TC_BODY(open_failure, tc)
746 ATF_CHECK_ERRNO(ENOENT, open("non-existent", O_RDONLY) == -1);
750 ATF_TC_HEAD(known_bug, tc)
752 atf_tc_set_md_var(tc, "descr",
753 "Reproduces a known bug");
755 ATF_TC_BODY(known_bug, tc)
757 atf_tc_expect_fail("See bug number foo/bar");
758 ATF_CHECK_EQ(3, 1 + 1);
759 atf_tc_expect_pass();
760 ATF_CHECK_EQ(3, 1 + 2);
765 ATF_TP_ADD_TC(tp, addition);
766 ATF_TP_ADD_TC(tp, string_formatting);
767 ATF_TP_ADD_TC(tp, open_failure);
768 ATF_TP_ADD_TC(tp, known_bug);
770 return atf_no_error();
774 .Xr atf-test-program 1 ,
775 .Xr atf-test-case 4 ,