| blob | d0579a9a754418520115a06510e363dc671260cc |
1 .\"
2 .\" Automated Testing Framework (atf)
3 .\"
4 .\" Copyright (c) 2008 The NetBSD Foundation, Inc.
5 .\" All rights reserved.
6 .\"
7 .\" Redistribution and use in source and binary forms, with or without
8 .\" modification, are permitted provided that the following conditions
9 .\" are met:
10 .\" 1. Redistributions of source code must retain the above copyright
11 .\" notice, this list of conditions and the following disclaimer.
12 .\" 2. Redistributions in binary form must reproduce the above copyright
13 .\" notice, this list of conditions and the following disclaimer in the
14 .\" documentation and/or other materials provided with the distribution.
15 .\"
16 .\" THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND
17 .\" CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
18 .\" INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
19 .\" MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 .\" IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY
21 .\" DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 .\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
23 .\" GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24 .\" INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
25 .\" IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
26 .\" OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
27 .\" IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 .\"
29 .Dd November 30, 2012
30 .Dt ATF-C++-API 3
31 .Os
32 .Sh NAME
33 .Nm atf-c++-api ,
34 .Nm ATF_ADD_TEST_CASE ,
35 .Nm ATF_CHECK_ERRNO ,
36 .Nm ATF_FAIL ,
37 .Nm ATF_INIT_TEST_CASES ,
38 .Nm ATF_PASS ,
39 .Nm ATF_REQUIRE ,
40 .Nm ATF_REQUIRE_EQ ,
41 .Nm ATF_REQUIRE_ERRNO ,
42 .Nm ATF_REQUIRE_IN ,
43 .Nm ATF_REQUIRE_MATCH ,
44 .Nm ATF_REQUIRE_NOT_IN ,
45 .Nm ATF_REQUIRE_THROW ,
46 .Nm ATF_REQUIRE_THROW_RE ,
47 .Nm ATF_SKIP ,
48 .Nm ATF_TEST_CASE ,
49 .Nm ATF_TEST_CASE_BODY ,
50 .Nm ATF_TEST_CASE_CLEANUP ,
51 .Nm ATF_TEST_CASE_HEAD ,
52 .Nm ATF_TEST_CASE_NAME ,
53 .Nm ATF_TEST_CASE_USE ,
54 .Nm ATF_TEST_CASE_WITH_CLEANUP ,
55 .Nm ATF_TEST_CASE_WITHOUT_HEAD ,
56 .Nm atf::utils::cat_file ,
57 .Nm atf::utils::compare_file ,
58 .Nm atf::utils::copy_file ,
59 .Nm atf::utils::create_file ,
60 .Nm atf::utils::file_exists ,
61 .Nm atf::utils::fork ,
62 .Nm atf::utils::grep_collection ,
63 .Nm atf::utils::grep_file ,
64 .Nm atf::utils::grep_string ,
65 .Nm atf::utils::redirect ,
66 .Nm atf::utils::wait
67 .Nd C++ API to write ATF-based test programs
68 .Sh SYNOPSIS
69 .In atf-c++.hpp
70 .Fn ATF_ADD_TEST_CASE "tcs" "name"
71 .Fn ATF_CHECK_ERRNO "exp_errno" "bool_expression"
72 .Fn ATF_FAIL "reason"
73 .Fn ATF_INIT_TEST_CASES "tcs"
74 .Fn ATF_PASS
75 .Fn ATF_REQUIRE "expression"
76 .Fn ATF_REQUIRE_EQ "expression_1" "expression_2"
77 .Fn ATF_REQUIRE_ERRNO "exp_errno" "bool_expression"
78 .Fn ATF_REQUIRE_IN "element" "collection"
79 .Fn ATF_REQUIRE_MATCH "regexp" "string_expression"
80 .Fn ATF_REQUIRE_NOT_IN "element" "collection"
81 .Fn ATF_REQUIRE_THROW "expected_exception" "statement"
82 .Fn ATF_REQUIRE_THROW_RE "expected_exception" "regexp" "statement"
83 .Fn ATF_SKIP "reason"
84 .Fn ATF_TEST_CASE "name"
85 .Fn ATF_TEST_CASE_BODY "name"
86 .Fn ATF_TEST_CASE_CLEANUP "name"
87 .Fn ATF_TEST_CASE_HEAD "name"
88 .Fn ATF_TEST_CASE_NAME "name"
89 .Fn ATF_TEST_CASE_USE "name"
90 .Fn ATF_TEST_CASE_WITH_CLEANUP "name"
91 .Fn ATF_TEST_CASE_WITHOUT_HEAD "name"
92 .Ft void
93 .Fo atf::utils::cat_file
94 .Fa "const std::string& path"
95 .Fa "const std::string& prefix"
96 .Fc
97 .Ft bool
98 .Fo atf::utils::compare_file
99 .Fa "const std::string& path"
100 .Fa "const std::string& contents"
101 .Fc
102 .Ft void
103 .Fo atf::utils::copy_file
104 .Fa "const std::string& source"
105 .Fa "const std::string& destination"
106 .Fc
107 .Ft void
108 .Fo atf::utils::create_file
109 .Fa "const std::string& path"
110 .Fa "const std::string& contents"
111 .Fc
112 .Ft void
113 .Fo atf::utils::file_exists
114 .Fa "const std::string& path"
115 .Fc
116 .Ft pid_t
117 .Fo atf::utils::fork
118 .Fa "void"
119 .Fc
120 .Ft bool
121 .Fo atf::utils::grep_collection
122 .Fa "const std::string& regexp"
123 .Fa "const Collection& collection"
124 .Fc
125 .Ft bool
126 .Fo atf::utils::grep_file
127 .Fa "const std::string& regexp"
128 .Fa "const std::string& path"
129 .Fc
130 .Ft bool
131 .Fo atf::utils::grep_string
132 .Fa "const std::string& regexp"
133 .Fa "const std::string& path"
134 .Fc
135 .Ft void
136 .Fo atf::utils::redirect
137 .Fa "const int fd"
138 .Fa "const std::string& path"
139 .Fc
140 .Ft void
141 .Fo atf::utils::wait
142 .Fa "const pid_t pid"
143 .Fa "const int expected_exit_status"
144 .Fa "const std::string& expected_stdout"
145 .Fa "const std::string& expected_stderr"
146 .Fc
147 .Sh DESCRIPTION
148 ATF provides a mostly-macro-based programming interface to implement test
149 programs in C or C++.
150 This interface is backed by a C++ implementation, but this fact is
151 hidden from the developer as much as possible through the use of
152 macros to simplify programming.
153 However, the use of C++ is not hidden everywhere and while you can
154 implement test cases without knowing anything at all about the object model
155 underneath the provided calls, you might need some minimum notions of the
156 language in very specific circumstances.
157 .Pp
158 C++-based test programs always follow this template:
159 .Bd -literal -offset indent
160 extern "C" {
161 .Ns ... C-specific includes go here ...
162 }
164 .Ns ... C++-specific includes go here ...
166 #include <atf-c++.hpp>
168 ATF_TEST_CASE(tc1);
169 ATF_TEST_CASE_HEAD(tc1)
170 {
171 ... first test case's header ...
172 }
173 ATF_TEST_CASE_BODY(tc1)
174 {
175 ... first test case's body ...
176 }
178 ATF_TEST_CASE_WITH_CLEANUP(tc2);
179 ATF_TEST_CASE_HEAD(tc2)
180 {
181 ... second test case's header ...
182 }
183 ATF_TEST_CASE_BODY(tc2)
184 {
185 ... second test case's body ...
186 }
187 ATF_TEST_CASE_CLEANUP(tc2)
188 {
189 ... second test case's cleanup ...
190 }
192 ATF_TEST_CASE(tc3);
193 ATF_TEST_CASE_BODY(tc3)
194 {
195 ... third test case's body ...
196 }
198 .Ns ... additional test cases ...
200 ATF_INIT_TEST_CASES(tcs)
201 {
202 ATF_ADD_TEST_CASE(tcs, tc1);
203 ATF_ADD_TEST_CASE(tcs, tc2);
204 ATF_ADD_TEST_CASE(tcs, tc3);
205 ... add additional test cases ...
206 }
207 .Ed
208 .Ss Definition of test cases
209 Test cases have an identifier and are composed of three different parts:
210 the header, the body and an optional cleanup routine, all of which are
211 described in
212 .Xr atf-test-case 4 .
213 To define test cases, one can use the
214 .Fn ATF_TEST_CASE ,
215 .Fn ATF_TEST_CASE_WITH_CLEANUP
216 or the
217 .Fn ATF_TEST_CASE_WITHOUT_HEAD
218 macros, which take a single parameter specifiying the test case's
219 name.
220 .Fn ATF_TEST_CASE ,
221 requires to define a head and a body for the test case,
222 .Fn ATF_TEST_CASE_WITH_CLEANUP
223 requires to define a head, a body and a cleanup for the test case and
224 .Fn ATF_TEST_CASE_WITHOUT_HEAD
225 requires only a body for the test case.
226 It is important to note that these
227 .Em do not
228 set the test case up for execution when the program is run.
229 In order to do so, a later registration is needed through the
230 .Fn ATF_ADD_TEST_CASE
231 macro detailed in
232 .Sx Program initialization .
233 .Pp
234 Later on, one must define the three parts of the body by means of three
235 functions.
236 Their headers are given by the
237 .Fn ATF_TEST_CASE_HEAD ,
238 .Fn ATF_TEST_CASE_BODY
239 and
240 .Fn ATF_TEST_CASE_CLEANUP
241 macros, all of which take the test case's name.
242 Following each of these, a block of code is expected, surrounded by the
243 opening and closing brackets.
244 .Pp
245 Additionally, the
246 .Fn ATF_TEST_CASE_NAME
247 macro can be used to obtain the name of the class corresponding to a
248 particular test case, as the name is internally manged by the library to
249 prevent clashes with other user identifiers.
250 Similarly, the
251 .Fn ATF_TEST_CASE_USE
252 macro can be executed on a particular test case to mark it as "used" and
253 thus prevent compiler warnings regarding unused symbols.
254 Note that
255 .Em you should never have to use these macros during regular operation.
256 .Ss Program initialization
257 The library provides a way to easily define the test program's
258 .Fn main
259 function.
260 You should never define one on your own, but rely on the
261 library to do it for you.
262 This is done by using the
263 .Fn ATF_INIT_TEST_CASES
264 macro, which is passed the name of the list that will hold the test cases.
265 This name can be whatever you want as long as it is a valid variable value.
266 .Pp
267 After the macro, you are supposed to provide the body of a function, which
268 should only use the
269 .Fn ATF_ADD_TEST_CASE
270 macro to register the test cases the test program will execute.
271 The first parameter of this macro matches the name you provided in the
272 former call.
273 .Ss Header definitions
274 The test case's header can define the meta-data by using the
275 .Fn set_md_var
276 method, which takes two parameters: the first one specifies the
277 meta-data variable to be set and the second one specifies its value.
278 Both of them are strings.
279 .Ss Configuration variables
280 The test case has read-only access to the current configuration variables
281 by means of the
282 .Ft bool
283 .Fn has_config_var
284 and the
285 .Ft std::string
286 .Fn get_config_var
287 methods, which can be called in any of the three parts of a test case.
288 .Ss Access to the source directory
289 It is possible to get the path to the test case's source directory from any
290 of its three components by querying the
291 .Sq srcdir
292 configuration variable.
293 .Ss Requiring programs
294 Aside from the
295 .Va require.progs
296 meta-data variable available in the header only, one can also check for
297 additional programs in the test case's body by using the
298 .Fn require_prog
299 function, which takes the base name or full path of a single binary.
300 Relative paths are forbidden.
301 If it is not found, the test case will be automatically skipped.
302 .Ss Test case finalization
303 The test case finalizes either when the body reaches its end, at which
304 point the test is assumed to have
305 .Em passed ,
306 or at any explicit call to
307 .Fn ATF_PASS ,
308 .Fn ATF_FAIL
309 or
310 .Fn ATF_SKIP .
311 These three macros terminate the execution of the test case immediately.
312 The cleanup routine will be processed afterwards in a completely automated
313 way, regardless of the test case's termination reason.
314 .Pp
315 .Fn ATF_PASS
316 does not take any parameters.
317 .Fn ATF_FAIL
318 and
319 .Fn ATF_SKIP
320 take a single string that describes why the test case failed or
321 was skipped, respectively.
322 It is very important to provide a clear error message in both cases so that
323 the user can quickly know why the test did not pass.
324 .Ss Expectations
325 Everything explained in the previous section changes when the test case
326 expectations are redefined by the programmer.
327 .Pp
328 Each test case has an internal state called
329 .Sq expect
330 that describes what the test case expectations are at any point in time.
331 The value of this property can change during execution by any of:
332 .Bl -tag -width indent
333 .It Fn expect_death "reason"
334 Expects the test case to exit prematurely regardless of the nature of the
335 exit.
336 .It Fn expect_exit "exitcode" "reason"
337 Expects the test case to exit cleanly.
338 If
339 .Va exitcode
340 is not
341 .Sq -1 ,
342 .Xr atf-run 1
343 will validate that the exit code of the test case matches the one provided
344 in this call.
345 Otherwise, the exact value will be ignored.
346 .It Fn expect_fail "reason"
347 Any failure (be it fatal or non-fatal) raised in this mode is recorded.
348 However, such failures do not report the test case as failed; instead, the
349 test case finalizes cleanly and is reported as
350 .Sq expected failure ;
351 this report includes the provided
352 .Fa reason
353 as part of it.
354 If no error is raised while running in this mode, then the test case is
355 reported as
356 .Sq failed .
357 .Pp
358 This mode is useful to reproduce actual known bugs in tests.
359 Whenever the developer fixes the bug later on, the test case will start
360 reporting a failure, signaling the developer that the test case must be
361 adjusted to the new conditions.
362 In this situation, it is useful, for example, to set
363 .Fa reason
364 as the bug number for tracking purposes.
365 .It Fn expect_pass
366 This is the normal mode of execution.
367 In this mode, any failure is reported as such to the user and the test case
368 is marked as
369 .Sq failed .
370 .It Fn expect_race "reason"
371 Any failure or timeout during the execution of the test case will be
372 considered as if a race condition has been triggered and reported as such.
373 If no problems arise, the test will continue execution as usual.
374 .It Fn expect_signal "signo" "reason"
375 Expects the test case to terminate due to the reception of a signal.
376 If
377 .Va signo
378 is not
379 .Sq -1 ,
380 .Xr atf-run 1
381 will validate that the signal that terminated the test case matches the one
382 provided in this call.
383 Otherwise, the exact value will be ignored.
384 .It Fn expect_timeout "reason"
385 Expects the test case to execute for longer than its timeout.
386 .El
387 .Ss Helper macros for common checks
388 The library provides several macros that are very handy in multiple
389 situations.
390 These basically check some condition after executing a given statement or
391 processing a given expression and, if the condition is not met, they
392 automatically call
393 .Fn ATF_FAIL
394 with an appropriate error message.
395 .Pp
396 .Fn ATF_REQUIRE
397 takes an expression and raises a failure if it evaluates to false.
398 .Pp
399 .Fn ATF_REQUIRE_EQ
400 takes two expressions and raises a failure if the two do not evaluate to
401 the same exact value.
402 .Pp
403 .Fn ATF_REQUIRE_IN
404 takes an element and a collection and validates that the element is present in
405 the collection.
406 .Pp
407 .Fn ATF_REQUIRE_MATCH
408 takes a regular expression and a string and raises a failure if the regular
409 expression does not match the string.
410 .Pp
411 .Fn ATF_REQUIRE_NOT_IN
412 takes an element and a collection and validates that the element is not present
413 in the collection.
414 .Pp
415 .Fn ATF_REQUIRE_THROW
416 takes the name of an exception and a statement and raises a failure if
417 the statement does not throw the specified exception.
418 .Fn ATF_REQUIRE_THROW_RE
419 takes the name of an exception, a regular expresion and a statement and raises a
420 failure if the statement does not throw the specified exception and if the
421 message of the exception does not match the regular expression.
422 .Pp
423 .Fn ATF_CHECK_ERRNO
424 and
425 .Fn ATF_REQUIRE_ERRNO
426 take, first, the error code that the check is expecting to find in the
427 .Va errno
428 variable and, second, a boolean expression that, if evaluates to true,
429 means that a call failed and
430 .Va errno
431 has to be checked against the first value.
432 .Ss Utility functions
433 The following functions are provided as part of the
434 .Nm
435 API to simplify the creation of a variety of tests.
436 In particular, these are useful to write tests for command-line interfaces.
437 .Pp
438 .Ft void
439 .Fo atf::utils::cat_file
440 .Fa "const std::string& path"
441 .Fa "const std::string& prefix"
442 .Fc
443 .Bd -offset indent
444 Prints the contents of
445 .Fa path
446 to the standard output, prefixing every line with the string in
447 .Fa prefix .
448 .Ed
449 .Pp
450 .Ft bool
451 .Fo atf::utils::compare_file
452 .Fa "const std::string& path"
453 .Fa "const std::string& contents"
454 .Fc
455 .Bd -offset indent
456 Returns true if the given
457 .Fa path
458 matches exactly the expected inlined
459 .Fa contents .
460 .Ed
461 .Pp
462 .Ft void
463 .Fo atf::utils::copy_file
464 .Fa "const std::string& source"
465 .Fa "const std::string& destination"
466 .Fc
467 .Bd -offset indent
468 Copies the file
469 .Fa source
470 to
471 .Fa destination .
472 The permissions of the file are preserved during the code.
473 .Ed
474 .Pp
475 .Ft void
476 .Fo atf::utils::create_file
477 .Fa "const std::string& path"
478 .Fa "const std::string& contents"
479 .Fc
480 .Bd -offset indent
481 Creates
482 .Fa file
483 with the text given in
484 .Fa contents .
485 .Ed
486 .Pp
487 .Ft void
488 .Fo atf::utils::file_exists
489 .Fa "const std::string& path"
490 .Fc
491 .Bd -offset indent
492 Checks if
493 .Fa path
494 exists.
495 .Ed
496 .Pp
497 .Ft pid_t
498 .Fo atf::utils::fork
499 .Fa "void"
500 .Fc
501 .Bd -offset indent
502 Forks a process and redirects the standard output and standard error of the
503 child to files for later validation with
504 .Fn atf::utils::wait .
505 Fails the test case if the fork fails, so this does not return an error.
506 .Ed
507 .Pp
508 .Ft bool
509 .Fo atf::utils::grep_collection
510 .Fa "const std::string& regexp"
511 .Fa "const Collection& collection"
512 .Fc
513 .Bd -offset indent
514 Searches for the regular expression
515 .Fa regexp
516 in any of the strings contained in the
517 .Fa collection .
518 This is a template that accepts any one-dimensional container of strings.
519 .Ed
520 .Pp
521 .Ft bool
522 .Fo atf::utils::grep_file
523 .Fa "const std::string& regexp"
524 .Fa "const std::string& path"
525 .Fc
526 .Bd -offset indent
527 Searches for the regular expression
528 .Fa regexp
529 in the file
530 .Fa path .
531 The variable arguments are used to construct the regular expression.
532 .Ed
533 .Pp
534 .Ft bool
535 .Fo atf::utils::grep_string
536 .Fa "const std::string& regexp"
537 .Fa "const std::string& str"
538 .Fc
539 .Bd -offset indent
540 Searches for the regular expression
541 .Fa regexp
542 in the string
543 .Fa str .
544 .Ed
545 .Ft void
546 .Fo atf::utils::redirect
547 .Fa "const int fd"
548 .Fa "const std::string& path"
549 .Fc
550 .Bd -offset indent
551 Redirects the given file descriptor
552 .Fa fd
553 to the file
554 .Fa path .
555 This function exits the process in case of an error and does not properly mark
556 the test case as failed.
557 As a result, it should only be used in subprocesses of the test case; specially
558 those spawned by
559 .Fn atf::utils::fork .
560 .Ed
561 .Pp
562 .Ft void
563 .Fo atf::utils::wait
564 .Fa "const pid_t pid"
565 .Fa "const int expected_exit_status"
566 .Fa "const std::string& expected_stdout"
567 .Fa "const std::string& expected_stderr"
568 .Fc
569 .Bd -offset indent
570 Waits and validates the result of a subprocess spawned with
571 .Fn atf::utils::wait .
572 The validation involves checking that the subprocess exited cleanly and returned
573 the code specified in
574 .Fa expected_exit_status
575 and that its standard output and standard error match the strings given in
576 .Fa expected_stdout
577 and
578 .Fa expected_stderr .
579 .Pp
580 If any of the
581 .Fa expected_stdout
582 or
583 .Fa expected_stderr
584 strings are prefixed with
585 .Sq save: ,
586 then they specify the name of the file into which to store the stdout or stderr
587 of the subprocess, and no comparison is performed.
588 .Ed
589 .Sh EXAMPLES
590 The following shows a complete test program with a single test case that
591 validates the addition operator:
592 .Bd -literal -offset indent
593 #include <atf-c++.hpp>
595 ATF_TEST_CASE(addition);
596 ATF_TEST_CASE_HEAD(addition)
597 {
598 set_md_var("descr", "Sample tests for the addition operator");
599 }
600 ATF_TEST_CASE_BODY(addition)
601 {
602 ATF_REQUIRE_EQ(0 + 0, 0);
603 ATF_REQUIRE_EQ(0 + 1, 1);
604 ATF_REQUIRE_EQ(1 + 0, 1);
606 ATF_REQUIRE_EQ(1 + 1, 2);
608 ATF_REQUIRE_EQ(100 + 200, 300);
609 }
611 ATF_TEST_CASE(open_failure);
612 ATF_TEST_CASE_HEAD(open_failure)
613 {
614 set_md_var("descr", "Sample tests for the open function");
615 }
616 ATF_TEST_CASE_BODY(open_failure)
617 {
618 ATF_REQUIRE_ERRNO(ENOENT, open("non-existent", O_RDONLY) == -1);
619 }
621 ATF_TEST_CASE(known_bug);
622 ATF_TEST_CASE_HEAD(known_bug)
623 {
624 set_md_var("descr", "Reproduces a known bug");
625 }
626 ATF_TEST_CASE_BODY(known_bug)
627 {
628 expect_fail("See bug number foo/bar");
629 ATF_REQUIRE_EQ(3, 1 + 1);
630 expect_pass();
631 ATF_REQUIRE_EQ(3, 1 + 2);
632 }
634 ATF_INIT_TEST_CASES(tcs)
635 {
636 ATF_ADD_TEST_CASE(tcs, addition);
637 ATF_ADD_TEST_CASE(tcs, open_failure);
638 ATF_ADD_TEST_CASE(tcs, known_bug);
639 }
640 .Ed
641 .Sh SEE ALSO
642 .Xr atf-test-program 1 ,
643 .Xr atf-test-case 4 ,
644 .Xr atf 7