2 // Automated Testing Framework (atf)
4 // Copyright (c) 2008 The NetBSD Foundation, Inc.
5 // All rights reserved.
7 // Redistribution and use in source and binary forms, with or without
8 // modification, are permitted provided that the following conditions
10 // 1. Redistributions of source code must retain the above copyright
11 // notice, this list of conditions and the following disclaimer.
12 // 2. Redistributions in binary form must reproduce the above copyright
13 // notice, this list of conditions and the following disclaimer in the
14 // documentation and/or other materials provided with the distribution.
16 // THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND
17 // CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
18 // INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
19 // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 // IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY
21 // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
23 // GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24 // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
25 // IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
26 // OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
27 // IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31 #include <sys/types.h>
50 #include "atf-c++/check.hpp"
51 #include "atf-c++/config.hpp"
53 #include "atf-c++/detail/application.hpp"
54 #include "atf-c++/detail/auto_array.hpp"
55 #include "atf-c++/detail/exceptions.hpp"
56 #include "atf-c++/detail/fs.hpp"
57 #include "atf-c++/detail/process.hpp"
58 #include "atf-c++/detail/sanity.hpp"
59 #include "atf-c++/detail/text.hpp"
61 // ------------------------------------------------------------------------
62 // Auxiliary functions.
63 // ------------------------------------------------------------------------
78 status_check(const status_check_t
& p_type
, const bool p_negated
,
101 output_check(const output_check_t
& p_type
, const bool p_negated
,
102 const std::string
& p_value
) :
110 class temp_file
: public std::ostream
{
111 std::auto_ptr
< atf::fs::path
> m_path
;
115 temp_file(const atf::fs::path
& p
) :
119 atf::auto_array
< char > buf(new char[p
.str().length() + 1]);
120 std::strcpy(buf
.get(), p
.c_str());
122 m_fd
= ::mkstemp(buf
.get());
124 throw atf::system_error("atf_check::temp_file::temp_file(" +
125 p
.str() + ")", "mkstemp(3) failed",
128 m_path
.reset(new atf::fs::path(buf
.get()));
136 } catch (const atf::system_error
&) {
137 // Ignore deletion errors.
148 write(const std::string
& text
)
150 if (::write(m_fd
, text
.c_str(), text
.size()) == -1)
151 throw atf::system_error("atf_check", "write(2) failed", errno
);
165 } // anonymous namespace
168 parse_exit_code(const std::string
& str
)
171 const int value
= atf::text::to_type
< int >(str
);
172 if (value
< 0 || value
> 255)
173 throw std::runtime_error("Unused reason");
175 } catch (const std::runtime_error
&) {
176 throw atf::application::usage_error("Invalid exit code for -s option; "
177 "must be an integer in range 0-255");
181 static struct name_number
{
184 } signal_names_to_numbers
[] = {
201 signal_name_to_number(const std::string
& str
)
203 struct name_number
* iter
= signal_names_to_numbers
;
205 while (signo
== INT_MIN
&& iter
->name
!= NULL
) {
206 if (str
== iter
->name
|| str
== std::string("sig") + iter
->name
)
215 parse_signal(const std::string
& str
)
217 const int signo
= signal_name_to_number(str
);
218 if (signo
== INT_MIN
) {
220 return atf::text::to_type
< int >(str
);
221 } catch (std::runtime_error
) {
222 throw atf::application::usage_error("Invalid signal name or number "
226 INV(signo
!= INT_MIN
);
231 parse_status_check_arg(const std::string
& arg
)
233 const std::string::size_type delimiter
= arg
.find(':');
234 bool negated
= (arg
.compare(0, 4, "not-") == 0);
235 const std::string action_str
= arg
.substr(0, delimiter
);
236 const std::string action
= negated
? action_str
.substr(4) : action_str
;
237 const std::string value_str
= (
238 delimiter
== std::string::npos
? "" : arg
.substr(delimiter
+ 1));
242 if (action
== "eq") {
243 // Deprecated; use exit instead. TODO: Remove after 0.10.
246 throw atf::application::usage_error("Cannot negate eq checker");
248 value
= parse_exit_code(value_str
);
249 } else if (action
== "exit") {
251 if (value_str
.empty())
254 value
= parse_exit_code(value_str
);
255 } else if (action
== "ignore") {
257 throw atf::application::usage_error("Cannot negate ignore checker");
260 } else if (action
== "ne") {
261 // Deprecated; use not-exit instead. TODO: Remove after 0.10.
264 throw atf::application::usage_error("Cannot negate ne checker");
266 value
= parse_exit_code(value_str
);
267 } else if (action
== "signal") {
269 if (value_str
.empty())
272 value
= parse_signal(value_str
);
274 throw atf::application::usage_error("Invalid status checker");
276 return status_check(type
, negated
, value
);
281 parse_output_check_arg(const std::string
& arg
)
283 const std::string::size_type delimiter
= arg
.find(':');
284 const bool negated
= (arg
.compare(0, 4, "not-") == 0);
285 const std::string action_str
= arg
.substr(0, delimiter
);
286 const std::string action
= negated
? action_str
.substr(4) : action_str
;
289 if (action
== "empty")
291 else if (action
== "file")
293 else if (action
== "ignore") {
295 throw atf::application::usage_error("Cannot negate ignore checker");
297 } else if (action
== "inline")
299 else if (action
== "match")
301 else if (action
== "save") {
303 throw atf::application::usage_error("Cannot negate save checker");
306 throw atf::application::usage_error("Invalid output checker");
308 return output_check(type
, negated
, arg
.substr(delimiter
+ 1));
313 flatten_argv(char* const* argv
)
317 char* const* arg
= &argv
[0];
318 while (*arg
!= NULL
) {
331 std::auto_ptr
< atf::check::check_result
>
332 execute(const char* const* argv
)
334 // TODO: This should go to stderr... but fixing it now may be hard as test
335 // cases out there might be relying on stderr being silent.
336 std::cout
<< "Executing command [ ";
337 for (int i
= 0; argv
[i
] != NULL
; ++i
)
338 std::cout
<< argv
[i
] << " ";
342 atf::process::argv_array
argva(argv
);
343 return atf::check::exec(argva
);
347 std::auto_ptr
< atf::check::check_result
>
348 execute_with_shell(char* const* argv
)
350 const std::string cmd
= flatten_argv(argv
);
352 const char* sh_argv
[4];
353 sh_argv
[0] = atf::config::get("atf_shell").c_str();
355 sh_argv
[2] = cmd
.c_str();
357 return execute(sh_argv
);
362 cat_file(const atf::fs::path
& path
)
364 std::ifstream
stream(path
.c_str());
366 throw std::runtime_error("Failed to open " + path
.str());
368 stream
>> std::noskipws
;
369 std::istream_iterator
< char > begin(stream
), end
;
370 std::ostream_iterator
< char > out(std::cerr
);
371 std::copy(begin
, end
, out
);
378 grep_file(const atf::fs::path
& path
, const std::string
& regexp
)
380 std::ifstream
stream(path
.c_str());
382 throw std::runtime_error("Failed to open " + path
.str());
387 while (!found
&& !std::getline(stream
, line
).fail()) {
388 if (atf::text::match(line
, regexp
))
399 file_empty(const atf::fs::path
& p
)
401 atf::fs::file_info
f(p
);
403 return (f
.get_size() == 0);
407 compare_files(const atf::fs::path
& p1
, const atf::fs::path
& p2
)
411 std::ifstream
f1(p1
.c_str());
413 throw std::runtime_error("Failed to open " + p1
.str());
415 std::ifstream
f2(p2
.c_str());
417 throw std::runtime_error("Failed to open " + p1
.str());
420 char buf1
[512], buf2
[512];
422 f1
.read(buf1
, sizeof(buf1
));
424 throw std::runtime_error("Failed to read from " + p1
.str());
426 f2
.read(buf2
, sizeof(buf2
));
428 throw std::runtime_error("Failed to read from " + p1
.str());
430 if ((f1
.gcount() == 0) && (f2
.gcount() == 0)) {
435 if ((f1
.gcount() != f2
.gcount()) ||
436 (std::memcmp(buf1
, buf2
, f1
.gcount()) != 0)) {
446 print_diff(const atf::fs::path
& p1
, const atf::fs::path
& p2
)
448 const atf::process::status s
=
449 atf::process::exec(atf::fs::path("diff"),
450 atf::process::argv_array("diff", "-u", p1
.c_str(),
452 atf::process::stream_connect(STDOUT_FILENO
,
454 atf::process::stream_inherit());
457 std::cerr
<< "Failed to run diff(3)\n";
459 if (s
.exitstatus() != 1)
460 std::cerr
<< "Error while running diff(3)\n";
465 decode(const std::string
& s
)
470 res
.reserve(s
.length());
473 while (i
< s
.length()) {
478 case 'a': c
= '\a'; break;
479 case 'b': c
= '\b'; break;
481 case 'e': c
= 033; break;
482 case 'f': c
= '\f'; break;
483 case 'n': c
= '\n'; break;
484 case 'r': c
= '\r'; break;
485 case 't': c
= '\t'; break;
486 case 'v': c
= '\v'; break;
492 while (--count
>= 0 && (unsigned)(s
[i
] - '0') < 8)
493 c
= (c
<< 3) + (s
[i
++] - '0');
510 run_status_check(const status_check
& sc
, const atf::check::check_result
& cr
)
514 if (sc
.type
== sc_exit
) {
515 if (cr
.exited() && sc
.value
!= INT_MIN
) {
516 const int status
= cr
.exitcode();
518 if (!sc
.negated
&& sc
.value
!= status
) {
519 std::cerr
<< "Fail: incorrect exit status: "
520 << status
<< ", expected: "
523 } else if (sc
.negated
&& sc
.value
== status
) {
524 std::cerr
<< "Fail: incorrect exit status: "
525 << status
<< ", expected: "
526 << "anything else\n";
530 } else if (cr
.exited() && sc
.value
== INT_MIN
) {
533 std::cerr
<< "Fail: program did not exit cleanly\n";
536 } else if (sc
.type
== sc_ignore
) {
538 } else if (sc
.type
== sc_signal
) {
539 if (cr
.signaled() && sc
.value
!= INT_MIN
) {
540 const int status
= cr
.termsig();
542 if (!sc
.negated
&& sc
.value
!= status
) {
543 std::cerr
<< "Fail: incorrect signal received: "
544 << status
<< ", expected: " << sc
.value
<< "\n";
546 } else if (sc
.negated
&& sc
.value
== status
) {
547 std::cerr
<< "Fail: incorrect signal received: "
548 << status
<< ", expected: "
549 << "anything else\n";
553 } else if (cr
.signaled() && sc
.value
== INT_MIN
) {
556 std::cerr
<< "Fail: program did not receive a signal\n";
564 if (result
== false) {
565 std::cerr
<< "stdout:\n";
566 cat_file(atf::fs::path(cr
.stdout_path()));
569 std::cerr
<< "stderr:\n";
570 cat_file(atf::fs::path(cr
.stderr_path()));
579 run_status_checks(const std::vector
< status_check
>& checks
,
580 const atf::check::check_result
& result
)
584 for (std::vector
< status_check
>::const_iterator iter
= checks
.begin();
585 !ok
&& iter
!= checks
.end(); iter
++) {
586 ok
|= run_status_check(*iter
, result
);
594 run_output_check(const output_check oc
, const atf::fs::path
& path
,
595 const std::string
& stdxxx
)
599 if (oc
.type
== oc_empty
) {
600 const bool is_empty
= file_empty(path
);
601 if (!oc
.negated
&& !is_empty
) {
602 std::cerr
<< "Fail: " << stdxxx
<< " not empty\n";
603 print_diff(atf::fs::path("/dev/null"), path
);
605 } else if (oc
.negated
&& is_empty
) {
606 std::cerr
<< "Fail: " << stdxxx
<< " is empty\n";
610 } else if (oc
.type
== oc_file
) {
611 const bool equals
= compare_files(path
, atf::fs::path(oc
.value
));
612 if (!oc
.negated
&& !equals
) {
613 std::cerr
<< "Fail: " << stdxxx
<< " does not match golden "
615 print_diff(atf::fs::path(oc
.value
), path
);
617 } else if (oc
.negated
&& equals
) {
618 std::cerr
<< "Fail: " << stdxxx
<< " matches golden output\n";
619 cat_file(atf::fs::path(oc
.value
));
623 } else if (oc
.type
== oc_ignore
) {
625 } else if (oc
.type
== oc_inline
) {
626 atf::fs::path path2
= atf::fs::path(atf::config::get("atf_workdir"))
628 temp_file
temp(path2
);
629 temp
.write(decode(oc
.value
));
632 const bool equals
= compare_files(path
, temp
.get_path());
633 if (!oc
.negated
&& !equals
) {
634 std::cerr
<< "Fail: " << stdxxx
<< " does not match expected "
636 print_diff(temp
.get_path(), path
);
638 } else if (oc
.negated
&& equals
) {
639 std::cerr
<< "Fail: " << stdxxx
<< " matches expected value\n";
640 cat_file(temp
.get_path());
644 } else if (oc
.type
== oc_match
) {
645 const bool matches
= grep_file(path
, oc
.value
);
646 if (!oc
.negated
&& !matches
) {
647 std::cerr
<< "Fail: regexp " + oc
.value
+ " not in " << stdxxx
651 } else if (oc
.negated
&& matches
) {
652 std::cerr
<< "Fail: regexp " + oc
.value
+ " is in " << stdxxx
658 } else if (oc
.type
== oc_save
) {
660 std::ifstream
ifs(path
.c_str(), std::fstream::binary
);
661 ifs
>> std::noskipws
;
662 std::istream_iterator
< char > begin(ifs
), end
;
664 std::ofstream
ofs(oc
.value
.c_str(), std::fstream::binary
665 | std::fstream::trunc
);
666 std::ostream_iterator
<char> obegin(ofs
);
668 std::copy(begin
, end
, obegin
);
680 run_output_checks(const std::vector
< output_check
>& checks
,
681 const atf::fs::path
& path
, const std::string
& stdxxx
)
685 for (std::vector
< output_check
>::const_iterator iter
= checks
.begin();
686 iter
!= checks
.end(); iter
++) {
687 ok
&= run_output_check(*iter
, path
, stdxxx
);
693 // ------------------------------------------------------------------------
694 // The "atf_check" application.
695 // ------------------------------------------------------------------------
699 class atf_check
: public atf::application::app
{
702 std::vector
< status_check
> m_status_checks
;
703 std::vector
< output_check
> m_stdout_checks
;
704 std::vector
< output_check
> m_stderr_checks
;
706 static const char* m_description
;
708 bool run_output_checks(const atf::check::check_result
&,
709 const std::string
&) const;
711 std::string
specific_args(void) const;
712 options_set
specific_options(void) const;
713 void process_option(int, const char*);
714 void process_option_s(const std::string
&);
721 } // anonymous namespace
723 const char* atf_check::m_description
=
724 "atf-check executes given command and analyzes its results.";
726 atf_check::atf_check(void) :
727 app(m_description
, "atf-check(1)"),
733 atf_check::run_output_checks(const atf::check::check_result
& r
,
734 const std::string
& stdxxx
)
737 if (stdxxx
== "stdout") {
738 return ::run_output_checks(m_stdout_checks
,
739 atf::fs::path(r
.stdout_path()), "stdout");
740 } else if (stdxxx
== "stderr") {
741 return ::run_output_checks(m_stderr_checks
,
742 atf::fs::path(r
.stderr_path()), "stderr");
750 atf_check::specific_args(void)
756 atf_check::options_set
757 atf_check::specific_options(void)
760 using atf::application::option
;
763 opts
.insert(option('s', "qual:value", "Handle status. Qualifier "
764 "must be one of: ignore exit:<num> signal:<name|num>"));
765 opts
.insert(option('o', "action:arg", "Handle stdout. Action must be "
766 "one of: empty ignore file:<path> inline:<val> match:regexp "
768 opts
.insert(option('e', "action:arg", "Handle stderr. Action must be "
769 "one of: empty ignore file:<path> inline:<val> match:regexp "
771 opts
.insert(option('x', "", "Execute command as a shell command"));
777 atf_check::process_option(int ch
, const char* arg
)
781 m_status_checks
.push_back(parse_status_check_arg(arg
));
785 m_stdout_checks
.push_back(parse_output_check_arg(arg
));
789 m_stderr_checks
.push_back(parse_output_check_arg(arg
));
802 atf_check::main(void)
805 throw atf::application::usage_error("No command specified");
807 int status
= EXIT_FAILURE
;
809 std::auto_ptr
< atf::check::check_result
> r
=
810 m_xflag
? execute_with_shell(m_argv
) : execute(m_argv
);
812 if (m_status_checks
.empty())
813 m_status_checks
.push_back(status_check(sc_exit
, false, EXIT_SUCCESS
));
814 else if (m_status_checks
.size() > 1) {
815 // TODO: Remove this restriction.
816 throw atf::application::usage_error("Cannot specify -s more than once");
819 if (m_stdout_checks
.empty())
820 m_stdout_checks
.push_back(output_check(oc_empty
, false, ""));
821 if (m_stderr_checks
.empty())
822 m_stderr_checks
.push_back(output_check(oc_empty
, false, ""));
824 if ((run_status_checks(m_status_checks
, *r
) == false) ||
825 (run_output_checks(*r
, "stderr") == false) ||
826 (run_output_checks(*r
, "stdout") == false))
827 status
= EXIT_FAILURE
;
829 status
= EXIT_SUCCESS
;
835 main(int argc
, char* const* argv
)
837 return atf_check().run(argc
, argv
);