Remove building with NOCRYPTO option
[minix3.git] / external / bsd / atf / dist / tools / atf-report_test.sh
blob9f0f47fc61cf998e694563c4ee2b9ace231eff18
2 # Automated Testing Framework (atf)
4 # Copyright (c) 2007 The NetBSD Foundation, Inc.
5 # All rights reserved.
7 # Redistribution and use in source and binary forms, with or without
8 # modification, are permitted provided that the following conditions
9 # are met:
10 # 1. Redistributions of source code must retain the above copyright
11 # notice, this list of conditions and the following disclaimer.
12 # 2. Redistributions in binary form must reproduce the above copyright
13 # notice, this list of conditions and the following disclaimer in the
14 # documentation and/or other materials provided with the distribution.
16 # THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND
17 # CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
18 # INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
19 # MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 # IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY
21 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
23 # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24 # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
25 # IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
26 # OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
27 # IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 create_helpers()
32 mkdir dir1
33 cp $(atf_get_srcdir)/pass_helper dir1/tp1
34 cp $(atf_get_srcdir)/fail_helper dir1/tp2
35 cp $(atf_get_srcdir)/pass_helper tp3
36 cp $(atf_get_srcdir)/fail_helper tp4
38 cat >tp5 <<EOF
39 #! $(atf-config -t atf_shell)
40 echo foo
41 EOF
42 chmod +x tp5
44 cat >Atffile <<EOF
45 Content-Type: application/X-atf-atffile; version="1"
47 prop: test-suite = atf
49 tp: dir1
50 tp: tp3
51 tp: tp4
52 tp: tp5
53 EOF
55 cat >dir1/Atffile <<EOF
56 Content-Type: application/X-atf-atffile; version="1"
58 prop: test-suite = atf
60 tp: tp1
61 tp: tp2
62 EOF
65 run_helpers()
67 mkdir etc
68 cat >etc/atf-run.hooks <<EOF
69 #! $(atf-config -t atf_shell)
71 info_start_hook()
73 atf_tps_writer_info "startinfo" "A value"
76 info_end_hook()
78 atf_tps_writer_info "endinfo" "Another value"
80 EOF
81 echo "Using atf-run to run helpers"
82 ATF_CONFDIR=$(pwd)/etc atf-run >tps.out 2>/dev/null
83 rm -rf etc
86 atf_test_case default
87 default_head()
89 atf_set "descr" "Checks that the default output uses the ticker" \
90 "format"
92 default_body()
94 create_helpers
95 run_helpers
97 # Check that the default output uses the ticker format.
98 atf_check -s eq:0 -o match:'test cases' -o match:'Failed test cases' \
99 -o match:'Summary for' -e empty -x 'atf-report <tps.out'
102 # XXX The test for all expect_ values should be intermixed with the other
103 # tests. However, to do that, we need to migrate to using C helpers for
104 # simplicity in raising signals...
105 atf_test_case expect
106 expect_body()
108 ln -s "$(atf_get_srcdir)/expect_helpers" .
109 cat >Atffile <<EOF
110 Content-Type: application/X-atf-atffile; version="1"
112 prop: test-suite = atf
114 tp: expect_helpers
116 run_helpers
118 # NO_CHECK_STYLE_BEGIN
119 cat >expout <<EOF
120 tc, #.#, expect_helpers, death_and_exit, expected_death, Exit case
121 tc, #.#, expect_helpers, death_and_signal, expected_death, Signal case
122 tc, #.#, expect_helpers, death_but_pass, failed, Test case was expected to terminate abruptly but it continued execution
123 tc, #.#, expect_helpers, exit_any_and_exit, expected_exit, Call will exit
124 tc, #.#, expect_helpers, exit_but_pass, failed, Test case was expected to exit cleanly but it continued execution
125 tc, #.#, expect_helpers, exit_code_and_exit, expected_exit, Call will exit
126 tc, #.#, expect_helpers, fail_and_fail_check, expected_failure, And fail again: 2 checks failed as expected; see output for more details
127 tc, #.#, expect_helpers, fail_and_fail_requirement, expected_failure, Fail reason: The failure
128 tc, #.#, expect_helpers, fail_but_pass, failed, Test case was expecting a failure but none were raised
129 tc, #.#, expect_helpers, pass_and_pass, passed
130 tc, #.#, expect_helpers, pass_but_fail_check, failed, 1 checks failed; see output for more details
131 tc, #.#, expect_helpers, pass_but_fail_requirement, failed, Some reason
132 tc, #.#, expect_helpers, signal_any_and_signal, expected_signal, Call will signal
133 tc, #.#, expect_helpers, signal_but_pass, failed, Test case was expected to receive a termination signal but it continued execution
134 tc, #.#, expect_helpers, signal_no_and_signal, expected_signal, Call will signal
135 tc, #.#, expect_helpers, timeout_and_hang, expected_timeout, Will overrun
136 tc, #.#, expect_helpers, timeout_but_pass, failed, Test case was expected to hang but it continued execution
137 tp, #.#, expect_helpers, failed
139 # NO_CHECK_STYLE_END
140 atf_check -s eq:0 -o file:expout -e empty -x \
141 "atf-report -o csv:- <tps.out | " \
142 "sed -E -e 's/[0-9]+.[0-9]{6}, /#.#, /'"
144 # NO_CHECK_STYLE_BEGIN
145 cat >expout <<EOF
146 expect_helpers (1/1): 17 test cases
147 death_and_exit: [#.#s] Expected failure: Exit case
148 death_and_signal: [#.#s] Expected failure: Signal case
149 death_but_pass: [#.#s] Failed: Test case was expected to terminate abruptly but it continued execution
150 exit_any_and_exit: [#.#s] Expected failure: Call will exit
151 exit_but_pass: [#.#s] Failed: Test case was expected to exit cleanly but it continued execution
152 exit_code_and_exit: [#.#s] Expected failure: Call will exit
153 fail_and_fail_check: [#.#s] Expected failure: And fail again: 2 checks failed as expected; see output for more details
154 fail_and_fail_requirement: [#.#s] Expected failure: Fail reason: The failure
155 fail_but_pass: [#.#s] Failed: Test case was expecting a failure but none were raised
156 pass_and_pass: [#.#s] Passed.
157 pass_but_fail_check: [#.#s] Failed: 1 checks failed; see output for more details
158 pass_but_fail_requirement: [#.#s] Failed: Some reason
159 signal_any_and_signal: [#.#s] Expected failure: Call will signal
160 signal_but_pass: [#.#s] Failed: Test case was expected to receive a termination signal but it continued execution
161 signal_no_and_signal: [#.#s] Expected failure: Call will signal
162 timeout_and_hang: [#.#s] Expected failure: Will overrun
163 timeout_but_pass: [#.#s] Failed: Test case was expected to hang but it continued execution
164 [#.#s]
166 Test cases for known bugs:
167 expect_helpers:death_and_exit: Exit case
168 expect_helpers:death_and_signal: Signal case
169 expect_helpers:exit_any_and_exit: Call will exit
170 expect_helpers:exit_code_and_exit: Call will exit
171 expect_helpers:fail_and_fail_check: And fail again: 2 checks failed as expected; see output for more details
172 expect_helpers:fail_and_fail_requirement: Fail reason: The failure
173 expect_helpers:signal_any_and_signal: Call will signal
174 expect_helpers:signal_no_and_signal: Call will signal
175 expect_helpers:timeout_and_hang: Will overrun
177 Failed test cases:
178 expect_helpers:death_but_pass, expect_helpers:exit_but_pass, expect_helpers:fail_but_pass, expect_helpers:pass_but_fail_check, expect_helpers:pass_but_fail_requirement, expect_helpers:signal_but_pass, expect_helpers:timeout_but_pass
180 Summary for 1 test programs:
181 1 passed test cases.
182 7 failed test cases.
183 9 expected failed test cases.
184 0 skipped test cases.
186 # NO_CHECK_STYLE_END
187 atf_check -s eq:0 -o file:expout -e empty -x \
188 "atf-report -o ticker:- <tps.out | " \
189 "sed -E -e 's/[0-9]+.[0-9]{6}/#.#/'"
191 # Just ensure that this does not crash for now...
192 atf_check -s eq:0 -o ignore -e empty -x "atf-report -o xml:- <tps.out"
195 atf_test_case oflag
196 oflag_head()
198 atf_set "descr" "Checks that the -o flag works"
200 oflag_body()
202 create_helpers
203 run_helpers
205 # Get the default output.
206 atf_check -s eq:0 -o save:stdout -e empty -x 'atf-report <tps.out'
207 mv stdout defout
209 # Check that changing the stdout output works.
210 atf_check -s eq:0 -o save:stdout -e empty -x 'atf-report -o csv:- <tps.out'
211 atf_check -s eq:1 -o empty -e empty cmp -s defout stdout
212 cp stdout expcsv
214 # Check that sending the output to a file does not write to stdout.
215 atf_check -s eq:0 -o empty -e empty -x 'atf-report -o csv:fmt.out <tps.out'
216 atf_check -s eq:0 -o empty -e empty cmp -s expcsv fmt.out
217 rm -f fmt.out
219 # Check that defining two outputs using the same format works.
220 atf_check -s eq:0 -o empty -e empty -x \
221 'atf-report -o csv:fmt.out -o csv:fmt2.out <tps.out'
222 atf_check -s eq:0 -o empty -e empty cmp -s expcsv fmt.out
223 atf_check -s eq:0 -o empty -e empty cmp -s fmt.out fmt2.out
224 rm -f fmt.out fmt2.out
226 # Check that defining two outputs using different formats works.
227 atf_check -s eq:0 -o empty -e empty -x \
228 'atf-report -o csv:fmt.out -o ticker:fmt2.out <tps.out'
229 atf_check -s eq:0 -o empty -e empty cmp -s expcsv fmt.out
230 atf_check -s eq:1 -o empty -e empty cmp -s fmt.out fmt2.out
231 atf_check -s eq:0 -o ignore -e empty grep "test cases" fmt2.out
232 atf_check -s eq:0 -o ignore -e empty grep "Failed test cases" fmt2.out
233 atf_check -s eq:0 -o ignore -e empty grep "Summary for" fmt2.out
234 rm -f fmt.out fmt2.out
236 # Check that defining two outputs over the same file does not work.
237 atf_check -s eq:1 -o empty -e match:'more than once' -x \
238 'atf-report -o csv:fmt.out -o ticker:fmt.out <tps.out'
239 rm -f fmt.out
241 # Check that defining two outputs over stdout (but using different
242 # paths) does not work.
243 atf_check -s eq:1 -o empty -e match:'more than once' -x \
244 'atf-report -o csv:- -o ticker:/dev/stdout <tps.out'
245 rm -f fmt.out
248 atf_test_case output_csv
249 output_csv_head()
251 atf_set "descr" "Checks the CSV output format"
253 output_csv_body()
255 create_helpers
256 run_helpers
258 # NO_CHECK_STYLE_BEGIN
259 cat >expout <<EOF
260 tc, #.#, dir1/tp1, main, passed
261 tp, #.#, dir1/tp1, passed
262 tc, #.#, dir1/tp2, main, failed, This always fails
263 tp, #.#, dir1/tp2, failed
264 tc, #.#, tp3, main, passed
265 tp, #.#, tp3, passed
266 tc, #.#, tp4, main, failed, This always fails
267 tp, #.#, tp4, failed
268 tp, #.#, tp5, bogus, Invalid format for test case list: 1: Unexpected token \`<<NEWLINE>>'; expected \`:'
270 # NO_CHECK_STYLE_END
272 atf_check -s eq:0 -o file:expout -e empty -x \
273 "atf-report -o csv:- <tps.out | sed -E -e 's/[0-9]+.[0-9]{6}, /#.#, /'"
276 atf_test_case output_ticker
277 output_ticker_head()
279 atf_set "descr" "Checks the ticker output format"
281 output_ticker_body()
283 create_helpers
284 run_helpers
286 # NO_CHECK_STYLE_BEGIN
287 cat >expout <<EOF
288 dir1/tp1 (1/5): 1 test cases
289 main: [#.#s] Passed.
290 [#.#s]
292 dir1/tp2 (2/5): 1 test cases
293 main: [#.#s] Failed: This always fails
294 [#.#s]
296 tp3 (3/5): 1 test cases
297 main: [#.#s] Passed.
298 [#.#s]
300 tp4 (4/5): 1 test cases
301 main: [#.#s] Failed: This always fails
302 [#.#s]
304 tp5 (5/5): 0 test cases
305 tp5: BOGUS TEST PROGRAM: Cannot trust its results because of \`Invalid format for test case list: 1: Unexpected token \`<<NEWLINE>>'; expected \`:''
306 [#.#s]
308 Failed (bogus) test programs:
311 Failed test cases:
312 dir1/tp2:main, tp4:main
314 Summary for 5 test programs:
315 2 passed test cases.
316 2 failed test cases.
317 0 expected failed test cases.
318 0 skipped test cases.
321 atf_check -s eq:0 -o file:expout -e empty -x \
322 "atf-report -o ticker:- <tps.out | sed -E -e 's/[0-9]+.[0-9]{6}/#.#/'"
324 # NO_CHECK_STYLE_END
326 atf_test_case output_xml
327 output_xml_head()
329 atf_set "descr" "Checks the XML output format"
331 output_xml_body()
333 create_helpers
334 run_helpers
336 # NO_CHECK_STYLE_BEGIN
337 cat >expout <<EOF
338 <?xml version="1.0" encoding="ISO-8859-1"?>
339 <!DOCTYPE tests-results PUBLIC "-//NetBSD//DTD ATF Tests Results 0.1//EN" "http://www.NetBSD.org/XML/atf/tests-results.dtd">
341 <tests-results>
342 <info class="startinfo">A value</info>
343 <tp id="dir1/tp1">
344 <tc id="main">
345 <passed />
346 <tc-time>#.#</tc-time></tc>
347 <tp-time>#.#</tp-time></tp>
348 <tp id="dir1/tp2">
349 <tc id="main">
350 <failed>This always fails</failed>
351 <tc-time>#.#</tc-time></tc>
352 <tp-time>#.#</tp-time></tp>
353 <tp id="tp3">
354 <tc id="main">
355 <passed />
356 <tc-time>#.#</tc-time></tc>
357 <tp-time>#.#</tp-time></tp>
358 <tp id="tp4">
359 <tc id="main">
360 <failed>This always fails</failed>
361 <tc-time>#.#</tc-time></tc>
362 <tp-time>#.#</tp-time></tp>
363 <tp id="tp5">
364 <failed>Invalid format for test case list: 1: Unexpected token \`&lt;&lt;NEWLINE&gt;&gt;'; expected \`:'</failed>
365 <tp-time>#.#</tp-time></tp>
366 <info class="endinfo">Another value</info>
367 </tests-results>
369 # NO_CHECK_STYLE_END
371 atf_check -s eq:0 -o file:expout -e empty -x \
372 "atf-report -o xml:- < tps.out | sed -E -e 's/>[0-9]+.[0-9]{6}</>#.#</'"
375 atf_test_case output_xml_space
376 output_xml_space_head()
378 atf_set "descr" "Checks that the XML output format properly preserves" \
379 "leading and trailing whitespace in stdout and stderr" \
380 "lines"
382 output_xml_space_body()
384 export TESTCASE=diff
385 cp $(atf_get_srcdir)/misc_helpers .
386 cat >Atffile <<EOF
387 Content-Type: application/X-atf-atffile; version="1"
389 prop: test-suite = atf
391 tp: misc_helpers
394 # NO_CHECK_STYLE_BEGIN
395 cat >expout <<EOF
396 <?xml version="1.0" encoding="ISO-8859-1"?>
397 <!DOCTYPE tests-results PUBLIC "-//NetBSD//DTD ATF Tests Results 0.1//EN" "http://www.NetBSD.org/XML/atf/tests-results.dtd">
399 <tests-results>
400 <info class="startinfo">A value</info>
401 <tp id="misc_helpers">
402 <tc id="diff">
403 <so>--- a 2007-11-04 14:00:41.000000000 +0100</so>
404 <so>+++ b 2007-11-04 14:00:48.000000000 +0100</so>
405 <so>@@ -1,7 +1,7 @@</so>
406 <so> This test is meant to simulate a diff.</so>
407 <so> Blank space at beginning of context lines must be preserved.</so>
408 <so> </so>
409 <so>-First original line.</so>
410 <so>-Second original line.</so>
411 <so>+First modified line.</so>
412 <so>+Second modified line.</so>
413 <so> </so>
414 <so> EOF</so>
415 <passed />
416 <tc-time>#.#</tc-time></tc>
417 <tp-time>#.#</tp-time></tp>
418 <info class="endinfo">Another value</info>
419 </tests-results>
421 # NO_CHECK_STYLE_END
423 run_helpers
424 atf_check -s eq:0 -o file:expout -e empty -x \
425 "atf-report -o xml:- <tps.out | sed -E -e 's/>[0-9]+.[0-9]{6}</>#.#</'"
428 atf_test_case too_many_args
429 too_many_args_body()
431 cat >experr <<EOF
432 atf-report: ERROR: No arguments allowed
434 atf_check -s eq:1 -o empty -e file:experr atf-report foo
437 atf_init_test_cases()
439 atf_add_test_case default
440 atf_add_test_case expect
441 atf_add_test_case oflag
442 atf_add_test_case output_csv
443 atf_add_test_case output_ticker
444 atf_add_test_case output_xml
445 atf_add_test_case output_xml_space
446 atf_add_test_case too_many_args
449 # vim: syntax=sh:expandtab:shiftwidth=4:softtabstop=4