tools/llvm: Do not build with symbols
[minix3.git] / external / bsd / kyua-testers / dist / atf_result_test.c
blob181d3e5aece931669aecaef0d9b9a22f81f7327b
1 // Copyright 2012 Google Inc.
2 // All rights reserved.
3 //
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are
6 // met:
7 //
8 // * Redistributions of source code must retain the above copyright
9 // notice, this list of conditions and the following disclaimer.
10 // * Redistributions in binary form must reproduce the above copyright
11 // notice, this list of conditions and the following disclaimer in the
12 // documentation and/or other materials provided with the distribution.
13 // * Neither the name of Google Inc. nor the names of its contributors
14 // may be used to endorse or promote products derived from this software
15 // without specific prior written permission.
17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 #include "atf_result.h"
31 #include <sys/resource.h>
32 #include <sys/wait.h>
34 #include <signal.h>
35 #include <stdlib.h>
36 #include <unistd.h>
38 #include <atf-c.h>
40 #include "error.h"
43 /// Evalutes an expression and ensures it does not return an error.
44 ///
45 /// \param expr A expression that must evaluate to kyua_error_t.
46 #define RE(expr) ATF_REQUIRE(!kyua_error_is_set(expr))
49 /// Generates a wait(2) status for a successful exit code.
50 ///
51 /// \param exitstatus The exit status to encode in the status.
52 ///
53 /// \return The wait(2) status.
54 static int
55 generate_wait_exitstatus(const int exitstatus)
57 const pid_t pid = fork();
58 ATF_REQUIRE(pid != -1);
59 if (pid == 0) {
60 exit(exitstatus);
61 } else {
62 int status;
63 ATF_REQUIRE(waitpid(pid, &status, 0) != -1);
64 return status;
69 /// Generates a wait(2) status for a termination due to a signal.
70 ///
71 /// \param signo The signal number to encode in the status.
72 ///
73 /// \return The wait(2) status.
74 static int
75 generate_wait_termsig(const int signo)
77 #if !defined(__minix)
78 // Explicitly disable core files to avoid inconsistent behaviors across
79 // operating systems. Some of the signal numbers passed to this function
80 // may have a meaning or not depending on the system, and this might mean
81 // that a core gets generated arbitrarily. As a result of this, our string
82 // comparisons below fail.
83 struct rlimit rl;
84 rl.rlim_cur = 0;
85 rl.rlim_max = RLIM_INFINITY;
86 if (setrlimit(RLIMIT_CORE, &rl) == -1)
87 atf_tc_skip("Failed to lower the core size limit");
88 #endif /* !defined(__minix) */
90 const pid_t pid = fork();
91 ATF_REQUIRE(pid != -1);
92 if (pid == 0) {
93 kill(getpid(), signo);
94 abort();
95 } else {
96 int status;
97 ATF_REQUIRE(waitpid(pid, &status, 0) != -1);
98 return status;
103 /// Validates an execution of kyua_atf_result_rewrite().
105 /// The code of this check is all within the macro instead of in a separate
106 /// function so that we get meaningful line numbers in test failure messages.
108 /// \param name Name of the test case.
109 /// \param input Text of the ATF result.
110 /// \param wait_status Exit status of the ATF test case, as returned by wait(2).
111 /// \param timed_out Whether the test case timed out or not.
112 /// \param exp_output Text of the expected generic result.
113 /// \param exp_success Expected value of the success result.
114 #define CHECK(name, input, wait_status, timed_out, exp_output, exp_success) \
115 ATF_TC_WITHOUT_HEAD(name); \
116 ATF_TC_BODY(name, tc) \
118 bool success; \
119 atf_utils_create_file("in.txt", "%s", input); \
120 RE(kyua_atf_result_rewrite("in.txt", "out.txt", wait_status, \
121 timed_out, &success)); \
122 atf_utils_cat_file("out.txt", "OUTPUT: "); \
123 ATF_REQUIRE(atf_utils_compare_file("out.txt", exp_output)); \
124 ATF_REQUIRE_EQ(exp_success, success); \
128 CHECK(rewrite__expected_death__exit_failure,
129 "expected_death: Some text\n",
130 generate_wait_exitstatus(EXIT_FAILURE), false,
131 "expected_failure: Some text\n",
132 true);
133 CHECK(rewrite__expected_death__signaled,
134 "expected_death: Foo bar\n",
135 generate_wait_termsig(1), false,
136 "expected_failure: Foo bar\n",
137 true);
138 CHECK(rewrite__expected_death__exit_success,
139 "expected_death: Some text\n",
140 generate_wait_exitstatus(EXIT_SUCCESS), false,
141 "failed: Test case expected to die but exited successfully\n",
142 false);
143 CHECK(rewrite__expected_death__no_reason,
144 "expected_death\n",
145 generate_wait_exitstatus(EXIT_FAILURE), false,
146 "broken: Test case should have reported a failure reason but didn't; "
147 "test case exited with code 1\n",
148 false);
149 CHECK(rewrite__expected_death__unexpected_arg,
150 "expected_death(23): Some text\n",
151 generate_wait_exitstatus(EXIT_FAILURE), false,
152 "broken: Unknown test case result status expected_death(23); "
153 "test case exited with code 1\n",
154 false);
157 CHECK(rewrite__expected_exit__exit_success,
158 "expected_exit: Some text\n",
159 generate_wait_exitstatus(EXIT_SUCCESS), false,
160 "expected_failure: Some text\n",
161 true);
162 CHECK(rewrite__expected_exit__exit_failure,
163 "expected_exit: Some other text\n",
164 generate_wait_exitstatus(EXIT_FAILURE), false,
165 "expected_failure: Some other text\n",
166 true);
167 CHECK(rewrite__expected_exit__arg_match,
168 "expected_exit(15): Some text\n",
169 generate_wait_exitstatus(15), false,
170 "expected_failure: Some text\n",
171 true);
172 CHECK(rewrite__expected_exit__arg_not_match,
173 "expected_exit(18): Some text\n",
174 generate_wait_exitstatus(15), false,
175 "failed: Test case expected to exit with code 18 but got code 15\n",
176 false);
177 CHECK(rewrite__expected_exit__signaled,
178 "expected_exit: Foo bar\n",
179 generate_wait_termsig(1), false,
180 "failed: Test case expected to exit normally but received signal 1\n",
181 false);
182 CHECK(rewrite__expected_exit__no_reason,
183 "expected_exit\n",
184 generate_wait_exitstatus(EXIT_FAILURE), false,
185 "broken: Test case should have reported a failure reason but didn't; "
186 "test case exited with code 1\n",
187 false);
188 CHECK(rewrite__expected_exit__bad_arg,
189 "expected_exit(abc): Some text\n",
190 generate_wait_exitstatus(25), false,
191 "broken: Invalid status argument (abc): not a number; test case exited "
192 "with code 25\n",
193 false);
196 CHECK(rewrite__expected_failure__ok,
197 "expected_failure: Some text\n",
198 generate_wait_exitstatus(EXIT_SUCCESS), false,
199 "expected_failure: Some text\n",
200 true);
201 CHECK(rewrite__expected_failure__failed,
202 "expected_failure: Some text\n",
203 generate_wait_exitstatus(EXIT_FAILURE), false,
204 "failed: Test case expected a failure but exited with error code 1\n",
205 false);
206 CHECK(rewrite__expected_failure__signaled,
207 "expected_failure: Foo bar\n",
208 generate_wait_termsig(1), false,
209 "failed: Test case expected a failure but received signal 1\n",
210 false);
211 CHECK(rewrite__expected_failure__no_reason,
212 "expected_failure\n",
213 generate_wait_exitstatus(EXIT_FAILURE), false,
214 "broken: Test case should have reported a failure reason but didn't; "
215 "test case exited with code 1\n",
216 false);
217 CHECK(rewrite__expected_failure__unexpected_arg,
218 "expected_failure(23): Some text\n",
219 generate_wait_exitstatus(EXIT_FAILURE), false,
220 "broken: Unknown test case result status expected_failure(23); "
221 "test case exited with code 1\n",
222 false);
225 CHECK(rewrite__expected_signal__ok,
226 "expected_signal: Some text\n",
227 generate_wait_termsig(6), false,
228 "expected_failure: Some text\n",
229 true);
230 CHECK(rewrite__expected_signal__arg_match,
231 "expected_signal(15): Some text\n",
232 generate_wait_termsig(15), false,
233 "expected_failure: Some text\n",
234 true);
235 CHECK(rewrite__expected_signal__arg_not_match,
236 "expected_signal(18): Some text\n",
237 generate_wait_termsig(15), false,
238 "failed: Test case expected to receive signal 18 but got 15\n",
239 false);
240 CHECK(rewrite__expected_signal__exited,
241 "expected_signal: Foo bar\n",
242 generate_wait_exitstatus(12), false,
243 "failed: Test case expected to receive a signal but exited with code 12\n",
244 false);
245 CHECK(rewrite__expected_signal__no_reason,
246 "expected_signal\n",
247 generate_wait_termsig(15), false,
248 "broken: Test case should have reported a failure reason but didn't; "
249 "test case received signal 15\n",
250 false);
251 CHECK(rewrite__expected_signal__bad_arg,
252 "expected_signal(abc): Some text\n",
253 generate_wait_termsig(25), false,
254 "broken: Invalid status argument (abc): not a number; test case received "
255 "signal 25\n",
256 false);
259 CHECK(rewrite__expected_timeout__ok,
260 "expected_timeout: Some text\n",
261 generate_wait_exitstatus(EXIT_SUCCESS), true,
262 "expected_failure: Some text\n",
263 true);
264 CHECK(rewrite__expected_timeout__exited,
265 "expected_timeout: Foo bar\n",
266 generate_wait_exitstatus(12), false,
267 "failed: Test case expected to time out but exited with code 12\n",
268 false);
269 CHECK(rewrite__expected_timeout__signaled,
270 "expected_timeout: Foo bar\n",
271 generate_wait_termsig(15), false,
272 "failed: Test case expected to time out but received signal 15\n",
273 false);
274 CHECK(rewrite__expected_timeout__no_reason,
275 "expected_timeout\n",
276 generate_wait_exitstatus(EXIT_FAILURE), false,
277 "broken: Test case should have reported a failure reason but didn't; "
278 "test case exited with code 1\n",
279 false);
280 CHECK(rewrite__expected_timeout__unexpected_arg,
281 "expected_timeout(23): Some text\n",
282 generate_wait_exitstatus(EXIT_FAILURE), false,
283 "broken: Unknown test case result status expected_timeout(23); "
284 "test case exited with code 1\n",
285 false);
288 CHECK(rewrite__failed__ok,
289 "failed: Some text\n",
290 generate_wait_exitstatus(EXIT_FAILURE), false,
291 "failed: Some text\n",
292 false);
293 CHECK(rewrite__failed__exit_success,
294 "failed: Some text\n",
295 generate_wait_exitstatus(EXIT_SUCCESS), false,
296 "broken: Test case reported a failed result but exited with a successful "
297 "exit code\n",
298 false);
299 CHECK(rewrite__failed__signaled,
300 "failed: Not used\n",
301 generate_wait_termsig(1), false,
302 "broken: Test case reported a failed result but received signal 1\n",
303 false);
304 CHECK(rewrite__failed__no_reason,
305 "failed\n",
306 generate_wait_exitstatus(EXIT_FAILURE), false,
307 "broken: Test case should have reported a failure reason but didn't; "
308 "test case exited with code 1\n",
309 false);
310 CHECK(rewrite__failed__unexpected_arg,
311 "failed(23): Some text\n",
312 generate_wait_exitstatus(EXIT_FAILURE), false,
313 "broken: Unknown test case result status failed(23); "
314 "test case exited with code 1\n",
315 false);
318 CHECK(rewrite__passed__ok,
319 "passed\n",
320 generate_wait_exitstatus(EXIT_SUCCESS), false,
321 "passed\n",
322 true);
323 CHECK(rewrite__passed__exit_failure,
324 "passed\n",
325 generate_wait_exitstatus(EXIT_FAILURE), false,
326 "broken: Test case reported a passed result but returned a non-zero "
327 "exit code 1\n",
328 false);
329 CHECK(rewrite__passed__signaled,
330 "passed\n",
331 generate_wait_termsig(1), false,
332 "broken: Test case reported a passed result but received signal 1\n",
333 false);
334 CHECK(rewrite__passed__unexpected_reason,
335 "passed: This should not be here\n",
336 generate_wait_exitstatus(EXIT_SUCCESS), false,
337 "broken: Found unexpected reason in passed test result; test case exited "
338 "with code 0\n",
339 false);
340 CHECK(rewrite__passed__unexpected_arg,
341 "passed(0)\n",
342 generate_wait_exitstatus(EXIT_SUCCESS), false,
343 "broken: Unknown test case result status passed(0); "
344 "test case exited with code 0\n",
345 false);
348 CHECK(rewrite__skipped__ok,
349 "skipped: Does not apply\n",
350 generate_wait_exitstatus(EXIT_SUCCESS), false,
351 "skipped: Does not apply\n",
352 true);
353 CHECK(rewrite__skipped__exit_failure,
354 "skipped: Some reason\n",
355 generate_wait_exitstatus(EXIT_FAILURE), false,
356 "broken: Test case reported a skipped result but returned a non-zero "
357 "exit code 1\n",
358 false);
359 CHECK(rewrite__skipped__signaled,
360 "skipped: Not used\n",
361 generate_wait_termsig(1), false,
362 "broken: Test case reported a skipped result but received signal 1\n",
363 false);
364 CHECK(rewrite__skipped__no_reason,
365 "skipped\n",
366 generate_wait_exitstatus(EXIT_SUCCESS), false,
367 "broken: Test case should have reported a failure reason but didn't; "
368 "test case exited with code 0\n",
369 false);
370 CHECK(rewrite__skipped__unexpected_arg,
371 "skipped(1): Some text\n",
372 generate_wait_exitstatus(EXIT_SUCCESS), false,
373 "broken: Unknown test case result status skipped(1); "
374 "test case exited with code 0\n",
375 false);
378 CHECK(rewrite__timed_out,
379 "this invalid result file should not be parsed\n",
380 generate_wait_exitstatus(EXIT_SUCCESS), true,
381 "broken: Test case body timed out\n",
382 false);
385 ATF_TC_WITHOUT_HEAD(rewrite__missing_file);
386 ATF_TC_BODY(rewrite__missing_file, tc)
388 bool success;
389 RE(kyua_atf_result_rewrite("in.txt", "out.txt",
390 generate_wait_exitstatus(EXIT_SUCCESS),
391 false, &success));
392 atf_utils_cat_file("out.txt", "OUTPUT: ");
393 ATF_REQUIRE(atf_utils_compare_file("out.txt",
394 "broken: Premature exit; test case exited with code 0\n"));
395 ATF_REQUIRE(!success);
397 CHECK(rewrite__empty,
399 generate_wait_exitstatus(EXIT_FAILURE), false,
400 "broken: Empty result file in.txt; test case exited with code 1\n",
401 false);
402 CHECK(rewrite__unknown_status,
403 "foo\n",
404 generate_wait_exitstatus(123), false,
405 "broken: Unknown test case result status foo; test case exited with "
406 "code 123\n",
407 false);
408 CHECK(rewrite__status_without_newline,
409 "failed",
410 generate_wait_termsig(1), false,
411 "broken: Missing newline in result file; test case received signal 1\n",
412 false);
413 CHECK(rewrite__status_multiline,
414 "failed: First line\nSecond line\nThird line",
415 generate_wait_exitstatus(EXIT_FAILURE), false,
416 "failed: First line<<NEWLINE>>Second line<<NEWLINE>>Third line\n",
417 false);
418 CHECK(rewrite__status_multiline_2,
419 "failed: First line\nSecond line\nThird line\n",
420 generate_wait_exitstatus(EXIT_FAILURE), false,
421 "failed: First line<<NEWLINE>>Second line<<NEWLINE>>Third line\n",
422 false);
425 /// Validates an execution of kyua_atf_result_cleanup_rewrite().
427 /// The code of this check is all within the macro instead of in a separate
428 /// function so that we get meaningful line numbers in test failure messages.
430 /// \param name Name of the test case.
431 /// \param wait_status Exit status of the ATF test case, as returned by wait(2).
432 /// \param timed_out Whether the test case timed out or not.
433 /// \param exp_output Text of the expected generic result.
434 /// \param exp_success Expected value of the success result.
435 #define CHECK_CLEANUP(name, wait_status, timed_out, exp_output, exp_success) \
436 ATF_TC_WITHOUT_HEAD(name); \
437 ATF_TC_BODY(name, tc) \
439 bool success; \
440 atf_utils_create_file("out.txt", "skipped: Preexistent file\n"); \
441 RE(kyua_atf_result_cleanup_rewrite("out.txt", wait_status, \
442 timed_out, &success)); \
443 atf_utils_cat_file("out.txt", "OUTPUT: "); \
444 ATF_REQUIRE(atf_utils_compare_file("out.txt", exp_output)); \
445 ATF_REQUIRE_EQ(exp_success, success); \
449 CHECK_CLEANUP(cleanup_rewrite__ok,
450 generate_wait_exitstatus(EXIT_SUCCESS), false,
451 "skipped: Preexistent file\n", // Previous file not overwritten.
452 true);
453 CHECK_CLEANUP(cleanup_rewrite__exit_failure,
454 generate_wait_exitstatus(EXIT_FAILURE), false,
455 "broken: Test case cleanup exited with code 1\n",
456 false);
457 CHECK_CLEANUP(cleanup_rewrite__signaled,
458 generate_wait_termsig(1), false,
459 "broken: Test case cleanup received signal 1\n",
460 false);
461 CHECK_CLEANUP(cleanup_rewrite__timed_out,
462 generate_wait_exitstatus(EXIT_SUCCESS), true,
463 "broken: Test case cleanup timed out\n",
464 false);
467 ATF_TP_ADD_TCS(tp)
469 ATF_TP_ADD_TC(tp, rewrite__expected_death__exit_failure);
470 ATF_TP_ADD_TC(tp, rewrite__expected_death__signaled);
471 ATF_TP_ADD_TC(tp, rewrite__expected_death__exit_success);
472 ATF_TP_ADD_TC(tp, rewrite__expected_death__no_reason);
473 ATF_TP_ADD_TC(tp, rewrite__expected_death__unexpected_arg);
475 ATF_TP_ADD_TC(tp, rewrite__expected_exit__exit_success);
476 ATF_TP_ADD_TC(tp, rewrite__expected_exit__exit_failure);
477 ATF_TP_ADD_TC(tp, rewrite__expected_exit__arg_match);
478 ATF_TP_ADD_TC(tp, rewrite__expected_exit__arg_not_match);
479 ATF_TP_ADD_TC(tp, rewrite__expected_exit__signaled);
480 ATF_TP_ADD_TC(tp, rewrite__expected_exit__no_reason);
481 ATF_TP_ADD_TC(tp, rewrite__expected_exit__bad_arg);
483 ATF_TP_ADD_TC(tp, rewrite__expected_failure__ok);
484 ATF_TP_ADD_TC(tp, rewrite__expected_failure__failed);
485 ATF_TP_ADD_TC(tp, rewrite__expected_failure__signaled);
486 ATF_TP_ADD_TC(tp, rewrite__expected_failure__no_reason);
487 ATF_TP_ADD_TC(tp, rewrite__expected_failure__unexpected_arg);
489 ATF_TP_ADD_TC(tp, rewrite__expected_signal__ok);
490 ATF_TP_ADD_TC(tp, rewrite__expected_signal__arg_match);
491 ATF_TP_ADD_TC(tp, rewrite__expected_signal__arg_not_match);
492 ATF_TP_ADD_TC(tp, rewrite__expected_signal__exited);
493 ATF_TP_ADD_TC(tp, rewrite__expected_signal__no_reason);
494 ATF_TP_ADD_TC(tp, rewrite__expected_signal__bad_arg);
496 ATF_TP_ADD_TC(tp, rewrite__expected_timeout__ok);
497 ATF_TP_ADD_TC(tp, rewrite__expected_timeout__exited);
498 ATF_TP_ADD_TC(tp, rewrite__expected_timeout__signaled);
499 ATF_TP_ADD_TC(tp, rewrite__expected_timeout__no_reason);
500 ATF_TP_ADD_TC(tp, rewrite__expected_timeout__unexpected_arg);
502 ATF_TP_ADD_TC(tp, rewrite__failed__ok);
503 ATF_TP_ADD_TC(tp, rewrite__failed__exit_success);
504 ATF_TP_ADD_TC(tp, rewrite__failed__signaled);
505 ATF_TP_ADD_TC(tp, rewrite__failed__no_reason);
506 ATF_TP_ADD_TC(tp, rewrite__failed__unexpected_arg);
508 ATF_TP_ADD_TC(tp, rewrite__passed__ok);
509 ATF_TP_ADD_TC(tp, rewrite__passed__exit_failure);
510 ATF_TP_ADD_TC(tp, rewrite__passed__signaled);
511 ATF_TP_ADD_TC(tp, rewrite__passed__unexpected_reason);
512 ATF_TP_ADD_TC(tp, rewrite__passed__unexpected_arg);
514 ATF_TP_ADD_TC(tp, rewrite__skipped__ok);
515 ATF_TP_ADD_TC(tp, rewrite__skipped__exit_failure);
516 ATF_TP_ADD_TC(tp, rewrite__skipped__signaled);
517 ATF_TP_ADD_TC(tp, rewrite__skipped__no_reason);
518 ATF_TP_ADD_TC(tp, rewrite__skipped__unexpected_arg);
520 ATF_TP_ADD_TC(tp, rewrite__timed_out);
522 ATF_TP_ADD_TC(tp, rewrite__missing_file);
523 ATF_TP_ADD_TC(tp, rewrite__empty);
524 ATF_TP_ADD_TC(tp, rewrite__unknown_status);
525 ATF_TP_ADD_TC(tp, rewrite__status_without_newline);
526 ATF_TP_ADD_TC(tp, rewrite__status_multiline);
527 ATF_TP_ADD_TC(tp, rewrite__status_multiline_2);
529 ATF_TP_ADD_TC(tp, cleanup_rewrite__ok);
530 ATF_TP_ADD_TC(tp, cleanup_rewrite__exit_failure);
531 ATF_TP_ADD_TC(tp, cleanup_rewrite__signaled);
532 ATF_TP_ADD_TC(tp, cleanup_rewrite__timed_out);
534 return atf_no_error();