1 // Copyright 2012 Google Inc.
2 // All rights reserved.
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are
8 // * Redistributions of source code must retain the above copyright
9 // notice, this list of conditions and the following disclaimer.
10 // * Redistributions in binary form must reproduce the above copyright
11 // notice, this list of conditions and the following disclaimer in the
12 // documentation and/or other materials provided with the distribution.
13 // * Neither the name of Google Inc. nor the names of its contributors
14 // may be used to endorse or promote products derived from this software
15 // without specific prior written permission.
17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
39 #include "atf_result.h"
45 #include "stacktrace.h"
49 /// Template for the creation of the temporary work directories.
50 #define WORKDIR_TEMPLATE "kyua.atf-tester.XXXXXX"
53 static void run_list(const char*, const int[2]) KYUA_DEFS_NORETURN
;
56 /// Executes the test program in list mode.
58 /// \param test_program Path to the test program to execute; should be absolute.
59 /// \param stdout_fds Pipe to write the output of the test program to.
61 run_list(const char* test_program
, const int stdout_fds
[2])
63 (void)close(stdout_fds
[0]);
65 if (stdout_fds
[1] != STDOUT_FILENO
) {
66 if (dup2(stdout_fds
[1], STDOUT_FILENO
) == -1)
67 err(EXIT_FAILURE
, "dup2 failed");
68 (void)close(stdout_fds
[1]);
71 if (dup2(STDOUT_FILENO
, STDERR_FILENO
) == -1)
72 err(EXIT_FAILURE
, "dup2 failed");
74 const char* const program_args
[] = { test_program
, "-l", NULL
};
75 kyua_run_exec(test_program
, program_args
);
79 /// Dumps the contents of the input file into the output.
81 /// \param input File from which to read.
82 /// \param output File to which to write.
84 /// \return An error if there is a problem.
86 dump_file(FILE* input
, FILE* output
)
91 while ((length
= fread(buffer
, 1, sizeof(buffer
), input
)) > 0) {
92 if (fwrite(buffer
, 1, length
, output
) != length
) {
93 return kyua_generic_error_new("Failed to write to output file");
97 return kyua_libc_error_new(errno
, "Failed to read test cases list");
99 return kyua_error_ok();
103 /// Creates a file within the work directory.
105 /// \param work_directory Path to the work directory.
106 /// \param name Name of the file to create.
107 /// \param mode Mode of the file, as specified by fopen(3).
108 /// \param [out] file Pointer to the created stream.
110 /// \return An error if there is a problem.
112 create_file_in_work_directory(const char* work_directory
, const char* name
,
113 const char* mode
, FILE** file
)
116 kyua_error_t error
= kyua_fs_concat(&path
, work_directory
, name
, NULL
);
117 if (kyua_error_is_set(error
))
120 FILE* tmp_file
= fopen(path
, mode
);
121 if (tmp_file
== NULL
) {
122 error
= kyua_libc_error_new(errno
, "Failed to create %s", path
);
128 assert(!kyua_error_is_set(error
));
136 /// Lists the test cases in a test program.
138 /// \param test_program Path to the test program for which to list the test
139 /// cases. Should be absolute.
140 /// \param run_params Execution parameters to configure the test process.
142 /// \return An error if the listing fails; OK otherwise.
144 list_test_cases(const char* test_program
, const kyua_run_params_t
* run_params
)
148 char* work_directory
;
149 error
= kyua_run_work_directory_enter(WORKDIR_TEMPLATE
,
150 run_params
->unprivileged_user
,
151 run_params
->unprivileged_group
,
153 if (kyua_error_is_set(error
))
155 kyua_run_params_t real_run_params
= *run_params
;
156 real_run_params
.work_directory
= work_directory
;
159 if (pipe(stdout_fds
) == -1) {
160 error
= kyua_libc_error_new(errno
, "pipe failed");
161 goto out_work_directory
;
165 error
= kyua_run_fork(&real_run_params
, &pid
);
166 if (!kyua_error_is_set(error
) && pid
== 0) {
167 run_list(test_program
, stdout_fds
);
169 assert(pid
!= -1 && pid
!= 0);
170 if (kyua_error_is_set(error
))
173 FILE* tmp_output
= NULL
; // Initialize to shut up gcc warning.
174 error
= create_file_in_work_directory(real_run_params
.work_directory
,
175 "list.txt", "w+", &tmp_output
);
176 if (kyua_error_is_set(error
))
179 close(stdout_fds
[1]); stdout_fds
[1] = -1;
180 kyua_error_t parse_error
= atf_list_parse(stdout_fds
[0], tmp_output
);
181 stdout_fds
[0] = -1; // Guaranteed closed by atf_list_parse.
182 // Delay reporting of parse errors to later. If we detect a problem while
183 // waiting for the test program, we know that the parsing has most likely
184 // failed and therefore the error with the program is more important for
185 // reporting purposes.
187 int status
; bool timed_out
;
188 error
= kyua_run_wait(pid
, &status
, &timed_out
);
189 if (kyua_error_is_set(error
))
191 if (!WIFEXITED(status
) || WEXITSTATUS(status
) != EXIT_SUCCESS
) {
192 error
= kyua_generic_error_new("Test program list did not return "
197 error
= kyua_error_subsume(error
, parse_error
);
198 if (!kyua_error_is_set(error
)) {
200 error
= dump_file(tmp_output
, stdout
);
206 if (stdout_fds
[0] != -1)
207 close(stdout_fds
[0]);
208 if (stdout_fds
[1] != -1)
209 close(stdout_fds
[1]);
211 error
= kyua_error_subsume(error
,
212 kyua_run_work_directory_leave(&work_directory
));
218 /// Counts the length of a user variables array.
220 /// \param user_variables The array of elements to be counted.
222 /// \return The length of the array.
224 count_variables(const char* const user_variables
[])
227 const char* const* iter
;
228 for (iter
= user_variables
; *iter
!= NULL
; ++iter
)
234 static void exec_body(const char* test_program
, const char* test_case
,
235 const char* result_file
,
236 const char* const user_variables
[]) KYUA_DEFS_NORETURN
;
239 /// Executes the body of a test case.
241 /// \param test_program Path to the test program to execute.
242 /// \param test_case Name of the test case to run.
243 /// \param result_file Path to the ATF result file to be created.
244 /// \param user_variables Set of configuration variables to pass to the test.
246 exec_body(const char* test_program
, const char* test_case
,
247 const char* result_file
, const char* const user_variables
[])
250 1 /* test_program */ +
251 2 /* -r result_file */
252 + 2 * count_variables(user_variables
) /* -v name=value */
253 + 1 /* test_case */ +
256 const char** args
= malloc(sizeof(const char*) * nargs
);
258 kyua_error_err(EXIT_FAILURE
, kyua_oom_error_new(),
259 "Failed to construct arguments list");
262 args
[i
++] = test_program
;
264 args
[i
++] = result_file
;
265 const char* const* iter
;
266 for (iter
= user_variables
; *iter
!= NULL
; ++iter
) {
270 args
[i
++] = test_case
;
274 kyua_run_exec(test_program
, args
);
278 /// Forks and executes the body of a test case in a controlled manner.
280 /// \param test_program Path to the test program to execute.
281 /// \param test_case Name of the test case to run.
282 /// \param result_file Path to the ATF result file to be created.
283 /// \param user_variables Set of configuration variables to pass to the test.
284 /// \param run_params Settings to control the subprocess.
285 /// \param [out] success Set to true if the test case runs properly and returns
286 /// a result that is to be considered as successful.
288 /// \return OK if all goes well, an error otherwise. Note that a failed test
289 /// case is denoted by setting success to false on exit, not by returning an
292 run_body(const char* test_program
, const char* test_case
,
293 const char* result_file
, const char* const user_variables
[],
294 const kyua_run_params_t
* run_params
, bool* success
)
298 char* tmp_result_file
;
299 error
= kyua_fs_concat(&tmp_result_file
, run_params
->work_directory
,
301 if (kyua_error_is_set(error
))
305 error
= kyua_run_fork(run_params
, &pid
);
306 if (!kyua_error_is_set(error
) && pid
== 0) {
307 exec_body(test_program
, test_case
, tmp_result_file
, user_variables
);
309 assert(pid
!= -1 && pid
!= 0);
310 if (kyua_error_is_set(error
))
311 goto out_tmp_result_file
;
313 int status
; bool timed_out
;
314 error
= kyua_run_wait(pid
, &status
, &timed_out
);
315 if (kyua_error_is_set(error
))
316 goto out_tmp_result_file
;
318 if (WIFSIGNALED(status
) && WCOREDUMP(status
)) {
319 kyua_stacktrace_dump(test_program
, pid
, run_params
, stderr
);
322 error
= kyua_atf_result_rewrite(tmp_result_file
, result_file
, status
,
326 free(tmp_result_file
);
332 static void exec_cleanup(const char* test_program
, const char* test_case
,
333 const char* const user_variables
[]) KYUA_DEFS_NORETURN
;
336 /// Executes the cleanup of a test case.
338 /// \param test_program Path to the test program to execute.
339 /// \param test_case Name of the test case to run.
340 /// \param user_variables Set of configuration variables to pass to the test.
342 exec_cleanup(const char* test_program
, const char* test_case
,
343 const char* const user_variables
[])
346 kyua_error_t error
= kyua_text_printf(&name
, "%s:cleanup", test_case
);
347 if (kyua_error_is_set(error
))
348 kyua_error_err(EXIT_FAILURE
, error
,
349 "Failed to construct argument list");
352 1 /* test_program */ +
353 + 2 * count_variables(user_variables
) /* -v name=value */
354 + 1 /* test_case */ +
357 const char** args
= malloc(sizeof(const char*) * nargs
);
359 kyua_error_err(EXIT_FAILURE
, kyua_oom_error_new(),
360 "Failed to construct arguments list");
363 args
[i
++] = test_program
;
364 const char* const* iter
;
365 for (iter
= user_variables
; *iter
!= NULL
; ++iter
) {
373 kyua_run_exec(test_program
, args
);
377 /// Forks and executes the cleanup of a test case in a controlled manner.
379 /// \param test_program Path to the test program to execute.
380 /// \param test_case Name of the test case to run.
381 /// \param result_file Path to the ATF result file created by the body of the
382 /// test case. The cleanup may update such file if it fails.
383 /// \param user_variables Set of configuration variables to pass to the test.
384 /// \param run_params Settings to control the subprocess.
385 /// \param body_success The success value returned by run_body().
386 /// \param [out] success Set to true if the test case runs properly and returns
387 /// a result that is to be considered as successful.
389 /// \return OK if all goes well, an error otherwise. Note that a failed test
390 /// case cleanup is denoted by setting success to false on exit, not by
391 /// returning an error.
393 run_cleanup(const char* test_program
, const char* test_case
,
394 const char* result_file
, const char* const user_variables
[],
395 const kyua_run_params_t
* run_params
, const bool body_success
,
401 error
= kyua_run_fork(run_params
, &pid
);
402 if (!kyua_error_is_set(error
) && pid
== 0) {
403 exec_cleanup(test_program
, test_case
, user_variables
);
405 assert(pid
!= -1 && pid
!= 0);
406 if (kyua_error_is_set(error
))
409 int status
; bool timed_out
;
410 error
= kyua_run_wait(pid
, &status
, &timed_out
);
411 if (kyua_error_is_set(error
))
414 if (WIFSIGNALED(status
) && WCOREDUMP(status
)) {
415 kyua_stacktrace_dump(test_program
, pid
, run_params
, stderr
);
419 // If the body has reported a successful result, we inspect the status
420 // of the cleanup routine. If the cleanup has failed, then we need to
421 // mark the test as broken. However, if the body itself had failed, we
422 // don't do this to give preference to the original result, which is
423 // probably more informative.
424 error
= kyua_atf_result_cleanup_rewrite(result_file
, status
,
433 /// Checks if the user variables indicate that a test has a cleanup routine.
435 /// This is an ugly hack to allow us to run the cleanup routine only when a test
436 /// case has it. When Kyua invokes the tester to generate the test case list,
437 /// the tester tells Kyua which tests have a cleanup routine. However, when the
438 /// tests are later run from here (as a different invocation) we cannot know if
439 /// the test had a cleanup routine or not. We rely on Kyua telling us this fact
440 /// by specifying has.cleanup=true in the variables.
442 /// \param user_variables Array of name=value pairs that describe the user
443 /// configuration variables for the test case.
445 has_cleanup(const char* const* user_variables
)
447 const char* const* iter
;
448 for (iter
= user_variables
; *iter
!= NULL
; ++iter
) {
449 if (strcmp(*iter
, "has.cleanup=false") == 0)
453 // The default is true because not running a cleanup routine when it exists
454 // is worse than running an empty routine when not told to do so.
459 /// Runs a single test cases of a test program.
461 /// \param test_program Path to the test program for which to list the test
462 /// cases. Should be absolute.
463 /// \param test_case Name of the test case to run.
464 /// \param result_file Path to the file to which to write the result of the
465 /// test. Should be absolute.
466 /// \param user_variables Array of name=value pairs that describe the user
467 /// configuration variables for the test case.
468 /// \param run_params Execution parameters to configure the test process.
469 /// \param [out] success Set to true if the test case reported a valid exit
470 /// condition (like "passed" or "skipped"); false otherwise. This is
471 /// only updated if the method returns OK.
473 /// \return An error if the listing fails; OK otherwise.
475 run_test_case(const char* test_program
, const char* test_case
,
476 const char* result_file
, const char* const user_variables
[],
477 const kyua_run_params_t
* run_params
, bool* success
)
481 char* work_directory
;
482 error
= kyua_run_work_directory_enter(WORKDIR_TEMPLATE
,
483 run_params
->unprivileged_user
,
484 run_params
->unprivileged_group
,
486 if (kyua_error_is_set(error
))
488 kyua_run_params_t real_run_params
= *run_params
;
489 real_run_params
.work_directory
= work_directory
;
491 error
= run_body(test_program
, test_case
, result_file
, user_variables
,
492 &real_run_params
, success
);
493 if (has_cleanup(user_variables
)) {
494 error
= run_cleanup(test_program
, test_case
, result_file
,
495 user_variables
, &real_run_params
, *success
,
499 error
= kyua_error_subsume(error
,
500 kyua_run_work_directory_leave(&work_directory
));
506 /// Definition of the tester.
507 static kyua_cli_tester_t atf_tester
= {
508 .list_test_cases
= list_test_cases
,
509 .run_test_case
= run_test_case
,
513 /// Tester entry point.
515 /// \param argc Number of command line arguments.
516 /// \param argv NULL-terminated array of command line arguments.
518 /// \return An exit code.
520 main(const int argc
, char* const* const argv
)
522 return kyua_cli_main(argc
, argv
, &atf_tester
);