2 * Copyright (c) 2003-2007 Tim Kientzle
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
17 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
18 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
19 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
20 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
21 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
23 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 * Various utility routines useful for test programs.
28 * Each test program is linked against this file.
38 * This same file is used pretty much verbatim for all test harnesses.
40 * The next few lines are the only differences.
42 #define PROGRAM "bsdtar" /* Name of program being tested. */
43 #define ENVBASE "BSDTAR" /* Prefix for environment variables. */
44 #undef EXTRA_DUMP /* How to dump extra data */
45 /* How to generate extra version info. */
46 #define EXTRA_VERSION (systemf("%s --version", testprog) ? "" : "")
47 __FBSDID("$FreeBSD: src/usr.bin/tar/test/main.c,v 1.3 2008/06/15 10:07:54 kientzle Exp $");
50 * "list.h" is simply created by "grep DEFINE_TEST"; it has
52 * DEFINE_TEST(test_function)
54 * Include it here with a suitable DEFINE_TEST to declare all of the
58 #define DEFINE_TEST(name) void name(void);
61 /* Interix doesn't define these in a standard header. */
67 /* Enable core dump on failure. */
68 static int dump_on_failure
= 0;
69 /* Default is to remove temp dirs for successful tests. */
70 static int keep_temp_files
= 0;
71 /* Default is to print some basic information about each test. */
72 static int quiet_flag
= 0;
73 /* Default is to summarize repeated failures. */
74 static int verbose
= 0;
75 /* Cumulative count of component failures. */
76 static int failures
= 0;
77 /* Cumulative count of skipped component tests. */
79 /* Cumulative count of assertions. */
80 static int assertions
= 0;
82 /* Directory where uuencoded reference files can be found. */
86 * My own implementation of the standard assert() macro emits the
87 * message in the same format as GCC (file:line: message).
88 * It also includes some additional useful information.
89 * This makes it a lot easier to skim through test failures in
92 * It also supports a few special features specifically to simplify
94 * failure(fmt, args) -- Stores a text string that gets
95 * printed if the following assertion fails, good for
96 * explaining subtle tests.
98 static char msg
[4096];
101 * For each test source file, we remember how many times each
102 * failure was reported.
104 static const char *failed_filename
= NULL
;
108 } failed_lines
[1000];
111 * Count this failure; return the number of previous failures.
114 previous_failures(const char *filename
, int line
)
119 if (failed_filename
== NULL
|| strcmp(failed_filename
, filename
) != 0)
120 memset(failed_lines
, 0, sizeof(failed_lines
));
121 failed_filename
= filename
;
123 for (i
= 0; i
< sizeof(failed_lines
)/sizeof(failed_lines
[0]); i
++) {
124 if (failed_lines
[i
].line
== line
) {
125 count
= failed_lines
[i
].count
;
126 failed_lines
[i
].count
++;
129 if (failed_lines
[i
].line
== 0) {
130 failed_lines
[i
].line
= line
;
131 failed_lines
[i
].count
= 1;
139 * Copy arguments into file-local variables.
141 static const char *test_filename
;
142 static int test_line
;
143 static void *test_extra
;
144 void test_setup(const char *filename
, int line
)
146 test_filename
= filename
;
151 * Inform user that we're skipping a test.
154 test_skipping(const char *fmt
, ...)
158 if (previous_failures(test_filename
, test_line
))
162 fprintf(stderr
, " *** SKIPPING: ");
163 vfprintf(stderr
, fmt
, ap
);
164 fprintf(stderr
, "\n");
169 /* Common handling of failed tests. */
171 report_failure(void *extra
)
173 if (msg
[0] != '\0') {
174 fprintf(stderr
, " Description: %s\n", msg
);
180 fprintf(stderr
, " detail: %s\n", EXTRA_DUMP(extra
));
182 (void)extra
; /* UNUSED */
185 if (dump_on_failure
) {
187 " *** forcing core dump so failure can be debugged ***\n");
194 * Summarize repeated failures in the just-completed test file.
195 * The reports above suppress multiple failures from the same source
196 * line; this reports on any tests that did fail multiple times.
199 summarize_comparator(const void *a0
, const void *b0
)
201 const struct line
*a
= a0
, *b
= b0
;
202 if (a
->line
== 0 && b
->line
== 0)
208 return (a
->line
- b
->line
);
216 qsort(failed_lines
, sizeof(failed_lines
)/sizeof(failed_lines
[0]),
217 sizeof(failed_lines
[0]), summarize_comparator
);
218 for (i
= 0; i
< sizeof(failed_lines
)/sizeof(failed_lines
[0]); i
++) {
219 if (failed_lines
[i
].line
== 0)
221 if (failed_lines
[i
].count
> 1)
222 fprintf(stderr
, "%s:%d: Failed %d times\n",
223 failed_filename
, failed_lines
[i
].line
,
224 failed_lines
[i
].count
);
226 /* Clear the failure history for the next file. */
227 memset(failed_lines
, 0, sizeof(failed_lines
));
230 /* Set up a message to display only after a test fails. */
232 failure(const char *fmt
, ...)
236 vsprintf(msg
, fmt
, ap
);
240 /* Generic assert() just displays the failed condition. */
242 test_assert(const char *file
, int line
, int value
, const char *condition
, void *extra
)
250 if (!verbose
&& previous_failures(file
, line
))
252 fprintf(stderr
, "%s:%d: Assertion failed\n", file
, line
);
253 fprintf(stderr
, " Condition: %s\n", condition
);
254 report_failure(extra
);
258 /* assertEqualInt() displays the values of the two integers. */
260 test_assert_equal_int(const char *file
, int line
,
261 int v1
, const char *e1
, int v2
, const char *e2
, void *extra
)
269 if (!verbose
&& previous_failures(file
, line
))
271 fprintf(stderr
, "%s:%d: Assertion failed: Ints not equal\n",
273 fprintf(stderr
, " %s=%d\n", e1
, v1
);
274 fprintf(stderr
, " %s=%d\n", e2
, v2
);
275 report_failure(extra
);
279 static void strdump(const char *p
)
282 fprintf(stderr
, "(null)");
285 fprintf(stderr
, "\"");
287 unsigned int c
= 0xff & *p
++;
289 case '\a': fprintf(stderr
, "\a"); break;
290 case '\b': fprintf(stderr
, "\b"); break;
291 case '\n': fprintf(stderr
, "\n"); break;
292 case '\r': fprintf(stderr
, "\r"); break;
294 if (c
>= 32 && c
< 127)
295 fprintf(stderr
, "%c", c
);
297 fprintf(stderr
, "\\x%02X", c
);
300 fprintf(stderr
, "\"");
303 /* assertEqualString() displays the values of the two strings. */
305 test_assert_equal_string(const char *file
, int line
,
306 const char *v1
, const char *e1
,
307 const char *v2
, const char *e2
,
311 if (v1
== NULL
|| v2
== NULL
) {
316 } else if (strcmp(v1
, v2
) == 0) {
321 if (!verbose
&& previous_failures(file
, line
))
323 fprintf(stderr
, "%s:%d: Assertion failed: Strings not equal\n",
325 fprintf(stderr
, " %s = ", e1
);
327 fprintf(stderr
, " (length %d)\n", v1
== NULL
? 0 : strlen(v1
));
328 fprintf(stderr
, " %s = ", e2
);
330 fprintf(stderr
, " (length %d)\n", v2
== NULL
? 0 : strlen(v2
));
331 report_failure(extra
);
335 static void wcsdump(const wchar_t *w
)
338 fprintf(stderr
, "(null)");
341 fprintf(stderr
, "\"");
342 while (*w
!= L
'\0') {
343 unsigned int c
= *w
++;
344 if (c
>= 32 && c
< 127)
345 fprintf(stderr
, "%c", c
);
347 fprintf(stderr
, "\\x%02X", c
);
348 else if (c
< 0x10000)
349 fprintf(stderr
, "\\u%04X", c
);
351 fprintf(stderr
, "\\U%08X", c
);
353 fprintf(stderr
, "\"");
356 /* assertEqualWString() displays the values of the two strings. */
358 test_assert_equal_wstring(const char *file
, int line
,
359 const wchar_t *v1
, const char *e1
,
360 const wchar_t *v2
, const char *e2
,
369 } else if (v2
== NULL
) {
374 } else if (wcscmp(v1
, v2
) == 0) {
379 if (!verbose
&& previous_failures(file
, line
))
381 fprintf(stderr
, "%s:%d: Assertion failed: Unicode strings not equal\n",
383 fprintf(stderr
, " %s = ", e1
);
385 fprintf(stderr
, "\n");
386 fprintf(stderr
, " %s = ", e2
);
388 fprintf(stderr
, "\n");
389 report_failure(extra
);
394 * Pretty standard hexdump routine. As a bonus, if ref != NULL, then
395 * any bytes in p that differ from ref will be highlighted with '_'
396 * before and after the hex value.
399 hexdump(const char *p
, const char *ref
, size_t l
, size_t offset
)
404 for(i
=0; i
< l
; i
+=16) {
405 fprintf(stderr
, "%04x", i
+ offset
);
407 for (j
= 0; j
< 16 && i
+ j
< l
; j
++) {
408 if (ref
!= NULL
&& p
[i
+ j
] != ref
[i
+ j
])
410 fprintf(stderr
, "%c%02x", sep
, 0xff & (int)p
[i
+j
]);
411 if (ref
!= NULL
&& p
[i
+ j
] == ref
[i
+ j
])
414 for (; j
< 16; j
++) {
415 fprintf(stderr
, "%c ", sep
);
418 fprintf(stderr
, "%c", sep
);
419 for (j
=0; j
< 16 && i
+ j
< l
; j
++) {
421 if (c
>= ' ' && c
<= 126)
422 fprintf(stderr
, "%c", c
);
424 fprintf(stderr
, ".");
426 fprintf(stderr
, "\n");
430 /* assertEqualMem() displays the values of the two memory blocks. */
431 /* TODO: For long blocks, hexdump the first bytes that actually differ. */
433 test_assert_equal_mem(const char *file
, int line
,
434 const char *v1
, const char *e1
,
435 const char *v2
, const char *e2
,
436 size_t l
, const char *ld
, void *extra
)
439 if (v1
== NULL
|| v2
== NULL
) {
444 } else if (memcmp(v1
, v2
, l
) == 0) {
449 if (!verbose
&& previous_failures(file
, line
))
451 fprintf(stderr
, "%s:%d: Assertion failed: memory not equal\n",
453 fprintf(stderr
, " size %s = %d\n", ld
, (int)l
);
454 fprintf(stderr
, " Dump of %s\n", e1
);
455 hexdump(v1
, v2
, l
< 32 ? l
: 32, 0);
456 fprintf(stderr
, " Dump of %s\n", e2
);
457 hexdump(v2
, v1
, l
< 32 ? l
: 32, 0);
458 fprintf(stderr
, "\n");
459 report_failure(extra
);
464 test_assert_empty_file(const char *f1fmt
, ...)
475 vsprintf(f1
, f1fmt
, ap
);
478 if (stat(f1
, &st
) != 0) {
479 fprintf(stderr
, "%s:%d: Could not stat: %s\n", test_filename
, test_line
, f1
);
480 report_failure(NULL
);
487 if (!verbose
&& previous_failures(test_filename
, test_line
))
490 fprintf(stderr
, "%s:%d: File not empty: %s\n", test_filename
, test_line
, f1
);
491 fprintf(stderr
, " File size: %d\n", (int)st
.st_size
);
492 fprintf(stderr
, " Contents:\n");
493 fd
= open(f1
, O_RDONLY
);
495 fprintf(stderr
, " Unable to open %s\n", f1
);
497 s
= sizeof(buff
) < st
.st_size
? sizeof(buff
) : st
.st_size
;
498 s
= read(fd
, buff
, s
);
499 hexdump(buff
, NULL
, s
, 0);
501 report_failure(NULL
);
505 /* assertEqualFile() asserts that two files have the same contents. */
506 /* TODO: hexdump the first bytes that actually differ. */
508 test_assert_equal_file(const char *f1
, const char *f2pattern
, ...)
517 va_start(ap
, f2pattern
);
518 vsprintf(f2
, f2pattern
, ap
);
521 fd1
= open(f1
, O_RDONLY
);
522 fd2
= open(f2
, O_RDONLY
);
524 n1
= read(fd1
, buff1
, sizeof(buff1
));
525 n2
= read(fd2
, buff2
, sizeof(buff2
));
528 if (n1
== 0 && n2
== 0)
530 if (memcmp(buff1
, buff2
, n1
) != 0)
534 if (!verbose
&& previous_failures(test_filename
, test_line
))
536 fprintf(stderr
, "%s:%d: Files are not identical\n",
537 test_filename
, test_line
);
538 fprintf(stderr
, " file1=\"%s\"\n", f1
);
539 fprintf(stderr
, " file2=\"%s\"\n", f2
);
540 report_failure(test_extra
);
545 test_assert_file_exists(const char *fpattern
, ...)
550 va_start(ap
, fpattern
);
551 vsprintf(f
, fpattern
, ap
);
554 if (!access(f
, F_OK
))
556 if (!previous_failures(test_filename
, test_line
)) {
557 fprintf(stderr
, "%s:%d: File doesn't exist\n",
558 test_filename
, test_line
);
559 fprintf(stderr
, " file=\"%s\"\n", f
);
560 report_failure(test_extra
);
566 test_assert_file_not_exists(const char *fpattern
, ...)
571 va_start(ap
, fpattern
);
572 vsprintf(f
, fpattern
, ap
);
577 if (!previous_failures(test_filename
, test_line
)) {
578 fprintf(stderr
, "%s:%d: File exists and shouldn't\n",
579 test_filename
, test_line
);
580 fprintf(stderr
, " file=\"%s\"\n", f
);
581 report_failure(test_extra
);
586 /* assertFileContents() asserts the contents of a file. */
588 test_assert_file_contents(const void *buff
, int s
, const char *fpattern
, ...)
596 va_start(ap
, fpattern
);
597 vsprintf(f
, fpattern
, ap
);
600 fd
= open(f
, O_RDONLY
);
601 contents
= malloc(s
* 2);
602 n
= read(fd
, contents
, s
* 2);
603 if (n
== s
&& memcmp(buff
, contents
, s
) == 0) {
608 if (!previous_failures(test_filename
, test_line
)) {
609 fprintf(stderr
, "%s:%d: File contents don't match\n",
610 test_filename
, test_line
);
611 fprintf(stderr
, " file=\"%s\"\n", f
);
613 hexdump(contents
, buff
, n
, 0);
615 fprintf(stderr
, " File empty, contents should be:\n");
616 hexdump(buff
, NULL
, s
, 0);
618 report_failure(test_extra
);
625 * Call standard system() call, but build up the command line using
626 * sprintf() conventions.
629 systemf(const char *fmt
, ...)
636 vsprintf(buff
, fmt
, ap
);
643 * Slurp a file into memory for ease of comparison and testing.
644 * Returns size of file in 'sizep' if non-NULL, null-terminates
645 * data in memory for ease of use.
648 slurpfile(size_t * sizep
, const char *fmt
, ...)
659 vsprintf(filename
, fmt
, ap
);
662 fd
= open(filename
, O_RDONLY
);
664 /* Note: No error; non-existent file is okay here. */
669 fprintf(stderr
, "Can't stat file %s\n", filename
);
673 p
= malloc(st
.st_size
+ 1);
675 fprintf(stderr
, "Can't allocate %ld bytes of memory to read file %s\n", (long int)st
.st_size
, filename
);
679 bytes_read
= read(fd
, p
, st
.st_size
);
680 if (bytes_read
< st
.st_size
) {
681 fprintf(stderr
, "Can't read file %s\n", filename
);
686 p
[st
.st_size
] = '\0';
688 *sizep
= (size_t)st
.st_size
;
694 * "list.h" is automatically generated; it just has a lot of lines like:
695 * DEFINE_TEST(function_name)
696 * It's used above to declare all of the test functions.
697 * We reuse it here to define a list of all tests (functions and names).
700 #define DEFINE_TEST(n) { n, #n },
701 struct { void (*func
)(void); const char *name
; } tests
[] = {
706 * Each test is run in a private work dir. Those work dirs
707 * do have consistent and predictable names, in case a group
708 * of tests need to collaborate. However, there is no provision
709 * for requiring that tests run in a certain order.
711 static int test_run(int i
, const char *tmpdir
)
713 int failures_before
= failures
;
716 printf("%d: %s\n", i
, tests
[i
].name
);
721 * Always explicitly chdir() in case the last test moved us to
726 "ERROR: Couldn't chdir to temp dir %s\n",
730 /* Create a temp directory for this specific test. */
731 if (mkdir(tests
[i
].name
, 0755)) {
733 "ERROR: Couldn't create temp dir ``%s''\n",
737 /* Chdir() to that work directory. */
738 if (chdir(tests
[i
].name
)) {
740 "ERROR: Couldn't chdir to temp dir ``%s''\n",
744 /* Explicitly reset the locale before each test. */
745 setlocale(LC_ALL
, "C");
746 /* Run the actual test. */
748 /* Summarize the results of this test. */
750 /* If there were no failures, we can remove the work dir. */
751 if (failures
== failures_before
) {
752 if (!keep_temp_files
&& chdir(tmpdir
) == 0) {
753 systemf("rm -rf %s", tests
[i
].name
);
756 /* Return appropriate status. */
757 return (failures
== failures_before
? 0 : 1);
760 static void usage(const char *program
)
762 static const int limit
= sizeof(tests
) / sizeof(tests
[0]);
765 printf("Usage: %s [options] <test> <test> ...\n", program
);
766 printf("Default is to run all tests.\n");
767 printf("Otherwise, specify the numbers of the tests you wish to run.\n");
768 printf("Options:\n");
769 printf(" -d Dump core after any failure, for debugging.\n");
770 printf(" -k Keep all temp files.\n");
771 printf(" Default: temp files for successful tests deleted.\n");
773 printf(" -p <path> Path to executable to be tested.\n");
774 printf(" Default: path taken from " ENVBASE
" environment variable.\n");
776 printf(" -q Quiet.\n");
777 printf(" -r <dir> Path to dir containing reference files.\n");
778 printf(" Default: Current directory.\n");
779 printf(" -v Verbose.\n");
780 printf("Available tests:\n");
781 for (i
= 0; i
< limit
; i
++)
782 printf(" %d: %s\n", i
, tests
[i
].name
);
786 #define UUDECODE(c) (((c) - 0x20) & 0x3f)
789 extract_reference_file(const char *name
)
794 sprintf(buff
, "%s/%s.uu", refdir
, name
);
795 in
= fopen(buff
, "r");
796 failure("Couldn't open reference file %s", buff
);
800 /* Read up to and including the 'begin' line. */
802 if (fgets(buff
, sizeof(buff
), in
) == NULL
) {
803 /* TODO: This is a failure. */
806 if (memcmp(buff
, "begin ", 6) == 0)
809 /* Now, decode the rest and write it. */
810 /* Not a lot of error checking here; the input better be right. */
811 out
= fopen(name
, "w");
812 while (fgets(buff
, sizeof(buff
), in
) != NULL
) {
816 if (memcmp(buff
, "end", 3) == 0)
819 bytes
= UUDECODE(*p
++);
822 /* Write out 1-3 bytes from that. */
824 n
= UUDECODE(*p
++) << 18;
825 n
|= UUDECODE(*p
++) << 12;
830 n
|= UUDECODE(*p
++) << 6;
831 fputc((n
>> 8) & 0xFF, out
);
836 fputc(n
& 0xFF, out
);
846 int main(int argc
, char **argv
)
848 static const int limit
= sizeof(tests
) / sizeof(tests
[0]);
849 int i
, tests_run
= 0, tests_failed
= 0, opt
;
851 char *refdir_alloc
= NULL
;
854 char tmpdir_timestamp
[256];
857 * Name of this program, used to build root of our temp directory
860 progname
= p
= argv
[0];
868 /* Get the target program from environment, if available. */
869 testprog
= getenv(ENVBASE
);
872 /* Allow -d to be controlled through the environment. */
873 if (getenv(ENVBASE
"_DEBUG") != NULL
)
876 /* Get the directory holding test files from environment. */
877 refdir
= getenv(ENVBASE
"_TEST_FILES");
882 while ((opt
= getopt(argc
, argv
, "dkp:qr:v")) != -1) {
915 * Sanity-check that our options make sense.
918 if (testprog
== NULL
)
923 * Create a temp directory for the following tests.
924 * Include the time the tests started as part of the name,
925 * to make it easier to track the results of multiple tests.
928 for (i
= 0; i
< 1000; i
++) {
929 strftime(tmpdir_timestamp
, sizeof(tmpdir_timestamp
),
932 sprintf(tmpdir
, "/tmp/%s.%s-%03d", progname
, tmpdir_timestamp
, i
);
933 if (mkdir(tmpdir
,0755) == 0)
937 fprintf(stderr
, "ERROR: Unable to create temp directory %s\n",
943 * If the user didn't specify a directory for locating
944 * reference files, use the current directory for that.
946 if (refdir
== NULL
) {
947 systemf("/bin/pwd > %s/refdir", tmpdir
);
948 refdir
= refdir_alloc
= slurpfile(NULL
, "%s/refdir", tmpdir
);
949 p
= refdir
+ strlen(refdir
);
950 while (p
[-1] == '\n') {
954 systemf("rm %s/refdir", tmpdir
);
958 * Banner with basic information.
961 printf("Running tests in: %s\n", tmpdir
);
962 printf("Reference files will be read from: %s\n", refdir
);
964 printf("Running tests on: %s\n", testprog
);
966 printf("Exercising: ");
968 printf("%s\n", EXTRA_VERSION
);
972 * Run some or all of the individual tests.
975 /* Default: Run all tests. */
976 for (i
= 0; i
< limit
; i
++) {
977 if (test_run(i
, tmpdir
))
982 while (*(argv
) != NULL
) {
984 if (**argv
< '0' || **argv
> '9' || i
< 0 || i
>= limit
) {
985 printf("*** INVALID Test %s\n", *argv
);
988 if (test_run(i
, tmpdir
))
997 * Report summary statistics.
1001 printf("%d of %d tests reported failures\n",
1002 tests_failed
, tests_run
);
1003 printf(" Total of %d assertions checked.\n", assertions
);
1004 printf(" Total of %d assertions failed.\n", failures
);
1005 printf(" Total of %d assertions skipped.\n", skips
);
1010 /* If the final tmpdir is empty, we can remove it. */
1011 /* This should be the usual case when all tests succeed. */
1014 return (tests_failed
);