1 /* GLib testing framework runner
2 * Copyright (C) 2007 Sven Herzberg
3 * Copyright (C) 2007 Tim Janik
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Lesser General Public
7 * License as published by the Free Software Foundation; either
8 * version 2 of the License, or (at your option) any later version.
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Lesser General Public License for more details.
15 * You should have received a copy of the GNU Lesser General Public
16 * License along with this library; if not, write to the
17 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 * Boston, MA 02111-1307, USA.
23 #include <glib-unix.h>
33 /* the read buffer size in bytes */
34 #define READ_BUFFER_SIZE 4096
36 /* --- prototypes --- */
37 static int main_selftest (int argc
,
39 static void parse_args (gint
*argc_p
,
42 /* --- variables --- */
43 static GIOChannel
*ioc_report
= NULL
;
44 static gboolean gtester_quiet
= FALSE
;
45 static gboolean gtester_verbose
= FALSE
;
46 static gboolean gtester_list_tests
= FALSE
;
47 static gboolean gtester_selftest
= FALSE
;
48 static gboolean subtest_running
= FALSE
;
49 static gint subtest_exitstatus
= 0;
50 static gboolean subtest_io_pending
= FALSE
;
51 static gboolean subtest_quiet
= TRUE
;
52 static gboolean subtest_verbose
= FALSE
;
53 static gboolean subtest_mode_fatal
= TRUE
;
54 static gboolean subtest_mode_perf
= FALSE
;
55 static gboolean subtest_mode_quick
= TRUE
;
56 static gboolean subtest_mode_undefined
= TRUE
;
57 static const gchar
*subtest_seedstr
= NULL
;
58 static gchar
*subtest_last_seed
= NULL
;
59 static GSList
*subtest_paths
= NULL
;
60 static GSList
*skipped_paths
= NULL
;
61 static GSList
*subtest_args
= NULL
;
62 static gboolean testcase_open
= FALSE
;
63 static guint testcase_count
= 0;
64 static guint testcase_fail_count
= 0;
65 static const gchar
*output_filename
= NULL
;
66 static guint log_indent
= 0;
67 static gint log_fd
= -1;
69 /* --- functions --- */
73 static const char spaces
[] = " ";
74 int l
= sizeof (spaces
) - 1;
76 return spaces
+ l
- n
;
79 static void G_GNUC_PRINTF (1, 2)
80 test_log_printfe (const char *format
,
86 va_start (args
, format
);
87 result
= g_markup_vprintf_escaped (format
, args
);
90 r
= write (log_fd
, result
, strlen (result
));
91 while (r
< 0 && errno
== EINTR
);
98 kill (getpid(), SIGTERM
);
103 testcase_close (long double duration
,
107 g_return_if_fail (testcase_open
> 0);
108 test_log_printfe ("%s<duration>%.6Lf</duration>\n", sindent (log_indent
), duration
);
109 test_log_printfe ("%s<status exit-status=\"%d\" n-forks=\"%d\" result=\"%s\"/>\n",
110 sindent (log_indent
), exit_status
, n_forks
,
111 exit_status
? "failed" : "success");
113 test_log_printfe ("%s</testcase>\n", sindent (log_indent
));
116 g_print ("%s\n", exit_status
? "FAIL" : "OK");
117 if (exit_status
&& subtest_last_seed
)
118 g_print ("GTester: last random seed: %s\n", subtest_last_seed
);
120 testcase_fail_count
+= 1;
121 if (subtest_mode_fatal
&& exit_status
)
126 test_log_msg (GTestLogMsg
*msg
)
128 switch (msg
->log_type
)
132 case G_TEST_LOG_NONE
:
134 case G_TEST_LOG_ERROR
:
135 strv
= g_strsplit (msg
->strings
[0], "\n", -1);
136 for (i
= 0; strv
[i
]; i
++)
137 test_log_printfe ("%s<error>%s</error>\n", sindent (log_indent
), strv
[i
]);
140 case G_TEST_LOG_START_BINARY
:
141 test_log_printfe ("%s<binary file=\"%s\"/>\n", sindent (log_indent
), msg
->strings
[0]);
142 subtest_last_seed
= g_strdup (msg
->strings
[1]);
143 test_log_printfe ("%s<random-seed>%s</random-seed>\n", sindent (log_indent
), subtest_last_seed
);
145 case G_TEST_LOG_LIST_CASE
:
146 g_print ("%s\n", msg
->strings
[0]);
148 case G_TEST_LOG_START_CASE
:
152 gchar
*sc
= g_strconcat (msg
->strings
[0], ":", NULL
);
153 gchar
*sleft
= g_strdup_printf ("%-68s", sc
);
155 g_print ("%70s ", sleft
);
158 g_return_if_fail (testcase_open
== 0);
160 test_log_printfe ("%s<testcase path=\"%s\">\n", sindent (log_indent
), msg
->strings
[0]);
163 case G_TEST_LOG_SKIP_CASE
:
164 if (FALSE
&& gtester_verbose
) /* enable to debug test case skipping logic */
166 gchar
*sc
= g_strconcat (msg
->strings
[0], ":", NULL
);
167 gchar
*sleft
= g_strdup_printf ("%-68s", sc
);
169 g_print ("%70s SKIPPED\n", sleft
);
172 test_log_printfe ("%s<testcase path=\"%s\" skipped=\"1\"/>\n", sindent (log_indent
), msg
->strings
[0]);
174 case G_TEST_LOG_STOP_CASE
:
175 testcase_close (msg
->nums
[2], (int) msg
->nums
[0], (int) msg
->nums
[1]);
177 case G_TEST_LOG_MIN_RESULT
:
178 case G_TEST_LOG_MAX_RESULT
:
179 test_log_printfe ("%s<performance minimize=\"%d\" maximize=\"%d\" value=\"%.16Lg\">\n",
180 sindent (log_indent
), msg
->log_type
== G_TEST_LOG_MIN_RESULT
, msg
->log_type
== G_TEST_LOG_MAX_RESULT
, msg
->nums
[0]);
181 test_log_printfe ("%s%s\n", sindent (log_indent
+ 2), msg
->strings
[0]);
182 test_log_printfe ("%s</performance>\n", sindent (log_indent
));
184 case G_TEST_LOG_MESSAGE
:
185 test_log_printfe ("%s<message>\n%s\n%s</message>\n", sindent (log_indent
), msg
->strings
[0], sindent (log_indent
));
191 child_report_cb (GIOChannel
*source
,
192 GIOCondition condition
,
195 GTestLogBuffer
*tlb
= data
;
196 GIOStatus status
= G_IO_STATUS_NORMAL
;
197 gboolean first_read_eof
= FALSE
, first_read
= TRUE
;
201 guint8 buffer
[READ_BUFFER_SIZE
];
202 GError
*error
= NULL
;
203 status
= g_io_channel_read_chars (source
, (gchar
*) buffer
, sizeof (buffer
), &length
, &error
);
204 if (first_read
&& (condition
& G_IO_IN
))
206 /* on some unixes (MacOS) we need to detect non-blocking fd EOF
207 * by an IO_IN select/poll followed by read()==0.
209 first_read_eof
= length
== 0;
215 g_test_log_buffer_push (tlb
, length
, buffer
);
218 msg
= g_test_log_buffer_pop (tlb
);
222 g_test_log_msg_free (msg
);
227 g_clear_error (&error
);
228 /* ignore the io channel status, which will report intermediate EOFs for non blocking fds */
232 /* g_print ("LASTIOSTATE: first_read_eof=%d condition=%d\n", first_read_eof, condition); */
233 if (first_read_eof
|| (condition
& (G_IO_ERR
| G_IO_HUP
)))
235 /* if there's no data to read and select() reports an error or hangup,
236 * the fd must have been closed remotely
238 subtest_io_pending
= FALSE
;
241 return TRUE
; /* keep polling */
245 child_watch_cb (GPid pid
,
249 g_spawn_close_pid (pid
);
250 if (WIFEXITED (status
)) /* normal exit */
251 subtest_exitstatus
= WEXITSTATUS (status
);
252 else /* signal or core dump, etc */
253 subtest_exitstatus
= 0xffffffff;
254 subtest_running
= FALSE
;
258 queue_gfree (GSList
**slistp
,
261 *slistp
= g_slist_prepend (*slistp
, string
);
266 unset_cloexec_fdp (gpointer fdp_data
)
268 int r
, *fdp
= fdp_data
;
270 r
= fcntl (*fdp
, F_SETFD
, 0 /* FD_CLOEXEC */);
271 while (r
< 0 && errno
== EINTR
);
275 launch_test_binary (const char *binary
,
279 GSList
*slist
, *free_list
= NULL
;
280 GError
*error
= NULL
;
284 gint report_pipe
[2] = { -1, -1 };
285 guint child_report_cb_id
= 0;
286 gboolean loop_pending
;
289 if (!g_unix_open_pipe (report_pipe
, FD_CLOEXEC
, &error
))
291 if (subtest_mode_fatal
)
292 g_error ("Failed to open pipe for test binary: %s: %s", binary
, error
->message
);
294 g_warning ("Failed to open pipe for test binary: %s: %s", binary
, error
->message
);
295 g_clear_error (&error
);
300 for (slist
= subtest_args
; slist
; slist
= slist
->next
)
307 if (!subtest_mode_fatal
)
309 if (subtest_mode_quick
)
313 if (subtest_mode_perf
)
315 if (!subtest_mode_undefined
)
317 if (gtester_list_tests
)
324 for (slist
= subtest_paths
; slist
; slist
= slist
->next
)
326 for (slist
= skipped_paths
; slist
; slist
= slist
->next
)
330 argv
= g_malloc ((argc
+ 2) * sizeof(gchar
*));
332 for (slist
= subtest_args
; slist
; slist
= slist
->next
)
333 argv
[i
++] = (gchar
*) slist
->data
;
334 /* argv[i++] = "--debug-log"; */
336 argv
[i
++] = "--quiet";
338 argv
[i
++] = "--verbose";
339 if (!subtest_mode_fatal
)
340 argv
[i
++] = "--keep-going";
341 if (subtest_mode_quick
)
342 argv
[i
++] = "-m=quick";
344 argv
[i
++] = "-m=slow";
345 if (subtest_mode_perf
)
346 argv
[i
++] = "-m=perf";
347 if (!subtest_mode_undefined
)
348 argv
[i
++] = "-m=no-undefined";
349 if (gtester_list_tests
)
352 argv
[i
++] = queue_gfree (&free_list
, g_strdup_printf ("--seed=%s", subtest_seedstr
));
353 argv
[i
++] = queue_gfree (&free_list
, g_strdup_printf ("--GTestLogFD=%u", report_pipe
[1]));
355 argv
[i
++] = queue_gfree (&free_list
, g_strdup_printf ("--GTestSkipCount=%u", skip_tests
));
356 for (slist
= subtest_paths
; slist
; slist
= slist
->next
)
357 argv
[i
++] = queue_gfree (&free_list
, g_strdup_printf ("-p=%s", (gchar
*) slist
->data
));
358 for (slist
= skipped_paths
; slist
; slist
= slist
->next
)
359 argv
[i
++] = queue_gfree (&free_list
, g_strdup_printf ("-s=%s", (gchar
*) slist
->data
));
362 g_spawn_async_with_pipes (NULL
, /* g_get_current_dir() */
365 G_SPAWN_DO_NOT_REAP_CHILD
, /* G_SPAWN_SEARCH_PATH */
366 unset_cloexec_fdp
, &report_pipe
[1], /* pre-exec callback */
368 NULL
, /* standard_input */
369 NULL
, /* standard_output */
370 NULL
, /* standard_error */
372 g_slist_foreach (free_list
, (void(*)(void*,void*)) g_free
, NULL
);
373 g_slist_free (free_list
);
375 close (report_pipe
[1]);
378 g_print ("(pid=%lu)\n", (unsigned long) pid
);
382 close (report_pipe
[0]);
383 if (subtest_mode_fatal
)
384 g_error ("Failed to execute test binary: %s: %s", argv
[0], error
->message
);
386 g_warning ("Failed to execute test binary: %s: %s", argv
[0], error
->message
);
387 g_clear_error (&error
);
393 subtest_running
= TRUE
;
394 subtest_io_pending
= TRUE
;
395 tlb
= g_test_log_buffer_new();
396 if (report_pipe
[0] >= 0)
398 ioc_report
= g_io_channel_unix_new (report_pipe
[0]);
399 g_io_channel_set_flags (ioc_report
, G_IO_FLAG_NONBLOCK
, NULL
);
400 g_io_channel_set_encoding (ioc_report
, NULL
, NULL
);
401 g_io_channel_set_buffered (ioc_report
, FALSE
);
402 child_report_cb_id
= g_io_add_watch_full (ioc_report
, G_PRIORITY_DEFAULT
- 1, G_IO_IN
| G_IO_ERR
| G_IO_HUP
, child_report_cb
, tlb
, NULL
);
403 g_io_channel_unref (ioc_report
);
405 g_child_watch_add_full (G_PRIORITY_DEFAULT
+ 1, pid
, child_watch_cb
, NULL
, NULL
);
407 loop_pending
= g_main_context_pending (NULL
);
408 while (subtest_running
|| /* FALSE once child exits */
409 subtest_io_pending
|| /* FALSE once ioc_report closes */
410 loop_pending
) /* TRUE while idler, etc are running */
412 /* g_print ("LOOPSTATE: subtest_running=%d subtest_io_pending=%d\n", subtest_running, subtest_io_pending); */
413 /* check for unexpected hangs that are not signalled on report_pipe */
414 if (!subtest_running
&& /* child exited */
415 subtest_io_pending
&& /* no EOF detected on report_pipe */
416 !loop_pending
) /* no IO events pending however */
418 g_main_context_iteration (NULL
, TRUE
);
419 loop_pending
= g_main_context_pending (NULL
);
422 g_source_remove (child_report_cb_id
);
423 close (report_pipe
[0]);
424 g_test_log_buffer_free (tlb
);
430 launch_test (const char *binary
)
432 gboolean success
= TRUE
;
433 GTimer
*btimer
= g_timer_new();
434 gboolean need_restart
;
438 g_print ("TEST: %s... ", binary
);
441 test_log_printfe ("%s<testbinary path=\"%s\">\n", sindent (log_indent
), binary
);
443 g_timer_start (btimer
);
444 subtest_exitstatus
= 0;
445 success
&= launch_test_binary (binary
, testcase_count
);
446 success
&= subtest_exitstatus
== 0;
447 need_restart
= testcase_open
!= 0;
449 testcase_close (0, -256, 0);
450 g_timer_stop (btimer
);
451 test_log_printfe ("%s<duration>%.6f</duration>\n", sindent (log_indent
), g_timer_elapsed (btimer
, NULL
));
453 test_log_printfe ("%s</testbinary>\n", sindent (log_indent
));
454 g_free (subtest_last_seed
);
455 subtest_last_seed
= NULL
;
458 /* restart test binary, skipping processed test cases */
462 /* count the inability to run a test as a failure */
463 if (!success
&& testcase_count
== 0)
464 testcase_fail_count
++;
467 g_print ("%s: %s\n", !success
? "FAIL" : "PASS", binary
);
468 g_timer_destroy (btimer
);
469 if (subtest_mode_fatal
&& !success
)
474 usage (gboolean just_version
)
478 g_print ("gtester version %d.%d.%d\n", GLIB_MAJOR_VERSION
, GLIB_MINOR_VERSION
, GLIB_MICRO_VERSION
);
481 g_print ("Usage:\n");
482 g_print ("gtester [OPTIONS] testprogram...\n\n");
483 /* 12345678901234567890123456789012345678901234567890123456789012345678901234567890 */
484 g_print ("Help Options:\n");
485 g_print (" -h, --help Show this help message\n\n");
486 g_print ("Utility Options:\n");
487 g_print (" -v, --version Print version informations\n");
488 g_print (" --g-fatal-warnings Make warnings fatal (abort)\n");
489 g_print (" -k, --keep-going Continue running after tests failed\n");
490 g_print (" -l List paths of available test cases\n");
491 g_print (" -m {perf|slow|thorough|quick} Run test cases according to mode\n");
492 g_print (" -m {undefined|no-undefined} Run test cases according to mode\n");
493 g_print (" -p=TESTPATH Only start test cases matching TESTPATH\n");
494 g_print (" -s=TESTPATH Skip test cases matching TESTPATH\n");
495 g_print (" --seed=SEEDSTRING Start tests with random seed SEEDSTRING\n");
496 g_print (" -o=LOGFILE Write the test log to LOGFILE\n");
497 g_print (" -q, --quiet Suppress per test binary output\n");
498 g_print (" --verbose Report success per testcase\n");
502 parse_args (gint
*argc_p
,
505 guint argc
= *argc_p
;
506 gchar
**argv
= *argv_p
;
508 /* parse known args */
509 for (i
= 1; i
< argc
; i
++)
511 if (strcmp (argv
[i
], "--g-fatal-warnings") == 0)
513 GLogLevelFlags fatal_mask
= (GLogLevelFlags
) g_log_set_always_fatal ((GLogLevelFlags
) G_LOG_FATAL_MASK
);
514 fatal_mask
= (GLogLevelFlags
) (fatal_mask
| G_LOG_LEVEL_WARNING
| G_LOG_LEVEL_CRITICAL
);
515 g_log_set_always_fatal (fatal_mask
);
518 else if (strcmp (argv
[i
], "--gtester-selftest") == 0)
520 gtester_selftest
= TRUE
;
522 break; /* stop parsing regular gtester arguments */
524 else if (strcmp (argv
[i
], "-h") == 0 || strcmp (argv
[i
], "--help") == 0)
530 else if (strcmp (argv
[i
], "-v") == 0 || strcmp (argv
[i
], "--version") == 0)
536 else if (strcmp (argv
[i
], "--keep-going") == 0 ||
537 strcmp (argv
[i
], "-k") == 0)
539 subtest_mode_fatal
= FALSE
;
542 else if (strcmp ("-p", argv
[i
]) == 0 || strncmp ("-p=", argv
[i
], 3) == 0)
544 gchar
*equal
= argv
[i
] + 2;
546 subtest_paths
= g_slist_prepend (subtest_paths
, equal
+ 1);
547 else if (i
+ 1 < argc
)
550 subtest_paths
= g_slist_prepend (subtest_paths
, argv
[i
]);
554 else if (strcmp ("-s", argv
[i
]) == 0 || strncmp ("-s=", argv
[i
], 3) == 0)
556 gchar
*equal
= argv
[i
] + 2;
558 skipped_paths
= g_slist_prepend (skipped_paths
, equal
+ 1);
559 else if (i
+ 1 < argc
)
562 skipped_paths
= g_slist_prepend (skipped_paths
, argv
[i
]);
566 else if (strcmp ("--test-arg", argv
[i
]) == 0 || strncmp ("--test-arg=", argv
[i
], 11) == 0)
568 gchar
*equal
= argv
[i
] + 10;
570 subtest_args
= g_slist_prepend (subtest_args
, equal
+ 1);
571 else if (i
+ 1 < argc
)
574 subtest_args
= g_slist_prepend (subtest_args
, argv
[i
]);
578 else if (strcmp ("-o", argv
[i
]) == 0 || strncmp ("-o=", argv
[i
], 3) == 0)
580 gchar
*equal
= argv
[i
] + 2;
582 output_filename
= equal
+ 1;
583 else if (i
+ 1 < argc
)
586 output_filename
= argv
[i
];
590 else if (strcmp ("-m", argv
[i
]) == 0 || strncmp ("-m=", argv
[i
], 3) == 0)
592 gchar
*equal
= argv
[i
] + 2;
593 const gchar
*mode
= "";
596 else if (i
+ 1 < argc
)
601 if (strcmp (mode
, "perf") == 0)
602 subtest_mode_perf
= TRUE
;
603 else if (strcmp (mode
, "slow") == 0 || strcmp (mode
, "thorough") == 0)
604 subtest_mode_quick
= FALSE
;
605 else if (strcmp (mode
, "quick") == 0)
607 subtest_mode_quick
= TRUE
;
608 subtest_mode_perf
= FALSE
;
610 else if (strcmp (mode
, "undefined") == 0)
611 subtest_mode_undefined
= TRUE
;
612 else if (strcmp (mode
, "no-undefined") == 0)
613 subtest_mode_undefined
= FALSE
;
615 g_error ("unknown test mode: -m %s", mode
);
618 else if (strcmp ("-q", argv
[i
]) == 0 || strcmp ("--quiet", argv
[i
]) == 0)
620 gtester_quiet
= TRUE
;
621 gtester_verbose
= FALSE
;
624 else if (strcmp ("--verbose", argv
[i
]) == 0)
626 gtester_quiet
= FALSE
;
627 gtester_verbose
= TRUE
;
630 else if (strcmp ("-l", argv
[i
]) == 0)
632 gtester_list_tests
= TRUE
;
635 else if (strcmp ("--seed", argv
[i
]) == 0 || strncmp ("--seed=", argv
[i
], 7) == 0)
637 gchar
*equal
= argv
[i
] + 6;
639 subtest_seedstr
= equal
+ 1;
640 else if (i
+ 1 < argc
)
643 subtest_seedstr
= argv
[i
];
650 for (i
= 1; i
< argc
; i
++)
666 g_set_prgname (argv
[0]);
667 parse_args (&argc
, &argv
);
668 if (gtester_selftest
)
669 return main_selftest (argc
, argv
);
679 log_fd
= g_open (output_filename
, O_WRONLY
| O_CREAT
| O_TRUNC
, 0666);
681 g_error ("Failed to open log file '%s': %s", output_filename
, g_strerror (errno
));
684 test_log_printfe ("<?xml version=\"1.0\"?>\n");
685 test_log_printfe ("%s<gtester>\n", sindent (log_indent
));
687 for (ui
= 1; ui
< argc
; ui
++)
689 const char *binary
= argv
[ui
];
690 launch_test (binary
);
691 /* we only get here on success or if !subtest_mode_fatal */
694 test_log_printfe ("%s</gtester>\n", sindent (log_indent
));
698 return testcase_fail_count
== 0 ? EXIT_SUCCESS
: EXIT_FAILURE
;
702 fixture_setup (guint
*fix
,
703 gconstpointer test_data
)
705 g_assert_cmphex (*fix
, ==, 0);
709 fixture_test (guint
*fix
,
710 gconstpointer test_data
)
712 g_assert_cmphex (*fix
, ==, 0xdeadbeef);
713 g_test_message ("This is a test message API test message.");
714 g_test_bug_base ("http://www.example.com/bugtracker/");
716 g_test_bug_base ("http://www.example.com/bugtracker?bugnum=%s;cmd=showbug");
720 fixture_teardown (guint
*fix
,
721 gconstpointer test_data
)
723 g_assert_cmphex (*fix
, ==, 0xdeadbeef);
727 main_selftest (int argc
,
730 /* gtester main() for --gtester-selftest invocations */
731 g_test_init (&argc
, &argv
, NULL
);
732 g_test_add ("/gtester/fixture-test", guint
, NULL
, fixture_setup
, fixture_test
, fixture_teardown
);