1 /* GLib testing framework runner
2 * Copyright (C) 2007 Sven Herzberg
3 * Copyright (C) 2007 Tim Janik
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Lesser General Public
7 * License as published by the Free Software Foundation; either
8 * version 2 of the License, or (at your option) any later version.
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Lesser General Public License for more details.
15 * You should have received a copy of the GNU Lesser General Public
16 * License along with this library; if not, write to the
17 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 * Boston, MA 02111-1307, USA.
30 /* the read buffer size in bytes */
31 #define READ_BUFFER_SIZE 4096
33 /* --- prototypes --- */
34 static int main_selftest (int argc
,
36 static void parse_args (gint
*argc_p
,
39 /* --- variables --- */
40 static GIOChannel
*ioc_report
= NULL
;
41 static gboolean gtester_quiet
= FALSE
;
42 static gboolean gtester_verbose
= FALSE
;
43 static gboolean gtester_list_tests
= FALSE
;
44 static gboolean gtester_selftest
= FALSE
;
45 static gboolean subtest_running
= FALSE
;
46 static gint subtest_exitstatus
= 0;
47 static gboolean subtest_io_pending
= FALSE
;
48 static gboolean subtest_quiet
= TRUE
;
49 static gboolean subtest_verbose
= FALSE
;
50 static gboolean subtest_mode_fatal
= TRUE
;
51 static gboolean subtest_mode_perf
= FALSE
;
52 static gboolean subtest_mode_quick
= TRUE
;
53 static gboolean subtest_mode_undefined
= TRUE
;
54 static const gchar
*subtest_seedstr
= NULL
;
55 static gchar
*subtest_last_seed
= NULL
;
56 static GSList
*subtest_paths
= NULL
;
57 static GSList
*skipped_paths
= NULL
;
58 static GSList
*subtest_args
= NULL
;
59 static gboolean testcase_open
= FALSE
;
60 static guint testcase_count
= 0;
61 static guint testcase_fail_count
= 0;
62 static const gchar
*output_filename
= NULL
;
63 static guint log_indent
= 0;
64 static gint log_fd
= -1;
66 /* --- functions --- */
70 static const char spaces
[] = " ";
71 int l
= sizeof (spaces
) - 1;
73 return spaces
+ l
- n
;
76 static void G_GNUC_PRINTF (1, 2)
77 test_log_printfe (const char *format
,
83 va_start (args
, format
);
84 result
= g_markup_vprintf_escaped (format
, args
);
87 r
= write (log_fd
, result
, strlen (result
));
88 while (r
< 0 && errno
== EINTR
);
95 kill (getpid(), SIGTERM
);
100 testcase_close (long double duration
,
104 g_return_if_fail (testcase_open
> 0);
105 test_log_printfe ("%s<duration>%.6Lf</duration>\n", sindent (log_indent
), duration
);
106 test_log_printfe ("%s<status exit-status=\"%d\" n-forks=\"%d\" result=\"%s\"/>\n",
107 sindent (log_indent
), exit_status
, n_forks
,
108 exit_status
? "failed" : "success");
110 test_log_printfe ("%s</testcase>\n", sindent (log_indent
));
113 g_print ("%s\n", exit_status
? "FAIL" : "OK");
114 if (exit_status
&& subtest_last_seed
)
115 g_print ("GTester: last random seed: %s\n", subtest_last_seed
);
117 testcase_fail_count
+= 1;
118 if (subtest_mode_fatal
&& exit_status
)
123 test_log_msg (GTestLogMsg
*msg
)
125 switch (msg
->log_type
)
129 case G_TEST_LOG_NONE
:
131 case G_TEST_LOG_ERROR
:
132 strv
= g_strsplit (msg
->strings
[0], "\n", -1);
133 for (i
= 0; strv
[i
]; i
++)
134 test_log_printfe ("%s<error>%s</error>\n", sindent (log_indent
), strv
[i
]);
137 case G_TEST_LOG_START_BINARY
:
138 test_log_printfe ("%s<binary file=\"%s\"/>\n", sindent (log_indent
), msg
->strings
[0]);
139 subtest_last_seed
= g_strdup (msg
->strings
[1]);
140 test_log_printfe ("%s<random-seed>%s</random-seed>\n", sindent (log_indent
), subtest_last_seed
);
142 case G_TEST_LOG_LIST_CASE
:
143 g_print ("%s\n", msg
->strings
[0]);
145 case G_TEST_LOG_START_CASE
:
149 gchar
*sc
= g_strconcat (msg
->strings
[0], ":", NULL
);
150 gchar
*sleft
= g_strdup_printf ("%-68s", sc
);
152 g_print ("%70s ", sleft
);
155 g_return_if_fail (testcase_open
== 0);
157 test_log_printfe ("%s<testcase path=\"%s\">\n", sindent (log_indent
), msg
->strings
[0]);
160 case G_TEST_LOG_SKIP_CASE
:
161 if (FALSE
&& gtester_verbose
) /* enable to debug test case skipping logic */
163 gchar
*sc
= g_strconcat (msg
->strings
[0], ":", NULL
);
164 gchar
*sleft
= g_strdup_printf ("%-68s", sc
);
166 g_print ("%70s SKIPPED\n", sleft
);
169 test_log_printfe ("%s<testcase path=\"%s\" skipped=\"1\"/>\n", sindent (log_indent
), msg
->strings
[0]);
171 case G_TEST_LOG_STOP_CASE
:
172 testcase_close (msg
->nums
[2], (int) msg
->nums
[0], (int) msg
->nums
[1]);
174 case G_TEST_LOG_MIN_RESULT
:
175 case G_TEST_LOG_MAX_RESULT
:
176 test_log_printfe ("%s<performance minimize=\"%d\" maximize=\"%d\" value=\"%.16Lg\">\n",
177 sindent (log_indent
), msg
->log_type
== G_TEST_LOG_MIN_RESULT
, msg
->log_type
== G_TEST_LOG_MAX_RESULT
, msg
->nums
[0]);
178 test_log_printfe ("%s%s\n", sindent (log_indent
+ 2), msg
->strings
[0]);
179 test_log_printfe ("%s</performance>\n", sindent (log_indent
));
181 case G_TEST_LOG_MESSAGE
:
182 test_log_printfe ("%s<message>\n%s\n%s</message>\n", sindent (log_indent
), msg
->strings
[0], sindent (log_indent
));
188 child_report_cb (GIOChannel
*source
,
189 GIOCondition condition
,
192 GTestLogBuffer
*tlb
= data
;
193 GIOStatus status
= G_IO_STATUS_NORMAL
;
194 gboolean first_read_eof
= FALSE
, first_read
= TRUE
;
198 guint8 buffer
[READ_BUFFER_SIZE
];
199 GError
*error
= NULL
;
200 status
= g_io_channel_read_chars (source
, (gchar
*) buffer
, sizeof (buffer
), &length
, &error
);
201 if (first_read
&& (condition
& G_IO_IN
))
203 /* on some unixes (MacOS) we need to detect non-blocking fd EOF
204 * by an IO_IN select/poll followed by read()==0.
206 first_read_eof
= length
== 0;
212 g_test_log_buffer_push (tlb
, length
, buffer
);
215 msg
= g_test_log_buffer_pop (tlb
);
219 g_test_log_msg_free (msg
);
224 g_clear_error (&error
);
225 /* ignore the io channel status, which will report intermediate EOFs for non blocking fds */
229 /* g_print ("LASTIOSTATE: first_read_eof=%d condition=%d\n", first_read_eof, condition); */
230 if (first_read_eof
|| (condition
& (G_IO_ERR
| G_IO_HUP
)))
232 /* if there's no data to read and select() reports an error or hangup,
233 * the fd must have been closed remotely
235 subtest_io_pending
= FALSE
;
238 return TRUE
; /* keep polling */
242 child_watch_cb (GPid pid
,
246 g_spawn_close_pid (pid
);
247 if (WIFEXITED (status
)) /* normal exit */
248 subtest_exitstatus
= WEXITSTATUS (status
);
249 else /* signal or core dump, etc */
250 subtest_exitstatus
= 0xffffffff;
251 subtest_running
= FALSE
;
255 queue_gfree (GSList
**slistp
,
258 *slistp
= g_slist_prepend (*slistp
, string
);
263 unset_cloexec_fdp (gpointer fdp_data
)
265 int r
, *fdp
= fdp_data
;
267 r
= fcntl (*fdp
, F_SETFD
, 0 /* FD_CLOEXEC */);
268 while (r
< 0 && errno
== EINTR
);
272 launch_test_binary (const char *binary
,
276 GSList
*slist
, *free_list
= NULL
;
277 GError
*error
= NULL
;
281 gint report_pipe
[2] = { -1, -1 };
282 guint child_report_cb_id
= 0;
283 gboolean loop_pending
;
286 if (pipe (report_pipe
) < 0)
288 if (subtest_mode_fatal
)
289 g_error ("Failed to open pipe for test binary: %s: %s", binary
, g_strerror (errno
));
291 g_warning ("Failed to open pipe for test binary: %s: %s", binary
, g_strerror (errno
));
296 for (slist
= subtest_args
; slist
; slist
= slist
->next
)
303 if (!subtest_mode_fatal
)
305 if (subtest_mode_quick
)
309 if (subtest_mode_perf
)
311 if (!subtest_mode_undefined
)
313 if (gtester_list_tests
)
320 for (slist
= subtest_paths
; slist
; slist
= slist
->next
)
322 for (slist
= skipped_paths
; slist
; slist
= slist
->next
)
326 argv
= g_malloc ((argc
+ 2) * sizeof(gchar
*));
328 for (slist
= subtest_args
; slist
; slist
= slist
->next
)
329 argv
[i
++] = (gchar
*) slist
->data
;
330 /* argv[i++] = "--debug-log"; */
332 argv
[i
++] = "--quiet";
334 argv
[i
++] = "--verbose";
335 if (!subtest_mode_fatal
)
336 argv
[i
++] = "--keep-going";
337 if (subtest_mode_quick
)
338 argv
[i
++] = "-m=quick";
340 argv
[i
++] = "-m=slow";
341 if (subtest_mode_perf
)
342 argv
[i
++] = "-m=perf";
343 if (!subtest_mode_undefined
)
344 argv
[i
++] = "-m=no-undefined";
345 if (gtester_list_tests
)
348 argv
[i
++] = queue_gfree (&free_list
, g_strdup_printf ("--seed=%s", subtest_seedstr
));
349 argv
[i
++] = queue_gfree (&free_list
, g_strdup_printf ("--GTestLogFD=%u", report_pipe
[1]));
351 argv
[i
++] = queue_gfree (&free_list
, g_strdup_printf ("--GTestSkipCount=%u", skip_tests
));
352 for (slist
= subtest_paths
; slist
; slist
= slist
->next
)
353 argv
[i
++] = queue_gfree (&free_list
, g_strdup_printf ("-p=%s", (gchar
*) slist
->data
));
354 for (slist
= skipped_paths
; slist
; slist
= slist
->next
)
355 argv
[i
++] = queue_gfree (&free_list
, g_strdup_printf ("-s=%s", (gchar
*) slist
->data
));
358 g_spawn_async_with_pipes (NULL
, /* g_get_current_dir() */
361 G_SPAWN_DO_NOT_REAP_CHILD
, /* G_SPAWN_SEARCH_PATH */
362 unset_cloexec_fdp
, &report_pipe
[1], /* pre-exec callback */
364 NULL
, /* standard_input */
365 NULL
, /* standard_output */
366 NULL
, /* standard_error */
368 g_slist_foreach (free_list
, (void(*)(void*,void*)) g_free
, NULL
);
369 g_slist_free (free_list
);
371 close (report_pipe
[1]);
374 g_print ("(pid=%lu)\n", (unsigned long) pid
);
378 close (report_pipe
[0]);
379 if (subtest_mode_fatal
)
380 g_error ("Failed to execute test binary: %s: %s", argv
[0], error
->message
);
382 g_warning ("Failed to execute test binary: %s: %s", argv
[0], error
->message
);
383 g_clear_error (&error
);
389 subtest_running
= TRUE
;
390 subtest_io_pending
= TRUE
;
391 tlb
= g_test_log_buffer_new();
392 if (report_pipe
[0] >= 0)
394 ioc_report
= g_io_channel_unix_new (report_pipe
[0]);
395 g_io_channel_set_flags (ioc_report
, G_IO_FLAG_NONBLOCK
, NULL
);
396 g_io_channel_set_encoding (ioc_report
, NULL
, NULL
);
397 g_io_channel_set_buffered (ioc_report
, FALSE
);
398 child_report_cb_id
= g_io_add_watch_full (ioc_report
, G_PRIORITY_DEFAULT
- 1, G_IO_IN
| G_IO_ERR
| G_IO_HUP
, child_report_cb
, tlb
, NULL
);
399 g_io_channel_unref (ioc_report
);
401 g_child_watch_add_full (G_PRIORITY_DEFAULT
+ 1, pid
, child_watch_cb
, NULL
, NULL
);
403 loop_pending
= g_main_context_pending (NULL
);
404 while (subtest_running
|| /* FALSE once child exits */
405 subtest_io_pending
|| /* FALSE once ioc_report closes */
406 loop_pending
) /* TRUE while idler, etc are running */
408 /* g_print ("LOOPSTATE: subtest_running=%d subtest_io_pending=%d\n", subtest_running, subtest_io_pending); */
409 /* check for unexpected hangs that are not signalled on report_pipe */
410 if (!subtest_running
&& /* child exited */
411 subtest_io_pending
&& /* no EOF detected on report_pipe */
412 !loop_pending
) /* no IO events pending however */
414 g_main_context_iteration (NULL
, TRUE
);
415 loop_pending
= g_main_context_pending (NULL
);
418 g_source_remove (child_report_cb_id
);
419 close (report_pipe
[0]);
420 g_test_log_buffer_free (tlb
);
426 launch_test (const char *binary
)
428 gboolean success
= TRUE
;
429 GTimer
*btimer
= g_timer_new();
430 gboolean need_restart
;
434 g_print ("TEST: %s... ", binary
);
437 test_log_printfe ("%s<testbinary path=\"%s\">\n", sindent (log_indent
), binary
);
439 g_timer_start (btimer
);
440 subtest_exitstatus
= 0;
441 success
&= launch_test_binary (binary
, testcase_count
);
442 success
&= subtest_exitstatus
== 0;
443 need_restart
= testcase_open
!= 0;
445 testcase_close (0, -256, 0);
446 g_timer_stop (btimer
);
447 test_log_printfe ("%s<duration>%.6f</duration>\n", sindent (log_indent
), g_timer_elapsed (btimer
, NULL
));
449 test_log_printfe ("%s</testbinary>\n", sindent (log_indent
));
450 g_free (subtest_last_seed
);
451 subtest_last_seed
= NULL
;
454 /* restart test binary, skipping processed test cases */
458 /* count the inability to run a test as a failure */
459 if (!success
&& testcase_count
== 0)
460 testcase_fail_count
++;
463 g_print ("%s: %s\n", !success
? "FAIL" : "PASS", binary
);
464 g_timer_destroy (btimer
);
465 if (subtest_mode_fatal
&& !success
)
470 usage (gboolean just_version
)
474 g_print ("gtester version %d.%d.%d\n", GLIB_MAJOR_VERSION
, GLIB_MINOR_VERSION
, GLIB_MICRO_VERSION
);
477 g_print ("Usage: gtester [OPTIONS] testprogram...\n");
478 /* 12345678901234567890123456789012345678901234567890123456789012345678901234567890 */
479 g_print ("Options:\n");
480 g_print (" -h, --help show this help message\n");
481 g_print (" -v, --version print version informations\n");
482 g_print (" --g-fatal-warnings make warnings fatal (abort)\n");
483 g_print (" -k, --keep-going continue running after tests failed\n");
484 g_print (" -l list paths of available test cases\n");
485 g_print (" -m=perf, -m=slow, -m=quick -m=thorough\n");
486 g_print (" run test cases in mode perf, slow/thorough or quick (default)\n");
487 g_print (" -m=no-undefined don't run test cases that provoke assertions\n");
488 g_print (" -p=TESTPATH only start test cases matching TESTPATH\n");
489 g_print (" -s=TESTPATH skip test cases matching TESTPATH\n");
490 g_print (" --seed=SEEDSTRING start all tests with random number seed SEEDSTRING\n");
491 g_print (" -o=LOGFILE write the test log to LOGFILE\n");
492 g_print (" -q, --quiet suppress per test binary output\n");
493 g_print (" --verbose report success per testcase\n");
497 parse_args (gint
*argc_p
,
500 guint argc
= *argc_p
;
501 gchar
**argv
= *argv_p
;
503 /* parse known args */
504 for (i
= 1; i
< argc
; i
++)
506 if (strcmp (argv
[i
], "--g-fatal-warnings") == 0)
508 GLogLevelFlags fatal_mask
= (GLogLevelFlags
) g_log_set_always_fatal ((GLogLevelFlags
) G_LOG_FATAL_MASK
);
509 fatal_mask
= (GLogLevelFlags
) (fatal_mask
| G_LOG_LEVEL_WARNING
| G_LOG_LEVEL_CRITICAL
);
510 g_log_set_always_fatal (fatal_mask
);
513 else if (strcmp (argv
[i
], "--gtester-selftest") == 0)
515 gtester_selftest
= TRUE
;
517 break; /* stop parsing regular gtester arguments */
519 else if (strcmp (argv
[i
], "-h") == 0 || strcmp (argv
[i
], "--help") == 0)
525 else if (strcmp (argv
[i
], "-v") == 0 || strcmp (argv
[i
], "--version") == 0)
531 else if (strcmp (argv
[i
], "--keep-going") == 0 ||
532 strcmp (argv
[i
], "-k") == 0)
534 subtest_mode_fatal
= FALSE
;
537 else if (strcmp ("-p", argv
[i
]) == 0 || strncmp ("-p=", argv
[i
], 3) == 0)
539 gchar
*equal
= argv
[i
] + 2;
541 subtest_paths
= g_slist_prepend (subtest_paths
, equal
+ 1);
542 else if (i
+ 1 < argc
)
545 subtest_paths
= g_slist_prepend (subtest_paths
, argv
[i
]);
549 else if (strcmp ("-s", argv
[i
]) == 0 || strncmp ("-s=", argv
[i
], 3) == 0)
551 gchar
*equal
= argv
[i
] + 2;
553 skipped_paths
= g_slist_prepend (skipped_paths
, equal
+ 1);
554 else if (i
+ 1 < argc
)
557 skipped_paths
= g_slist_prepend (skipped_paths
, argv
[i
]);
561 else if (strcmp ("--test-arg", argv
[i
]) == 0 || strncmp ("--test-arg=", argv
[i
], 11) == 0)
563 gchar
*equal
= argv
[i
] + 10;
565 subtest_args
= g_slist_prepend (subtest_args
, equal
+ 1);
566 else if (i
+ 1 < argc
)
569 subtest_args
= g_slist_prepend (subtest_args
, argv
[i
]);
573 else if (strcmp ("-o", argv
[i
]) == 0 || strncmp ("-o=", argv
[i
], 3) == 0)
575 gchar
*equal
= argv
[i
] + 2;
577 output_filename
= equal
+ 1;
578 else if (i
+ 1 < argc
)
581 output_filename
= argv
[i
];
585 else if (strcmp ("-m", argv
[i
]) == 0 || strncmp ("-m=", argv
[i
], 3) == 0)
587 gchar
*equal
= argv
[i
] + 2;
588 const gchar
*mode
= "";
591 else if (i
+ 1 < argc
)
596 if (strcmp (mode
, "perf") == 0)
597 subtest_mode_perf
= TRUE
;
598 else if (strcmp (mode
, "slow") == 0 || strcmp (mode
, "thorough") == 0)
599 subtest_mode_quick
= FALSE
;
600 else if (strcmp (mode
, "quick") == 0)
602 subtest_mode_quick
= TRUE
;
603 subtest_mode_perf
= FALSE
;
605 else if (strcmp (mode
, "undefined") == 0)
606 subtest_mode_undefined
= TRUE
;
607 else if (strcmp (mode
, "no-undefined") == 0)
608 subtest_mode_undefined
= FALSE
;
610 g_error ("unknown test mode: -m %s", mode
);
613 else if (strcmp ("-q", argv
[i
]) == 0 || strcmp ("--quiet", argv
[i
]) == 0)
615 gtester_quiet
= TRUE
;
616 gtester_verbose
= FALSE
;
619 else if (strcmp ("--verbose", argv
[i
]) == 0)
621 gtester_quiet
= FALSE
;
622 gtester_verbose
= TRUE
;
625 else if (strcmp ("-l", argv
[i
]) == 0)
627 gtester_list_tests
= TRUE
;
630 else if (strcmp ("--seed", argv
[i
]) == 0 || strncmp ("--seed=", argv
[i
], 7) == 0)
632 gchar
*equal
= argv
[i
] + 6;
634 subtest_seedstr
= equal
+ 1;
635 else if (i
+ 1 < argc
)
638 subtest_seedstr
= argv
[i
];
645 for (i
= 1; i
< argc
; i
++)
661 g_set_prgname (argv
[0]);
662 parse_args (&argc
, &argv
);
663 if (gtester_selftest
)
664 return main_selftest (argc
, argv
);
674 log_fd
= g_open (output_filename
, O_WRONLY
| O_CREAT
| O_TRUNC
, 0666);
676 g_error ("Failed to open log file '%s': %s", output_filename
, g_strerror (errno
));
679 test_log_printfe ("<?xml version=\"1.0\"?>\n");
680 test_log_printfe ("%s<gtester>\n", sindent (log_indent
));
682 for (ui
= 1; ui
< argc
; ui
++)
684 const char *binary
= argv
[ui
];
685 launch_test (binary
);
686 /* we only get here on success or if !subtest_mode_fatal */
689 test_log_printfe ("%s</gtester>\n", sindent (log_indent
));
693 return testcase_fail_count
== 0 ? EXIT_SUCCESS
: EXIT_FAILURE
;
697 fixture_setup (guint
*fix
,
698 gconstpointer test_data
)
700 g_assert_cmphex (*fix
, ==, 0);
704 fixture_test (guint
*fix
,
705 gconstpointer test_data
)
707 g_assert_cmphex (*fix
, ==, 0xdeadbeef);
708 g_test_message ("This is a test message API test message.");
709 g_test_bug_base ("http://www.example.com/bugtracker/");
711 g_test_bug_base ("http://www.example.com/bugtracker?bugnum=%s;cmd=showbug");
715 fixture_teardown (guint
*fix
,
716 gconstpointer test_data
)
718 g_assert_cmphex (*fix
, ==, 0xdeadbeef);
722 main_selftest (int argc
,
725 /* gtester main() for --gtester-selftest invocations */
726 g_test_init (&argc
, &argv
, NULL
);
727 g_test_add ("/gtester/fixture-test", guint
, NULL
, fixture_setup
, fixture_test
, fixture_teardown
);