src/ext/tinytest.c

   1 /* tinytest.c -- Copyright 2009-2012 Nick Mathewson
   2  *
   3  * Redistribution and use in source and binary forms, with or without
   4  * modification, are permitted provided that the following conditions
   5  * are met:
   6  * 1. Redistributions of source code must retain the above copyright
   7  *    notice, this list of conditions and the following disclaimer.
   8  * 2. Redistributions in binary form must reproduce the above copyright
   9  *    notice, this list of conditions and the following disclaimer in the
  10  *    documentation and/or other materials provided with the distribution.
  11  * 3. The name of the author may not be used to endorse or promote products
  12  *    derived from this software without specific prior written permission.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  15  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  16  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  17  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  18  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  19  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  20  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  21  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  23  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  24  */
  25 #ifdef TINYTEST_LOCAL
  26 #include "tinytest_local.h"
  27 #endif
  28 #define TINYTEST_POSTFORK
  29
  30 #include <stdio.h>
  31 #include <stdlib.h>
  32 #include <string.h>
  33 #include <assert.h>
  34
  35 #ifndef NO_FORKING
  36
  37 #ifdef _WIN32
  38 #include <windows.h>
  39 #else
  40 #include <sys/types.h>
  41 #include <sys/wait.h>
  42 #include <unistd.h>
  43 #endif
  44
  45 #if defined(__APPLE__) && defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__)
  46 #if (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1060 && \
  47     __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1070)
  48 /* Workaround for a stupid bug in OSX 10.6 */
  49 #define FORK_BREAKS_GCOV
  50 #include <vproc.h>
  51 #endif
  52 #endif
  53
  54 #endif /* !NO_FORKING */
  55
  56 #ifndef __GNUC__
  57 #define __attribute__(x)
  58 #endif
  59
  60 #include "tinytest.h"
  61 #include "tinytest_macros.h"
  62
  63 #define LONGEST_TEST_NAME 16384
  64
  65 static int in_tinytest_main = 0; /**< true if we're in tinytest_main().*/
  66 static int n_ok = 0; /**< Number of tests that have passed */
  67 static int n_bad = 0; /**< Number of tests that have failed. */
  68 static int n_skipped = 0; /**< Number of tests that have been skipped. */
  69
  70 static int opt_forked = 0; /**< True iff we're called from inside a win32 fork*/
  71 static int opt_nofork = 0; /**< Suppress calls to fork() for debugging. */
  72 static int opt_verbosity = 1; /**< -==quiet,0==terse,1==normal,2==verbose */
  73 static const char *verbosity_flag = "";
  74
  75 static const struct testlist_alias_t *cfg_aliases=NULL;
  76
  77 enum outcome { SKIP=2, OK=1, FAIL=0 };
  78 static enum outcome cur_test_outcome = 0;
  79 /** prefix of the current test group */
  80 static const char *cur_test_prefix = NULL;
  81 /** Name of the current test, if we haven't logged is yet. Used for --quiet */
  82 static const char *cur_test_name = NULL;
  83
  84 #ifdef _WIN32
  85 /* Copy of argv[0] for win32. */
  86 static char commandname[MAX_PATH+1];
  87 #endif
  88
  89 static void usage(struct testgroup_t *groups, int list_groups)
  90   __attribute__((noreturn));
  91 static int process_test_option(struct testgroup_t *groups, const char *test);
  92
  93 static enum outcome
  94 testcase_run_bare_(const struct testcase_t *testcase)
  95 {
  96         void *env = NULL;
  97         int outcome;
  98         if (testcase->setup) {
  99                 env = testcase->setup->setup_fn(testcase);
 100                 if (!env)
 101                         return FAIL;
 102                 else if (env == (void*)TT_SKIP)
 103                         return SKIP;
 104         }
 105
 106         cur_test_outcome = OK;
 107         testcase->fn(env);
 108         outcome = cur_test_outcome;
 109
 110         if (testcase->setup) {
 111                 if (testcase->setup->cleanup_fn(testcase, env) == 0)
 112                         outcome = FAIL;
 113         }
 114
 115         return outcome;
 116 }
 117
 118 #define MAGIC_EXITCODE 42
 119
 120 #ifndef NO_FORKING
 121
 122 #ifdef TINYTEST_POSTFORK
 123 void tinytest_prefork(void);
 124 void tinytest_postfork(void);
 125 #else
 126 static void tinytest_prefork(void) { }
 127 static void tinytest_postfork(void) { }
 128 #endif
 129
 130 static enum outcome
 131 testcase_run_forked_(const struct testgroup_t *group,
 132                      const struct testcase_t *testcase)
 133 {
 134 #ifdef _WIN32
 135         /* Fork? On Win32?  How primitive!  We'll do what the smart kids do:
 136            we'll invoke our own exe (whose name we recall from the command
 137            line) with a command line that tells it to run just the test we
 138            want, and this time without forking.
 139
 140            (No, threads aren't an option.  The whole point of forking is to
 141            share no state between tests.)
 142          */
 143         int ok;
 144         char buffer[LONGEST_TEST_NAME+256];
 145         STARTUPINFOA si;
 146         PROCESS_INFORMATION info;
 147         DWORD exitcode;
 148
 149         if (!in_tinytest_main) {
 150                 printf("\nERROR.  On Windows, testcase_run_forked_ must be"
 151                        " called from within tinytest_main.\n");
 152                 abort();
 153         }
 154         if (opt_verbosity>0)
 155                 printf("[forking] ");
 156
 157         snprintf(buffer, sizeof(buffer), "\"%s\" --RUNNING-FORKED %s %s%s",
 158                  commandname, verbosity_flag, group->prefix, testcase->name);
 159
 160         memset(&si, 0, sizeof(si));
 161         memset(&info, 0, sizeof(info));
 162         si.cb = sizeof(si);
 163
 164         ok = CreateProcessA(commandname, buffer, NULL, NULL, 0,
 165                            0, NULL, NULL, &si, &info);
 166         if (!ok) {
 167                 printf("CreateProcess failed!\n");
 168                 return 0;
 169         }
 170         WaitForSingleObject(info.hProcess, INFINITE);
 171         GetExitCodeProcess(info.hProcess, &exitcode);
 172         CloseHandle(info.hProcess);
 173         CloseHandle(info.hThread);
 174         if (exitcode == 0)
 175                 return OK;
 176         else if (exitcode == MAGIC_EXITCODE)
 177                 return SKIP;
 178         else
 179                 return FAIL;
 180 #else
 181         int outcome_pipe[2];
 182         pid_t pid;
 183         (void)group;
 184
 185         if (pipe(outcome_pipe))
 186                 perror("opening pipe");
 187
 188         if (opt_verbosity>0)
 189                 printf("[forking] ");
 190         tinytest_prefork();
 191         pid = fork();
 192 #ifdef FORK_BREAKS_GCOV
 193         vproc_transaction_begin(0);
 194 #endif
 195         tinytest_postfork();
 196         if (!pid) {
 197                 /* child. */
 198                 int test_r, write_r;
 199                 char b[1];
 200                 close(outcome_pipe[0]);
 201                 test_r = testcase_run_bare_(testcase);
 202                 assert(0<=(int)test_r && (int)test_r<=2);
 203                 b[0] = "NYS"[test_r];
 204                 write_r = (int)write(outcome_pipe[1], b, 1);
 205                 if (write_r != 1) {
 206                         perror("write outcome to pipe");
 207                         exit(1);
 208                 }
 209                 exit(0);
 210                 return FAIL; /* unreachable */
 211         } else {
 212                 /* parent */
 213                 int status, r;
 214                 char b[1];
 215                 /* Close this now, so that if the other side closes it,
 216                  * our read fails. */
 217                 close(outcome_pipe[1]);
 218                 r = (int)read(outcome_pipe[0], b, 1);
 219                 if (r == 0) {
 220                         printf("[Lost connection!] ");
 221                         return FAIL;
 222                 } else if (r != 1) {
 223                         perror("read outcome from pipe");
 224                 }
 225                 r = waitpid(pid, &status, 0);
 226                 close(outcome_pipe[0]);
 227                 if (r == -1) {
 228                         perror("waitpid");
 229                         return FAIL;
 230                 }
 231                 if (! WIFEXITED(status) || WEXITSTATUS(status) != 0) {
 232                         printf("[did not exit cleanly.]");
 233                         return FAIL;
 234                 }
 235                 return b[0]=='Y' ? OK : (b[0]=='S' ? SKIP : FAIL);
 236         }
 237 #endif
 238 }
 239
 240 #endif /* !NO_FORKING */
 241
 242 int
 243 testcase_run_one(const struct testgroup_t *group,
 244                  const struct testcase_t *testcase)
 245 {
 246         enum outcome outcome;
 247
 248         if (testcase->flags & (TT_SKIP|TT_OFF_BY_DEFAULT)) {
 249                 if (opt_verbosity>0)
 250                         printf("%s%s: %s\n",
 251                            group->prefix, testcase->name,
 252                            (testcase->flags & TT_SKIP) ? "SKIPPED" : "DISABLED");
 253                 ++n_skipped;
 254                 return SKIP;
 255         }
 256
 257         if (opt_verbosity>0 && !opt_forked) {
 258                 printf("%s%s: ", group->prefix, testcase->name);
 259         } else {
 260                 if (opt_verbosity==0) printf(".");
 261                 cur_test_prefix = group->prefix;
 262                 cur_test_name = testcase->name;
 263         }
 264
 265 #ifndef NO_FORKING
 266         if ((testcase->flags & TT_FORK) && !(opt_forked||opt_nofork)) {
 267                 outcome = testcase_run_forked_(group, testcase);
 268         } else {
 269 #else
 270         {
 271 #endif
 272                 outcome = testcase_run_bare_(testcase);
 273         }
 274
 275         if (outcome == OK) {
 276                 ++n_ok;
 277                 if (opt_verbosity>0 && !opt_forked)
 278                         puts(opt_verbosity==1?"OK":"");
 279         } else if (outcome == SKIP) {
 280                 ++n_skipped;
 281                 if (opt_verbosity>0 && !opt_forked)
 282                         puts("SKIPPED");
 283         } else {
 284                 ++n_bad;
 285                 if (!opt_forked)
 286                         printf("\n  [%s FAILED]\n", testcase->name);
 287         }
 288
 289         if (opt_forked) {
 290                 exit(outcome==OK ? 0 : (outcome==SKIP?MAGIC_EXITCODE : 1));
 291                 return 1; /* unreachable */
 292         } else {
 293                 return (int)outcome;
 294         }
 295 }
 296
 297 int
 298 tinytest_set_flag_(struct testgroup_t *groups, const char *arg, int set, unsigned long flag)
 299 {
 300         int i, j;
 301         size_t length = LONGEST_TEST_NAME;
 302         char fullname[LONGEST_TEST_NAME];
 303         int found=0;
 304         if (strstr(arg, ".."))
 305                 length = strstr(arg,"..")-arg;
 306         for (i=0; groups[i].prefix; ++i) {
 307                 for (j=0; groups[i].cases[j].name; ++j) {
 308                         struct testcase_t *testcase = &groups[i].cases[j];
 309                         snprintf(fullname, sizeof(fullname), "%s%s",
 310                                  groups[i].prefix, testcase->name);
 311                         if (!flag) { /* Hack! */
 312                                 printf("    %s", fullname);
 313                                 if (testcase->flags & TT_OFF_BY_DEFAULT)
 314                                         puts("   (Off by default)");
 315                                 else if (testcase->flags & TT_SKIP)
 316                                         puts("  (DISABLED)");
 317                                 else
 318                                         puts("");
 319                         }
 320                         if (!strncmp(fullname, arg, length)) {
 321                                 if (set)
 322                                         testcase->flags |= flag;
 323                                 else
 324                                         testcase->flags &= ~flag;
 325                                 ++found;
 326                         }
 327                 }
 328         }
 329         return found;
 330 }
 331
 332 static void
 333 usage(struct testgroup_t *groups, int list_groups)
 334 {
 335         puts("Options are: [--verbose|--quiet|--terse] [--no-fork]");
 336         puts("  Specify tests by name, or using a prefix ending with '..'");
 337         puts("  To skip a test, prefix its name with a colon.");
 338         puts("  To enable a disabled test, prefix its name with a plus.");
 339         puts("  Use --list-tests for a list of tests.");
 340         if (list_groups) {
 341                 puts("Known tests are:");
 342                 tinytest_set_flag_(groups, "..", 1, 0);
 343         }
 344         exit(0);
 345 }
 346
 347 static int
 348 process_test_alias(struct testgroup_t *groups, const char *test)
 349 {
 350         int i, j, n, r;
 351         for (i=0; cfg_aliases && cfg_aliases[i].name; ++i) {
 352                 if (!strcmp(cfg_aliases[i].name, test)) {
 353                         n = 0;
 354                         for (j = 0; cfg_aliases[i].tests[j]; ++j) {
 355                                 r = process_test_option(groups, cfg_aliases[i].tests[j]);
 356                                 if (r<0)
 357                                         return -1;
 358                                 n += r;
 359                         }
 360                         return n;
 361                 }
 362         }
 363         printf("No such test alias as @%s!",test);
 364         return -1;
 365 }
 366
 367 static int
 368 process_test_option(struct testgroup_t *groups, const char *test)
 369 {
 370         int flag = TT_ENABLED_;
 371         int n = 0;
 372         if (test[0] == '@') {
 373                 return process_test_alias(groups, test + 1);
 374         } else if (test[0] == ':') {
 375                 ++test;
 376                 flag = TT_SKIP;
 377         } else if (test[0] == '+') {
 378                 ++test;
 379                 ++n;
 380                 if (!tinytest_set_flag_(groups, test, 0, TT_OFF_BY_DEFAULT)) {
 381                         printf("No such test as %s!\n", test);
 382                         return -1;
 383                 }
 384         } else {
 385                 ++n;
 386         }
 387         if (!tinytest_set_flag_(groups, test, 1, flag)) {
 388                 printf("No such test as %s!\n", test);
 389                 return -1;
 390         }
 391         return n;
 392 }
 393
 394 void
 395 tinytest_set_aliases(const struct testlist_alias_t *aliases)
 396 {
 397         cfg_aliases = aliases;
 398 }
 399
 400 int
 401 tinytest_main(int c, const char **v, struct testgroup_t *groups)
 402 {
 403         int i, j, n=0;
 404
 405 #ifdef _WIN32
 406         const char *sp = strrchr(v[0], '.');
 407         const char *extension = "";
 408         if (!sp || stricmp(sp, ".exe"))
 409                 extension = ".exe"; /* Add an exe so CreateProcess will work */
 410         snprintf(commandname, sizeof(commandname), "%s%s", v[0], extension);
 411         commandname[MAX_PATH]='\0';
 412 #endif
 413         for (i=1; i<c; ++i) {
 414                 if (v[i][0] == '-') {
 415                         if (!strcmp(v[i], "--RUNNING-FORKED")) {
 416                                 opt_forked = 1;
 417                         } else if (!strcmp(v[i], "--no-fork")) {
 418                                 opt_nofork = 1;
 419                         } else if (!strcmp(v[i], "--quiet")) {
 420                                 opt_verbosity = -1;
 421                                 verbosity_flag = "--quiet";
 422                         } else if (!strcmp(v[i], "--verbose")) {
 423                                 opt_verbosity = 2;
 424                                 verbosity_flag = "--verbose";
 425                         } else if (!strcmp(v[i], "--terse")) {
 426                                 opt_verbosity = 0;
 427                                 verbosity_flag = "--terse";
 428                         } else if (!strcmp(v[i], "--help")) {
 429                                 usage(groups, 0);
 430                         } else if (!strcmp(v[i], "--list-tests")) {
 431                                 usage(groups, 1);
 432                         } else {
 433                                 printf("Unknown option %s.  Try --help\n",v[i]);
 434                                 return -1;
 435                         }
 436                 } else {
 437                         int r = process_test_option(groups, v[i]);
 438                         if (r<0)
 439                                 return -1;
 440                         n += r;
 441                 }
 442         }
 443         if (!n)
 444                 tinytest_set_flag_(groups, "..", 1, TT_ENABLED_);
 445
 446 #ifdef _IONBF
 447         setvbuf(stdout, NULL, _IONBF, 0);
 448 #endif
 449
 450         ++in_tinytest_main;
 451         for (i=0; groups[i].prefix; ++i)
 452                 for (j=0; groups[i].cases[j].name; ++j)
 453                         if (groups[i].cases[j].flags & TT_ENABLED_)
 454                                 testcase_run_one(&groups[i],
 455                                                  &groups[i].cases[j]);
 456
 457         --in_tinytest_main;
 458
 459         if (opt_verbosity==0)
 460                 puts("");
 461
 462         if (n_bad)
 463                 printf("%d/%d TESTS FAILED. (%d skipped)\n", n_bad,
 464                        n_bad+n_ok,n_skipped);
 465         else if (opt_verbosity >= 1)
 466                 printf("%d tests ok.  (%d skipped)\n", n_ok, n_skipped);
 467
 468         return (n_bad == 0) ? 0 : 1;
 469 }
 470
 471 int
 472 tinytest_get_verbosity_(void)
 473 {
 474         return opt_verbosity;
 475 }
 476
 477 void
 478 tinytest_set_test_failed_(void)
 479 {
 480         if (opt_verbosity <= 0 && cur_test_name) {
 481                 if (opt_verbosity==0) puts("");
 482                 printf("%s%s: ", cur_test_prefix, cur_test_name);
 483                 cur_test_name = NULL;
 484         }
 485         cur_test_outcome = 0;
 486 }
 487
 488 void
 489 tinytest_set_test_skipped_(void)
 490 {
 491         if (cur_test_outcome==OK)
 492                 cur_test_outcome = SKIP;
 493 }
 494
 495 int
 496 tinytest_cur_test_has_failed(void)
 497 {
 498         return (cur_test_outcome == FAIL);
 499 }
 500
 501 char *
 502 tinytest_format_hex_(const void *val_, unsigned long len)
 503 {
 504         const unsigned char *val = val_;
 505         char *result, *cp;
 506         size_t i;
 507         int ellipses = 0;
 508
 509         if (!val)
 510                 return strdup("null");
 511         if (len > 1024) {
 512                 ellipses = 3;
 513                 len = 1024;
 514         }
 515         if (!(result = malloc(len*2+4)))
 516                 return strdup("<allocation failure>");
 517         cp = result;
 518         for (i=0;i<len;++i) {
 519                 *cp++ = "0123456789ABCDEF"[(val[i] >> 4)&0x0f];
 520                 *cp++ = "0123456789ABCDEF"[val[i] & 0x0f];
 521         }
 522         while (ellipses--)
 523                 *cp++ = '.';
 524         *cp = 0;
 525         return result;
 526 }