src/harness.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause
   3  *
   4  * Copyright (c) 2024 Alessio Chiapperini
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27 #include <sys/time.h>
  28
  29 #include <inttypes.h>
  30 #include <stdio.h>
  31 #include <stdlib.h>
  32 #include <string.h>
  33 #include <time.h>
  34
  35 #include "harness.h"
  36
  37 #define US_PER_SECOND 1000000
  38
  39 #define SEED_CONSTANT 0xA3
  40 #define UNI_32BIT_INV 2.3283064365386962890625e-10
  41 #define VNI_32BIT_INV 4.6566128730773925781250e-10 /* UNI_32BIT_INV * 2 */
  42
  43 struct harness_report {
  44         struct timeval total_time;
  45         struct timeval suite_time;
  46         int npass;
  47         int nfail;
  48         int nskip;
  49         int nerror;
  50 };
  51
  52 static void
  53 timeval_sub(struct timeval t1, struct timeval t2, struct timeval *res)
  54 {
  55         res->tv_usec = t2.tv_usec - t1.tv_usec;
  56         res->tv_sec  = t2.tv_sec - t1.tv_sec;
  57         if (res->tv_sec > 0 && res->tv_usec < 0) {
  58                 res->tv_usec += US_PER_SECOND;
  59                 res->tv_sec--;
  60         } else if (res->tv_sec < 0 && res->tv_usec > 0) {
  61                 res->tv_usec -= US_PER_SECOND;
  62                 res->tv_sec++;
  63         }
  64 }
  65
  66 static void
  67 timeval_add(struct timeval t1, struct timeval t2, struct timeval *res)
  68 {
  69         res->tv_usec = t2.tv_usec + t1.tv_usec;
  70         res->tv_sec  = t2.tv_sec + t1.tv_sec;
  71         if (res->tv_usec >= US_PER_SECOND) {
  72                 res->tv_usec -= US_PER_SECOND;
  73                 res->tv_sec++;
  74         } else if (res->tv_usec <= -US_PER_SECOND) {
  75                 res->tv_usec += US_PER_SECOND;
  76                 res->tv_sec--;
  77         }
  78 }
  79
  80 static uint32_t state[2] = {0, SEED_CONSTANT};
  81
  82 static void
  83 random_init(void)
  84 {
  85         state[0] = (uint32_t)time(0);
  86         state[1] = SEED_CONSTANT;
  87 }
  88
  89 static inline uint32_t
  90 rotl(const uint32_t x, int k)
  91 {
  92         return ((x << k) | (x >> (32 - k)));
  93 }
  94
  95 static float
  96 int_to_float(uint32_t rand)
  97 {
  98         union { uint32_t u32; float f; } u = { .u32 = rand >> 9 | 0x3f800000 };
  99         return (u.f - 1.0f);
 100 }
 101
 102 uint32_t
 103 random_next(void)
 104 {
 105         const uint32_t s0 = state[0];
 106         uint32_t s1 = state[1];
 107         const uint32_t result = rotl(s0 * 0x9E3779BB, 5) * 5;
 108
 109         s1 ^= s0;
 110         state[0] = rotl(s0, 26) ^ s1 ^ (s1 << 9); /* a, b */
 111         state[1] = rotl(s1, 13); /* c */
 112
 113         return (result);
 114 }
 115
 116 uint32_t
 117 random_bounded(uint32_t range)
 118 {
 119         uint32_t x = random_next();
 120         uint64_t m = (uint64_t)x * (uint64_t)range;
 121         uint32_t l = (uint32_t)m;
 122         if (l < range) {
 123                 uint32_t t = -range;
 124                 if (t >= range) {
 125                         t -= range;
 126                         if (t >= range) {
 127                                 t %= range;
 128                         }
 129                 }
 130                 while (l < t) {
 131                         x = random_next();
 132                         m = (uint64_t)x * (uint64_t)range;
 133                         l = (uint32_t)m;
 134                 }
 135         }
 136         return (m >> 32);
 137 }
 138
 139 float
 140 random_float(void)
 141 {
 142         return (int_to_float(random_next()));
 143 }
 144
 145 static struct harness_cfg *cfg = NULL;
 146 static struct harness_suite test_suites[MAX_SUITES] = {0};
 147
 148 int
 149 harness_init(struct harness_cfg *harness_cfg)
 150 {
 151         if (harness_cfg == NULL) {
 152                 (void)fprintf(stderr, "harness: harness_cfg cannot be NULL\n");
 153                 return (HARNESS_FAIL);
 154         }
 155
 156         cfg = harness_cfg;
 157         cfg->nsuites = 0;
 158         cfg->ntests = 0;
 159
 160         random_init();
 161         if (cfg->seed == 0) {
 162                 cfg->seed = random_next();
 163         }
 164
 165         return (HARNESS_PASS);
 166 }
 167
 168 int
 169 harness_add_test(char *test_suite, char *test_name, test_function *test,
 170     setup_function *setup, teardown_function *teardown, void *user_data)
 171 {
 172         int found = 0;
 173         int suite_idx = -1;
 174         int nsuites = cfg->nsuites;
 175         int ntests = 0;
 176
 177         if (nsuites >= MAX_SUITES) {
 178                 (void)fprintf(stderr, "harness: Unable to add more suites\n");
 179                 return (HARNESS_FAIL);
 180         }
 181
 182         for (suite_idx = 0; suite_idx < nsuites; suite_idx++) {
 183                 if (test_suites[suite_idx].suite_name == NULL) {
 184                         (void)fprintf(stderr, "harness: suite name is NULL\n");
 185                         return (HARNESS_FAIL);
 186                 }
 187
 188                 if (strcmp(test_suite,
 189                     test_suites[suite_idx].suite_name) == 0) {
 190                         found = 1;
 191                         break;
 192                 }
 193         }
 194
 195         if (found == 0) {
 196                 suite_idx = nsuites;
 197                 test_suites[nsuites].suite_name = test_suite;
 198         }
 199
 200         ntests = test_suites[suite_idx].ntests;
 201         if (ntests >= MAX_TESTS) {
 202                 (void)fprintf(stderr, "harness: Unable to add more tests "
 203                     "to suite %s\n", test_suites[nsuites].suite_name);
 204                 return (HARNESS_FAIL);
 205         }
 206         test_suites[suite_idx].tests[ntests].test_function = test;
 207         test_suites[suite_idx].tests[ntests].test_name = test_name;
 208         test_suites[suite_idx].tests[ntests].user_data = user_data;
 209         test_suites[suite_idx].tests[ntests].setup = setup;
 210         test_suites[suite_idx].tests[ntests].teardown = teardown;
 211         test_suites[suite_idx].ntests++;
 212         cfg->ntests++;
 213
 214         /* Increment the number of suites only if the suite is new */
 215         if (found == 0) {
 216                 cfg->nsuites++;
 217         }
 218
 219         return (HARNESS_PASS);
 220 }
 221
 222 int
 223 harness_add_suite(char *test_suite, struct harness_test *tests) {
 224         int res = HARNESS_PASS;
 225         for (struct harness_test *t = tests; t != NULL &&
 226             t->test_function != NULL; t++) {
 227                 res = harness_add_test(test_suite, t->test_name,
 228                     t->test_function, t->setup, t->teardown, t->user_data);
 229
 230                 if (res == 1) {
 231                         return (HARNESS_FAIL);
 232                 }
 233         }
 234
 235         return (res);
 236 }
 237
 238 void
 239 harness_list_tests(char *test_suite)
 240 {
 241         int s;
 242
 243         if (test_suite != NULL) {
 244                 for (s = 0; s < cfg->nsuites; s++) {
 245                         if (!strcmp(test_suites[s].suite_name, test_suite)) {
 246                                 break;
 247                         }
 248                 }
 249                 (void)fprintf(stdout, "%s\n", test_suite);
 250
 251                 for (int t = 0; t < test_suites[s].ntests; t++) {
 252                         (void)fprintf(stdout, "  %s\n",
 253                             test_suites[s].tests[t].test_name);
 254                 }
 255         } else {
 256                 for (s = 0; s < cfg->nsuites; s++) {
 257                         (void)fprintf(stdout, "%s\n",
 258                             test_suites[s].suite_name);
 259                         for (int t = 0; t < test_suites[s].ntests; t++) {
 260                                 (void)fprintf(stdout, "  %s\n",
 261                                     test_suites[s].tests[t].test_name);
 262                         }
 263                 }
 264         }
 265 }
 266
 267 static int
 268 harness_run_test(int s, int t, struct harness_report *report)
 269 {
 270         struct timeval test_time = {0};
 271         struct timeval start_time = {0};
 272         struct timeval end_time = {0};
 273
 274         int res = 0;
 275         char *verdict = 0;
 276
 277         (void)fprintf(stdout, "[ %-8s ] %s.%s\n", "RUN",
 278                 test_suites[s].suite_name,
 279                 test_suites[s].tests[t].test_name);
 280
 281         (void)memset(&test_time, 0, sizeof(struct timeval));
 282
 283         if (test_suites[s].tests[t].setup != NULL) {
 284                 res = test_suites[s].tests[t].setup(
 285                     test_suites[s].tests[t].user_data);
 286                 if (res != HARNESS_PASS) {
 287                         goto print_report;
 288                 }
 289         }
 290
 291         gettimeofday(&start_time, NULL);
 292         res = test_suites[s].tests[t].test_function(
 293                 test_suites[s].tests[t].user_data);
 294         gettimeofday(&end_time, NULL);
 295         timeval_sub(start_time, end_time, &test_time);
 296
 297 print_report:
 298         switch (res) {
 299         case HARNESS_PASS:
 300                 verdict = "PASS";
 301                 report->npass++;
 302                 break;
 303         case HARNESS_FAIL:
 304                 verdict = "FAIL";
 305                 report->nfail++;
 306                 break;
 307         case HARNESS_SKIP:
 308                 verdict = "SKIP";
 309                 report->nskip++;
 310                 break;
 311         default:
 312                 verdict = "ERROR";
 313                 report->nerror++;
 314                 break;
 315         }
 316
 317         (void)fprintf(stdout, "[ %8s ] %s.%s "
 318                 "(%ld ms)\n",
 319                 verdict,
 320                 test_suites[s].suite_name,
 321                 test_suites[s].tests[t].test_name,
 322                 (test_time.tv_usec / 1000) +
 323                 test_time.tv_sec * 1000);
 324         timeval_add(report->suite_time, test_time, &report->suite_time);
 325
 326         if (test_suites[s].tests[t].teardown != NULL) {
 327                 test_suites[s].tests[t].teardown(
 328                         test_suites[s].tests[t].user_data);
 329         }
 330
 331         return (res);
 332 }
 333
 334 int
 335 harness_run(void *user_data)
 336 {
 337         struct harness_report report = {0};
 338
 339         if (cfg->setup != NULL) {
 340                 cfg->setup(user_data);
 341                 (void)fprintf(stdout, "[----------] Test environment "
 342                     "set-up.\n");
 343         }
 344
 345         (void)fprintf(stdout, "[==========] Running %d test cases from %d "
 346             "test suites with seed 0x%08" PRIx32 ".\n", cfg->ntests,
 347             cfg->nsuites, cfg->seed);
 348
 349         for (int s = 0; s < cfg->nsuites; s++) {
 350                 (void)fprintf(stdout, "[----------] %d tests from %s.\n",
 351                     test_suites[s].ntests, test_suites[s].suite_name);
 352
 353                 (void)memset(&report.suite_time, 0, sizeof(struct timeval));
 354                 for (int t = 0; t < test_suites[s].ntests; t++) {
 355                         (void)harness_run_test(s, t, &report);
 356                 }
 357                 (void)fprintf(stdout, "[----------] %d tests from %s (%ld ms "
 358                     "total).\n",
 359                     test_suites[s].ntests, test_suites[s].suite_name,
 360                     (report.suite_time.tv_usec / 1000) +
 361                     report.suite_time.tv_sec * 1000);
 362                 timeval_add(report.total_time, report.suite_time,
 363                     &report.total_time);
 364         }
 365
 366         (void)fprintf(stdout, "[==========] %d test cases from %d test suites "
 367             "ran (%ld ms total).\n", cfg->ntests, cfg->nsuites,
 368             (report.total_time.tv_usec / 1000) +
 369             report.total_time.tv_sec * 1000);
 370         (void)fprintf(stdout, "[  PASSED  ] %d tests.\n", report.npass);
 371         (void)fprintf(stdout, "[  FAILED  ] %d tests.\n", report.nfail);
 372         (void)fprintf(stdout, "[  SKIPPED ] %d tests.\n", report.nskip);
 373         (void)fprintf(stdout, "[  ERRORS  ] %d tests.\n", report.nerror);
 374
 375         if (cfg->teardown != NULL) {
 376                 cfg->teardown(user_data);
 377                 (void)fprintf(stdout, "[----------] Test environment "
 378                     "teardown.\n");
 379         }
 380
 381         if (report.nfail != 0) {
 382                 return (EXIT_FAILURE);
 383         }
 384
 385         return (EXIT_SUCCESS);
 386 }
 387
 388 void
 389 harness_destroy(struct harness_cfg *harness_cfg)
 390 {
 391         for (int i = 0; i < cfg->nsuites; i++) {
 392                 (void)memset(test_suites[i].tests, 0,
 393                     sizeof(struct harness_test));
 394         }
 395         (void)memset(test_suites, 0, sizeof(test_suites));
 396
 397         cfg = NULL;
 398         *harness_cfg = (struct harness_cfg) {0};
 399 }