tools/testing/selftests/bpf/test_loader.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
   3 #include <linux/capability.h>
   4 #include <stdlib.h>
   5 #include <regex.h>
   6 #include <test_progs.h>
   7 #include <bpf/btf.h>
   8
   9 #include "autoconf_helper.h"
  10 #include "disasm_helpers.h"
  11 #include "unpriv_helpers.h"
  12 #include "cap_helpers.h"
  13 #include "jit_disasm_helpers.h"
  14
  15 #define str_has_pfx(str, pfx) \
  16         (strncmp(str, pfx, __builtin_constant_p(pfx) ? sizeof(pfx) - 1 : strlen(pfx)) == 0)
  17
  18 #define TEST_LOADER_LOG_BUF_SZ 2097152
  19
  20 #define TEST_TAG_EXPECT_FAILURE "comment:test_expect_failure"
  21 #define TEST_TAG_EXPECT_SUCCESS "comment:test_expect_success"
  22 #define TEST_TAG_EXPECT_MSG_PFX "comment:test_expect_msg="
  23 #define TEST_TAG_EXPECT_XLATED_PFX "comment:test_expect_xlated="
  24 #define TEST_TAG_EXPECT_FAILURE_UNPRIV "comment:test_expect_failure_unpriv"
  25 #define TEST_TAG_EXPECT_SUCCESS_UNPRIV "comment:test_expect_success_unpriv"
  26 #define TEST_TAG_EXPECT_MSG_PFX_UNPRIV "comment:test_expect_msg_unpriv="
  27 #define TEST_TAG_EXPECT_XLATED_PFX_UNPRIV "comment:test_expect_xlated_unpriv="
  28 #define TEST_TAG_LOG_LEVEL_PFX "comment:test_log_level="
  29 #define TEST_TAG_PROG_FLAGS_PFX "comment:test_prog_flags="
  30 #define TEST_TAG_DESCRIPTION_PFX "comment:test_description="
  31 #define TEST_TAG_RETVAL_PFX "comment:test_retval="
  32 #define TEST_TAG_RETVAL_PFX_UNPRIV "comment:test_retval_unpriv="
  33 #define TEST_TAG_AUXILIARY "comment:test_auxiliary"
  34 #define TEST_TAG_AUXILIARY_UNPRIV "comment:test_auxiliary_unpriv"
  35 #define TEST_BTF_PATH "comment:test_btf_path="
  36 #define TEST_TAG_ARCH "comment:test_arch="
  37 #define TEST_TAG_JITED_PFX "comment:test_jited="
  38 #define TEST_TAG_JITED_PFX_UNPRIV "comment:test_jited_unpriv="
  39 #define TEST_TAG_CAPS_UNPRIV "comment:test_caps_unpriv="
  40
  41 /* Warning: duplicated in bpf_misc.h */
  42 #define POINTER_VALUE   0xcafe4all
  43 #define TEST_DATA_LEN   64
  44
  45 #ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
  46 #define EFFICIENT_UNALIGNED_ACCESS 1
  47 #else
  48 #define EFFICIENT_UNALIGNED_ACCESS 0
  49 #endif
  50
  51 static int sysctl_unpriv_disabled = -1;
  52
  53 enum mode {
  54         PRIV = 1,
  55         UNPRIV = 2
  56 };
  57
  58 struct expect_msg {
  59         const char *substr; /* substring match */
  60         regex_t regex;
  61         bool is_regex;
  62         bool on_next_line;
  63 };
  64
  65 struct expected_msgs {
  66         struct expect_msg *patterns;
  67         size_t cnt;
  68 };
  69
  70 struct test_subspec {
  71         char *name;
  72         bool expect_failure;
  73         struct expected_msgs expect_msgs;
  74         struct expected_msgs expect_xlated;
  75         struct expected_msgs jited;
  76         int retval;
  77         bool execute;
  78         __u64 caps;
  79 };
  80
  81 struct test_spec {
  82         const char *prog_name;
  83         struct test_subspec priv;
  84         struct test_subspec unpriv;
  85         const char *btf_custom_path;
  86         int log_level;
  87         int prog_flags;
  88         int mode_mask;
  89         int arch_mask;
  90         bool auxiliary;
  91         bool valid;
  92 };
  93
  94 static int tester_init(struct test_loader *tester)
  95 {
  96         if (!tester->log_buf) {
  97                 tester->log_buf_sz = TEST_LOADER_LOG_BUF_SZ;
  98                 tester->log_buf = calloc(tester->log_buf_sz, 1);
  99                 if (!ASSERT_OK_PTR(tester->log_buf, "tester_log_buf"))
 100                         return -ENOMEM;
 101         }
 102
 103         return 0;
 104 }
 105
 106 void test_loader_fini(struct test_loader *tester)
 107 {
 108         if (!tester)
 109                 return;
 110
 111         free(tester->log_buf);
 112 }
 113
 114 static void free_msgs(struct expected_msgs *msgs)
 115 {
 116         int i;
 117
 118         for (i = 0; i < msgs->cnt; i++)
 119                 if (msgs->patterns[i].is_regex)
 120                         regfree(&msgs->patterns[i].regex);
 121         free(msgs->patterns);
 122         msgs->patterns = NULL;
 123         msgs->cnt = 0;
 124 }
 125
 126 static void free_test_spec(struct test_spec *spec)
 127 {
 128         /* Deallocate expect_msgs arrays. */
 129         free_msgs(&spec->priv.expect_msgs);
 130         free_msgs(&spec->unpriv.expect_msgs);
 131         free_msgs(&spec->priv.expect_xlated);
 132         free_msgs(&spec->unpriv.expect_xlated);
 133         free_msgs(&spec->priv.jited);
 134         free_msgs(&spec->unpriv.jited);
 135
 136         free(spec->priv.name);
 137         free(spec->unpriv.name);
 138         spec->priv.name = NULL;
 139         spec->unpriv.name = NULL;
 140 }
 141
 142 /* Compiles regular expression matching pattern.
 143  * Pattern has a special syntax:
 144  *
 145  *   pattern := (<verbatim text> | regex)*
 146  *   regex := "{{" <posix extended regular expression> "}}"
 147  *
 148  * In other words, pattern is a verbatim text with inclusion
 149  * of regular expressions enclosed in "{{" "}}" pairs.
 150  * For example, pattern "foo{{[0-9]+}}" matches strings like
 151  * "foo0", "foo007", etc.
 152  */
 153 static int compile_regex(const char *pattern, regex_t *regex)
 154 {
 155         char err_buf[256], buf[256] = {}, *ptr, *buf_end;
 156         const char *original_pattern = pattern;
 157         bool in_regex = false;
 158         int err;
 159
 160         buf_end = buf + sizeof(buf);
 161         ptr = buf;
 162         while (*pattern && ptr < buf_end - 2) {
 163                 if (!in_regex && str_has_pfx(pattern, "{{")) {
 164                         in_regex = true;
 165                         pattern += 2;
 166                         continue;
 167                 }
 168                 if (in_regex && str_has_pfx(pattern, "}}")) {
 169                         in_regex = false;
 170                         pattern += 2;
 171                         continue;
 172                 }
 173                 if (in_regex) {
 174                         *ptr++ = *pattern++;
 175                         continue;
 176                 }
 177                 /* list of characters that need escaping for extended posix regex */
 178                 if (strchr(".[]\\()*+?{}|^$", *pattern)) {
 179                         *ptr++ = '\\';
 180                         *ptr++ = *pattern++;
 181                         continue;
 182                 }
 183                 *ptr++ = *pattern++;
 184         }
 185         if (*pattern) {
 186                 PRINT_FAIL("Regexp too long: '%s'\n", original_pattern);
 187                 return -EINVAL;
 188         }
 189         if (in_regex) {
 190                 PRINT_FAIL("Regexp has open '{{' but no closing '}}': '%s'\n", original_pattern);
 191                 return -EINVAL;
 192         }
 193         err = regcomp(regex, buf, REG_EXTENDED | REG_NEWLINE);
 194         if (err != 0) {
 195                 regerror(err, regex, err_buf, sizeof(err_buf));
 196                 PRINT_FAIL("Regexp compilation error in '%s': '%s'\n", buf, err_buf);
 197                 return -EINVAL;
 198         }
 199         return 0;
 200 }
 201
 202 static int __push_msg(const char *pattern, bool on_next_line, struct expected_msgs *msgs)
 203 {
 204         struct expect_msg *msg;
 205         void *tmp;
 206         int err;
 207
 208         tmp = realloc(msgs->patterns,
 209                       (1 + msgs->cnt) * sizeof(struct expect_msg));
 210         if (!tmp) {
 211                 ASSERT_FAIL("failed to realloc memory for messages\n");
 212                 return -ENOMEM;
 213         }
 214         msgs->patterns = tmp;
 215         msg = &msgs->patterns[msgs->cnt];
 216         msg->on_next_line = on_next_line;
 217         msg->substr = pattern;
 218         msg->is_regex = false;
 219         if (strstr(pattern, "{{")) {
 220                 err = compile_regex(pattern, &msg->regex);
 221                 if (err)
 222                         return err;
 223                 msg->is_regex = true;
 224         }
 225         msgs->cnt += 1;
 226         return 0;
 227 }
 228
 229 static int clone_msgs(struct expected_msgs *from, struct expected_msgs *to)
 230 {
 231         struct expect_msg *msg;
 232         int i, err;
 233
 234         for (i = 0; i < from->cnt; i++) {
 235                 msg = &from->patterns[i];
 236                 err = __push_msg(msg->substr, msg->on_next_line, to);
 237                 if (err)
 238                         return err;
 239         }
 240         return 0;
 241 }
 242
 243 static int push_msg(const char *substr, struct expected_msgs *msgs)
 244 {
 245         return __push_msg(substr, false, msgs);
 246 }
 247
 248 static int push_disasm_msg(const char *regex_str, bool *on_next_line, struct expected_msgs *msgs)
 249 {
 250         int err;
 251
 252         if (strcmp(regex_str, "...") == 0) {
 253                 *on_next_line = false;
 254                 return 0;
 255         }
 256         err = __push_msg(regex_str, *on_next_line, msgs);
 257         if (err)
 258                 return err;
 259         *on_next_line = true;
 260         return 0;
 261 }
 262
 263 static int parse_int(const char *str, int *val, const char *name)
 264 {
 265         char *end;
 266         long tmp;
 267
 268         errno = 0;
 269         if (str_has_pfx(str, "0x"))
 270                 tmp = strtol(str + 2, &end, 16);
 271         else
 272                 tmp = strtol(str, &end, 10);
 273         if (errno || end[0] != '\0') {
 274                 PRINT_FAIL("failed to parse %s from '%s'\n", name, str);
 275                 return -EINVAL;
 276         }
 277         *val = tmp;
 278         return 0;
 279 }
 280
 281 static int parse_caps(const char *str, __u64 *val, const char *name)
 282 {
 283         int cap_flag = 0;
 284         char *token = NULL, *saveptr = NULL;
 285
 286         char *str_cpy = strdup(str);
 287         if (str_cpy == NULL) {
 288                 PRINT_FAIL("Memory allocation failed\n");
 289                 return -EINVAL;
 290         }
 291
 292         token = strtok_r(str_cpy, "|", &saveptr);
 293         while (token != NULL) {
 294                 errno = 0;
 295                 if (!strncmp("CAP_", token, sizeof("CAP_") - 1)) {
 296                         PRINT_FAIL("define %s constant in bpf_misc.h, failed to parse caps\n", token);
 297                         return -EINVAL;
 298                 }
 299                 cap_flag = strtol(token, NULL, 10);
 300                 if (!cap_flag || errno) {
 301                         PRINT_FAIL("failed to parse caps %s\n", name);
 302                         return -EINVAL;
 303                 }
 304                 *val |= (1ULL << cap_flag);
 305                 token = strtok_r(NULL, "|", &saveptr);
 306         }
 307
 308         free(str_cpy);
 309         return 0;
 310 }
 311
 312 static int parse_retval(const char *str, int *val, const char *name)
 313 {
 314         struct {
 315                 char *name;
 316                 int val;
 317         } named_values[] = {
 318                 { "INT_MIN"      , INT_MIN },
 319                 { "POINTER_VALUE", POINTER_VALUE },
 320                 { "TEST_DATA_LEN", TEST_DATA_LEN },
 321         };
 322         int i;
 323
 324         for (i = 0; i < ARRAY_SIZE(named_values); ++i) {
 325                 if (strcmp(str, named_values[i].name) != 0)
 326                         continue;
 327                 *val = named_values[i].val;
 328                 return 0;
 329         }
 330
 331         return parse_int(str, val, name);
 332 }
 333
 334 static void update_flags(int *flags, int flag, bool clear)
 335 {
 336         if (clear)
 337                 *flags &= ~flag;
 338         else
 339                 *flags |= flag;
 340 }
 341
 342 /* Matches a string of form '<pfx>[^=]=.*' and returns it's suffix.
 343  * Used to parse btf_decl_tag values.
 344  * Such values require unique prefix because compiler does not add
 345  * same __attribute__((btf_decl_tag(...))) twice.
 346  * Test suite uses two-component tags for such cases:
 347  *
 348  *   <pfx> __COUNTER__ '='
 349  *
 350  * For example, two consecutive __msg tags '__msg("foo") __msg("foo")'
 351  * would be encoded as:
 352  *
 353  *   [18] DECL_TAG 'comment:test_expect_msg=0=foo' type_id=15 component_idx=-1
 354  *   [19] DECL_TAG 'comment:test_expect_msg=1=foo' type_id=15 component_idx=-1
 355  *
 356  * And the purpose of this function is to extract 'foo' from the above.
 357  */
 358 static const char *skip_dynamic_pfx(const char *s, const char *pfx)
 359 {
 360         const char *msg;
 361
 362         if (strncmp(s, pfx, strlen(pfx)) != 0)
 363                 return NULL;
 364         msg = s + strlen(pfx);
 365         msg = strchr(msg, '=');
 366         if (!msg)
 367                 return NULL;
 368         return msg + 1;
 369 }
 370
 371 enum arch {
 372         ARCH_UNKNOWN    = 0x1,
 373         ARCH_X86_64     = 0x2,
 374         ARCH_ARM64      = 0x4,
 375         ARCH_RISCV64    = 0x8,
 376 };
 377
 378 static int get_current_arch(void)
 379 {
 380 #if defined(__x86_64__)
 381         return ARCH_X86_64;
 382 #elif defined(__aarch64__)
 383         return ARCH_ARM64;
 384 #elif defined(__riscv) && __riscv_xlen == 64
 385         return ARCH_RISCV64;
 386 #endif
 387         return ARCH_UNKNOWN;
 388 }
 389
 390 /* Uses btf_decl_tag attributes to describe the expected test
 391  * behavior, see bpf_misc.h for detailed description of each attribute
 392  * and attribute combinations.
 393  */
 394 static int parse_test_spec(struct test_loader *tester,
 395                            struct bpf_object *obj,
 396                            struct bpf_program *prog,
 397                            struct test_spec *spec)
 398 {
 399         const char *description = NULL;
 400         bool has_unpriv_result = false;
 401         bool has_unpriv_retval = false;
 402         bool unpriv_xlated_on_next_line = true;
 403         bool xlated_on_next_line = true;
 404         bool unpriv_jit_on_next_line;
 405         bool jit_on_next_line;
 406         bool collect_jit = false;
 407         int func_id, i, err = 0;
 408         u32 arch_mask = 0;
 409         struct btf *btf;
 410         enum arch arch;
 411
 412         memset(spec, 0, sizeof(*spec));
 413
 414         spec->prog_name = bpf_program__name(prog);
 415         spec->prog_flags = testing_prog_flags();
 416
 417         btf = bpf_object__btf(obj);
 418         if (!btf) {
 419                 ASSERT_FAIL("BPF object has no BTF");
 420                 return -EINVAL;
 421         }
 422
 423         func_id = btf__find_by_name_kind(btf, spec->prog_name, BTF_KIND_FUNC);
 424         if (func_id < 0) {
 425                 ASSERT_FAIL("failed to find FUNC BTF type for '%s'", spec->prog_name);
 426                 return -EINVAL;
 427         }
 428
 429         for (i = 1; i < btf__type_cnt(btf); i++) {
 430                 const char *s, *val, *msg;
 431                 const struct btf_type *t;
 432                 bool clear;
 433                 int flags;
 434
 435                 t = btf__type_by_id(btf, i);
 436                 if (!btf_is_decl_tag(t))
 437                         continue;
 438
 439                 if (t->type != func_id || btf_decl_tag(t)->component_idx != -1)
 440                         continue;
 441
 442                 s = btf__str_by_offset(btf, t->name_off);
 443                 if (str_has_pfx(s, TEST_TAG_DESCRIPTION_PFX)) {
 444                         description = s + sizeof(TEST_TAG_DESCRIPTION_PFX) - 1;
 445                 } else if (strcmp(s, TEST_TAG_EXPECT_FAILURE) == 0) {
 446                         spec->priv.expect_failure = true;
 447                         spec->mode_mask |= PRIV;
 448                 } else if (strcmp(s, TEST_TAG_EXPECT_SUCCESS) == 0) {
 449                         spec->priv.expect_failure = false;
 450                         spec->mode_mask |= PRIV;
 451                 } else if (strcmp(s, TEST_TAG_EXPECT_FAILURE_UNPRIV) == 0) {
 452                         spec->unpriv.expect_failure = true;
 453                         spec->mode_mask |= UNPRIV;
 454                         has_unpriv_result = true;
 455                 } else if (strcmp(s, TEST_TAG_EXPECT_SUCCESS_UNPRIV) == 0) {
 456                         spec->unpriv.expect_failure = false;
 457                         spec->mode_mask |= UNPRIV;
 458                         has_unpriv_result = true;
 459                 } else if (strcmp(s, TEST_TAG_AUXILIARY) == 0) {
 460                         spec->auxiliary = true;
 461                         spec->mode_mask |= PRIV;
 462                 } else if (strcmp(s, TEST_TAG_AUXILIARY_UNPRIV) == 0) {
 463                         spec->auxiliary = true;
 464                         spec->mode_mask |= UNPRIV;
 465                 } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_MSG_PFX))) {
 466                         err = push_msg(msg, &spec->priv.expect_msgs);
 467                         if (err)
 468                                 goto cleanup;
 469                         spec->mode_mask |= PRIV;
 470                 } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_MSG_PFX_UNPRIV))) {
 471                         err = push_msg(msg, &spec->unpriv.expect_msgs);
 472                         if (err)
 473                                 goto cleanup;
 474                         spec->mode_mask |= UNPRIV;
 475                 } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_JITED_PFX))) {
 476                         if (arch_mask == 0) {
 477                                 PRINT_FAIL("__jited used before __arch_*");
 478                                 goto cleanup;
 479                         }
 480                         if (collect_jit) {
 481                                 err = push_disasm_msg(msg, &jit_on_next_line,
 482                                                       &spec->priv.jited);
 483                                 if (err)
 484                                         goto cleanup;
 485                                 spec->mode_mask |= PRIV;
 486                         }
 487                 } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_JITED_PFX_UNPRIV))) {
 488                         if (arch_mask == 0) {
 489                                 PRINT_FAIL("__unpriv_jited used before __arch_*");
 490                                 goto cleanup;
 491                         }
 492                         if (collect_jit) {
 493                                 err = push_disasm_msg(msg, &unpriv_jit_on_next_line,
 494                                                       &spec->unpriv.jited);
 495                                 if (err)
 496                                         goto cleanup;
 497                                 spec->mode_mask |= UNPRIV;
 498                         }
 499                 } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_XLATED_PFX))) {
 500                         err = push_disasm_msg(msg, &xlated_on_next_line,
 501                                               &spec->priv.expect_xlated);
 502                         if (err)
 503                                 goto cleanup;
 504                         spec->mode_mask |= PRIV;
 505                 } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_XLATED_PFX_UNPRIV))) {
 506                         err = push_disasm_msg(msg, &unpriv_xlated_on_next_line,
 507                                               &spec->unpriv.expect_xlated);
 508                         if (err)
 509                                 goto cleanup;
 510                         spec->mode_mask |= UNPRIV;
 511                 } else if (str_has_pfx(s, TEST_TAG_RETVAL_PFX)) {
 512                         val = s + sizeof(TEST_TAG_RETVAL_PFX) - 1;
 513                         err = parse_retval(val, &spec->priv.retval, "__retval");
 514                         if (err)
 515                                 goto cleanup;
 516                         spec->priv.execute = true;
 517                         spec->mode_mask |= PRIV;
 518                 } else if (str_has_pfx(s, TEST_TAG_RETVAL_PFX_UNPRIV)) {
 519                         val = s + sizeof(TEST_TAG_RETVAL_PFX_UNPRIV) - 1;
 520                         err = parse_retval(val, &spec->unpriv.retval, "__retval_unpriv");
 521                         if (err)
 522                                 goto cleanup;
 523                         spec->mode_mask |= UNPRIV;
 524                         spec->unpriv.execute = true;
 525                         has_unpriv_retval = true;
 526                 } else if (str_has_pfx(s, TEST_TAG_LOG_LEVEL_PFX)) {
 527                         val = s + sizeof(TEST_TAG_LOG_LEVEL_PFX) - 1;
 528                         err = parse_int(val, &spec->log_level, "test log level");
 529                         if (err)
 530                                 goto cleanup;
 531                 } else if (str_has_pfx(s, TEST_TAG_PROG_FLAGS_PFX)) {
 532                         val = s + sizeof(TEST_TAG_PROG_FLAGS_PFX) - 1;
 533
 534                         clear = val[0] == '!';
 535                         if (clear)
 536                                 val++;
 537
 538                         if (strcmp(val, "BPF_F_STRICT_ALIGNMENT") == 0) {
 539                                 update_flags(&spec->prog_flags, BPF_F_STRICT_ALIGNMENT, clear);
 540                         } else if (strcmp(val, "BPF_F_ANY_ALIGNMENT") == 0) {
 541                                 update_flags(&spec->prog_flags, BPF_F_ANY_ALIGNMENT, clear);
 542                         } else if (strcmp(val, "BPF_F_TEST_RND_HI32") == 0) {
 543                                 update_flags(&spec->prog_flags, BPF_F_TEST_RND_HI32, clear);
 544                         } else if (strcmp(val, "BPF_F_TEST_STATE_FREQ") == 0) {
 545                                 update_flags(&spec->prog_flags, BPF_F_TEST_STATE_FREQ, clear);
 546                         } else if (strcmp(val, "BPF_F_SLEEPABLE") == 0) {
 547                                 update_flags(&spec->prog_flags, BPF_F_SLEEPABLE, clear);
 548                         } else if (strcmp(val, "BPF_F_XDP_HAS_FRAGS") == 0) {
 549                                 update_flags(&spec->prog_flags, BPF_F_XDP_HAS_FRAGS, clear);
 550                         } else if (strcmp(val, "BPF_F_TEST_REG_INVARIANTS") == 0) {
 551                                 update_flags(&spec->prog_flags, BPF_F_TEST_REG_INVARIANTS, clear);
 552                         } else /* assume numeric value */ {
 553                                 err = parse_int(val, &flags, "test prog flags");
 554                                 if (err)
 555                                         goto cleanup;
 556                                 update_flags(&spec->prog_flags, flags, clear);
 557                         }
 558                 } else if (str_has_pfx(s, TEST_TAG_ARCH)) {
 559                         val = s + sizeof(TEST_TAG_ARCH) - 1;
 560                         if (strcmp(val, "X86_64") == 0) {
 561                                 arch = ARCH_X86_64;
 562                         } else if (strcmp(val, "ARM64") == 0) {
 563                                 arch = ARCH_ARM64;
 564                         } else if (strcmp(val, "RISCV64") == 0) {
 565                                 arch = ARCH_RISCV64;
 566                         } else {
 567                                 PRINT_FAIL("bad arch spec: '%s'", val);
 568                                 err = -EINVAL;
 569                                 goto cleanup;
 570                         }
 571                         arch_mask |= arch;
 572                         collect_jit = get_current_arch() == arch;
 573                         unpriv_jit_on_next_line = true;
 574                         jit_on_next_line = true;
 575                 } else if (str_has_pfx(s, TEST_BTF_PATH)) {
 576                         spec->btf_custom_path = s + sizeof(TEST_BTF_PATH) - 1;
 577                 } else if (str_has_pfx(s, TEST_TAG_CAPS_UNPRIV)) {
 578                         val = s + sizeof(TEST_TAG_CAPS_UNPRIV) - 1;
 579                         err = parse_caps(val, &spec->unpriv.caps, "test caps");
 580                         if (err)
 581                                 goto cleanup;
 582                         spec->mode_mask |= UNPRIV;
 583                 }
 584         }
 585
 586         spec->arch_mask = arch_mask ?: -1;
 587
 588         if (spec->mode_mask == 0)
 589                 spec->mode_mask = PRIV;
 590
 591         if (!description)
 592                 description = spec->prog_name;
 593
 594         if (spec->mode_mask & PRIV) {
 595                 spec->priv.name = strdup(description);
 596                 if (!spec->priv.name) {
 597                         PRINT_FAIL("failed to allocate memory for priv.name\n");
 598                         err = -ENOMEM;
 599                         goto cleanup;
 600                 }
 601         }
 602
 603         if (spec->mode_mask & UNPRIV) {
 604                 int descr_len = strlen(description);
 605                 const char *suffix = " @unpriv";
 606                 char *name;
 607
 608                 name = malloc(descr_len + strlen(suffix) + 1);
 609                 if (!name) {
 610                         PRINT_FAIL("failed to allocate memory for unpriv.name\n");
 611                         err = -ENOMEM;
 612                         goto cleanup;
 613                 }
 614
 615                 strcpy(name, description);
 616                 strcpy(&name[descr_len], suffix);
 617                 spec->unpriv.name = name;
 618         }
 619
 620         if (spec->mode_mask & (PRIV | UNPRIV)) {
 621                 if (!has_unpriv_result)
 622                         spec->unpriv.expect_failure = spec->priv.expect_failure;
 623
 624                 if (!has_unpriv_retval) {
 625                         spec->unpriv.retval = spec->priv.retval;
 626                         spec->unpriv.execute = spec->priv.execute;
 627                 }
 628
 629                 if (spec->unpriv.expect_msgs.cnt == 0)
 630                         clone_msgs(&spec->priv.expect_msgs, &spec->unpriv.expect_msgs);
 631                 if (spec->unpriv.expect_xlated.cnt == 0)
 632                         clone_msgs(&spec->priv.expect_xlated, &spec->unpriv.expect_xlated);
 633                 if (spec->unpriv.jited.cnt == 0)
 634                         clone_msgs(&spec->priv.jited, &spec->unpriv.jited);
 635         }
 636
 637         spec->valid = true;
 638
 639         return 0;
 640
 641 cleanup:
 642         free_test_spec(spec);
 643         return err;
 644 }
 645
 646 static void prepare_case(struct test_loader *tester,
 647                          struct test_spec *spec,
 648                          struct bpf_object *obj,
 649                          struct bpf_program *prog)
 650 {
 651         int min_log_level = 0, prog_flags;
 652
 653         if (env.verbosity > VERBOSE_NONE)
 654                 min_log_level = 1;
 655         if (env.verbosity > VERBOSE_VERY)
 656                 min_log_level = 2;
 657
 658         bpf_program__set_log_buf(prog, tester->log_buf, tester->log_buf_sz);
 659
 660         /* Make sure we set at least minimal log level, unless test requires
 661          * even higher level already. Make sure to preserve independent log
 662          * level 4 (verifier stats), though.
 663          */
 664         if ((spec->log_level & 3) < min_log_level)
 665                 bpf_program__set_log_level(prog, (spec->log_level & 4) | min_log_level);
 666         else
 667                 bpf_program__set_log_level(prog, spec->log_level);
 668
 669         prog_flags = bpf_program__flags(prog);
 670         bpf_program__set_flags(prog, prog_flags | spec->prog_flags);
 671
 672         tester->log_buf[0] = '\0';
 673 }
 674
 675 static void emit_verifier_log(const char *log_buf, bool force)
 676 {
 677         if (!force && env.verbosity == VERBOSE_NONE)
 678                 return;
 679         fprintf(stdout, "VERIFIER LOG:\n=============\n%s=============\n", log_buf);
 680 }
 681
 682 static void emit_xlated(const char *xlated, bool force)
 683 {
 684         if (!force && env.verbosity == VERBOSE_NONE)
 685                 return;
 686         fprintf(stdout, "XLATED:\n=============\n%s=============\n", xlated);
 687 }
 688
 689 static void emit_jited(const char *jited, bool force)
 690 {
 691         if (!force && env.verbosity == VERBOSE_NONE)
 692                 return;
 693         fprintf(stdout, "JITED:\n=============\n%s=============\n", jited);
 694 }
 695
 696 static void validate_msgs(char *log_buf, struct expected_msgs *msgs,
 697                           void (*emit_fn)(const char *buf, bool force))
 698 {
 699         const char *log = log_buf, *prev_match;
 700         regmatch_t reg_match[1];
 701         int prev_match_line;
 702         int match_line;
 703         int i, j, err;
 704
 705         prev_match_line = -1;
 706         match_line = 0;
 707         prev_match = log;
 708         for (i = 0; i < msgs->cnt; i++) {
 709                 struct expect_msg *msg = &msgs->patterns[i];
 710                 const char *match = NULL, *pat_status;
 711                 bool wrong_line = false;
 712
 713                 if (!msg->is_regex) {
 714                         match = strstr(log, msg->substr);
 715                         if (match)
 716                                 log = match + strlen(msg->substr);
 717                 } else {
 718                         err = regexec(&msg->regex, log, 1, reg_match, 0);
 719                         if (err == 0) {
 720                                 match = log + reg_match[0].rm_so;
 721                                 log += reg_match[0].rm_eo;
 722                         }
 723                 }
 724
 725                 if (match) {
 726                         for (; prev_match < match; ++prev_match)
 727                                 if (*prev_match == '\n')
 728                                         ++match_line;
 729                         wrong_line = msg->on_next_line && prev_match_line >= 0 &&
 730                                      prev_match_line + 1 != match_line;
 731                 }
 732
 733                 if (!match || wrong_line) {
 734                         PRINT_FAIL("expect_msg\n");
 735                         if (env.verbosity == VERBOSE_NONE)
 736                                 emit_fn(log_buf, true /*force*/);
 737                         for (j = 0; j <= i; j++) {
 738                                 msg = &msgs->patterns[j];
 739                                 if (j < i)
 740                                         pat_status = "MATCHED   ";
 741                                 else if (wrong_line)
 742                                         pat_status = "WRONG LINE";
 743                                 else
 744                                         pat_status = "EXPECTED  ";
 745                                 msg = &msgs->patterns[j];
 746                                 fprintf(stderr, "%s %s: '%s'\n",
 747                                         pat_status,
 748                                         msg->is_regex ? " REGEX" : "SUBSTR",
 749                                         msg->substr);
 750                         }
 751                         if (wrong_line) {
 752                                 fprintf(stderr,
 753                                         "expecting match at line %d, actual match is at line %d\n",
 754                                         prev_match_line + 1, match_line);
 755                         }
 756                         break;
 757                 }
 758
 759                 prev_match_line = match_line;
 760         }
 761 }
 762
 763 struct cap_state {
 764         __u64 old_caps;
 765         bool initialized;
 766 };
 767
 768 static int drop_capabilities(struct cap_state *caps)
 769 {
 770         const __u64 caps_to_drop = (1ULL << CAP_SYS_ADMIN | 1ULL << CAP_NET_ADMIN |
 771                                     1ULL << CAP_PERFMON   | 1ULL << CAP_BPF);
 772         int err;
 773
 774         err = cap_disable_effective(caps_to_drop, &caps->old_caps);
 775         if (err) {
 776                 PRINT_FAIL("failed to drop capabilities: %i, %s\n", err, strerror(err));
 777                 return err;
 778         }
 779
 780         caps->initialized = true;
 781         return 0;
 782 }
 783
 784 static int restore_capabilities(struct cap_state *caps)
 785 {
 786         int err;
 787
 788         if (!caps->initialized)
 789                 return 0;
 790
 791         err = cap_enable_effective(caps->old_caps, NULL);
 792         if (err)
 793                 PRINT_FAIL("failed to restore capabilities: %i, %s\n", err, strerror(err));
 794         caps->initialized = false;
 795         return err;
 796 }
 797
 798 static bool can_execute_unpriv(struct test_loader *tester, struct test_spec *spec)
 799 {
 800         if (sysctl_unpriv_disabled < 0)
 801                 sysctl_unpriv_disabled = get_unpriv_disabled() ? 1 : 0;
 802         if (sysctl_unpriv_disabled)
 803                 return false;
 804         if ((spec->prog_flags & BPF_F_ANY_ALIGNMENT) && !EFFICIENT_UNALIGNED_ACCESS)
 805                 return false;
 806         return true;
 807 }
 808
 809 static bool is_unpriv_capable_map(struct bpf_map *map)
 810 {
 811         enum bpf_map_type type;
 812         __u32 flags;
 813
 814         type = bpf_map__type(map);
 815
 816         switch (type) {
 817         case BPF_MAP_TYPE_HASH:
 818         case BPF_MAP_TYPE_PERCPU_HASH:
 819         case BPF_MAP_TYPE_HASH_OF_MAPS:
 820                 flags = bpf_map__map_flags(map);
 821                 return !(flags & BPF_F_ZERO_SEED);
 822         case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE:
 823         case BPF_MAP_TYPE_ARRAY:
 824         case BPF_MAP_TYPE_RINGBUF:
 825         case BPF_MAP_TYPE_PROG_ARRAY:
 826         case BPF_MAP_TYPE_CGROUP_ARRAY:
 827         case BPF_MAP_TYPE_PERCPU_ARRAY:
 828         case BPF_MAP_TYPE_USER_RINGBUF:
 829         case BPF_MAP_TYPE_ARRAY_OF_MAPS:
 830         case BPF_MAP_TYPE_CGROUP_STORAGE:
 831         case BPF_MAP_TYPE_PERF_EVENT_ARRAY:
 832                 return true;
 833         default:
 834                 return false;
 835         }
 836 }
 837
 838 static int do_prog_test_run(int fd_prog, int *retval, bool empty_opts)
 839 {
 840         __u8 tmp_out[TEST_DATA_LEN << 2] = {};
 841         __u8 tmp_in[TEST_DATA_LEN] = {};
 842         int err, saved_errno;
 843         LIBBPF_OPTS(bpf_test_run_opts, topts,
 844                 .data_in = tmp_in,
 845                 .data_size_in = sizeof(tmp_in),
 846                 .data_out = tmp_out,
 847                 .data_size_out = sizeof(tmp_out),
 848                 .repeat = 1,
 849         );
 850
 851         if (empty_opts) {
 852                 memset(&topts, 0, sizeof(struct bpf_test_run_opts));
 853                 topts.sz = sizeof(struct bpf_test_run_opts);
 854         }
 855         err = bpf_prog_test_run_opts(fd_prog, &topts);
 856         saved_errno = errno;
 857
 858         if (err) {
 859                 PRINT_FAIL("FAIL: Unexpected bpf_prog_test_run error: %d (%s) ",
 860                            saved_errno, strerror(saved_errno));
 861                 return err;
 862         }
 863
 864         ASSERT_OK(0, "bpf_prog_test_run");
 865         *retval = topts.retval;
 866
 867         return 0;
 868 }
 869
 870 static bool should_do_test_run(struct test_spec *spec, struct test_subspec *subspec)
 871 {
 872         if (!subspec->execute)
 873                 return false;
 874
 875         if (subspec->expect_failure)
 876                 return false;
 877
 878         if ((spec->prog_flags & BPF_F_ANY_ALIGNMENT) && !EFFICIENT_UNALIGNED_ACCESS) {
 879                 if (env.verbosity != VERBOSE_NONE)
 880                         printf("alignment prevents execution\n");
 881                 return false;
 882         }
 883
 884         return true;
 885 }
 886
 887 /* Get a disassembly of BPF program after verifier applies all rewrites */
 888 static int get_xlated_program_text(int prog_fd, char *text, size_t text_sz)
 889 {
 890         struct bpf_insn *insn_start = NULL, *insn, *insn_end;
 891         __u32 insns_cnt = 0, i;
 892         char buf[64];
 893         FILE *out = NULL;
 894         int err;
 895
 896         err = get_xlated_program(prog_fd, &insn_start, &insns_cnt);
 897         if (!ASSERT_OK(err, "get_xlated_program"))
 898                 goto out;
 899         out = fmemopen(text, text_sz, "w");
 900         if (!ASSERT_OK_PTR(out, "open_memstream"))
 901                 goto out;
 902         insn_end = insn_start + insns_cnt;
 903         insn = insn_start;
 904         while (insn < insn_end) {
 905                 i = insn - insn_start;
 906                 insn = disasm_insn(insn, buf, sizeof(buf));
 907                 fprintf(out, "%d: %s\n", i, buf);
 908         }
 909         fflush(out);
 910
 911 out:
 912         free(insn_start);
 913         if (out)
 914                 fclose(out);
 915         return err;
 916 }
 917
 918 /* this function is forced noinline and has short generic name to look better
 919  * in test_progs output (in case of a failure)
 920  */
 921 static noinline
 922 void run_subtest(struct test_loader *tester,
 923                  struct bpf_object_open_opts *open_opts,
 924                  const void *obj_bytes,
 925                  size_t obj_byte_cnt,
 926                  struct test_spec *specs,
 927                  struct test_spec *spec,
 928                  bool unpriv)
 929 {
 930         struct test_subspec *subspec = unpriv ? &spec->unpriv : &spec->priv;
 931         struct bpf_program *tprog = NULL, *tprog_iter;
 932         struct bpf_link *link, *links[32] = {};
 933         struct test_spec *spec_iter;
 934         struct cap_state caps = {};
 935         struct bpf_object *tobj;
 936         struct bpf_map *map;
 937         int retval, err, i;
 938         int links_cnt = 0;
 939         bool should_load;
 940
 941         if (!test__start_subtest(subspec->name))
 942                 return;
 943
 944         if ((get_current_arch() & spec->arch_mask) == 0) {
 945                 test__skip();
 946                 return;
 947         }
 948
 949         if (unpriv) {
 950                 if (!can_execute_unpriv(tester, spec)) {
 951                         test__skip();
 952                         test__end_subtest();
 953                         return;
 954                 }
 955                 if (drop_capabilities(&caps)) {
 956                         test__end_subtest();
 957                         return;
 958                 }
 959                 if (subspec->caps) {
 960                         err = cap_enable_effective(subspec->caps, NULL);
 961                         if (err) {
 962                                 PRINT_FAIL("failed to set capabilities: %i, %s\n", err, strerror(err));
 963                                 goto subtest_cleanup;
 964                         }
 965                 }
 966         }
 967
 968         /* Implicitly reset to NULL if next test case doesn't specify */
 969         open_opts->btf_custom_path = spec->btf_custom_path;
 970
 971         tobj = bpf_object__open_mem(obj_bytes, obj_byte_cnt, open_opts);
 972         if (!ASSERT_OK_PTR(tobj, "obj_open_mem")) /* shouldn't happen */
 973                 goto subtest_cleanup;
 974
 975         i = 0;
 976         bpf_object__for_each_program(tprog_iter, tobj) {
 977                 spec_iter = &specs[i++];
 978                 should_load = false;
 979
 980                 if (spec_iter->valid) {
 981                         if (strcmp(bpf_program__name(tprog_iter), spec->prog_name) == 0) {
 982                                 tprog = tprog_iter;
 983                                 should_load = true;
 984                         }
 985
 986                         if (spec_iter->auxiliary &&
 987                             spec_iter->mode_mask & (unpriv ? UNPRIV : PRIV))
 988                                 should_load = true;
 989                 }
 990
 991                 bpf_program__set_autoload(tprog_iter, should_load);
 992         }
 993
 994         prepare_case(tester, spec, tobj, tprog);
 995
 996         /* By default bpf_object__load() automatically creates all
 997          * maps declared in the skeleton. Some map types are only
 998          * allowed in priv mode. Disable autoload for such maps in
 999          * unpriv mode.
1000          */
1001         bpf_object__for_each_map(map, tobj)
1002                 bpf_map__set_autocreate(map, !unpriv || is_unpriv_capable_map(map));
1003
1004         err = bpf_object__load(tobj);
1005         if (subspec->expect_failure) {
1006                 if (!ASSERT_ERR(err, "unexpected_load_success")) {
1007                         emit_verifier_log(tester->log_buf, false /*force*/);
1008                         goto tobj_cleanup;
1009                 }
1010         } else {
1011                 if (!ASSERT_OK(err, "unexpected_load_failure")) {
1012                         emit_verifier_log(tester->log_buf, true /*force*/);
1013                         goto tobj_cleanup;
1014                 }
1015         }
1016         emit_verifier_log(tester->log_buf, false /*force*/);
1017         validate_msgs(tester->log_buf, &subspec->expect_msgs, emit_verifier_log);
1018
1019         if (subspec->expect_xlated.cnt) {
1020                 err = get_xlated_program_text(bpf_program__fd(tprog),
1021                                               tester->log_buf, tester->log_buf_sz);
1022                 if (err)
1023                         goto tobj_cleanup;
1024                 emit_xlated(tester->log_buf, false /*force*/);
1025                 validate_msgs(tester->log_buf, &subspec->expect_xlated, emit_xlated);
1026         }
1027
1028         if (subspec->jited.cnt) {
1029                 err = get_jited_program_text(bpf_program__fd(tprog),
1030                                              tester->log_buf, tester->log_buf_sz);
1031                 if (err == -EOPNOTSUPP) {
1032                         printf("%s:SKIP: jited programs disassembly is not supported,\n", __func__);
1033                         printf("%s:SKIP: tests are built w/o LLVM development libs\n", __func__);
1034                         test__skip();
1035                         goto tobj_cleanup;
1036                 }
1037                 if (!ASSERT_EQ(err, 0, "get_jited_program_text"))
1038                         goto tobj_cleanup;
1039                 emit_jited(tester->log_buf, false /*force*/);
1040                 validate_msgs(tester->log_buf, &subspec->jited, emit_jited);
1041         }
1042
1043         if (should_do_test_run(spec, subspec)) {
1044                 /* For some reason test_verifier executes programs
1045                  * with all capabilities restored. Do the same here.
1046                  */
1047                 if (restore_capabilities(&caps))
1048                         goto tobj_cleanup;
1049
1050                 /* Do bpf_map__attach_struct_ops() for each struct_ops map.
1051                  * This should trigger bpf_struct_ops->reg callback on kernel side.
1052                  */
1053                 bpf_object__for_each_map(map, tobj) {
1054                         if (!bpf_map__autocreate(map) ||
1055                             bpf_map__type(map) != BPF_MAP_TYPE_STRUCT_OPS)
1056                                 continue;
1057                         if (links_cnt >= ARRAY_SIZE(links)) {
1058                                 PRINT_FAIL("too many struct_ops maps");
1059                                 goto tobj_cleanup;
1060                         }
1061                         link = bpf_map__attach_struct_ops(map);
1062                         if (!link) {
1063                                 PRINT_FAIL("bpf_map__attach_struct_ops failed for map %s: err=%d\n",
1064                                            bpf_map__name(map), err);
1065                                 goto tobj_cleanup;
1066                         }
1067                         links[links_cnt++] = link;
1068                 }
1069
1070                 if (tester->pre_execution_cb) {
1071                         err = tester->pre_execution_cb(tobj);
1072                         if (err) {
1073                                 PRINT_FAIL("pre_execution_cb failed: %d\n", err);
1074                                 goto tobj_cleanup;
1075                         }
1076                 }
1077
1078                 do_prog_test_run(bpf_program__fd(tprog), &retval,
1079                                  bpf_program__type(tprog) == BPF_PROG_TYPE_SYSCALL ? true : false);
1080                 if (retval != subspec->retval && subspec->retval != POINTER_VALUE) {
1081                         PRINT_FAIL("Unexpected retval: %d != %d\n", retval, subspec->retval);
1082                         goto tobj_cleanup;
1083                 }
1084                 /* redo bpf_map__attach_struct_ops for each test */
1085                 while (links_cnt > 0)
1086                         bpf_link__destroy(links[--links_cnt]);
1087         }
1088
1089 tobj_cleanup:
1090         while (links_cnt > 0)
1091                 bpf_link__destroy(links[--links_cnt]);
1092         bpf_object__close(tobj);
1093 subtest_cleanup:
1094         test__end_subtest();
1095         restore_capabilities(&caps);
1096 }
1097
1098 static void process_subtest(struct test_loader *tester,
1099                             const char *skel_name,
1100                             skel_elf_bytes_fn elf_bytes_factory)
1101 {
1102         LIBBPF_OPTS(bpf_object_open_opts, open_opts, .object_name = skel_name);
1103         struct test_spec *specs = NULL;
1104         struct bpf_object *obj = NULL;
1105         struct bpf_program *prog;
1106         const void *obj_bytes;
1107         int err, i, nr_progs;
1108         size_t obj_byte_cnt;
1109
1110         if (tester_init(tester) < 0)
1111                 return; /* failed to initialize tester */
1112
1113         obj_bytes = elf_bytes_factory(&obj_byte_cnt);
1114         obj = bpf_object__open_mem(obj_bytes, obj_byte_cnt, &open_opts);
1115         if (!ASSERT_OK_PTR(obj, "obj_open_mem"))
1116                 return;
1117
1118         nr_progs = 0;
1119         bpf_object__for_each_program(prog, obj)
1120                 ++nr_progs;
1121
1122         specs = calloc(nr_progs, sizeof(struct test_spec));
1123         if (!ASSERT_OK_PTR(specs, "specs_alloc"))
1124                 return;
1125
1126         i = 0;
1127         bpf_object__for_each_program(prog, obj) {
1128                 /* ignore tests for which  we can't derive test specification */
1129                 err = parse_test_spec(tester, obj, prog, &specs[i++]);
1130                 if (err)
1131                         PRINT_FAIL("Can't parse test spec for program '%s'\n",
1132                                    bpf_program__name(prog));
1133         }
1134
1135         i = 0;
1136         bpf_object__for_each_program(prog, obj) {
1137                 struct test_spec *spec = &specs[i++];
1138
1139                 if (!spec->valid || spec->auxiliary)
1140                         continue;
1141
1142                 if (spec->mode_mask & PRIV)
1143                         run_subtest(tester, &open_opts, obj_bytes, obj_byte_cnt,
1144                                     specs, spec, false);
1145                 if (spec->mode_mask & UNPRIV)
1146                         run_subtest(tester, &open_opts, obj_bytes, obj_byte_cnt,
1147                                     specs, spec, true);
1148
1149         }
1150
1151         for (i = 0; i < nr_progs; ++i)
1152                 free_test_spec(&specs[i]);
1153         free(specs);
1154         bpf_object__close(obj);
1155 }
1156
1157 void test_loader__run_subtests(struct test_loader *tester,
1158                                const char *skel_name,
1159                                skel_elf_bytes_fn elf_bytes_factory)
1160 {
1161         /* see comment in run_subtest() for why we do this function nesting */
1162         process_subtest(tester, skel_name, elf_bytes_factory);
1163 }