tools/testing/selftests/bpf/test_loader.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
   3 #include <linux/capability.h>
   4 #include <stdlib.h>
   5 #include <regex.h>
   6 #include <test_progs.h>
   7 #include <bpf/btf.h>
   8
   9 #include "autoconf_helper.h"
  10 #include "disasm_helpers.h"
  11 #include "unpriv_helpers.h"
  12 #include "cap_helpers.h"
  13 #include "jit_disasm_helpers.h"
  14
  15 #define str_has_pfx(str, pfx) \
  16         (strncmp(str, pfx, __builtin_constant_p(pfx) ? sizeof(pfx) - 1 : strlen(pfx)) == 0)
  17
  18 #define TEST_LOADER_LOG_BUF_SZ 2097152
  19
  20 #define TEST_TAG_EXPECT_FAILURE "comment:test_expect_failure"
  21 #define TEST_TAG_EXPECT_SUCCESS "comment:test_expect_success"
  22 #define TEST_TAG_EXPECT_MSG_PFX "comment:test_expect_msg="
  23 #define TEST_TAG_EXPECT_XLATED_PFX "comment:test_expect_xlated="
  24 #define TEST_TAG_EXPECT_FAILURE_UNPRIV "comment:test_expect_failure_unpriv"
  25 #define TEST_TAG_EXPECT_SUCCESS_UNPRIV "comment:test_expect_success_unpriv"
  26 #define TEST_TAG_EXPECT_MSG_PFX_UNPRIV "comment:test_expect_msg_unpriv="
  27 #define TEST_TAG_EXPECT_XLATED_PFX_UNPRIV "comment:test_expect_xlated_unpriv="
  28 #define TEST_TAG_LOG_LEVEL_PFX "comment:test_log_level="
  29 #define TEST_TAG_PROG_FLAGS_PFX "comment:test_prog_flags="
  30 #define TEST_TAG_DESCRIPTION_PFX "comment:test_description="
  31 #define TEST_TAG_RETVAL_PFX "comment:test_retval="
  32 #define TEST_TAG_RETVAL_PFX_UNPRIV "comment:test_retval_unpriv="
  33 #define TEST_TAG_AUXILIARY "comment:test_auxiliary"
  34 #define TEST_TAG_AUXILIARY_UNPRIV "comment:test_auxiliary_unpriv"
  35 #define TEST_BTF_PATH "comment:test_btf_path="
  36 #define TEST_TAG_ARCH "comment:test_arch="
  37 #define TEST_TAG_JITED_PFX "comment:test_jited="
  38 #define TEST_TAG_JITED_PFX_UNPRIV "comment:test_jited_unpriv="
  39
  40 /* Warning: duplicated in bpf_misc.h */
  41 #define POINTER_VALUE   0xcafe4all
  42 #define TEST_DATA_LEN   64
  43
  44 #ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
  45 #define EFFICIENT_UNALIGNED_ACCESS 1
  46 #else
  47 #define EFFICIENT_UNALIGNED_ACCESS 0
  48 #endif
  49
  50 static int sysctl_unpriv_disabled = -1;
  51
  52 enum mode {
  53         PRIV = 1,
  54         UNPRIV = 2
  55 };
  56
  57 struct expect_msg {
  58         const char *substr; /* substring match */
  59         regex_t regex;
  60         bool is_regex;
  61         bool on_next_line;
  62 };
  63
  64 struct expected_msgs {
  65         struct expect_msg *patterns;
  66         size_t cnt;
  67 };
  68
  69 struct test_subspec {
  70         char *name;
  71         bool expect_failure;
  72         struct expected_msgs expect_msgs;
  73         struct expected_msgs expect_xlated;
  74         struct expected_msgs jited;
  75         int retval;
  76         bool execute;
  77 };
  78
  79 struct test_spec {
  80         const char *prog_name;
  81         struct test_subspec priv;
  82         struct test_subspec unpriv;
  83         const char *btf_custom_path;
  84         int log_level;
  85         int prog_flags;
  86         int mode_mask;
  87         int arch_mask;
  88         bool auxiliary;
  89         bool valid;
  90 };
  91
  92 static int tester_init(struct test_loader *tester)
  93 {
  94         if (!tester->log_buf) {
  95                 tester->log_buf_sz = TEST_LOADER_LOG_BUF_SZ;
  96                 tester->log_buf = calloc(tester->log_buf_sz, 1);
  97                 if (!ASSERT_OK_PTR(tester->log_buf, "tester_log_buf"))
  98                         return -ENOMEM;
  99         }
 100
 101         return 0;
 102 }
 103
 104 void test_loader_fini(struct test_loader *tester)
 105 {
 106         if (!tester)
 107                 return;
 108
 109         free(tester->log_buf);
 110 }
 111
 112 static void free_msgs(struct expected_msgs *msgs)
 113 {
 114         int i;
 115
 116         for (i = 0; i < msgs->cnt; i++)
 117                 if (msgs->patterns[i].is_regex)
 118                         regfree(&msgs->patterns[i].regex);
 119         free(msgs->patterns);
 120         msgs->patterns = NULL;
 121         msgs->cnt = 0;
 122 }
 123
 124 static void free_test_spec(struct test_spec *spec)
 125 {
 126         /* Deallocate expect_msgs arrays. */
 127         free_msgs(&spec->priv.expect_msgs);
 128         free_msgs(&spec->unpriv.expect_msgs);
 129         free_msgs(&spec->priv.expect_xlated);
 130         free_msgs(&spec->unpriv.expect_xlated);
 131         free_msgs(&spec->priv.jited);
 132         free_msgs(&spec->unpriv.jited);
 133
 134         free(spec->priv.name);
 135         free(spec->unpriv.name);
 136         spec->priv.name = NULL;
 137         spec->unpriv.name = NULL;
 138 }
 139
 140 /* Compiles regular expression matching pattern.
 141  * Pattern has a special syntax:
 142  *
 143  *   pattern := (<verbatim text> | regex)*
 144  *   regex := "{{" <posix extended regular expression> "}}"
 145  *
 146  * In other words, pattern is a verbatim text with inclusion
 147  * of regular expressions enclosed in "{{" "}}" pairs.
 148  * For example, pattern "foo{{[0-9]+}}" matches strings like
 149  * "foo0", "foo007", etc.
 150  */
 151 static int compile_regex(const char *pattern, regex_t *regex)
 152 {
 153         char err_buf[256], buf[256] = {}, *ptr, *buf_end;
 154         const char *original_pattern = pattern;
 155         bool in_regex = false;
 156         int err;
 157
 158         buf_end = buf + sizeof(buf);
 159         ptr = buf;
 160         while (*pattern && ptr < buf_end - 2) {
 161                 if (!in_regex && str_has_pfx(pattern, "{{")) {
 162                         in_regex = true;
 163                         pattern += 2;
 164                         continue;
 165                 }
 166                 if (in_regex && str_has_pfx(pattern, "}}")) {
 167                         in_regex = false;
 168                         pattern += 2;
 169                         continue;
 170                 }
 171                 if (in_regex) {
 172                         *ptr++ = *pattern++;
 173                         continue;
 174                 }
 175                 /* list of characters that need escaping for extended posix regex */
 176                 if (strchr(".[]\\()*+?{}|^$", *pattern)) {
 177                         *ptr++ = '\\';
 178                         *ptr++ = *pattern++;
 179                         continue;
 180                 }
 181                 *ptr++ = *pattern++;
 182         }
 183         if (*pattern) {
 184                 PRINT_FAIL("Regexp too long: '%s'\n", original_pattern);
 185                 return -EINVAL;
 186         }
 187         if (in_regex) {
 188                 PRINT_FAIL("Regexp has open '{{' but no closing '}}': '%s'\n", original_pattern);
 189                 return -EINVAL;
 190         }
 191         err = regcomp(regex, buf, REG_EXTENDED | REG_NEWLINE);
 192         if (err != 0) {
 193                 regerror(err, regex, err_buf, sizeof(err_buf));
 194                 PRINT_FAIL("Regexp compilation error in '%s': '%s'\n", buf, err_buf);
 195                 return -EINVAL;
 196         }
 197         return 0;
 198 }
 199
 200 static int __push_msg(const char *pattern, bool on_next_line, struct expected_msgs *msgs)
 201 {
 202         struct expect_msg *msg;
 203         void *tmp;
 204         int err;
 205
 206         tmp = realloc(msgs->patterns,
 207                       (1 + msgs->cnt) * sizeof(struct expect_msg));
 208         if (!tmp) {
 209                 ASSERT_FAIL("failed to realloc memory for messages\n");
 210                 return -ENOMEM;
 211         }
 212         msgs->patterns = tmp;
 213         msg = &msgs->patterns[msgs->cnt];
 214         msg->on_next_line = on_next_line;
 215         msg->substr = pattern;
 216         msg->is_regex = false;
 217         if (strstr(pattern, "{{")) {
 218                 err = compile_regex(pattern, &msg->regex);
 219                 if (err)
 220                         return err;
 221                 msg->is_regex = true;
 222         }
 223         msgs->cnt += 1;
 224         return 0;
 225 }
 226
 227 static int clone_msgs(struct expected_msgs *from, struct expected_msgs *to)
 228 {
 229         struct expect_msg *msg;
 230         int i, err;
 231
 232         for (i = 0; i < from->cnt; i++) {
 233                 msg = &from->patterns[i];
 234                 err = __push_msg(msg->substr, msg->on_next_line, to);
 235                 if (err)
 236                         return err;
 237         }
 238         return 0;
 239 }
 240
 241 static int push_msg(const char *substr, struct expected_msgs *msgs)
 242 {
 243         return __push_msg(substr, false, msgs);
 244 }
 245
 246 static int push_disasm_msg(const char *regex_str, bool *on_next_line, struct expected_msgs *msgs)
 247 {
 248         int err;
 249
 250         if (strcmp(regex_str, "...") == 0) {
 251                 *on_next_line = false;
 252                 return 0;
 253         }
 254         err = __push_msg(regex_str, *on_next_line, msgs);
 255         if (err)
 256                 return err;
 257         *on_next_line = true;
 258         return 0;
 259 }
 260
 261 static int parse_int(const char *str, int *val, const char *name)
 262 {
 263         char *end;
 264         long tmp;
 265
 266         errno = 0;
 267         if (str_has_pfx(str, "0x"))
 268                 tmp = strtol(str + 2, &end, 16);
 269         else
 270                 tmp = strtol(str, &end, 10);
 271         if (errno || end[0] != '\0') {
 272                 PRINT_FAIL("failed to parse %s from '%s'\n", name, str);
 273                 return -EINVAL;
 274         }
 275         *val = tmp;
 276         return 0;
 277 }
 278
 279 static int parse_retval(const char *str, int *val, const char *name)
 280 {
 281         struct {
 282                 char *name;
 283                 int val;
 284         } named_values[] = {
 285                 { "INT_MIN"      , INT_MIN },
 286                 { "POINTER_VALUE", POINTER_VALUE },
 287                 { "TEST_DATA_LEN", TEST_DATA_LEN },
 288         };
 289         int i;
 290
 291         for (i = 0; i < ARRAY_SIZE(named_values); ++i) {
 292                 if (strcmp(str, named_values[i].name) != 0)
 293                         continue;
 294                 *val = named_values[i].val;
 295                 return 0;
 296         }
 297
 298         return parse_int(str, val, name);
 299 }
 300
 301 static void update_flags(int *flags, int flag, bool clear)
 302 {
 303         if (clear)
 304                 *flags &= ~flag;
 305         else
 306                 *flags |= flag;
 307 }
 308
 309 /* Matches a string of form '<pfx>[^=]=.*' and returns it's suffix.
 310  * Used to parse btf_decl_tag values.
 311  * Such values require unique prefix because compiler does not add
 312  * same __attribute__((btf_decl_tag(...))) twice.
 313  * Test suite uses two-component tags for such cases:
 314  *
 315  *   <pfx> __COUNTER__ '='
 316  *
 317  * For example, two consecutive __msg tags '__msg("foo") __msg("foo")'
 318  * would be encoded as:
 319  *
 320  *   [18] DECL_TAG 'comment:test_expect_msg=0=foo' type_id=15 component_idx=-1
 321  *   [19] DECL_TAG 'comment:test_expect_msg=1=foo' type_id=15 component_idx=-1
 322  *
 323  * And the purpose of this function is to extract 'foo' from the above.
 324  */
 325 static const char *skip_dynamic_pfx(const char *s, const char *pfx)
 326 {
 327         const char *msg;
 328
 329         if (strncmp(s, pfx, strlen(pfx)) != 0)
 330                 return NULL;
 331         msg = s + strlen(pfx);
 332         msg = strchr(msg, '=');
 333         if (!msg)
 334                 return NULL;
 335         return msg + 1;
 336 }
 337
 338 enum arch {
 339         ARCH_UNKNOWN    = 0x1,
 340         ARCH_X86_64     = 0x2,
 341         ARCH_ARM64      = 0x4,
 342         ARCH_RISCV64    = 0x8,
 343 };
 344
 345 static int get_current_arch(void)
 346 {
 347 #if defined(__x86_64__)
 348         return ARCH_X86_64;
 349 #elif defined(__aarch64__)
 350         return ARCH_ARM64;
 351 #elif defined(__riscv) && __riscv_xlen == 64
 352         return ARCH_RISCV64;
 353 #endif
 354         return ARCH_UNKNOWN;
 355 }
 356
 357 /* Uses btf_decl_tag attributes to describe the expected test
 358  * behavior, see bpf_misc.h for detailed description of each attribute
 359  * and attribute combinations.
 360  */
 361 static int parse_test_spec(struct test_loader *tester,
 362                            struct bpf_object *obj,
 363                            struct bpf_program *prog,
 364                            struct test_spec *spec)
 365 {
 366         const char *description = NULL;
 367         bool has_unpriv_result = false;
 368         bool has_unpriv_retval = false;
 369         bool unpriv_xlated_on_next_line = true;
 370         bool xlated_on_next_line = true;
 371         bool unpriv_jit_on_next_line;
 372         bool jit_on_next_line;
 373         bool collect_jit = false;
 374         int func_id, i, err = 0;
 375         u32 arch_mask = 0;
 376         struct btf *btf;
 377         enum arch arch;
 378
 379         memset(spec, 0, sizeof(*spec));
 380
 381         spec->prog_name = bpf_program__name(prog);
 382         spec->prog_flags = testing_prog_flags();
 383
 384         btf = bpf_object__btf(obj);
 385         if (!btf) {
 386                 ASSERT_FAIL("BPF object has no BTF");
 387                 return -EINVAL;
 388         }
 389
 390         func_id = btf__find_by_name_kind(btf, spec->prog_name, BTF_KIND_FUNC);
 391         if (func_id < 0) {
 392                 ASSERT_FAIL("failed to find FUNC BTF type for '%s'", spec->prog_name);
 393                 return -EINVAL;
 394         }
 395
 396         for (i = 1; i < btf__type_cnt(btf); i++) {
 397                 const char *s, *val, *msg;
 398                 const struct btf_type *t;
 399                 bool clear;
 400                 int flags;
 401
 402                 t = btf__type_by_id(btf, i);
 403                 if (!btf_is_decl_tag(t))
 404                         continue;
 405
 406                 if (t->type != func_id || btf_decl_tag(t)->component_idx != -1)
 407                         continue;
 408
 409                 s = btf__str_by_offset(btf, t->name_off);
 410                 if (str_has_pfx(s, TEST_TAG_DESCRIPTION_PFX)) {
 411                         description = s + sizeof(TEST_TAG_DESCRIPTION_PFX) - 1;
 412                 } else if (strcmp(s, TEST_TAG_EXPECT_FAILURE) == 0) {
 413                         spec->priv.expect_failure = true;
 414                         spec->mode_mask |= PRIV;
 415                 } else if (strcmp(s, TEST_TAG_EXPECT_SUCCESS) == 0) {
 416                         spec->priv.expect_failure = false;
 417                         spec->mode_mask |= PRIV;
 418                 } else if (strcmp(s, TEST_TAG_EXPECT_FAILURE_UNPRIV) == 0) {
 419                         spec->unpriv.expect_failure = true;
 420                         spec->mode_mask |= UNPRIV;
 421                         has_unpriv_result = true;
 422                 } else if (strcmp(s, TEST_TAG_EXPECT_SUCCESS_UNPRIV) == 0) {
 423                         spec->unpriv.expect_failure = false;
 424                         spec->mode_mask |= UNPRIV;
 425                         has_unpriv_result = true;
 426                 } else if (strcmp(s, TEST_TAG_AUXILIARY) == 0) {
 427                         spec->auxiliary = true;
 428                         spec->mode_mask |= PRIV;
 429                 } else if (strcmp(s, TEST_TAG_AUXILIARY_UNPRIV) == 0) {
 430                         spec->auxiliary = true;
 431                         spec->mode_mask |= UNPRIV;
 432                 } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_MSG_PFX))) {
 433                         err = push_msg(msg, &spec->priv.expect_msgs);
 434                         if (err)
 435                                 goto cleanup;
 436                         spec->mode_mask |= PRIV;
 437                 } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_MSG_PFX_UNPRIV))) {
 438                         err = push_msg(msg, &spec->unpriv.expect_msgs);
 439                         if (err)
 440                                 goto cleanup;
 441                         spec->mode_mask |= UNPRIV;
 442                 } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_JITED_PFX))) {
 443                         if (arch_mask == 0) {
 444                                 PRINT_FAIL("__jited used before __arch_*");
 445                                 goto cleanup;
 446                         }
 447                         if (collect_jit) {
 448                                 err = push_disasm_msg(msg, &jit_on_next_line,
 449                                                       &spec->priv.jited);
 450                                 if (err)
 451                                         goto cleanup;
 452                                 spec->mode_mask |= PRIV;
 453                         }
 454                 } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_JITED_PFX_UNPRIV))) {
 455                         if (arch_mask == 0) {
 456                                 PRINT_FAIL("__unpriv_jited used before __arch_*");
 457                                 goto cleanup;
 458                         }
 459                         if (collect_jit) {
 460                                 err = push_disasm_msg(msg, &unpriv_jit_on_next_line,
 461                                                       &spec->unpriv.jited);
 462                                 if (err)
 463                                         goto cleanup;
 464                                 spec->mode_mask |= UNPRIV;
 465                         }
 466                 } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_XLATED_PFX))) {
 467                         err = push_disasm_msg(msg, &xlated_on_next_line,
 468                                               &spec->priv.expect_xlated);
 469                         if (err)
 470                                 goto cleanup;
 471                         spec->mode_mask |= PRIV;
 472                 } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_XLATED_PFX_UNPRIV))) {
 473                         err = push_disasm_msg(msg, &unpriv_xlated_on_next_line,
 474                                               &spec->unpriv.expect_xlated);
 475                         if (err)
 476                                 goto cleanup;
 477                         spec->mode_mask |= UNPRIV;
 478                 } else if (str_has_pfx(s, TEST_TAG_RETVAL_PFX)) {
 479                         val = s + sizeof(TEST_TAG_RETVAL_PFX) - 1;
 480                         err = parse_retval(val, &spec->priv.retval, "__retval");
 481                         if (err)
 482                                 goto cleanup;
 483                         spec->priv.execute = true;
 484                         spec->mode_mask |= PRIV;
 485                 } else if (str_has_pfx(s, TEST_TAG_RETVAL_PFX_UNPRIV)) {
 486                         val = s + sizeof(TEST_TAG_RETVAL_PFX_UNPRIV) - 1;
 487                         err = parse_retval(val, &spec->unpriv.retval, "__retval_unpriv");
 488                         if (err)
 489                                 goto cleanup;
 490                         spec->mode_mask |= UNPRIV;
 491                         spec->unpriv.execute = true;
 492                         has_unpriv_retval = true;
 493                 } else if (str_has_pfx(s, TEST_TAG_LOG_LEVEL_PFX)) {
 494                         val = s + sizeof(TEST_TAG_LOG_LEVEL_PFX) - 1;
 495                         err = parse_int(val, &spec->log_level, "test log level");
 496                         if (err)
 497                                 goto cleanup;
 498                 } else if (str_has_pfx(s, TEST_TAG_PROG_FLAGS_PFX)) {
 499                         val = s + sizeof(TEST_TAG_PROG_FLAGS_PFX) - 1;
 500
 501                         clear = val[0] == '!';
 502                         if (clear)
 503                                 val++;
 504
 505                         if (strcmp(val, "BPF_F_STRICT_ALIGNMENT") == 0) {
 506                                 update_flags(&spec->prog_flags, BPF_F_STRICT_ALIGNMENT, clear);
 507                         } else if (strcmp(val, "BPF_F_ANY_ALIGNMENT") == 0) {
 508                                 update_flags(&spec->prog_flags, BPF_F_ANY_ALIGNMENT, clear);
 509                         } else if (strcmp(val, "BPF_F_TEST_RND_HI32") == 0) {
 510                                 update_flags(&spec->prog_flags, BPF_F_TEST_RND_HI32, clear);
 511                         } else if (strcmp(val, "BPF_F_TEST_STATE_FREQ") == 0) {
 512                                 update_flags(&spec->prog_flags, BPF_F_TEST_STATE_FREQ, clear);
 513                         } else if (strcmp(val, "BPF_F_SLEEPABLE") == 0) {
 514                                 update_flags(&spec->prog_flags, BPF_F_SLEEPABLE, clear);
 515                         } else if (strcmp(val, "BPF_F_XDP_HAS_FRAGS") == 0) {
 516                                 update_flags(&spec->prog_flags, BPF_F_XDP_HAS_FRAGS, clear);
 517                         } else if (strcmp(val, "BPF_F_TEST_REG_INVARIANTS") == 0) {
 518                                 update_flags(&spec->prog_flags, BPF_F_TEST_REG_INVARIANTS, clear);
 519                         } else /* assume numeric value */ {
 520                                 err = parse_int(val, &flags, "test prog flags");
 521                                 if (err)
 522                                         goto cleanup;
 523                                 update_flags(&spec->prog_flags, flags, clear);
 524                         }
 525                 } else if (str_has_pfx(s, TEST_TAG_ARCH)) {
 526                         val = s + sizeof(TEST_TAG_ARCH) - 1;
 527                         if (strcmp(val, "X86_64") == 0) {
 528                                 arch = ARCH_X86_64;
 529                         } else if (strcmp(val, "ARM64") == 0) {
 530                                 arch = ARCH_ARM64;
 531                         } else if (strcmp(val, "RISCV64") == 0) {
 532                                 arch = ARCH_RISCV64;
 533                         } else {
 534                                 PRINT_FAIL("bad arch spec: '%s'", val);
 535                                 err = -EINVAL;
 536                                 goto cleanup;
 537                         }
 538                         arch_mask |= arch;
 539                         collect_jit = get_current_arch() == arch;
 540                         unpriv_jit_on_next_line = true;
 541                         jit_on_next_line = true;
 542                 } else if (str_has_pfx(s, TEST_BTF_PATH)) {
 543                         spec->btf_custom_path = s + sizeof(TEST_BTF_PATH) - 1;
 544                 }
 545         }
 546
 547         spec->arch_mask = arch_mask ?: -1;
 548
 549         if (spec->mode_mask == 0)
 550                 spec->mode_mask = PRIV;
 551
 552         if (!description)
 553                 description = spec->prog_name;
 554
 555         if (spec->mode_mask & PRIV) {
 556                 spec->priv.name = strdup(description);
 557                 if (!spec->priv.name) {
 558                         PRINT_FAIL("failed to allocate memory for priv.name\n");
 559                         err = -ENOMEM;
 560                         goto cleanup;
 561                 }
 562         }
 563
 564         if (spec->mode_mask & UNPRIV) {
 565                 int descr_len = strlen(description);
 566                 const char *suffix = " @unpriv";
 567                 char *name;
 568
 569                 name = malloc(descr_len + strlen(suffix) + 1);
 570                 if (!name) {
 571                         PRINT_FAIL("failed to allocate memory for unpriv.name\n");
 572                         err = -ENOMEM;
 573                         goto cleanup;
 574                 }
 575
 576                 strcpy(name, description);
 577                 strcpy(&name[descr_len], suffix);
 578                 spec->unpriv.name = name;
 579         }
 580
 581         if (spec->mode_mask & (PRIV | UNPRIV)) {
 582                 if (!has_unpriv_result)
 583                         spec->unpriv.expect_failure = spec->priv.expect_failure;
 584
 585                 if (!has_unpriv_retval) {
 586                         spec->unpriv.retval = spec->priv.retval;
 587                         spec->unpriv.execute = spec->priv.execute;
 588                 }
 589
 590                 if (spec->unpriv.expect_msgs.cnt == 0)
 591                         clone_msgs(&spec->priv.expect_msgs, &spec->unpriv.expect_msgs);
 592                 if (spec->unpriv.expect_xlated.cnt == 0)
 593                         clone_msgs(&spec->priv.expect_xlated, &spec->unpriv.expect_xlated);
 594                 if (spec->unpriv.jited.cnt == 0)
 595                         clone_msgs(&spec->priv.jited, &spec->unpriv.jited);
 596         }
 597
 598         spec->valid = true;
 599
 600         return 0;
 601
 602 cleanup:
 603         free_test_spec(spec);
 604         return err;
 605 }
 606
 607 static void prepare_case(struct test_loader *tester,
 608                          struct test_spec *spec,
 609                          struct bpf_object *obj,
 610                          struct bpf_program *prog)
 611 {
 612         int min_log_level = 0, prog_flags;
 613
 614         if (env.verbosity > VERBOSE_NONE)
 615                 min_log_level = 1;
 616         if (env.verbosity > VERBOSE_VERY)
 617                 min_log_level = 2;
 618
 619         bpf_program__set_log_buf(prog, tester->log_buf, tester->log_buf_sz);
 620
 621         /* Make sure we set at least minimal log level, unless test requires
 622          * even higher level already. Make sure to preserve independent log
 623          * level 4 (verifier stats), though.
 624          */
 625         if ((spec->log_level & 3) < min_log_level)
 626                 bpf_program__set_log_level(prog, (spec->log_level & 4) | min_log_level);
 627         else
 628                 bpf_program__set_log_level(prog, spec->log_level);
 629
 630         prog_flags = bpf_program__flags(prog);
 631         bpf_program__set_flags(prog, prog_flags | spec->prog_flags);
 632
 633         tester->log_buf[0] = '\0';
 634 }
 635
 636 static void emit_verifier_log(const char *log_buf, bool force)
 637 {
 638         if (!force && env.verbosity == VERBOSE_NONE)
 639                 return;
 640         fprintf(stdout, "VERIFIER LOG:\n=============\n%s=============\n", log_buf);
 641 }
 642
 643 static void emit_xlated(const char *xlated, bool force)
 644 {
 645         if (!force && env.verbosity == VERBOSE_NONE)
 646                 return;
 647         fprintf(stdout, "XLATED:\n=============\n%s=============\n", xlated);
 648 }
 649
 650 static void emit_jited(const char *jited, bool force)
 651 {
 652         if (!force && env.verbosity == VERBOSE_NONE)
 653                 return;
 654         fprintf(stdout, "JITED:\n=============\n%s=============\n", jited);
 655 }
 656
 657 static void validate_msgs(char *log_buf, struct expected_msgs *msgs,
 658                           void (*emit_fn)(const char *buf, bool force))
 659 {
 660         const char *log = log_buf, *prev_match;
 661         regmatch_t reg_match[1];
 662         int prev_match_line;
 663         int match_line;
 664         int i, j, err;
 665
 666         prev_match_line = -1;
 667         match_line = 0;
 668         prev_match = log;
 669         for (i = 0; i < msgs->cnt; i++) {
 670                 struct expect_msg *msg = &msgs->patterns[i];
 671                 const char *match = NULL, *pat_status;
 672                 bool wrong_line = false;
 673
 674                 if (!msg->is_regex) {
 675                         match = strstr(log, msg->substr);
 676                         if (match)
 677                                 log = match + strlen(msg->substr);
 678                 } else {
 679                         err = regexec(&msg->regex, log, 1, reg_match, 0);
 680                         if (err == 0) {
 681                                 match = log + reg_match[0].rm_so;
 682                                 log += reg_match[0].rm_eo;
 683                         }
 684                 }
 685
 686                 if (match) {
 687                         for (; prev_match < match; ++prev_match)
 688                                 if (*prev_match == '\n')
 689                                         ++match_line;
 690                         wrong_line = msg->on_next_line && prev_match_line >= 0 &&
 691                                      prev_match_line + 1 != match_line;
 692                 }
 693
 694                 if (!match || wrong_line) {
 695                         PRINT_FAIL("expect_msg\n");
 696                         if (env.verbosity == VERBOSE_NONE)
 697                                 emit_fn(log_buf, true /*force*/);
 698                         for (j = 0; j <= i; j++) {
 699                                 msg = &msgs->patterns[j];
 700                                 if (j < i)
 701                                         pat_status = "MATCHED   ";
 702                                 else if (wrong_line)
 703                                         pat_status = "WRONG LINE";
 704                                 else
 705                                         pat_status = "EXPECTED  ";
 706                                 msg = &msgs->patterns[j];
 707                                 fprintf(stderr, "%s %s: '%s'\n",
 708                                         pat_status,
 709                                         msg->is_regex ? " REGEX" : "SUBSTR",
 710                                         msg->substr);
 711                         }
 712                         if (wrong_line) {
 713                                 fprintf(stderr,
 714                                         "expecting match at line %d, actual match is at line %d\n",
 715                                         prev_match_line + 1, match_line);
 716                         }
 717                         break;
 718                 }
 719
 720                 prev_match_line = match_line;
 721         }
 722 }
 723
 724 struct cap_state {
 725         __u64 old_caps;
 726         bool initialized;
 727 };
 728
 729 static int drop_capabilities(struct cap_state *caps)
 730 {
 731         const __u64 caps_to_drop = (1ULL << CAP_SYS_ADMIN | 1ULL << CAP_NET_ADMIN |
 732                                     1ULL << CAP_PERFMON   | 1ULL << CAP_BPF);
 733         int err;
 734
 735         err = cap_disable_effective(caps_to_drop, &caps->old_caps);
 736         if (err) {
 737                 PRINT_FAIL("failed to drop capabilities: %i, %s\n", err, strerror(err));
 738                 return err;
 739         }
 740
 741         caps->initialized = true;
 742         return 0;
 743 }
 744
 745 static int restore_capabilities(struct cap_state *caps)
 746 {
 747         int err;
 748
 749         if (!caps->initialized)
 750                 return 0;
 751
 752         err = cap_enable_effective(caps->old_caps, NULL);
 753         if (err)
 754                 PRINT_FAIL("failed to restore capabilities: %i, %s\n", err, strerror(err));
 755         caps->initialized = false;
 756         return err;
 757 }
 758
 759 static bool can_execute_unpriv(struct test_loader *tester, struct test_spec *spec)
 760 {
 761         if (sysctl_unpriv_disabled < 0)
 762                 sysctl_unpriv_disabled = get_unpriv_disabled() ? 1 : 0;
 763         if (sysctl_unpriv_disabled)
 764                 return false;
 765         if ((spec->prog_flags & BPF_F_ANY_ALIGNMENT) && !EFFICIENT_UNALIGNED_ACCESS)
 766                 return false;
 767         return true;
 768 }
 769
 770 static bool is_unpriv_capable_map(struct bpf_map *map)
 771 {
 772         enum bpf_map_type type;
 773         __u32 flags;
 774
 775         type = bpf_map__type(map);
 776
 777         switch (type) {
 778         case BPF_MAP_TYPE_HASH:
 779         case BPF_MAP_TYPE_PERCPU_HASH:
 780         case BPF_MAP_TYPE_HASH_OF_MAPS:
 781                 flags = bpf_map__map_flags(map);
 782                 return !(flags & BPF_F_ZERO_SEED);
 783         case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE:
 784         case BPF_MAP_TYPE_ARRAY:
 785         case BPF_MAP_TYPE_RINGBUF:
 786         case BPF_MAP_TYPE_PROG_ARRAY:
 787         case BPF_MAP_TYPE_CGROUP_ARRAY:
 788         case BPF_MAP_TYPE_PERCPU_ARRAY:
 789         case BPF_MAP_TYPE_USER_RINGBUF:
 790         case BPF_MAP_TYPE_ARRAY_OF_MAPS:
 791         case BPF_MAP_TYPE_CGROUP_STORAGE:
 792         case BPF_MAP_TYPE_PERF_EVENT_ARRAY:
 793                 return true;
 794         default:
 795                 return false;
 796         }
 797 }
 798
 799 static int do_prog_test_run(int fd_prog, int *retval, bool empty_opts)
 800 {
 801         __u8 tmp_out[TEST_DATA_LEN << 2] = {};
 802         __u8 tmp_in[TEST_DATA_LEN] = {};
 803         int err, saved_errno;
 804         LIBBPF_OPTS(bpf_test_run_opts, topts,
 805                 .data_in = tmp_in,
 806                 .data_size_in = sizeof(tmp_in),
 807                 .data_out = tmp_out,
 808                 .data_size_out = sizeof(tmp_out),
 809                 .repeat = 1,
 810         );
 811
 812         if (empty_opts) {
 813                 memset(&topts, 0, sizeof(struct bpf_test_run_opts));
 814                 topts.sz = sizeof(struct bpf_test_run_opts);
 815         }
 816         err = bpf_prog_test_run_opts(fd_prog, &topts);
 817         saved_errno = errno;
 818
 819         if (err) {
 820                 PRINT_FAIL("FAIL: Unexpected bpf_prog_test_run error: %d (%s) ",
 821                            saved_errno, strerror(saved_errno));
 822                 return err;
 823         }
 824
 825         ASSERT_OK(0, "bpf_prog_test_run");
 826         *retval = topts.retval;
 827
 828         return 0;
 829 }
 830
 831 static bool should_do_test_run(struct test_spec *spec, struct test_subspec *subspec)
 832 {
 833         if (!subspec->execute)
 834                 return false;
 835
 836         if (subspec->expect_failure)
 837                 return false;
 838
 839         if ((spec->prog_flags & BPF_F_ANY_ALIGNMENT) && !EFFICIENT_UNALIGNED_ACCESS) {
 840                 if (env.verbosity != VERBOSE_NONE)
 841                         printf("alignment prevents execution\n");
 842                 return false;
 843         }
 844
 845         return true;
 846 }
 847
 848 /* Get a disassembly of BPF program after verifier applies all rewrites */
 849 static int get_xlated_program_text(int prog_fd, char *text, size_t text_sz)
 850 {
 851         struct bpf_insn *insn_start = NULL, *insn, *insn_end;
 852         __u32 insns_cnt = 0, i;
 853         char buf[64];
 854         FILE *out = NULL;
 855         int err;
 856
 857         err = get_xlated_program(prog_fd, &insn_start, &insns_cnt);
 858         if (!ASSERT_OK(err, "get_xlated_program"))
 859                 goto out;
 860         out = fmemopen(text, text_sz, "w");
 861         if (!ASSERT_OK_PTR(out, "open_memstream"))
 862                 goto out;
 863         insn_end = insn_start + insns_cnt;
 864         insn = insn_start;
 865         while (insn < insn_end) {
 866                 i = insn - insn_start;
 867                 insn = disasm_insn(insn, buf, sizeof(buf));
 868                 fprintf(out, "%d: %s\n", i, buf);
 869         }
 870         fflush(out);
 871
 872 out:
 873         free(insn_start);
 874         if (out)
 875                 fclose(out);
 876         return err;
 877 }
 878
 879 /* this function is forced noinline and has short generic name to look better
 880  * in test_progs output (in case of a failure)
 881  */
 882 static noinline
 883 void run_subtest(struct test_loader *tester,
 884                  struct bpf_object_open_opts *open_opts,
 885                  const void *obj_bytes,
 886                  size_t obj_byte_cnt,
 887                  struct test_spec *specs,
 888                  struct test_spec *spec,
 889                  bool unpriv)
 890 {
 891         struct test_subspec *subspec = unpriv ? &spec->unpriv : &spec->priv;
 892         struct bpf_program *tprog = NULL, *tprog_iter;
 893         struct bpf_link *link, *links[32] = {};
 894         struct test_spec *spec_iter;
 895         struct cap_state caps = {};
 896         struct bpf_object *tobj;
 897         struct bpf_map *map;
 898         int retval, err, i;
 899         int links_cnt = 0;
 900         bool should_load;
 901
 902         if (!test__start_subtest(subspec->name))
 903                 return;
 904
 905         if ((get_current_arch() & spec->arch_mask) == 0) {
 906                 test__skip();
 907                 return;
 908         }
 909
 910         if (unpriv) {
 911                 if (!can_execute_unpriv(tester, spec)) {
 912                         test__skip();
 913                         test__end_subtest();
 914                         return;
 915                 }
 916                 if (drop_capabilities(&caps)) {
 917                         test__end_subtest();
 918                         return;
 919                 }
 920         }
 921
 922         /* Implicitly reset to NULL if next test case doesn't specify */
 923         open_opts->btf_custom_path = spec->btf_custom_path;
 924
 925         tobj = bpf_object__open_mem(obj_bytes, obj_byte_cnt, open_opts);
 926         if (!ASSERT_OK_PTR(tobj, "obj_open_mem")) /* shouldn't happen */
 927                 goto subtest_cleanup;
 928
 929         i = 0;
 930         bpf_object__for_each_program(tprog_iter, tobj) {
 931                 spec_iter = &specs[i++];
 932                 should_load = false;
 933
 934                 if (spec_iter->valid) {
 935                         if (strcmp(bpf_program__name(tprog_iter), spec->prog_name) == 0) {
 936                                 tprog = tprog_iter;
 937                                 should_load = true;
 938                         }
 939
 940                         if (spec_iter->auxiliary &&
 941                             spec_iter->mode_mask & (unpriv ? UNPRIV : PRIV))
 942                                 should_load = true;
 943                 }
 944
 945                 bpf_program__set_autoload(tprog_iter, should_load);
 946         }
 947
 948         prepare_case(tester, spec, tobj, tprog);
 949
 950         /* By default bpf_object__load() automatically creates all
 951          * maps declared in the skeleton. Some map types are only
 952          * allowed in priv mode. Disable autoload for such maps in
 953          * unpriv mode.
 954          */
 955         bpf_object__for_each_map(map, tobj)
 956                 bpf_map__set_autocreate(map, !unpriv || is_unpriv_capable_map(map));
 957
 958         err = bpf_object__load(tobj);
 959         if (subspec->expect_failure) {
 960                 if (!ASSERT_ERR(err, "unexpected_load_success")) {
 961                         emit_verifier_log(tester->log_buf, false /*force*/);
 962                         goto tobj_cleanup;
 963                 }
 964         } else {
 965                 if (!ASSERT_OK(err, "unexpected_load_failure")) {
 966                         emit_verifier_log(tester->log_buf, true /*force*/);
 967                         goto tobj_cleanup;
 968                 }
 969         }
 970         emit_verifier_log(tester->log_buf, false /*force*/);
 971         validate_msgs(tester->log_buf, &subspec->expect_msgs, emit_verifier_log);
 972
 973         if (subspec->expect_xlated.cnt) {
 974                 err = get_xlated_program_text(bpf_program__fd(tprog),
 975                                               tester->log_buf, tester->log_buf_sz);
 976                 if (err)
 977                         goto tobj_cleanup;
 978                 emit_xlated(tester->log_buf, false /*force*/);
 979                 validate_msgs(tester->log_buf, &subspec->expect_xlated, emit_xlated);
 980         }
 981
 982         if (subspec->jited.cnt) {
 983                 err = get_jited_program_text(bpf_program__fd(tprog),
 984                                              tester->log_buf, tester->log_buf_sz);
 985                 if (err == -EOPNOTSUPP) {
 986                         printf("%s:SKIP: jited programs disassembly is not supported,\n", __func__);
 987                         printf("%s:SKIP: tests are built w/o LLVM development libs\n", __func__);
 988                         test__skip();
 989                         goto tobj_cleanup;
 990                 }
 991                 if (!ASSERT_EQ(err, 0, "get_jited_program_text"))
 992                         goto tobj_cleanup;
 993                 emit_jited(tester->log_buf, false /*force*/);
 994                 validate_msgs(tester->log_buf, &subspec->jited, emit_jited);
 995         }
 996
 997         if (should_do_test_run(spec, subspec)) {
 998                 /* For some reason test_verifier executes programs
 999                  * with all capabilities restored. Do the same here.
1000                  */
1001                 if (restore_capabilities(&caps))
1002                         goto tobj_cleanup;
1003
1004                 /* Do bpf_map__attach_struct_ops() for each struct_ops map.
1005                  * This should trigger bpf_struct_ops->reg callback on kernel side.
1006                  */
1007                 bpf_object__for_each_map(map, tobj) {
1008                         if (!bpf_map__autocreate(map) ||
1009                             bpf_map__type(map) != BPF_MAP_TYPE_STRUCT_OPS)
1010                                 continue;
1011                         if (links_cnt >= ARRAY_SIZE(links)) {
1012                                 PRINT_FAIL("too many struct_ops maps");
1013                                 goto tobj_cleanup;
1014                         }
1015                         link = bpf_map__attach_struct_ops(map);
1016                         if (!link) {
1017                                 PRINT_FAIL("bpf_map__attach_struct_ops failed for map %s: err=%d\n",
1018                                            bpf_map__name(map), err);
1019                                 goto tobj_cleanup;
1020                         }
1021                         links[links_cnt++] = link;
1022                 }
1023
1024                 if (tester->pre_execution_cb) {
1025                         err = tester->pre_execution_cb(tobj);
1026                         if (err) {
1027                                 PRINT_FAIL("pre_execution_cb failed: %d\n", err);
1028                                 goto tobj_cleanup;
1029                         }
1030                 }
1031
1032                 do_prog_test_run(bpf_program__fd(tprog), &retval,
1033                                  bpf_program__type(tprog) == BPF_PROG_TYPE_SYSCALL ? true : false);
1034                 if (retval != subspec->retval && subspec->retval != POINTER_VALUE) {
1035                         PRINT_FAIL("Unexpected retval: %d != %d\n", retval, subspec->retval);
1036                         goto tobj_cleanup;
1037                 }
1038                 /* redo bpf_map__attach_struct_ops for each test */
1039                 while (links_cnt > 0)
1040                         bpf_link__destroy(links[--links_cnt]);
1041         }
1042
1043 tobj_cleanup:
1044         while (links_cnt > 0)
1045                 bpf_link__destroy(links[--links_cnt]);
1046         bpf_object__close(tobj);
1047 subtest_cleanup:
1048         test__end_subtest();
1049         restore_capabilities(&caps);
1050 }
1051
1052 static void process_subtest(struct test_loader *tester,
1053                             const char *skel_name,
1054                             skel_elf_bytes_fn elf_bytes_factory)
1055 {
1056         LIBBPF_OPTS(bpf_object_open_opts, open_opts, .object_name = skel_name);
1057         struct test_spec *specs = NULL;
1058         struct bpf_object *obj = NULL;
1059         struct bpf_program *prog;
1060         const void *obj_bytes;
1061         int err, i, nr_progs;
1062         size_t obj_byte_cnt;
1063
1064         if (tester_init(tester) < 0)
1065                 return; /* failed to initialize tester */
1066
1067         obj_bytes = elf_bytes_factory(&obj_byte_cnt);
1068         obj = bpf_object__open_mem(obj_bytes, obj_byte_cnt, &open_opts);
1069         if (!ASSERT_OK_PTR(obj, "obj_open_mem"))
1070                 return;
1071
1072         nr_progs = 0;
1073         bpf_object__for_each_program(prog, obj)
1074                 ++nr_progs;
1075
1076         specs = calloc(nr_progs, sizeof(struct test_spec));
1077         if (!ASSERT_OK_PTR(specs, "specs_alloc"))
1078                 return;
1079
1080         i = 0;
1081         bpf_object__for_each_program(prog, obj) {
1082                 /* ignore tests for which  we can't derive test specification */
1083                 err = parse_test_spec(tester, obj, prog, &specs[i++]);
1084                 if (err)
1085                         PRINT_FAIL("Can't parse test spec for program '%s'\n",
1086                                    bpf_program__name(prog));
1087         }
1088
1089         i = 0;
1090         bpf_object__for_each_program(prog, obj) {
1091                 struct test_spec *spec = &specs[i++];
1092
1093                 if (!spec->valid || spec->auxiliary)
1094                         continue;
1095
1096                 if (spec->mode_mask & PRIV)
1097                         run_subtest(tester, &open_opts, obj_bytes, obj_byte_cnt,
1098                                     specs, spec, false);
1099                 if (spec->mode_mask & UNPRIV)
1100                         run_subtest(tester, &open_opts, obj_bytes, obj_byte_cnt,
1101                                     specs, spec, true);
1102
1103         }
1104
1105         for (i = 0; i < nr_progs; ++i)
1106                 free_test_spec(&specs[i]);
1107         free(specs);
1108         bpf_object__close(obj);
1109 }
1110
1111 void test_loader__run_subtests(struct test_loader *tester,
1112                                const char *skel_name,
1113                                skel_elf_bytes_fn elf_bytes_factory)
1114 {
1115         /* see comment in run_subtest() for why we do this function nesting */
1116         process_subtest(tester, skel_name, elf_bytes_factory);
1117 }