tools/testing/selftests/bpf/test_select_reuseport.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (c) 2018 Facebook */
   3
   4 #include <stdlib.h>
   5 #include <unistd.h>
   6 #include <stdbool.h>
   7 #include <string.h>
   8 #include <errno.h>
   9 #include <assert.h>
  10 #include <fcntl.h>
  11 #include <linux/bpf.h>
  12 #include <linux/err.h>
  13 #include <linux/types.h>
  14 #include <linux/if_ether.h>
  15 #include <sys/types.h>
  16 #include <sys/epoll.h>
  17 #include <sys/socket.h>
  18 #include <netinet/in.h>
  19 #include <bpf/bpf.h>
  20 #include <bpf/libbpf.h>
  21 #include "bpf_rlimit.h"
  22 #include "bpf_util.h"
  23 #include "test_select_reuseport_common.h"
  24
  25 #define MIN_TCPHDR_LEN 20
  26 #define UDPHDR_LEN 8
  27
  28 #define TCP_SYNCOOKIE_SYSCTL "/proc/sys/net/ipv4/tcp_syncookies"
  29 #define TCP_FO_SYSCTL "/proc/sys/net/ipv4/tcp_fastopen"
  30 #define REUSEPORT_ARRAY_SIZE 32
  31
  32 static int result_map, tmp_index_ovr_map, linum_map, data_check_map;
  33 static enum result expected_results[NR_RESULTS];
  34 static int sk_fds[REUSEPORT_ARRAY_SIZE];
  35 static int reuseport_array, outer_map;
  36 static int select_by_skb_data_prog;
  37 static int saved_tcp_syncookie;
  38 static struct bpf_object *obj;
  39 static int saved_tcp_fo;
  40 static __u32 index_zero;
  41 static int epfd;
  42
  43 static union sa46 {
  44         struct sockaddr_in6 v6;
  45         struct sockaddr_in v4;
  46         sa_family_t family;
  47 } srv_sa;
  48
  49 #define CHECK(condition, tag, format...) ({                             \
  50         int __ret = !!(condition);                                      \
  51         if (__ret) {                                                    \
  52                 printf("%s(%d):FAIL:%s ", __func__, __LINE__, tag);     \
  53                 printf(format);                                         \
  54                 exit(-1);                                               \
  55         }                                                               \
  56 })
  57
  58 static void create_maps(void)
  59 {
  60         struct bpf_create_map_attr attr = {};
  61
  62         /* Creating reuseport_array */
  63         attr.name = "reuseport_array";
  64         attr.map_type = BPF_MAP_TYPE_REUSEPORT_SOCKARRAY;
  65         attr.key_size = sizeof(__u32);
  66         attr.value_size = sizeof(__u32);
  67         attr.max_entries = REUSEPORT_ARRAY_SIZE;
  68
  69         reuseport_array = bpf_create_map_xattr(&attr);
  70         CHECK(reuseport_array == -1, "creating reuseport_array",
  71               "reuseport_array:%d errno:%d\n", reuseport_array, errno);
  72
  73         /* Creating outer_map */
  74         attr.name = "outer_map";
  75         attr.map_type = BPF_MAP_TYPE_ARRAY_OF_MAPS;
  76         attr.key_size = sizeof(__u32);
  77         attr.value_size = sizeof(__u32);
  78         attr.max_entries = 1;
  79         attr.inner_map_fd = reuseport_array;
  80         outer_map = bpf_create_map_xattr(&attr);
  81         CHECK(outer_map == -1, "creating outer_map",
  82               "outer_map:%d errno:%d\n", outer_map, errno);
  83 }
  84
  85 static void prepare_bpf_obj(void)
  86 {
  87         struct bpf_program *prog;
  88         struct bpf_map *map;
  89         int err;
  90         struct bpf_object_open_attr attr = {
  91                 .file = "test_select_reuseport_kern.o",
  92                 .prog_type = BPF_PROG_TYPE_SK_REUSEPORT,
  93         };
  94
  95         obj = bpf_object__open_xattr(&attr);
  96         CHECK(IS_ERR_OR_NULL(obj), "open test_select_reuseport_kern.o",
  97               "obj:%p PTR_ERR(obj):%ld\n", obj, PTR_ERR(obj));
  98
  99         prog = bpf_program__next(NULL, obj);
 100         CHECK(!prog, "get first bpf_program", "!prog\n");
 101         bpf_program__set_type(prog, attr.prog_type);
 102
 103         map = bpf_object__find_map_by_name(obj, "outer_map");
 104         CHECK(!map, "find outer_map", "!map\n");
 105         err = bpf_map__reuse_fd(map, outer_map);
 106         CHECK(err, "reuse outer_map", "err:%d\n", err);
 107
 108         err = bpf_object__load(obj);
 109         CHECK(err, "load bpf_object", "err:%d\n", err);
 110
 111         select_by_skb_data_prog = bpf_program__fd(prog);
 112         CHECK(select_by_skb_data_prog == -1, "get prog fd",
 113               "select_by_skb_data_prog:%d\n", select_by_skb_data_prog);
 114
 115         map = bpf_object__find_map_by_name(obj, "result_map");
 116         CHECK(!map, "find result_map", "!map\n");
 117         result_map = bpf_map__fd(map);
 118         CHECK(result_map == -1, "get result_map fd",
 119               "result_map:%d\n", result_map);
 120
 121         map = bpf_object__find_map_by_name(obj, "tmp_index_ovr_map");
 122         CHECK(!map, "find tmp_index_ovr_map", "!map\n");
 123         tmp_index_ovr_map = bpf_map__fd(map);
 124         CHECK(tmp_index_ovr_map == -1, "get tmp_index_ovr_map fd",
 125               "tmp_index_ovr_map:%d\n", tmp_index_ovr_map);
 126
 127         map = bpf_object__find_map_by_name(obj, "linum_map");
 128         CHECK(!map, "find linum_map", "!map\n");
 129         linum_map = bpf_map__fd(map);
 130         CHECK(linum_map == -1, "get linum_map fd",
 131               "linum_map:%d\n", linum_map);
 132
 133         map = bpf_object__find_map_by_name(obj, "data_check_map");
 134         CHECK(!map, "find data_check_map", "!map\n");
 135         data_check_map = bpf_map__fd(map);
 136         CHECK(data_check_map == -1, "get data_check_map fd",
 137               "data_check_map:%d\n", data_check_map);
 138 }
 139
 140 static void sa46_init_loopback(union sa46 *sa, sa_family_t family)
 141 {
 142         memset(sa, 0, sizeof(*sa));
 143         sa->family = family;
 144         if (sa->family == AF_INET6)
 145                 sa->v6.sin6_addr = in6addr_loopback;
 146         else
 147                 sa->v4.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
 148 }
 149
 150 static void sa46_init_inany(union sa46 *sa, sa_family_t family)
 151 {
 152         memset(sa, 0, sizeof(*sa));
 153         sa->family = family;
 154         if (sa->family == AF_INET6)
 155                 sa->v6.sin6_addr = in6addr_any;
 156         else
 157                 sa->v4.sin_addr.s_addr = INADDR_ANY;
 158 }
 159
 160 static int read_int_sysctl(const char *sysctl)
 161 {
 162         char buf[16];
 163         int fd, ret;
 164
 165         fd = open(sysctl, 0);
 166         CHECK(fd == -1, "open(sysctl)", "sysctl:%s fd:%d errno:%d\n",
 167               sysctl, fd, errno);
 168
 169         ret = read(fd, buf, sizeof(buf));
 170         CHECK(ret <= 0, "read(sysctl)", "sysctl:%s ret:%d errno:%d\n",
 171               sysctl, ret, errno);
 172         close(fd);
 173
 174         return atoi(buf);
 175 }
 176
 177 static void write_int_sysctl(const char *sysctl, int v)
 178 {
 179         int fd, ret, size;
 180         char buf[16];
 181
 182         fd = open(sysctl, O_RDWR);
 183         CHECK(fd == -1, "open(sysctl)", "sysctl:%s fd:%d errno:%d\n",
 184               sysctl, fd, errno);
 185
 186         size = snprintf(buf, sizeof(buf), "%d", v);
 187         ret = write(fd, buf, size);
 188         CHECK(ret != size, "write(sysctl)",
 189               "sysctl:%s ret:%d size:%d errno:%d\n", sysctl, ret, size, errno);
 190         close(fd);
 191 }
 192
 193 static void restore_sysctls(void)
 194 {
 195         write_int_sysctl(TCP_FO_SYSCTL, saved_tcp_fo);
 196         write_int_sysctl(TCP_SYNCOOKIE_SYSCTL, saved_tcp_syncookie);
 197 }
 198
 199 static void enable_fastopen(void)
 200 {
 201         int fo;
 202
 203         fo = read_int_sysctl(TCP_FO_SYSCTL);
 204         write_int_sysctl(TCP_FO_SYSCTL, fo | 7);
 205 }
 206
 207 static void enable_syncookie(void)
 208 {
 209         write_int_sysctl(TCP_SYNCOOKIE_SYSCTL, 2);
 210 }
 211
 212 static void disable_syncookie(void)
 213 {
 214         write_int_sysctl(TCP_SYNCOOKIE_SYSCTL, 0);
 215 }
 216
 217 static __u32 get_linum(void)
 218 {
 219         __u32 linum;
 220         int err;
 221
 222         err = bpf_map_lookup_elem(linum_map, &index_zero, &linum);
 223         CHECK(err == -1, "lookup_elem(linum_map)", "err:%d errno:%d\n",
 224               err, errno);
 225
 226         return linum;
 227 }
 228
 229 static void check_data(int type, sa_family_t family, const struct cmd *cmd,
 230                        int cli_fd)
 231 {
 232         struct data_check expected = {}, result;
 233         union sa46 cli_sa;
 234         socklen_t addrlen;
 235         int err;
 236
 237         addrlen = sizeof(cli_sa);
 238         err = getsockname(cli_fd, (struct sockaddr *)&cli_sa,
 239                           &addrlen);
 240         CHECK(err == -1, "getsockname(cli_fd)", "err:%d errno:%d\n",
 241               err, errno);
 242
 243         err = bpf_map_lookup_elem(data_check_map, &index_zero, &result);
 244         CHECK(err == -1, "lookup_elem(data_check_map)", "err:%d errno:%d\n",
 245               err, errno);
 246
 247         if (type == SOCK_STREAM) {
 248                 expected.len = MIN_TCPHDR_LEN;
 249                 expected.ip_protocol = IPPROTO_TCP;
 250         } else {
 251                 expected.len = UDPHDR_LEN;
 252                 expected.ip_protocol = IPPROTO_UDP;
 253         }
 254
 255         if (family == AF_INET6) {
 256                 expected.eth_protocol = htons(ETH_P_IPV6);
 257                 expected.bind_inany = !srv_sa.v6.sin6_addr.s6_addr32[3] &&
 258                         !srv_sa.v6.sin6_addr.s6_addr32[2] &&
 259                         !srv_sa.v6.sin6_addr.s6_addr32[1] &&
 260                         !srv_sa.v6.sin6_addr.s6_addr32[0];
 261
 262                 memcpy(&expected.skb_addrs[0], cli_sa.v6.sin6_addr.s6_addr32,
 263                        sizeof(cli_sa.v6.sin6_addr));
 264                 memcpy(&expected.skb_addrs[4], &in6addr_loopback,
 265                        sizeof(in6addr_loopback));
 266                 expected.skb_ports[0] = cli_sa.v6.sin6_port;
 267                 expected.skb_ports[1] = srv_sa.v6.sin6_port;
 268         } else {
 269                 expected.eth_protocol = htons(ETH_P_IP);
 270                 expected.bind_inany = !srv_sa.v4.sin_addr.s_addr;
 271
 272                 expected.skb_addrs[0] = cli_sa.v4.sin_addr.s_addr;
 273                 expected.skb_addrs[1] = htonl(INADDR_LOOPBACK);
 274                 expected.skb_ports[0] = cli_sa.v4.sin_port;
 275                 expected.skb_ports[1] = srv_sa.v4.sin_port;
 276         }
 277
 278         if (memcmp(&result, &expected, offsetof(struct data_check,
 279                                                 equal_check_end))) {
 280                 printf("unexpected data_check\n");
 281                 printf("  result: (0x%x, %u, %u)\n",
 282                        result.eth_protocol, result.ip_protocol,
 283                        result.bind_inany);
 284                 printf("expected: (0x%x, %u, %u)\n",
 285                        expected.eth_protocol, expected.ip_protocol,
 286                        expected.bind_inany);
 287                 CHECK(1, "data_check result != expected",
 288                       "bpf_prog_linum:%u\n", get_linum());
 289         }
 290
 291         CHECK(!result.hash, "data_check result.hash empty",
 292               "result.hash:%u", result.hash);
 293
 294         expected.len += cmd ? sizeof(*cmd) : 0;
 295         if (type == SOCK_STREAM)
 296                 CHECK(expected.len > result.len, "expected.len > result.len",
 297                       "expected.len:%u result.len:%u bpf_prog_linum:%u\n",
 298                       expected.len, result.len, get_linum());
 299         else
 300                 CHECK(expected.len != result.len, "expected.len != result.len",
 301                       "expected.len:%u result.len:%u bpf_prog_linum:%u\n",
 302                       expected.len, result.len, get_linum());
 303 }
 304
 305 static void check_results(void)
 306 {
 307         __u32 results[NR_RESULTS];
 308         __u32 i, broken = 0;
 309         int err;
 310
 311         for (i = 0; i < NR_RESULTS; i++) {
 312                 err = bpf_map_lookup_elem(result_map, &i, &results[i]);
 313                 CHECK(err == -1, "lookup_elem(result_map)",
 314                       "i:%u err:%d errno:%d\n", i, err, errno);
 315         }
 316
 317         for (i = 0; i < NR_RESULTS; i++) {
 318                 if (results[i] != expected_results[i]) {
 319                         broken = i;
 320                         break;
 321                 }
 322         }
 323
 324         if (i == NR_RESULTS)
 325                 return;
 326
 327         printf("unexpected result\n");
 328         printf(" result: [");
 329         printf("%u", results[0]);
 330         for (i = 1; i < NR_RESULTS; i++)
 331                 printf(", %u", results[i]);
 332         printf("]\n");
 333
 334         printf("expected: [");
 335         printf("%u", expected_results[0]);
 336         for (i = 1; i < NR_RESULTS; i++)
 337                 printf(", %u", expected_results[i]);
 338         printf("]\n");
 339
 340         CHECK(expected_results[broken] != results[broken],
 341               "unexpected result",
 342               "expected_results[%u] != results[%u] bpf_prog_linum:%u\n",
 343               broken, broken, get_linum());
 344 }
 345
 346 static int send_data(int type, sa_family_t family, void *data, size_t len,
 347                      enum result expected)
 348 {
 349         union sa46 cli_sa;
 350         int fd, err;
 351
 352         fd = socket(family, type, 0);
 353         CHECK(fd == -1, "socket()", "fd:%d errno:%d\n", fd, errno);
 354
 355         sa46_init_loopback(&cli_sa, family);
 356         err = bind(fd, (struct sockaddr *)&cli_sa, sizeof(cli_sa));
 357         CHECK(fd == -1, "bind(cli_sa)", "err:%d errno:%d\n", err, errno);
 358
 359         err = sendto(fd, data, len, MSG_FASTOPEN, (struct sockaddr *)&srv_sa,
 360                      sizeof(srv_sa));
 361         CHECK(err != len && expected >= PASS,
 362               "sendto()", "family:%u err:%d errno:%d expected:%d\n",
 363               family, err, errno, expected);
 364
 365         return fd;
 366 }
 367
 368 static void do_test(int type, sa_family_t family, struct cmd *cmd,
 369                     enum result expected)
 370 {
 371         int nev, srv_fd, cli_fd;
 372         struct epoll_event ev;
 373         struct cmd rcv_cmd;
 374         ssize_t nread;
 375
 376         cli_fd = send_data(type, family, cmd, cmd ? sizeof(*cmd) : 0,
 377                            expected);
 378         nev = epoll_wait(epfd, &ev, 1, expected >= PASS ? 5 : 0);
 379         CHECK((nev <= 0 && expected >= PASS) ||
 380               (nev > 0 && expected < PASS),
 381               "nev <> expected",
 382               "nev:%d expected:%d type:%d family:%d data:(%d, %d)\n",
 383               nev, expected, type, family,
 384               cmd ? cmd->reuseport_index : -1,
 385               cmd ? cmd->pass_on_failure : -1);
 386         check_results();
 387         check_data(type, family, cmd, cli_fd);
 388
 389         if (expected < PASS)
 390                 return;
 391
 392         CHECK(expected != PASS_ERR_SK_SELECT_REUSEPORT &&
 393               cmd->reuseport_index != ev.data.u32,
 394               "check cmd->reuseport_index",
 395               "cmd:(%u, %u) ev.data.u32:%u\n",
 396               cmd->pass_on_failure, cmd->reuseport_index, ev.data.u32);
 397
 398         srv_fd = sk_fds[ev.data.u32];
 399         if (type == SOCK_STREAM) {
 400                 int new_fd = accept(srv_fd, NULL, 0);
 401
 402                 CHECK(new_fd == -1, "accept(srv_fd)",
 403                       "ev.data.u32:%u new_fd:%d errno:%d\n",
 404                       ev.data.u32, new_fd, errno);
 405
 406                 nread = recv(new_fd, &rcv_cmd, sizeof(rcv_cmd), MSG_DONTWAIT);
 407                 CHECK(nread != sizeof(rcv_cmd),
 408                       "recv(new_fd)",
 409                       "ev.data.u32:%u nread:%zd sizeof(rcv_cmd):%zu errno:%d\n",
 410                       ev.data.u32, nread, sizeof(rcv_cmd), errno);
 411
 412                 close(new_fd);
 413         } else {
 414                 nread = recv(srv_fd, &rcv_cmd, sizeof(rcv_cmd), MSG_DONTWAIT);
 415                 CHECK(nread != sizeof(rcv_cmd),
 416                       "recv(sk_fds)",
 417                       "ev.data.u32:%u nread:%zd sizeof(rcv_cmd):%zu errno:%d\n",
 418                       ev.data.u32, nread, sizeof(rcv_cmd), errno);
 419         }
 420
 421         close(cli_fd);
 422 }
 423
 424 static void test_err_inner_map(int type, sa_family_t family)
 425 {
 426         struct cmd cmd = {
 427                 .reuseport_index = 0,
 428                 .pass_on_failure = 0,
 429         };
 430
 431         printf("%s: ", __func__);
 432         expected_results[DROP_ERR_INNER_MAP]++;
 433         do_test(type, family, &cmd, DROP_ERR_INNER_MAP);
 434         printf("OK\n");
 435 }
 436
 437 static void test_err_skb_data(int type, sa_family_t family)
 438 {
 439         printf("%s: ", __func__);
 440         expected_results[DROP_ERR_SKB_DATA]++;
 441         do_test(type, family, NULL, DROP_ERR_SKB_DATA);
 442         printf("OK\n");
 443 }
 444
 445 static void test_err_sk_select_port(int type, sa_family_t family)
 446 {
 447         struct cmd cmd = {
 448                 .reuseport_index = REUSEPORT_ARRAY_SIZE,
 449                 .pass_on_failure = 0,
 450         };
 451
 452         printf("%s: ", __func__);
 453         expected_results[DROP_ERR_SK_SELECT_REUSEPORT]++;
 454         do_test(type, family, &cmd, DROP_ERR_SK_SELECT_REUSEPORT);
 455         printf("OK\n");
 456 }
 457
 458 static void test_pass(int type, sa_family_t family)
 459 {
 460         struct cmd cmd;
 461         int i;
 462
 463         printf("%s: ", __func__);
 464         cmd.pass_on_failure = 0;
 465         for (i = 0; i < REUSEPORT_ARRAY_SIZE; i++) {
 466                 expected_results[PASS]++;
 467                 cmd.reuseport_index = i;
 468                 do_test(type, family, &cmd, PASS);
 469         }
 470         printf("OK\n");
 471 }
 472
 473 static void test_syncookie(int type, sa_family_t family)
 474 {
 475         int err, tmp_index = 1;
 476         struct cmd cmd = {
 477                 .reuseport_index = 0,
 478                 .pass_on_failure = 0,
 479         };
 480
 481         if (type != SOCK_STREAM)
 482                 return;
 483
 484         printf("%s: ", __func__);
 485         /*
 486          * +1 for TCP-SYN and
 487          * +1 for the TCP-ACK (ack the syncookie)
 488          */
 489         expected_results[PASS] += 2;
 490         enable_syncookie();
 491         /*
 492          * Simulate TCP-SYN and TCP-ACK are handled by two different sk:
 493          * TCP-SYN: select sk_fds[tmp_index = 1] tmp_index is from the
 494          *          tmp_index_ovr_map
 495          * TCP-ACK: select sk_fds[reuseport_index = 0] reuseport_index
 496          *          is from the cmd.reuseport_index
 497          */
 498         err = bpf_map_update_elem(tmp_index_ovr_map, &index_zero,
 499                                   &tmp_index, BPF_ANY);
 500         CHECK(err == -1, "update_elem(tmp_index_ovr_map, 0, 1)",
 501               "err:%d errno:%d\n", err, errno);
 502         do_test(type, family, &cmd, PASS);
 503         err = bpf_map_lookup_elem(tmp_index_ovr_map, &index_zero,
 504                                   &tmp_index);
 505         CHECK(err == -1 || tmp_index != -1,
 506               "lookup_elem(tmp_index_ovr_map)",
 507               "err:%d errno:%d tmp_index:%d\n",
 508               err, errno, tmp_index);
 509         disable_syncookie();
 510         printf("OK\n");
 511 }
 512
 513 static void test_pass_on_err(int type, sa_family_t family)
 514 {
 515         struct cmd cmd = {
 516                 .reuseport_index = REUSEPORT_ARRAY_SIZE,
 517                 .pass_on_failure = 1,
 518         };
 519
 520         printf("%s: ", __func__);
 521         expected_results[PASS_ERR_SK_SELECT_REUSEPORT] += 1;
 522         do_test(type, family, &cmd, PASS_ERR_SK_SELECT_REUSEPORT);
 523         printf("OK\n");
 524 }
 525
 526 static void test_detach_bpf(int type, sa_family_t family)
 527 {
 528 #ifdef SO_DETACH_REUSEPORT_BPF
 529         __u32 nr_run_before = 0, nr_run_after = 0, tmp, i;
 530         struct epoll_event ev;
 531         int cli_fd, err, nev;
 532         struct cmd cmd = {};
 533         int optvalue = 0;
 534
 535         printf("%s: ", __func__);
 536         err = setsockopt(sk_fds[0], SOL_SOCKET, SO_DETACH_REUSEPORT_BPF,
 537                          &optvalue, sizeof(optvalue));
 538         CHECK(err == -1, "setsockopt(SO_DETACH_REUSEPORT_BPF)",
 539               "err:%d errno:%d\n", err, errno);
 540
 541         err = setsockopt(sk_fds[1], SOL_SOCKET, SO_DETACH_REUSEPORT_BPF,
 542                          &optvalue, sizeof(optvalue));
 543         CHECK(err == 0 || errno != ENOENT, "setsockopt(SO_DETACH_REUSEPORT_BPF)",
 544               "err:%d errno:%d\n", err, errno);
 545
 546         for (i = 0; i < NR_RESULTS; i++) {
 547                 err = bpf_map_lookup_elem(result_map, &i, &tmp);
 548                 CHECK(err == -1, "lookup_elem(result_map)",
 549                       "i:%u err:%d errno:%d\n", i, err, errno);
 550                 nr_run_before += tmp;
 551         }
 552
 553         cli_fd = send_data(type, family, &cmd, sizeof(cmd), PASS);
 554         nev = epoll_wait(epfd, &ev, 1, 5);
 555         CHECK(nev <= 0, "nev <= 0",
 556               "nev:%d expected:1 type:%d family:%d data:(0, 0)\n",
 557               nev,  type, family);
 558
 559         for (i = 0; i < NR_RESULTS; i++) {
 560                 err = bpf_map_lookup_elem(result_map, &i, &tmp);
 561                 CHECK(err == -1, "lookup_elem(result_map)",
 562                       "i:%u err:%d errno:%d\n", i, err, errno);
 563                 nr_run_after += tmp;
 564         }
 565
 566         CHECK(nr_run_before != nr_run_after,
 567               "nr_run_before != nr_run_after",
 568               "nr_run_before:%u nr_run_after:%u\n",
 569               nr_run_before, nr_run_after);
 570
 571         printf("OK\n");
 572         close(cli_fd);
 573 #else
 574         printf("%s: SKIP\n", __func__);
 575 #endif
 576 }
 577
 578 static void prepare_sk_fds(int type, sa_family_t family, bool inany)
 579 {
 580         const int first = REUSEPORT_ARRAY_SIZE - 1;
 581         int i, err, optval = 1;
 582         struct epoll_event ev;
 583         socklen_t addrlen;
 584
 585         if (inany)
 586                 sa46_init_inany(&srv_sa, family);
 587         else
 588                 sa46_init_loopback(&srv_sa, family);
 589         addrlen = sizeof(srv_sa);
 590
 591         /*
 592          * The sk_fds[] is filled from the back such that the order
 593          * is exactly opposite to the (struct sock_reuseport *)reuse->socks[].
 594          */
 595         for (i = first; i >= 0; i--) {
 596                 sk_fds[i] = socket(family, type, 0);
 597                 CHECK(sk_fds[i] == -1, "socket()", "sk_fds[%d]:%d errno:%d\n",
 598                       i, sk_fds[i], errno);
 599                 err = setsockopt(sk_fds[i], SOL_SOCKET, SO_REUSEPORT,
 600                                  &optval, sizeof(optval));
 601                 CHECK(err == -1, "setsockopt(SO_REUSEPORT)",
 602                       "sk_fds[%d] err:%d errno:%d\n",
 603                       i, err, errno);
 604
 605                 if (i == first) {
 606                         err = setsockopt(sk_fds[i], SOL_SOCKET,
 607                                          SO_ATTACH_REUSEPORT_EBPF,
 608                                          &select_by_skb_data_prog,
 609                                          sizeof(select_by_skb_data_prog));
 610                         CHECK(err == -1, "setsockopt(SO_ATTACH_REUEPORT_EBPF)",
 611                               "err:%d errno:%d\n", err, errno);
 612                 }
 613
 614                 err = bind(sk_fds[i], (struct sockaddr *)&srv_sa, addrlen);
 615                 CHECK(err == -1, "bind()", "sk_fds[%d] err:%d errno:%d\n",
 616                       i, err, errno);
 617
 618                 if (type == SOCK_STREAM) {
 619                         err = listen(sk_fds[i], 10);
 620                         CHECK(err == -1, "listen()",
 621                               "sk_fds[%d] err:%d errno:%d\n",
 622                               i, err, errno);
 623                 }
 624
 625                 err = bpf_map_update_elem(reuseport_array, &i, &sk_fds[i],
 626                                           BPF_NOEXIST);
 627                 CHECK(err == -1, "update_elem(reuseport_array)",
 628                       "sk_fds[%d] err:%d errno:%d\n", i, err, errno);
 629
 630                 if (i == first) {
 631                         socklen_t addrlen = sizeof(srv_sa);
 632
 633                         err = getsockname(sk_fds[i], (struct sockaddr *)&srv_sa,
 634                                           &addrlen);
 635                         CHECK(err == -1, "getsockname()",
 636                               "sk_fds[%d] err:%d errno:%d\n", i, err, errno);
 637                 }
 638         }
 639
 640         epfd = epoll_create(1);
 641         CHECK(epfd == -1, "epoll_create(1)",
 642               "epfd:%d errno:%d\n", epfd, errno);
 643
 644         ev.events = EPOLLIN;
 645         for (i = 0; i < REUSEPORT_ARRAY_SIZE; i++) {
 646                 ev.data.u32 = i;
 647                 err = epoll_ctl(epfd, EPOLL_CTL_ADD, sk_fds[i], &ev);
 648                 CHECK(err, "epoll_ctl(EPOLL_CTL_ADD)", "sk_fds[%d]\n", i);
 649         }
 650 }
 651
 652 static void setup_per_test(int type, unsigned short family, bool inany)
 653 {
 654         int ovr = -1, err;
 655
 656         prepare_sk_fds(type, family, inany);
 657         err = bpf_map_update_elem(tmp_index_ovr_map, &index_zero, &ovr,
 658                                   BPF_ANY);
 659         CHECK(err == -1, "update_elem(tmp_index_ovr_map, 0, -1)",
 660               "err:%d errno:%d\n", err, errno);
 661 }
 662
 663 static void cleanup_per_test(void)
 664 {
 665         int i, err;
 666
 667         for (i = 0; i < REUSEPORT_ARRAY_SIZE; i++)
 668                 close(sk_fds[i]);
 669         close(epfd);
 670
 671         err = bpf_map_delete_elem(outer_map, &index_zero);
 672         CHECK(err == -1, "delete_elem(outer_map)",
 673               "err:%d errno:%d\n", err, errno);
 674 }
 675
 676 static void cleanup(void)
 677 {
 678         close(outer_map);
 679         close(reuseport_array);
 680         bpf_object__close(obj);
 681 }
 682
 683 static void test_all(void)
 684 {
 685         /* Extra SOCK_STREAM to test bind_inany==true */
 686         const int types[] = { SOCK_STREAM, SOCK_DGRAM, SOCK_STREAM };
 687         const char * const type_strings[] = { "TCP", "UDP", "TCP" };
 688         const char * const family_strings[] = { "IPv6", "IPv4" };
 689         const unsigned short families[] = { AF_INET6, AF_INET };
 690         const bool bind_inany[] = { false, false, true };
 691         int t, f, err;
 692
 693         for (f = 0; f < ARRAY_SIZE(families); f++) {
 694                 unsigned short family = families[f];
 695
 696                 for (t = 0; t < ARRAY_SIZE(types); t++) {
 697                         bool inany = bind_inany[t];
 698                         int type = types[t];
 699
 700                         printf("######## %s/%s %s ########\n",
 701                                family_strings[f], type_strings[t],
 702                                 inany ? " INANY  " : "LOOPBACK");
 703
 704                         setup_per_test(type, family, inany);
 705
 706                         test_err_inner_map(type, family);
 707
 708                         /* Install reuseport_array to the outer_map */
 709                         err = bpf_map_update_elem(outer_map, &index_zero,
 710                                                   &reuseport_array, BPF_ANY);
 711                         CHECK(err == -1, "update_elem(outer_map)",
 712                               "err:%d errno:%d\n", err, errno);
 713
 714                         test_err_skb_data(type, family);
 715                         test_err_sk_select_port(type, family);
 716                         test_pass(type, family);
 717                         test_syncookie(type, family);
 718                         test_pass_on_err(type, family);
 719                         /* Must be the last test */
 720                         test_detach_bpf(type, family);
 721
 722                         cleanup_per_test();
 723                         printf("\n");
 724                 }
 725         }
 726 }
 727
 728 int main(int argc, const char **argv)
 729 {
 730         create_maps();
 731         prepare_bpf_obj();
 732         saved_tcp_fo = read_int_sysctl(TCP_FO_SYSCTL);
 733         saved_tcp_syncookie = read_int_sysctl(TCP_SYNCOOKIE_SYSCTL);
 734         enable_fastopen();
 735         disable_syncookie();
 736         atexit(restore_sysctls);
 737
 738         test_all();
 739
 740         cleanup();
 741         return 0;
 742 }