1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) 2017-2018 Covalent IO, Inc. http://covalent.io
5 #include <sys/socket.h>
7 #include <sys/select.h>
8 #include <netinet/in.h>
21 #include <sys/resource.h>
22 #include <sys/types.h>
23 #include <sys/sendfile.h>
25 #include <linux/netlink.h>
26 #include <linux/socket.h>
27 #include <linux/sock_diag.h>
28 #include <linux/bpf.h>
29 #include <linux/if_link.h>
30 #include <linux/tls.h>
37 #include <bpf/libbpf.h>
40 #include "bpf_rlimit.h"
41 #include "cgroup_helpers.h"
44 static void running_handler(int a
);
53 /* randomly selected ports for testing on lo */
57 #define BPF_SOCKMAP_FILENAME "test_sockmap_kern.o"
58 #define BPF_SOCKHASH_FILENAME "test_sockhash_kern.o"
59 #define CG_PATH "/sockmap"
62 int s1
, s2
, c1
, c2
, p1
, p2
;
67 struct bpf_map
*maps
[8];
73 int txmsg_redir_noisy
;
88 static const struct option long_options
[] = {
89 {"help", no_argument
, NULL
, 'h' },
90 {"cgroup", required_argument
, NULL
, 'c' },
91 {"rate", required_argument
, NULL
, 'r' },
92 {"verbose", no_argument
, NULL
, 'v' },
93 {"iov_count", required_argument
, NULL
, 'i' },
94 {"length", required_argument
, NULL
, 'l' },
95 {"test", required_argument
, NULL
, 't' },
96 {"data_test", no_argument
, NULL
, 'd' },
97 {"txmsg", no_argument
, &txmsg_pass
, 1 },
98 {"txmsg_noisy", no_argument
, &txmsg_noisy
, 1 },
99 {"txmsg_redir", no_argument
, &txmsg_redir
, 1 },
100 {"txmsg_redir_noisy", no_argument
, &txmsg_redir_noisy
, 1},
101 {"txmsg_drop", no_argument
, &txmsg_drop
, 1 },
102 {"txmsg_apply", required_argument
, NULL
, 'a'},
103 {"txmsg_cork", required_argument
, NULL
, 'k'},
104 {"txmsg_start", required_argument
, NULL
, 's'},
105 {"txmsg_end", required_argument
, NULL
, 'e'},
106 {"txmsg_start_push", required_argument
, NULL
, 'p'},
107 {"txmsg_end_push", required_argument
, NULL
, 'q'},
108 {"txmsg_start_pop", required_argument
, NULL
, 'w'},
109 {"txmsg_pop", required_argument
, NULL
, 'x'},
110 {"txmsg_ingress", no_argument
, &txmsg_ingress
, 1 },
111 {"txmsg_skb", no_argument
, &txmsg_skb
, 1 },
112 {"ktls", no_argument
, &ktls
, 1 },
113 {"peek", no_argument
, &peek_flag
, 1 },
117 static void usage(char *argv
[])
121 printf(" Usage: %s --cgroup <cgroup_path>\n", argv
[0]);
122 printf(" options:\n");
123 for (i
= 0; long_options
[i
].name
!= 0; i
++) {
124 printf(" --%-12s", long_options
[i
].name
);
125 if (long_options
[i
].flag
!= NULL
)
126 printf(" flag (internal value:%d)\n",
127 *long_options
[i
].flag
);
129 printf(" -%c\n", long_options
[i
].val
);
134 char *sock_to_string(int s
)
152 static int sockmap_init_ktls(int verbose
, int s
)
154 struct tls12_crypto_info_aes_gcm_128 tls_tx
= {
156 .version
= TLS_1_2_VERSION
,
157 .cipher_type
= TLS_CIPHER_AES_GCM_128
,
160 struct tls12_crypto_info_aes_gcm_128 tls_rx
= {
162 .version
= TLS_1_2_VERSION
,
163 .cipher_type
= TLS_CIPHER_AES_GCM_128
,
166 int so_buf
= 6553500;
169 err
= setsockopt(s
, 6, TCP_ULP
, "tls", sizeof("tls"));
171 fprintf(stderr
, "setsockopt: TCP_ULP(%s) failed with error %i\n", sock_to_string(s
), err
);
174 err
= setsockopt(s
, SOL_TLS
, TLS_TX
, (void *)&tls_tx
, sizeof(tls_tx
));
176 fprintf(stderr
, "setsockopt: TLS_TX(%s) failed with error %i\n", sock_to_string(s
), err
);
179 err
= setsockopt(s
, SOL_TLS
, TLS_RX
, (void *)&tls_rx
, sizeof(tls_rx
));
181 fprintf(stderr
, "setsockopt: TLS_RX(%s) failed with error %i\n", sock_to_string(s
), err
);
184 err
= setsockopt(s
, SOL_SOCKET
, SO_SNDBUF
, &so_buf
, sizeof(so_buf
));
186 fprintf(stderr
, "setsockopt: (%s) failed sndbuf with error %i\n", sock_to_string(s
), err
);
189 err
= setsockopt(s
, SOL_SOCKET
, SO_RCVBUF
, &so_buf
, sizeof(so_buf
));
191 fprintf(stderr
, "setsockopt: (%s) failed rcvbuf with error %i\n", sock_to_string(s
), err
);
196 fprintf(stdout
, "socket(%s) kTLS enabled\n", sock_to_string(s
));
199 static int sockmap_init_sockets(int verbose
)
202 struct sockaddr_in addr
;
203 int *fds
[4] = {&s1
, &s2
, &c1
, &c2
};
205 s1
= s2
= p1
= p2
= c1
= c2
= 0;
208 for (i
= 0; i
< 4; i
++) {
209 *fds
[i
] = socket(AF_INET
, SOCK_STREAM
, 0);
211 perror("socket s1 failed()");
217 for (i
= 0; i
< 2; i
++) {
218 err
= setsockopt(*fds
[i
], SOL_SOCKET
, SO_REUSEADDR
,
219 (char *)&one
, sizeof(one
));
221 perror("setsockopt failed()");
226 /* Non-blocking sockets */
227 for (i
= 0; i
< 2; i
++) {
228 err
= ioctl(*fds
[i
], FIONBIO
, (char *)&one
);
230 perror("ioctl s1 failed()");
235 /* Bind server sockets */
236 memset(&addr
, 0, sizeof(struct sockaddr_in
));
237 addr
.sin_family
= AF_INET
;
238 addr
.sin_addr
.s_addr
= inet_addr("127.0.0.1");
240 addr
.sin_port
= htons(S1_PORT
);
241 err
= bind(s1
, (struct sockaddr
*)&addr
, sizeof(addr
));
243 perror("bind s1 failed()\n");
247 addr
.sin_port
= htons(S2_PORT
);
248 err
= bind(s2
, (struct sockaddr
*)&addr
, sizeof(addr
));
250 perror("bind s2 failed()\n");
254 /* Listen server sockets */
255 addr
.sin_port
= htons(S1_PORT
);
256 err
= listen(s1
, 32);
258 perror("listen s1 failed()\n");
262 addr
.sin_port
= htons(S2_PORT
);
263 err
= listen(s2
, 32);
265 perror("listen s1 failed()\n");
269 /* Initiate Connect */
270 addr
.sin_port
= htons(S1_PORT
);
271 err
= connect(c1
, (struct sockaddr
*)&addr
, sizeof(addr
));
272 if (err
< 0 && errno
!= EINPROGRESS
) {
273 perror("connect c1 failed()\n");
277 addr
.sin_port
= htons(S2_PORT
);
278 err
= connect(c2
, (struct sockaddr
*)&addr
, sizeof(addr
));
279 if (err
< 0 && errno
!= EINPROGRESS
) {
280 perror("connect c2 failed()\n");
282 } else if (err
< 0) {
286 /* Accept Connecrtions */
287 p1
= accept(s1
, NULL
, NULL
);
289 perror("accept s1 failed()\n");
293 p2
= accept(s2
, NULL
, NULL
);
295 perror("accept s1 failed()\n");
300 printf("connected sockets: c1 <-> p1, c2 <-> p2\n");
301 printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n",
310 struct timespec start
;
314 struct sockmap_options
{
325 static int msg_loop_sendpage(int fd
, int iov_length
, int cnt
,
327 struct sockmap_options
*opt
)
329 bool drop
= opt
->drop_expected
;
334 file
= fopen(".sendpage_tst.tmp", "w+");
335 for (i
= 0; i
< iov_length
* cnt
; i
++, k
++)
336 fwrite(&k
, sizeof(char), 1, file
);
338 fseek(file
, 0, SEEK_SET
);
341 fp
= open(".sendpage_tst.tmp", O_RDONLY
);
342 clock_gettime(CLOCK_MONOTONIC
, &s
->start
);
343 for (i
= 0; i
< cnt
; i
++) {
344 int sent
= sendfile(fd
, fp
, NULL
, iov_length
);
346 if (!drop
&& sent
< 0) {
347 perror("send loop error:");
350 } else if (drop
&& sent
>= 0) {
351 printf("sendpage loop error expected: %i\n", sent
);
357 s
->bytes_sent
+= sent
;
359 clock_gettime(CLOCK_MONOTONIC
, &s
->end
);
364 static void msg_free_iov(struct msghdr
*msg
)
368 for (i
= 0; i
< msg
->msg_iovlen
; i
++)
369 free(msg
->msg_iov
[i
].iov_base
);
375 static int msg_alloc_iov(struct msghdr
*msg
,
376 int iov_count
, int iov_length
,
377 bool data
, bool xmit
)
383 iov
= calloc(iov_count
, sizeof(struct iovec
));
387 for (i
= 0; i
< iov_count
; i
++) {
388 unsigned char *d
= calloc(iov_length
, sizeof(char));
391 fprintf(stderr
, "iov_count %i/%i OOM\n", i
, iov_count
);
395 iov
[i
].iov_len
= iov_length
;
400 for (j
= 0; j
< iov_length
; j
++)
406 msg
->msg_iovlen
= iov_count
;
410 for (i
--; i
>= 0 ; i
--)
411 free(msg
->msg_iov
[i
].iov_base
);
415 static int msg_verify_data(struct msghdr
*msg
, int size
, int chunk_sz
)
417 int i
, j
, bytes_cnt
= 0;
420 for (i
= 0; i
< msg
->msg_iovlen
; i
++) {
421 unsigned char *d
= msg
->msg_iov
[i
].iov_base
;
424 j
< msg
->msg_iov
[i
].iov_len
&& size
; j
++) {
427 "detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n",
428 i
, j
, d
[j
], k
- 1, d
[j
+1], k
);
432 if (bytes_cnt
== chunk_sz
) {
442 static int msg_loop(int fd
, int iov_count
, int iov_length
, int cnt
,
443 struct msg_stats
*s
, bool tx
,
444 struct sockmap_options
*opt
)
446 struct msghdr msg
= {0}, msg_peek
= {0};
447 int err
, i
, flags
= MSG_NOSIGNAL
;
448 bool drop
= opt
->drop_expected
;
449 bool data
= opt
->data_test
;
451 err
= msg_alloc_iov(&msg
, iov_count
, iov_length
, data
, tx
);
455 err
= msg_alloc_iov(&msg_peek
, iov_count
, iov_length
, data
, tx
);
461 clock_gettime(CLOCK_MONOTONIC
, &s
->start
);
462 for (i
= 0; i
< cnt
; i
++) {
463 int sent
= sendmsg(fd
, &msg
, flags
);
465 if (!drop
&& sent
< 0) {
466 perror("send loop error:");
468 } else if (drop
&& sent
>= 0) {
469 printf("send loop error expected: %i\n", sent
);
474 s
->bytes_sent
+= sent
;
476 clock_gettime(CLOCK_MONOTONIC
, &s
->end
);
478 int slct
, recvp
= 0, recv
, max_fd
= fd
;
479 float total_bytes
, txmsg_pop_total
;
480 int fd_flags
= O_NONBLOCK
;
481 struct timeval timeout
;
485 /* Account for pop bytes noting each iteration of apply will
486 * call msg_pop_data helper so we need to account for this
487 * by calculating the number of apply iterations. Note user
488 * of the tool can create cases where no data is sent by
489 * manipulating pop/push/pull/etc. For example txmsg_apply 1
490 * with txmsg_pop 1 will try to apply 1B at a time but each
491 * iteration will then pop 1B so no data will ever be sent.
492 * This is really only useful for testing edge cases in code
495 total_bytes
= (float)iov_count
* (float)iov_length
* (float)cnt
;
496 txmsg_pop_total
= txmsg_pop
;
498 txmsg_pop_total
*= (total_bytes
/ txmsg_apply
);
499 total_bytes
-= txmsg_pop_total
;
500 err
= clock_gettime(CLOCK_MONOTONIC
, &s
->start
);
502 perror("recv start time: ");
503 while (s
->bytes_recvd
< total_bytes
) {
506 timeout
.tv_usec
= 300000;
516 slct
= select(max_fd
+ 1, &w
, NULL
, NULL
, &timeout
);
519 clock_gettime(CLOCK_MONOTONIC
, &s
->end
);
523 fprintf(stderr
, "unexpected timeout: recved %zu/%f pop_total %f\n", s
->bytes_recvd
, total_bytes
, txmsg_pop_total
);
525 clock_gettime(CLOCK_MONOTONIC
, &s
->end
);
532 recvp
= recvmsg(fd
, &msg_peek
, flags
);
534 if (errno
!= EWOULDBLOCK
) {
535 clock_gettime(CLOCK_MONOTONIC
, &s
->end
);
542 recv
= recvmsg(fd
, &msg
, flags
);
544 if (errno
!= EWOULDBLOCK
) {
545 clock_gettime(CLOCK_MONOTONIC
, &s
->end
);
546 perror("recv failed()\n");
551 s
->bytes_recvd
+= recv
;
554 int chunk_sz
= opt
->sendpage
?
556 iov_length
* iov_count
;
558 errno
= msg_verify_data(&msg
, recv
, chunk_sz
);
560 perror("data verify msg failed\n");
564 errno
= msg_verify_data(&msg_peek
,
568 perror("data verify msg_peek failed\n");
574 clock_gettime(CLOCK_MONOTONIC
, &s
->end
);
578 msg_free_iov(&msg_peek
);
582 msg_free_iov(&msg_peek
);
586 static float giga
= 1000000000;
588 static inline float sentBps(struct msg_stats s
)
590 return s
.bytes_sent
/ (s
.end
.tv_sec
- s
.start
.tv_sec
);
593 static inline float recvdBps(struct msg_stats s
)
595 return s
.bytes_recvd
/ (s
.end
.tv_sec
- s
.start
.tv_sec
);
598 static int sendmsg_test(struct sockmap_options
*opt
)
600 float sent_Bps
= 0, recvd_Bps
= 0;
601 int rx_fd
, txpid
, rxpid
, err
= 0;
602 struct msg_stats s
= {0};
603 int iov_count
= opt
->iov_count
;
604 int iov_buf
= opt
->iov_length
;
605 int rx_status
, tx_status
;
616 /* Redirecting into non-TLS socket which sends into a TLS
617 * socket is not a valid test. So in this case lets not
618 * enable kTLS but still run the test.
620 if (!txmsg_redir
|| (txmsg_redir
&& txmsg_ingress
)) {
621 err
= sockmap_init_ktls(opt
->verbose
, rx_fd
);
625 err
= sockmap_init_ktls(opt
->verbose
, c1
);
632 if (opt
->drop_expected
)
637 err
= msg_loop(rx_fd
, iov_count
, iov_buf
,
638 cnt
, &s
, false, opt
);
641 "msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n",
642 iov_count
, iov_buf
, cnt
, err
);
643 if (s
.end
.tv_sec
- s
.start
.tv_sec
) {
644 sent_Bps
= sentBps(s
);
645 recvd_Bps
= recvdBps(s
);
649 "rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s %s\n",
650 s
.bytes_sent
, sent_Bps
, sent_Bps
/giga
,
651 s
.bytes_recvd
, recvd_Bps
, recvd_Bps
/giga
,
652 peek_flag
? "(peek_msg)" : "");
653 if (err
&& txmsg_cork
)
656 } else if (rxpid
== -1) {
657 perror("msg_loop_rx: ");
664 err
= msg_loop_sendpage(c1
, iov_buf
, cnt
, &s
, opt
);
666 err
= msg_loop(c1
, iov_count
, iov_buf
,
671 "msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n",
672 iov_count
, iov_buf
, cnt
, err
);
673 if (s
.end
.tv_sec
- s
.start
.tv_sec
) {
674 sent_Bps
= sentBps(s
);
675 recvd_Bps
= recvdBps(s
);
679 "tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n",
680 s
.bytes_sent
, sent_Bps
, sent_Bps
/giga
,
681 s
.bytes_recvd
, recvd_Bps
, recvd_Bps
/giga
);
683 } else if (txpid
== -1) {
684 perror("msg_loop_tx: ");
688 assert(waitpid(rxpid
, &rx_status
, 0) == rxpid
);
689 assert(waitpid(txpid
, &tx_status
, 0) == txpid
);
690 if (WIFEXITED(rx_status
)) {
691 err
= WEXITSTATUS(rx_status
);
693 fprintf(stderr
, "rx thread exited with err %d. ", err
);
697 if (WIFEXITED(tx_status
)) {
698 err
= WEXITSTATUS(tx_status
);
700 fprintf(stderr
, "tx thread exited with err %d. ", err
);
706 static int forever_ping_pong(int rate
, struct sockmap_options
*opt
)
708 struct timeval timeout
;
709 char buf
[1024] = {0};
715 /* Ping/Pong data from client to server */
716 sc
= send(c1
, buf
, sizeof(buf
), 0);
718 perror("send failed()\n");
723 int s
, rc
, i
, max_fd
= p2
;
733 s
= select(max_fd
+ 1, &w
, NULL
, NULL
, &timeout
);
738 fprintf(stderr
, "unexpected timeout\n");
742 for (i
= 0; i
<= max_fd
&& s
> 0; ++i
) {
743 if (!FD_ISSET(i
, &w
))
748 rc
= recv(i
, buf
, sizeof(buf
), 0);
750 if (errno
!= EWOULDBLOCK
) {
751 perror("recv failed()\n");
761 sc
= send(i
, buf
, rc
, 0);
763 perror("send failed()\n");
789 static int run_options(struct sockmap_options
*options
, int cg_fd
, int test
)
791 int i
, key
, next_key
, err
, tx_prog_fd
= -1, zero
= 0;
793 /* If base test skip BPF setup */
794 if (test
== BASE
|| test
== BASE_SENDPAGE
)
797 /* Attach programs to sockmap */
798 err
= bpf_prog_attach(prog_fd
[0], map_fd
[0],
799 BPF_SK_SKB_STREAM_PARSER
, 0);
802 "ERROR: bpf_prog_attach (sockmap %i->%i): %d (%s)\n",
803 prog_fd
[0], map_fd
[0], err
, strerror(errno
));
807 err
= bpf_prog_attach(prog_fd
[1], map_fd
[0],
808 BPF_SK_SKB_STREAM_VERDICT
, 0);
810 fprintf(stderr
, "ERROR: bpf_prog_attach (sockmap): %d (%s)\n",
811 err
, strerror(errno
));
815 /* Attach to cgroups */
816 err
= bpf_prog_attach(prog_fd
[2], cg_fd
, BPF_CGROUP_SOCK_OPS
, 0);
818 fprintf(stderr
, "ERROR: bpf_prog_attach (groups): %d (%s)\n",
819 err
, strerror(errno
));
824 err
= sockmap_init_sockets(options
->verbose
);
826 fprintf(stderr
, "ERROR: test socket failed: %d\n", err
);
830 /* Attach txmsg program to sockmap */
832 tx_prog_fd
= prog_fd
[3];
833 else if (txmsg_noisy
)
834 tx_prog_fd
= prog_fd
[4];
835 else if (txmsg_redir
)
836 tx_prog_fd
= prog_fd
[5];
837 else if (txmsg_redir_noisy
)
838 tx_prog_fd
= prog_fd
[6];
840 tx_prog_fd
= prog_fd
[9];
841 /* apply and cork must be last */
842 else if (txmsg_apply
)
843 tx_prog_fd
= prog_fd
[7];
845 tx_prog_fd
= prog_fd
[8];
852 err
= bpf_prog_attach(tx_prog_fd
,
853 map_fd
[1], BPF_SK_MSG_VERDICT
, 0);
856 "ERROR: bpf_prog_attach (txmsg): %d (%s)\n",
857 err
, strerror(errno
));
861 err
= bpf_map_update_elem(map_fd
[1], &i
, &c1
, BPF_ANY
);
864 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",
865 err
, strerror(errno
));
869 if (txmsg_redir
|| txmsg_redir_noisy
)
874 err
= bpf_map_update_elem(map_fd
[2], &i
, &redir_fd
, BPF_ANY
);
877 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",
878 err
, strerror(errno
));
883 err
= bpf_map_update_elem(map_fd
[3],
884 &i
, &txmsg_apply
, BPF_ANY
);
887 "ERROR: bpf_map_update_elem (apply_bytes): %d (%s\n",
888 err
, strerror(errno
));
894 err
= bpf_map_update_elem(map_fd
[4],
895 &i
, &txmsg_cork
, BPF_ANY
);
898 "ERROR: bpf_map_update_elem (cork_bytes): %d (%s\n",
899 err
, strerror(errno
));
905 err
= bpf_map_update_elem(map_fd
[5],
906 &i
, &txmsg_start
, BPF_ANY
);
909 "ERROR: bpf_map_update_elem (txmsg_start): %d (%s)\n",
910 err
, strerror(errno
));
917 err
= bpf_map_update_elem(map_fd
[5],
918 &i
, &txmsg_end
, BPF_ANY
);
921 "ERROR: bpf_map_update_elem (txmsg_end): %d (%s)\n",
922 err
, strerror(errno
));
927 if (txmsg_start_push
) {
929 err
= bpf_map_update_elem(map_fd
[5],
930 &i
, &txmsg_start_push
, BPF_ANY
);
933 "ERROR: bpf_map_update_elem (txmsg_start_push): %d (%s)\n",
934 err
, strerror(errno
));
939 if (txmsg_end_push
) {
941 err
= bpf_map_update_elem(map_fd
[5],
942 &i
, &txmsg_end_push
, BPF_ANY
);
945 "ERROR: bpf_map_update_elem %i@%i (txmsg_end_push): %d (%s)\n",
946 txmsg_end_push
, i
, err
, strerror(errno
));
951 if (txmsg_start_pop
) {
953 err
= bpf_map_update_elem(map_fd
[5],
954 &i
, &txmsg_start_pop
, BPF_ANY
);
957 "ERROR: bpf_map_update_elem %i@%i (txmsg_start_pop): %d (%s)\n",
958 txmsg_start_pop
, i
, err
, strerror(errno
));
963 bpf_map_update_elem(map_fd
[5],
964 &i
, &txmsg_start_pop
, BPF_ANY
);
969 err
= bpf_map_update_elem(map_fd
[5],
970 &i
, &txmsg_pop
, BPF_ANY
);
973 "ERROR: bpf_map_update_elem %i@%i (txmsg_pop): %d (%s)\n",
974 txmsg_pop
, i
, err
, strerror(errno
));
979 bpf_map_update_elem(map_fd
[5],
980 &i
, &txmsg_pop
, BPF_ANY
);
985 int in
= BPF_F_INGRESS
;
988 err
= bpf_map_update_elem(map_fd
[6], &i
, &in
, BPF_ANY
);
991 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
992 err
, strerror(errno
));
995 err
= bpf_map_update_elem(map_fd
[1], &i
, &p1
, BPF_ANY
);
998 "ERROR: bpf_map_update_elem (p1 txmsg): %d (%s)\n",
999 err
, strerror(errno
));
1001 err
= bpf_map_update_elem(map_fd
[2], &i
, &p1
, BPF_ANY
);
1004 "ERROR: bpf_map_update_elem (p1 redir): %d (%s)\n",
1005 err
, strerror(errno
));
1009 err
= bpf_map_update_elem(map_fd
[2], &i
, &p2
, BPF_ANY
);
1012 "ERROR: bpf_map_update_elem (p2 txmsg): %d (%s)\n",
1013 err
, strerror(errno
));
1018 int skb_fd
= (test
== SENDMSG
|| test
== SENDPAGE
) ?
1020 int ingress
= BPF_F_INGRESS
;
1023 err
= bpf_map_update_elem(map_fd
[7],
1024 &i
, &ingress
, BPF_ANY
);
1027 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1028 err
, strerror(errno
));
1032 err
= bpf_map_update_elem(map_fd
[0],
1033 &i
, &skb_fd
, BPF_ANY
);
1036 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
1037 err
, strerror(errno
));
1043 options
->drop_expected
= true;
1045 if (test
== PING_PONG
)
1046 err
= forever_ping_pong(options
->rate
, options
);
1047 else if (test
== SENDMSG
) {
1048 options
->base
= false;
1049 options
->sendpage
= false;
1050 err
= sendmsg_test(options
);
1051 } else if (test
== SENDPAGE
) {
1052 options
->base
= false;
1053 options
->sendpage
= true;
1054 err
= sendmsg_test(options
);
1055 } else if (test
== BASE
) {
1056 options
->base
= true;
1057 options
->sendpage
= false;
1058 err
= sendmsg_test(options
);
1059 } else if (test
== BASE_SENDPAGE
) {
1060 options
->base
= true;
1061 options
->sendpage
= true;
1062 err
= sendmsg_test(options
);
1064 fprintf(stderr
, "unknown test\n");
1066 /* Detatch and zero all the maps */
1067 bpf_prog_detach2(prog_fd
[2], cg_fd
, BPF_CGROUP_SOCK_OPS
);
1068 bpf_prog_detach2(prog_fd
[0], map_fd
[0], BPF_SK_SKB_STREAM_PARSER
);
1069 bpf_prog_detach2(prog_fd
[1], map_fd
[0], BPF_SK_SKB_STREAM_VERDICT
);
1070 if (tx_prog_fd
>= 0)
1071 bpf_prog_detach2(tx_prog_fd
, map_fd
[1], BPF_SK_MSG_VERDICT
);
1073 for (i
= 0; i
< 8; i
++) {
1075 bpf_map_update_elem(map_fd
[i
], &key
, &zero
, BPF_ANY
);
1076 while (bpf_map_get_next_key(map_fd
[i
], &key
, &next_key
) == 0) {
1077 bpf_map_update_elem(map_fd
[i
], &key
, &zero
, BPF_ANY
);
1091 static char *test_to_str(int test
)
1102 #define OPTSTRING 60
1103 static void test_options(char *options
)
1105 char tstr
[OPTSTRING
];
1107 memset(options
, 0, OPTSTRING
);
1110 strncat(options
, "pass,", OPTSTRING
);
1112 strncat(options
, "pass_noisy,", OPTSTRING
);
1114 strncat(options
, "redir,", OPTSTRING
);
1115 if (txmsg_redir_noisy
)
1116 strncat(options
, "redir_noisy,", OPTSTRING
);
1118 strncat(options
, "drop,", OPTSTRING
);
1120 snprintf(tstr
, OPTSTRING
, "apply %d,", txmsg_apply
);
1121 strncat(options
, tstr
, OPTSTRING
);
1124 snprintf(tstr
, OPTSTRING
, "cork %d,", txmsg_cork
);
1125 strncat(options
, tstr
, OPTSTRING
);
1128 snprintf(tstr
, OPTSTRING
, "start %d,", txmsg_start
);
1129 strncat(options
, tstr
, OPTSTRING
);
1132 snprintf(tstr
, OPTSTRING
, "end %d,", txmsg_end
);
1133 strncat(options
, tstr
, OPTSTRING
);
1135 if (txmsg_start_pop
) {
1136 snprintf(tstr
, OPTSTRING
, "pop (%d,%d),",
1137 txmsg_start_pop
, txmsg_start_pop
+ txmsg_pop
);
1138 strncat(options
, tstr
, OPTSTRING
);
1141 strncat(options
, "ingress,", OPTSTRING
);
1143 strncat(options
, "skb,", OPTSTRING
);
1145 strncat(options
, "ktls,", OPTSTRING
);
1147 strncat(options
, "peek,", OPTSTRING
);
1150 static int __test_exec(int cgrp
, int test
, struct sockmap_options
*opt
)
1152 char *options
= calloc(OPTSTRING
, sizeof(char));
1155 if (test
== SENDPAGE
)
1156 opt
->sendpage
= true;
1158 opt
->sendpage
= false;
1161 opt
->drop_expected
= true;
1163 opt
->drop_expected
= false;
1165 test_options(options
);
1168 "[TEST %i]: (%i, %i, %i, %s, %s): ",
1169 test_cnt
, opt
->rate
, opt
->iov_count
, opt
->iov_length
,
1170 test_to_str(test
), options
);
1172 err
= run_options(opt
, cgrp
, test
);
1173 fprintf(stdout
, "%s\n", !err
? "PASS" : "FAILED");
1175 !err
? passed
++ : failed
++;
1180 static int test_exec(int cgrp
, struct sockmap_options
*opt
)
1182 int err
= __test_exec(cgrp
, SENDMSG
, opt
);
1187 err
= __test_exec(cgrp
, SENDPAGE
, opt
);
1192 static int test_loop(int cgrp
)
1194 struct sockmap_options opt
;
1200 opt
.sendpage
= false;
1201 opt
.data_test
= false;
1202 opt
.drop_expected
= false;
1208 for (i
= 1; i
< 100; i
+= 33) {
1209 for (l
= 1; l
< 100; l
+= 33) {
1213 err
= test_exec(cgrp
, &opt
);
1223 static int test_txmsg(int cgrp
)
1227 txmsg_pass
= txmsg_noisy
= txmsg_redir_noisy
= txmsg_drop
= 0;
1228 txmsg_apply
= txmsg_cork
= 0;
1229 txmsg_ingress
= txmsg_skb
= 0;
1232 err
= test_loop(cgrp
);
1238 err
= test_loop(cgrp
);
1244 err
= test_loop(cgrp
);
1251 err
= test_loop(cgrp
);
1263 static int test_send(struct sockmap_options
*opt
, int cgrp
)
1267 opt
->iov_length
= 1;
1270 err
= test_exec(cgrp
, opt
);
1274 opt
->iov_length
= 1;
1275 opt
->iov_count
= 1024;
1277 err
= test_exec(cgrp
, opt
);
1281 opt
->iov_length
= 1024;
1284 err
= test_exec(cgrp
, opt
);
1288 opt
->iov_length
= 1;
1291 err
= test_exec(cgrp
, opt
);
1295 opt
->iov_length
= 256;
1296 opt
->iov_count
= 1024;
1298 err
= test_exec(cgrp
, opt
);
1304 opt
->iov_length
= 5;
1305 err
= test_exec(cgrp
, opt
);
1313 static int test_mixed(int cgrp
)
1315 struct sockmap_options opt
= {0};
1318 txmsg_pass
= txmsg_noisy
= txmsg_redir_noisy
= txmsg_drop
= 0;
1319 txmsg_apply
= txmsg_cork
= 0;
1320 txmsg_start
= txmsg_end
= 0;
1321 txmsg_start_push
= txmsg_end_push
= 0;
1322 txmsg_start_pop
= txmsg_pop
= 0;
1324 /* Test small and large iov_count values with pass/redir/apply/cork */
1329 err
= test_send(&opt
, cgrp
);
1337 err
= test_send(&opt
, cgrp
);
1345 err
= test_send(&opt
, cgrp
);
1353 err
= test_send(&opt
, cgrp
);
1361 err
= test_send(&opt
, cgrp
);
1369 err
= test_send(&opt
, cgrp
);
1377 err
= test_send(&opt
, cgrp
);
1385 err
= test_send(&opt
, cgrp
);
1393 err
= test_send(&opt
, cgrp
);
1401 err
= test_send(&opt
, cgrp
);
1409 err
= test_send(&opt
, cgrp
);
1417 err
= test_send(&opt
, cgrp
);
1425 err
= test_send(&opt
, cgrp
);
1432 static int test_start_end(int cgrp
)
1434 struct sockmap_options opt
= {0};
1437 /* Test basic start/end with lots of iov_count and iov_lengths */
1440 txmsg_start_push
= 1;
1442 txmsg_start_pop
= 1;
1444 err
= test_txmsg(cgrp
);
1448 /* Cut a byte of pushed data but leave reamining in place */
1451 txmsg_start_push
= 1;
1453 txmsg_start_pop
= 1;
1455 err
= test_txmsg(cgrp
);
1459 /* Test start/end with cork */
1462 opt
.iov_length
= 100;
1465 txmsg_start_pop
= 0;
1468 for (i
= 99; i
<= 1600; i
+= 500) {
1471 txmsg_start_push
= 0;
1473 err
= test_exec(cgrp
, &opt
);
1478 /* Test pop data in middle of cork */
1479 for (i
= 99; i
<= 1600; i
+= 500) {
1480 txmsg_start_pop
= 10;
1482 err
= test_exec(cgrp
, &opt
);
1486 txmsg_start_pop
= 0;
1489 /* Test start/end with cork but pull data in middle */
1490 for (i
= 199; i
<= 1600; i
+= 500) {
1493 txmsg_start_push
= 100;
1495 err
= test_exec(cgrp
, &opt
);
1500 /* Test start/end with cork pulling last sg entry */
1503 txmsg_start_push
= 1500;
1504 txmsg_end_push
= 1600;
1505 err
= test_exec(cgrp
, &opt
);
1509 /* Test pop with cork pulling last sg entry */
1510 txmsg_start_pop
= 1500;
1512 err
= test_exec(cgrp
, &opt
);
1515 txmsg_start_pop
= 0;
1518 /* Test start/end pull of single byte in last page */
1521 txmsg_start_push
= 1111;
1522 txmsg_end_push
= 1112;
1523 err
= test_exec(cgrp
, &opt
);
1527 /* Test pop of single byte in last page */
1528 txmsg_start_pop
= 1111;
1530 err
= test_exec(cgrp
, &opt
);
1534 /* Test start/end with end < start */
1537 txmsg_start_push
= 1111;
1539 err
= test_exec(cgrp
, &opt
);
1543 /* Test start/end with end > data */
1546 txmsg_start_push
= 0;
1547 txmsg_end_push
= 1601;
1548 err
= test_exec(cgrp
, &opt
);
1552 /* Test start/end with start > data */
1555 txmsg_start_push
= 1601;
1556 txmsg_end_push
= 1600;
1557 err
= test_exec(cgrp
, &opt
);
1561 /* Test pop with start > data */
1562 txmsg_start_pop
= 1601;
1564 err
= test_exec(cgrp
, &opt
);
1568 /* Test pop with pop range > data */
1569 txmsg_start_pop
= 1599;
1571 err
= test_exec(cgrp
, &opt
);
1579 char *map_names
[] = {
1590 int prog_attach_type
[] = {
1591 BPF_SK_SKB_STREAM_PARSER
,
1592 BPF_SK_SKB_STREAM_VERDICT
,
1593 BPF_CGROUP_SOCK_OPS
,
1604 BPF_PROG_TYPE_SK_SKB
,
1605 BPF_PROG_TYPE_SK_SKB
,
1606 BPF_PROG_TYPE_SOCK_OPS
,
1607 BPF_PROG_TYPE_SK_MSG
,
1608 BPF_PROG_TYPE_SK_MSG
,
1609 BPF_PROG_TYPE_SK_MSG
,
1610 BPF_PROG_TYPE_SK_MSG
,
1611 BPF_PROG_TYPE_SK_MSG
,
1612 BPF_PROG_TYPE_SK_MSG
,
1613 BPF_PROG_TYPE_SK_MSG
,
1616 static int populate_progs(char *bpf_file
)
1618 struct bpf_program
*prog
;
1619 struct bpf_object
*obj
;
1623 obj
= bpf_object__open(bpf_file
);
1624 err
= libbpf_get_error(obj
);
1628 libbpf_strerror(err
, err_buf
, sizeof(err_buf
));
1629 printf("Unable to load eBPF objects in file '%s' : %s\n",
1634 bpf_object__for_each_program(prog
, obj
) {
1635 bpf_program__set_type(prog
, prog_type
[i
]);
1636 bpf_program__set_expected_attach_type(prog
,
1637 prog_attach_type
[i
]);
1641 i
= bpf_object__load(obj
);
1643 bpf_object__for_each_program(prog
, obj
) {
1644 prog_fd
[i
] = bpf_program__fd(prog
);
1648 for (i
= 0; i
< sizeof(map_fd
)/sizeof(int); i
++) {
1649 maps
[i
] = bpf_object__find_map_by_name(obj
, map_names
[i
]);
1650 map_fd
[i
] = bpf_map__fd(maps
[i
]);
1651 if (map_fd
[i
] < 0) {
1652 fprintf(stderr
, "load_bpf_file: (%i) %s\n",
1653 map_fd
[i
], strerror(errno
));
1661 static int __test_suite(int cg_fd
, char *bpf_file
)
1663 int err
, cleanup
= cg_fd
;
1665 err
= populate_progs(bpf_file
);
1667 fprintf(stderr
, "ERROR: (%i) load bpf failed\n", err
);
1672 if (setup_cgroup_environment()) {
1673 fprintf(stderr
, "ERROR: cgroup env failed\n");
1677 cg_fd
= create_and_get_cgroup(CG_PATH
);
1680 "ERROR: (%i) open cg path failed: %s\n",
1685 if (join_cgroup(CG_PATH
)) {
1686 fprintf(stderr
, "ERROR: failed to join cgroup\n");
1691 /* Tests basic commands and APIs with range of iov values */
1692 txmsg_start
= txmsg_end
= txmsg_start_push
= txmsg_end_push
= 0;
1693 err
= test_txmsg(cg_fd
);
1697 /* Tests interesting combinations of APIs used together */
1698 err
= test_mixed(cg_fd
);
1702 /* Tests pull_data API using start/end API */
1703 err
= test_start_end(cg_fd
);
1708 printf("Summary: %i PASSED %i FAILED\n", passed
, failed
);
1710 cleanup_cgroup_environment();
1716 static int test_suite(int cg_fd
)
1720 err
= __test_suite(cg_fd
, BPF_SOCKMAP_FILENAME
);
1723 err
= __test_suite(cg_fd
, BPF_SOCKHASH_FILENAME
);
1730 int main(int argc
, char **argv
)
1732 int iov_count
= 1, length
= 1024, rate
= 1;
1733 struct sockmap_options options
= {0};
1734 int opt
, longindex
, err
, cg_fd
= 0;
1735 char *bpf_file
= BPF_SOCKMAP_FILENAME
;
1736 int test
= PING_PONG
;
1739 return test_suite(-1);
1741 while ((opt
= getopt_long(argc
, argv
, ":dhvc:r:i:l:t:p:q:",
1742 long_options
, &longindex
)) != -1) {
1745 txmsg_start
= atoi(optarg
);
1748 txmsg_end
= atoi(optarg
);
1751 txmsg_start_push
= atoi(optarg
);
1754 txmsg_end_push
= atoi(optarg
);
1757 txmsg_start_pop
= atoi(optarg
);
1760 txmsg_pop
= atoi(optarg
);
1763 txmsg_apply
= atoi(optarg
);
1766 txmsg_cork
= atoi(optarg
);
1769 cg_fd
= open(optarg
, O_DIRECTORY
, O_RDONLY
);
1772 "ERROR: (%i) open cg path failed: %s\n",
1778 rate
= atoi(optarg
);
1781 options
.verbose
= 1;
1784 iov_count
= atoi(optarg
);
1787 length
= atoi(optarg
);
1790 options
.data_test
= true;
1793 if (strcmp(optarg
, "ping") == 0) {
1795 } else if (strcmp(optarg
, "sendmsg") == 0) {
1797 } else if (strcmp(optarg
, "base") == 0) {
1799 } else if (strcmp(optarg
, "base_sendpage") == 0) {
1800 test
= BASE_SENDPAGE
;
1801 } else if (strcmp(optarg
, "sendpage") == 0) {
1817 if (argc
<= 3 && cg_fd
)
1818 return test_suite(cg_fd
);
1821 fprintf(stderr
, "%s requires cgroup option: --cgroup <path>\n",
1826 err
= populate_progs(bpf_file
);
1828 fprintf(stderr
, "populate program: (%s) %s\n",
1829 bpf_file
, strerror(errno
));
1835 signal(SIGINT
, running_handler
);
1837 options
.iov_count
= iov_count
;
1838 options
.iov_length
= length
;
1839 options
.rate
= rate
;
1841 err
= run_options(&options
, cg_fd
, test
);
1846 void running_handler(int a
)