1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2017 - 2018 Intel Corporation. */
4 #include <asm/barrier.h>
9 #include <linux/compiler.h>
10 #include <linux/if_link.h>
11 #include <linux/if_xdp.h>
12 #include <linux/if_ether.h>
14 #include <net/ethernet.h>
24 #include <sys/resource.h>
25 #include <sys/socket.h>
26 #include <sys/types.h>
30 #include "bpf/libbpf.h"
46 #define NUM_FRAMES (4 * 1024)
49 #define DEBUG_HEXDUMP 0
55 static unsigned long prev_time
;
63 static enum benchmark_type opt_bench
= BENCH_RXDROP
;
64 static u32 opt_xdp_flags
= XDP_FLAGS_UPDATE_IF_NOEXIST
;
65 static const char *opt_if
= "";
66 static int opt_ifindex
;
69 static int opt_interval
= 1;
70 static u32 opt_xdp_bind_flags
;
73 struct xsk_umem_info
{
74 struct xsk_ring_prod fq
;
75 struct xsk_ring_cons cq
;
76 struct xsk_umem
*umem
;
80 struct xsk_socket_info
{
81 struct xsk_ring_cons rx
;
82 struct xsk_ring_prod tx
;
83 struct xsk_umem_info
*umem
;
84 struct xsk_socket
*xsk
;
85 unsigned long rx_npkts
;
86 unsigned long tx_npkts
;
87 unsigned long prev_rx_npkts
;
88 unsigned long prev_tx_npkts
;
93 struct xsk_socket_info
*xsks
[MAX_SOCKS
];
95 static unsigned long get_nsecs(void)
99 clock_gettime(CLOCK_MONOTONIC
, &ts
);
100 return ts
.tv_sec
* 1000000000UL + ts
.tv_nsec
;
103 static void print_benchmark(bool running
)
105 const char *bench_str
= "INVALID";
107 if (opt_bench
== BENCH_RXDROP
)
108 bench_str
= "rxdrop";
109 else if (opt_bench
== BENCH_TXONLY
)
110 bench_str
= "txonly";
111 else if (opt_bench
== BENCH_L2FWD
)
114 printf("%s:%d %s ", opt_if
, opt_queue
, bench_str
);
115 if (opt_xdp_flags
& XDP_FLAGS_SKB_MODE
)
117 else if (opt_xdp_flags
& XDP_FLAGS_DRV_MODE
)
126 printf("running...");
131 static void dump_stats(void)
133 unsigned long now
= get_nsecs();
134 long dt
= now
- prev_time
;
139 for (i
= 0; i
< num_socks
&& xsks
[i
]; i
++) {
140 char *fmt
= "%-15s %'-11.0f %'-11lu\n";
141 double rx_pps
, tx_pps
;
143 rx_pps
= (xsks
[i
]->rx_npkts
- xsks
[i
]->prev_rx_npkts
) *
145 tx_pps
= (xsks
[i
]->tx_npkts
- xsks
[i
]->prev_tx_npkts
) *
148 printf("\n sock%d@", i
);
149 print_benchmark(false);
152 printf("%-15s %-11s %-11s %-11.2f\n", "", "pps", "pkts",
154 printf(fmt
, "rx", rx_pps
, xsks
[i
]->rx_npkts
);
155 printf(fmt
, "tx", tx_pps
, xsks
[i
]->tx_npkts
);
157 xsks
[i
]->prev_rx_npkts
= xsks
[i
]->rx_npkts
;
158 xsks
[i
]->prev_tx_npkts
= xsks
[i
]->tx_npkts
;
162 static void *poller(void *arg
)
173 static void remove_xdp_program(void)
175 __u32 curr_prog_id
= 0;
177 if (bpf_get_link_xdp_id(opt_ifindex
, &curr_prog_id
, opt_xdp_flags
)) {
178 printf("bpf_get_link_xdp_id failed\n");
181 if (prog_id
== curr_prog_id
)
182 bpf_set_link_xdp_fd(opt_ifindex
, -1, opt_xdp_flags
);
183 else if (!curr_prog_id
)
184 printf("couldn't find a prog id on a given interface\n");
186 printf("program on interface changed, not removing\n");
189 static void int_exit(int sig
)
191 struct xsk_umem
*umem
= xsks
[0]->umem
->umem
;
196 xsk_socket__delete(xsks
[0]->xsk
);
197 (void)xsk_umem__delete(umem
);
198 remove_xdp_program();
203 static void __exit_with_error(int error
, const char *file
, const char *func
,
206 fprintf(stderr
, "%s:%s:%i: errno: %d/\"%s\"\n", file
, func
,
207 line
, error
, strerror(error
));
209 remove_xdp_program();
213 #define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, \
216 static const char pkt_data
[] =
217 "\x3c\xfd\xfe\x9e\x7f\x71\xec\xb1\xd7\x98\x3a\xc0\x08\x00\x45\x00"
218 "\x00\x2e\x00\x00\x00\x00\x40\x11\x88\x97\x05\x08\x07\x08\xc8\x14"
219 "\x1e\x04\x10\x92\x10\x92\x00\x1a\x6d\xa3\x34\x33\x1f\x69\x40\x6b"
220 "\x54\x59\xb6\x14\x2d\x11\x44\xbf\xaf\xd9\xbe\xaa";
222 static void swap_mac_addresses(void *data
)
224 struct ether_header
*eth
= (struct ether_header
*)data
;
225 struct ether_addr
*src_addr
= (struct ether_addr
*)ð
->ether_shost
;
226 struct ether_addr
*dst_addr
= (struct ether_addr
*)ð
->ether_dhost
;
227 struct ether_addr tmp
;
230 *src_addr
= *dst_addr
;
234 static void hex_dump(void *pkt
, size_t length
, u64 addr
)
236 const unsigned char *address
= (unsigned char *)pkt
;
237 const unsigned char *line
= address
;
238 size_t line_size
= 32;
246 sprintf(buf
, "addr=%llu", addr
);
247 printf("length = %zu\n", length
);
248 printf("%s | ", buf
);
249 while (length
-- > 0) {
250 printf("%02X ", *address
++);
251 if (!(++i
% line_size
) || (length
== 0 && i
% line_size
)) {
253 while (i
++ % line_size
)
256 printf(" | "); /* right close */
257 while (line
< address
) {
259 printf("%c", (c
< 33 || c
== 255) ? 0x2E : c
);
263 printf("%s | ", buf
);
269 static size_t gen_eth_frame(struct xsk_umem_info
*umem
, u64 addr
)
271 memcpy(xsk_umem__get_data(umem
->buffer
, addr
), pkt_data
,
272 sizeof(pkt_data
) - 1);
273 return sizeof(pkt_data
) - 1;
276 static struct xsk_umem_info
*xsk_configure_umem(void *buffer
, u64 size
)
278 struct xsk_umem_info
*umem
;
281 umem
= calloc(1, sizeof(*umem
));
283 exit_with_error(errno
);
285 ret
= xsk_umem__create(&umem
->umem
, buffer
, size
, &umem
->fq
, &umem
->cq
,
288 exit_with_error(-ret
);
290 umem
->buffer
= buffer
;
294 static struct xsk_socket_info
*xsk_configure_socket(struct xsk_umem_info
*umem
)
296 struct xsk_socket_config cfg
;
297 struct xsk_socket_info
*xsk
;
302 xsk
= calloc(1, sizeof(*xsk
));
304 exit_with_error(errno
);
307 cfg
.rx_size
= XSK_RING_CONS__DEFAULT_NUM_DESCS
;
308 cfg
.tx_size
= XSK_RING_PROD__DEFAULT_NUM_DESCS
;
309 cfg
.libbpf_flags
= 0;
310 cfg
.xdp_flags
= opt_xdp_flags
;
311 cfg
.bind_flags
= opt_xdp_bind_flags
;
312 ret
= xsk_socket__create(&xsk
->xsk
, opt_if
, opt_queue
, umem
->umem
,
313 &xsk
->rx
, &xsk
->tx
, &cfg
);
315 exit_with_error(-ret
);
317 ret
= bpf_get_link_xdp_id(opt_ifindex
, &prog_id
, opt_xdp_flags
);
319 exit_with_error(-ret
);
321 ret
= xsk_ring_prod__reserve(&xsk
->umem
->fq
,
322 XSK_RING_PROD__DEFAULT_NUM_DESCS
,
324 if (ret
!= XSK_RING_PROD__DEFAULT_NUM_DESCS
)
325 exit_with_error(-ret
);
327 i
< XSK_RING_PROD__DEFAULT_NUM_DESCS
*
328 XSK_UMEM__DEFAULT_FRAME_SIZE
;
329 i
+= XSK_UMEM__DEFAULT_FRAME_SIZE
)
330 *xsk_ring_prod__fill_addr(&xsk
->umem
->fq
, idx
++) = i
;
331 xsk_ring_prod__submit(&xsk
->umem
->fq
,
332 XSK_RING_PROD__DEFAULT_NUM_DESCS
);
337 static struct option long_options
[] = {
338 {"rxdrop", no_argument
, 0, 'r'},
339 {"txonly", no_argument
, 0, 't'},
340 {"l2fwd", no_argument
, 0, 'l'},
341 {"interface", required_argument
, 0, 'i'},
342 {"queue", required_argument
, 0, 'q'},
343 {"poll", no_argument
, 0, 'p'},
344 {"xdp-skb", no_argument
, 0, 'S'},
345 {"xdp-native", no_argument
, 0, 'N'},
346 {"interval", required_argument
, 0, 'n'},
347 {"zero-copy", no_argument
, 0, 'z'},
348 {"copy", no_argument
, 0, 'c'},
352 static void usage(const char *prog
)
355 " Usage: %s [OPTIONS]\n"
357 " -r, --rxdrop Discard all incoming packets (default)\n"
358 " -t, --txonly Only send packets\n"
359 " -l, --l2fwd MAC swap L2 forwarding\n"
360 " -i, --interface=n Run on interface n\n"
361 " -q, --queue=n Use queue n (default 0)\n"
362 " -p, --poll Use poll syscall\n"
363 " -S, --xdp-skb=n Use XDP skb-mod\n"
364 " -N, --xdp-native=n Enfore XDP native mode\n"
365 " -n, --interval=n Specify statistics update interval (default 1 sec).\n"
366 " -z, --zero-copy Force zero-copy mode.\n"
367 " -c, --copy Force copy mode.\n"
369 fprintf(stderr
, str
, prog
);
373 static void parse_command_line(int argc
, char **argv
)
380 c
= getopt_long(argc
, argv
, "Frtli:q:psSNn:cz", long_options
,
387 opt_bench
= BENCH_RXDROP
;
390 opt_bench
= BENCH_TXONLY
;
393 opt_bench
= BENCH_L2FWD
;
399 opt_queue
= atoi(optarg
);
405 opt_xdp_flags
|= XDP_FLAGS_SKB_MODE
;
406 opt_xdp_bind_flags
|= XDP_COPY
;
409 opt_xdp_flags
|= XDP_FLAGS_DRV_MODE
;
412 opt_interval
= atoi(optarg
);
415 opt_xdp_bind_flags
|= XDP_ZEROCOPY
;
418 opt_xdp_bind_flags
|= XDP_COPY
;
421 opt_xdp_flags
&= ~XDP_FLAGS_UPDATE_IF_NOEXIST
;
424 usage(basename(argv
[0]));
428 opt_ifindex
= if_nametoindex(opt_if
);
430 fprintf(stderr
, "ERROR: interface \"%s\" does not exist\n",
432 usage(basename(argv
[0]));
437 static void kick_tx(struct xsk_socket_info
*xsk
)
441 ret
= sendto(xsk_socket__fd(xsk
->xsk
), NULL
, 0, MSG_DONTWAIT
, NULL
, 0);
442 if (ret
>= 0 || errno
== ENOBUFS
|| errno
== EAGAIN
|| errno
== EBUSY
)
444 exit_with_error(errno
);
447 static inline void complete_tx_l2fwd(struct xsk_socket_info
*xsk
)
449 u32 idx_cq
= 0, idx_fq
= 0;
453 if (!xsk
->outstanding_tx
)
457 ndescs
= (xsk
->outstanding_tx
> BATCH_SIZE
) ? BATCH_SIZE
:
460 /* re-add completed Tx buffers */
461 rcvd
= xsk_ring_cons__peek(&xsk
->umem
->cq
, ndescs
, &idx_cq
);
466 ret
= xsk_ring_prod__reserve(&xsk
->umem
->fq
, rcvd
, &idx_fq
);
467 while (ret
!= rcvd
) {
469 exit_with_error(-ret
);
470 ret
= xsk_ring_prod__reserve(&xsk
->umem
->fq
, rcvd
,
473 for (i
= 0; i
< rcvd
; i
++)
474 *xsk_ring_prod__fill_addr(&xsk
->umem
->fq
, idx_fq
++) =
475 *xsk_ring_cons__comp_addr(&xsk
->umem
->cq
,
478 xsk_ring_prod__submit(&xsk
->umem
->fq
, rcvd
);
479 xsk_ring_cons__release(&xsk
->umem
->cq
, rcvd
);
480 xsk
->outstanding_tx
-= rcvd
;
481 xsk
->tx_npkts
+= rcvd
;
485 static inline void complete_tx_only(struct xsk_socket_info
*xsk
)
490 if (!xsk
->outstanding_tx
)
495 rcvd
= xsk_ring_cons__peek(&xsk
->umem
->cq
, BATCH_SIZE
, &idx
);
497 xsk_ring_cons__release(&xsk
->umem
->cq
, rcvd
);
498 xsk
->outstanding_tx
-= rcvd
;
499 xsk
->tx_npkts
+= rcvd
;
503 static void rx_drop(struct xsk_socket_info
*xsk
)
505 unsigned int rcvd
, i
;
506 u32 idx_rx
= 0, idx_fq
= 0;
509 rcvd
= xsk_ring_cons__peek(&xsk
->rx
, BATCH_SIZE
, &idx_rx
);
513 ret
= xsk_ring_prod__reserve(&xsk
->umem
->fq
, rcvd
, &idx_fq
);
514 while (ret
!= rcvd
) {
516 exit_with_error(-ret
);
517 ret
= xsk_ring_prod__reserve(&xsk
->umem
->fq
, rcvd
, &idx_fq
);
520 for (i
= 0; i
< rcvd
; i
++) {
521 u64 addr
= xsk_ring_cons__rx_desc(&xsk
->rx
, idx_rx
)->addr
;
522 u32 len
= xsk_ring_cons__rx_desc(&xsk
->rx
, idx_rx
++)->len
;
523 char *pkt
= xsk_umem__get_data(xsk
->umem
->buffer
, addr
);
525 hex_dump(pkt
, len
, addr
);
526 *xsk_ring_prod__fill_addr(&xsk
->umem
->fq
, idx_fq
++) = addr
;
529 xsk_ring_prod__submit(&xsk
->umem
->fq
, rcvd
);
530 xsk_ring_cons__release(&xsk
->rx
, rcvd
);
531 xsk
->rx_npkts
+= rcvd
;
534 static void rx_drop_all(void)
536 struct pollfd fds
[MAX_SOCKS
+ 1];
537 int i
, ret
, timeout
, nfds
= 1;
539 memset(fds
, 0, sizeof(fds
));
541 for (i
= 0; i
< num_socks
; i
++) {
542 fds
[i
].fd
= xsk_socket__fd(xsks
[i
]->xsk
);
543 fds
[i
].events
= POLLIN
;
544 timeout
= 1000; /* 1sn */
549 ret
= poll(fds
, nfds
, timeout
);
554 for (i
= 0; i
< num_socks
; i
++)
559 static void tx_only(struct xsk_socket_info
*xsk
)
561 int timeout
, ret
, nfds
= 1;
562 struct pollfd fds
[nfds
+ 1];
563 u32 idx
, frame_nb
= 0;
565 memset(fds
, 0, sizeof(fds
));
566 fds
[0].fd
= xsk_socket__fd(xsk
->xsk
);
567 fds
[0].events
= POLLOUT
;
568 timeout
= 1000; /* 1sn */
572 ret
= poll(fds
, nfds
, timeout
);
576 if (!(fds
[0].revents
& POLLOUT
))
580 if (xsk_ring_prod__reserve(&xsk
->tx
, BATCH_SIZE
, &idx
) ==
584 for (i
= 0; i
< BATCH_SIZE
; i
++) {
585 xsk_ring_prod__tx_desc(&xsk
->tx
, idx
+ i
)->addr
587 XSK_UMEM__DEFAULT_FRAME_SHIFT
;
588 xsk_ring_prod__tx_desc(&xsk
->tx
, idx
+ i
)->len
=
589 sizeof(pkt_data
) - 1;
592 xsk_ring_prod__submit(&xsk
->tx
, BATCH_SIZE
);
593 xsk
->outstanding_tx
+= BATCH_SIZE
;
594 frame_nb
+= BATCH_SIZE
;
595 frame_nb
%= NUM_FRAMES
;
598 complete_tx_only(xsk
);
602 static void l2fwd(struct xsk_socket_info
*xsk
)
605 unsigned int rcvd
, i
;
606 u32 idx_rx
= 0, idx_tx
= 0;
610 complete_tx_l2fwd(xsk
);
612 rcvd
= xsk_ring_cons__peek(&xsk
->rx
, BATCH_SIZE
,
618 ret
= xsk_ring_prod__reserve(&xsk
->tx
, rcvd
, &idx_tx
);
619 while (ret
!= rcvd
) {
621 exit_with_error(-ret
);
622 ret
= xsk_ring_prod__reserve(&xsk
->tx
, rcvd
, &idx_tx
);
625 for (i
= 0; i
< rcvd
; i
++) {
626 u64 addr
= xsk_ring_cons__rx_desc(&xsk
->rx
,
628 u32 len
= xsk_ring_cons__rx_desc(&xsk
->rx
,
630 char *pkt
= xsk_umem__get_data(xsk
->umem
->buffer
, addr
);
632 swap_mac_addresses(pkt
);
634 hex_dump(pkt
, len
, addr
);
635 xsk_ring_prod__tx_desc(&xsk
->tx
, idx_tx
)->addr
= addr
;
636 xsk_ring_prod__tx_desc(&xsk
->tx
, idx_tx
++)->len
= len
;
639 xsk_ring_prod__submit(&xsk
->tx
, rcvd
);
640 xsk_ring_cons__release(&xsk
->rx
, rcvd
);
642 xsk
->rx_npkts
+= rcvd
;
643 xsk
->outstanding_tx
+= rcvd
;
647 int main(int argc
, char **argv
)
649 struct rlimit r
= {RLIM_INFINITY
, RLIM_INFINITY
};
650 struct xsk_umem_info
*umem
;
655 parse_command_line(argc
, argv
);
657 if (setrlimit(RLIMIT_MEMLOCK
, &r
)) {
658 fprintf(stderr
, "ERROR: setrlimit(RLIMIT_MEMLOCK) \"%s\"\n",
663 ret
= posix_memalign(&bufs
, getpagesize(), /* PAGE_SIZE aligned */
664 NUM_FRAMES
* XSK_UMEM__DEFAULT_FRAME_SIZE
);
666 exit_with_error(ret
);
668 /* Create sockets... */
669 umem
= xsk_configure_umem(bufs
,
670 NUM_FRAMES
* XSK_UMEM__DEFAULT_FRAME_SIZE
);
671 xsks
[num_socks
++] = xsk_configure_socket(umem
);
673 if (opt_bench
== BENCH_TXONLY
) {
676 for (i
= 0; i
< NUM_FRAMES
* XSK_UMEM__DEFAULT_FRAME_SIZE
;
677 i
+= XSK_UMEM__DEFAULT_FRAME_SIZE
)
678 (void)gen_eth_frame(umem
, i
);
681 signal(SIGINT
, int_exit
);
682 signal(SIGTERM
, int_exit
);
683 signal(SIGABRT
, int_exit
);
685 setlocale(LC_ALL
, "");
687 ret
= pthread_create(&pt
, NULL
, poller
, NULL
);
689 exit_with_error(ret
);
691 prev_time
= get_nsecs();
693 if (opt_bench
== BENCH_RXDROP
)
695 else if (opt_bench
== BENCH_TXONLY
)