tools/lib/bpf/xsk.h

   1 /* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
   2
   3 /*
   4  * AF_XDP user-space access library.
   5  *
   6  * Copyright(c) 2018 - 2019 Intel Corporation.
   7  *
   8  * Author(s): Magnus Karlsson <magnus.karlsson@intel.com>
   9  */
  10
  11 #ifndef __LIBBPF_XSK_H
  12 #define __LIBBPF_XSK_H
  13
  14 #include <stdio.h>
  15 #include <stdint.h>
  16 #include <linux/if_xdp.h>
  17
  18 #include "libbpf.h"
  19 #include "libbpf_util.h"
  20
  21 #ifdef __cplusplus
  22 extern "C" {
  23 #endif
  24
  25 /* Do not access these members directly. Use the functions below. */
  26 #define DEFINE_XSK_RING(name) \
  27 struct name { \
  28         __u32 cached_prod; \
  29         __u32 cached_cons; \
  30         __u32 mask; \
  31         __u32 size; \
  32         __u32 *producer; \
  33         __u32 *consumer; \
  34         void *ring; \
  35         __u32 *flags; \
  36 }
  37
  38 DEFINE_XSK_RING(xsk_ring_prod);
  39 DEFINE_XSK_RING(xsk_ring_cons);
  40
  41 /* For a detailed explanation on the memory barriers associated with the
  42  * ring, please take a look at net/xdp/xsk_queue.h.
  43  */
  44
  45 struct xsk_umem;
  46 struct xsk_socket;
  47
  48 static inline __u64 *xsk_ring_prod__fill_addr(struct xsk_ring_prod *fill,
  49                                               __u32 idx)
  50 {
  51         __u64 *addrs = (__u64 *)fill->ring;
  52
  53         return &addrs[idx & fill->mask];
  54 }
  55
  56 static inline const __u64 *
  57 xsk_ring_cons__comp_addr(const struct xsk_ring_cons *comp, __u32 idx)
  58 {
  59         const __u64 *addrs = (const __u64 *)comp->ring;
  60
  61         return &addrs[idx & comp->mask];
  62 }
  63
  64 static inline struct xdp_desc *xsk_ring_prod__tx_desc(struct xsk_ring_prod *tx,
  65                                                       __u32 idx)
  66 {
  67         struct xdp_desc *descs = (struct xdp_desc *)tx->ring;
  68
  69         return &descs[idx & tx->mask];
  70 }
  71
  72 static inline const struct xdp_desc *
  73 xsk_ring_cons__rx_desc(const struct xsk_ring_cons *rx, __u32 idx)
  74 {
  75         const struct xdp_desc *descs = (const struct xdp_desc *)rx->ring;
  76
  77         return &descs[idx & rx->mask];
  78 }
  79
  80 static inline int xsk_ring_prod__needs_wakeup(const struct xsk_ring_prod *r)
  81 {
  82         return *r->flags & XDP_RING_NEED_WAKEUP;
  83 }
  84
  85 static inline __u32 xsk_prod_nb_free(struct xsk_ring_prod *r, __u32 nb)
  86 {
  87         __u32 free_entries = r->cached_cons - r->cached_prod;
  88
  89         if (free_entries >= nb)
  90                 return free_entries;
  91
  92         /* Refresh the local tail pointer.
  93          * cached_cons is r->size bigger than the real consumer pointer so
  94          * that this addition can be avoided in the more frequently
  95          * executed code that computs free_entries in the beginning of
  96          * this function. Without this optimization it whould have been
  97          * free_entries = r->cached_prod - r->cached_cons + r->size.
  98          */
  99         r->cached_cons = *r->consumer + r->size;
 100
 101         return r->cached_cons - r->cached_prod;
 102 }
 103
 104 static inline __u32 xsk_cons_nb_avail(struct xsk_ring_cons *r, __u32 nb)
 105 {
 106         __u32 entries = r->cached_prod - r->cached_cons;
 107
 108         if (entries == 0) {
 109                 r->cached_prod = *r->producer;
 110                 entries = r->cached_prod - r->cached_cons;
 111         }
 112
 113         return (entries > nb) ? nb : entries;
 114 }
 115
 116 static inline __u32 xsk_ring_prod__reserve(struct xsk_ring_prod *prod, __u32 nb, __u32 *idx)
 117 {
 118         if (xsk_prod_nb_free(prod, nb) < nb)
 119                 return 0;
 120
 121         *idx = prod->cached_prod;
 122         prod->cached_prod += nb;
 123
 124         return nb;
 125 }
 126
 127 static inline void xsk_ring_prod__submit(struct xsk_ring_prod *prod, __u32 nb)
 128 {
 129         /* Make sure everything has been written to the ring before indicating
 130          * this to the kernel by writing the producer pointer.
 131          */
 132         libbpf_smp_wmb();
 133
 134         *prod->producer += nb;
 135 }
 136
 137 static inline __u32 xsk_ring_cons__peek(struct xsk_ring_cons *cons, __u32 nb, __u32 *idx)
 138 {
 139         __u32 entries = xsk_cons_nb_avail(cons, nb);
 140
 141         if (entries > 0) {
 142                 /* Make sure we do not speculatively read the data before
 143                  * we have received the packet buffers from the ring.
 144                  */
 145                 libbpf_smp_rmb();
 146
 147                 *idx = cons->cached_cons;
 148                 cons->cached_cons += entries;
 149         }
 150
 151         return entries;
 152 }
 153
 154 static inline void xsk_ring_cons__cancel(struct xsk_ring_cons *cons, __u32 nb)
 155 {
 156         cons->cached_cons -= nb;
 157 }
 158
 159 static inline void xsk_ring_cons__release(struct xsk_ring_cons *cons, __u32 nb)
 160 {
 161         /* Make sure data has been read before indicating we are done
 162          * with the entries by updating the consumer pointer.
 163          */
 164         libbpf_smp_rwmb();
 165
 166         *cons->consumer += nb;
 167 }
 168
 169 static inline void *xsk_umem__get_data(void *umem_area, __u64 addr)
 170 {
 171         return &((char *)umem_area)[addr];
 172 }
 173
 174 static inline __u64 xsk_umem__extract_addr(__u64 addr)
 175 {
 176         return addr & XSK_UNALIGNED_BUF_ADDR_MASK;
 177 }
 178
 179 static inline __u64 xsk_umem__extract_offset(__u64 addr)
 180 {
 181         return addr >> XSK_UNALIGNED_BUF_OFFSET_SHIFT;
 182 }
 183
 184 static inline __u64 xsk_umem__add_offset_to_addr(__u64 addr)
 185 {
 186         return xsk_umem__extract_addr(addr) + xsk_umem__extract_offset(addr);
 187 }
 188
 189 LIBBPF_API int xsk_umem__fd(const struct xsk_umem *umem);
 190 LIBBPF_API int xsk_socket__fd(const struct xsk_socket *xsk);
 191
 192 #define XSK_RING_CONS__DEFAULT_NUM_DESCS      2048
 193 #define XSK_RING_PROD__DEFAULT_NUM_DESCS      2048
 194 #define XSK_UMEM__DEFAULT_FRAME_SHIFT    12 /* 4096 bytes */
 195 #define XSK_UMEM__DEFAULT_FRAME_SIZE     (1 << XSK_UMEM__DEFAULT_FRAME_SHIFT)
 196 #define XSK_UMEM__DEFAULT_FRAME_HEADROOM 0
 197 #define XSK_UMEM__DEFAULT_FLAGS 0
 198
 199 struct xsk_umem_config {
 200         __u32 fill_size;
 201         __u32 comp_size;
 202         __u32 frame_size;
 203         __u32 frame_headroom;
 204         __u32 flags;
 205 };
 206
 207 LIBBPF_API int xsk_setup_xdp_prog(int ifindex,
 208                                   int *xsks_map_fd);
 209 LIBBPF_API int xsk_socket__update_xskmap(struct xsk_socket *xsk,
 210                                          int xsks_map_fd);
 211
 212 /* Flags for the libbpf_flags field. */
 213 #define XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD (1 << 0)
 214
 215 struct xsk_socket_config {
 216         __u32 rx_size;
 217         __u32 tx_size;
 218         __u32 libbpf_flags;
 219         __u32 xdp_flags;
 220         __u16 bind_flags;
 221 };
 222
 223 /* Set config to NULL to get the default configuration. */
 224 LIBBPF_API int xsk_umem__create(struct xsk_umem **umem,
 225                                 void *umem_area, __u64 size,
 226                                 struct xsk_ring_prod *fill,
 227                                 struct xsk_ring_cons *comp,
 228                                 const struct xsk_umem_config *config);
 229 LIBBPF_API int xsk_umem__create_v0_0_2(struct xsk_umem **umem,
 230                                        void *umem_area, __u64 size,
 231                                        struct xsk_ring_prod *fill,
 232                                        struct xsk_ring_cons *comp,
 233                                        const struct xsk_umem_config *config);
 234 LIBBPF_API int xsk_umem__create_v0_0_4(struct xsk_umem **umem,
 235                                        void *umem_area, __u64 size,
 236                                        struct xsk_ring_prod *fill,
 237                                        struct xsk_ring_cons *comp,
 238                                        const struct xsk_umem_config *config);
 239 LIBBPF_API int xsk_socket__create(struct xsk_socket **xsk,
 240                                   const char *ifname, __u32 queue_id,
 241                                   struct xsk_umem *umem,
 242                                   struct xsk_ring_cons *rx,
 243                                   struct xsk_ring_prod *tx,
 244                                   const struct xsk_socket_config *config);
 245 LIBBPF_API int
 246 xsk_socket__create_shared(struct xsk_socket **xsk_ptr,
 247                           const char *ifname,
 248                           __u32 queue_id, struct xsk_umem *umem,
 249                           struct xsk_ring_cons *rx,
 250                           struct xsk_ring_prod *tx,
 251                           struct xsk_ring_prod *fill,
 252                           struct xsk_ring_cons *comp,
 253                           const struct xsk_socket_config *config);
 254
 255 /* Returns 0 for success and -EBUSY if the umem is still in use. */
 256 LIBBPF_API int xsk_umem__delete(struct xsk_umem *umem);
 257 LIBBPF_API void xsk_socket__delete(struct xsk_socket *xsk);
 258
 259 #ifdef __cplusplus
 260 } /* extern "C" */
 261 #endif
 262
 263 #endif /* __LIBBPF_XSK_H */