drivers/net/ethernet/microsoft/mana/mana_bpf.c

   1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
   2 /* Copyright (c) 2021, Microsoft Corporation. */
   3
   4 #include <linux/inetdevice.h>
   5 #include <linux/etherdevice.h>
   6 #include <linux/mm.h>
   7 #include <linux/bpf.h>
   8 #include <linux/bpf_trace.h>
   9 #include <net/xdp.h>
  10
  11 #include <net/mana/mana.h>
  12
  13 void mana_xdp_tx(struct sk_buff *skb, struct net_device *ndev)
  14 {
  15         u16 txq_idx = skb_get_queue_mapping(skb);
  16         struct netdev_queue *ndevtxq;
  17         int rc;
  18
  19         __skb_push(skb, ETH_HLEN);
  20
  21         ndevtxq = netdev_get_tx_queue(ndev, txq_idx);
  22         __netif_tx_lock(ndevtxq, smp_processor_id());
  23
  24         rc = mana_start_xmit(skb, ndev);
  25
  26         __netif_tx_unlock(ndevtxq);
  27
  28         if (dev_xmit_complete(rc))
  29                 return;
  30
  31         dev_kfree_skb_any(skb);
  32         ndev->stats.tx_dropped++;
  33 }
  34
  35 static int mana_xdp_xmit_fm(struct net_device *ndev, struct xdp_frame *frame,
  36                             u16 q_idx)
  37 {
  38         struct sk_buff *skb;
  39
  40         skb = xdp_build_skb_from_frame(frame, ndev);
  41         if (unlikely(!skb))
  42                 return -ENOMEM;
  43
  44         skb_set_queue_mapping(skb, q_idx);
  45
  46         mana_xdp_tx(skb, ndev);
  47
  48         return 0;
  49 }
  50
  51 int mana_xdp_xmit(struct net_device *ndev, int n, struct xdp_frame **frames,
  52                   u32 flags)
  53 {
  54         struct mana_port_context *apc = netdev_priv(ndev);
  55         struct mana_stats_tx *tx_stats;
  56         int i, count = 0;
  57         u16 q_idx;
  58
  59         if (unlikely(!apc->port_is_up))
  60                 return 0;
  61
  62         q_idx = smp_processor_id() % ndev->real_num_tx_queues;
  63
  64         for (i = 0; i < n; i++) {
  65                 if (mana_xdp_xmit_fm(ndev, frames[i], q_idx))
  66                         break;
  67
  68                 count++;
  69         }
  70
  71         tx_stats = &apc->tx_qp[q_idx].txq.stats;
  72
  73         u64_stats_update_begin(&tx_stats->syncp);
  74         tx_stats->xdp_xmit += count;
  75         u64_stats_update_end(&tx_stats->syncp);
  76
  77         return count;
  78 }
  79
  80 u32 mana_run_xdp(struct net_device *ndev, struct mana_rxq *rxq,
  81                  struct xdp_buff *xdp, void *buf_va, uint pkt_len)
  82 {
  83         struct mana_stats_rx *rx_stats;
  84         struct bpf_prog *prog;
  85         u32 act = XDP_PASS;
  86
  87         rcu_read_lock();
  88         prog = rcu_dereference(rxq->bpf_prog);
  89
  90         if (!prog)
  91                 goto out;
  92
  93         xdp_init_buff(xdp, PAGE_SIZE, &rxq->xdp_rxq);
  94         xdp_prepare_buff(xdp, buf_va, XDP_PACKET_HEADROOM, pkt_len, false);
  95
  96         act = bpf_prog_run_xdp(prog, xdp);
  97
  98         rx_stats = &rxq->stats;
  99
 100         switch (act) {
 101         case XDP_PASS:
 102         case XDP_TX:
 103         case XDP_DROP:
 104                 break;
 105
 106         case XDP_REDIRECT:
 107                 rxq->xdp_rc = xdp_do_redirect(ndev, xdp, prog);
 108                 if (!rxq->xdp_rc) {
 109                         rxq->xdp_flush = true;
 110
 111                         u64_stats_update_begin(&rx_stats->syncp);
 112                         rx_stats->packets++;
 113                         rx_stats->bytes += pkt_len;
 114                         rx_stats->xdp_redirect++;
 115                         u64_stats_update_end(&rx_stats->syncp);
 116
 117                         break;
 118                 }
 119
 120                 fallthrough;
 121
 122         case XDP_ABORTED:
 123                 trace_xdp_exception(ndev, prog, act);
 124                 break;
 125
 126         default:
 127                 bpf_warn_invalid_xdp_action(ndev, prog, act);
 128         }
 129
 130 out:
 131         rcu_read_unlock();
 132
 133         return act;
 134 }
 135
 136 struct bpf_prog *mana_xdp_get(struct mana_port_context *apc)
 137 {
 138         ASSERT_RTNL();
 139
 140         return apc->bpf_prog;
 141 }
 142
 143 static struct bpf_prog *mana_chn_xdp_get(struct mana_port_context *apc)
 144 {
 145         return rtnl_dereference(apc->rxqs[0]->bpf_prog);
 146 }
 147
 148 /* Set xdp program on channels */
 149 void mana_chn_setxdp(struct mana_port_context *apc, struct bpf_prog *prog)
 150 {
 151         struct bpf_prog *old_prog = mana_chn_xdp_get(apc);
 152         unsigned int num_queues = apc->num_queues;
 153         int i;
 154
 155         ASSERT_RTNL();
 156
 157         if (old_prog == prog)
 158                 return;
 159
 160         if (prog)
 161                 bpf_prog_add(prog, num_queues);
 162
 163         for (i = 0; i < num_queues; i++)
 164                 rcu_assign_pointer(apc->rxqs[i]->bpf_prog, prog);
 165
 166         if (old_prog)
 167                 for (i = 0; i < num_queues; i++)
 168                         bpf_prog_put(old_prog);
 169 }
 170
 171 static int mana_xdp_set(struct net_device *ndev, struct bpf_prog *prog,
 172                         struct netlink_ext_ack *extack)
 173 {
 174         struct mana_port_context *apc = netdev_priv(ndev);
 175         struct bpf_prog *old_prog;
 176         struct gdma_context *gc;
 177
 178         gc = apc->ac->gdma_dev->gdma_context;
 179
 180         old_prog = mana_xdp_get(apc);
 181
 182         if (!old_prog && !prog)
 183                 return 0;
 184
 185         if (prog && ndev->mtu > MANA_XDP_MTU_MAX) {
 186                 netdev_err(ndev, "XDP: mtu:%u too large, mtu_max:%lu\n",
 187                            ndev->mtu, MANA_XDP_MTU_MAX);
 188                 NL_SET_ERR_MSG_MOD(extack, "XDP: mtu too large");
 189
 190                 return -EOPNOTSUPP;
 191         }
 192
 193         /* One refcnt of the prog is hold by the caller already, so
 194          * don't increase refcnt for this one.
 195          */
 196         apc->bpf_prog = prog;
 197
 198         if (old_prog)
 199                 bpf_prog_put(old_prog);
 200
 201         if (apc->port_is_up)
 202                 mana_chn_setxdp(apc, prog);
 203
 204         if (prog)
 205                 ndev->max_mtu = MANA_XDP_MTU_MAX;
 206         else
 207                 ndev->max_mtu = gc->adapter_mtu - ETH_HLEN;
 208
 209         return 0;
 210 }
 211
 212 int mana_bpf(struct net_device *ndev, struct netdev_bpf *bpf)
 213 {
 214         struct netlink_ext_ack *extack = bpf->extack;
 215         int ret;
 216
 217         switch (bpf->command) {
 218         case XDP_SETUP_PROG:
 219                 return mana_xdp_set(ndev, bpf->prog, extack);
 220
 221         default:
 222                 return -EOPNOTSUPP;
 223         }
 224
 225         return ret;
 226 }