drivers/net/ethernet/netronome/nfp/nfp_net_dp.c

   1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
   2 /* Copyright (C) 2015-2019 Netronome Systems, Inc. */
   3
   4 #include "nfp_app.h"
   5 #include "nfp_net_dp.h"
   6 #include "nfp_net_xsk.h"
   7
   8 /**
   9  * nfp_net_rx_alloc_one() - Allocate and map page frag for RX
  10  * @dp:         NFP Net data path struct
  11  * @dma_addr:   Pointer to storage for DMA address (output param)
  12  *
  13  * This function will allcate a new page frag, map it for DMA.
  14  *
  15  * Return: allocated page frag or NULL on failure.
  16  */
  17 void *nfp_net_rx_alloc_one(struct nfp_net_dp *dp, dma_addr_t *dma_addr)
  18 {
  19         void *frag;
  20
  21         if (!dp->xdp_prog) {
  22                 frag = netdev_alloc_frag(dp->fl_bufsz);
  23         } else {
  24                 struct page *page;
  25
  26                 page = alloc_page(GFP_KERNEL);
  27                 frag = page ? page_address(page) : NULL;
  28         }
  29         if (!frag) {
  30                 nn_dp_warn(dp, "Failed to alloc receive page frag\n");
  31                 return NULL;
  32         }
  33
  34         *dma_addr = nfp_net_dma_map_rx(dp, frag);
  35         if (dma_mapping_error(dp->dev, *dma_addr)) {
  36                 nfp_net_free_frag(frag, dp->xdp_prog);
  37                 nn_dp_warn(dp, "Failed to map DMA RX buffer\n");
  38                 return NULL;
  39         }
  40
  41         return frag;
  42 }
  43
  44 /**
  45  * nfp_net_tx_ring_init() - Fill in the boilerplate for a TX ring
  46  * @tx_ring:  TX ring structure
  47  * @dp:       NFP Net data path struct
  48  * @r_vec:    IRQ vector servicing this ring
  49  * @idx:      Ring index
  50  * @is_xdp:   Is this an XDP TX ring?
  51  */
  52 static void
  53 nfp_net_tx_ring_init(struct nfp_net_tx_ring *tx_ring, struct nfp_net_dp *dp,
  54                      struct nfp_net_r_vector *r_vec, unsigned int idx,
  55                      bool is_xdp)
  56 {
  57         struct nfp_net *nn = r_vec->nfp_net;
  58
  59         tx_ring->idx = idx;
  60         tx_ring->r_vec = r_vec;
  61         tx_ring->is_xdp = is_xdp;
  62         u64_stats_init(&tx_ring->r_vec->tx_sync);
  63
  64         tx_ring->qcidx = tx_ring->idx * nn->stride_tx;
  65         tx_ring->txrwb = dp->txrwb ? &dp->txrwb[idx] : NULL;
  66         tx_ring->qcp_q = nn->tx_bar + NFP_QCP_QUEUE_OFF(tx_ring->qcidx);
  67 }
  68
  69 /**
  70  * nfp_net_rx_ring_init() - Fill in the boilerplate for a RX ring
  71  * @rx_ring:  RX ring structure
  72  * @r_vec:    IRQ vector servicing this ring
  73  * @idx:      Ring index
  74  */
  75 static void
  76 nfp_net_rx_ring_init(struct nfp_net_rx_ring *rx_ring,
  77                      struct nfp_net_r_vector *r_vec, unsigned int idx)
  78 {
  79         struct nfp_net *nn = r_vec->nfp_net;
  80
  81         rx_ring->idx = idx;
  82         rx_ring->r_vec = r_vec;
  83         u64_stats_init(&rx_ring->r_vec->rx_sync);
  84
  85         rx_ring->fl_qcidx = rx_ring->idx * nn->stride_rx;
  86         rx_ring->qcp_fl = nn->rx_bar + NFP_QCP_QUEUE_OFF(rx_ring->fl_qcidx);
  87 }
  88
  89 /**
  90  * nfp_net_rx_ring_reset() - Reflect in SW state of freelist after disable
  91  * @rx_ring:    RX ring structure
  92  *
  93  * Assumes that the device is stopped, must be idempotent.
  94  */
  95 void nfp_net_rx_ring_reset(struct nfp_net_rx_ring *rx_ring)
  96 {
  97         unsigned int wr_idx, last_idx;
  98
  99         /* wr_p == rd_p means ring was never fed FL bufs.  RX rings are always
 100          * kept at cnt - 1 FL bufs.
 101          */
 102         if (rx_ring->wr_p == 0 && rx_ring->rd_p == 0)
 103                 return;
 104
 105         /* Move the empty entry to the end of the list */
 106         wr_idx = D_IDX(rx_ring, rx_ring->wr_p);
 107         last_idx = rx_ring->cnt - 1;
 108         if (rx_ring->r_vec->xsk_pool) {
 109                 rx_ring->xsk_rxbufs[wr_idx] = rx_ring->xsk_rxbufs[last_idx];
 110                 memset(&rx_ring->xsk_rxbufs[last_idx], 0,
 111                        sizeof(*rx_ring->xsk_rxbufs));
 112         } else {
 113                 rx_ring->rxbufs[wr_idx] = rx_ring->rxbufs[last_idx];
 114                 memset(&rx_ring->rxbufs[last_idx], 0, sizeof(*rx_ring->rxbufs));
 115         }
 116
 117         memset(rx_ring->rxds, 0, rx_ring->size);
 118         rx_ring->wr_p = 0;
 119         rx_ring->rd_p = 0;
 120 }
 121
 122 /**
 123  * nfp_net_rx_ring_bufs_free() - Free any buffers currently on the RX ring
 124  * @dp:         NFP Net data path struct
 125  * @rx_ring:    RX ring to remove buffers from
 126  *
 127  * Assumes that the device is stopped and buffers are in [0, ring->cnt - 1)
 128  * entries.  After device is disabled nfp_net_rx_ring_reset() must be called
 129  * to restore required ring geometry.
 130  */
 131 static void
 132 nfp_net_rx_ring_bufs_free(struct nfp_net_dp *dp,
 133                           struct nfp_net_rx_ring *rx_ring)
 134 {
 135         unsigned int i;
 136
 137         if (nfp_net_has_xsk_pool_slow(dp, rx_ring->idx))
 138                 return;
 139
 140         for (i = 0; i < rx_ring->cnt - 1; i++) {
 141                 /* NULL skb can only happen when initial filling of the ring
 142                  * fails to allocate enough buffers and calls here to free
 143                  * already allocated ones.
 144                  */
 145                 if (!rx_ring->rxbufs[i].frag)
 146                         continue;
 147
 148                 nfp_net_dma_unmap_rx(dp, rx_ring->rxbufs[i].dma_addr);
 149                 nfp_net_free_frag(rx_ring->rxbufs[i].frag, dp->xdp_prog);
 150                 rx_ring->rxbufs[i].dma_addr = 0;
 151                 rx_ring->rxbufs[i].frag = NULL;
 152         }
 153 }
 154
 155 /**
 156  * nfp_net_rx_ring_bufs_alloc() - Fill RX ring with buffers (don't give to FW)
 157  * @dp:         NFP Net data path struct
 158  * @rx_ring:    RX ring to remove buffers from
 159  */
 160 static int
 161 nfp_net_rx_ring_bufs_alloc(struct nfp_net_dp *dp,
 162                            struct nfp_net_rx_ring *rx_ring)
 163 {
 164         struct nfp_net_rx_buf *rxbufs;
 165         unsigned int i;
 166
 167         if (nfp_net_has_xsk_pool_slow(dp, rx_ring->idx))
 168                 return 0;
 169
 170         rxbufs = rx_ring->rxbufs;
 171
 172         for (i = 0; i < rx_ring->cnt - 1; i++) {
 173                 rxbufs[i].frag = nfp_net_rx_alloc_one(dp, &rxbufs[i].dma_addr);
 174                 if (!rxbufs[i].frag) {
 175                         nfp_net_rx_ring_bufs_free(dp, rx_ring);
 176                         return -ENOMEM;
 177                 }
 178         }
 179
 180         return 0;
 181 }
 182
 183 int nfp_net_tx_rings_prepare(struct nfp_net *nn, struct nfp_net_dp *dp)
 184 {
 185         unsigned int r;
 186
 187         dp->tx_rings = kcalloc(dp->num_tx_rings, sizeof(*dp->tx_rings),
 188                                GFP_KERNEL);
 189         if (!dp->tx_rings)
 190                 return -ENOMEM;
 191
 192         if (dp->ctrl & NFP_NET_CFG_CTRL_TXRWB) {
 193                 dp->txrwb = dma_alloc_coherent(dp->dev,
 194                                                dp->num_tx_rings * sizeof(u64),
 195                                                &dp->txrwb_dma, GFP_KERNEL);
 196                 if (!dp->txrwb)
 197                         goto err_free_rings;
 198         }
 199
 200         for (r = 0; r < dp->num_tx_rings; r++) {
 201                 int bias = 0;
 202
 203                 if (r >= dp->num_stack_tx_rings)
 204                         bias = dp->num_stack_tx_rings;
 205
 206                 nfp_net_tx_ring_init(&dp->tx_rings[r], dp,
 207                                      &nn->r_vecs[r - bias], r, bias);
 208
 209                 if (nfp_net_tx_ring_alloc(dp, &dp->tx_rings[r]))
 210                         goto err_free_prev;
 211
 212                 if (nfp_net_tx_ring_bufs_alloc(dp, &dp->tx_rings[r]))
 213                         goto err_free_ring;
 214         }
 215
 216         return 0;
 217
 218 err_free_prev:
 219         while (r--) {
 220                 nfp_net_tx_ring_bufs_free(dp, &dp->tx_rings[r]);
 221 err_free_ring:
 222                 nfp_net_tx_ring_free(dp, &dp->tx_rings[r]);
 223         }
 224         if (dp->txrwb)
 225                 dma_free_coherent(dp->dev, dp->num_tx_rings * sizeof(u64),
 226                                   dp->txrwb, dp->txrwb_dma);
 227 err_free_rings:
 228         kfree(dp->tx_rings);
 229         return -ENOMEM;
 230 }
 231
 232 void nfp_net_tx_rings_free(struct nfp_net_dp *dp)
 233 {
 234         unsigned int r;
 235
 236         for (r = 0; r < dp->num_tx_rings; r++) {
 237                 nfp_net_tx_ring_bufs_free(dp, &dp->tx_rings[r]);
 238                 nfp_net_tx_ring_free(dp, &dp->tx_rings[r]);
 239         }
 240
 241         if (dp->txrwb)
 242                 dma_free_coherent(dp->dev, dp->num_tx_rings * sizeof(u64),
 243                                   dp->txrwb, dp->txrwb_dma);
 244         kfree(dp->tx_rings);
 245 }
 246
 247 /**
 248  * nfp_net_rx_ring_free() - Free resources allocated to a RX ring
 249  * @rx_ring:  RX ring to free
 250  */
 251 static void nfp_net_rx_ring_free(struct nfp_net_rx_ring *rx_ring)
 252 {
 253         struct nfp_net_r_vector *r_vec = rx_ring->r_vec;
 254         struct nfp_net_dp *dp = &r_vec->nfp_net->dp;
 255
 256         if (dp->netdev)
 257                 xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
 258
 259         if (nfp_net_has_xsk_pool_slow(dp, rx_ring->idx))
 260                 kvfree(rx_ring->xsk_rxbufs);
 261         else
 262                 kvfree(rx_ring->rxbufs);
 263
 264         if (rx_ring->rxds)
 265                 dma_free_coherent(dp->dev, rx_ring->size,
 266                                   rx_ring->rxds, rx_ring->dma);
 267
 268         rx_ring->cnt = 0;
 269         rx_ring->rxbufs = NULL;
 270         rx_ring->xsk_rxbufs = NULL;
 271         rx_ring->rxds = NULL;
 272         rx_ring->dma = 0;
 273         rx_ring->size = 0;
 274 }
 275
 276 /**
 277  * nfp_net_rx_ring_alloc() - Allocate resource for a RX ring
 278  * @dp:       NFP Net data path struct
 279  * @rx_ring:  RX ring to allocate
 280  *
 281  * Return: 0 on success, negative errno otherwise.
 282  */
 283 static int
 284 nfp_net_rx_ring_alloc(struct nfp_net_dp *dp, struct nfp_net_rx_ring *rx_ring)
 285 {
 286         enum xdp_mem_type mem_type;
 287         size_t rxbuf_sw_desc_sz;
 288         int err;
 289
 290         if (nfp_net_has_xsk_pool_slow(dp, rx_ring->idx)) {
 291                 mem_type = MEM_TYPE_XSK_BUFF_POOL;
 292                 rxbuf_sw_desc_sz = sizeof(*rx_ring->xsk_rxbufs);
 293         } else {
 294                 mem_type = MEM_TYPE_PAGE_ORDER0;
 295                 rxbuf_sw_desc_sz = sizeof(*rx_ring->rxbufs);
 296         }
 297
 298         if (dp->netdev) {
 299                 err = xdp_rxq_info_reg(&rx_ring->xdp_rxq, dp->netdev,
 300                                        rx_ring->idx, rx_ring->r_vec->napi.napi_id);
 301                 if (err < 0)
 302                         return err;
 303
 304                 err = xdp_rxq_info_reg_mem_model(&rx_ring->xdp_rxq, mem_type, NULL);
 305                 if (err)
 306                         goto err_alloc;
 307         }
 308
 309         rx_ring->cnt = dp->rxd_cnt;
 310         rx_ring->size = array_size(rx_ring->cnt, sizeof(*rx_ring->rxds));
 311         rx_ring->rxds = dma_alloc_coherent(dp->dev, rx_ring->size,
 312                                            &rx_ring->dma,
 313                                            GFP_KERNEL | __GFP_NOWARN);
 314         if (!rx_ring->rxds) {
 315                 netdev_warn(dp->netdev, "failed to allocate RX descriptor ring memory, requested descriptor count: %d, consider lowering descriptor count\n",
 316                             rx_ring->cnt);
 317                 goto err_alloc;
 318         }
 319
 320         if (nfp_net_has_xsk_pool_slow(dp, rx_ring->idx)) {
 321                 rx_ring->xsk_rxbufs = kvcalloc(rx_ring->cnt, rxbuf_sw_desc_sz,
 322                                                GFP_KERNEL);
 323                 if (!rx_ring->xsk_rxbufs)
 324                         goto err_alloc;
 325         } else {
 326                 rx_ring->rxbufs = kvcalloc(rx_ring->cnt, rxbuf_sw_desc_sz,
 327                                            GFP_KERNEL);
 328                 if (!rx_ring->rxbufs)
 329                         goto err_alloc;
 330         }
 331
 332         return 0;
 333
 334 err_alloc:
 335         nfp_net_rx_ring_free(rx_ring);
 336         return -ENOMEM;
 337 }
 338
 339 int nfp_net_rx_rings_prepare(struct nfp_net *nn, struct nfp_net_dp *dp)
 340 {
 341         unsigned int r;
 342
 343         dp->rx_rings = kcalloc(dp->num_rx_rings, sizeof(*dp->rx_rings),
 344                                GFP_KERNEL);
 345         if (!dp->rx_rings)
 346                 return -ENOMEM;
 347
 348         for (r = 0; r < dp->num_rx_rings; r++) {
 349                 nfp_net_rx_ring_init(&dp->rx_rings[r], &nn->r_vecs[r], r);
 350
 351                 if (nfp_net_rx_ring_alloc(dp, &dp->rx_rings[r]))
 352                         goto err_free_prev;
 353
 354                 if (nfp_net_rx_ring_bufs_alloc(dp, &dp->rx_rings[r]))
 355                         goto err_free_ring;
 356         }
 357
 358         return 0;
 359
 360 err_free_prev:
 361         while (r--) {
 362                 nfp_net_rx_ring_bufs_free(dp, &dp->rx_rings[r]);
 363 err_free_ring:
 364                 nfp_net_rx_ring_free(&dp->rx_rings[r]);
 365         }
 366         kfree(dp->rx_rings);
 367         return -ENOMEM;
 368 }
 369
 370 void nfp_net_rx_rings_free(struct nfp_net_dp *dp)
 371 {
 372         unsigned int r;
 373
 374         for (r = 0; r < dp->num_rx_rings; r++) {
 375                 nfp_net_rx_ring_bufs_free(dp, &dp->rx_rings[r]);
 376                 nfp_net_rx_ring_free(&dp->rx_rings[r]);
 377         }
 378
 379         kfree(dp->rx_rings);
 380 }
 381
 382 void
 383 nfp_net_rx_ring_hw_cfg_write(struct nfp_net *nn,
 384                              struct nfp_net_rx_ring *rx_ring, unsigned int idx)
 385 {
 386         /* Write the DMA address, size and MSI-X info to the device */
 387         nn_writeq(nn, NFP_NET_CFG_RXR_ADDR(idx), rx_ring->dma);
 388         nn_writeb(nn, NFP_NET_CFG_RXR_SZ(idx), ilog2(rx_ring->cnt));
 389         nn_writeb(nn, NFP_NET_CFG_RXR_VEC(idx), rx_ring->r_vec->irq_entry);
 390 }
 391
 392 void
 393 nfp_net_tx_ring_hw_cfg_write(struct nfp_net *nn,
 394                              struct nfp_net_tx_ring *tx_ring, unsigned int idx)
 395 {
 396         nn_writeq(nn, NFP_NET_CFG_TXR_ADDR(idx), tx_ring->dma);
 397         if (tx_ring->txrwb) {
 398                 *tx_ring->txrwb = 0;
 399                 nn_writeq(nn, NFP_NET_CFG_TXR_WB_ADDR(idx),
 400                           nn->dp.txrwb_dma + idx * sizeof(u64));
 401         }
 402         nn_writeb(nn, NFP_NET_CFG_TXR_SZ(idx), ilog2(tx_ring->cnt));
 403         nn_writeb(nn, NFP_NET_CFG_TXR_VEC(idx), tx_ring->r_vec->irq_entry);
 404 }
 405
 406 void nfp_net_vec_clear_ring_data(struct nfp_net *nn, unsigned int idx)
 407 {
 408         nn_writeq(nn, NFP_NET_CFG_RXR_ADDR(idx), 0);
 409         nn_writeb(nn, NFP_NET_CFG_RXR_SZ(idx), 0);
 410         nn_writeb(nn, NFP_NET_CFG_RXR_VEC(idx), 0);
 411
 412         nn_writeq(nn, NFP_NET_CFG_TXR_ADDR(idx), 0);
 413         nn_writeq(nn, NFP_NET_CFG_TXR_WB_ADDR(idx), 0);
 414         nn_writeb(nn, NFP_NET_CFG_TXR_SZ(idx), 0);
 415         nn_writeb(nn, NFP_NET_CFG_TXR_VEC(idx), 0);
 416 }
 417
 418 netdev_tx_t nfp_net_tx(struct sk_buff *skb, struct net_device *netdev)
 419 {
 420         struct nfp_net *nn = netdev_priv(netdev);
 421
 422         return nn->dp.ops->xmit(skb, netdev);
 423 }
 424
 425 bool __nfp_ctrl_tx(struct nfp_net *nn, struct sk_buff *skb)
 426 {
 427         struct nfp_net_r_vector *r_vec = &nn->r_vecs[0];
 428
 429         return nn->dp.ops->ctrl_tx_one(nn, r_vec, skb, false);
 430 }
 431
 432 bool nfp_ctrl_tx(struct nfp_net *nn, struct sk_buff *skb)
 433 {
 434         struct nfp_net_r_vector *r_vec = &nn->r_vecs[0];
 435         bool ret;
 436
 437         spin_lock_bh(&r_vec->lock);
 438         ret = nn->dp.ops->ctrl_tx_one(nn, r_vec, skb, false);
 439         spin_unlock_bh(&r_vec->lock);
 440
 441         return ret;
 442 }
 443
 444 bool nfp_net_vlan_strip(struct sk_buff *skb, const struct nfp_net_rx_desc *rxd,
 445                         const struct nfp_meta_parsed *meta)
 446 {
 447         u16 tpid = 0, tci = 0;
 448
 449         if (rxd->rxd.flags & PCIE_DESC_RX_VLAN) {
 450                 tpid = ETH_P_8021Q;
 451                 tci = le16_to_cpu(rxd->rxd.vlan);
 452         } else if (meta->vlan.stripped) {
 453                 if (meta->vlan.tpid == NFP_NET_VLAN_CTAG)
 454                         tpid = ETH_P_8021Q;
 455                 else if (meta->vlan.tpid == NFP_NET_VLAN_STAG)
 456                         tpid = ETH_P_8021AD;
 457                 else
 458                         return false;
 459
 460                 tci = meta->vlan.tci;
 461         }
 462         if (tpid)
 463                 __vlan_hwaccel_put_tag(skb, htons(tpid), tci);
 464
 465         return true;
 466 }