drivers/net/ethernet/huawei/hinic/hinic_rx.c

   1 /*
   2  * Huawei HiNIC PCI Express Linux driver
   3  * Copyright(c) 2017 Huawei Technologies Co., Ltd
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * for more details.
  13  *
  14  */
  15
  16 #include <linux/kernel.h>
  17 #include <linux/types.h>
  18 #include <linux/errno.h>
  19 #include <linux/pci.h>
  20 #include <linux/device.h>
  21 #include <linux/netdevice.h>
  22 #include <linux/etherdevice.h>
  23 #include <linux/u64_stats_sync.h>
  24 #include <linux/slab.h>
  25 #include <linux/interrupt.h>
  26 #include <linux/skbuff.h>
  27 #include <linux/dma-mapping.h>
  28 #include <linux/prefetch.h>
  29 #include <linux/cpumask.h>
  30 #include <asm/barrier.h>
  31
  32 #include "hinic_common.h"
  33 #include "hinic_hw_if.h"
  34 #include "hinic_hw_wqe.h"
  35 #include "hinic_hw_wq.h"
  36 #include "hinic_hw_qp.h"
  37 #include "hinic_hw_dev.h"
  38 #include "hinic_rx.h"
  39 #include "hinic_dev.h"
  40
  41 #define RX_IRQ_NO_PENDING               0
  42 #define RX_IRQ_NO_COALESC               0
  43 #define RX_IRQ_NO_LLI_TIMER             0
  44 #define RX_IRQ_NO_CREDIT                0
  45 #define RX_IRQ_NO_RESEND_TIMER          0
  46
  47 /**
  48  * hinic_rxq_clean_stats - Clean the statistics of specific queue
  49  * @rxq: Logical Rx Queue
  50  **/
  51 void hinic_rxq_clean_stats(struct hinic_rxq *rxq)
  52 {
  53         struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats;
  54
  55         u64_stats_update_begin(&rxq_stats->syncp);
  56         rxq_stats->pkts  = 0;
  57         rxq_stats->bytes = 0;
  58         u64_stats_update_end(&rxq_stats->syncp);
  59 }
  60
  61 /**
  62  * hinic_rxq_get_stats - get statistics of Rx Queue
  63  * @rxq: Logical Rx Queue
  64  * @stats: return updated stats here
  65  **/
  66 void hinic_rxq_get_stats(struct hinic_rxq *rxq, struct hinic_rxq_stats *stats)
  67 {
  68         struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats;
  69         unsigned int start;
  70
  71         u64_stats_update_begin(&stats->syncp);
  72         do {
  73                 start = u64_stats_fetch_begin(&rxq_stats->syncp);
  74                 stats->pkts = rxq_stats->pkts;
  75                 stats->bytes = rxq_stats->bytes;
  76         } while (u64_stats_fetch_retry(&rxq_stats->syncp, start));
  77         u64_stats_update_end(&stats->syncp);
  78 }
  79
  80 /**
  81  * rxq_stats_init - Initialize the statistics of specific queue
  82  * @rxq: Logical Rx Queue
  83  **/
  84 static void rxq_stats_init(struct hinic_rxq *rxq)
  85 {
  86         struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats;
  87
  88         u64_stats_init(&rxq_stats->syncp);
  89         hinic_rxq_clean_stats(rxq);
  90 }
  91
  92 /**
  93  * rx_alloc_skb - allocate skb and map it to dma address
  94  * @rxq: rx queue
  95  * @dma_addr: returned dma address for the skb
  96  *
  97  * Return skb
  98  **/
  99 static struct sk_buff *rx_alloc_skb(struct hinic_rxq *rxq,
 100                                     dma_addr_t *dma_addr)
 101 {
 102         struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);
 103         struct hinic_hwdev *hwdev = nic_dev->hwdev;
 104         struct hinic_hwif *hwif = hwdev->hwif;
 105         struct pci_dev *pdev = hwif->pdev;
 106         struct sk_buff *skb;
 107         dma_addr_t addr;
 108         int err;
 109
 110         skb = netdev_alloc_skb_ip_align(rxq->netdev, rxq->rq->buf_sz);
 111         if (!skb) {
 112                 netdev_err(rxq->netdev, "Failed to allocate Rx SKB\n");
 113                 return NULL;
 114         }
 115
 116         addr = dma_map_single(&pdev->dev, skb->data, rxq->rq->buf_sz,
 117                               DMA_FROM_DEVICE);
 118         err = dma_mapping_error(&pdev->dev, addr);
 119         if (err) {
 120                 dev_err(&pdev->dev, "Failed to map Rx DMA, err = %d\n", err);
 121                 goto err_rx_map;
 122         }
 123
 124         *dma_addr = addr;
 125         return skb;
 126
 127 err_rx_map:
 128         dev_kfree_skb_any(skb);
 129         return NULL;
 130 }
 131
 132 /**
 133  * rx_unmap_skb - unmap the dma address of the skb
 134  * @rxq: rx queue
 135  * @dma_addr: dma address of the skb
 136  **/
 137 static void rx_unmap_skb(struct hinic_rxq *rxq, dma_addr_t dma_addr)
 138 {
 139         struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);
 140         struct hinic_hwdev *hwdev = nic_dev->hwdev;
 141         struct hinic_hwif *hwif = hwdev->hwif;
 142         struct pci_dev *pdev = hwif->pdev;
 143
 144         dma_unmap_single(&pdev->dev, dma_addr, rxq->rq->buf_sz,
 145                          DMA_FROM_DEVICE);
 146 }
 147
 148 /**
 149  * rx_free_skb - unmap and free skb
 150  * @rxq: rx queue
 151  * @skb: skb to free
 152  * @dma_addr: dma address of the skb
 153  **/
 154 static void rx_free_skb(struct hinic_rxq *rxq, struct sk_buff *skb,
 155                         dma_addr_t dma_addr)
 156 {
 157         rx_unmap_skb(rxq, dma_addr);
 158         dev_kfree_skb_any(skb);
 159 }
 160
 161 /**
 162  * rx_alloc_pkts - allocate pkts in rx queue
 163  * @rxq: rx queue
 164  *
 165  * Return number of skbs allocated
 166  **/
 167 static int rx_alloc_pkts(struct hinic_rxq *rxq)
 168 {
 169         struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);
 170         struct hinic_rq_wqe *rq_wqe;
 171         unsigned int free_wqebbs;
 172         struct hinic_sge sge;
 173         dma_addr_t dma_addr;
 174         struct sk_buff *skb;
 175         u16 prod_idx;
 176         int i;
 177
 178         free_wqebbs = hinic_get_rq_free_wqebbs(rxq->rq);
 179
 180         /* Limit the allocation chunks */
 181         if (free_wqebbs > nic_dev->rx_weight)
 182                 free_wqebbs = nic_dev->rx_weight;
 183
 184         for (i = 0; i < free_wqebbs; i++) {
 185                 skb = rx_alloc_skb(rxq, &dma_addr);
 186                 if (!skb) {
 187                         netdev_err(rxq->netdev, "Failed to alloc Rx skb\n");
 188                         goto skb_out;
 189                 }
 190
 191                 hinic_set_sge(&sge, dma_addr, skb->len);
 192
 193                 rq_wqe = hinic_rq_get_wqe(rxq->rq, HINIC_RQ_WQE_SIZE,
 194                                           &prod_idx);
 195                 if (!rq_wqe) {
 196                         rx_free_skb(rxq, skb, dma_addr);
 197                         goto skb_out;
 198                 }
 199
 200                 hinic_rq_prepare_wqe(rxq->rq, prod_idx, rq_wqe, &sge);
 201
 202                 hinic_rq_write_wqe(rxq->rq, prod_idx, rq_wqe, skb);
 203         }
 204
 205 skb_out:
 206         if (i) {
 207                 wmb();  /* write all the wqes before update PI */
 208
 209                 hinic_rq_update(rxq->rq, prod_idx);
 210         }
 211
 212         tasklet_schedule(&rxq->rx_task);
 213         return i;
 214 }
 215
 216 /**
 217  * free_all_rx_skbs - free all skbs in rx queue
 218  * @rxq: rx queue
 219  **/
 220 static void free_all_rx_skbs(struct hinic_rxq *rxq)
 221 {
 222         struct hinic_rq *rq = rxq->rq;
 223         struct hinic_hw_wqe *hw_wqe;
 224         struct hinic_sge sge;
 225         u16 ci;
 226
 227         while ((hw_wqe = hinic_read_wqe(rq->wq, HINIC_RQ_WQE_SIZE, &ci))) {
 228                 if (IS_ERR(hw_wqe))
 229                         break;
 230
 231                 hinic_rq_get_sge(rq, &hw_wqe->rq_wqe, ci, &sge);
 232
 233                 hinic_put_wqe(rq->wq, HINIC_RQ_WQE_SIZE);
 234
 235                 rx_free_skb(rxq, rq->saved_skb[ci], hinic_sge_to_dma(&sge));
 236         }
 237 }
 238
 239 /**
 240  * rx_alloc_task - tasklet for queue allocation
 241  * @data: rx queue
 242  **/
 243 static void rx_alloc_task(unsigned long data)
 244 {
 245         struct hinic_rxq *rxq = (struct hinic_rxq *)data;
 246
 247         (void)rx_alloc_pkts(rxq);
 248 }
 249
 250 /**
 251  * rx_recv_jumbo_pkt - Rx handler for jumbo pkt
 252  * @rxq: rx queue
 253  * @head_skb: the first skb in the list
 254  * @left_pkt_len: left size of the pkt exclude head skb
 255  * @ci: consumer index
 256  *
 257  * Return number of wqes that used for the left of the pkt
 258  **/
 259 static int rx_recv_jumbo_pkt(struct hinic_rxq *rxq, struct sk_buff *head_skb,
 260                              unsigned int left_pkt_len, u16 ci)
 261 {
 262         struct sk_buff *skb, *curr_skb = head_skb;
 263         struct hinic_rq_wqe *rq_wqe;
 264         unsigned int curr_len;
 265         struct hinic_sge sge;
 266         int num_wqes = 0;
 267
 268         while (left_pkt_len > 0) {
 269                 rq_wqe = hinic_rq_read_next_wqe(rxq->rq, HINIC_RQ_WQE_SIZE,
 270                                                 &skb, &ci);
 271
 272                 num_wqes++;
 273
 274                 hinic_rq_get_sge(rxq->rq, rq_wqe, ci, &sge);
 275
 276                 rx_unmap_skb(rxq, hinic_sge_to_dma(&sge));
 277
 278                 prefetch(skb->data);
 279
 280                 curr_len = (left_pkt_len > HINIC_RX_BUF_SZ) ? HINIC_RX_BUF_SZ :
 281                             left_pkt_len;
 282
 283                 left_pkt_len -= curr_len;
 284
 285                 __skb_put(skb, curr_len);
 286
 287                 if (curr_skb == head_skb)
 288                         skb_shinfo(head_skb)->frag_list = skb;
 289                 else
 290                         curr_skb->next = skb;
 291
 292                 head_skb->len += skb->len;
 293                 head_skb->data_len += skb->len;
 294                 head_skb->truesize += skb->truesize;
 295
 296                 curr_skb = skb;
 297         }
 298
 299         return num_wqes;
 300 }
 301
 302 /**
 303  * rxq_recv - Rx handler
 304  * @rxq: rx queue
 305  * @budget: maximum pkts to process
 306  *
 307  * Return number of pkts received
 308  **/
 309 static int rxq_recv(struct hinic_rxq *rxq, int budget)
 310 {
 311         struct hinic_qp *qp = container_of(rxq->rq, struct hinic_qp, rq);
 312         u64 pkt_len = 0, rx_bytes = 0;
 313         struct hinic_rq_wqe *rq_wqe;
 314         int num_wqes, pkts = 0;
 315         struct hinic_sge sge;
 316         struct sk_buff *skb;
 317         u16 ci;
 318
 319         while (pkts < budget) {
 320                 num_wqes = 0;
 321
 322                 rq_wqe = hinic_rq_read_wqe(rxq->rq, HINIC_RQ_WQE_SIZE, &skb,
 323                                            &ci);
 324                 if (!rq_wqe)
 325                         break;
 326
 327                 hinic_rq_get_sge(rxq->rq, rq_wqe, ci, &sge);
 328
 329                 rx_unmap_skb(rxq, hinic_sge_to_dma(&sge));
 330
 331                 prefetch(skb->data);
 332
 333                 pkt_len = sge.len;
 334
 335                 if (pkt_len <= HINIC_RX_BUF_SZ) {
 336                         __skb_put(skb, pkt_len);
 337                 } else {
 338                         __skb_put(skb, HINIC_RX_BUF_SZ);
 339                         num_wqes = rx_recv_jumbo_pkt(rxq, skb, pkt_len -
 340                                                      HINIC_RX_BUF_SZ, ci);
 341                 }
 342
 343                 hinic_rq_put_wqe(rxq->rq, ci,
 344                                  (num_wqes + 1) * HINIC_RQ_WQE_SIZE);
 345
 346                 skb_record_rx_queue(skb, qp->q_id);
 347                 skb->protocol = eth_type_trans(skb, rxq->netdev);
 348
 349                 napi_gro_receive(&rxq->napi, skb);
 350
 351                 pkts++;
 352                 rx_bytes += pkt_len;
 353         }
 354
 355         if (pkts)
 356                 tasklet_schedule(&rxq->rx_task); /* rx_alloc_pkts */
 357
 358         u64_stats_update_begin(&rxq->rxq_stats.syncp);
 359         rxq->rxq_stats.pkts += pkts;
 360         rxq->rxq_stats.bytes += rx_bytes;
 361         u64_stats_update_end(&rxq->rxq_stats.syncp);
 362
 363         return pkts;
 364 }
 365
 366 static int rx_poll(struct napi_struct *napi, int budget)
 367 {
 368         struct hinic_rxq *rxq = container_of(napi, struct hinic_rxq, napi);
 369         struct hinic_rq *rq = rxq->rq;
 370         int pkts;
 371
 372         pkts = rxq_recv(rxq, budget);
 373         if (pkts >= budget)
 374                 return budget;
 375
 376         napi_complete(napi);
 377         enable_irq(rq->irq);
 378         return pkts;
 379 }
 380
 381 static void rx_add_napi(struct hinic_rxq *rxq)
 382 {
 383         struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);
 384
 385         netif_napi_add(rxq->netdev, &rxq->napi, rx_poll, nic_dev->rx_weight);
 386         napi_enable(&rxq->napi);
 387 }
 388
 389 static void rx_del_napi(struct hinic_rxq *rxq)
 390 {
 391         napi_disable(&rxq->napi);
 392         netif_napi_del(&rxq->napi);
 393 }
 394
 395 static irqreturn_t rx_irq(int irq, void *data)
 396 {
 397         struct hinic_rxq *rxq = (struct hinic_rxq *)data;
 398         struct hinic_rq *rq = rxq->rq;
 399         struct hinic_dev *nic_dev;
 400
 401         /* Disable the interrupt until napi will be completed */
 402         disable_irq_nosync(rq->irq);
 403
 404         nic_dev = netdev_priv(rxq->netdev);
 405         hinic_hwdev_msix_cnt_set(nic_dev->hwdev, rq->msix_entry);
 406
 407         napi_schedule(&rxq->napi);
 408         return IRQ_HANDLED;
 409 }
 410
 411 static int rx_request_irq(struct hinic_rxq *rxq)
 412 {
 413         struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);
 414         struct hinic_hwdev *hwdev = nic_dev->hwdev;
 415         struct hinic_rq *rq = rxq->rq;
 416         struct hinic_qp *qp;
 417         struct cpumask mask;
 418         int err;
 419
 420         rx_add_napi(rxq);
 421
 422         hinic_hwdev_msix_set(hwdev, rq->msix_entry,
 423                              RX_IRQ_NO_PENDING, RX_IRQ_NO_COALESC,
 424                              RX_IRQ_NO_LLI_TIMER, RX_IRQ_NO_CREDIT,
 425                              RX_IRQ_NO_RESEND_TIMER);
 426
 427         err = request_irq(rq->irq, rx_irq, 0, rxq->irq_name, rxq);
 428         if (err) {
 429                 rx_del_napi(rxq);
 430                 return err;
 431         }
 432
 433         qp = container_of(rq, struct hinic_qp, rq);
 434         cpumask_set_cpu(qp->q_id % num_online_cpus(), &mask);
 435         return irq_set_affinity_hint(rq->irq, &mask);
 436 }
 437
 438 static void rx_free_irq(struct hinic_rxq *rxq)
 439 {
 440         struct hinic_rq *rq = rxq->rq;
 441
 442         free_irq(rq->irq, rxq);
 443         rx_del_napi(rxq);
 444 }
 445
 446 /**
 447  * hinic_init_rxq - Initialize the Rx Queue
 448  * @rxq: Logical Rx Queue
 449  * @rq: Hardware Rx Queue to connect the Logical queue with
 450  * @netdev: network device to connect the Logical queue with
 451  *
 452  * Return 0 - Success, negative - Failure
 453  **/
 454 int hinic_init_rxq(struct hinic_rxq *rxq, struct hinic_rq *rq,
 455                    struct net_device *netdev)
 456 {
 457         struct hinic_qp *qp = container_of(rq, struct hinic_qp, rq);
 458         int err, pkts, irqname_len;
 459
 460         rxq->netdev = netdev;
 461         rxq->rq = rq;
 462
 463         rxq_stats_init(rxq);
 464
 465         irqname_len = snprintf(NULL, 0, "hinic_rxq%d", qp->q_id) + 1;
 466         rxq->irq_name = devm_kzalloc(&netdev->dev, irqname_len, GFP_KERNEL);
 467         if (!rxq->irq_name)
 468                 return -ENOMEM;
 469
 470         sprintf(rxq->irq_name, "hinic_rxq%d", qp->q_id);
 471
 472         tasklet_init(&rxq->rx_task, rx_alloc_task, (unsigned long)rxq);
 473
 474         pkts = rx_alloc_pkts(rxq);
 475         if (!pkts) {
 476                 err = -ENOMEM;
 477                 goto err_rx_pkts;
 478         }
 479
 480         err = rx_request_irq(rxq);
 481         if (err) {
 482                 netdev_err(netdev, "Failed to request Rx irq\n");
 483                 goto err_req_rx_irq;
 484         }
 485
 486         return 0;
 487
 488 err_req_rx_irq:
 489 err_rx_pkts:
 490         tasklet_kill(&rxq->rx_task);
 491         free_all_rx_skbs(rxq);
 492         devm_kfree(&netdev->dev, rxq->irq_name);
 493         return err;
 494 }
 495
 496 /**
 497  * hinic_clean_rxq - Clean the Rx Queue
 498  * @rxq: Logical Rx Queue
 499  **/
 500 void hinic_clean_rxq(struct hinic_rxq *rxq)
 501 {
 502         struct net_device *netdev = rxq->netdev;
 503
 504         rx_free_irq(rxq);
 505
 506         tasklet_kill(&rxq->rx_task);
 507         free_all_rx_skbs(rxq);
 508         devm_kfree(&netdev->dev, rxq->irq_name);
 509 }