drivers/net/ethernet/huawei/hinic/hinic_hw_qp.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Huawei HiNIC PCI Express Linux driver
   4  * Copyright(c) 2017 Huawei Technologies Co., Ltd
   5  */
   6
   7 #include <linux/kernel.h>
   8 #include <linux/types.h>
   9 #include <linux/pci.h>
  10 #include <linux/device.h>
  11 #include <linux/dma-mapping.h>
  12 #include <linux/vmalloc.h>
  13 #include <linux/errno.h>
  14 #include <linux/sizes.h>
  15 #include <linux/atomic.h>
  16 #include <linux/skbuff.h>
  17 #include <linux/io.h>
  18 #include <asm/barrier.h>
  19 #include <asm/byteorder.h>
  20
  21 #include "hinic_common.h"
  22 #include "hinic_hw_if.h"
  23 #include "hinic_hw_wqe.h"
  24 #include "hinic_hw_wq.h"
  25 #include "hinic_hw_qp_ctxt.h"
  26 #include "hinic_hw_qp.h"
  27 #include "hinic_hw_io.h"
  28
  29 #define SQ_DB_OFF               SZ_2K
  30
  31 /* The number of cache line to prefetch Until threshold state */
  32 #define WQ_PREFETCH_MAX         2
  33 /* The number of cache line to prefetch After threshold state */
  34 #define WQ_PREFETCH_MIN         1
  35 /* Threshold state */
  36 #define WQ_PREFETCH_THRESHOLD   256
  37
  38 /* sizes of the SQ/RQ ctxt */
  39 #define Q_CTXT_SIZE             48
  40 #define CTXT_RSVD               240
  41
  42 #define SQ_CTXT_OFFSET(max_sqs, max_rqs, q_id)  \
  43                 (((max_rqs) + (max_sqs)) * CTXT_RSVD + (q_id) * Q_CTXT_SIZE)
  44
  45 #define RQ_CTXT_OFFSET(max_sqs, max_rqs, q_id)  \
  46                 (((max_rqs) + (max_sqs)) * CTXT_RSVD + \
  47                  (max_sqs + (q_id)) * Q_CTXT_SIZE)
  48
  49 #define SIZE_16BYTES(size)              (ALIGN(size, 16) >> 4)
  50 #define SIZE_8BYTES(size)               (ALIGN(size, 8) >> 3)
  51 #define SECT_SIZE_FROM_8BYTES(size)     ((size) << 3)
  52
  53 #define SQ_DB_PI_HI_SHIFT       8
  54 #define SQ_DB_PI_HI(prod_idx)   ((prod_idx) >> SQ_DB_PI_HI_SHIFT)
  55
  56 #define SQ_DB_PI_LOW_MASK       0xFF
  57 #define SQ_DB_PI_LOW(prod_idx)  ((prod_idx) & SQ_DB_PI_LOW_MASK)
  58
  59 #define SQ_DB_ADDR(sq, pi)      ((u64 *)((sq)->db_base) + SQ_DB_PI_LOW(pi))
  60
  61 #define SQ_MASKED_IDX(sq, idx)  ((idx) & (sq)->wq->mask)
  62 #define RQ_MASKED_IDX(rq, idx)  ((idx) & (rq)->wq->mask)
  63
  64 enum sq_wqe_type {
  65         SQ_NORMAL_WQE = 0,
  66 };
  67
  68 enum rq_completion_fmt {
  69         RQ_COMPLETE_SGE = 1
  70 };
  71
  72 void hinic_qp_prepare_header(struct hinic_qp_ctxt_header *qp_ctxt_hdr,
  73                              enum hinic_qp_ctxt_type ctxt_type,
  74                              u16 num_queues, u16 max_queues)
  75 {
  76         u16 max_sqs = max_queues;
  77         u16 max_rqs = max_queues;
  78
  79         qp_ctxt_hdr->num_queues = num_queues;
  80         qp_ctxt_hdr->queue_type = ctxt_type;
  81
  82         if (ctxt_type == HINIC_QP_CTXT_TYPE_SQ)
  83                 qp_ctxt_hdr->addr_offset = SQ_CTXT_OFFSET(max_sqs, max_rqs, 0);
  84         else
  85                 qp_ctxt_hdr->addr_offset = RQ_CTXT_OFFSET(max_sqs, max_rqs, 0);
  86
  87         qp_ctxt_hdr->addr_offset = SIZE_16BYTES(qp_ctxt_hdr->addr_offset);
  88
  89         hinic_cpu_to_be32(qp_ctxt_hdr, sizeof(*qp_ctxt_hdr));
  90 }
  91
  92 void hinic_sq_prepare_ctxt(struct hinic_sq_ctxt *sq_ctxt,
  93                            struct hinic_sq *sq, u16 global_qid)
  94 {
  95         u32 wq_page_pfn_hi, wq_page_pfn_lo, wq_block_pfn_hi, wq_block_pfn_lo;
  96         u64 wq_page_addr, wq_page_pfn, wq_block_pfn;
  97         u16 pi_start, ci_start;
  98         struct hinic_wq *wq;
  99
 100         wq = sq->wq;
 101         ci_start = atomic_read(&wq->cons_idx);
 102         pi_start = atomic_read(&wq->prod_idx);
 103
 104         /* Read the first page paddr from the WQ page paddr ptrs */
 105         wq_page_addr = be64_to_cpu(*wq->block_vaddr);
 106
 107         wq_page_pfn = HINIC_WQ_PAGE_PFN(wq_page_addr);
 108         wq_page_pfn_hi = upper_32_bits(wq_page_pfn);
 109         wq_page_pfn_lo = lower_32_bits(wq_page_pfn);
 110
 111         /* If only one page, use 0-level CLA */
 112         if (wq->num_q_pages == 1)
 113                 wq_block_pfn = HINIC_WQ_BLOCK_PFN(wq_page_addr);
 114         else
 115                 wq_block_pfn = HINIC_WQ_BLOCK_PFN(wq->block_paddr);
 116
 117         wq_block_pfn_hi = upper_32_bits(wq_block_pfn);
 118         wq_block_pfn_lo = lower_32_bits(wq_block_pfn);
 119
 120         sq_ctxt->ceq_attr = HINIC_SQ_CTXT_CEQ_ATTR_SET(global_qid,
 121                                                        GLOBAL_SQ_ID) |
 122                             HINIC_SQ_CTXT_CEQ_ATTR_SET(0, EN);
 123
 124         sq_ctxt->ci_wrapped = HINIC_SQ_CTXT_CI_SET(ci_start, IDX) |
 125                               HINIC_SQ_CTXT_CI_SET(1, WRAPPED);
 126
 127         sq_ctxt->wq_hi_pfn_pi =
 128                         HINIC_SQ_CTXT_WQ_PAGE_SET(wq_page_pfn_hi, HI_PFN) |
 129                         HINIC_SQ_CTXT_WQ_PAGE_SET(pi_start, PI);
 130
 131         sq_ctxt->wq_lo_pfn = wq_page_pfn_lo;
 132
 133         sq_ctxt->pref_cache =
 134                 HINIC_SQ_CTXT_PREF_SET(WQ_PREFETCH_MIN, CACHE_MIN) |
 135                 HINIC_SQ_CTXT_PREF_SET(WQ_PREFETCH_MAX, CACHE_MAX) |
 136                 HINIC_SQ_CTXT_PREF_SET(WQ_PREFETCH_THRESHOLD, CACHE_THRESHOLD);
 137
 138         sq_ctxt->pref_wrapped = 1;
 139
 140         sq_ctxt->pref_wq_hi_pfn_ci =
 141                 HINIC_SQ_CTXT_PREF_SET(ci_start, CI) |
 142                 HINIC_SQ_CTXT_PREF_SET(wq_page_pfn_hi, WQ_HI_PFN);
 143
 144         sq_ctxt->pref_wq_lo_pfn = wq_page_pfn_lo;
 145
 146         sq_ctxt->wq_block_hi_pfn =
 147                 HINIC_SQ_CTXT_WQ_BLOCK_SET(wq_block_pfn_hi, HI_PFN);
 148
 149         sq_ctxt->wq_block_lo_pfn = wq_block_pfn_lo;
 150
 151         hinic_cpu_to_be32(sq_ctxt, sizeof(*sq_ctxt));
 152 }
 153
 154 void hinic_rq_prepare_ctxt(struct hinic_rq_ctxt *rq_ctxt,
 155                            struct hinic_rq *rq, u16 global_qid)
 156 {
 157         u32 wq_page_pfn_hi, wq_page_pfn_lo, wq_block_pfn_hi, wq_block_pfn_lo;
 158         u64 wq_page_addr, wq_page_pfn, wq_block_pfn;
 159         u16 pi_start, ci_start;
 160         struct hinic_wq *wq;
 161
 162         wq = rq->wq;
 163         ci_start = atomic_read(&wq->cons_idx);
 164         pi_start = atomic_read(&wq->prod_idx);
 165
 166         /* Read the first page paddr from the WQ page paddr ptrs */
 167         wq_page_addr = be64_to_cpu(*wq->block_vaddr);
 168
 169         wq_page_pfn = HINIC_WQ_PAGE_PFN(wq_page_addr);
 170         wq_page_pfn_hi = upper_32_bits(wq_page_pfn);
 171         wq_page_pfn_lo = lower_32_bits(wq_page_pfn);
 172
 173         wq_block_pfn = HINIC_WQ_BLOCK_PFN(wq->block_paddr);
 174         wq_block_pfn_hi = upper_32_bits(wq_block_pfn);
 175         wq_block_pfn_lo = lower_32_bits(wq_block_pfn);
 176
 177         rq_ctxt->ceq_attr = HINIC_RQ_CTXT_CEQ_ATTR_SET(0, EN) |
 178                             HINIC_RQ_CTXT_CEQ_ATTR_SET(1, WRAPPED);
 179
 180         rq_ctxt->pi_intr_attr = HINIC_RQ_CTXT_PI_SET(pi_start, IDX) |
 181                                 HINIC_RQ_CTXT_PI_SET(rq->msix_entry, INTR);
 182
 183         rq_ctxt->wq_hi_pfn_ci = HINIC_RQ_CTXT_WQ_PAGE_SET(wq_page_pfn_hi,
 184                                                           HI_PFN) |
 185                                 HINIC_RQ_CTXT_WQ_PAGE_SET(ci_start, CI);
 186
 187         rq_ctxt->wq_lo_pfn = wq_page_pfn_lo;
 188
 189         rq_ctxt->pref_cache =
 190                 HINIC_RQ_CTXT_PREF_SET(WQ_PREFETCH_MIN, CACHE_MIN) |
 191                 HINIC_RQ_CTXT_PREF_SET(WQ_PREFETCH_MAX, CACHE_MAX) |
 192                 HINIC_RQ_CTXT_PREF_SET(WQ_PREFETCH_THRESHOLD, CACHE_THRESHOLD);
 193
 194         rq_ctxt->pref_wrapped = 1;
 195
 196         rq_ctxt->pref_wq_hi_pfn_ci =
 197                 HINIC_RQ_CTXT_PREF_SET(wq_page_pfn_hi, WQ_HI_PFN) |
 198                 HINIC_RQ_CTXT_PREF_SET(ci_start, CI);
 199
 200         rq_ctxt->pref_wq_lo_pfn = wq_page_pfn_lo;
 201
 202         rq_ctxt->pi_paddr_hi = upper_32_bits(rq->pi_dma_addr);
 203         rq_ctxt->pi_paddr_lo = lower_32_bits(rq->pi_dma_addr);
 204
 205         rq_ctxt->wq_block_hi_pfn =
 206                 HINIC_RQ_CTXT_WQ_BLOCK_SET(wq_block_pfn_hi, HI_PFN);
 207
 208         rq_ctxt->wq_block_lo_pfn = wq_block_pfn_lo;
 209
 210         hinic_cpu_to_be32(rq_ctxt, sizeof(*rq_ctxt));
 211 }
 212
 213 /**
 214  * alloc_sq_skb_arr - allocate sq array for saved skb
 215  * @sq: HW Send Queue
 216  *
 217  * Return 0 - Success, negative - Failure
 218  **/
 219 static int alloc_sq_skb_arr(struct hinic_sq *sq)
 220 {
 221         struct hinic_wq *wq = sq->wq;
 222         size_t skb_arr_size;
 223
 224         skb_arr_size = wq->q_depth * sizeof(*sq->saved_skb);
 225         sq->saved_skb = vzalloc(skb_arr_size);
 226         if (!sq->saved_skb)
 227                 return -ENOMEM;
 228
 229         return 0;
 230 }
 231
 232 /**
 233  * free_sq_skb_arr - free sq array for saved skb
 234  * @sq: HW Send Queue
 235  **/
 236 static void free_sq_skb_arr(struct hinic_sq *sq)
 237 {
 238         vfree(sq->saved_skb);
 239 }
 240
 241 /**
 242  * alloc_rq_skb_arr - allocate rq array for saved skb
 243  * @rq: HW Receive Queue
 244  *
 245  * Return 0 - Success, negative - Failure
 246  **/
 247 static int alloc_rq_skb_arr(struct hinic_rq *rq)
 248 {
 249         struct hinic_wq *wq = rq->wq;
 250         size_t skb_arr_size;
 251
 252         skb_arr_size = wq->q_depth * sizeof(*rq->saved_skb);
 253         rq->saved_skb = vzalloc(skb_arr_size);
 254         if (!rq->saved_skb)
 255                 return -ENOMEM;
 256
 257         return 0;
 258 }
 259
 260 /**
 261  * free_rq_skb_arr - free rq array for saved skb
 262  * @rq: HW Receive Queue
 263  **/
 264 static void free_rq_skb_arr(struct hinic_rq *rq)
 265 {
 266         vfree(rq->saved_skb);
 267 }
 268
 269 /**
 270  * hinic_init_sq - Initialize HW Send Queue
 271  * @sq: HW Send Queue
 272  * @hwif: HW Interface for accessing HW
 273  * @wq: Work Queue for the data of the SQ
 274  * @entry: msix entry for sq
 275  * @ci_addr: address for reading the current HW consumer index
 276  * @ci_dma_addr: dma address for reading the current HW consumer index
 277  * @db_base: doorbell base address
 278  *
 279  * Return 0 - Success, negative - Failure
 280  **/
 281 int hinic_init_sq(struct hinic_sq *sq, struct hinic_hwif *hwif,
 282                   struct hinic_wq *wq, struct msix_entry *entry,
 283                   void *ci_addr, dma_addr_t ci_dma_addr,
 284                   void __iomem *db_base)
 285 {
 286         sq->hwif = hwif;
 287
 288         sq->wq = wq;
 289
 290         sq->irq = entry->vector;
 291         sq->msix_entry = entry->entry;
 292
 293         sq->hw_ci_addr = ci_addr;
 294         sq->hw_ci_dma_addr = ci_dma_addr;
 295
 296         sq->db_base = db_base + SQ_DB_OFF;
 297
 298         return alloc_sq_skb_arr(sq);
 299 }
 300
 301 /**
 302  * hinic_clean_sq - Clean HW Send Queue's Resources
 303  * @sq: Send Queue
 304  **/
 305 void hinic_clean_sq(struct hinic_sq *sq)
 306 {
 307         free_sq_skb_arr(sq);
 308 }
 309
 310 /**
 311  * alloc_rq_cqe - allocate rq completion queue elements
 312  * @rq: HW Receive Queue
 313  *
 314  * Return 0 - Success, negative - Failure
 315  **/
 316 static int alloc_rq_cqe(struct hinic_rq *rq)
 317 {
 318         struct hinic_hwif *hwif = rq->hwif;
 319         struct pci_dev *pdev = hwif->pdev;
 320         size_t cqe_dma_size, cqe_size;
 321         struct hinic_wq *wq = rq->wq;
 322         int j, i;
 323
 324         cqe_size = wq->q_depth * sizeof(*rq->cqe);
 325         rq->cqe = vzalloc(cqe_size);
 326         if (!rq->cqe)
 327                 return -ENOMEM;
 328
 329         cqe_dma_size = wq->q_depth * sizeof(*rq->cqe_dma);
 330         rq->cqe_dma = vzalloc(cqe_dma_size);
 331         if (!rq->cqe_dma)
 332                 goto err_cqe_dma_arr_alloc;
 333
 334         for (i = 0; i < wq->q_depth; i++) {
 335                 rq->cqe[i] = dma_alloc_coherent(&pdev->dev,
 336                                                 sizeof(*rq->cqe[i]),
 337                                                 &rq->cqe_dma[i], GFP_KERNEL);
 338                 if (!rq->cqe[i])
 339                         goto err_cqe_alloc;
 340         }
 341
 342         return 0;
 343
 344 err_cqe_alloc:
 345         for (j = 0; j < i; j++)
 346                 dma_free_coherent(&pdev->dev, sizeof(*rq->cqe[j]), rq->cqe[j],
 347                                   rq->cqe_dma[j]);
 348
 349         vfree(rq->cqe_dma);
 350
 351 err_cqe_dma_arr_alloc:
 352         vfree(rq->cqe);
 353         return -ENOMEM;
 354 }
 355
 356 /**
 357  * free_rq_cqe - free rq completion queue elements
 358  * @rq: HW Receive Queue
 359  **/
 360 static void free_rq_cqe(struct hinic_rq *rq)
 361 {
 362         struct hinic_hwif *hwif = rq->hwif;
 363         struct pci_dev *pdev = hwif->pdev;
 364         struct hinic_wq *wq = rq->wq;
 365         int i;
 366
 367         for (i = 0; i < wq->q_depth; i++)
 368                 dma_free_coherent(&pdev->dev, sizeof(*rq->cqe[i]), rq->cqe[i],
 369                                   rq->cqe_dma[i]);
 370
 371         vfree(rq->cqe_dma);
 372         vfree(rq->cqe);
 373 }
 374
 375 /**
 376  * hinic_init_rq - Initialize HW Receive Queue
 377  * @rq: HW Receive Queue
 378  * @hwif: HW Interface for accessing HW
 379  * @wq: Work Queue for the data of the RQ
 380  * @entry: msix entry for rq
 381  *
 382  * Return 0 - Success, negative - Failure
 383  **/
 384 int hinic_init_rq(struct hinic_rq *rq, struct hinic_hwif *hwif,
 385                   struct hinic_wq *wq, struct msix_entry *entry)
 386 {
 387         struct pci_dev *pdev = hwif->pdev;
 388         size_t pi_size;
 389         int err;
 390
 391         rq->hwif = hwif;
 392
 393         rq->wq = wq;
 394
 395         rq->irq = entry->vector;
 396         rq->msix_entry = entry->entry;
 397
 398         rq->buf_sz = HINIC_RX_BUF_SZ;
 399
 400         err = alloc_rq_skb_arr(rq);
 401         if (err) {
 402                 dev_err(&pdev->dev, "Failed to allocate rq priv data\n");
 403                 return err;
 404         }
 405
 406         err = alloc_rq_cqe(rq);
 407         if (err) {
 408                 dev_err(&pdev->dev, "Failed to allocate rq cqe\n");
 409                 goto err_alloc_rq_cqe;
 410         }
 411
 412         /* HW requirements: Must be at least 32 bit */
 413         pi_size = ALIGN(sizeof(*rq->pi_virt_addr), sizeof(u32));
 414         rq->pi_virt_addr = dma_alloc_coherent(&pdev->dev, pi_size,
 415                                               &rq->pi_dma_addr, GFP_KERNEL);
 416         if (!rq->pi_virt_addr) {
 417                 dev_err(&pdev->dev, "Failed to allocate PI address\n");
 418                 err = -ENOMEM;
 419                 goto err_pi_virt;
 420         }
 421
 422         return 0;
 423
 424 err_pi_virt:
 425         free_rq_cqe(rq);
 426
 427 err_alloc_rq_cqe:
 428         free_rq_skb_arr(rq);
 429         return err;
 430 }
 431
 432 /**
 433  * hinic_clean_rq - Clean HW Receive Queue's Resources
 434  * @rq: HW Receive Queue
 435  **/
 436 void hinic_clean_rq(struct hinic_rq *rq)
 437 {
 438         struct hinic_hwif *hwif = rq->hwif;
 439         struct pci_dev *pdev = hwif->pdev;
 440         size_t pi_size;
 441
 442         pi_size = ALIGN(sizeof(*rq->pi_virt_addr), sizeof(u32));
 443         dma_free_coherent(&pdev->dev, pi_size, rq->pi_virt_addr,
 444                           rq->pi_dma_addr);
 445
 446         free_rq_cqe(rq);
 447         free_rq_skb_arr(rq);
 448 }
 449
 450 /**
 451  * hinic_get_sq_free_wqebbs - return number of free wqebbs for use
 452  * @sq: send queue
 453  *
 454  * Return number of free wqebbs
 455  **/
 456 int hinic_get_sq_free_wqebbs(struct hinic_sq *sq)
 457 {
 458         struct hinic_wq *wq = sq->wq;
 459
 460         return atomic_read(&wq->delta) - 1;
 461 }
 462
 463 /**
 464  * hinic_get_rq_free_wqebbs - return number of free wqebbs for use
 465  * @rq: recv queue
 466  *
 467  * Return number of free wqebbs
 468  **/
 469 int hinic_get_rq_free_wqebbs(struct hinic_rq *rq)
 470 {
 471         struct hinic_wq *wq = rq->wq;
 472
 473         return atomic_read(&wq->delta) - 1;
 474 }
 475
 476 static void sq_prepare_ctrl(struct hinic_sq_ctrl *ctrl, u16 prod_idx,
 477                             int nr_descs)
 478 {
 479         u32 ctrl_size, task_size, bufdesc_size;
 480
 481         ctrl_size = SIZE_8BYTES(sizeof(struct hinic_sq_ctrl));
 482         task_size = SIZE_8BYTES(sizeof(struct hinic_sq_task));
 483         bufdesc_size = nr_descs * sizeof(struct hinic_sq_bufdesc);
 484         bufdesc_size = SIZE_8BYTES(bufdesc_size);
 485
 486         ctrl->ctrl_info = HINIC_SQ_CTRL_SET(bufdesc_size, BUFDESC_SECT_LEN) |
 487                           HINIC_SQ_CTRL_SET(task_size, TASKSECT_LEN)        |
 488                           HINIC_SQ_CTRL_SET(SQ_NORMAL_WQE, DATA_FORMAT)     |
 489                           HINIC_SQ_CTRL_SET(ctrl_size, LEN);
 490
 491         ctrl->queue_info = HINIC_SQ_CTRL_SET(HINIC_MSS_DEFAULT,
 492                                              QUEUE_INFO_MSS) |
 493                            HINIC_SQ_CTRL_SET(1, QUEUE_INFO_UC);
 494 }
 495
 496 static void sq_prepare_task(struct hinic_sq_task *task)
 497 {
 498         task->pkt_info0 = 0;
 499         task->pkt_info1 = 0;
 500         task->pkt_info2 = 0;
 501
 502         task->ufo_v6_identify = 0;
 503
 504         task->pkt_info4 = HINIC_SQ_TASK_INFO4_SET(HINIC_L2TYPE_ETH, L2TYPE);
 505
 506         task->zero_pad = 0;
 507 }
 508
 509 void hinic_task_set_l2hdr(struct hinic_sq_task *task, u32 len)
 510 {
 511         task->pkt_info0 |= HINIC_SQ_TASK_INFO0_SET(len, L2HDR_LEN);
 512 }
 513
 514 void hinic_task_set_outter_l3(struct hinic_sq_task *task,
 515                               enum hinic_l3_offload_type l3_type,
 516                               u32 network_len)
 517 {
 518         task->pkt_info2 |= HINIC_SQ_TASK_INFO2_SET(l3_type, OUTER_L3TYPE) |
 519                            HINIC_SQ_TASK_INFO2_SET(network_len, OUTER_L3LEN);
 520 }
 521
 522 void hinic_task_set_inner_l3(struct hinic_sq_task *task,
 523                              enum hinic_l3_offload_type l3_type,
 524                              u32 network_len)
 525 {
 526         task->pkt_info0 |= HINIC_SQ_TASK_INFO0_SET(l3_type, INNER_L3TYPE);
 527         task->pkt_info1 |= HINIC_SQ_TASK_INFO1_SET(network_len, INNER_L3LEN);
 528 }
 529
 530 void hinic_task_set_tunnel_l4(struct hinic_sq_task *task,
 531                               enum hinic_l4_tunnel_type l4_type,
 532                               u32 tunnel_len)
 533 {
 534         task->pkt_info2 |= HINIC_SQ_TASK_INFO2_SET(l4_type, TUNNEL_L4TYPE) |
 535                            HINIC_SQ_TASK_INFO2_SET(tunnel_len, TUNNEL_L4LEN);
 536 }
 537
 538 void hinic_set_cs_inner_l4(struct hinic_sq_task *task, u32 *queue_info,
 539                            enum hinic_l4_offload_type l4_offload,
 540                            u32 l4_len, u32 offset)
 541 {
 542         u32 tcp_udp_cs = 0, sctp = 0;
 543         u32 mss = HINIC_MSS_DEFAULT;
 544
 545         if (l4_offload == TCP_OFFLOAD_ENABLE ||
 546             l4_offload == UDP_OFFLOAD_ENABLE)
 547                 tcp_udp_cs = 1;
 548         else if (l4_offload == SCTP_OFFLOAD_ENABLE)
 549                 sctp = 1;
 550
 551         task->pkt_info0 |= HINIC_SQ_TASK_INFO0_SET(l4_offload, L4_OFFLOAD);
 552         task->pkt_info1 |= HINIC_SQ_TASK_INFO1_SET(l4_len, INNER_L4LEN);
 553
 554         *queue_info |= HINIC_SQ_CTRL_SET(offset, QUEUE_INFO_PLDOFF) |
 555                        HINIC_SQ_CTRL_SET(tcp_udp_cs, QUEUE_INFO_TCPUDP_CS) |
 556                        HINIC_SQ_CTRL_SET(sctp, QUEUE_INFO_SCTP);
 557
 558         *queue_info = HINIC_SQ_CTRL_CLEAR(*queue_info, QUEUE_INFO_MSS);
 559         *queue_info |= HINIC_SQ_CTRL_SET(mss, QUEUE_INFO_MSS);
 560 }
 561
 562 void hinic_set_tso_inner_l4(struct hinic_sq_task *task, u32 *queue_info,
 563                             enum hinic_l4_offload_type l4_offload,
 564                             u32 l4_len, u32 offset, u32 ip_ident, u32 mss)
 565 {
 566         u32 tso = 0, ufo = 0;
 567
 568         if (l4_offload == TCP_OFFLOAD_ENABLE)
 569                 tso = 1;
 570         else if (l4_offload == UDP_OFFLOAD_ENABLE)
 571                 ufo = 1;
 572
 573         task->ufo_v6_identify = ip_ident;
 574
 575         task->pkt_info0 |= HINIC_SQ_TASK_INFO0_SET(l4_offload, L4_OFFLOAD);
 576         task->pkt_info0 |= HINIC_SQ_TASK_INFO0_SET(tso || ufo, TSO_FLAG);
 577         task->pkt_info1 |= HINIC_SQ_TASK_INFO1_SET(l4_len, INNER_L4LEN);
 578
 579         *queue_info |= HINIC_SQ_CTRL_SET(offset, QUEUE_INFO_PLDOFF) |
 580                        HINIC_SQ_CTRL_SET(tso, QUEUE_INFO_TSO) |
 581                        HINIC_SQ_CTRL_SET(ufo, QUEUE_INFO_UFO) |
 582                        HINIC_SQ_CTRL_SET(!!l4_offload, QUEUE_INFO_TCPUDP_CS);
 583
 584         /* set MSS value */
 585         *queue_info = HINIC_SQ_CTRL_CLEAR(*queue_info, QUEUE_INFO_MSS);
 586         *queue_info |= HINIC_SQ_CTRL_SET(mss, QUEUE_INFO_MSS);
 587 }
 588
 589 /**
 590  * hinic_sq_prepare_wqe - prepare wqe before insert to the queue
 591  * @sq: send queue
 592  * @prod_idx: pi value
 593  * @sq_wqe: wqe to prepare
 594  * @sges: sges for use by the wqe for send for buf addresses
 595  * @nr_sges: number of sges
 596  **/
 597 void hinic_sq_prepare_wqe(struct hinic_sq *sq, u16 prod_idx,
 598                           struct hinic_sq_wqe *sq_wqe, struct hinic_sge *sges,
 599                           int nr_sges)
 600 {
 601         int i;
 602
 603         sq_prepare_ctrl(&sq_wqe->ctrl, prod_idx, nr_sges);
 604
 605         sq_prepare_task(&sq_wqe->task);
 606
 607         for (i = 0; i < nr_sges; i++)
 608                 sq_wqe->buf_descs[i].sge = sges[i];
 609 }
 610
 611 /**
 612  * sq_prepare_db - prepare doorbell to write
 613  * @sq: send queue
 614  * @prod_idx: pi value for the doorbell
 615  * @cos: cos of the doorbell
 616  *
 617  * Return db value
 618  **/
 619 static u32 sq_prepare_db(struct hinic_sq *sq, u16 prod_idx, unsigned int cos)
 620 {
 621         struct hinic_qp *qp = container_of(sq, struct hinic_qp, sq);
 622         u8 hi_prod_idx = SQ_DB_PI_HI(SQ_MASKED_IDX(sq, prod_idx));
 623
 624         /* Data should be written to HW in Big Endian Format */
 625         return cpu_to_be32(HINIC_SQ_DB_INFO_SET(hi_prod_idx, PI_HI)     |
 626                            HINIC_SQ_DB_INFO_SET(HINIC_DB_SQ_TYPE, TYPE) |
 627                            HINIC_SQ_DB_INFO_SET(HINIC_DATA_PATH, PATH)  |
 628                            HINIC_SQ_DB_INFO_SET(cos, COS)               |
 629                            HINIC_SQ_DB_INFO_SET(qp->q_id, QID));
 630 }
 631
 632 /**
 633  * hinic_sq_write_db- write doorbell
 634  * @sq: send queue
 635  * @prod_idx: pi value for the doorbell
 636  * @wqe_size: wqe size
 637  * @cos: cos of the wqe
 638  **/
 639 void hinic_sq_write_db(struct hinic_sq *sq, u16 prod_idx, unsigned int wqe_size,
 640                        unsigned int cos)
 641 {
 642         struct hinic_wq *wq = sq->wq;
 643
 644         /* increment prod_idx to the next */
 645         prod_idx += ALIGN(wqe_size, wq->wqebb_size) / wq->wqebb_size;
 646         prod_idx = SQ_MASKED_IDX(sq, prod_idx);
 647
 648         wmb();  /* Write all before the doorbell */
 649
 650         writel(sq_prepare_db(sq, prod_idx, cos), SQ_DB_ADDR(sq, prod_idx));
 651 }
 652
 653 /**
 654  * hinic_sq_get_wqe - get wqe ptr in the current pi and update the pi
 655  * @sq: sq to get wqe from
 656  * @wqe_size: wqe size
 657  * @prod_idx: returned pi
 658  *
 659  * Return wqe pointer
 660  **/
 661 struct hinic_sq_wqe *hinic_sq_get_wqe(struct hinic_sq *sq,
 662                                       unsigned int wqe_size, u16 *prod_idx)
 663 {
 664         struct hinic_hw_wqe *hw_wqe = hinic_get_wqe(sq->wq, wqe_size,
 665                                                     prod_idx);
 666
 667         if (IS_ERR(hw_wqe))
 668                 return NULL;
 669
 670         return &hw_wqe->sq_wqe;
 671 }
 672
 673 /**
 674  * hinic_sq_return_wqe - return the wqe to the sq
 675  * @sq: send queue
 676  * @wqe_size: the size of the wqe
 677  **/
 678 void hinic_sq_return_wqe(struct hinic_sq *sq, unsigned int wqe_size)
 679 {
 680         hinic_return_wqe(sq->wq, wqe_size);
 681 }
 682
 683 /**
 684  * hinic_sq_write_wqe - write the wqe to the sq
 685  * @sq: send queue
 686  * @prod_idx: pi of the wqe
 687  * @sq_wqe: the wqe to write
 688  * @skb: skb to save
 689  * @wqe_size: the size of the wqe
 690  **/
 691 void hinic_sq_write_wqe(struct hinic_sq *sq, u16 prod_idx,
 692                         struct hinic_sq_wqe *sq_wqe,
 693                         struct sk_buff *skb, unsigned int wqe_size)
 694 {
 695         struct hinic_hw_wqe *hw_wqe = (struct hinic_hw_wqe *)sq_wqe;
 696
 697         sq->saved_skb[prod_idx] = skb;
 698
 699         /* The data in the HW should be in Big Endian Format */
 700         hinic_cpu_to_be32(sq_wqe, wqe_size);
 701
 702         hinic_write_wqe(sq->wq, hw_wqe, wqe_size);
 703 }
 704
 705 /**
 706  * hinic_sq_read_wqebb - read wqe ptr in the current ci and update the ci, the
 707  * wqe only have one wqebb
 708  * @sq: send queue
 709  * @skb: return skb that was saved
 710  * @wqe_size: the wqe size ptr
 711  * @cons_idx: consumer index of the wqe
 712  *
 713  * Return wqe in ci position
 714  **/
 715 struct hinic_sq_wqe *hinic_sq_read_wqebb(struct hinic_sq *sq,
 716                                          struct sk_buff **skb,
 717                                          unsigned int *wqe_size, u16 *cons_idx)
 718 {
 719         struct hinic_hw_wqe *hw_wqe;
 720         struct hinic_sq_wqe *sq_wqe;
 721         struct hinic_sq_ctrl *ctrl;
 722         unsigned int buf_sect_len;
 723         u32 ctrl_info;
 724
 725         /* read the ctrl section for getting wqe size */
 726         hw_wqe = hinic_read_wqe(sq->wq, sizeof(*ctrl), cons_idx);
 727         if (IS_ERR(hw_wqe))
 728                 return NULL;
 729
 730         *skb = sq->saved_skb[*cons_idx];
 731
 732         sq_wqe = &hw_wqe->sq_wqe;
 733         ctrl = &sq_wqe->ctrl;
 734         ctrl_info = be32_to_cpu(ctrl->ctrl_info);
 735         buf_sect_len = HINIC_SQ_CTRL_GET(ctrl_info, BUFDESC_SECT_LEN);
 736
 737         *wqe_size = sizeof(*ctrl) + sizeof(sq_wqe->task);
 738         *wqe_size += SECT_SIZE_FROM_8BYTES(buf_sect_len);
 739         *wqe_size = ALIGN(*wqe_size, sq->wq->wqebb_size);
 740
 741         return &hw_wqe->sq_wqe;
 742 }
 743
 744 /**
 745  * hinic_sq_read_wqe - read wqe ptr in the current ci and update the ci
 746  * @sq: send queue
 747  * @skb: return skb that was saved
 748  * @wqe_size: the size of the wqe
 749  * @cons_idx: consumer index of the wqe
 750  *
 751  * Return wqe in ci position
 752  **/
 753 struct hinic_sq_wqe *hinic_sq_read_wqe(struct hinic_sq *sq,
 754                                        struct sk_buff **skb,
 755                                        unsigned int wqe_size, u16 *cons_idx)
 756 {
 757         struct hinic_hw_wqe *hw_wqe;
 758
 759         hw_wqe = hinic_read_wqe(sq->wq, wqe_size, cons_idx);
 760         *skb = sq->saved_skb[*cons_idx];
 761
 762         return &hw_wqe->sq_wqe;
 763 }
 764
 765 /**
 766  * hinic_sq_put_wqe - release the ci for new wqes
 767  * @sq: send queue
 768  * @wqe_size: the size of the wqe
 769  **/
 770 void hinic_sq_put_wqe(struct hinic_sq *sq, unsigned int wqe_size)
 771 {
 772         hinic_put_wqe(sq->wq, wqe_size);
 773 }
 774
 775 /**
 776  * hinic_sq_get_sges - get sges from the wqe
 777  * @sq_wqe: wqe to get the sges from its buffer addresses
 778  * @sges: returned sges
 779  * @nr_sges: number sges to return
 780  **/
 781 void hinic_sq_get_sges(struct hinic_sq_wqe *sq_wqe, struct hinic_sge *sges,
 782                        int nr_sges)
 783 {
 784         int i;
 785
 786         for (i = 0; i < nr_sges && i < HINIC_MAX_SQ_BUFDESCS; i++) {
 787                 sges[i] = sq_wqe->buf_descs[i].sge;
 788                 hinic_be32_to_cpu(&sges[i], sizeof(sges[i]));
 789         }
 790 }
 791
 792 /**
 793  * hinic_rq_get_wqe - get wqe ptr in the current pi and update the pi
 794  * @rq: rq to get wqe from
 795  * @wqe_size: wqe size
 796  * @prod_idx: returned pi
 797  *
 798  * Return wqe pointer
 799  **/
 800 struct hinic_rq_wqe *hinic_rq_get_wqe(struct hinic_rq *rq,
 801                                       unsigned int wqe_size, u16 *prod_idx)
 802 {
 803         struct hinic_hw_wqe *hw_wqe = hinic_get_wqe(rq->wq, wqe_size,
 804                                                     prod_idx);
 805
 806         if (IS_ERR(hw_wqe))
 807                 return NULL;
 808
 809         return &hw_wqe->rq_wqe;
 810 }
 811
 812 /**
 813  * hinic_rq_write_wqe - write the wqe to the rq
 814  * @rq: recv queue
 815  * @prod_idx: pi of the wqe
 816  * @rq_wqe: the wqe to write
 817  * @skb: skb to save
 818  **/
 819 void hinic_rq_write_wqe(struct hinic_rq *rq, u16 prod_idx,
 820                         struct hinic_rq_wqe *rq_wqe, struct sk_buff *skb)
 821 {
 822         struct hinic_hw_wqe *hw_wqe = (struct hinic_hw_wqe *)rq_wqe;
 823
 824         rq->saved_skb[prod_idx] = skb;
 825
 826         /* The data in the HW should be in Big Endian Format */
 827         hinic_cpu_to_be32(rq_wqe, sizeof(*rq_wqe));
 828
 829         hinic_write_wqe(rq->wq, hw_wqe, sizeof(*rq_wqe));
 830 }
 831
 832 /**
 833  * hinic_rq_read_wqe - read wqe ptr in the current ci and update the ci
 834  * @rq: recv queue
 835  * @wqe_size: the size of the wqe
 836  * @skb: return saved skb
 837  * @cons_idx: consumer index of the wqe
 838  *
 839  * Return wqe in ci position
 840  **/
 841 struct hinic_rq_wqe *hinic_rq_read_wqe(struct hinic_rq *rq,
 842                                        unsigned int wqe_size,
 843                                        struct sk_buff **skb, u16 *cons_idx)
 844 {
 845         struct hinic_hw_wqe *hw_wqe;
 846         struct hinic_rq_cqe *cqe;
 847         int rx_done;
 848         u32 status;
 849
 850         hw_wqe = hinic_read_wqe(rq->wq, wqe_size, cons_idx);
 851         if (IS_ERR(hw_wqe))
 852                 return NULL;
 853
 854         cqe = rq->cqe[*cons_idx];
 855
 856         status = be32_to_cpu(cqe->status);
 857
 858         rx_done = HINIC_RQ_CQE_STATUS_GET(status, RXDONE);
 859         if (!rx_done)
 860                 return NULL;
 861
 862         *skb = rq->saved_skb[*cons_idx];
 863
 864         return &hw_wqe->rq_wqe;
 865 }
 866
 867 /**
 868  * hinic_rq_read_next_wqe - increment ci and read the wqe in ci position
 869  * @rq: recv queue
 870  * @wqe_size: the size of the wqe
 871  * @skb: return saved skb
 872  * @cons_idx: consumer index in the wq
 873  *
 874  * Return wqe in incremented ci position
 875  **/
 876 struct hinic_rq_wqe *hinic_rq_read_next_wqe(struct hinic_rq *rq,
 877                                             unsigned int wqe_size,
 878                                             struct sk_buff **skb,
 879                                             u16 *cons_idx)
 880 {
 881         struct hinic_wq *wq = rq->wq;
 882         struct hinic_hw_wqe *hw_wqe;
 883         unsigned int num_wqebbs;
 884
 885         wqe_size = ALIGN(wqe_size, wq->wqebb_size);
 886         num_wqebbs = wqe_size / wq->wqebb_size;
 887
 888         *cons_idx = RQ_MASKED_IDX(rq, *cons_idx + num_wqebbs);
 889
 890         *skb = rq->saved_skb[*cons_idx];
 891
 892         hw_wqe = hinic_read_wqe_direct(wq, *cons_idx);
 893
 894         return &hw_wqe->rq_wqe;
 895 }
 896
 897 /**
 898  * hinic_put_wqe - release the ci for new wqes
 899  * @rq: recv queue
 900  * @cons_idx: consumer index of the wqe
 901  * @wqe_size: the size of the wqe
 902  **/
 903 void hinic_rq_put_wqe(struct hinic_rq *rq, u16 cons_idx,
 904                       unsigned int wqe_size)
 905 {
 906         struct hinic_rq_cqe *cqe = rq->cqe[cons_idx];
 907         u32 status = be32_to_cpu(cqe->status);
 908
 909         status = HINIC_RQ_CQE_STATUS_CLEAR(status, RXDONE);
 910
 911         /* Rx WQE size is 1 WQEBB, no wq shadow*/
 912         cqe->status = cpu_to_be32(status);
 913
 914         wmb();          /* clear done flag */
 915
 916         hinic_put_wqe(rq->wq, wqe_size);
 917 }
 918
 919 /**
 920  * hinic_rq_get_sge - get sge from the wqe
 921  * @rq: recv queue
 922  * @rq_wqe: wqe to get the sge from its buf address
 923  * @cons_idx: consumer index
 924  * @sge: returned sge
 925  **/
 926 void hinic_rq_get_sge(struct hinic_rq *rq, struct hinic_rq_wqe *rq_wqe,
 927                       u16 cons_idx, struct hinic_sge *sge)
 928 {
 929         struct hinic_rq_cqe *cqe = rq->cqe[cons_idx];
 930         u32 len = be32_to_cpu(cqe->len);
 931
 932         sge->hi_addr = be32_to_cpu(rq_wqe->buf_desc.hi_addr);
 933         sge->lo_addr = be32_to_cpu(rq_wqe->buf_desc.lo_addr);
 934         sge->len = HINIC_RQ_CQE_SGE_GET(len, LEN);
 935 }
 936
 937 /**
 938  * hinic_rq_prepare_wqe - prepare wqe before insert to the queue
 939  * @rq: recv queue
 940  * @prod_idx: pi value
 941  * @rq_wqe: the wqe
 942  * @sge: sge for use by the wqe for recv buf address
 943  **/
 944 void hinic_rq_prepare_wqe(struct hinic_rq *rq, u16 prod_idx,
 945                           struct hinic_rq_wqe *rq_wqe, struct hinic_sge *sge)
 946 {
 947         struct hinic_rq_cqe_sect *cqe_sect = &rq_wqe->cqe_sect;
 948         struct hinic_rq_bufdesc *buf_desc = &rq_wqe->buf_desc;
 949         struct hinic_rq_cqe *cqe = rq->cqe[prod_idx];
 950         struct hinic_rq_ctrl *ctrl = &rq_wqe->ctrl;
 951         dma_addr_t cqe_dma = rq->cqe_dma[prod_idx];
 952
 953         ctrl->ctrl_info =
 954                 HINIC_RQ_CTRL_SET(SIZE_8BYTES(sizeof(*ctrl)), LEN) |
 955                 HINIC_RQ_CTRL_SET(SIZE_8BYTES(sizeof(*cqe_sect)),
 956                                   COMPLETE_LEN)                    |
 957                 HINIC_RQ_CTRL_SET(SIZE_8BYTES(sizeof(*buf_desc)),
 958                                   BUFDESC_SECT_LEN)                |
 959                 HINIC_RQ_CTRL_SET(RQ_COMPLETE_SGE, COMPLETE_FORMAT);
 960
 961         hinic_set_sge(&cqe_sect->sge, cqe_dma, sizeof(*cqe));
 962
 963         buf_desc->hi_addr = sge->hi_addr;
 964         buf_desc->lo_addr = sge->lo_addr;
 965 }
 966
 967 /**
 968  * hinic_rq_update - update pi of the rq
 969  * @rq: recv queue
 970  * @prod_idx: pi value
 971  **/
 972 void hinic_rq_update(struct hinic_rq *rq, u16 prod_idx)
 973 {
 974         *rq->pi_virt_addr = cpu_to_be16(RQ_MASKED_IDX(rq, prod_idx + 1));
 975 }