drivers/net/ethernet/huawei/hinic/hinic_hw_qp.c

   1 /*
   2  * Huawei HiNIC PCI Express Linux driver
   3  * Copyright(c) 2017 Huawei Technologies Co., Ltd
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * for more details.
  13  *
  14  */
  15
  16 #include <linux/kernel.h>
  17 #include <linux/types.h>
  18 #include <linux/pci.h>
  19 #include <linux/device.h>
  20 #include <linux/dma-mapping.h>
  21 #include <linux/vmalloc.h>
  22 #include <linux/errno.h>
  23 #include <linux/sizes.h>
  24 #include <linux/atomic.h>
  25 #include <linux/skbuff.h>
  26 #include <linux/io.h>
  27 #include <asm/barrier.h>
  28 #include <asm/byteorder.h>
  29
  30 #include "hinic_common.h"
  31 #include "hinic_hw_if.h"
  32 #include "hinic_hw_wqe.h"
  33 #include "hinic_hw_wq.h"
  34 #include "hinic_hw_qp_ctxt.h"
  35 #include "hinic_hw_qp.h"
  36 #include "hinic_hw_io.h"
  37
  38 #define SQ_DB_OFF               SZ_2K
  39
  40 /* The number of cache line to prefetch Until threshold state */
  41 #define WQ_PREFETCH_MAX         2
  42 /* The number of cache line to prefetch After threshold state */
  43 #define WQ_PREFETCH_MIN         1
  44 /* Threshold state */
  45 #define WQ_PREFETCH_THRESHOLD   256
  46
  47 /* sizes of the SQ/RQ ctxt */
  48 #define Q_CTXT_SIZE             48
  49 #define CTXT_RSVD               240
  50
  51 #define SQ_CTXT_OFFSET(max_sqs, max_rqs, q_id)  \
  52                 (((max_rqs) + (max_sqs)) * CTXT_RSVD + (q_id) * Q_CTXT_SIZE)
  53
  54 #define RQ_CTXT_OFFSET(max_sqs, max_rqs, q_id)  \
  55                 (((max_rqs) + (max_sqs)) * CTXT_RSVD + \
  56                  (max_sqs + (q_id)) * Q_CTXT_SIZE)
  57
  58 #define SIZE_16BYTES(size)              (ALIGN(size, 16) >> 4)
  59 #define SIZE_8BYTES(size)               (ALIGN(size, 8) >> 3)
  60 #define SECT_SIZE_FROM_8BYTES(size)     ((size) << 3)
  61
  62 #define SQ_DB_PI_HI_SHIFT       8
  63 #define SQ_DB_PI_HI(prod_idx)   ((prod_idx) >> SQ_DB_PI_HI_SHIFT)
  64
  65 #define SQ_DB_PI_LOW_MASK       0xFF
  66 #define SQ_DB_PI_LOW(prod_idx)  ((prod_idx) & SQ_DB_PI_LOW_MASK)
  67
  68 #define SQ_DB_ADDR(sq, pi)      ((u64 *)((sq)->db_base) + SQ_DB_PI_LOW(pi))
  69
  70 #define SQ_MASKED_IDX(sq, idx)  ((idx) & (sq)->wq->mask)
  71 #define RQ_MASKED_IDX(rq, idx)  ((idx) & (rq)->wq->mask)
  72
  73 #define TX_MAX_MSS_DEFAULT      0x3E00
  74
  75 enum sq_wqe_type {
  76         SQ_NORMAL_WQE = 0,
  77 };
  78
  79 enum rq_completion_fmt {
  80         RQ_COMPLETE_SGE = 1
  81 };
  82
  83 void hinic_qp_prepare_header(struct hinic_qp_ctxt_header *qp_ctxt_hdr,
  84                              enum hinic_qp_ctxt_type ctxt_type,
  85                              u16 num_queues, u16 max_queues)
  86 {
  87         u16 max_sqs = max_queues;
  88         u16 max_rqs = max_queues;
  89
  90         qp_ctxt_hdr->num_queues = num_queues;
  91         qp_ctxt_hdr->queue_type = ctxt_type;
  92
  93         if (ctxt_type == HINIC_QP_CTXT_TYPE_SQ)
  94                 qp_ctxt_hdr->addr_offset = SQ_CTXT_OFFSET(max_sqs, max_rqs, 0);
  95         else
  96                 qp_ctxt_hdr->addr_offset = RQ_CTXT_OFFSET(max_sqs, max_rqs, 0);
  97
  98         qp_ctxt_hdr->addr_offset = SIZE_16BYTES(qp_ctxt_hdr->addr_offset);
  99
 100         hinic_cpu_to_be32(qp_ctxt_hdr, sizeof(*qp_ctxt_hdr));
 101 }
 102
 103 void hinic_sq_prepare_ctxt(struct hinic_sq_ctxt *sq_ctxt,
 104                            struct hinic_sq *sq, u16 global_qid)
 105 {
 106         u32 wq_page_pfn_hi, wq_page_pfn_lo, wq_block_pfn_hi, wq_block_pfn_lo;
 107         u64 wq_page_addr, wq_page_pfn, wq_block_pfn;
 108         u16 pi_start, ci_start;
 109         struct hinic_wq *wq;
 110
 111         wq = sq->wq;
 112         ci_start = atomic_read(&wq->cons_idx);
 113         pi_start = atomic_read(&wq->prod_idx);
 114
 115         /* Read the first page paddr from the WQ page paddr ptrs */
 116         wq_page_addr = be64_to_cpu(*wq->block_vaddr);
 117
 118         wq_page_pfn = HINIC_WQ_PAGE_PFN(wq_page_addr);
 119         wq_page_pfn_hi = upper_32_bits(wq_page_pfn);
 120         wq_page_pfn_lo = lower_32_bits(wq_page_pfn);
 121
 122         wq_block_pfn = HINIC_WQ_BLOCK_PFN(wq->block_paddr);
 123         wq_block_pfn_hi = upper_32_bits(wq_block_pfn);
 124         wq_block_pfn_lo = lower_32_bits(wq_block_pfn);
 125
 126         sq_ctxt->ceq_attr = HINIC_SQ_CTXT_CEQ_ATTR_SET(global_qid,
 127                                                        GLOBAL_SQ_ID) |
 128                             HINIC_SQ_CTXT_CEQ_ATTR_SET(0, EN);
 129
 130         sq_ctxt->ci_wrapped = HINIC_SQ_CTXT_CI_SET(ci_start, IDX) |
 131                               HINIC_SQ_CTXT_CI_SET(1, WRAPPED);
 132
 133         sq_ctxt->wq_hi_pfn_pi =
 134                         HINIC_SQ_CTXT_WQ_PAGE_SET(wq_page_pfn_hi, HI_PFN) |
 135                         HINIC_SQ_CTXT_WQ_PAGE_SET(pi_start, PI);
 136
 137         sq_ctxt->wq_lo_pfn = wq_page_pfn_lo;
 138
 139         sq_ctxt->pref_cache =
 140                 HINIC_SQ_CTXT_PREF_SET(WQ_PREFETCH_MIN, CACHE_MIN) |
 141                 HINIC_SQ_CTXT_PREF_SET(WQ_PREFETCH_MAX, CACHE_MAX) |
 142                 HINIC_SQ_CTXT_PREF_SET(WQ_PREFETCH_THRESHOLD, CACHE_THRESHOLD);
 143
 144         sq_ctxt->pref_wrapped = 1;
 145
 146         sq_ctxt->pref_wq_hi_pfn_ci =
 147                 HINIC_SQ_CTXT_PREF_SET(ci_start, CI) |
 148                 HINIC_SQ_CTXT_PREF_SET(wq_page_pfn_hi, WQ_HI_PFN);
 149
 150         sq_ctxt->pref_wq_lo_pfn = wq_page_pfn_lo;
 151
 152         sq_ctxt->wq_block_hi_pfn =
 153                 HINIC_SQ_CTXT_WQ_BLOCK_SET(wq_block_pfn_hi, HI_PFN);
 154
 155         sq_ctxt->wq_block_lo_pfn = wq_block_pfn_lo;
 156
 157         hinic_cpu_to_be32(sq_ctxt, sizeof(*sq_ctxt));
 158 }
 159
 160 void hinic_rq_prepare_ctxt(struct hinic_rq_ctxt *rq_ctxt,
 161                            struct hinic_rq *rq, u16 global_qid)
 162 {
 163         u32 wq_page_pfn_hi, wq_page_pfn_lo, wq_block_pfn_hi, wq_block_pfn_lo;
 164         u64 wq_page_addr, wq_page_pfn, wq_block_pfn;
 165         u16 pi_start, ci_start;
 166         struct hinic_wq *wq;
 167
 168         wq = rq->wq;
 169         ci_start = atomic_read(&wq->cons_idx);
 170         pi_start = atomic_read(&wq->prod_idx);
 171
 172         /* Read the first page paddr from the WQ page paddr ptrs */
 173         wq_page_addr = be64_to_cpu(*wq->block_vaddr);
 174
 175         wq_page_pfn = HINIC_WQ_PAGE_PFN(wq_page_addr);
 176         wq_page_pfn_hi = upper_32_bits(wq_page_pfn);
 177         wq_page_pfn_lo = lower_32_bits(wq_page_pfn);
 178
 179         wq_block_pfn = HINIC_WQ_BLOCK_PFN(wq->block_paddr);
 180         wq_block_pfn_hi = upper_32_bits(wq_block_pfn);
 181         wq_block_pfn_lo = lower_32_bits(wq_block_pfn);
 182
 183         rq_ctxt->ceq_attr = HINIC_RQ_CTXT_CEQ_ATTR_SET(0, EN) |
 184                             HINIC_RQ_CTXT_CEQ_ATTR_SET(1, WRAPPED);
 185
 186         rq_ctxt->pi_intr_attr = HINIC_RQ_CTXT_PI_SET(pi_start, IDX) |
 187                                 HINIC_RQ_CTXT_PI_SET(rq->msix_entry, INTR);
 188
 189         rq_ctxt->wq_hi_pfn_ci = HINIC_RQ_CTXT_WQ_PAGE_SET(wq_page_pfn_hi,
 190                                                           HI_PFN) |
 191                                 HINIC_RQ_CTXT_WQ_PAGE_SET(ci_start, CI);
 192
 193         rq_ctxt->wq_lo_pfn = wq_page_pfn_lo;
 194
 195         rq_ctxt->pref_cache =
 196                 HINIC_RQ_CTXT_PREF_SET(WQ_PREFETCH_MIN, CACHE_MIN) |
 197                 HINIC_RQ_CTXT_PREF_SET(WQ_PREFETCH_MAX, CACHE_MAX) |
 198                 HINIC_RQ_CTXT_PREF_SET(WQ_PREFETCH_THRESHOLD, CACHE_THRESHOLD);
 199
 200         rq_ctxt->pref_wrapped = 1;
 201
 202         rq_ctxt->pref_wq_hi_pfn_ci =
 203                 HINIC_RQ_CTXT_PREF_SET(wq_page_pfn_hi, WQ_HI_PFN) |
 204                 HINIC_RQ_CTXT_PREF_SET(ci_start, CI);
 205
 206         rq_ctxt->pref_wq_lo_pfn = wq_page_pfn_lo;
 207
 208         rq_ctxt->pi_paddr_hi = upper_32_bits(rq->pi_dma_addr);
 209         rq_ctxt->pi_paddr_lo = lower_32_bits(rq->pi_dma_addr);
 210
 211         rq_ctxt->wq_block_hi_pfn =
 212                 HINIC_RQ_CTXT_WQ_BLOCK_SET(wq_block_pfn_hi, HI_PFN);
 213
 214         rq_ctxt->wq_block_lo_pfn = wq_block_pfn_lo;
 215
 216         hinic_cpu_to_be32(rq_ctxt, sizeof(*rq_ctxt));
 217 }
 218
 219 /**
 220  * alloc_sq_skb_arr - allocate sq array for saved skb
 221  * @sq: HW Send Queue
 222  *
 223  * Return 0 - Success, negative - Failure
 224  **/
 225 static int alloc_sq_skb_arr(struct hinic_sq *sq)
 226 {
 227         struct hinic_wq *wq = sq->wq;
 228         size_t skb_arr_size;
 229
 230         skb_arr_size = wq->q_depth * sizeof(*sq->saved_skb);
 231         sq->saved_skb = vzalloc(skb_arr_size);
 232         if (!sq->saved_skb)
 233                 return -ENOMEM;
 234
 235         return 0;
 236 }
 237
 238 /**
 239  * free_sq_skb_arr - free sq array for saved skb
 240  * @sq: HW Send Queue
 241  **/
 242 static void free_sq_skb_arr(struct hinic_sq *sq)
 243 {
 244         vfree(sq->saved_skb);
 245 }
 246
 247 /**
 248  * alloc_rq_skb_arr - allocate rq array for saved skb
 249  * @rq: HW Receive Queue
 250  *
 251  * Return 0 - Success, negative - Failure
 252  **/
 253 static int alloc_rq_skb_arr(struct hinic_rq *rq)
 254 {
 255         struct hinic_wq *wq = rq->wq;
 256         size_t skb_arr_size;
 257
 258         skb_arr_size = wq->q_depth * sizeof(*rq->saved_skb);
 259         rq->saved_skb = vzalloc(skb_arr_size);
 260         if (!rq->saved_skb)
 261                 return -ENOMEM;
 262
 263         return 0;
 264 }
 265
 266 /**
 267  * free_rq_skb_arr - free rq array for saved skb
 268  * @rq: HW Receive Queue
 269  **/
 270 static void free_rq_skb_arr(struct hinic_rq *rq)
 271 {
 272         vfree(rq->saved_skb);
 273 }
 274
 275 /**
 276  * hinic_init_sq - Initialize HW Send Queue
 277  * @sq: HW Send Queue
 278  * @hwif: HW Interface for accessing HW
 279  * @wq: Work Queue for the data of the SQ
 280  * @entry: msix entry for sq
 281  * @ci_addr: address for reading the current HW consumer index
 282  * @ci_dma_addr: dma address for reading the current HW consumer index
 283  * @db_base: doorbell base address
 284  *
 285  * Return 0 - Success, negative - Failure
 286  **/
 287 int hinic_init_sq(struct hinic_sq *sq, struct hinic_hwif *hwif,
 288                   struct hinic_wq *wq, struct msix_entry *entry,
 289                   void *ci_addr, dma_addr_t ci_dma_addr,
 290                   void __iomem *db_base)
 291 {
 292         sq->hwif = hwif;
 293
 294         sq->wq = wq;
 295
 296         sq->irq = entry->vector;
 297         sq->msix_entry = entry->entry;
 298
 299         sq->hw_ci_addr = ci_addr;
 300         sq->hw_ci_dma_addr = ci_dma_addr;
 301
 302         sq->db_base = db_base + SQ_DB_OFF;
 303
 304         return alloc_sq_skb_arr(sq);
 305 }
 306
 307 /**
 308  * hinic_clean_sq - Clean HW Send Queue's Resources
 309  * @sq: Send Queue
 310  **/
 311 void hinic_clean_sq(struct hinic_sq *sq)
 312 {
 313         free_sq_skb_arr(sq);
 314 }
 315
 316 /**
 317  * alloc_rq_cqe - allocate rq completion queue elements
 318  * @rq: HW Receive Queue
 319  *
 320  * Return 0 - Success, negative - Failure
 321  **/
 322 static int alloc_rq_cqe(struct hinic_rq *rq)
 323 {
 324         struct hinic_hwif *hwif = rq->hwif;
 325         struct pci_dev *pdev = hwif->pdev;
 326         size_t cqe_dma_size, cqe_size;
 327         struct hinic_wq *wq = rq->wq;
 328         int j, i;
 329
 330         cqe_size = wq->q_depth * sizeof(*rq->cqe);
 331         rq->cqe = vzalloc(cqe_size);
 332         if (!rq->cqe)
 333                 return -ENOMEM;
 334
 335         cqe_dma_size = wq->q_depth * sizeof(*rq->cqe_dma);
 336         rq->cqe_dma = vzalloc(cqe_dma_size);
 337         if (!rq->cqe_dma)
 338                 goto err_cqe_dma_arr_alloc;
 339
 340         for (i = 0; i < wq->q_depth; i++) {
 341                 rq->cqe[i] = dma_zalloc_coherent(&pdev->dev,
 342                                                  sizeof(*rq->cqe[i]),
 343                                                  &rq->cqe_dma[i], GFP_KERNEL);
 344                 if (!rq->cqe[i])
 345                         goto err_cqe_alloc;
 346         }
 347
 348         return 0;
 349
 350 err_cqe_alloc:
 351         for (j = 0; j < i; j++)
 352                 dma_free_coherent(&pdev->dev, sizeof(*rq->cqe[j]), rq->cqe[j],
 353                                   rq->cqe_dma[j]);
 354
 355         vfree(rq->cqe_dma);
 356
 357 err_cqe_dma_arr_alloc:
 358         vfree(rq->cqe);
 359         return -ENOMEM;
 360 }
 361
 362 /**
 363  * free_rq_cqe - free rq completion queue elements
 364  * @rq: HW Receive Queue
 365  **/
 366 static void free_rq_cqe(struct hinic_rq *rq)
 367 {
 368         struct hinic_hwif *hwif = rq->hwif;
 369         struct pci_dev *pdev = hwif->pdev;
 370         struct hinic_wq *wq = rq->wq;
 371         int i;
 372
 373         for (i = 0; i < wq->q_depth; i++)
 374                 dma_free_coherent(&pdev->dev, sizeof(*rq->cqe[i]), rq->cqe[i],
 375                                   rq->cqe_dma[i]);
 376
 377         vfree(rq->cqe_dma);
 378         vfree(rq->cqe);
 379 }
 380
 381 /**
 382  * hinic_init_rq - Initialize HW Receive Queue
 383  * @rq: HW Receive Queue
 384  * @hwif: HW Interface for accessing HW
 385  * @wq: Work Queue for the data of the RQ
 386  * @entry: msix entry for rq
 387  *
 388  * Return 0 - Success, negative - Failure
 389  **/
 390 int hinic_init_rq(struct hinic_rq *rq, struct hinic_hwif *hwif,
 391                   struct hinic_wq *wq, struct msix_entry *entry)
 392 {
 393         struct pci_dev *pdev = hwif->pdev;
 394         size_t pi_size;
 395         int err;
 396
 397         rq->hwif = hwif;
 398
 399         rq->wq = wq;
 400
 401         rq->irq = entry->vector;
 402         rq->msix_entry = entry->entry;
 403
 404         rq->buf_sz = HINIC_RX_BUF_SZ;
 405
 406         err = alloc_rq_skb_arr(rq);
 407         if (err) {
 408                 dev_err(&pdev->dev, "Failed to allocate rq priv data\n");
 409                 return err;
 410         }
 411
 412         err = alloc_rq_cqe(rq);
 413         if (err) {
 414                 dev_err(&pdev->dev, "Failed to allocate rq cqe\n");
 415                 goto err_alloc_rq_cqe;
 416         }
 417
 418         /* HW requirements: Must be at least 32 bit */
 419         pi_size = ALIGN(sizeof(*rq->pi_virt_addr), sizeof(u32));
 420         rq->pi_virt_addr = dma_zalloc_coherent(&pdev->dev, pi_size,
 421                                                &rq->pi_dma_addr, GFP_KERNEL);
 422         if (!rq->pi_virt_addr) {
 423                 dev_err(&pdev->dev, "Failed to allocate PI address\n");
 424                 err = -ENOMEM;
 425                 goto err_pi_virt;
 426         }
 427
 428         return 0;
 429
 430 err_pi_virt:
 431         free_rq_cqe(rq);
 432
 433 err_alloc_rq_cqe:
 434         free_rq_skb_arr(rq);
 435         return err;
 436 }
 437
 438 /**
 439  * hinic_clean_rq - Clean HW Receive Queue's Resources
 440  * @rq: HW Receive Queue
 441  **/
 442 void hinic_clean_rq(struct hinic_rq *rq)
 443 {
 444         struct hinic_hwif *hwif = rq->hwif;
 445         struct pci_dev *pdev = hwif->pdev;
 446         size_t pi_size;
 447
 448         pi_size = ALIGN(sizeof(*rq->pi_virt_addr), sizeof(u32));
 449         dma_free_coherent(&pdev->dev, pi_size, rq->pi_virt_addr,
 450                           rq->pi_dma_addr);
 451
 452         free_rq_cqe(rq);
 453         free_rq_skb_arr(rq);
 454 }
 455
 456 /**
 457  * hinic_get_sq_free_wqebbs - return number of free wqebbs for use
 458  * @sq: send queue
 459  *
 460  * Return number of free wqebbs
 461  **/
 462 int hinic_get_sq_free_wqebbs(struct hinic_sq *sq)
 463 {
 464         struct hinic_wq *wq = sq->wq;
 465
 466         return atomic_read(&wq->delta) - 1;
 467 }
 468
 469 /**
 470  * hinic_get_rq_free_wqebbs - return number of free wqebbs for use
 471  * @rq: recv queue
 472  *
 473  * Return number of free wqebbs
 474  **/
 475 int hinic_get_rq_free_wqebbs(struct hinic_rq *rq)
 476 {
 477         struct hinic_wq *wq = rq->wq;
 478
 479         return atomic_read(&wq->delta) - 1;
 480 }
 481
 482 static void sq_prepare_ctrl(struct hinic_sq_ctrl *ctrl, u16 prod_idx,
 483                             int nr_descs)
 484 {
 485         u32 ctrl_size, task_size, bufdesc_size;
 486
 487         ctrl_size = SIZE_8BYTES(sizeof(struct hinic_sq_ctrl));
 488         task_size = SIZE_8BYTES(sizeof(struct hinic_sq_task));
 489         bufdesc_size = nr_descs * sizeof(struct hinic_sq_bufdesc);
 490         bufdesc_size = SIZE_8BYTES(bufdesc_size);
 491
 492         ctrl->ctrl_info = HINIC_SQ_CTRL_SET(bufdesc_size, BUFDESC_SECT_LEN) |
 493                           HINIC_SQ_CTRL_SET(task_size, TASKSECT_LEN)        |
 494                           HINIC_SQ_CTRL_SET(SQ_NORMAL_WQE, DATA_FORMAT)     |
 495                           HINIC_SQ_CTRL_SET(ctrl_size, LEN);
 496
 497         ctrl->queue_info = HINIC_SQ_CTRL_SET(TX_MAX_MSS_DEFAULT,
 498                                              QUEUE_INFO_MSS);
 499 }
 500
 501 static void sq_prepare_task(struct hinic_sq_task *task)
 502 {
 503         task->pkt_info0 =
 504                 HINIC_SQ_TASK_INFO0_SET(0, L2HDR_LEN) |
 505                 HINIC_SQ_TASK_INFO0_SET(HINIC_L4_OFF_DISABLE, L4_OFFLOAD) |
 506                 HINIC_SQ_TASK_INFO0_SET(HINIC_OUTER_L3TYPE_UNKNOWN,
 507                                         INNER_L3TYPE) |
 508                 HINIC_SQ_TASK_INFO0_SET(HINIC_VLAN_OFF_DISABLE,
 509                                         VLAN_OFFLOAD) |
 510                 HINIC_SQ_TASK_INFO0_SET(HINIC_PKT_NOT_PARSED, PARSE_FLAG);
 511
 512         task->pkt_info1 =
 513                 HINIC_SQ_TASK_INFO1_SET(HINIC_MEDIA_UNKNOWN, MEDIA_TYPE) |
 514                 HINIC_SQ_TASK_INFO1_SET(0, INNER_L4_LEN) |
 515                 HINIC_SQ_TASK_INFO1_SET(0, INNER_L3_LEN);
 516
 517         task->pkt_info2 =
 518                 HINIC_SQ_TASK_INFO2_SET(0, TUNNEL_L4_LEN) |
 519                 HINIC_SQ_TASK_INFO2_SET(0, OUTER_L3_LEN)  |
 520                 HINIC_SQ_TASK_INFO2_SET(HINIC_TUNNEL_L4TYPE_UNKNOWN,
 521                                         TUNNEL_L4TYPE)    |
 522                 HINIC_SQ_TASK_INFO2_SET(HINIC_OUTER_L3TYPE_UNKNOWN,
 523                                         OUTER_L3TYPE);
 524
 525         task->ufo_v6_identify = 0;
 526
 527         task->pkt_info4 = HINIC_SQ_TASK_INFO4_SET(HINIC_L2TYPE_ETH, L2TYPE);
 528
 529         task->zero_pad = 0;
 530 }
 531
 532 /**
 533  * hinic_sq_prepare_wqe - prepare wqe before insert to the queue
 534  * @sq: send queue
 535  * @prod_idx: pi value
 536  * @sq_wqe: wqe to prepare
 537  * @sges: sges for use by the wqe for send for buf addresses
 538  * @nr_sges: number of sges
 539  **/
 540 void hinic_sq_prepare_wqe(struct hinic_sq *sq, u16 prod_idx,
 541                           struct hinic_sq_wqe *sq_wqe, struct hinic_sge *sges,
 542                           int nr_sges)
 543 {
 544         int i;
 545
 546         sq_prepare_ctrl(&sq_wqe->ctrl, prod_idx, nr_sges);
 547
 548         sq_prepare_task(&sq_wqe->task);
 549
 550         for (i = 0; i < nr_sges; i++)
 551                 sq_wqe->buf_descs[i].sge = sges[i];
 552 }
 553
 554 /**
 555  * sq_prepare_db - prepare doorbell to write
 556  * @sq: send queue
 557  * @prod_idx: pi value for the doorbell
 558  * @cos: cos of the doorbell
 559  *
 560  * Return db value
 561  **/
 562 static u32 sq_prepare_db(struct hinic_sq *sq, u16 prod_idx, unsigned int cos)
 563 {
 564         struct hinic_qp *qp = container_of(sq, struct hinic_qp, sq);
 565         u8 hi_prod_idx = SQ_DB_PI_HI(SQ_MASKED_IDX(sq, prod_idx));
 566
 567         /* Data should be written to HW in Big Endian Format */
 568         return cpu_to_be32(HINIC_SQ_DB_INFO_SET(hi_prod_idx, PI_HI)     |
 569                            HINIC_SQ_DB_INFO_SET(HINIC_DB_SQ_TYPE, TYPE) |
 570                            HINIC_SQ_DB_INFO_SET(HINIC_DATA_PATH, PATH)  |
 571                            HINIC_SQ_DB_INFO_SET(cos, COS)               |
 572                            HINIC_SQ_DB_INFO_SET(qp->q_id, QID));
 573 }
 574
 575 /**
 576  * hinic_sq_write_db- write doorbell
 577  * @sq: send queue
 578  * @prod_idx: pi value for the doorbell
 579  * @wqe_size: wqe size
 580  * @cos: cos of the wqe
 581  **/
 582 void hinic_sq_write_db(struct hinic_sq *sq, u16 prod_idx, unsigned int wqe_size,
 583                        unsigned int cos)
 584 {
 585         struct hinic_wq *wq = sq->wq;
 586
 587         /* increment prod_idx to the next */
 588         prod_idx += ALIGN(wqe_size, wq->wqebb_size) / wq->wqebb_size;
 589
 590         wmb();  /* Write all before the doorbell */
 591
 592         writel(sq_prepare_db(sq, prod_idx, cos), SQ_DB_ADDR(sq, prod_idx));
 593 }
 594
 595 /**
 596  * hinic_sq_get_wqe - get wqe ptr in the current pi and update the pi
 597  * @sq: sq to get wqe from
 598  * @wqe_size: wqe size
 599  * @prod_idx: returned pi
 600  *
 601  * Return wqe pointer
 602  **/
 603 struct hinic_sq_wqe *hinic_sq_get_wqe(struct hinic_sq *sq,
 604                                       unsigned int wqe_size, u16 *prod_idx)
 605 {
 606         struct hinic_hw_wqe *hw_wqe = hinic_get_wqe(sq->wq, wqe_size,
 607                                                     prod_idx);
 608
 609         if (IS_ERR(hw_wqe))
 610                 return NULL;
 611
 612         return &hw_wqe->sq_wqe;
 613 }
 614
 615 /**
 616  * hinic_sq_write_wqe - write the wqe to the sq
 617  * @sq: send queue
 618  * @prod_idx: pi of the wqe
 619  * @sq_wqe: the wqe to write
 620  * @skb: skb to save
 621  * @wqe_size: the size of the wqe
 622  **/
 623 void hinic_sq_write_wqe(struct hinic_sq *sq, u16 prod_idx,
 624                         struct hinic_sq_wqe *sq_wqe,
 625                         struct sk_buff *skb, unsigned int wqe_size)
 626 {
 627         struct hinic_hw_wqe *hw_wqe = (struct hinic_hw_wqe *)sq_wqe;
 628
 629         sq->saved_skb[prod_idx] = skb;
 630
 631         /* The data in the HW should be in Big Endian Format */
 632         hinic_cpu_to_be32(sq_wqe, wqe_size);
 633
 634         hinic_write_wqe(sq->wq, hw_wqe, wqe_size);
 635 }
 636
 637 /**
 638  * hinic_sq_read_wqe - read wqe ptr in the current ci and update the ci
 639  * @sq: send queue
 640  * @skb: return skb that was saved
 641  * @wqe_size: the size of the wqe
 642  * @cons_idx: consumer index of the wqe
 643  *
 644  * Return wqe in ci position
 645  **/
 646 struct hinic_sq_wqe *hinic_sq_read_wqe(struct hinic_sq *sq,
 647                                        struct sk_buff **skb,
 648                                        unsigned int *wqe_size, u16 *cons_idx)
 649 {
 650         struct hinic_hw_wqe *hw_wqe;
 651         struct hinic_sq_wqe *sq_wqe;
 652         struct hinic_sq_ctrl *ctrl;
 653         unsigned int buf_sect_len;
 654         u32 ctrl_info;
 655
 656         /* read the ctrl section for getting wqe size */
 657         hw_wqe = hinic_read_wqe(sq->wq, sizeof(*ctrl), cons_idx);
 658         if (IS_ERR(hw_wqe))
 659                 return NULL;
 660
 661         sq_wqe = &hw_wqe->sq_wqe;
 662         ctrl = &sq_wqe->ctrl;
 663         ctrl_info = be32_to_cpu(ctrl->ctrl_info);
 664         buf_sect_len = HINIC_SQ_CTRL_GET(ctrl_info, BUFDESC_SECT_LEN);
 665
 666         *wqe_size = sizeof(*ctrl) + sizeof(sq_wqe->task);
 667         *wqe_size += SECT_SIZE_FROM_8BYTES(buf_sect_len);
 668
 669         *skb = sq->saved_skb[*cons_idx];
 670
 671         /* using the real wqe size to read wqe again */
 672         hw_wqe = hinic_read_wqe(sq->wq, *wqe_size, cons_idx);
 673
 674         return &hw_wqe->sq_wqe;
 675 }
 676
 677 /**
 678  * hinic_sq_put_wqe - release the ci for new wqes
 679  * @sq: send queue
 680  * @wqe_size: the size of the wqe
 681  **/
 682 void hinic_sq_put_wqe(struct hinic_sq *sq, unsigned int wqe_size)
 683 {
 684         hinic_put_wqe(sq->wq, wqe_size);
 685 }
 686
 687 /**
 688  * hinic_sq_get_sges - get sges from the wqe
 689  * @sq_wqe: wqe to get the sges from its buffer addresses
 690  * @sges: returned sges
 691  * @nr_sges: number sges to return
 692  **/
 693 void hinic_sq_get_sges(struct hinic_sq_wqe *sq_wqe, struct hinic_sge *sges,
 694                        int nr_sges)
 695 {
 696         int i;
 697
 698         for (i = 0; i < nr_sges && i < HINIC_MAX_SQ_BUFDESCS; i++) {
 699                 sges[i] = sq_wqe->buf_descs[i].sge;
 700                 hinic_be32_to_cpu(&sges[i], sizeof(sges[i]));
 701         }
 702 }
 703
 704 /**
 705  * hinic_rq_get_wqe - get wqe ptr in the current pi and update the pi
 706  * @rq: rq to get wqe from
 707  * @wqe_size: wqe size
 708  * @prod_idx: returned pi
 709  *
 710  * Return wqe pointer
 711  **/
 712 struct hinic_rq_wqe *hinic_rq_get_wqe(struct hinic_rq *rq,
 713                                       unsigned int wqe_size, u16 *prod_idx)
 714 {
 715         struct hinic_hw_wqe *hw_wqe = hinic_get_wqe(rq->wq, wqe_size,
 716                                                     prod_idx);
 717
 718         if (IS_ERR(hw_wqe))
 719                 return NULL;
 720
 721         return &hw_wqe->rq_wqe;
 722 }
 723
 724 /**
 725  * hinic_rq_write_wqe - write the wqe to the rq
 726  * @rq: recv queue
 727  * @prod_idx: pi of the wqe
 728  * @rq_wqe: the wqe to write
 729  * @skb: skb to save
 730  **/
 731 void hinic_rq_write_wqe(struct hinic_rq *rq, u16 prod_idx,
 732                         struct hinic_rq_wqe *rq_wqe, struct sk_buff *skb)
 733 {
 734         struct hinic_hw_wqe *hw_wqe = (struct hinic_hw_wqe *)rq_wqe;
 735
 736         rq->saved_skb[prod_idx] = skb;
 737
 738         /* The data in the HW should be in Big Endian Format */
 739         hinic_cpu_to_be32(rq_wqe, sizeof(*rq_wqe));
 740
 741         hinic_write_wqe(rq->wq, hw_wqe, sizeof(*rq_wqe));
 742 }
 743
 744 /**
 745  * hinic_rq_read_wqe - read wqe ptr in the current ci and update the ci
 746  * @rq: recv queue
 747  * @wqe_size: the size of the wqe
 748  * @skb: return saved skb
 749  * @cons_idx: consumer index of the wqe
 750  *
 751  * Return wqe in ci position
 752  **/
 753 struct hinic_rq_wqe *hinic_rq_read_wqe(struct hinic_rq *rq,
 754                                        unsigned int wqe_size,
 755                                        struct sk_buff **skb, u16 *cons_idx)
 756 {
 757         struct hinic_hw_wqe *hw_wqe;
 758         struct hinic_rq_cqe *cqe;
 759         int rx_done;
 760         u32 status;
 761
 762         hw_wqe = hinic_read_wqe(rq->wq, wqe_size, cons_idx);
 763         if (IS_ERR(hw_wqe))
 764                 return NULL;
 765
 766         cqe = rq->cqe[*cons_idx];
 767
 768         status = be32_to_cpu(cqe->status);
 769
 770         rx_done = HINIC_RQ_CQE_STATUS_GET(status, RXDONE);
 771         if (!rx_done)
 772                 return NULL;
 773
 774         *skb = rq->saved_skb[*cons_idx];
 775
 776         return &hw_wqe->rq_wqe;
 777 }
 778
 779 /**
 780  * hinic_rq_read_next_wqe - increment ci and read the wqe in ci position
 781  * @rq: recv queue
 782  * @wqe_size: the size of the wqe
 783  * @skb: return saved skb
 784  * @cons_idx: consumer index in the wq
 785  *
 786  * Return wqe in incremented ci position
 787  **/
 788 struct hinic_rq_wqe *hinic_rq_read_next_wqe(struct hinic_rq *rq,
 789                                             unsigned int wqe_size,
 790                                             struct sk_buff **skb,
 791                                             u16 *cons_idx)
 792 {
 793         struct hinic_wq *wq = rq->wq;
 794         struct hinic_hw_wqe *hw_wqe;
 795         unsigned int num_wqebbs;
 796
 797         wqe_size = ALIGN(wqe_size, wq->wqebb_size);
 798         num_wqebbs = wqe_size / wq->wqebb_size;
 799
 800         *cons_idx = RQ_MASKED_IDX(rq, *cons_idx + num_wqebbs);
 801
 802         *skb = rq->saved_skb[*cons_idx];
 803
 804         hw_wqe = hinic_read_wqe_direct(wq, *cons_idx);
 805
 806         return &hw_wqe->rq_wqe;
 807 }
 808
 809 /**
 810  * hinic_put_wqe - release the ci for new wqes
 811  * @rq: recv queue
 812  * @cons_idx: consumer index of the wqe
 813  * @wqe_size: the size of the wqe
 814  **/
 815 void hinic_rq_put_wqe(struct hinic_rq *rq, u16 cons_idx,
 816                       unsigned int wqe_size)
 817 {
 818         struct hinic_rq_cqe *cqe = rq->cqe[cons_idx];
 819         u32 status = be32_to_cpu(cqe->status);
 820
 821         status = HINIC_RQ_CQE_STATUS_CLEAR(status, RXDONE);
 822
 823         /* Rx WQE size is 1 WQEBB, no wq shadow*/
 824         cqe->status = cpu_to_be32(status);
 825
 826         wmb();          /* clear done flag */
 827
 828         hinic_put_wqe(rq->wq, wqe_size);
 829 }
 830
 831 /**
 832  * hinic_rq_get_sge - get sge from the wqe
 833  * @rq: recv queue
 834  * @rq_wqe: wqe to get the sge from its buf address
 835  * @cons_idx: consumer index
 836  * @sge: returned sge
 837  **/
 838 void hinic_rq_get_sge(struct hinic_rq *rq, struct hinic_rq_wqe *rq_wqe,
 839                       u16 cons_idx, struct hinic_sge *sge)
 840 {
 841         struct hinic_rq_cqe *cqe = rq->cqe[cons_idx];
 842         u32 len = be32_to_cpu(cqe->len);
 843
 844         sge->hi_addr = be32_to_cpu(rq_wqe->buf_desc.hi_addr);
 845         sge->lo_addr = be32_to_cpu(rq_wqe->buf_desc.lo_addr);
 846         sge->len = HINIC_RQ_CQE_SGE_GET(len, LEN);
 847 }
 848
 849 /**
 850  * hinic_rq_prepare_wqe - prepare wqe before insert to the queue
 851  * @rq: recv queue
 852  * @prod_idx: pi value
 853  * @rq_wqe: the wqe
 854  * @sge: sge for use by the wqe for recv buf address
 855  **/
 856 void hinic_rq_prepare_wqe(struct hinic_rq *rq, u16 prod_idx,
 857                           struct hinic_rq_wqe *rq_wqe, struct hinic_sge *sge)
 858 {
 859         struct hinic_rq_cqe_sect *cqe_sect = &rq_wqe->cqe_sect;
 860         struct hinic_rq_bufdesc *buf_desc = &rq_wqe->buf_desc;
 861         struct hinic_rq_cqe *cqe = rq->cqe[prod_idx];
 862         struct hinic_rq_ctrl *ctrl = &rq_wqe->ctrl;
 863         dma_addr_t cqe_dma = rq->cqe_dma[prod_idx];
 864
 865         ctrl->ctrl_info =
 866                 HINIC_RQ_CTRL_SET(SIZE_8BYTES(sizeof(*ctrl)), LEN) |
 867                 HINIC_RQ_CTRL_SET(SIZE_8BYTES(sizeof(*cqe_sect)),
 868                                   COMPLETE_LEN)                    |
 869                 HINIC_RQ_CTRL_SET(SIZE_8BYTES(sizeof(*buf_desc)),
 870                                   BUFDESC_SECT_LEN)                |
 871                 HINIC_RQ_CTRL_SET(RQ_COMPLETE_SGE, COMPLETE_FORMAT);
 872
 873         hinic_set_sge(&cqe_sect->sge, cqe_dma, sizeof(*cqe));
 874
 875         buf_desc->hi_addr = sge->hi_addr;
 876         buf_desc->lo_addr = sge->lo_addr;
 877 }
 878
 879 /**
 880  * hinic_rq_update - update pi of the rq
 881  * @rq: recv queue
 882  * @prod_idx: pi value
 883  **/
 884 void hinic_rq_update(struct hinic_rq *rq, u16 prod_idx)
 885 {
 886         *rq->pi_virt_addr = cpu_to_be16(RQ_MASKED_IDX(rq, prod_idx + 1));
 887 }