?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'for_linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost
[cris-mirror.git] / drivers / infiniband / hw / qedr / verbs.c
blob53f00dbf313f757941d32451ae23e62305f9cf53
1 /* QLogic qedr NIC Driver
2 * Copyright (c) 2015-2016 QLogic Corporation
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and /or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32 #include <linux/dma-mapping.h>
33 #include <linux/crc32.h>
34 #include <net/ip.h>
35 #include <net/ipv6.h>
36 #include <net/udp.h>
37 #include <linux/iommu.h>
38
39 #include <rdma/ib_verbs.h>
40 #include <rdma/ib_user_verbs.h>
41 #include <rdma/iw_cm.h>
42 #include <rdma/ib_umem.h>
43 #include <rdma/ib_addr.h>
44 #include <rdma/ib_cache.h>
45
46 #include <linux/qed/common_hsi.h>
47 #include "qedr_hsi_rdma.h"
48 #include <linux/qed/qed_if.h>
49 #include "qedr.h"
50 #include "verbs.h"
51 #include <rdma/qedr-abi.h>
52 #include "qedr_roce_cm.h"
53
54 #define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
55
56 static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src,
57 size_t len)
58 {
59 size_t min_len = min_t(size_t, len, udata->outlen);
60
61 return ib_copy_to_udata(udata, src, min_len);
62 }
63
64 int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
65 {
66 if (index > QEDR_ROCE_PKEY_TABLE_LEN)
67 return -EINVAL;
68
69 *pkey = QEDR_ROCE_PKEY_DEFAULT;
70 return 0;
71 }
72
73 int qedr_iw_query_gid(struct ib_device *ibdev, u8 port,
74 int index, union ib_gid *sgid)
75 {
76 struct qedr_dev *dev = get_qedr_dev(ibdev);
77
78 memset(sgid->raw, 0, sizeof(sgid->raw));
79 ether_addr_copy(sgid->raw, dev->ndev->dev_addr);
80
81 DP_DEBUG(dev, QEDR_MSG_INIT, "QUERY sgid[%d]=%llx:%llx\n", index,
82 sgid->global.interface_id, sgid->global.subnet_prefix);
83
84 return 0;
85 }
86
87 int qedr_query_gid(struct ib_device *ibdev, u8 port, int index,
88 union ib_gid *sgid)
89 {
90 struct qedr_dev *dev = get_qedr_dev(ibdev);
91 int rc = 0;
92
93 if (!rdma_cap_roce_gid_table(ibdev, port))
94 return -ENODEV;
95
96 rc = ib_get_cached_gid(ibdev, port, index, sgid, NULL);
97 if (rc == -EAGAIN) {
98 memcpy(sgid, &zgid, sizeof(*sgid));
99 return 0;
100 }
101
102 DP_DEBUG(dev, QEDR_MSG_INIT, "query gid: index=%d %llx:%llx\n", index,
103 sgid->global.interface_id, sgid->global.subnet_prefix);
104
105 return rc;
106 }
107
108 int qedr_add_gid(struct ib_device *device, u8 port_num,
109 unsigned int index, const union ib_gid *gid,
110 const struct ib_gid_attr *attr, void **context)
111 {
112 if (!rdma_cap_roce_gid_table(device, port_num))
113 return -EINVAL;
114
115 if (port_num > QEDR_MAX_PORT)
116 return -EINVAL;
117
118 if (!context)
119 return -EINVAL;
120
121 return 0;
122 }
123
124 int qedr_del_gid(struct ib_device *device, u8 port_num,
125 unsigned int index, void **context)
126 {
127 if (!rdma_cap_roce_gid_table(device, port_num))
128 return -EINVAL;
129
130 if (port_num > QEDR_MAX_PORT)
131 return -EINVAL;
132
133 if (!context)
134 return -EINVAL;
135
136 return 0;
137 }
138
139 int qedr_query_device(struct ib_device *ibdev,
140 struct ib_device_attr *attr, struct ib_udata *udata)
141 {
142 struct qedr_dev *dev = get_qedr_dev(ibdev);
143 struct qedr_device_attr *qattr = &dev->attr;
144
145 if (!dev->rdma_ctx) {
146 DP_ERR(dev,
147 "qedr_query_device called with invalid params rdma_ctx=%p\n",
148 dev->rdma_ctx);
149 return -EINVAL;
150 }
151
152 memset(attr, 0, sizeof(*attr));
153
154 attr->fw_ver = qattr->fw_ver;
155 attr->sys_image_guid = qattr->sys_image_guid;
156 attr->max_mr_size = qattr->max_mr_size;
157 attr->page_size_cap = qattr->page_size_caps;
158 attr->vendor_id = qattr->vendor_id;
159 attr->vendor_part_id = qattr->vendor_part_id;
160 attr->hw_ver = qattr->hw_ver;
161 attr->max_qp = qattr->max_qp;
162 attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe);
163 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
164 IB_DEVICE_RC_RNR_NAK_GEN |
165 IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS;
166
167 attr->max_sge = qattr->max_sge;
168 attr->max_sge_rd = qattr->max_sge;
169 attr->max_cq = qattr->max_cq;
170 attr->max_cqe = qattr->max_cqe;
171 attr->max_mr = qattr->max_mr;
172 attr->max_mw = qattr->max_mw;
173 attr->max_pd = qattr->max_pd;
174 attr->atomic_cap = dev->atomic_cap;
175 attr->max_fmr = qattr->max_fmr;
176 attr->max_map_per_fmr = 16;
177 attr->max_qp_init_rd_atom =
178 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
179 attr->max_qp_rd_atom =
180 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
181 attr->max_qp_init_rd_atom);
182
183 attr->max_srq = qattr->max_srq;
184 attr->max_srq_sge = qattr->max_srq_sge;
185 attr->max_srq_wr = qattr->max_srq_wr;
186
187 attr->local_ca_ack_delay = qattr->dev_ack_delay;
188 attr->max_fast_reg_page_list_len = qattr->max_mr / 8;
189 attr->max_pkeys = QEDR_ROCE_PKEY_MAX;
190 attr->max_ah = qattr->max_ah;
191
192 return 0;
193 }
194
195 #define QEDR_SPEED_SDR (1)
196 #define QEDR_SPEED_DDR (2)
197 #define QEDR_SPEED_QDR (4)
198 #define QEDR_SPEED_FDR10 (8)
199 #define QEDR_SPEED_FDR (16)
200 #define QEDR_SPEED_EDR (32)
201
202 static inline void get_link_speed_and_width(int speed, u8 *ib_speed,
203 u8 *ib_width)
204 {
205 switch (speed) {
206 case 1000:
207 *ib_speed = QEDR_SPEED_SDR;
208 *ib_width = IB_WIDTH_1X;
209 break;
210 case 10000:
211 *ib_speed = QEDR_SPEED_QDR;
212 *ib_width = IB_WIDTH_1X;
213 break;
214
215 case 20000:
216 *ib_speed = QEDR_SPEED_DDR;
217 *ib_width = IB_WIDTH_4X;
218 break;
219
220 case 25000:
221 *ib_speed = QEDR_SPEED_EDR;
222 *ib_width = IB_WIDTH_1X;
223 break;
224
225 case 40000:
226 *ib_speed = QEDR_SPEED_QDR;
227 *ib_width = IB_WIDTH_4X;
228 break;
229
230 case 50000:
231 *ib_speed = QEDR_SPEED_QDR;
232 *ib_width = IB_WIDTH_4X;
233 break;
234
235 case 100000:
236 *ib_speed = QEDR_SPEED_EDR;
237 *ib_width = IB_WIDTH_4X;
238 break;
239
240 default:
241 /* Unsupported */
242 *ib_speed = QEDR_SPEED_SDR;
243 *ib_width = IB_WIDTH_1X;
244 }
245 }
246
247 int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr)
248 {
249 struct qedr_dev *dev;
250 struct qed_rdma_port *rdma_port;
251
252 dev = get_qedr_dev(ibdev);
253 if (port > 1) {
254 DP_ERR(dev, "invalid_port=0x%x\n", port);
255 return -EINVAL;
256 }
257
258 if (!dev->rdma_ctx) {
259 DP_ERR(dev, "rdma_ctx is NULL\n");
260 return -EINVAL;
261 }
262
263 rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);
264
265 /* *attr being zeroed by the caller, avoid zeroing it here */
266 if (rdma_port->port_state == QED_RDMA_PORT_UP) {
267 attr->state = IB_PORT_ACTIVE;
268 attr->phys_state = 5;
269 } else {
270 attr->state = IB_PORT_DOWN;
271 attr->phys_state = 3;
272 }
273 attr->max_mtu = IB_MTU_4096;
274 attr->active_mtu = iboe_get_mtu(dev->ndev->mtu);
275 attr->lid = 0;
276 attr->lmc = 0;
277 attr->sm_lid = 0;
278 attr->sm_sl = 0;
279 attr->port_cap_flags = IB_PORT_IP_BASED_GIDS;
280 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
281 attr->gid_tbl_len = 1;
282 attr->pkey_tbl_len = 1;
283 } else {
284 attr->gid_tbl_len = QEDR_MAX_SGID;
285 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
286 }
287 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
288 attr->qkey_viol_cntr = 0;
289 get_link_speed_and_width(rdma_port->link_speed,
290 &attr->active_speed, &attr->active_width);
291 attr->max_msg_sz = rdma_port->max_msg_size;
292 attr->max_vl_num = 4;
293
294 return 0;
295 }
296
297 int qedr_modify_port(struct ib_device *ibdev, u8 port, int mask,
298 struct ib_port_modify *props)
299 {
300 struct qedr_dev *dev;
301
302 dev = get_qedr_dev(ibdev);
303 if (port > 1) {
304 DP_ERR(dev, "invalid_port=0x%x\n", port);
305 return -EINVAL;
306 }
307
308 return 0;
309 }
310
311 static int qedr_add_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
312 unsigned long len)
313 {
314 struct qedr_mm *mm;
315
316 mm = kzalloc(sizeof(*mm), GFP_KERNEL);
317 if (!mm)
318 return -ENOMEM;
319
320 mm->key.phy_addr = phy_addr;
321 /* This function might be called with a length which is not a multiple
322 * of PAGE_SIZE, while the mapping is PAGE_SIZE grained and the kernel
323 * forces this granularity by increasing the requested size if needed.
324 * When qedr_mmap is called, it will search the list with the updated
325 * length as a key. To prevent search failures, the length is rounded up
326 * in advance to PAGE_SIZE.
327 */
328 mm->key.len = roundup(len, PAGE_SIZE);
329 INIT_LIST_HEAD(&mm->entry);
330
331 mutex_lock(&uctx->mm_list_lock);
332 list_add(&mm->entry, &uctx->mm_head);
333 mutex_unlock(&uctx->mm_list_lock);
334
335 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
336 "added (addr=0x%llx,len=0x%lx) for ctx=%p\n",
337 (unsigned long long)mm->key.phy_addr,
338 (unsigned long)mm->key.len, uctx);
339
340 return 0;
341 }
342
343 static bool qedr_search_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
344 unsigned long len)
345 {
346 bool found = false;
347 struct qedr_mm *mm;
348
349 mutex_lock(&uctx->mm_list_lock);
350 list_for_each_entry(mm, &uctx->mm_head, entry) {
351 if (len != mm->key.len || phy_addr != mm->key.phy_addr)
352 continue;
353
354 found = true;
355 break;
356 }
357 mutex_unlock(&uctx->mm_list_lock);
358 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
359 "searched for (addr=0x%llx,len=0x%lx) for ctx=%p, result=%d\n",
360 mm->key.phy_addr, mm->key.len, uctx, found);
361
362 return found;
363 }
364
365 struct ib_ucontext *qedr_alloc_ucontext(struct ib_device *ibdev,
366 struct ib_udata *udata)
367 {
368 int rc;
369 struct qedr_ucontext *ctx;
370 struct qedr_alloc_ucontext_resp uresp;
371 struct qedr_dev *dev = get_qedr_dev(ibdev);
372 struct qed_rdma_add_user_out_params oparams;
373
374 if (!udata)
375 return ERR_PTR(-EFAULT);
376
377 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
378 if (!ctx)
379 return ERR_PTR(-ENOMEM);
380
381 rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);
382 if (rc) {
383 DP_ERR(dev,
384 "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n",
385 rc);
386 goto err;
387 }
388
389 ctx->dpi = oparams.dpi;
390 ctx->dpi_addr = oparams.dpi_addr;
391 ctx->dpi_phys_addr = oparams.dpi_phys_addr;
392 ctx->dpi_size = oparams.dpi_size;
393 INIT_LIST_HEAD(&ctx->mm_head);
394 mutex_init(&ctx->mm_list_lock);
395
396 memset(&uresp, 0, sizeof(uresp));
397
398 uresp.dpm_enabled = dev->user_dpm_enabled;
399 uresp.wids_enabled = 1;
400 uresp.wid_count = oparams.wid_count;
401 uresp.db_pa = ctx->dpi_phys_addr;
402 uresp.db_size = ctx->dpi_size;
403 uresp.max_send_wr = dev->attr.max_sqe;
404 uresp.max_recv_wr = dev->attr.max_rqe;
405 uresp.max_srq_wr = dev->attr.max_srq_wr;
406 uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE;
407 uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE;
408 uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
409 uresp.max_cqes = QEDR_MAX_CQES;
410
411 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
412 if (rc)
413 goto err;
414
415 ctx->dev = dev;
416
417 rc = qedr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size);
418 if (rc)
419 goto err;
420
421 DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n",
422 &ctx->ibucontext);
423 return &ctx->ibucontext;
424
425 err:
426 kfree(ctx);
427 return ERR_PTR(rc);
428 }
429
430 int qedr_dealloc_ucontext(struct ib_ucontext *ibctx)
431 {
432 struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx);
433 struct qedr_mm *mm, *tmp;
434 int status = 0;
435
436 DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",
437 uctx);
438 uctx->dev->ops->rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi);
439
440 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
441 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
442 "deleted (addr=0x%llx,len=0x%lx) for ctx=%p\n",
443 mm->key.phy_addr, mm->key.len, uctx);
444 list_del(&mm->entry);
445 kfree(mm);
446 }
447
448 kfree(uctx);
449 return status;
450 }
451
452 int qedr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
453 {
454 struct qedr_ucontext *ucontext = get_qedr_ucontext(context);
455 struct qedr_dev *dev = get_qedr_dev(context->device);
456 unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT;
457 u64 unmapped_db = dev->db_phys_addr;
458 unsigned long len = (vma->vm_end - vma->vm_start);
459 int rc = 0;
460 bool found;
461
462 DP_DEBUG(dev, QEDR_MSG_INIT,
463 "qedr_mmap called vm_page=0x%lx vm_pgoff=0x%lx unmapped_db=0x%llx db_size=%x, len=%lx\n",
464 vm_page, vma->vm_pgoff, unmapped_db, dev->db_size, len);
465 if (vma->vm_start & (PAGE_SIZE - 1)) {
466 DP_ERR(dev, "Vma_start not page aligned = %ld\n",
467 vma->vm_start);
468 return -EINVAL;
469 }
470
471 found = qedr_search_mmap(ucontext, vm_page, len);
472 if (!found) {
473 DP_ERR(dev, "Vma_pgoff not found in mapped array = %ld\n",
474 vma->vm_pgoff);
475 return -EINVAL;
476 }
477
478 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping doorbell bar\n");
479
480 if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db +
481 dev->db_size))) {
482 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping doorbell bar\n");
483 if (vma->vm_flags & VM_READ) {
484 DP_ERR(dev, "Trying to map doorbell bar for read\n");
485 return -EPERM;
486 }
487
488 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
489
490 rc = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
491 PAGE_SIZE, vma->vm_page_prot);
492 } else {
493 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping chains\n");
494 rc = remap_pfn_range(vma, vma->vm_start,
495 vma->vm_pgoff, len, vma->vm_page_prot);
496 }
497 DP_DEBUG(dev, QEDR_MSG_INIT, "qedr_mmap return code: %d\n", rc);
498 return rc;
499 }
500
501 struct ib_pd *qedr_alloc_pd(struct ib_device *ibdev,
502 struct ib_ucontext *context, struct ib_udata *udata)
503 {
504 struct qedr_dev *dev = get_qedr_dev(ibdev);
505 struct qedr_pd *pd;
506 u16 pd_id;
507 int rc;
508
509 DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
510 (udata && context) ? "User Lib" : "Kernel");
511
512 if (!dev->rdma_ctx) {
513 DP_ERR(dev, "invalid RDMA context\n");
514 return ERR_PTR(-EINVAL);
515 }
516
517 pd = kzalloc(sizeof(*pd), GFP_KERNEL);
518 if (!pd)
519 return ERR_PTR(-ENOMEM);
520
521 rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
522 if (rc)
523 goto err;
524
525 pd->pd_id = pd_id;
526
527 if (udata && context) {
528 struct qedr_alloc_pd_uresp uresp;
529
530 uresp.pd_id = pd_id;
531
532 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
533 if (rc) {
534 DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
535 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id);
536 goto err;
537 }
538
539 pd->uctx = get_qedr_ucontext(context);
540 pd->uctx->pd = pd;
541 }
542
543 return &pd->ibpd;
544
545 err:
546 kfree(pd);
547 return ERR_PTR(rc);
548 }
549
550 int qedr_dealloc_pd(struct ib_pd *ibpd)
551 {
552 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
553 struct qedr_pd *pd = get_qedr_pd(ibpd);
554
555 if (!pd) {
556 pr_err("Invalid PD received in dealloc_pd\n");
557 return -EINVAL;
558 }
559
560 DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
561 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
562
563 kfree(pd);
564
565 return 0;
566 }
567
568 static void qedr_free_pbl(struct qedr_dev *dev,
569 struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl)
570 {
571 struct pci_dev *pdev = dev->pdev;
572 int i;
573
574 for (i = 0; i < pbl_info->num_pbls; i++) {
575 if (!pbl[i].va)
576 continue;
577 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
578 pbl[i].va, pbl[i].pa);
579 }
580
581 kfree(pbl);
582 }
583
584 #define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
585 #define MAX_FW_PBL_PAGE_SIZE (64 * 1024)
586
587 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
588 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
589 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)
590
591 static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
592 struct qedr_pbl_info *pbl_info,
593 gfp_t flags)
594 {
595 struct pci_dev *pdev = dev->pdev;
596 struct qedr_pbl *pbl_table;
597 dma_addr_t *pbl_main_tbl;
598 dma_addr_t pa;
599 void *va;
600 int i;
601
602 pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);
603 if (!pbl_table)
604 return ERR_PTR(-ENOMEM);
605
606 for (i = 0; i < pbl_info->num_pbls; i++) {
607 va = dma_zalloc_coherent(&pdev->dev, pbl_info->pbl_size,
608 &pa, flags);
609 if (!va)
610 goto err;
611
612 pbl_table[i].va = va;
613 pbl_table[i].pa = pa;
614 }
615
616 /* Two-Layer PBLs, if we have more than one pbl we need to initialize
617 * the first one with physical pointers to all of the rest
618 */
619 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
620 for (i = 0; i < pbl_info->num_pbls - 1; i++)
621 pbl_main_tbl[i] = pbl_table[i + 1].pa;
622
623 return pbl_table;
624
625 err:
626 for (i--; i >= 0; i--)
627 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
628 pbl_table[i].va, pbl_table[i].pa);
629
630 qedr_free_pbl(dev, pbl_info, pbl_table);
631
632 return ERR_PTR(-ENOMEM);
633 }
634
635 static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
636 struct qedr_pbl_info *pbl_info,
637 u32 num_pbes, int two_layer_capable)
638 {
639 u32 pbl_capacity;
640 u32 pbl_size;
641 u32 num_pbls;
642
643 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
644 if (num_pbes > MAX_PBES_TWO_LAYER) {
645 DP_ERR(dev, "prepare pbl table: too many pages %d\n",
646 num_pbes);
647 return -EINVAL;
648 }
649
650 /* calculate required pbl page size */
651 pbl_size = MIN_FW_PBL_PAGE_SIZE;
652 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
653 NUM_PBES_ON_PAGE(pbl_size);
654
655 while (pbl_capacity < num_pbes) {
656 pbl_size *= 2;
657 pbl_capacity = pbl_size / sizeof(u64);
658 pbl_capacity = pbl_capacity * pbl_capacity;
659 }
660
661 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
662 num_pbls++; /* One for the layer0 ( points to the pbls) */
663 pbl_info->two_layered = true;
664 } else {
665 /* One layered PBL */
666 num_pbls = 1;
667 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
668 roundup_pow_of_two((num_pbes * sizeof(u64))));
669 pbl_info->two_layered = false;
670 }
671
672 pbl_info->num_pbls = num_pbls;
673 pbl_info->pbl_size = pbl_size;
674 pbl_info->num_pbes = num_pbes;
675
676 DP_DEBUG(dev, QEDR_MSG_MR,
677 "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
678 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
679
680 return 0;
681 }
682
683 static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
684 struct qedr_pbl *pbl,
685 struct qedr_pbl_info *pbl_info, u32 pg_shift)
686 {
687 int shift, pg_cnt, pages, pbe_cnt, total_num_pbes = 0;
688 u32 fw_pg_cnt, fw_pg_per_umem_pg;
689 struct qedr_pbl *pbl_tbl;
690 struct scatterlist *sg;
691 struct regpair *pbe;
692 u64 pg_addr;
693 int entry;
694
695 if (!pbl_info->num_pbes)
696 return;
697
698 /* If we have a two layered pbl, the first pbl points to the rest
699 * of the pbls and the first entry lays on the second pbl in the table
700 */
701 if (pbl_info->two_layered)
702 pbl_tbl = &pbl[1];
703 else
704 pbl_tbl = pbl;
705
706 pbe = (struct regpair *)pbl_tbl->va;
707 if (!pbe) {
708 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
709 return;
710 }
711
712 pbe_cnt = 0;
713
714 shift = umem->page_shift;
715
716 fw_pg_per_umem_pg = BIT(umem->page_shift - pg_shift);
717
718 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
719 pages = sg_dma_len(sg) >> shift;
720 pg_addr = sg_dma_address(sg);
721 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) {
722 for (fw_pg_cnt = 0; fw_pg_cnt < fw_pg_per_umem_pg;) {
723 pbe->lo = cpu_to_le32(pg_addr);
724 pbe->hi = cpu_to_le32(upper_32_bits(pg_addr));
725
726 pg_addr += BIT(pg_shift);
727 pbe_cnt++;
728 total_num_pbes++;
729 pbe++;
730
731 if (total_num_pbes == pbl_info->num_pbes)
732 return;
733
734 /* If the given pbl is full storing the pbes,
735 * move to next pbl.
736 */
737 if (pbe_cnt ==
738 (pbl_info->pbl_size / sizeof(u64))) {
739 pbl_tbl++;
740 pbe = (struct regpair *)pbl_tbl->va;
741 pbe_cnt = 0;
742 }
743
744 fw_pg_cnt++;
745 }
746 }
747 }
748 }
749
750 static int qedr_copy_cq_uresp(struct qedr_dev *dev,
751 struct qedr_cq *cq, struct ib_udata *udata)
752 {
753 struct qedr_create_cq_uresp uresp;
754 int rc;
755
756 memset(&uresp, 0, sizeof(uresp));
757
758 uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
759 uresp.icid = cq->icid;
760
761 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
762 if (rc)
763 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
764
765 return rc;
766 }
767
768 static void consume_cqe(struct qedr_cq *cq)
769 {
770 if (cq->latest_cqe == cq->toggle_cqe)
771 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
772
773 cq->latest_cqe = qed_chain_consume(&cq->pbl);
774 }
775
776 static inline int qedr_align_cq_entries(int entries)
777 {
778 u64 size, aligned_size;
779
780 /* We allocate an extra entry that we don't report to the FW. */
781 size = (entries + 1) * QEDR_CQE_SIZE;
782 aligned_size = ALIGN(size, PAGE_SIZE);
783
784 return aligned_size / QEDR_CQE_SIZE;
785 }
786
787 static inline int qedr_init_user_queue(struct ib_ucontext *ib_ctx,
788 struct qedr_dev *dev,
789 struct qedr_userq *q,
790 u64 buf_addr, size_t buf_len,
791 int access, int dmasync,
792 int alloc_and_init)
793 {
794 u32 fw_pages;
795 int rc;
796
797 q->buf_addr = buf_addr;
798 q->buf_len = buf_len;
799 q->umem = ib_umem_get(ib_ctx, q->buf_addr, q->buf_len, access, dmasync);
800 if (IS_ERR(q->umem)) {
801 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
802 PTR_ERR(q->umem));
803 return PTR_ERR(q->umem);
804 }
805
806 fw_pages = ib_umem_page_count(q->umem) <<
807 (q->umem->page_shift - FW_PAGE_SHIFT);
808
809 rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0);
810 if (rc)
811 goto err0;
812
813 if (alloc_and_init) {
814 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
815 if (IS_ERR(q->pbl_tbl)) {
816 rc = PTR_ERR(q->pbl_tbl);
817 goto err0;
818 }
819 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info,
820 FW_PAGE_SHIFT);
821 } else {
822 q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL);
823 if (!q->pbl_tbl) {
824 rc = -ENOMEM;
825 goto err0;
826 }
827 }
828
829 return 0;
830
831 err0:
832 ib_umem_release(q->umem);
833 q->umem = NULL;
834
835 return rc;
836 }
837
838 static inline void qedr_init_cq_params(struct qedr_cq *cq,
839 struct qedr_ucontext *ctx,
840 struct qedr_dev *dev, int vector,
841 int chain_entries, int page_cnt,
842 u64 pbl_ptr,
843 struct qed_rdma_create_cq_in_params
844 *params)
845 {
846 memset(params, 0, sizeof(*params));
847 params->cq_handle_hi = upper_32_bits((uintptr_t)cq);
848 params->cq_handle_lo = lower_32_bits((uintptr_t)cq);
849 params->cnq_id = vector;
850 params->cq_size = chain_entries - 1;
851 params->dpi = (ctx) ? ctx->dpi : dev->dpi;
852 params->pbl_num_pages = page_cnt;
853 params->pbl_ptr = pbl_ptr;
854 params->pbl_two_level = 0;
855 }
856
857 static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags)
858 {
859 /* Flush data before signalling doorbell */
860 wmb();
861 cq->db.data.agg_flags = flags;
862 cq->db.data.value = cpu_to_le32(cons);
863 writeq(cq->db.raw, cq->db_addr);
864
865 /* Make sure write would stick */
866 mmiowb();
867 }
868
869 int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
870 {
871 struct qedr_cq *cq = get_qedr_cq(ibcq);
872 unsigned long sflags;
873 struct qedr_dev *dev;
874
875 dev = get_qedr_dev(ibcq->device);
876
877 if (cq->destroyed) {
878 DP_ERR(dev,
879 "warning: arm was invoked after destroy for cq %p (icid=%d)\n",
880 cq, cq->icid);
881 return -EINVAL;
882 }
883
884
885 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
886 return 0;
887
888 spin_lock_irqsave(&cq->cq_lock, sflags);
889
890 cq->arm_flags = 0;
891
892 if (flags & IB_CQ_SOLICITED)
893 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
894
895 if (flags & IB_CQ_NEXT_COMP)
896 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
897
898 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
899
900 spin_unlock_irqrestore(&cq->cq_lock, sflags);
901
902 return 0;
903 }
904
905 struct ib_cq *qedr_create_cq(struct ib_device *ibdev,
906 const struct ib_cq_init_attr *attr,
907 struct ib_ucontext *ib_ctx, struct ib_udata *udata)
908 {
909 struct qedr_ucontext *ctx = get_qedr_ucontext(ib_ctx);
910 struct qed_rdma_destroy_cq_out_params destroy_oparams;
911 struct qed_rdma_destroy_cq_in_params destroy_iparams;
912 struct qedr_dev *dev = get_qedr_dev(ibdev);
913 struct qed_rdma_create_cq_in_params params;
914 struct qedr_create_cq_ureq ureq;
915 int vector = attr->comp_vector;
916 int entries = attr->cqe;
917 struct qedr_cq *cq;
918 int chain_entries;
919 int page_cnt;
920 u64 pbl_ptr;
921 u16 icid;
922 int rc;
923
924 DP_DEBUG(dev, QEDR_MSG_INIT,
925 "create_cq: called from %s. entries=%d, vector=%d\n",
926 udata ? "User Lib" : "Kernel", entries, vector);
927
928 if (entries > QEDR_MAX_CQES) {
929 DP_ERR(dev,
930 "create cq: the number of entries %d is too high. Must be equal or below %d.\n",
931 entries, QEDR_MAX_CQES);
932 return ERR_PTR(-EINVAL);
933 }
934
935 chain_entries = qedr_align_cq_entries(entries);
936 chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);
937
938 cq = kzalloc(sizeof(*cq), GFP_KERNEL);
939 if (!cq)
940 return ERR_PTR(-ENOMEM);
941
942 if (udata) {
943 memset(&ureq, 0, sizeof(ureq));
944 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
945 DP_ERR(dev,
946 "create cq: problem copying data from user space\n");
947 goto err0;
948 }
949
950 if (!ureq.len) {
951 DP_ERR(dev,
952 "create cq: cannot create a cq with 0 entries\n");
953 goto err0;
954 }
955
956 cq->cq_type = QEDR_CQ_TYPE_USER;
957
958 rc = qedr_init_user_queue(ib_ctx, dev, &cq->q, ureq.addr,
959 ureq.len, IB_ACCESS_LOCAL_WRITE,
960 1, 1);
961 if (rc)
962 goto err0;
963
964 pbl_ptr = cq->q.pbl_tbl->pa;
965 page_cnt = cq->q.pbl_info.num_pbes;
966
967 cq->ibcq.cqe = chain_entries;
968 } else {
969 cq->cq_type = QEDR_CQ_TYPE_KERNEL;
970
971 rc = dev->ops->common->chain_alloc(dev->cdev,
972 QED_CHAIN_USE_TO_CONSUME,
973 QED_CHAIN_MODE_PBL,
974 QED_CHAIN_CNT_TYPE_U32,
975 chain_entries,
976 sizeof(union rdma_cqe),
977 &cq->pbl, NULL);
978 if (rc)
979 goto err1;
980
981 page_cnt = qed_chain_get_page_cnt(&cq->pbl);
982 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl);
983 cq->ibcq.cqe = cq->pbl.capacity;
984 }
985
986 qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
987 pbl_ptr, &params);
988
989 rc = dev->ops->rdma_create_cq(dev->rdma_ctx, &params, &icid);
990 if (rc)
991 goto err2;
992
993 cq->icid = icid;
994 cq->sig = QEDR_CQ_MAGIC_NUMBER;
995 spin_lock_init(&cq->cq_lock);
996
997 if (ib_ctx) {
998 rc = qedr_copy_cq_uresp(dev, cq, udata);
999 if (rc)
1000 goto err3;
1001 } else {
1002 /* Generate doorbell address. */
1003 cq->db_addr = dev->db_addr +
1004 DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
1005 cq->db.data.icid = cq->icid;
1006 cq->db.data.params = DB_AGG_CMD_SET <<
1007 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
1008
1009 /* point to the very last element, passing it we will toggle */
1010 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl);
1011 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
1012 cq->latest_cqe = NULL;
1013 consume_cqe(cq);
1014 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
1015 }
1016
1017 DP_DEBUG(dev, QEDR_MSG_CQ,
1018 "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
1019 cq->icid, cq, params.cq_size);
1020
1021 return &cq->ibcq;
1022
1023 err3:
1024 destroy_iparams.icid = cq->icid;
1025 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
1026 &destroy_oparams);
1027 err2:
1028 if (udata)
1029 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1030 else
1031 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1032 err1:
1033 if (udata)
1034 ib_umem_release(cq->q.umem);
1035 err0:
1036 kfree(cq);
1037 return ERR_PTR(-EINVAL);
1038 }
1039
1040 int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
1041 {
1042 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1043 struct qedr_cq *cq = get_qedr_cq(ibcq);
1044
1045 DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq);
1046
1047 return 0;
1048 }
1049
1050 #define QEDR_DESTROY_CQ_MAX_ITERATIONS (10)
1051 #define QEDR_DESTROY_CQ_ITER_DURATION (10)
1052
1053 int qedr_destroy_cq(struct ib_cq *ibcq)
1054 {
1055 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1056 struct qed_rdma_destroy_cq_out_params oparams;
1057 struct qed_rdma_destroy_cq_in_params iparams;
1058 struct qedr_cq *cq = get_qedr_cq(ibcq);
1059 int iter;
1060 int rc;
1061
1062 DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid);
1063
1064 cq->destroyed = 1;
1065
1066 /* GSIs CQs are handled by driver, so they don't exist in the FW */
1067 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
1068 goto done;
1069
1070 iparams.icid = cq->icid;
1071 rc = dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
1072 if (rc)
1073 return rc;
1074
1075 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1076
1077 if (ibcq->uobject && ibcq->uobject->context) {
1078 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1079 ib_umem_release(cq->q.umem);
1080 }
1081
1082 /* We don't want the IRQ handler to handle a non-existing CQ so we
1083 * wait until all CNQ interrupts, if any, are received. This will always
1084 * happen and will always happen very fast. If not, then a serious error
1085 * has occured. That is why we can use a long delay.
1086 * We spin for a short time so we don’t lose time on context switching
1087 * in case all the completions are handled in that span. Otherwise
1088 * we sleep for a while and check again. Since the CNQ may be
1089 * associated with (only) the current CPU we use msleep to allow the
1090 * current CPU to be freed.
1091 * The CNQ notification is increased in qedr_irq_handler().
1092 */
1093 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1094 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1095 udelay(QEDR_DESTROY_CQ_ITER_DURATION);
1096 iter--;
1097 }
1098
1099 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1100 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1101 msleep(QEDR_DESTROY_CQ_ITER_DURATION);
1102 iter--;
1103 }
1104
1105 if (oparams.num_cq_notif != cq->cnq_notif)
1106 goto err;
1107
1108 /* Note that we don't need to have explicit code to wait for the
1109 * completion of the event handler because it is invoked from the EQ.
1110 * Since the destroy CQ ramrod has also been received on the EQ we can
1111 * be certain that there's no event handler in process.
1112 */
1113 done:
1114 cq->sig = ~cq->sig;
1115
1116 kfree(cq);
1117
1118 return 0;
1119
1120 err:
1121 DP_ERR(dev,
1122 "CQ %p (icid=%d) not freed, expecting %d ints but got %d ints\n",
1123 cq, cq->icid, oparams.num_cq_notif, cq->cnq_notif);
1124
1125 return -EINVAL;
1126 }
1127
1128 static inline int get_gid_info_from_table(struct ib_qp *ibqp,
1129 struct ib_qp_attr *attr,
1130 int attr_mask,
1131 struct qed_rdma_modify_qp_in_params
1132 *qp_params)
1133 {
1134 enum rdma_network_type nw_type;
1135 struct ib_gid_attr gid_attr;
1136 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
1137 union ib_gid gid;
1138 u32 ipv4_addr;
1139 int rc = 0;
1140 int i;
1141
1142 rc = ib_get_cached_gid(ibqp->device,
1143 rdma_ah_get_port_num(&attr->ah_attr),
1144 grh->sgid_index, &gid, &gid_attr);
1145 if (rc)
1146 return rc;
1147
1148 if (!memcmp(&gid, &zgid, sizeof(gid)))
1149 return -ENOENT;
1150
1151 if (gid_attr.ndev) {
1152 qp_params->vlan_id = rdma_vlan_dev_vlan_id(gid_attr.ndev);
1153
1154 dev_put(gid_attr.ndev);
1155 nw_type = ib_gid_to_network_type(gid_attr.gid_type, &gid);
1156 switch (nw_type) {
1157 case RDMA_NETWORK_IPV6:
1158 memcpy(&qp_params->sgid.bytes[0], &gid.raw[0],
1159 sizeof(qp_params->sgid));
1160 memcpy(&qp_params->dgid.bytes[0],
1161 &grh->dgid,
1162 sizeof(qp_params->dgid));
1163 qp_params->roce_mode = ROCE_V2_IPV6;
1164 SET_FIELD(qp_params->modify_flags,
1165 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1166 break;
1167 case RDMA_NETWORK_IB:
1168 memcpy(&qp_params->sgid.bytes[0], &gid.raw[0],
1169 sizeof(qp_params->sgid));
1170 memcpy(&qp_params->dgid.bytes[0],
1171 &grh->dgid,
1172 sizeof(qp_params->dgid));
1173 qp_params->roce_mode = ROCE_V1;
1174 break;
1175 case RDMA_NETWORK_IPV4:
1176 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid));
1177 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid));
1178 ipv4_addr = qedr_get_ipv4_from_gid(gid.raw);
1179 qp_params->sgid.ipv4_addr = ipv4_addr;
1180 ipv4_addr =
1181 qedr_get_ipv4_from_gid(grh->dgid.raw);
1182 qp_params->dgid.ipv4_addr = ipv4_addr;
1183 SET_FIELD(qp_params->modify_flags,
1184 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1185 qp_params->roce_mode = ROCE_V2_IPV4;
1186 break;
1187 }
1188 }
1189
1190 for (i = 0; i < 4; i++) {
1191 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
1192 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
1193 }
1194
1195 if (qp_params->vlan_id >= VLAN_CFI_MASK)
1196 qp_params->vlan_id = 0;
1197
1198 return 0;
1199 }
1200
1201 static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
1202 struct ib_qp_init_attr *attrs)
1203 {
1204 struct qedr_device_attr *qattr = &dev->attr;
1205
1206 /* QP0... attrs->qp_type == IB_QPT_GSI */
1207 if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) {
1208 DP_DEBUG(dev, QEDR_MSG_QP,
1209 "create qp: unsupported qp type=0x%x requested\n",
1210 attrs->qp_type);
1211 return -EINVAL;
1212 }
1213
1214 if (attrs->cap.max_send_wr > qattr->max_sqe) {
1215 DP_ERR(dev,
1216 "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n",
1217 attrs->cap.max_send_wr, qattr->max_sqe);
1218 return -EINVAL;
1219 }
1220
1221 if (attrs->cap.max_inline_data > qattr->max_inline) {
1222 DP_ERR(dev,
1223 "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n",
1224 attrs->cap.max_inline_data, qattr->max_inline);
1225 return -EINVAL;
1226 }
1227
1228 if (attrs->cap.max_send_sge > qattr->max_sge) {
1229 DP_ERR(dev,
1230 "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n",
1231 attrs->cap.max_send_sge, qattr->max_sge);
1232 return -EINVAL;
1233 }
1234
1235 if (attrs->cap.max_recv_sge > qattr->max_sge) {
1236 DP_ERR(dev,
1237 "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n",
1238 attrs->cap.max_recv_sge, qattr->max_sge);
1239 return -EINVAL;
1240 }
1241
1242 /* Unprivileged user space cannot create special QP */
1243 if (ibpd->uobject && attrs->qp_type == IB_QPT_GSI) {
1244 DP_ERR(dev,
1245 "create qp: userspace can't create special QPs of type=0x%x\n",
1246 attrs->qp_type);
1247 return -EINVAL;
1248 }
1249
1250 return 0;
1251 }
1252
1253 static void qedr_copy_rq_uresp(struct qedr_dev *dev,
1254 struct qedr_create_qp_uresp *uresp,
1255 struct qedr_qp *qp)
1256 {
1257 /* iWARP requires two doorbells per RQ. */
1258 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1259 uresp->rq_db_offset =
1260 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1261 uresp->rq_db2_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1262 } else {
1263 uresp->rq_db_offset =
1264 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1265 }
1266
1267 uresp->rq_icid = qp->icid;
1268 }
1269
1270 static void qedr_copy_sq_uresp(struct qedr_dev *dev,
1271 struct qedr_create_qp_uresp *uresp,
1272 struct qedr_qp *qp)
1273 {
1274 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1275
1276 /* iWARP uses the same cid for rq and sq */
1277 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1278 uresp->sq_icid = qp->icid;
1279 else
1280 uresp->sq_icid = qp->icid + 1;
1281 }
1282
1283 static int qedr_copy_qp_uresp(struct qedr_dev *dev,
1284 struct qedr_qp *qp, struct ib_udata *udata)
1285 {
1286 struct qedr_create_qp_uresp uresp;
1287 int rc;
1288
1289 memset(&uresp, 0, sizeof(uresp));
1290 qedr_copy_sq_uresp(dev, &uresp, qp);
1291 qedr_copy_rq_uresp(dev, &uresp, qp);
1292
1293 uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
1294 uresp.qp_id = qp->qp_id;
1295
1296 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1297 if (rc)
1298 DP_ERR(dev,
1299 "create qp: failed a copy to user space with qp icid=0x%x.\n",
1300 qp->icid);
1301
1302 return rc;
1303 }
1304
1305 static void qedr_set_common_qp_params(struct qedr_dev *dev,
1306 struct qedr_qp *qp,
1307 struct qedr_pd *pd,
1308 struct ib_qp_init_attr *attrs)
1309 {
1310 spin_lock_init(&qp->q_lock);
1311 atomic_set(&qp->refcnt, 1);
1312 qp->pd = pd;
1313 qp->qp_type = attrs->qp_type;
1314 qp->max_inline_data = attrs->cap.max_inline_data;
1315 qp->sq.max_sges = attrs->cap.max_send_sge;
1316 qp->state = QED_ROCE_QP_STATE_RESET;
1317 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
1318 qp->sq_cq = get_qedr_cq(attrs->send_cq);
1319 qp->rq_cq = get_qedr_cq(attrs->recv_cq);
1320 qp->dev = dev;
1321 qp->rq.max_sges = attrs->cap.max_recv_sge;
1322
1323 DP_DEBUG(dev, QEDR_MSG_QP,
1324 "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
1325 qp->rq.max_sges, qp->rq_cq->icid);
1326 DP_DEBUG(dev, QEDR_MSG_QP,
1327 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n",
1328 pd->pd_id, qp->qp_type, qp->max_inline_data,
1329 qp->state, qp->signaled, (attrs->srq) ? 1 : 0);
1330 DP_DEBUG(dev, QEDR_MSG_QP,
1331 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1332 qp->sq.max_sges, qp->sq_cq->icid);
1333 }
1334
1335 static void qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1336 {
1337 qp->sq.db = dev->db_addr +
1338 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1339 qp->sq.db_data.data.icid = qp->icid + 1;
1340 qp->rq.db = dev->db_addr +
1341 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1342 qp->rq.db_data.data.icid = qp->icid;
1343 }
1344
1345 static inline void
1346 qedr_init_common_qp_in_params(struct qedr_dev *dev,
1347 struct qedr_pd *pd,
1348 struct qedr_qp *qp,
1349 struct ib_qp_init_attr *attrs,
1350 bool fmr_and_reserved_lkey,
1351 struct qed_rdma_create_qp_in_params *params)
1352 {
1353 /* QP handle to be written in an async event */
1354 params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp);
1355 params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp);
1356
1357 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
1358 params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;
1359 params->pd = pd->pd_id;
1360 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
1361 params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid;
1362 params->stats_queue = 0;
1363 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1364 params->srq_id = 0;
1365 params->use_srq = false;
1366 }
1367
1368 static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
1369 {
1370 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. "
1371 "qp=%p. "
1372 "sq_addr=0x%llx, "
1373 "sq_len=%zd, "
1374 "rq_addr=0x%llx, "
1375 "rq_len=%zd"
1376 "\n",
1377 qp,
1378 qp->usq.buf_addr,
1379 qp->usq.buf_len, qp->urq.buf_addr, qp->urq.buf_len);
1380 }
1381
1382 static int qedr_idr_add(struct qedr_dev *dev, void *ptr, u32 id)
1383 {
1384 int rc;
1385
1386 if (!rdma_protocol_iwarp(&dev->ibdev, 1))
1387 return 0;
1388
1389 idr_preload(GFP_KERNEL);
1390 spin_lock_irq(&dev->idr_lock);
1391
1392 rc = idr_alloc(&dev->qpidr, ptr, id, id + 1, GFP_ATOMIC);
1393
1394 spin_unlock_irq(&dev->idr_lock);
1395 idr_preload_end();
1396
1397 return rc < 0 ? rc : 0;
1398 }
1399
1400 static void qedr_idr_remove(struct qedr_dev *dev, u32 id)
1401 {
1402 if (!rdma_protocol_iwarp(&dev->ibdev, 1))
1403 return;
1404
1405 spin_lock_irq(&dev->idr_lock);
1406 idr_remove(&dev->qpidr, id);
1407 spin_unlock_irq(&dev->idr_lock);
1408 }
1409
1410 static inline void
1411 qedr_iwarp_populate_user_qp(struct qedr_dev *dev,
1412 struct qedr_qp *qp,
1413 struct qed_rdma_create_qp_out_params *out_params)
1414 {
1415 qp->usq.pbl_tbl->va = out_params->sq_pbl_virt;
1416 qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys;
1417
1418 qedr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl,
1419 &qp->usq.pbl_info, FW_PAGE_SHIFT);
1420
1421 qp->urq.pbl_tbl->va = out_params->rq_pbl_virt;
1422 qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys;
1423
1424 qedr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl,
1425 &qp->urq.pbl_info, FW_PAGE_SHIFT);
1426 }
1427
1428 static void qedr_cleanup_user(struct qedr_dev *dev, struct qedr_qp *qp)
1429 {
1430 if (qp->usq.umem)
1431 ib_umem_release(qp->usq.umem);
1432 qp->usq.umem = NULL;
1433
1434 if (qp->urq.umem)
1435 ib_umem_release(qp->urq.umem);
1436 qp->urq.umem = NULL;
1437 }
1438
1439 static int qedr_create_user_qp(struct qedr_dev *dev,
1440 struct qedr_qp *qp,
1441 struct ib_pd *ibpd,
1442 struct ib_udata *udata,
1443 struct ib_qp_init_attr *attrs)
1444 {
1445 struct qed_rdma_create_qp_in_params in_params;
1446 struct qed_rdma_create_qp_out_params out_params;
1447 struct qedr_pd *pd = get_qedr_pd(ibpd);
1448 struct ib_ucontext *ib_ctx = NULL;
1449 struct qedr_create_qp_ureq ureq;
1450 int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1);
1451 int rc = -EINVAL;
1452
1453 ib_ctx = ibpd->uobject->context;
1454
1455 memset(&ureq, 0, sizeof(ureq));
1456 rc = ib_copy_from_udata(&ureq, udata, sizeof(ureq));
1457 if (rc) {
1458 DP_ERR(dev, "Problem copying data from user space\n");
1459 return rc;
1460 }
1461
1462 /* SQ - read access only (0), dma sync not required (0) */
1463 rc = qedr_init_user_queue(ib_ctx, dev, &qp->usq, ureq.sq_addr,
1464 ureq.sq_len, 0, 0, alloc_and_init);
1465 if (rc)
1466 return rc;
1467
1468 /* RQ - read access only (0), dma sync not required (0) */
1469 rc = qedr_init_user_queue(ib_ctx, dev, &qp->urq, ureq.rq_addr,
1470 ureq.rq_len, 0, 0, alloc_and_init);
1471 if (rc)
1472 return rc;
1473
1474 memset(&in_params, 0, sizeof(in_params));
1475 qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
1476 in_params.qp_handle_lo = ureq.qp_handle_lo;
1477 in_params.qp_handle_hi = ureq.qp_handle_hi;
1478 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
1479 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
1480 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
1481 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1482
1483 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1484 &in_params, &out_params);
1485
1486 if (!qp->qed_qp) {
1487 rc = -ENOMEM;
1488 goto err1;
1489 }
1490
1491 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1492 qedr_iwarp_populate_user_qp(dev, qp, &out_params);
1493
1494 qp->qp_id = out_params.qp_id;
1495 qp->icid = out_params.icid;
1496
1497 rc = qedr_copy_qp_uresp(dev, qp, udata);
1498 if (rc)
1499 goto err;
1500
1501 qedr_qp_user_print(dev, qp);
1502
1503 return 0;
1504 err:
1505 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1506 if (rc)
1507 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1508
1509 err1:
1510 qedr_cleanup_user(dev, qp);
1511 return rc;
1512 }
1513
1514 static void qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1515 {
1516 qp->sq.db = dev->db_addr +
1517 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1518 qp->sq.db_data.data.icid = qp->icid;
1519
1520 qp->rq.db = dev->db_addr +
1521 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1522 qp->rq.db_data.data.icid = qp->icid;
1523 qp->rq.iwarp_db2 = dev->db_addr +
1524 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1525 qp->rq.iwarp_db2_data.data.icid = qp->icid;
1526 qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD;
1527 }
1528
1529 static int
1530 qedr_roce_create_kernel_qp(struct qedr_dev *dev,
1531 struct qedr_qp *qp,
1532 struct qed_rdma_create_qp_in_params *in_params,
1533 u32 n_sq_elems, u32 n_rq_elems)
1534 {
1535 struct qed_rdma_create_qp_out_params out_params;
1536 int rc;
1537
1538 rc = dev->ops->common->chain_alloc(dev->cdev,
1539 QED_CHAIN_USE_TO_PRODUCE,
1540 QED_CHAIN_MODE_PBL,
1541 QED_CHAIN_CNT_TYPE_U32,
1542 n_sq_elems,
1543 QEDR_SQE_ELEMENT_SIZE,
1544 &qp->sq.pbl, NULL);
1545
1546 if (rc)
1547 return rc;
1548
1549 in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl);
1550 in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl);
1551
1552 rc = dev->ops->common->chain_alloc(dev->cdev,
1553 QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1554 QED_CHAIN_MODE_PBL,
1555 QED_CHAIN_CNT_TYPE_U32,
1556 n_rq_elems,
1557 QEDR_RQE_ELEMENT_SIZE,
1558 &qp->rq.pbl, NULL);
1559 if (rc)
1560 return rc;
1561
1562 in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl);
1563 in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl);
1564
1565 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1566 in_params, &out_params);
1567
1568 if (!qp->qed_qp)
1569 return -EINVAL;
1570
1571 qp->qp_id = out_params.qp_id;
1572 qp->icid = out_params.icid;
1573
1574 qedr_set_roce_db_info(dev, qp);
1575 return rc;
1576 }
1577
1578 static int
1579 qedr_iwarp_create_kernel_qp(struct qedr_dev *dev,
1580 struct qedr_qp *qp,
1581 struct qed_rdma_create_qp_in_params *in_params,
1582 u32 n_sq_elems, u32 n_rq_elems)
1583 {
1584 struct qed_rdma_create_qp_out_params out_params;
1585 struct qed_chain_ext_pbl ext_pbl;
1586 int rc;
1587
1588 in_params->sq_num_pages = QED_CHAIN_PAGE_CNT(n_sq_elems,
1589 QEDR_SQE_ELEMENT_SIZE,
1590 QED_CHAIN_MODE_PBL);
1591 in_params->rq_num_pages = QED_CHAIN_PAGE_CNT(n_rq_elems,
1592 QEDR_RQE_ELEMENT_SIZE,
1593 QED_CHAIN_MODE_PBL);
1594
1595 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1596 in_params, &out_params);
1597
1598 if (!qp->qed_qp)
1599 return -EINVAL;
1600
1601 /* Now we allocate the chain */
1602 ext_pbl.p_pbl_virt = out_params.sq_pbl_virt;
1603 ext_pbl.p_pbl_phys = out_params.sq_pbl_phys;
1604
1605 rc = dev->ops->common->chain_alloc(dev->cdev,
1606 QED_CHAIN_USE_TO_PRODUCE,
1607 QED_CHAIN_MODE_PBL,
1608 QED_CHAIN_CNT_TYPE_U32,
1609 n_sq_elems,
1610 QEDR_SQE_ELEMENT_SIZE,
1611 &qp->sq.pbl, &ext_pbl);
1612
1613 if (rc)
1614 goto err;
1615
1616 ext_pbl.p_pbl_virt = out_params.rq_pbl_virt;
1617 ext_pbl.p_pbl_phys = out_params.rq_pbl_phys;
1618
1619 rc = dev->ops->common->chain_alloc(dev->cdev,
1620 QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1621 QED_CHAIN_MODE_PBL,
1622 QED_CHAIN_CNT_TYPE_U32,
1623 n_rq_elems,
1624 QEDR_RQE_ELEMENT_SIZE,
1625 &qp->rq.pbl, &ext_pbl);
1626
1627 if (rc)
1628 goto err;
1629
1630 qp->qp_id = out_params.qp_id;
1631 qp->icid = out_params.icid;
1632
1633 qedr_set_iwarp_db_info(dev, qp);
1634 return rc;
1635
1636 err:
1637 dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1638
1639 return rc;
1640 }
1641
1642 static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp)
1643 {
1644 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
1645 kfree(qp->wqe_wr_id);
1646
1647 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
1648 kfree(qp->rqe_wr_id);
1649 }
1650
1651 static int qedr_create_kernel_qp(struct qedr_dev *dev,
1652 struct qedr_qp *qp,
1653 struct ib_pd *ibpd,
1654 struct ib_qp_init_attr *attrs)
1655 {
1656 struct qed_rdma_create_qp_in_params in_params;
1657 struct qedr_pd *pd = get_qedr_pd(ibpd);
1658 int rc = -EINVAL;
1659 u32 n_rq_elems;
1660 u32 n_sq_elems;
1661 u32 n_sq_entries;
1662
1663 memset(&in_params, 0, sizeof(in_params));
1664
1665 /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in
1666 * the ring. The ring should allow at least a single WR, even if the
1667 * user requested none, due to allocation issues.
1668 * We should add an extra WR since the prod and cons indices of
1669 * wqe_wr_id are managed in such a way that the WQ is considered full
1670 * when (prod+1)%max_wr==cons. We currently don't do that because we
1671 * double the number of entries due an iSER issue that pushes far more
1672 * WRs than indicated. If we decline its ib_post_send() then we get
1673 * error prints in the dmesg we'd like to avoid.
1674 */
1675 qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
1676 dev->attr.max_sqe);
1677
1678 qp->wqe_wr_id = kzalloc(qp->sq.max_wr * sizeof(*qp->wqe_wr_id),
1679 GFP_KERNEL);
1680 if (!qp->wqe_wr_id) {
1681 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n");
1682 return -ENOMEM;
1683 }
1684
1685 /* QP handle to be written in CQE */
1686 in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp);
1687 in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp);
1688
1689 /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in
1690 * the ring. There ring should allow at least a single WR, even if the
1691 * user requested none, due to allocation issues.
1692 */
1693 qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1);
1694
1695 /* Allocate driver internal RQ array */
1696 qp->rqe_wr_id = kzalloc(qp->rq.max_wr * sizeof(*qp->rqe_wr_id),
1697 GFP_KERNEL);
1698 if (!qp->rqe_wr_id) {
1699 DP_ERR(dev,
1700 "create qp: failed RQ shadow memory allocation\n");
1701 kfree(qp->wqe_wr_id);
1702 return -ENOMEM;
1703 }
1704
1705 qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
1706
1707 n_sq_entries = attrs->cap.max_send_wr;
1708 n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);
1709 n_sq_entries = max_t(u32, n_sq_entries, 1);
1710 n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE;
1711
1712 n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE;
1713
1714 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1715 rc = qedr_iwarp_create_kernel_qp(dev, qp, &in_params,
1716 n_sq_elems, n_rq_elems);
1717 else
1718 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params,
1719 n_sq_elems, n_rq_elems);
1720 if (rc)
1721 qedr_cleanup_kernel(dev, qp);
1722
1723 return rc;
1724 }
1725
1726 struct ib_qp *qedr_create_qp(struct ib_pd *ibpd,
1727 struct ib_qp_init_attr *attrs,
1728 struct ib_udata *udata)
1729 {
1730 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
1731 struct qedr_pd *pd = get_qedr_pd(ibpd);
1732 struct qedr_qp *qp;
1733 struct ib_qp *ibqp;
1734 int rc = 0;
1735
1736 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
1737 udata ? "user library" : "kernel", pd);
1738
1739 rc = qedr_check_qp_attrs(ibpd, dev, attrs);
1740 if (rc)
1741 return ERR_PTR(rc);
1742
1743 if (attrs->srq)
1744 return ERR_PTR(-EINVAL);
1745
1746 DP_DEBUG(dev, QEDR_MSG_QP,
1747 "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
1748 udata ? "user library" : "kernel", attrs->event_handler, pd,
1749 get_qedr_cq(attrs->send_cq),
1750 get_qedr_cq(attrs->send_cq)->icid,
1751 get_qedr_cq(attrs->recv_cq),
1752 get_qedr_cq(attrs->recv_cq)->icid);
1753
1754 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
1755 if (!qp) {
1756 DP_ERR(dev, "create qp: failed allocating memory\n");
1757 return ERR_PTR(-ENOMEM);
1758 }
1759
1760 qedr_set_common_qp_params(dev, qp, pd, attrs);
1761
1762 if (attrs->qp_type == IB_QPT_GSI) {
1763 ibqp = qedr_create_gsi_qp(dev, attrs, qp);
1764 if (IS_ERR(ibqp))
1765 kfree(qp);
1766 return ibqp;
1767 }
1768
1769 if (udata)
1770 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs);
1771 else
1772 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs);
1773
1774 if (rc)
1775 goto err;
1776
1777 qp->ibqp.qp_num = qp->qp_id;
1778
1779 rc = qedr_idr_add(dev, qp, qp->qp_id);
1780 if (rc)
1781 goto err;
1782
1783 return &qp->ibqp;
1784
1785 err:
1786 kfree(qp);
1787
1788 return ERR_PTR(-EFAULT);
1789 }
1790
1791 static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)
1792 {
1793 switch (qp_state) {
1794 case QED_ROCE_QP_STATE_RESET:
1795 return IB_QPS_RESET;
1796 case QED_ROCE_QP_STATE_INIT:
1797 return IB_QPS_INIT;
1798 case QED_ROCE_QP_STATE_RTR:
1799 return IB_QPS_RTR;
1800 case QED_ROCE_QP_STATE_RTS:
1801 return IB_QPS_RTS;
1802 case QED_ROCE_QP_STATE_SQD:
1803 return IB_QPS_SQD;
1804 case QED_ROCE_QP_STATE_ERR:
1805 return IB_QPS_ERR;
1806 case QED_ROCE_QP_STATE_SQE:
1807 return IB_QPS_SQE;
1808 }
1809 return IB_QPS_ERR;
1810 }
1811
1812 static enum qed_roce_qp_state qedr_get_state_from_ibqp(
1813 enum ib_qp_state qp_state)
1814 {
1815 switch (qp_state) {
1816 case IB_QPS_RESET:
1817 return QED_ROCE_QP_STATE_RESET;
1818 case IB_QPS_INIT:
1819 return QED_ROCE_QP_STATE_INIT;
1820 case IB_QPS_RTR:
1821 return QED_ROCE_QP_STATE_RTR;
1822 case IB_QPS_RTS:
1823 return QED_ROCE_QP_STATE_RTS;
1824 case IB_QPS_SQD:
1825 return QED_ROCE_QP_STATE_SQD;
1826 case IB_QPS_ERR:
1827 return QED_ROCE_QP_STATE_ERR;
1828 default:
1829 return QED_ROCE_QP_STATE_ERR;
1830 }
1831 }
1832
1833 static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
1834 {
1835 qed_chain_reset(&qph->pbl);
1836 qph->prod = 0;
1837 qph->cons = 0;
1838 qph->wqe_cons = 0;
1839 qph->db_data.data.value = cpu_to_le16(0);
1840 }
1841
1842 static int qedr_update_qp_state(struct qedr_dev *dev,
1843 struct qedr_qp *qp,
1844 enum qed_roce_qp_state new_state)
1845 {
1846 int status = 0;
1847
1848 if (new_state == qp->state)
1849 return 0;
1850
1851 switch (qp->state) {
1852 case QED_ROCE_QP_STATE_RESET:
1853 switch (new_state) {
1854 case QED_ROCE_QP_STATE_INIT:
1855 qp->prev_wqe_size = 0;
1856 qedr_reset_qp_hwq_info(&qp->sq);
1857 qedr_reset_qp_hwq_info(&qp->rq);
1858 break;
1859 default:
1860 status = -EINVAL;
1861 break;
1862 };
1863 break;
1864 case QED_ROCE_QP_STATE_INIT:
1865 switch (new_state) {
1866 case QED_ROCE_QP_STATE_RTR:
1867 /* Update doorbell (in case post_recv was
1868 * done before move to RTR)
1869 */
1870
1871 if (rdma_protocol_roce(&dev->ibdev, 1)) {
1872 wmb();
1873 writel(qp->rq.db_data.raw, qp->rq.db);
1874 /* Make sure write takes effect */
1875 mmiowb();
1876 }
1877 break;
1878 case QED_ROCE_QP_STATE_ERR:
1879 break;
1880 default:
1881 /* Invalid state change. */
1882 status = -EINVAL;
1883 break;
1884 };
1885 break;
1886 case QED_ROCE_QP_STATE_RTR:
1887 /* RTR->XXX */
1888 switch (new_state) {
1889 case QED_ROCE_QP_STATE_RTS:
1890 break;
1891 case QED_ROCE_QP_STATE_ERR:
1892 break;
1893 default:
1894 /* Invalid state change. */
1895 status = -EINVAL;
1896 break;
1897 };
1898 break;
1899 case QED_ROCE_QP_STATE_RTS:
1900 /* RTS->XXX */
1901 switch (new_state) {
1902 case QED_ROCE_QP_STATE_SQD:
1903 break;
1904 case QED_ROCE_QP_STATE_ERR:
1905 break;
1906 default:
1907 /* Invalid state change. */
1908 status = -EINVAL;
1909 break;
1910 };
1911 break;
1912 case QED_ROCE_QP_STATE_SQD:
1913 /* SQD->XXX */
1914 switch (new_state) {
1915 case QED_ROCE_QP_STATE_RTS:
1916 case QED_ROCE_QP_STATE_ERR:
1917 break;
1918 default:
1919 /* Invalid state change. */
1920 status = -EINVAL;
1921 break;
1922 };
1923 break;
1924 case QED_ROCE_QP_STATE_ERR:
1925 /* ERR->XXX */
1926 switch (new_state) {
1927 case QED_ROCE_QP_STATE_RESET:
1928 if ((qp->rq.prod != qp->rq.cons) ||
1929 (qp->sq.prod != qp->sq.cons)) {
1930 DP_NOTICE(dev,
1931 "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n",
1932 qp->rq.prod, qp->rq.cons, qp->sq.prod,
1933 qp->sq.cons);
1934 status = -EINVAL;
1935 }
1936 break;
1937 default:
1938 status = -EINVAL;
1939 break;
1940 };
1941 break;
1942 default:
1943 status = -EINVAL;
1944 break;
1945 };
1946
1947 return status;
1948 }
1949
1950 int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1951 int attr_mask, struct ib_udata *udata)
1952 {
1953 struct qedr_qp *qp = get_qedr_qp(ibqp);
1954 struct qed_rdma_modify_qp_in_params qp_params = { 0 };
1955 struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
1956 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
1957 enum ib_qp_state old_qp_state, new_qp_state;
1958 int rc = 0;
1959
1960 DP_DEBUG(dev, QEDR_MSG_QP,
1961 "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
1962 attr->qp_state);
1963
1964 old_qp_state = qedr_get_ibqp_state(qp->state);
1965 if (attr_mask & IB_QP_STATE)
1966 new_qp_state = attr->qp_state;
1967 else
1968 new_qp_state = old_qp_state;
1969
1970 if (rdma_protocol_roce(&dev->ibdev, 1)) {
1971 if (!ib_modify_qp_is_ok(old_qp_state, new_qp_state,
1972 ibqp->qp_type, attr_mask,
1973 IB_LINK_LAYER_ETHERNET)) {
1974 DP_ERR(dev,
1975 "modify qp: invalid attribute mask=0x%x specified for\n"
1976 "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n",
1977 attr_mask, qp->qp_id, ibqp->qp_type,
1978 old_qp_state, new_qp_state);
1979 rc = -EINVAL;
1980 goto err;
1981 }
1982 }
1983
1984 /* Translate the masks... */
1985 if (attr_mask & IB_QP_STATE) {
1986 SET_FIELD(qp_params.modify_flags,
1987 QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
1988 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state);
1989 }
1990
1991 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
1992 qp_params.sqd_async = true;
1993
1994 if (attr_mask & IB_QP_PKEY_INDEX) {
1995 SET_FIELD(qp_params.modify_flags,
1996 QED_ROCE_MODIFY_QP_VALID_PKEY, 1);
1997 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) {
1998 rc = -EINVAL;
1999 goto err;
2000 }
2001
2002 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT;
2003 }
2004
2005 if (attr_mask & IB_QP_QKEY)
2006 qp->qkey = attr->qkey;
2007
2008 if (attr_mask & IB_QP_ACCESS_FLAGS) {
2009 SET_FIELD(qp_params.modify_flags,
2010 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
2011 qp_params.incoming_rdma_read_en = attr->qp_access_flags &
2012 IB_ACCESS_REMOTE_READ;
2013 qp_params.incoming_rdma_write_en = attr->qp_access_flags &
2014 IB_ACCESS_REMOTE_WRITE;
2015 qp_params.incoming_atomic_en = attr->qp_access_flags &
2016 IB_ACCESS_REMOTE_ATOMIC;
2017 }
2018
2019 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
2020 if (attr_mask & IB_QP_PATH_MTU) {
2021 if (attr->path_mtu < IB_MTU_256 ||
2022 attr->path_mtu > IB_MTU_4096) {
2023 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n");
2024 rc = -EINVAL;
2025 goto err;
2026 }
2027 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
2028 ib_mtu_enum_to_int(iboe_get_mtu
2029 (dev->ndev->mtu)));
2030 }
2031
2032 if (!qp->mtu) {
2033 qp->mtu =
2034 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2035 pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);
2036 }
2037
2038 SET_FIELD(qp_params.modify_flags,
2039 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1);
2040
2041 qp_params.traffic_class_tos = grh->traffic_class;
2042 qp_params.flow_label = grh->flow_label;
2043 qp_params.hop_limit_ttl = grh->hop_limit;
2044
2045 qp->sgid_idx = grh->sgid_index;
2046
2047 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params);
2048 if (rc) {
2049 DP_ERR(dev,
2050 "modify qp: problems with GID index %d (rc=%d)\n",
2051 grh->sgid_index, rc);
2052 return rc;
2053 }
2054
2055 rc = qedr_get_dmac(dev, &attr->ah_attr,
2056 qp_params.remote_mac_addr);
2057 if (rc)
2058 return rc;
2059
2060 qp_params.use_local_mac = true;
2061 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr);
2062
2063 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n",
2064 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
2065 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
2066 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n",
2067 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
2068 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
2069 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n",
2070 qp_params.remote_mac_addr);
2071
2072 qp_params.mtu = qp->mtu;
2073 qp_params.lb_indication = false;
2074 }
2075
2076 if (!qp_params.mtu) {
2077 /* Stay with current MTU */
2078 if (qp->mtu)
2079 qp_params.mtu = qp->mtu;
2080 else
2081 qp_params.mtu =
2082 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2083 }
2084
2085 if (attr_mask & IB_QP_TIMEOUT) {
2086 SET_FIELD(qp_params.modify_flags,
2087 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
2088
2089 qp_params.ack_timeout = attr->timeout;
2090 if (attr->timeout) {
2091 u32 temp;
2092
2093 temp = 4096 * (1UL << attr->timeout) / 1000 / 1000;
2094 /* FW requires [msec] */
2095 qp_params.ack_timeout = temp;
2096 } else {
2097 /* Infinite */
2098 qp_params.ack_timeout = 0;
2099 }
2100 }
2101 if (attr_mask & IB_QP_RETRY_CNT) {
2102 SET_FIELD(qp_params.modify_flags,
2103 QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
2104 qp_params.retry_cnt = attr->retry_cnt;
2105 }
2106
2107 if (attr_mask & IB_QP_RNR_RETRY) {
2108 SET_FIELD(qp_params.modify_flags,
2109 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1);
2110 qp_params.rnr_retry_cnt = attr->rnr_retry;
2111 }
2112
2113 if (attr_mask & IB_QP_RQ_PSN) {
2114 SET_FIELD(qp_params.modify_flags,
2115 QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1);
2116 qp_params.rq_psn = attr->rq_psn;
2117 qp->rq_psn = attr->rq_psn;
2118 }
2119
2120 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
2121 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {
2122 rc = -EINVAL;
2123 DP_ERR(dev,
2124 "unsupported max_rd_atomic=%d, supported=%d\n",
2125 attr->max_rd_atomic,
2126 dev->attr.max_qp_req_rd_atomic_resc);
2127 goto err;
2128 }
2129
2130 SET_FIELD(qp_params.modify_flags,
2131 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1);
2132 qp_params.max_rd_atomic_req = attr->max_rd_atomic;
2133 }
2134
2135 if (attr_mask & IB_QP_MIN_RNR_TIMER) {
2136 SET_FIELD(qp_params.modify_flags,
2137 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1);
2138 qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
2139 }
2140
2141 if (attr_mask & IB_QP_SQ_PSN) {
2142 SET_FIELD(qp_params.modify_flags,
2143 QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1);
2144 qp_params.sq_psn = attr->sq_psn;
2145 qp->sq_psn = attr->sq_psn;
2146 }
2147
2148 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
2149 if (attr->max_dest_rd_atomic >
2150 dev->attr.max_qp_resp_rd_atomic_resc) {
2151 DP_ERR(dev,
2152 "unsupported max_dest_rd_atomic=%d, supported=%d\n",
2153 attr->max_dest_rd_atomic,
2154 dev->attr.max_qp_resp_rd_atomic_resc);
2155
2156 rc = -EINVAL;
2157 goto err;
2158 }
2159
2160 SET_FIELD(qp_params.modify_flags,
2161 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1);
2162 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
2163 }
2164
2165 if (attr_mask & IB_QP_DEST_QPN) {
2166 SET_FIELD(qp_params.modify_flags,
2167 QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1);
2168
2169 qp_params.dest_qp = attr->dest_qp_num;
2170 qp->dest_qp_num = attr->dest_qp_num;
2171 }
2172
2173 if (qp->qp_type != IB_QPT_GSI)
2174 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
2175 qp->qed_qp, &qp_params);
2176
2177 if (attr_mask & IB_QP_STATE) {
2178 if ((qp->qp_type != IB_QPT_GSI) && (!udata))
2179 rc = qedr_update_qp_state(dev, qp, qp_params.new_state);
2180 qp->state = qp_params.new_state;
2181 }
2182
2183 err:
2184 return rc;
2185 }
2186
2187 static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params)
2188 {
2189 int ib_qp_acc_flags = 0;
2190
2191 if (params->incoming_rdma_write_en)
2192 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
2193 if (params->incoming_rdma_read_en)
2194 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
2195 if (params->incoming_atomic_en)
2196 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
2197 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
2198 return ib_qp_acc_flags;
2199 }
2200
2201 int qedr_query_qp(struct ib_qp *ibqp,
2202 struct ib_qp_attr *qp_attr,
2203 int attr_mask, struct ib_qp_init_attr *qp_init_attr)
2204 {
2205 struct qed_rdma_query_qp_out_params params;
2206 struct qedr_qp *qp = get_qedr_qp(ibqp);
2207 struct qedr_dev *dev = qp->dev;
2208 int rc = 0;
2209
2210 memset(&params, 0, sizeof(params));
2211
2212 rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, &params);
2213 if (rc)
2214 goto err;
2215
2216 memset(qp_attr, 0, sizeof(*qp_attr));
2217 memset(qp_init_attr, 0, sizeof(*qp_init_attr));
2218
2219 qp_attr->qp_state = qedr_get_ibqp_state(params.state);
2220 qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state);
2221 qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu);
2222 qp_attr->path_mig_state = IB_MIG_MIGRATED;
2223 qp_attr->rq_psn = params.rq_psn;
2224 qp_attr->sq_psn = params.sq_psn;
2225 qp_attr->dest_qp_num = params.dest_qp;
2226
2227 qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(&params);
2228
2229 qp_attr->cap.max_send_wr = qp->sq.max_wr;
2230 qp_attr->cap.max_recv_wr = qp->rq.max_wr;
2231 qp_attr->cap.max_send_sge = qp->sq.max_sges;
2232 qp_attr->cap.max_recv_sge = qp->rq.max_sges;
2233 qp_attr->cap.max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE;
2234 qp_init_attr->cap = qp_attr->cap;
2235
2236 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
2237 rdma_ah_set_grh(&qp_attr->ah_attr, NULL,
2238 params.flow_label, qp->sgid_idx,
2239 params.hop_limit_ttl, params.traffic_class_tos);
2240 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, &params.dgid.bytes[0]);
2241 rdma_ah_set_port_num(&qp_attr->ah_attr, 1);
2242 rdma_ah_set_sl(&qp_attr->ah_attr, 0);
2243 qp_attr->timeout = params.timeout;
2244 qp_attr->rnr_retry = params.rnr_retry;
2245 qp_attr->retry_cnt = params.retry_cnt;
2246 qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
2247 qp_attr->pkey_index = params.pkey_index;
2248 qp_attr->port_num = 1;
2249 rdma_ah_set_path_bits(&qp_attr->ah_attr, 0);
2250 rdma_ah_set_static_rate(&qp_attr->ah_attr, 0);
2251 qp_attr->alt_pkey_index = 0;
2252 qp_attr->alt_port_num = 0;
2253 qp_attr->alt_timeout = 0;
2254 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
2255
2256 qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0;
2257 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
2258 qp_attr->max_rd_atomic = params.max_rd_atomic;
2259 qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0;
2260
2261 DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n",
2262 qp_attr->cap.max_inline_data);
2263
2264 err:
2265 return rc;
2266 }
2267
2268 static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp)
2269 {
2270 int rc = 0;
2271
2272 if (qp->qp_type != IB_QPT_GSI) {
2273 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2274 if (rc)
2275 return rc;
2276 }
2277
2278 if (qp->ibqp.uobject && qp->ibqp.uobject->context)
2279 qedr_cleanup_user(dev, qp);
2280 else
2281 qedr_cleanup_kernel(dev, qp);
2282
2283 return 0;
2284 }
2285
2286 int qedr_destroy_qp(struct ib_qp *ibqp)
2287 {
2288 struct qedr_qp *qp = get_qedr_qp(ibqp);
2289 struct qedr_dev *dev = qp->dev;
2290 struct ib_qp_attr attr;
2291 int attr_mask = 0;
2292 int rc = 0;
2293
2294 DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2295 qp, qp->qp_type);
2296
2297 if (rdma_protocol_roce(&dev->ibdev, 1)) {
2298 if ((qp->state != QED_ROCE_QP_STATE_RESET) &&
2299 (qp->state != QED_ROCE_QP_STATE_ERR) &&
2300 (qp->state != QED_ROCE_QP_STATE_INIT)) {
2301
2302 attr.qp_state = IB_QPS_ERR;
2303 attr_mask |= IB_QP_STATE;
2304
2305 /* Change the QP state to ERROR */
2306 qedr_modify_qp(ibqp, &attr, attr_mask, NULL);
2307 }
2308 } else {
2309 /* Wait for the connect/accept to complete */
2310 if (qp->ep) {
2311 int wait_count = 1;
2312
2313 while (qp->ep->during_connect) {
2314 DP_DEBUG(dev, QEDR_MSG_QP,
2315 "Still in during connect/accept\n");
2316
2317 msleep(100);
2318 if (wait_count++ > 200) {
2319 DP_NOTICE(dev,
2320 "during connect timeout\n");
2321 break;
2322 }
2323 }
2324 }
2325 }
2326
2327 if (qp->qp_type == IB_QPT_GSI)
2328 qedr_destroy_gsi_qp(dev);
2329
2330 qedr_free_qp_resources(dev, qp);
2331
2332 if (atomic_dec_and_test(&qp->refcnt)) {
2333 qedr_idr_remove(dev, qp->qp_id);
2334 kfree(qp);
2335 }
2336 return rc;
2337 }
2338
2339 struct ib_ah *qedr_create_ah(struct ib_pd *ibpd, struct rdma_ah_attr *attr,
2340 struct ib_udata *udata)
2341 {
2342 struct qedr_ah *ah;
2343
2344 ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
2345 if (!ah)
2346 return ERR_PTR(-ENOMEM);
2347
2348 ah->attr = *attr;
2349
2350 return &ah->ibah;
2351 }
2352
2353 int qedr_destroy_ah(struct ib_ah *ibah)
2354 {
2355 struct qedr_ah *ah = get_qedr_ah(ibah);
2356
2357 kfree(ah);
2358 return 0;
2359 }
2360
2361 static void free_mr_info(struct qedr_dev *dev, struct mr_info *info)
2362 {
2363 struct qedr_pbl *pbl, *tmp;
2364
2365 if (info->pbl_table)
2366 list_add_tail(&info->pbl_table->list_entry,
2367 &info->free_pbl_list);
2368
2369 if (!list_empty(&info->inuse_pbl_list))
2370 list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
2371
2372 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
2373 list_del(&pbl->list_entry);
2374 qedr_free_pbl(dev, &info->pbl_info, pbl);
2375 }
2376 }
2377
2378 static int init_mr_info(struct qedr_dev *dev, struct mr_info *info,
2379 size_t page_list_len, bool two_layered)
2380 {
2381 struct qedr_pbl *tmp;
2382 int rc;
2383
2384 INIT_LIST_HEAD(&info->free_pbl_list);
2385 INIT_LIST_HEAD(&info->inuse_pbl_list);
2386
2387 rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info,
2388 page_list_len, two_layered);
2389 if (rc)
2390 goto done;
2391
2392 info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2393 if (IS_ERR(info->pbl_table)) {
2394 rc = PTR_ERR(info->pbl_table);
2395 goto done;
2396 }
2397
2398 DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n",
2399 &info->pbl_table->pa);
2400
2401 /* in usual case we use 2 PBLs, so we add one to free
2402 * list and allocating another one
2403 */
2404 tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2405 if (IS_ERR(tmp)) {
2406 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n");
2407 goto done;
2408 }
2409
2410 list_add_tail(&tmp->list_entry, &info->free_pbl_list);
2411
2412 DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa);
2413
2414 done:
2415 if (rc)
2416 free_mr_info(dev, info);
2417
2418 return rc;
2419 }
2420
2421 struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
2422 u64 usr_addr, int acc, struct ib_udata *udata)
2423 {
2424 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2425 struct qedr_mr *mr;
2426 struct qedr_pd *pd;
2427 int rc = -ENOMEM;
2428
2429 pd = get_qedr_pd(ibpd);
2430 DP_DEBUG(dev, QEDR_MSG_MR,
2431 "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
2432 pd->pd_id, start, len, usr_addr, acc);
2433
2434 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
2435 return ERR_PTR(-EINVAL);
2436
2437 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2438 if (!mr)
2439 return ERR_PTR(rc);
2440
2441 mr->type = QEDR_MR_USER;
2442
2443 mr->umem = ib_umem_get(ibpd->uobject->context, start, len, acc, 0);
2444 if (IS_ERR(mr->umem)) {
2445 rc = -EFAULT;
2446 goto err0;
2447 }
2448
2449 rc = init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1);
2450 if (rc)
2451 goto err1;
2452
2453 qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
2454 &mr->info.pbl_info, mr->umem->page_shift);
2455
2456 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2457 if (rc) {
2458 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2459 goto err1;
2460 }
2461
2462 /* Index only, 18 bit long, lkey = itid << 8 | key */
2463 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2464 mr->hw_mr.key = 0;
2465 mr->hw_mr.pd = pd->pd_id;
2466 mr->hw_mr.local_read = 1;
2467 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2468 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2469 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2470 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2471 mr->hw_mr.mw_bind = false;
2472 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
2473 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2474 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2475 mr->hw_mr.page_size_log = mr->umem->page_shift;
2476 mr->hw_mr.fbo = ib_umem_offset(mr->umem);
2477 mr->hw_mr.length = len;
2478 mr->hw_mr.vaddr = usr_addr;
2479 mr->hw_mr.zbva = false;
2480 mr->hw_mr.phy_mr = false;
2481 mr->hw_mr.dma_mr = false;
2482
2483 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2484 if (rc) {
2485 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2486 goto err2;
2487 }
2488
2489 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2490 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2491 mr->hw_mr.remote_atomic)
2492 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2493
2494 DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n",
2495 mr->ibmr.lkey);
2496 return &mr->ibmr;
2497
2498 err2:
2499 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2500 err1:
2501 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2502 err0:
2503 kfree(mr);
2504 return ERR_PTR(rc);
2505 }
2506
2507 int qedr_dereg_mr(struct ib_mr *ib_mr)
2508 {
2509 struct qedr_mr *mr = get_qedr_mr(ib_mr);
2510 struct qedr_dev *dev = get_qedr_dev(ib_mr->device);
2511 int rc = 0;
2512
2513 rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid);
2514 if (rc)
2515 return rc;
2516
2517 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2518
2519 if ((mr->type != QEDR_MR_DMA) && (mr->type != QEDR_MR_FRMR))
2520 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2521
2522 /* it could be user registered memory. */
2523 if (mr->umem)
2524 ib_umem_release(mr->umem);
2525
2526 kfree(mr);
2527
2528 return rc;
2529 }
2530
2531 static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd,
2532 int max_page_list_len)
2533 {
2534 struct qedr_pd *pd = get_qedr_pd(ibpd);
2535 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2536 struct qedr_mr *mr;
2537 int rc = -ENOMEM;
2538
2539 DP_DEBUG(dev, QEDR_MSG_MR,
2540 "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id,
2541 max_page_list_len);
2542
2543 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2544 if (!mr)
2545 return ERR_PTR(rc);
2546
2547 mr->dev = dev;
2548 mr->type = QEDR_MR_FRMR;
2549
2550 rc = init_mr_info(dev, &mr->info, max_page_list_len, 1);
2551 if (rc)
2552 goto err0;
2553
2554 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2555 if (rc) {
2556 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2557 goto err0;
2558 }
2559
2560 /* Index only, 18 bit long, lkey = itid << 8 | key */
2561 mr->hw_mr.tid_type = QED_RDMA_TID_FMR;
2562 mr->hw_mr.key = 0;
2563 mr->hw_mr.pd = pd->pd_id;
2564 mr->hw_mr.local_read = 1;
2565 mr->hw_mr.local_write = 0;
2566 mr->hw_mr.remote_read = 0;
2567 mr->hw_mr.remote_write = 0;
2568 mr->hw_mr.remote_atomic = 0;
2569 mr->hw_mr.mw_bind = false;
2570 mr->hw_mr.pbl_ptr = 0;
2571 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2572 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2573 mr->hw_mr.fbo = 0;
2574 mr->hw_mr.length = 0;
2575 mr->hw_mr.vaddr = 0;
2576 mr->hw_mr.zbva = false;
2577 mr->hw_mr.phy_mr = true;
2578 mr->hw_mr.dma_mr = false;
2579
2580 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2581 if (rc) {
2582 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2583 goto err1;
2584 }
2585
2586 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2587 mr->ibmr.rkey = mr->ibmr.lkey;
2588
2589 DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey);
2590 return mr;
2591
2592 err1:
2593 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2594 err0:
2595 kfree(mr);
2596 return ERR_PTR(rc);
2597 }
2598
2599 struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd,
2600 enum ib_mr_type mr_type, u32 max_num_sg)
2601 {
2602 struct qedr_mr *mr;
2603
2604 if (mr_type != IB_MR_TYPE_MEM_REG)
2605 return ERR_PTR(-EINVAL);
2606
2607 mr = __qedr_alloc_mr(ibpd, max_num_sg);
2608
2609 if (IS_ERR(mr))
2610 return ERR_PTR(-EINVAL);
2611
2612 return &mr->ibmr;
2613 }
2614
2615 static int qedr_set_page(struct ib_mr *ibmr, u64 addr)
2616 {
2617 struct qedr_mr *mr = get_qedr_mr(ibmr);
2618 struct qedr_pbl *pbl_table;
2619 struct regpair *pbe;
2620 u32 pbes_in_page;
2621
2622 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
2623 DP_ERR(mr->dev, "qedr_set_page failes when %d\n", mr->npages);
2624 return -ENOMEM;
2625 }
2626
2627 DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n",
2628 mr->npages, addr);
2629
2630 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
2631 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
2632 pbe = (struct regpair *)pbl_table->va;
2633 pbe += mr->npages % pbes_in_page;
2634 pbe->lo = cpu_to_le32((u32)addr);
2635 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
2636
2637 mr->npages++;
2638
2639 return 0;
2640 }
2641
2642 static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info)
2643 {
2644 int work = info->completed - info->completed_handled - 1;
2645
2646 DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work);
2647 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
2648 struct qedr_pbl *pbl;
2649
2650 /* Free all the page list that are possible to be freed
2651 * (all the ones that were invalidated), under the assumption
2652 * that if an FMR was completed successfully that means that
2653 * if there was an invalidate operation before it also ended
2654 */
2655 pbl = list_first_entry(&info->inuse_pbl_list,
2656 struct qedr_pbl, list_entry);
2657 list_move_tail(&pbl->list_entry, &info->free_pbl_list);
2658 info->completed_handled++;
2659 }
2660 }
2661
2662 int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
2663 int sg_nents, unsigned int *sg_offset)
2664 {
2665 struct qedr_mr *mr = get_qedr_mr(ibmr);
2666
2667 mr->npages = 0;
2668
2669 handle_completed_mrs(mr->dev, &mr->info);
2670 return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page);
2671 }
2672
2673 struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc)
2674 {
2675 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2676 struct qedr_pd *pd = get_qedr_pd(ibpd);
2677 struct qedr_mr *mr;
2678 int rc;
2679
2680 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2681 if (!mr)
2682 return ERR_PTR(-ENOMEM);
2683
2684 mr->type = QEDR_MR_DMA;
2685
2686 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2687 if (rc) {
2688 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2689 goto err1;
2690 }
2691
2692 /* index only, 18 bit long, lkey = itid << 8 | key */
2693 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2694 mr->hw_mr.pd = pd->pd_id;
2695 mr->hw_mr.local_read = 1;
2696 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2697 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2698 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2699 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2700 mr->hw_mr.dma_mr = true;
2701
2702 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2703 if (rc) {
2704 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2705 goto err2;
2706 }
2707
2708 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2709 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2710 mr->hw_mr.remote_atomic)
2711 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2712
2713 DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey);
2714 return &mr->ibmr;
2715
2716 err2:
2717 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2718 err1:
2719 kfree(mr);
2720 return ERR_PTR(rc);
2721 }
2722
2723 static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq)
2724 {
2725 return (((wq->prod + 1) % wq->max_wr) == wq->cons);
2726 }
2727
2728 static int sge_data_len(struct ib_sge *sg_list, int num_sge)
2729 {
2730 int i, len = 0;
2731
2732 for (i = 0; i < num_sge; i++)
2733 len += sg_list[i].length;
2734
2735 return len;
2736 }
2737
2738 static void swap_wqe_data64(u64 *p)
2739 {
2740 int i;
2741
2742 for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
2743 *p = cpu_to_be64(cpu_to_le64(*p));
2744 }
2745
2746 static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev,
2747 struct qedr_qp *qp, u8 *wqe_size,
2748 struct ib_send_wr *wr,
2749 struct ib_send_wr **bad_wr, u8 *bits,
2750 u8 bit)
2751 {
2752 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
2753 char *seg_prt, *wqe;
2754 int i, seg_siz;
2755
2756 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
2757 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size);
2758 *bad_wr = wr;
2759 return 0;
2760 }
2761
2762 if (!data_size)
2763 return data_size;
2764
2765 *bits |= bit;
2766
2767 seg_prt = NULL;
2768 wqe = NULL;
2769 seg_siz = 0;
2770
2771 /* Copy data inline */
2772 for (i = 0; i < wr->num_sge; i++) {
2773 u32 len = wr->sg_list[i].length;
2774 void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
2775
2776 while (len > 0) {
2777 u32 cur;
2778
2779 /* New segment required */
2780 if (!seg_siz) {
2781 wqe = (char *)qed_chain_produce(&qp->sq.pbl);
2782 seg_prt = wqe;
2783 seg_siz = sizeof(struct rdma_sq_common_wqe);
2784 (*wqe_size)++;
2785 }
2786
2787 /* Calculate currently allowed length */
2788 cur = min_t(u32, len, seg_siz);
2789 memcpy(seg_prt, src, cur);
2790
2791 /* Update segment variables */
2792 seg_prt += cur;
2793 seg_siz -= cur;
2794
2795 /* Update sge variables */
2796 src += cur;
2797 len -= cur;
2798
2799 /* Swap fully-completed segments */
2800 if (!seg_siz)
2801 swap_wqe_data64((u64 *)wqe);
2802 }
2803 }
2804
2805 /* swap last not completed segment */
2806 if (seg_siz)
2807 swap_wqe_data64((u64 *)wqe);
2808
2809 return data_size;
2810 }
2811
2812 #define RQ_SGE_SET(sge, vaddr, vlength, vflags) \
2813 do { \
2814 DMA_REGPAIR_LE(sge->addr, vaddr); \
2815 (sge)->length = cpu_to_le32(vlength); \
2816 (sge)->flags = cpu_to_le32(vflags); \
2817 } while (0)
2818
2819 #define SRQ_HDR_SET(hdr, vwr_id, num_sge) \
2820 do { \
2821 DMA_REGPAIR_LE(hdr->wr_id, vwr_id); \
2822 (hdr)->num_sges = num_sge; \
2823 } while (0)
2824
2825 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \
2826 do { \
2827 DMA_REGPAIR_LE(sge->addr, vaddr); \
2828 (sge)->length = cpu_to_le32(vlength); \
2829 (sge)->l_key = cpu_to_le32(vlkey); \
2830 } while (0)
2831
2832 static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size,
2833 struct ib_send_wr *wr)
2834 {
2835 u32 data_size = 0;
2836 int i;
2837
2838 for (i = 0; i < wr->num_sge; i++) {
2839 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl);
2840
2841 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr);
2842 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
2843 sge->length = cpu_to_le32(wr->sg_list[i].length);
2844 data_size += wr->sg_list[i].length;
2845 }
2846
2847 if (wqe_size)
2848 *wqe_size += wr->num_sge;
2849
2850 return data_size;
2851 }
2852
2853 static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev,
2854 struct qedr_qp *qp,
2855 struct rdma_sq_rdma_wqe_1st *rwqe,
2856 struct rdma_sq_rdma_wqe_2nd *rwqe2,
2857 struct ib_send_wr *wr,
2858 struct ib_send_wr **bad_wr)
2859 {
2860 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
2861 DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr);
2862
2863 if (wr->send_flags & IB_SEND_INLINE &&
2864 (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
2865 wr->opcode == IB_WR_RDMA_WRITE)) {
2866 u8 flags = 0;
2867
2868 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
2869 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr,
2870 bad_wr, &rwqe->flags, flags);
2871 }
2872
2873 return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr);
2874 }
2875
2876 static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev,
2877 struct qedr_qp *qp,
2878 struct rdma_sq_send_wqe_1st *swqe,
2879 struct rdma_sq_send_wqe_2st *swqe2,
2880 struct ib_send_wr *wr,
2881 struct ib_send_wr **bad_wr)
2882 {
2883 memset(swqe2, 0, sizeof(*swqe2));
2884 if (wr->send_flags & IB_SEND_INLINE) {
2885 u8 flags = 0;
2886
2887 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
2888 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr,
2889 bad_wr, &swqe->flags, flags);
2890 }
2891
2892 return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr);
2893 }
2894
2895 static int qedr_prepare_reg(struct qedr_qp *qp,
2896 struct rdma_sq_fmr_wqe_1st *fwqe1,
2897 struct ib_reg_wr *wr)
2898 {
2899 struct qedr_mr *mr = get_qedr_mr(wr->mr);
2900 struct rdma_sq_fmr_wqe_2nd *fwqe2;
2901
2902 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl);
2903 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
2904 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
2905 fwqe1->l_key = wr->key;
2906
2907 fwqe2->access_ctrl = 0;
2908
2909 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
2910 !!(wr->access & IB_ACCESS_REMOTE_READ));
2911 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
2912 !!(wr->access & IB_ACCESS_REMOTE_WRITE));
2913 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
2914 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
2915 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
2916 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
2917 !!(wr->access & IB_ACCESS_LOCAL_WRITE));
2918 fwqe2->fmr_ctrl = 0;
2919
2920 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
2921 ilog2(mr->ibmr.page_size) - 12);
2922
2923 fwqe2->length_hi = 0;
2924 fwqe2->length_lo = mr->ibmr.length;
2925 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
2926 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
2927
2928 qp->wqe_wr_id[qp->sq.prod].mr = mr;
2929
2930 return 0;
2931 }
2932
2933 static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
2934 {
2935 switch (opcode) {
2936 case IB_WR_RDMA_WRITE:
2937 case IB_WR_RDMA_WRITE_WITH_IMM:
2938 return IB_WC_RDMA_WRITE;
2939 case IB_WR_SEND_WITH_IMM:
2940 case IB_WR_SEND:
2941 case IB_WR_SEND_WITH_INV:
2942 return IB_WC_SEND;
2943 case IB_WR_RDMA_READ:
2944 case IB_WR_RDMA_READ_WITH_INV:
2945 return IB_WC_RDMA_READ;
2946 case IB_WR_ATOMIC_CMP_AND_SWP:
2947 return IB_WC_COMP_SWAP;
2948 case IB_WR_ATOMIC_FETCH_AND_ADD:
2949 return IB_WC_FETCH_ADD;
2950 case IB_WR_REG_MR:
2951 return IB_WC_REG_MR;
2952 case IB_WR_LOCAL_INV:
2953 return IB_WC_LOCAL_INV;
2954 default:
2955 return IB_WC_SEND;
2956 }
2957 }
2958
2959 static inline bool qedr_can_post_send(struct qedr_qp *qp, struct ib_send_wr *wr)
2960 {
2961 int wq_is_full, err_wr, pbl_is_full;
2962 struct qedr_dev *dev = qp->dev;
2963
2964 /* prevent SQ overflow and/or processing of a bad WR */
2965 err_wr = wr->num_sge > qp->sq.max_sges;
2966 wq_is_full = qedr_wq_is_full(&qp->sq);
2967 pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) <
2968 QEDR_MAX_SQE_ELEMENTS_PER_SQE;
2969 if (wq_is_full || err_wr || pbl_is_full) {
2970 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) {
2971 DP_ERR(dev,
2972 "error: WQ is full. Post send on QP %p failed (this error appears only once)\n",
2973 qp);
2974 qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL;
2975 }
2976
2977 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) {
2978 DP_ERR(dev,
2979 "error: WR is bad. Post send on QP %p failed (this error appears only once)\n",
2980 qp);
2981 qp->err_bitmap |= QEDR_QP_ERR_BAD_SR;
2982 }
2983
2984 if (pbl_is_full &&
2985 !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) {
2986 DP_ERR(dev,
2987 "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n",
2988 qp);
2989 qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL;
2990 }
2991 return false;
2992 }
2993 return true;
2994 }
2995
2996 static int __qedr_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
2997 struct ib_send_wr **bad_wr)
2998 {
2999 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3000 struct qedr_qp *qp = get_qedr_qp(ibqp);
3001 struct rdma_sq_atomic_wqe_1st *awqe1;
3002 struct rdma_sq_atomic_wqe_2nd *awqe2;
3003 struct rdma_sq_atomic_wqe_3rd *awqe3;
3004 struct rdma_sq_send_wqe_2st *swqe2;
3005 struct rdma_sq_local_inv_wqe *iwqe;
3006 struct rdma_sq_rdma_wqe_2nd *rwqe2;
3007 struct rdma_sq_send_wqe_1st *swqe;
3008 struct rdma_sq_rdma_wqe_1st *rwqe;
3009 struct rdma_sq_fmr_wqe_1st *fwqe1;
3010 struct rdma_sq_common_wqe *wqe;
3011 u32 length;
3012 int rc = 0;
3013 bool comp;
3014
3015 if (!qedr_can_post_send(qp, wr)) {
3016 *bad_wr = wr;
3017 return -ENOMEM;
3018 }
3019
3020 wqe = qed_chain_produce(&qp->sq.pbl);
3021 qp->wqe_wr_id[qp->sq.prod].signaled =
3022 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
3023
3024 wqe->flags = 0;
3025 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG,
3026 !!(wr->send_flags & IB_SEND_SOLICITED));
3027 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled;
3028 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
3029 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,
3030 !!(wr->send_flags & IB_SEND_FENCE));
3031 wqe->prev_wqe_size = qp->prev_wqe_size;
3032
3033 qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode);
3034
3035 switch (wr->opcode) {
3036 case IB_WR_SEND_WITH_IMM:
3037 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
3038 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3039 swqe->wqe_size = 2;
3040 swqe2 = qed_chain_produce(&qp->sq.pbl);
3041
3042 swqe->inv_key_or_imm_data = cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
3043 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3044 wr, bad_wr);
3045 swqe->length = cpu_to_le32(length);
3046 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3047 qp->prev_wqe_size = swqe->wqe_size;
3048 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3049 break;
3050 case IB_WR_SEND:
3051 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
3052 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3053
3054 swqe->wqe_size = 2;
3055 swqe2 = qed_chain_produce(&qp->sq.pbl);
3056 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3057 wr, bad_wr);
3058 swqe->length = cpu_to_le32(length);
3059 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3060 qp->prev_wqe_size = swqe->wqe_size;
3061 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3062 break;
3063 case IB_WR_SEND_WITH_INV:
3064 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
3065 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3066 swqe2 = qed_chain_produce(&qp->sq.pbl);
3067 swqe->wqe_size = 2;
3068 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey);
3069 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3070 wr, bad_wr);
3071 swqe->length = cpu_to_le32(length);
3072 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3073 qp->prev_wqe_size = swqe->wqe_size;
3074 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3075 break;
3076
3077 case IB_WR_RDMA_WRITE_WITH_IMM:
3078 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
3079 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3080
3081 rwqe->wqe_size = 2;
3082 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
3083 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3084 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3085 wr, bad_wr);
3086 rwqe->length = cpu_to_le32(length);
3087 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3088 qp->prev_wqe_size = rwqe->wqe_size;
3089 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3090 break;
3091 case IB_WR_RDMA_WRITE:
3092 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
3093 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3094
3095 rwqe->wqe_size = 2;
3096 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3097 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3098 wr, bad_wr);
3099 rwqe->length = cpu_to_le32(length);
3100 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3101 qp->prev_wqe_size = rwqe->wqe_size;
3102 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3103 break;
3104 case IB_WR_RDMA_READ_WITH_INV:
3105 SET_FIELD2(wqe->flags, RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG, 1);
3106 /* fallthrough -- same is identical to RDMA READ */
3107
3108 case IB_WR_RDMA_READ:
3109 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
3110 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3111
3112 rwqe->wqe_size = 2;
3113 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3114 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3115 wr, bad_wr);
3116 rwqe->length = cpu_to_le32(length);
3117 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3118 qp->prev_wqe_size = rwqe->wqe_size;
3119 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3120 break;
3121
3122 case IB_WR_ATOMIC_CMP_AND_SWP:
3123 case IB_WR_ATOMIC_FETCH_AND_ADD:
3124 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe;
3125 awqe1->wqe_size = 4;
3126
3127 awqe2 = qed_chain_produce(&qp->sq.pbl);
3128 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr);
3129 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
3130
3131 awqe3 = qed_chain_produce(&qp->sq.pbl);
3132
3133 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
3134 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
3135 DMA_REGPAIR_LE(awqe3->swap_data,
3136 atomic_wr(wr)->compare_add);
3137 } else {
3138 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
3139 DMA_REGPAIR_LE(awqe3->swap_data,
3140 atomic_wr(wr)->swap);
3141 DMA_REGPAIR_LE(awqe3->cmp_data,
3142 atomic_wr(wr)->compare_add);
3143 }
3144
3145 qedr_prepare_sq_sges(qp, NULL, wr);
3146
3147 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;
3148 qp->prev_wqe_size = awqe1->wqe_size;
3149 break;
3150
3151 case IB_WR_LOCAL_INV:
3152 iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
3153 iwqe->wqe_size = 1;
3154
3155 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
3156 iwqe->inv_l_key = wr->ex.invalidate_rkey;
3157 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size;
3158 qp->prev_wqe_size = iwqe->wqe_size;
3159 break;
3160 case IB_WR_REG_MR:
3161 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n");
3162 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
3163 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
3164 fwqe1->wqe_size = 2;
3165
3166 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr));
3167 if (rc) {
3168 DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc);
3169 *bad_wr = wr;
3170 break;
3171 }
3172
3173 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
3174 qp->prev_wqe_size = fwqe1->wqe_size;
3175 break;
3176 default:
3177 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode);
3178 rc = -EINVAL;
3179 *bad_wr = wr;
3180 break;
3181 }
3182
3183 if (*bad_wr) {
3184 u16 value;
3185
3186 /* Restore prod to its position before
3187 * this WR was processed
3188 */
3189 value = le16_to_cpu(qp->sq.db_data.data.value);
3190 qed_chain_set_prod(&qp->sq.pbl, value, wqe);
3191
3192 /* Restore prev_wqe_size */
3193 qp->prev_wqe_size = wqe->prev_wqe_size;
3194 rc = -EINVAL;
3195 DP_ERR(dev, "POST SEND FAILED\n");
3196 }
3197
3198 return rc;
3199 }
3200
3201 int qedr_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
3202 struct ib_send_wr **bad_wr)
3203 {
3204 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3205 struct qedr_qp *qp = get_qedr_qp(ibqp);
3206 unsigned long flags;
3207 int rc = 0;
3208
3209 *bad_wr = NULL;
3210
3211 if (qp->qp_type == IB_QPT_GSI)
3212 return qedr_gsi_post_send(ibqp, wr, bad_wr);
3213
3214 spin_lock_irqsave(&qp->q_lock, flags);
3215
3216 if (rdma_protocol_roce(&dev->ibdev, 1)) {
3217 if ((qp->state != QED_ROCE_QP_STATE_RTS) &&
3218 (qp->state != QED_ROCE_QP_STATE_ERR) &&
3219 (qp->state != QED_ROCE_QP_STATE_SQD)) {
3220 spin_unlock_irqrestore(&qp->q_lock, flags);
3221 *bad_wr = wr;
3222 DP_DEBUG(dev, QEDR_MSG_CQ,
3223 "QP in wrong state! QP icid=0x%x state %d\n",
3224 qp->icid, qp->state);
3225 return -EINVAL;
3226 }
3227 }
3228
3229 while (wr) {
3230 rc = __qedr_post_send(ibqp, wr, bad_wr);
3231 if (rc)
3232 break;
3233
3234 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
3235
3236 qedr_inc_sw_prod(&qp->sq);
3237
3238 qp->sq.db_data.data.value++;
3239
3240 wr = wr->next;
3241 }
3242
3243 /* Trigger doorbell
3244 * If there was a failure in the first WR then it will be triggered in
3245 * vane. However this is not harmful (as long as the producer value is
3246 * unchanged). For performance reasons we avoid checking for this
3247 * redundant doorbell.
3248 */
3249 wmb();
3250 writel(qp->sq.db_data.raw, qp->sq.db);
3251
3252 /* Make sure write sticks */
3253 mmiowb();
3254
3255 spin_unlock_irqrestore(&qp->q_lock, flags);
3256
3257 return rc;
3258 }
3259
3260 int qedr_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
3261 struct ib_recv_wr **bad_wr)
3262 {
3263 struct qedr_qp *qp = get_qedr_qp(ibqp);
3264 struct qedr_dev *dev = qp->dev;
3265 unsigned long flags;
3266 int status = 0;
3267
3268 if (qp->qp_type == IB_QPT_GSI)
3269 return qedr_gsi_post_recv(ibqp, wr, bad_wr);
3270
3271 spin_lock_irqsave(&qp->q_lock, flags);
3272
3273 if (qp->state == QED_ROCE_QP_STATE_RESET) {
3274 spin_unlock_irqrestore(&qp->q_lock, flags);
3275 *bad_wr = wr;
3276 return -EINVAL;
3277 }
3278
3279 while (wr) {
3280 int i;
3281
3282 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) <
3283 QEDR_MAX_RQE_ELEMENTS_PER_RQE ||
3284 wr->num_sge > qp->rq.max_sges) {
3285 DP_ERR(dev, "Can't post WR (%d < %d) || (%d > %d)\n",
3286 qed_chain_get_elem_left_u32(&qp->rq.pbl),
3287 QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge,
3288 qp->rq.max_sges);
3289 status = -ENOMEM;
3290 *bad_wr = wr;
3291 break;
3292 }
3293 for (i = 0; i < wr->num_sge; i++) {
3294 u32 flags = 0;
3295 struct rdma_rq_sge *rqe =
3296 qed_chain_produce(&qp->rq.pbl);
3297
3298 /* First one must include the number
3299 * of SGE in the list
3300 */
3301 if (!i)
3302 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES,
3303 wr->num_sge);
3304
3305 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY,
3306 wr->sg_list[i].lkey);
3307
3308 RQ_SGE_SET(rqe, wr->sg_list[i].addr,
3309 wr->sg_list[i].length, flags);
3310 }
3311
3312 /* Special case of no sges. FW requires between 1-4 sges...
3313 * in this case we need to post 1 sge with length zero. this is
3314 * because rdma write with immediate consumes an RQ.
3315 */
3316 if (!wr->num_sge) {
3317 u32 flags = 0;
3318 struct rdma_rq_sge *rqe =
3319 qed_chain_produce(&qp->rq.pbl);
3320
3321 /* First one must include the number
3322 * of SGE in the list
3323 */
3324 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, 0);
3325 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
3326
3327 RQ_SGE_SET(rqe, 0, 0, flags);
3328 i = 1;
3329 }
3330
3331 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
3332 qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
3333
3334 qedr_inc_sw_prod(&qp->rq);
3335
3336 /* Flush all the writes before signalling doorbell */
3337 wmb();
3338
3339 qp->rq.db_data.data.value++;
3340
3341 writel(qp->rq.db_data.raw, qp->rq.db);
3342
3343 /* Make sure write sticks */
3344 mmiowb();
3345
3346 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
3347 writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2);
3348 mmiowb(); /* for second doorbell */
3349 }
3350
3351 wr = wr->next;
3352 }
3353
3354 spin_unlock_irqrestore(&qp->q_lock, flags);
3355
3356 return status;
3357 }
3358
3359 static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe)
3360 {
3361 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3362
3363 return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) ==
3364 cq->pbl_toggle;
3365 }
3366
3367 static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe)
3368 {
3369 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3370 struct qedr_qp *qp;
3371
3372 qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi,
3373 resp_cqe->qp_handle.lo,
3374 u64);
3375 return qp;
3376 }
3377
3378 static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe)
3379 {
3380 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3381
3382 return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
3383 }
3384
3385 /* Return latest CQE (needs processing) */
3386 static union rdma_cqe *get_cqe(struct qedr_cq *cq)
3387 {
3388 return cq->latest_cqe;
3389 }
3390
3391 /* In fmr we need to increase the number of fmr completed counter for the fmr
3392 * algorithm determining whether we can free a pbl or not.
3393 * we need to perform this whether the work request was signaled or not. for
3394 * this purpose we call this function from the condition that checks if a wr
3395 * should be skipped, to make sure we don't miss it ( possibly this fmr
3396 * operation was not signalted)
3397 */
3398 static inline void qedr_chk_if_fmr(struct qedr_qp *qp)
3399 {
3400 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
3401 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3402 }
3403
3404 static int process_req(struct qedr_dev *dev, struct qedr_qp *qp,
3405 struct qedr_cq *cq, int num_entries,
3406 struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status,
3407 int force)
3408 {
3409 u16 cnt = 0;
3410
3411 while (num_entries && qp->sq.wqe_cons != hw_cons) {
3412 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
3413 qedr_chk_if_fmr(qp);
3414 /* skip WC */
3415 goto next_cqe;
3416 }
3417
3418 /* fill WC */
3419 wc->status = status;
3420 wc->vendor_err = 0;
3421 wc->wc_flags = 0;
3422 wc->src_qp = qp->id;
3423 wc->qp = &qp->ibqp;
3424
3425 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
3426 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
3427
3428 switch (wc->opcode) {
3429 case IB_WC_RDMA_WRITE:
3430 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3431 break;
3432 case IB_WC_COMP_SWAP:
3433 case IB_WC_FETCH_ADD:
3434 wc->byte_len = 8;
3435 break;
3436 case IB_WC_REG_MR:
3437 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3438 break;
3439 case IB_WC_RDMA_READ:
3440 case IB_WC_SEND:
3441 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3442 break;
3443 default:
3444 break;
3445 }
3446
3447 num_entries--;
3448 wc++;
3449 cnt++;
3450 next_cqe:
3451 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
3452 qed_chain_consume(&qp->sq.pbl);
3453 qedr_inc_sw_cons(&qp->sq);
3454 }
3455
3456 return cnt;
3457 }
3458
3459 static int qedr_poll_cq_req(struct qedr_dev *dev,
3460 struct qedr_qp *qp, struct qedr_cq *cq,
3461 int num_entries, struct ib_wc *wc,
3462 struct rdma_cqe_requester *req)
3463 {
3464 int cnt = 0;
3465
3466 switch (req->status) {
3467 case RDMA_CQE_REQ_STS_OK:
3468 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3469 IB_WC_SUCCESS, 0);
3470 break;
3471 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
3472 if (qp->state != QED_ROCE_QP_STATE_ERR)
3473 DP_DEBUG(dev, QEDR_MSG_CQ,
3474 "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3475 cq->icid, qp->icid);
3476 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3477 IB_WC_WR_FLUSH_ERR, 1);
3478 break;
3479 default:
3480 /* process all WQE before the cosumer */
3481 qp->state = QED_ROCE_QP_STATE_ERR;
3482 cnt = process_req(dev, qp, cq, num_entries, wc,
3483 req->sq_cons - 1, IB_WC_SUCCESS, 0);
3484 wc += cnt;
3485 /* if we have extra WC fill it with actual error info */
3486 if (cnt < num_entries) {
3487 enum ib_wc_status wc_status;
3488
3489 switch (req->status) {
3490 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
3491 DP_ERR(dev,
3492 "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3493 cq->icid, qp->icid);
3494 wc_status = IB_WC_BAD_RESP_ERR;
3495 break;
3496 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
3497 DP_ERR(dev,
3498 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3499 cq->icid, qp->icid);
3500 wc_status = IB_WC_LOC_LEN_ERR;
3501 break;
3502 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
3503 DP_ERR(dev,
3504 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3505 cq->icid, qp->icid);
3506 wc_status = IB_WC_LOC_QP_OP_ERR;
3507 break;
3508 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
3509 DP_ERR(dev,
3510 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3511 cq->icid, qp->icid);
3512 wc_status = IB_WC_LOC_PROT_ERR;
3513 break;
3514 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
3515 DP_ERR(dev,
3516 "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3517 cq->icid, qp->icid);
3518 wc_status = IB_WC_MW_BIND_ERR;
3519 break;
3520 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
3521 DP_ERR(dev,
3522 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3523 cq->icid, qp->icid);
3524 wc_status = IB_WC_REM_INV_REQ_ERR;
3525 break;
3526 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
3527 DP_ERR(dev,
3528 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3529 cq->icid, qp->icid);
3530 wc_status = IB_WC_REM_ACCESS_ERR;
3531 break;
3532 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
3533 DP_ERR(dev,
3534 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3535 cq->icid, qp->icid);
3536 wc_status = IB_WC_REM_OP_ERR;
3537 break;
3538 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
3539 DP_ERR(dev,
3540 "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3541 cq->icid, qp->icid);
3542 wc_status = IB_WC_RNR_RETRY_EXC_ERR;
3543 break;
3544 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
3545 DP_ERR(dev,
3546 "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3547 cq->icid, qp->icid);
3548 wc_status = IB_WC_RETRY_EXC_ERR;
3549 break;
3550 default:
3551 DP_ERR(dev,
3552 "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3553 cq->icid, qp->icid);
3554 wc_status = IB_WC_GENERAL_ERR;
3555 }
3556 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
3557 wc_status, 1);
3558 }
3559 }
3560
3561 return cnt;
3562 }
3563
3564 static inline int qedr_cqe_resp_status_to_ib(u8 status)
3565 {
3566 switch (status) {
3567 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
3568 return IB_WC_LOC_ACCESS_ERR;
3569 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
3570 return IB_WC_LOC_LEN_ERR;
3571 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
3572 return IB_WC_LOC_QP_OP_ERR;
3573 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
3574 return IB_WC_LOC_PROT_ERR;
3575 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
3576 return IB_WC_MW_BIND_ERR;
3577 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
3578 return IB_WC_REM_INV_RD_REQ_ERR;
3579 case RDMA_CQE_RESP_STS_OK:
3580 return IB_WC_SUCCESS;
3581 default:
3582 return IB_WC_GENERAL_ERR;
3583 }
3584 }
3585
3586 static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp,
3587 struct ib_wc *wc)
3588 {
3589 wc->status = IB_WC_SUCCESS;
3590 wc->byte_len = le32_to_cpu(resp->length);
3591
3592 if (resp->flags & QEDR_RESP_IMM) {
3593 wc->ex.imm_data = cpu_to_be32(le32_to_cpu(resp->imm_data_or_inv_r_Key));
3594 wc->wc_flags |= IB_WC_WITH_IMM;
3595
3596 if (resp->flags & QEDR_RESP_RDMA)
3597 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
3598
3599 if (resp->flags & QEDR_RESP_INV)
3600 return -EINVAL;
3601
3602 } else if (resp->flags & QEDR_RESP_INV) {
3603 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key);
3604 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
3605
3606 if (resp->flags & QEDR_RESP_RDMA)
3607 return -EINVAL;
3608
3609 } else if (resp->flags & QEDR_RESP_RDMA) {
3610 return -EINVAL;
3611 }
3612
3613 return 0;
3614 }
3615
3616 static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
3617 struct qedr_cq *cq, struct ib_wc *wc,
3618 struct rdma_cqe_responder *resp, u64 wr_id)
3619 {
3620 /* Must fill fields before qedr_set_ok_cqe_resp_wc() */
3621 wc->opcode = IB_WC_RECV;
3622 wc->wc_flags = 0;
3623
3624 if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) {
3625 if (qedr_set_ok_cqe_resp_wc(resp, wc))
3626 DP_ERR(dev,
3627 "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n",
3628 cq, cq->icid, resp->flags);
3629
3630 } else {
3631 wc->status = qedr_cqe_resp_status_to_ib(resp->status);
3632 if (wc->status == IB_WC_GENERAL_ERR)
3633 DP_ERR(dev,
3634 "CQ %p (icid=%d) contains an invalid CQE status %d\n",
3635 cq, cq->icid, resp->status);
3636 }
3637
3638 /* Fill the rest of the WC */
3639 wc->vendor_err = 0;
3640 wc->src_qp = qp->id;
3641 wc->qp = &qp->ibqp;
3642 wc->wr_id = wr_id;
3643 }
3644
3645 static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
3646 struct qedr_cq *cq, struct ib_wc *wc,
3647 struct rdma_cqe_responder *resp)
3648 {
3649 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
3650
3651 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
3652
3653 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
3654 qed_chain_consume(&qp->rq.pbl);
3655 qedr_inc_sw_cons(&qp->rq);
3656
3657 return 1;
3658 }
3659
3660 static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq,
3661 int num_entries, struct ib_wc *wc, u16 hw_cons)
3662 {
3663 u16 cnt = 0;
3664
3665 while (num_entries && qp->rq.wqe_cons != hw_cons) {
3666 /* fill WC */
3667 wc->status = IB_WC_WR_FLUSH_ERR;
3668 wc->vendor_err = 0;
3669 wc->wc_flags = 0;
3670 wc->src_qp = qp->id;
3671 wc->byte_len = 0;
3672 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
3673 wc->qp = &qp->ibqp;
3674 num_entries--;
3675 wc++;
3676 cnt++;
3677 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
3678 qed_chain_consume(&qp->rq.pbl);
3679 qedr_inc_sw_cons(&qp->rq);
3680 }
3681
3682 return cnt;
3683 }
3684
3685 static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
3686 struct rdma_cqe_responder *resp, int *update)
3687 {
3688 if (le16_to_cpu(resp->rq_cons) == qp->rq.wqe_cons) {
3689 consume_cqe(cq);
3690 *update |= 1;
3691 }
3692 }
3693
3694 static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp,
3695 struct qedr_cq *cq, int num_entries,
3696 struct ib_wc *wc, struct rdma_cqe_responder *resp,
3697 int *update)
3698 {
3699 int cnt;
3700
3701 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
3702 cnt = process_resp_flush(qp, cq, num_entries, wc,
3703 resp->rq_cons);
3704 try_consume_resp_cqe(cq, qp, resp, update);
3705 } else {
3706 cnt = process_resp_one(dev, qp, cq, wc, resp);
3707 consume_cqe(cq);
3708 *update |= 1;
3709 }
3710
3711 return cnt;
3712 }
3713
3714 static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
3715 struct rdma_cqe_requester *req, int *update)
3716 {
3717 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
3718 consume_cqe(cq);
3719 *update |= 1;
3720 }
3721 }
3722
3723 int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
3724 {
3725 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
3726 struct qedr_cq *cq = get_qedr_cq(ibcq);
3727 union rdma_cqe *cqe = cq->latest_cqe;
3728 u32 old_cons, new_cons;
3729 unsigned long flags;
3730 int update = 0;
3731 int done = 0;
3732
3733 if (cq->destroyed) {
3734 DP_ERR(dev,
3735 "warning: poll was invoked after destroy for cq %p (icid=%d)\n",
3736 cq, cq->icid);
3737 return 0;
3738 }
3739
3740 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
3741 return qedr_gsi_poll_cq(ibcq, num_entries, wc);
3742
3743 spin_lock_irqsave(&cq->cq_lock, flags);
3744 old_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
3745 while (num_entries && is_valid_cqe(cq, cqe)) {
3746 struct qedr_qp *qp;
3747 int cnt = 0;
3748
3749 /* prevent speculative reads of any field of CQE */
3750 rmb();
3751
3752 qp = cqe_get_qp(cqe);
3753 if (!qp) {
3754 WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe);
3755 break;
3756 }
3757
3758 wc->qp = &qp->ibqp;
3759
3760 switch (cqe_get_type(cqe)) {
3761 case RDMA_CQE_TYPE_REQUESTER:
3762 cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc,
3763 &cqe->req);
3764 try_consume_req_cqe(cq, qp, &cqe->req, &update);
3765 break;
3766 case RDMA_CQE_TYPE_RESPONDER_RQ:
3767 cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc,
3768 &cqe->resp, &update);
3769 break;
3770 case RDMA_CQE_TYPE_INVALID:
3771 default:
3772 DP_ERR(dev, "Error: invalid CQE type = %d\n",
3773 cqe_get_type(cqe));
3774 }
3775 num_entries -= cnt;
3776 wc += cnt;
3777 done += cnt;
3778
3779 cqe = get_cqe(cq);
3780 }
3781 new_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
3782
3783 cq->cq_cons += new_cons - old_cons;
3784
3785 if (update)
3786 /* doorbell notifies abount latest VALID entry,
3787 * but chain already point to the next INVALID one
3788 */
3789 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
3790
3791 spin_unlock_irqrestore(&cq->cq_lock, flags);
3792 return done;
3793 }
3794
3795 int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags,
3796 u8 port_num,
3797 const struct ib_wc *in_wc,
3798 const struct ib_grh *in_grh,
3799 const struct ib_mad_hdr *mad_hdr,
3800 size_t in_mad_size, struct ib_mad_hdr *out_mad,
3801 size_t *out_mad_size, u16 *out_mad_pkey_index)
3802 {
3803 struct qedr_dev *dev = get_qedr_dev(ibdev);
3804
3805 DP_DEBUG(dev, QEDR_MSG_GSI,
3806 "QEDR_PROCESS_MAD in_mad %x %x %x %x %x %x %x %x\n",
3807 mad_hdr->attr_id, mad_hdr->base_version, mad_hdr->attr_mod,
3808 mad_hdr->class_specific, mad_hdr->class_version,
3809 mad_hdr->method, mad_hdr->mgmt_class, mad_hdr->status);
3810 return IB_MAD_RESULT_SUCCESS;
3811 }
CRIS linux kernel tree