?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'block-5.11-2021-01-10' of git://git.kernel.dk/linux-block
[linux/fpc-iii.git] / drivers / infiniband / hw / mlx5 / wr.c
blobd6038fb6c50c655aa3048479ef6ad6d343113397
1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3 * Copyright (c) 2020, Mellanox Technologies inc. All rights reserved.
4 */
5
6 #include <linux/gfp.h>
7 #include <linux/mlx5/qp.h>
8 #include <linux/mlx5/driver.h>
9 #include "wr.h"
10
11 static const u32 mlx5_ib_opcode[] = {
12 [IB_WR_SEND] = MLX5_OPCODE_SEND,
13 [IB_WR_LSO] = MLX5_OPCODE_LSO,
14 [IB_WR_SEND_WITH_IMM] = MLX5_OPCODE_SEND_IMM,
15 [IB_WR_RDMA_WRITE] = MLX5_OPCODE_RDMA_WRITE,
16 [IB_WR_RDMA_WRITE_WITH_IMM] = MLX5_OPCODE_RDMA_WRITE_IMM,
17 [IB_WR_RDMA_READ] = MLX5_OPCODE_RDMA_READ,
18 [IB_WR_ATOMIC_CMP_AND_SWP] = MLX5_OPCODE_ATOMIC_CS,
19 [IB_WR_ATOMIC_FETCH_AND_ADD] = MLX5_OPCODE_ATOMIC_FA,
20 [IB_WR_SEND_WITH_INV] = MLX5_OPCODE_SEND_INVAL,
21 [IB_WR_LOCAL_INV] = MLX5_OPCODE_UMR,
22 [IB_WR_REG_MR] = MLX5_OPCODE_UMR,
23 [IB_WR_MASKED_ATOMIC_CMP_AND_SWP] = MLX5_OPCODE_ATOMIC_MASKED_CS,
24 [IB_WR_MASKED_ATOMIC_FETCH_AND_ADD] = MLX5_OPCODE_ATOMIC_MASKED_FA,
25 [MLX5_IB_WR_UMR] = MLX5_OPCODE_UMR,
26 };
27
28 /* handle_post_send_edge - Check if we get to SQ edge. If yes, update to the
29 * next nearby edge and get new address translation for current WQE position.
30 * @sq - SQ buffer.
31 * @seg: Current WQE position (16B aligned).
32 * @wqe_sz: Total current WQE size [16B].
33 * @cur_edge: Updated current edge.
34 */
35 static inline void handle_post_send_edge(struct mlx5_ib_wq *sq, void **seg,
36 u32 wqe_sz, void **cur_edge)
37 {
38 u32 idx;
39
40 if (likely(*seg != *cur_edge))
41 return;
42
43 idx = (sq->cur_post + (wqe_sz >> 2)) & (sq->wqe_cnt - 1);
44 *cur_edge = get_sq_edge(sq, idx);
45
46 *seg = mlx5_frag_buf_get_wqe(&sq->fbc, idx);
47 }
48
49 /* memcpy_send_wqe - copy data from src to WQE and update the relevant WQ's
50 * pointers. At the end @seg is aligned to 16B regardless the copied size.
51 * @sq - SQ buffer.
52 * @cur_edge: Updated current edge.
53 * @seg: Current WQE position (16B aligned).
54 * @wqe_sz: Total current WQE size [16B].
55 * @src: Pointer to copy from.
56 * @n: Number of bytes to copy.
57 */
58 static inline void memcpy_send_wqe(struct mlx5_ib_wq *sq, void **cur_edge,
59 void **seg, u32 *wqe_sz, const void *src,
60 size_t n)
61 {
62 while (likely(n)) {
63 size_t leftlen = *cur_edge - *seg;
64 size_t copysz = min_t(size_t, leftlen, n);
65 size_t stride;
66
67 memcpy(*seg, src, copysz);
68
69 n -= copysz;
70 src += copysz;
71 stride = !n ? ALIGN(copysz, 16) : copysz;
72 *seg += stride;
73 *wqe_sz += stride >> 4;
74 handle_post_send_edge(sq, seg, *wqe_sz, cur_edge);
75 }
76 }
77
78 static int mlx5_wq_overflow(struct mlx5_ib_wq *wq, int nreq,
79 struct ib_cq *ib_cq)
80 {
81 struct mlx5_ib_cq *cq;
82 unsigned int cur;
83
84 cur = wq->head - wq->tail;
85 if (likely(cur + nreq < wq->max_post))
86 return 0;
87
88 cq = to_mcq(ib_cq);
89 spin_lock(&cq->lock);
90 cur = wq->head - wq->tail;
91 spin_unlock(&cq->lock);
92
93 return cur + nreq >= wq->max_post;
94 }
95
96 static __always_inline void set_raddr_seg(struct mlx5_wqe_raddr_seg *rseg,
97 u64 remote_addr, u32 rkey)
98 {
99 rseg->raddr = cpu_to_be64(remote_addr);
100 rseg->rkey = cpu_to_be32(rkey);
101 rseg->reserved = 0;
102 }
103
104 static void set_eth_seg(const struct ib_send_wr *wr, struct mlx5_ib_qp *qp,
105 void **seg, int *size, void **cur_edge)
106 {
107 struct mlx5_wqe_eth_seg *eseg = *seg;
108
109 memset(eseg, 0, sizeof(struct mlx5_wqe_eth_seg));
110
111 if (wr->send_flags & IB_SEND_IP_CSUM)
112 eseg->cs_flags = MLX5_ETH_WQE_L3_CSUM |
113 MLX5_ETH_WQE_L4_CSUM;
114
115 if (wr->opcode == IB_WR_LSO) {
116 struct ib_ud_wr *ud_wr = container_of(wr, struct ib_ud_wr, wr);
117 size_t left, copysz;
118 void *pdata = ud_wr->header;
119 size_t stride;
120
121 left = ud_wr->hlen;
122 eseg->mss = cpu_to_be16(ud_wr->mss);
123 eseg->inline_hdr.sz = cpu_to_be16(left);
124
125 /* memcpy_send_wqe should get a 16B align address. Hence, we
126 * first copy up to the current edge and then, if needed,
127 * continue to memcpy_send_wqe.
128 */
129 copysz = min_t(u64, *cur_edge - (void *)eseg->inline_hdr.start,
130 left);
131 memcpy(eseg->inline_hdr.start, pdata, copysz);
132 stride = ALIGN(sizeof(struct mlx5_wqe_eth_seg) -
133 sizeof(eseg->inline_hdr.start) + copysz, 16);
134 *size += stride / 16;
135 *seg += stride;
136
137 if (copysz < left) {
138 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
139 left -= copysz;
140 pdata += copysz;
141 memcpy_send_wqe(&qp->sq, cur_edge, seg, size, pdata,
142 left);
143 }
144
145 return;
146 }
147
148 *seg += sizeof(struct mlx5_wqe_eth_seg);
149 *size += sizeof(struct mlx5_wqe_eth_seg) / 16;
150 }
151
152 static void set_datagram_seg(struct mlx5_wqe_datagram_seg *dseg,
153 const struct ib_send_wr *wr)
154 {
155 memcpy(&dseg->av, &to_mah(ud_wr(wr)->ah)->av, sizeof(struct mlx5_av));
156 dseg->av.dqp_dct =
157 cpu_to_be32(ud_wr(wr)->remote_qpn | MLX5_EXTENDED_UD_AV);
158 dseg->av.key.qkey.qkey = cpu_to_be32(ud_wr(wr)->remote_qkey);
159 }
160
161 static void set_data_ptr_seg(struct mlx5_wqe_data_seg *dseg, struct ib_sge *sg)
162 {
163 dseg->byte_count = cpu_to_be32(sg->length);
164 dseg->lkey = cpu_to_be32(sg->lkey);
165 dseg->addr = cpu_to_be64(sg->addr);
166 }
167
168 static u64 get_xlt_octo(u64 bytes)
169 {
170 return ALIGN(bytes, MLX5_IB_UMR_XLT_ALIGNMENT) /
171 MLX5_IB_UMR_OCTOWORD;
172 }
173
174 static __be64 frwr_mkey_mask(bool atomic)
175 {
176 u64 result;
177
178 result = MLX5_MKEY_MASK_LEN |
179 MLX5_MKEY_MASK_PAGE_SIZE |
180 MLX5_MKEY_MASK_START_ADDR |
181 MLX5_MKEY_MASK_EN_RINVAL |
182 MLX5_MKEY_MASK_KEY |
183 MLX5_MKEY_MASK_LR |
184 MLX5_MKEY_MASK_LW |
185 MLX5_MKEY_MASK_RR |
186 MLX5_MKEY_MASK_RW |
187 MLX5_MKEY_MASK_SMALL_FENCE |
188 MLX5_MKEY_MASK_FREE;
189
190 if (atomic)
191 result |= MLX5_MKEY_MASK_A;
192
193 return cpu_to_be64(result);
194 }
195
196 static __be64 sig_mkey_mask(void)
197 {
198 u64 result;
199
200 result = MLX5_MKEY_MASK_LEN |
201 MLX5_MKEY_MASK_PAGE_SIZE |
202 MLX5_MKEY_MASK_START_ADDR |
203 MLX5_MKEY_MASK_EN_SIGERR |
204 MLX5_MKEY_MASK_EN_RINVAL |
205 MLX5_MKEY_MASK_KEY |
206 MLX5_MKEY_MASK_LR |
207 MLX5_MKEY_MASK_LW |
208 MLX5_MKEY_MASK_RR |
209 MLX5_MKEY_MASK_RW |
210 MLX5_MKEY_MASK_SMALL_FENCE |
211 MLX5_MKEY_MASK_FREE |
212 MLX5_MKEY_MASK_BSF_EN;
213
214 return cpu_to_be64(result);
215 }
216
217 static void set_reg_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr,
218 struct mlx5_ib_mr *mr, u8 flags, bool atomic)
219 {
220 int size = (mr->ndescs + mr->meta_ndescs) * mr->desc_size;
221
222 memset(umr, 0, sizeof(*umr));
223
224 umr->flags = flags;
225 umr->xlt_octowords = cpu_to_be16(get_xlt_octo(size));
226 umr->mkey_mask = frwr_mkey_mask(atomic);
227 }
228
229 static void set_linv_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr)
230 {
231 memset(umr, 0, sizeof(*umr));
232 umr->mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE);
233 umr->flags = MLX5_UMR_INLINE;
234 }
235
236 static __be64 get_umr_enable_mr_mask(void)
237 {
238 u64 result;
239
240 result = MLX5_MKEY_MASK_KEY |
241 MLX5_MKEY_MASK_FREE;
242
243 return cpu_to_be64(result);
244 }
245
246 static __be64 get_umr_disable_mr_mask(void)
247 {
248 u64 result;
249
250 result = MLX5_MKEY_MASK_FREE;
251
252 return cpu_to_be64(result);
253 }
254
255 static __be64 get_umr_update_translation_mask(void)
256 {
257 u64 result;
258
259 result = MLX5_MKEY_MASK_LEN |
260 MLX5_MKEY_MASK_PAGE_SIZE |
261 MLX5_MKEY_MASK_START_ADDR;
262
263 return cpu_to_be64(result);
264 }
265
266 static __be64 get_umr_update_access_mask(int atomic,
267 int relaxed_ordering_write,
268 int relaxed_ordering_read)
269 {
270 u64 result;
271
272 result = MLX5_MKEY_MASK_LR |
273 MLX5_MKEY_MASK_LW |
274 MLX5_MKEY_MASK_RR |
275 MLX5_MKEY_MASK_RW;
276
277 if (atomic)
278 result |= MLX5_MKEY_MASK_A;
279
280 if (relaxed_ordering_write)
281 result |= MLX5_MKEY_MASK_RELAXED_ORDERING_WRITE;
282
283 if (relaxed_ordering_read)
284 result |= MLX5_MKEY_MASK_RELAXED_ORDERING_READ;
285
286 return cpu_to_be64(result);
287 }
288
289 static __be64 get_umr_update_pd_mask(void)
290 {
291 u64 result;
292
293 result = MLX5_MKEY_MASK_PD;
294
295 return cpu_to_be64(result);
296 }
297
298 static int umr_check_mkey_mask(struct mlx5_ib_dev *dev, u64 mask)
299 {
300 if (mask & MLX5_MKEY_MASK_PAGE_SIZE &&
301 MLX5_CAP_GEN(dev->mdev, umr_modify_entity_size_disabled))
302 return -EPERM;
303
304 if (mask & MLX5_MKEY_MASK_A &&
305 MLX5_CAP_GEN(dev->mdev, umr_modify_atomic_disabled))
306 return -EPERM;
307
308 if (mask & MLX5_MKEY_MASK_RELAXED_ORDERING_WRITE &&
309 !MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write_umr))
310 return -EPERM;
311
312 if (mask & MLX5_MKEY_MASK_RELAXED_ORDERING_READ &&
313 !MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read_umr))
314 return -EPERM;
315
316 return 0;
317 }
318
319 static int set_reg_umr_segment(struct mlx5_ib_dev *dev,
320 struct mlx5_wqe_umr_ctrl_seg *umr,
321 const struct ib_send_wr *wr)
322 {
323 const struct mlx5_umr_wr *umrwr = umr_wr(wr);
324
325 memset(umr, 0, sizeof(*umr));
326
327 if (!umrwr->ignore_free_state) {
328 if (wr->send_flags & MLX5_IB_SEND_UMR_FAIL_IF_FREE)
329 /* fail if free */
330 umr->flags = MLX5_UMR_CHECK_FREE;
331 else
332 /* fail if not free */
333 umr->flags = MLX5_UMR_CHECK_NOT_FREE;
334 }
335
336 umr->xlt_octowords = cpu_to_be16(get_xlt_octo(umrwr->xlt_size));
337 if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_XLT) {
338 u64 offset = get_xlt_octo(umrwr->offset);
339
340 umr->xlt_offset = cpu_to_be16(offset & 0xffff);
341 umr->xlt_offset_47_16 = cpu_to_be32(offset >> 16);
342 umr->flags |= MLX5_UMR_TRANSLATION_OFFSET_EN;
343 }
344 if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_TRANSLATION)
345 umr->mkey_mask |= get_umr_update_translation_mask();
346 if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS) {
347 umr->mkey_mask |= get_umr_update_access_mask(
348 !!(MLX5_CAP_GEN(dev->mdev, atomic)),
349 !!(MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write_umr)),
350 !!(MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read_umr)));
351 umr->mkey_mask |= get_umr_update_pd_mask();
352 }
353 if (wr->send_flags & MLX5_IB_SEND_UMR_ENABLE_MR)
354 umr->mkey_mask |= get_umr_enable_mr_mask();
355 if (wr->send_flags & MLX5_IB_SEND_UMR_DISABLE_MR)
356 umr->mkey_mask |= get_umr_disable_mr_mask();
357
358 if (!wr->num_sge)
359 umr->flags |= MLX5_UMR_INLINE;
360
361 return umr_check_mkey_mask(dev, be64_to_cpu(umr->mkey_mask));
362 }
363
364 static u8 get_umr_flags(int acc)
365 {
366 return (acc & IB_ACCESS_REMOTE_ATOMIC ? MLX5_PERM_ATOMIC : 0) |
367 (acc & IB_ACCESS_REMOTE_WRITE ? MLX5_PERM_REMOTE_WRITE : 0) |
368 (acc & IB_ACCESS_REMOTE_READ ? MLX5_PERM_REMOTE_READ : 0) |
369 (acc & IB_ACCESS_LOCAL_WRITE ? MLX5_PERM_LOCAL_WRITE : 0) |
370 MLX5_PERM_LOCAL_READ | MLX5_PERM_UMR_EN;
371 }
372
373 static void set_reg_mkey_seg(struct mlx5_mkey_seg *seg,
374 struct mlx5_ib_mr *mr,
375 u32 key, int access)
376 {
377 int ndescs = ALIGN(mr->ndescs + mr->meta_ndescs, 8) >> 1;
378
379 memset(seg, 0, sizeof(*seg));
380
381 if (mr->access_mode == MLX5_MKC_ACCESS_MODE_MTT)
382 seg->log2_page_size = ilog2(mr->ibmr.page_size);
383 else if (mr->access_mode == MLX5_MKC_ACCESS_MODE_KLMS)
384 /* KLMs take twice the size of MTTs */
385 ndescs *= 2;
386
387 seg->flags = get_umr_flags(access) | mr->access_mode;
388 seg->qpn_mkey7_0 = cpu_to_be32((key & 0xff) | 0xffffff00);
389 seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL);
390 seg->start_addr = cpu_to_be64(mr->ibmr.iova);
391 seg->len = cpu_to_be64(mr->ibmr.length);
392 seg->xlt_oct_size = cpu_to_be32(ndescs);
393 }
394
395 static void set_linv_mkey_seg(struct mlx5_mkey_seg *seg)
396 {
397 memset(seg, 0, sizeof(*seg));
398 seg->status = MLX5_MKEY_STATUS_FREE;
399 }
400
401 static void set_reg_mkey_segment(struct mlx5_ib_dev *dev,
402 struct mlx5_mkey_seg *seg,
403 const struct ib_send_wr *wr)
404 {
405 const struct mlx5_umr_wr *umrwr = umr_wr(wr);
406
407 memset(seg, 0, sizeof(*seg));
408 if (wr->send_flags & MLX5_IB_SEND_UMR_DISABLE_MR)
409 MLX5_SET(mkc, seg, free, 1);
410
411 MLX5_SET(mkc, seg, a,
412 !!(umrwr->access_flags & IB_ACCESS_REMOTE_ATOMIC));
413 MLX5_SET(mkc, seg, rw,
414 !!(umrwr->access_flags & IB_ACCESS_REMOTE_WRITE));
415 MLX5_SET(mkc, seg, rr, !!(umrwr->access_flags & IB_ACCESS_REMOTE_READ));
416 MLX5_SET(mkc, seg, lw, !!(umrwr->access_flags & IB_ACCESS_LOCAL_WRITE));
417 MLX5_SET(mkc, seg, lr, 1);
418 if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write_umr))
419 MLX5_SET(mkc, seg, relaxed_ordering_write,
420 !!(umrwr->access_flags & IB_ACCESS_RELAXED_ORDERING));
421 if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read_umr))
422 MLX5_SET(mkc, seg, relaxed_ordering_read,
423 !!(umrwr->access_flags & IB_ACCESS_RELAXED_ORDERING));
424
425 if (umrwr->pd)
426 MLX5_SET(mkc, seg, pd, to_mpd(umrwr->pd)->pdn);
427 if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_TRANSLATION &&
428 !umrwr->length)
429 MLX5_SET(mkc, seg, length64, 1);
430
431 MLX5_SET64(mkc, seg, start_addr, umrwr->virt_addr);
432 MLX5_SET64(mkc, seg, len, umrwr->length);
433 MLX5_SET(mkc, seg, log_page_size, umrwr->page_shift);
434 MLX5_SET(mkc, seg, qpn, 0xffffff);
435 MLX5_SET(mkc, seg, mkey_7_0, mlx5_mkey_variant(umrwr->mkey));
436 }
437
438 static void set_reg_data_seg(struct mlx5_wqe_data_seg *dseg,
439 struct mlx5_ib_mr *mr,
440 struct mlx5_ib_pd *pd)
441 {
442 int bcount = mr->desc_size * (mr->ndescs + mr->meta_ndescs);
443
444 dseg->addr = cpu_to_be64(mr->desc_map);
445 dseg->byte_count = cpu_to_be32(ALIGN(bcount, 64));
446 dseg->lkey = cpu_to_be32(pd->ibpd.local_dma_lkey);
447 }
448
449 static __be32 send_ieth(const struct ib_send_wr *wr)
450 {
451 switch (wr->opcode) {
452 case IB_WR_SEND_WITH_IMM:
453 case IB_WR_RDMA_WRITE_WITH_IMM:
454 return wr->ex.imm_data;
455
456 case IB_WR_SEND_WITH_INV:
457 return cpu_to_be32(wr->ex.invalidate_rkey);
458
459 default:
460 return 0;
461 }
462 }
463
464 static u8 calc_sig(void *wqe, int size)
465 {
466 u8 *p = wqe;
467 u8 res = 0;
468 int i;
469
470 for (i = 0; i < size; i++)
471 res ^= p[i];
472
473 return ~res;
474 }
475
476 static u8 wq_sig(void *wqe)
477 {
478 return calc_sig(wqe, (*((u8 *)wqe + 8) & 0x3f) << 4);
479 }
480
481 static int set_data_inl_seg(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr,
482 void **wqe, int *wqe_sz, void **cur_edge)
483 {
484 struct mlx5_wqe_inline_seg *seg;
485 size_t offset;
486 int inl = 0;
487 int i;
488
489 seg = *wqe;
490 *wqe += sizeof(*seg);
491 offset = sizeof(*seg);
492
493 for (i = 0; i < wr->num_sge; i++) {
494 size_t len = wr->sg_list[i].length;
495 void *addr = (void *)(unsigned long)(wr->sg_list[i].addr);
496
497 inl += len;
498
499 if (unlikely(inl > qp->max_inline_data))
500 return -ENOMEM;
501
502 while (likely(len)) {
503 size_t leftlen;
504 size_t copysz;
505
506 handle_post_send_edge(&qp->sq, wqe,
507 *wqe_sz + (offset >> 4),
508 cur_edge);
509
510 leftlen = *cur_edge - *wqe;
511 copysz = min_t(size_t, leftlen, len);
512
513 memcpy(*wqe, addr, copysz);
514 len -= copysz;
515 addr += copysz;
516 *wqe += copysz;
517 offset += copysz;
518 }
519 }
520
521 seg->byte_count = cpu_to_be32(inl | MLX5_INLINE_SEG);
522
523 *wqe_sz += ALIGN(inl + sizeof(seg->byte_count), 16) / 16;
524
525 return 0;
526 }
527
528 static u16 prot_field_size(enum ib_signature_type type)
529 {
530 switch (type) {
531 case IB_SIG_TYPE_T10_DIF:
532 return MLX5_DIF_SIZE;
533 default:
534 return 0;
535 }
536 }
537
538 static u8 bs_selector(int block_size)
539 {
540 switch (block_size) {
541 case 512: return 0x1;
542 case 520: return 0x2;
543 case 4096: return 0x3;
544 case 4160: return 0x4;
545 case 1073741824: return 0x5;
546 default: return 0;
547 }
548 }
549
550 static void mlx5_fill_inl_bsf(struct ib_sig_domain *domain,
551 struct mlx5_bsf_inl *inl)
552 {
553 /* Valid inline section and allow BSF refresh */
554 inl->vld_refresh = cpu_to_be16(MLX5_BSF_INL_VALID |
555 MLX5_BSF_REFRESH_DIF);
556 inl->dif_apptag = cpu_to_be16(domain->sig.dif.app_tag);
557 inl->dif_reftag = cpu_to_be32(domain->sig.dif.ref_tag);
558 /* repeating block */
559 inl->rp_inv_seed = MLX5_BSF_REPEAT_BLOCK;
560 inl->sig_type = domain->sig.dif.bg_type == IB_T10DIF_CRC ?
561 MLX5_DIF_CRC : MLX5_DIF_IPCS;
562
563 if (domain->sig.dif.ref_remap)
564 inl->dif_inc_ref_guard_check |= MLX5_BSF_INC_REFTAG;
565
566 if (domain->sig.dif.app_escape) {
567 if (domain->sig.dif.ref_escape)
568 inl->dif_inc_ref_guard_check |= MLX5_BSF_APPREF_ESCAPE;
569 else
570 inl->dif_inc_ref_guard_check |= MLX5_BSF_APPTAG_ESCAPE;
571 }
572
573 inl->dif_app_bitmask_check =
574 cpu_to_be16(domain->sig.dif.apptag_check_mask);
575 }
576
577 static int mlx5_set_bsf(struct ib_mr *sig_mr,
578 struct ib_sig_attrs *sig_attrs,
579 struct mlx5_bsf *bsf, u32 data_size)
580 {
581 struct mlx5_core_sig_ctx *msig = to_mmr(sig_mr)->sig;
582 struct mlx5_bsf_basic *basic = &bsf->basic;
583 struct ib_sig_domain *mem = &sig_attrs->mem;
584 struct ib_sig_domain *wire = &sig_attrs->wire;
585
586 memset(bsf, 0, sizeof(*bsf));
587
588 /* Basic + Extended + Inline */
589 basic->bsf_size_sbs = 1 << 7;
590 /* Input domain check byte mask */
591 basic->check_byte_mask = sig_attrs->check_mask;
592 basic->raw_data_size = cpu_to_be32(data_size);
593
594 /* Memory domain */
595 switch (sig_attrs->mem.sig_type) {
596 case IB_SIG_TYPE_NONE:
597 break;
598 case IB_SIG_TYPE_T10_DIF:
599 basic->mem.bs_selector = bs_selector(mem->sig.dif.pi_interval);
600 basic->m_bfs_psv = cpu_to_be32(msig->psv_memory.psv_idx);
601 mlx5_fill_inl_bsf(mem, &bsf->m_inl);
602 break;
603 default:
604 return -EINVAL;
605 }
606
607 /* Wire domain */
608 switch (sig_attrs->wire.sig_type) {
609 case IB_SIG_TYPE_NONE:
610 break;
611 case IB_SIG_TYPE_T10_DIF:
612 if (mem->sig.dif.pi_interval == wire->sig.dif.pi_interval &&
613 mem->sig_type == wire->sig_type) {
614 /* Same block structure */
615 basic->bsf_size_sbs |= 1 << 4;
616 if (mem->sig.dif.bg_type == wire->sig.dif.bg_type)
617 basic->wire.copy_byte_mask |= MLX5_CPY_GRD_MASK;
618 if (mem->sig.dif.app_tag == wire->sig.dif.app_tag)
619 basic->wire.copy_byte_mask |= MLX5_CPY_APP_MASK;
620 if (mem->sig.dif.ref_tag == wire->sig.dif.ref_tag)
621 basic->wire.copy_byte_mask |= MLX5_CPY_REF_MASK;
622 } else
623 basic->wire.bs_selector =
624 bs_selector(wire->sig.dif.pi_interval);
625
626 basic->w_bfs_psv = cpu_to_be32(msig->psv_wire.psv_idx);
627 mlx5_fill_inl_bsf(wire, &bsf->w_inl);
628 break;
629 default:
630 return -EINVAL;
631 }
632
633 return 0;
634 }
635
636
637 static int set_sig_data_segment(const struct ib_send_wr *send_wr,
638 struct ib_mr *sig_mr,
639 struct ib_sig_attrs *sig_attrs,
640 struct mlx5_ib_qp *qp, void **seg, int *size,
641 void **cur_edge)
642 {
643 struct mlx5_bsf *bsf;
644 u32 data_len;
645 u32 data_key;
646 u64 data_va;
647 u32 prot_len = 0;
648 u32 prot_key = 0;
649 u64 prot_va = 0;
650 bool prot = false;
651 int ret;
652 int wqe_size;
653 struct mlx5_ib_mr *mr = to_mmr(sig_mr);
654 struct mlx5_ib_mr *pi_mr = mr->pi_mr;
655
656 data_len = pi_mr->data_length;
657 data_key = pi_mr->ibmr.lkey;
658 data_va = pi_mr->data_iova;
659 if (pi_mr->meta_ndescs) {
660 prot_len = pi_mr->meta_length;
661 prot_key = pi_mr->ibmr.lkey;
662 prot_va = pi_mr->pi_iova;
663 prot = true;
664 }
665
666 if (!prot || (data_key == prot_key && data_va == prot_va &&
667 data_len == prot_len)) {
668 /**
669 * Source domain doesn't contain signature information
670 * or data and protection are interleaved in memory.
671 * So need construct:
672 * ------------------
673 * | data_klm |
674 * ------------------
675 * | BSF |
676 * ------------------
677 **/
678 struct mlx5_klm *data_klm = *seg;
679
680 data_klm->bcount = cpu_to_be32(data_len);
681 data_klm->key = cpu_to_be32(data_key);
682 data_klm->va = cpu_to_be64(data_va);
683 wqe_size = ALIGN(sizeof(*data_klm), 64);
684 } else {
685 /**
686 * Source domain contains signature information
687 * So need construct a strided block format:
688 * ---------------------------
689 * | stride_block_ctrl |
690 * ---------------------------
691 * | data_klm |
692 * ---------------------------
693 * | prot_klm |
694 * ---------------------------
695 * | BSF |
696 * ---------------------------
697 **/
698 struct mlx5_stride_block_ctrl_seg *sblock_ctrl;
699 struct mlx5_stride_block_entry *data_sentry;
700 struct mlx5_stride_block_entry *prot_sentry;
701 u16 block_size = sig_attrs->mem.sig.dif.pi_interval;
702 int prot_size;
703
704 sblock_ctrl = *seg;
705 data_sentry = (void *)sblock_ctrl + sizeof(*sblock_ctrl);
706 prot_sentry = (void *)data_sentry + sizeof(*data_sentry);
707
708 prot_size = prot_field_size(sig_attrs->mem.sig_type);
709 if (!prot_size) {
710 pr_err("Bad block size given: %u\n", block_size);
711 return -EINVAL;
712 }
713 sblock_ctrl->bcount_per_cycle = cpu_to_be32(block_size +
714 prot_size);
715 sblock_ctrl->op = cpu_to_be32(MLX5_STRIDE_BLOCK_OP);
716 sblock_ctrl->repeat_count = cpu_to_be32(data_len / block_size);
717 sblock_ctrl->num_entries = cpu_to_be16(2);
718
719 data_sentry->bcount = cpu_to_be16(block_size);
720 data_sentry->key = cpu_to_be32(data_key);
721 data_sentry->va = cpu_to_be64(data_va);
722 data_sentry->stride = cpu_to_be16(block_size);
723
724 prot_sentry->bcount = cpu_to_be16(prot_size);
725 prot_sentry->key = cpu_to_be32(prot_key);
726 prot_sentry->va = cpu_to_be64(prot_va);
727 prot_sentry->stride = cpu_to_be16(prot_size);
728
729 wqe_size = ALIGN(sizeof(*sblock_ctrl) + sizeof(*data_sentry) +
730 sizeof(*prot_sentry), 64);
731 }
732
733 *seg += wqe_size;
734 *size += wqe_size / 16;
735 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
736
737 bsf = *seg;
738 ret = mlx5_set_bsf(sig_mr, sig_attrs, bsf, data_len);
739 if (ret)
740 return -EINVAL;
741
742 *seg += sizeof(*bsf);
743 *size += sizeof(*bsf) / 16;
744 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
745
746 return 0;
747 }
748
749 static void set_sig_mkey_segment(struct mlx5_mkey_seg *seg,
750 struct ib_mr *sig_mr, int access_flags,
751 u32 size, u32 length, u32 pdn)
752 {
753 u32 sig_key = sig_mr->rkey;
754 u8 sigerr = to_mmr(sig_mr)->sig->sigerr_count & 1;
755
756 memset(seg, 0, sizeof(*seg));
757
758 seg->flags = get_umr_flags(access_flags) | MLX5_MKC_ACCESS_MODE_KLMS;
759 seg->qpn_mkey7_0 = cpu_to_be32((sig_key & 0xff) | 0xffffff00);
760 seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL | sigerr << 26 |
761 MLX5_MKEY_BSF_EN | pdn);
762 seg->len = cpu_to_be64(length);
763 seg->xlt_oct_size = cpu_to_be32(get_xlt_octo(size));
764 seg->bsfs_octo_size = cpu_to_be32(MLX5_MKEY_BSF_OCTO_SIZE);
765 }
766
767 static void set_sig_umr_segment(struct mlx5_wqe_umr_ctrl_seg *umr,
768 u32 size)
769 {
770 memset(umr, 0, sizeof(*umr));
771
772 umr->flags = MLX5_FLAGS_INLINE | MLX5_FLAGS_CHECK_FREE;
773 umr->xlt_octowords = cpu_to_be16(get_xlt_octo(size));
774 umr->bsf_octowords = cpu_to_be16(MLX5_MKEY_BSF_OCTO_SIZE);
775 umr->mkey_mask = sig_mkey_mask();
776 }
777
778 static int set_pi_umr_wr(const struct ib_send_wr *send_wr,
779 struct mlx5_ib_qp *qp, void **seg, int *size,
780 void **cur_edge)
781 {
782 const struct ib_reg_wr *wr = reg_wr(send_wr);
783 struct mlx5_ib_mr *sig_mr = to_mmr(wr->mr);
784 struct mlx5_ib_mr *pi_mr = sig_mr->pi_mr;
785 struct ib_sig_attrs *sig_attrs = sig_mr->ibmr.sig_attrs;
786 u32 pdn = to_mpd(qp->ibqp.pd)->pdn;
787 u32 xlt_size;
788 int region_len, ret;
789
790 if (unlikely(send_wr->num_sge != 0) ||
791 unlikely(wr->access & IB_ACCESS_REMOTE_ATOMIC) ||
792 unlikely(!sig_mr->sig) || unlikely(!qp->ibqp.integrity_en) ||
793 unlikely(!sig_mr->sig->sig_status_checked))
794 return -EINVAL;
795
796 /* length of the protected region, data + protection */
797 region_len = pi_mr->ibmr.length;
798
799 /**
800 * KLM octoword size - if protection was provided
801 * then we use strided block format (3 octowords),
802 * else we use single KLM (1 octoword)
803 **/
804 if (sig_attrs->mem.sig_type != IB_SIG_TYPE_NONE)
805 xlt_size = 0x30;
806 else
807 xlt_size = sizeof(struct mlx5_klm);
808
809 set_sig_umr_segment(*seg, xlt_size);
810 *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
811 *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
812 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
813
814 set_sig_mkey_segment(*seg, wr->mr, wr->access, xlt_size, region_len,
815 pdn);
816 *seg += sizeof(struct mlx5_mkey_seg);
817 *size += sizeof(struct mlx5_mkey_seg) / 16;
818 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
819
820 ret = set_sig_data_segment(send_wr, wr->mr, sig_attrs, qp, seg, size,
821 cur_edge);
822 if (ret)
823 return ret;
824
825 sig_mr->sig->sig_status_checked = false;
826 return 0;
827 }
828
829 static int set_psv_wr(struct ib_sig_domain *domain,
830 u32 psv_idx, void **seg, int *size)
831 {
832 struct mlx5_seg_set_psv *psv_seg = *seg;
833
834 memset(psv_seg, 0, sizeof(*psv_seg));
835 psv_seg->psv_num = cpu_to_be32(psv_idx);
836 switch (domain->sig_type) {
837 case IB_SIG_TYPE_NONE:
838 break;
839 case IB_SIG_TYPE_T10_DIF:
840 psv_seg->transient_sig = cpu_to_be32(domain->sig.dif.bg << 16 |
841 domain->sig.dif.app_tag);
842 psv_seg->ref_tag = cpu_to_be32(domain->sig.dif.ref_tag);
843 break;
844 default:
845 pr_err("Bad signature type (%d) is given.\n",
846 domain->sig_type);
847 return -EINVAL;
848 }
849
850 *seg += sizeof(*psv_seg);
851 *size += sizeof(*psv_seg) / 16;
852
853 return 0;
854 }
855
856 static int set_reg_wr(struct mlx5_ib_qp *qp,
857 const struct ib_reg_wr *wr,
858 void **seg, int *size, void **cur_edge,
859 bool check_not_free)
860 {
861 struct mlx5_ib_mr *mr = to_mmr(wr->mr);
862 struct mlx5_ib_pd *pd = to_mpd(qp->ibqp.pd);
863 struct mlx5_ib_dev *dev = to_mdev(pd->ibpd.device);
864 int mr_list_size = (mr->ndescs + mr->meta_ndescs) * mr->desc_size;
865 bool umr_inline = mr_list_size <= MLX5_IB_SQ_UMR_INLINE_THRESHOLD;
866 bool atomic = wr->access & IB_ACCESS_REMOTE_ATOMIC;
867 u8 flags = 0;
868
869 /* Matches access in mlx5_set_umr_free_mkey() */
870 if (!mlx5_ib_can_reconfig_with_umr(dev, 0, wr->access)) {
871 mlx5_ib_warn(
872 to_mdev(qp->ibqp.device),
873 "Fast update for MR access flags is not possible\n");
874 return -EINVAL;
875 }
876
877 if (unlikely(wr->wr.send_flags & IB_SEND_INLINE)) {
878 mlx5_ib_warn(to_mdev(qp->ibqp.device),
879 "Invalid IB_SEND_INLINE send flag\n");
880 return -EINVAL;
881 }
882
883 if (check_not_free)
884 flags |= MLX5_UMR_CHECK_NOT_FREE;
885 if (umr_inline)
886 flags |= MLX5_UMR_INLINE;
887
888 set_reg_umr_seg(*seg, mr, flags, atomic);
889 *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
890 *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
891 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
892
893 set_reg_mkey_seg(*seg, mr, wr->key, wr->access);
894 *seg += sizeof(struct mlx5_mkey_seg);
895 *size += sizeof(struct mlx5_mkey_seg) / 16;
896 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
897
898 if (umr_inline) {
899 memcpy_send_wqe(&qp->sq, cur_edge, seg, size, mr->descs,
900 mr_list_size);
901 *size = ALIGN(*size, MLX5_SEND_WQE_BB >> 4);
902 } else {
903 set_reg_data_seg(*seg, mr, pd);
904 *seg += sizeof(struct mlx5_wqe_data_seg);
905 *size += (sizeof(struct mlx5_wqe_data_seg) / 16);
906 }
907 return 0;
908 }
909
910 static void set_linv_wr(struct mlx5_ib_qp *qp, void **seg, int *size,
911 void **cur_edge)
912 {
913 set_linv_umr_seg(*seg);
914 *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
915 *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
916 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
917 set_linv_mkey_seg(*seg);
918 *seg += sizeof(struct mlx5_mkey_seg);
919 *size += sizeof(struct mlx5_mkey_seg) / 16;
920 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
921 }
922
923 static void dump_wqe(struct mlx5_ib_qp *qp, u32 idx, int size_16)
924 {
925 __be32 *p = NULL;
926 int i, j;
927
928 pr_debug("dump WQE index %u:\n", idx);
929 for (i = 0, j = 0; i < size_16 * 4; i += 4, j += 4) {
930 if ((i & 0xf) == 0) {
931 p = mlx5_frag_buf_get_wqe(&qp->sq.fbc, idx);
932 pr_debug("WQBB at %p:\n", (void *)p);
933 j = 0;
934 idx = (idx + 1) & (qp->sq.wqe_cnt - 1);
935 }
936 pr_debug("%08x %08x %08x %08x\n", be32_to_cpu(p[j]),
937 be32_to_cpu(p[j + 1]), be32_to_cpu(p[j + 2]),
938 be32_to_cpu(p[j + 3]));
939 }
940 }
941
942 static int __begin_wqe(struct mlx5_ib_qp *qp, void **seg,
943 struct mlx5_wqe_ctrl_seg **ctrl,
944 const struct ib_send_wr *wr, unsigned int *idx,
945 int *size, void **cur_edge, int nreq,
946 bool send_signaled, bool solicited)
947 {
948 if (unlikely(mlx5_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)))
949 return -ENOMEM;
950
951 *idx = qp->sq.cur_post & (qp->sq.wqe_cnt - 1);
952 *seg = mlx5_frag_buf_get_wqe(&qp->sq.fbc, *idx);
953 *ctrl = *seg;
954 *(uint32_t *)(*seg + 8) = 0;
955 (*ctrl)->imm = send_ieth(wr);
956 (*ctrl)->fm_ce_se = qp->sq_signal_bits |
957 (send_signaled ? MLX5_WQE_CTRL_CQ_UPDATE : 0) |
958 (solicited ? MLX5_WQE_CTRL_SOLICITED : 0);
959
960 *seg += sizeof(**ctrl);
961 *size = sizeof(**ctrl) / 16;
962 *cur_edge = qp->sq.cur_edge;
963
964 return 0;
965 }
966
967 static int begin_wqe(struct mlx5_ib_qp *qp, void **seg,
968 struct mlx5_wqe_ctrl_seg **ctrl,
969 const struct ib_send_wr *wr, unsigned int *idx, int *size,
970 void **cur_edge, int nreq)
971 {
972 return __begin_wqe(qp, seg, ctrl, wr, idx, size, cur_edge, nreq,
973 wr->send_flags & IB_SEND_SIGNALED,
974 wr->send_flags & IB_SEND_SOLICITED);
975 }
976
977 static void finish_wqe(struct mlx5_ib_qp *qp,
978 struct mlx5_wqe_ctrl_seg *ctrl,
979 void *seg, u8 size, void *cur_edge,
980 unsigned int idx, u64 wr_id, int nreq, u8 fence,
981 u32 mlx5_opcode)
982 {
983 u8 opmod = 0;
984
985 ctrl->opmod_idx_opcode = cpu_to_be32(((u32)(qp->sq.cur_post) << 8) |
986 mlx5_opcode | ((u32)opmod << 24));
987 ctrl->qpn_ds = cpu_to_be32(size | (qp->trans_qp.base.mqp.qpn << 8));
988 ctrl->fm_ce_se |= fence;
989 if (unlikely(qp->flags_en & MLX5_QP_FLAG_SIGNATURE))
990 ctrl->signature = wq_sig(ctrl);
991
992 qp->sq.wrid[idx] = wr_id;
993 qp->sq.w_list[idx].opcode = mlx5_opcode;
994 qp->sq.wqe_head[idx] = qp->sq.head + nreq;
995 qp->sq.cur_post += DIV_ROUND_UP(size * 16, MLX5_SEND_WQE_BB);
996 qp->sq.w_list[idx].next = qp->sq.cur_post;
997
998 /* We save the edge which was possibly updated during the WQE
999 * construction, into SQ's cache.
1000 */
1001 seg = PTR_ALIGN(seg, MLX5_SEND_WQE_BB);
1002 qp->sq.cur_edge = (unlikely(seg == cur_edge)) ?
1003 get_sq_edge(&qp->sq, qp->sq.cur_post &
1004 (qp->sq.wqe_cnt - 1)) :
1005 cur_edge;
1006 }
1007
1008 static void handle_rdma_op(const struct ib_send_wr *wr, void **seg, int *size)
1009 {
1010 set_raddr_seg(*seg, rdma_wr(wr)->remote_addr, rdma_wr(wr)->rkey);
1011 *seg += sizeof(struct mlx5_wqe_raddr_seg);
1012 *size += sizeof(struct mlx5_wqe_raddr_seg) / 16;
1013 }
1014
1015 static void handle_local_inv(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr,
1016 struct mlx5_wqe_ctrl_seg **ctrl, void **seg,
1017 int *size, void **cur_edge, unsigned int idx)
1018 {
1019 qp->sq.wr_data[idx] = IB_WR_LOCAL_INV;
1020 (*ctrl)->imm = cpu_to_be32(wr->ex.invalidate_rkey);
1021 set_linv_wr(qp, seg, size, cur_edge);
1022 }
1023
1024 static int handle_reg_mr(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr,
1025 struct mlx5_wqe_ctrl_seg **ctrl, void **seg, int *size,
1026 void **cur_edge, unsigned int idx)
1027 {
1028 qp->sq.wr_data[idx] = IB_WR_REG_MR;
1029 (*ctrl)->imm = cpu_to_be32(reg_wr(wr)->key);
1030 return set_reg_wr(qp, reg_wr(wr), seg, size, cur_edge, true);
1031 }
1032
1033 static int handle_psv(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
1034 const struct ib_send_wr *wr,
1035 struct mlx5_wqe_ctrl_seg **ctrl, void **seg, int *size,
1036 void **cur_edge, unsigned int *idx, int nreq,
1037 struct ib_sig_domain *domain, u32 psv_index,
1038 u8 next_fence)
1039 {
1040 int err;
1041
1042 /*
1043 * SET_PSV WQEs are not signaled and solicited on error.
1044 */
1045 err = __begin_wqe(qp, seg, ctrl, wr, idx, size, cur_edge, nreq,
1046 false, true);
1047 if (unlikely(err)) {
1048 mlx5_ib_warn(dev, "\n");
1049 err = -ENOMEM;
1050 goto out;
1051 }
1052 err = set_psv_wr(domain, psv_index, seg, size);
1053 if (unlikely(err)) {
1054 mlx5_ib_warn(dev, "\n");
1055 goto out;
1056 }
1057 finish_wqe(qp, *ctrl, *seg, *size, *cur_edge, *idx, wr->wr_id, nreq,
1058 next_fence, MLX5_OPCODE_SET_PSV);
1059
1060 out:
1061 return err;
1062 }
1063
1064 static int handle_reg_mr_integrity(struct mlx5_ib_dev *dev,
1065 struct mlx5_ib_qp *qp,
1066 const struct ib_send_wr *wr,
1067 struct mlx5_wqe_ctrl_seg **ctrl, void **seg,
1068 int *size, void **cur_edge,
1069 unsigned int *idx, int nreq, u8 fence,
1070 u8 next_fence)
1071 {
1072 struct mlx5_ib_mr *mr;
1073 struct mlx5_ib_mr *pi_mr;
1074 struct mlx5_ib_mr pa_pi_mr;
1075 struct ib_sig_attrs *sig_attrs;
1076 struct ib_reg_wr reg_pi_wr;
1077 int err;
1078
1079 qp->sq.wr_data[*idx] = IB_WR_REG_MR_INTEGRITY;
1080
1081 mr = to_mmr(reg_wr(wr)->mr);
1082 pi_mr = mr->pi_mr;
1083
1084 if (pi_mr) {
1085 memset(&reg_pi_wr, 0,
1086 sizeof(struct ib_reg_wr));
1087
1088 reg_pi_wr.mr = &pi_mr->ibmr;
1089 reg_pi_wr.access = reg_wr(wr)->access;
1090 reg_pi_wr.key = pi_mr->ibmr.rkey;
1091
1092 (*ctrl)->imm = cpu_to_be32(reg_pi_wr.key);
1093 /* UMR for data + prot registration */
1094 err = set_reg_wr(qp, &reg_pi_wr, seg, size, cur_edge, false);
1095 if (unlikely(err))
1096 goto out;
1097
1098 finish_wqe(qp, *ctrl, *seg, *size, *cur_edge, *idx, wr->wr_id,
1099 nreq, fence, MLX5_OPCODE_UMR);
1100
1101 err = begin_wqe(qp, seg, ctrl, wr, idx, size, cur_edge, nreq);
1102 if (unlikely(err)) {
1103 mlx5_ib_warn(dev, "\n");
1104 err = -ENOMEM;
1105 goto out;
1106 }
1107 } else {
1108 memset(&pa_pi_mr, 0, sizeof(struct mlx5_ib_mr));
1109 /* No UMR, use local_dma_lkey */
1110 pa_pi_mr.ibmr.lkey = mr->ibmr.pd->local_dma_lkey;
1111 pa_pi_mr.ndescs = mr->ndescs;
1112 pa_pi_mr.data_length = mr->data_length;
1113 pa_pi_mr.data_iova = mr->data_iova;
1114 if (mr->meta_ndescs) {
1115 pa_pi_mr.meta_ndescs = mr->meta_ndescs;
1116 pa_pi_mr.meta_length = mr->meta_length;
1117 pa_pi_mr.pi_iova = mr->pi_iova;
1118 }
1119
1120 pa_pi_mr.ibmr.length = mr->ibmr.length;
1121 mr->pi_mr = &pa_pi_mr;
1122 }
1123 (*ctrl)->imm = cpu_to_be32(mr->ibmr.rkey);
1124 /* UMR for sig MR */
1125 err = set_pi_umr_wr(wr, qp, seg, size, cur_edge);
1126 if (unlikely(err)) {
1127 mlx5_ib_warn(dev, "\n");
1128 goto out;
1129 }
1130 finish_wqe(qp, *ctrl, *seg, *size, *cur_edge, *idx, wr->wr_id, nreq,
1131 fence, MLX5_OPCODE_UMR);
1132
1133 sig_attrs = mr->ibmr.sig_attrs;
1134 err = handle_psv(dev, qp, wr, ctrl, seg, size, cur_edge, idx, nreq,
1135 &sig_attrs->mem, mr->sig->psv_memory.psv_idx,
1136 next_fence);
1137 if (unlikely(err))
1138 goto out;
1139
1140 err = handle_psv(dev, qp, wr, ctrl, seg, size, cur_edge, idx, nreq,
1141 &sig_attrs->wire, mr->sig->psv_wire.psv_idx,
1142 next_fence);
1143 if (unlikely(err))
1144 goto out;
1145
1146 qp->next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;
1147
1148 out:
1149 return err;
1150 }
1151
1152 static int handle_qpt_rc(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
1153 const struct ib_send_wr *wr,
1154 struct mlx5_wqe_ctrl_seg **ctrl, void **seg, int *size,
1155 void **cur_edge, unsigned int *idx, int nreq, u8 fence,
1156 u8 next_fence, int *num_sge)
1157 {
1158 int err = 0;
1159
1160 switch (wr->opcode) {
1161 case IB_WR_RDMA_READ:
1162 case IB_WR_RDMA_WRITE:
1163 case IB_WR_RDMA_WRITE_WITH_IMM:
1164 handle_rdma_op(wr, seg, size);
1165 break;
1166
1167 case IB_WR_ATOMIC_CMP_AND_SWP:
1168 case IB_WR_ATOMIC_FETCH_AND_ADD:
1169 case IB_WR_MASKED_ATOMIC_CMP_AND_SWP:
1170 mlx5_ib_warn(dev, "Atomic operations are not supported yet\n");
1171 err = -EOPNOTSUPP;
1172 goto out;
1173
1174 case IB_WR_LOCAL_INV:
1175 handle_local_inv(qp, wr, ctrl, seg, size, cur_edge, *idx);
1176 *num_sge = 0;
1177 break;
1178
1179 case IB_WR_REG_MR:
1180 err = handle_reg_mr(qp, wr, ctrl, seg, size, cur_edge, *idx);
1181 if (unlikely(err))
1182 goto out;
1183 *num_sge = 0;
1184 break;
1185
1186 case IB_WR_REG_MR_INTEGRITY:
1187 err = handle_reg_mr_integrity(dev, qp, wr, ctrl, seg, size,
1188 cur_edge, idx, nreq, fence,
1189 next_fence);
1190 if (unlikely(err))
1191 goto out;
1192 *num_sge = 0;
1193 break;
1194
1195 default:
1196 break;
1197 }
1198
1199 out:
1200 return err;
1201 }
1202
1203 static void handle_qpt_uc(const struct ib_send_wr *wr, void **seg, int *size)
1204 {
1205 switch (wr->opcode) {
1206 case IB_WR_RDMA_WRITE:
1207 case IB_WR_RDMA_WRITE_WITH_IMM:
1208 handle_rdma_op(wr, seg, size);
1209 break;
1210 default:
1211 break;
1212 }
1213 }
1214
1215 static void handle_qpt_hw_gsi(struct mlx5_ib_qp *qp,
1216 const struct ib_send_wr *wr, void **seg,
1217 int *size, void **cur_edge)
1218 {
1219 set_datagram_seg(*seg, wr);
1220 *seg += sizeof(struct mlx5_wqe_datagram_seg);
1221 *size += sizeof(struct mlx5_wqe_datagram_seg) / 16;
1222 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
1223 }
1224
1225 static void handle_qpt_ud(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr,
1226 void **seg, int *size, void **cur_edge)
1227 {
1228 set_datagram_seg(*seg, wr);
1229 *seg += sizeof(struct mlx5_wqe_datagram_seg);
1230 *size += sizeof(struct mlx5_wqe_datagram_seg) / 16;
1231 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
1232
1233 /* handle qp that supports ud offload */
1234 if (qp->flags & IB_QP_CREATE_IPOIB_UD_LSO) {
1235 struct mlx5_wqe_eth_pad *pad;
1236
1237 pad = *seg;
1238 memset(pad, 0, sizeof(struct mlx5_wqe_eth_pad));
1239 *seg += sizeof(struct mlx5_wqe_eth_pad);
1240 *size += sizeof(struct mlx5_wqe_eth_pad) / 16;
1241 set_eth_seg(wr, qp, seg, size, cur_edge);
1242 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
1243 }
1244 }
1245
1246 static int handle_qpt_reg_umr(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
1247 const struct ib_send_wr *wr,
1248 struct mlx5_wqe_ctrl_seg **ctrl, void **seg,
1249 int *size, void **cur_edge, unsigned int idx)
1250 {
1251 int err = 0;
1252
1253 if (unlikely(wr->opcode != MLX5_IB_WR_UMR)) {
1254 err = -EINVAL;
1255 mlx5_ib_warn(dev, "bad opcode %d\n", wr->opcode);
1256 goto out;
1257 }
1258
1259 qp->sq.wr_data[idx] = MLX5_IB_WR_UMR;
1260 (*ctrl)->imm = cpu_to_be32(umr_wr(wr)->mkey);
1261 err = set_reg_umr_segment(dev, *seg, wr);
1262 if (unlikely(err))
1263 goto out;
1264 *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
1265 *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
1266 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
1267 set_reg_mkey_segment(dev, *seg, wr);
1268 *seg += sizeof(struct mlx5_mkey_seg);
1269 *size += sizeof(struct mlx5_mkey_seg) / 16;
1270 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
1271 out:
1272 return err;
1273 }
1274
1275 int mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
1276 const struct ib_send_wr **bad_wr, bool drain)
1277 {
1278 struct mlx5_wqe_ctrl_seg *ctrl = NULL; /* compiler warning */
1279 struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
1280 struct mlx5_core_dev *mdev = dev->mdev;
1281 struct mlx5_ib_qp *qp;
1282 struct mlx5_wqe_xrc_seg *xrc;
1283 struct mlx5_bf *bf;
1284 void *cur_edge;
1285 int size;
1286 unsigned long flags;
1287 unsigned int idx;
1288 int err = 0;
1289 int num_sge;
1290 void *seg;
1291 int nreq;
1292 int i;
1293 u8 next_fence = 0;
1294 u8 fence;
1295
1296 if (unlikely(mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR &&
1297 !drain)) {
1298 *bad_wr = wr;
1299 return -EIO;
1300 }
1301
1302 if (unlikely(ibqp->qp_type == IB_QPT_GSI))
1303 return mlx5_ib_gsi_post_send(ibqp, wr, bad_wr);
1304
1305 qp = to_mqp(ibqp);
1306 bf = &qp->bf;
1307
1308 spin_lock_irqsave(&qp->sq.lock, flags);
1309
1310 for (nreq = 0; wr; nreq++, wr = wr->next) {
1311 if (unlikely(wr->opcode >= ARRAY_SIZE(mlx5_ib_opcode))) {
1312 mlx5_ib_warn(dev, "\n");
1313 err = -EINVAL;
1314 *bad_wr = wr;
1315 goto out;
1316 }
1317
1318 num_sge = wr->num_sge;
1319 if (unlikely(num_sge > qp->sq.max_gs)) {
1320 mlx5_ib_warn(dev, "\n");
1321 err = -EINVAL;
1322 *bad_wr = wr;
1323 goto out;
1324 }
1325
1326 err = begin_wqe(qp, &seg, &ctrl, wr, &idx, &size, &cur_edge,
1327 nreq);
1328 if (err) {
1329 mlx5_ib_warn(dev, "\n");
1330 err = -ENOMEM;
1331 *bad_wr = wr;
1332 goto out;
1333 }
1334
1335 if (wr->opcode == IB_WR_REG_MR ||
1336 wr->opcode == IB_WR_REG_MR_INTEGRITY) {
1337 fence = dev->umr_fence;
1338 next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;
1339 } else {
1340 if (wr->send_flags & IB_SEND_FENCE) {
1341 if (qp->next_fence)
1342 fence = MLX5_FENCE_MODE_SMALL_AND_FENCE;
1343 else
1344 fence = MLX5_FENCE_MODE_FENCE;
1345 } else {
1346 fence = qp->next_fence;
1347 }
1348 }
1349
1350 switch (ibqp->qp_type) {
1351 case IB_QPT_XRC_INI:
1352 xrc = seg;
1353 seg += sizeof(*xrc);
1354 size += sizeof(*xrc) / 16;
1355 fallthrough;
1356 case IB_QPT_RC:
1357 err = handle_qpt_rc(dev, qp, wr, &ctrl, &seg, &size,
1358 &cur_edge, &idx, nreq, fence,
1359 next_fence, &num_sge);
1360 if (unlikely(err)) {
1361 *bad_wr = wr;
1362 goto out;
1363 } else if (wr->opcode == IB_WR_REG_MR_INTEGRITY) {
1364 goto skip_psv;
1365 }
1366 break;
1367
1368 case IB_QPT_UC:
1369 handle_qpt_uc(wr, &seg, &size);
1370 break;
1371 case IB_QPT_SMI:
1372 if (unlikely(!mdev->port_caps[qp->port - 1].has_smi)) {
1373 mlx5_ib_warn(dev, "Send SMP MADs is not allowed\n");
1374 err = -EPERM;
1375 *bad_wr = wr;
1376 goto out;
1377 }
1378 fallthrough;
1379 case MLX5_IB_QPT_HW_GSI:
1380 handle_qpt_hw_gsi(qp, wr, &seg, &size, &cur_edge);
1381 break;
1382 case IB_QPT_UD:
1383 handle_qpt_ud(qp, wr, &seg, &size, &cur_edge);
1384 break;
1385 case MLX5_IB_QPT_REG_UMR:
1386 err = handle_qpt_reg_umr(dev, qp, wr, &ctrl, &seg,
1387 &size, &cur_edge, idx);
1388 if (unlikely(err))
1389 goto out;
1390 break;
1391
1392 default:
1393 break;
1394 }
1395
1396 if (wr->send_flags & IB_SEND_INLINE && num_sge) {
1397 err = set_data_inl_seg(qp, wr, &seg, &size, &cur_edge);
1398 if (unlikely(err)) {
1399 mlx5_ib_warn(dev, "\n");
1400 *bad_wr = wr;
1401 goto out;
1402 }
1403 } else {
1404 for (i = 0; i < num_sge; i++) {
1405 handle_post_send_edge(&qp->sq, &seg, size,
1406 &cur_edge);
1407 if (unlikely(!wr->sg_list[i].length))
1408 continue;
1409
1410 set_data_ptr_seg(
1411 (struct mlx5_wqe_data_seg *)seg,
1412 wr->sg_list + i);
1413 size += sizeof(struct mlx5_wqe_data_seg) / 16;
1414 seg += sizeof(struct mlx5_wqe_data_seg);
1415 }
1416 }
1417
1418 qp->next_fence = next_fence;
1419 finish_wqe(qp, ctrl, seg, size, cur_edge, idx, wr->wr_id, nreq,
1420 fence, mlx5_ib_opcode[wr->opcode]);
1421 skip_psv:
1422 if (0)
1423 dump_wqe(qp, idx, size);
1424 }
1425
1426 out:
1427 if (likely(nreq)) {
1428 qp->sq.head += nreq;
1429
1430 /* Make sure that descriptors are written before
1431 * updating doorbell record and ringing the doorbell
1432 */
1433 wmb();
1434
1435 qp->db.db[MLX5_SND_DBR] = cpu_to_be32(qp->sq.cur_post);
1436
1437 /* Make sure doorbell record is visible to the HCA before
1438 * we hit doorbell.
1439 */
1440 wmb();
1441
1442 mlx5_write64((__be32 *)ctrl, bf->bfreg->map + bf->offset);
1443 /* Make sure doorbells don't leak out of SQ spinlock
1444 * and reach the HCA out of order.
1445 */
1446 bf->offset ^= bf->buf_size;
1447 }
1448
1449 spin_unlock_irqrestore(&qp->sq.lock, flags);
1450
1451 return err;
1452 }
1453
1454 static void set_sig_seg(struct mlx5_rwqe_sig *sig, int max_gs)
1455 {
1456 sig->signature = calc_sig(sig, (max_gs + 1) << 2);
1457 }
1458
1459 int mlx5_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
1460 const struct ib_recv_wr **bad_wr, bool drain)
1461 {
1462 struct mlx5_ib_qp *qp = to_mqp(ibqp);
1463 struct mlx5_wqe_data_seg *scat;
1464 struct mlx5_rwqe_sig *sig;
1465 struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
1466 struct mlx5_core_dev *mdev = dev->mdev;
1467 unsigned long flags;
1468 int err = 0;
1469 int nreq;
1470 int ind;
1471 int i;
1472
1473 if (unlikely(mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR &&
1474 !drain)) {
1475 *bad_wr = wr;
1476 return -EIO;
1477 }
1478
1479 if (unlikely(ibqp->qp_type == IB_QPT_GSI))
1480 return mlx5_ib_gsi_post_recv(ibqp, wr, bad_wr);
1481
1482 spin_lock_irqsave(&qp->rq.lock, flags);
1483
1484 ind = qp->rq.head & (qp->rq.wqe_cnt - 1);
1485
1486 for (nreq = 0; wr; nreq++, wr = wr->next) {
1487 if (mlx5_wq_overflow(&qp->rq, nreq, qp->ibqp.recv_cq)) {
1488 err = -ENOMEM;
1489 *bad_wr = wr;
1490 goto out;
1491 }
1492
1493 if (unlikely(wr->num_sge > qp->rq.max_gs)) {
1494 err = -EINVAL;
1495 *bad_wr = wr;
1496 goto out;
1497 }
1498
1499 scat = mlx5_frag_buf_get_wqe(&qp->rq.fbc, ind);
1500 if (qp->flags_en & MLX5_QP_FLAG_SIGNATURE)
1501 scat++;
1502
1503 for (i = 0; i < wr->num_sge; i++)
1504 set_data_ptr_seg(scat + i, wr->sg_list + i);
1505
1506 if (i < qp->rq.max_gs) {
1507 scat[i].byte_count = 0;
1508 scat[i].lkey = cpu_to_be32(MLX5_INVALID_LKEY);
1509 scat[i].addr = 0;
1510 }
1511
1512 if (qp->flags_en & MLX5_QP_FLAG_SIGNATURE) {
1513 sig = (struct mlx5_rwqe_sig *)scat;
1514 set_sig_seg(sig, qp->rq.max_gs);
1515 }
1516
1517 qp->rq.wrid[ind] = wr->wr_id;
1518
1519 ind = (ind + 1) & (qp->rq.wqe_cnt - 1);
1520 }
1521
1522 out:
1523 if (likely(nreq)) {
1524 qp->rq.head += nreq;
1525
1526 /* Make sure that descriptors are written before
1527 * doorbell record.
1528 */
1529 wmb();
1530
1531 *qp->db.db = cpu_to_be32(qp->rq.head & 0xffff);
1532 }
1533
1534 spin_unlock_irqrestore(&qp->rq.lock, flags);
1535
1536 return err;
1537 }
Linux with added FPC-III support