?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
interconnect: qcom: Fix Kconfig indentation
[linux/fpc-iii.git] / drivers / infiniband / hw / hns / hns_roce_hw_v2.c
blobcb8071a3e0d57f1ae54396c0ffff993ae1486c81
1 /*
2 * Copyright (c) 2016-2017 Hisilicon Limited.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33 #include <linux/acpi.h>
34 #include <linux/etherdevice.h>
35 #include <linux/interrupt.h>
36 #include <linux/kernel.h>
37 #include <linux/types.h>
38 #include <net/addrconf.h>
39 #include <rdma/ib_addr.h>
40 #include <rdma/ib_cache.h>
41 #include <rdma/ib_umem.h>
42 #include <rdma/uverbs_ioctl.h>
43
44 #include "hnae3.h"
45 #include "hns_roce_common.h"
46 #include "hns_roce_device.h"
47 #include "hns_roce_cmd.h"
48 #include "hns_roce_hem.h"
49 #include "hns_roce_hw_v2.h"
50
51 static void set_data_seg_v2(struct hns_roce_v2_wqe_data_seg *dseg,
52 struct ib_sge *sg)
53 {
54 dseg->lkey = cpu_to_le32(sg->lkey);
55 dseg->addr = cpu_to_le64(sg->addr);
56 dseg->len = cpu_to_le32(sg->length);
57 }
58
59 static void set_frmr_seg(struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
60 struct hns_roce_wqe_frmr_seg *fseg,
61 const struct ib_reg_wr *wr)
62 {
63 struct hns_roce_mr *mr = to_hr_mr(wr->mr);
64
65 /* use ib_access_flags */
66 roce_set_bit(rc_sq_wqe->byte_4,
67 V2_RC_FRMR_WQE_BYTE_4_BIND_EN_S,
68 wr->access & IB_ACCESS_MW_BIND ? 1 : 0);
69 roce_set_bit(rc_sq_wqe->byte_4,
70 V2_RC_FRMR_WQE_BYTE_4_ATOMIC_S,
71 wr->access & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0);
72 roce_set_bit(rc_sq_wqe->byte_4,
73 V2_RC_FRMR_WQE_BYTE_4_RR_S,
74 wr->access & IB_ACCESS_REMOTE_READ ? 1 : 0);
75 roce_set_bit(rc_sq_wqe->byte_4,
76 V2_RC_FRMR_WQE_BYTE_4_RW_S,
77 wr->access & IB_ACCESS_REMOTE_WRITE ? 1 : 0);
78 roce_set_bit(rc_sq_wqe->byte_4,
79 V2_RC_FRMR_WQE_BYTE_4_LW_S,
80 wr->access & IB_ACCESS_LOCAL_WRITE ? 1 : 0);
81
82 /* Data structure reuse may lead to confusion */
83 rc_sq_wqe->msg_len = cpu_to_le32(mr->pbl_ba & 0xffffffff);
84 rc_sq_wqe->inv_key = cpu_to_le32(mr->pbl_ba >> 32);
85
86 rc_sq_wqe->byte_16 = cpu_to_le32(wr->mr->length & 0xffffffff);
87 rc_sq_wqe->byte_20 = cpu_to_le32(wr->mr->length >> 32);
88 rc_sq_wqe->rkey = cpu_to_le32(wr->key);
89 rc_sq_wqe->va = cpu_to_le64(wr->mr->iova);
90
91 fseg->pbl_size = cpu_to_le32(mr->pbl_size);
92 roce_set_field(fseg->mode_buf_pg_sz,
93 V2_RC_FRMR_WQE_BYTE_40_PBL_BUF_PG_SZ_M,
94 V2_RC_FRMR_WQE_BYTE_40_PBL_BUF_PG_SZ_S,
95 mr->pbl_buf_pg_sz + PG_SHIFT_OFFSET);
96 roce_set_bit(fseg->mode_buf_pg_sz,
97 V2_RC_FRMR_WQE_BYTE_40_BLK_MODE_S, 0);
98 }
99
100 static void set_atomic_seg(struct hns_roce_wqe_atomic_seg *aseg,
101 const struct ib_atomic_wr *wr)
102 {
103 if (wr->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP) {
104 aseg->fetchadd_swap_data = cpu_to_le64(wr->swap);
105 aseg->cmp_data = cpu_to_le64(wr->compare_add);
106 } else {
107 aseg->fetchadd_swap_data = cpu_to_le64(wr->compare_add);
108 aseg->cmp_data = 0;
109 }
110 }
111
112 static void set_extend_sge(struct hns_roce_qp *qp, const struct ib_send_wr *wr,
113 unsigned int *sge_ind)
114 {
115 struct hns_roce_v2_wqe_data_seg *dseg;
116 struct ib_sge *sg;
117 int num_in_wqe = 0;
118 int extend_sge_num;
119 int fi_sge_num;
120 int se_sge_num;
121 int shift;
122 int i;
123
124 if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC)
125 num_in_wqe = HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE;
126 extend_sge_num = wr->num_sge - num_in_wqe;
127 sg = wr->sg_list + num_in_wqe;
128 shift = qp->hr_buf.page_shift;
129
130 /*
131 * Check whether wr->num_sge sges are in the same page. If not, we
132 * should calculate how many sges in the first page and the second
133 * page.
134 */
135 dseg = get_send_extend_sge(qp, (*sge_ind) & (qp->sge.sge_cnt - 1));
136 fi_sge_num = (round_up((uintptr_t)dseg, 1 << shift) -
137 (uintptr_t)dseg) /
138 sizeof(struct hns_roce_v2_wqe_data_seg);
139 if (extend_sge_num > fi_sge_num) {
140 se_sge_num = extend_sge_num - fi_sge_num;
141 for (i = 0; i < fi_sge_num; i++) {
142 set_data_seg_v2(dseg++, sg + i);
143 (*sge_ind)++;
144 }
145 dseg = get_send_extend_sge(qp,
146 (*sge_ind) & (qp->sge.sge_cnt - 1));
147 for (i = 0; i < se_sge_num; i++) {
148 set_data_seg_v2(dseg++, sg + fi_sge_num + i);
149 (*sge_ind)++;
150 }
151 } else {
152 for (i = 0; i < extend_sge_num; i++) {
153 set_data_seg_v2(dseg++, sg + i);
154 (*sge_ind)++;
155 }
156 }
157 }
158
159 static int set_rwqe_data_seg(struct ib_qp *ibqp, const struct ib_send_wr *wr,
160 struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
161 void *wqe, unsigned int *sge_ind,
162 const struct ib_send_wr **bad_wr)
163 {
164 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
165 struct hns_roce_v2_wqe_data_seg *dseg = wqe;
166 struct hns_roce_qp *qp = to_hr_qp(ibqp);
167 int i;
168
169 if (wr->send_flags & IB_SEND_INLINE && wr->num_sge) {
170 if (le32_to_cpu(rc_sq_wqe->msg_len) >
171 hr_dev->caps.max_sq_inline) {
172 *bad_wr = wr;
173 dev_err(hr_dev->dev, "inline len(1-%d)=%d, illegal",
174 rc_sq_wqe->msg_len, hr_dev->caps.max_sq_inline);
175 return -EINVAL;
176 }
177
178 if (wr->opcode == IB_WR_RDMA_READ) {
179 *bad_wr = wr;
180 dev_err(hr_dev->dev, "Not support inline data!\n");
181 return -EINVAL;
182 }
183
184 for (i = 0; i < wr->num_sge; i++) {
185 memcpy(wqe, ((void *)wr->sg_list[i].addr),
186 wr->sg_list[i].length);
187 wqe += wr->sg_list[i].length;
188 }
189
190 roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_INLINE_S,
191 1);
192 } else {
193 if (wr->num_sge <= HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) {
194 for (i = 0; i < wr->num_sge; i++) {
195 if (likely(wr->sg_list[i].length)) {
196 set_data_seg_v2(dseg, wr->sg_list + i);
197 dseg++;
198 }
199 }
200 } else {
201 roce_set_field(rc_sq_wqe->byte_20,
202 V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M,
203 V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S,
204 (*sge_ind) & (qp->sge.sge_cnt - 1));
205
206 for (i = 0; i < HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE; i++) {
207 if (likely(wr->sg_list[i].length)) {
208 set_data_seg_v2(dseg, wr->sg_list + i);
209 dseg++;
210 }
211 }
212
213 set_extend_sge(qp, wr, sge_ind);
214 }
215
216 roce_set_field(rc_sq_wqe->byte_16,
217 V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M,
218 V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S, wr->num_sge);
219 }
220
221 return 0;
222 }
223
224 static int hns_roce_v2_modify_qp(struct ib_qp *ibqp,
225 const struct ib_qp_attr *attr,
226 int attr_mask, enum ib_qp_state cur_state,
227 enum ib_qp_state new_state);
228
229 static int hns_roce_v2_post_send(struct ib_qp *ibqp,
230 const struct ib_send_wr *wr,
231 const struct ib_send_wr **bad_wr)
232 {
233 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
234 struct hns_roce_ah *ah = to_hr_ah(ud_wr(wr)->ah);
235 struct hns_roce_v2_ud_send_wqe *ud_sq_wqe;
236 struct hns_roce_v2_rc_send_wqe *rc_sq_wqe;
237 struct hns_roce_qp *qp = to_hr_qp(ibqp);
238 struct hns_roce_wqe_frmr_seg *fseg;
239 struct device *dev = hr_dev->dev;
240 struct hns_roce_v2_db sq_db;
241 struct ib_qp_attr attr;
242 unsigned int sge_ind;
243 unsigned int owner_bit;
244 unsigned long flags;
245 unsigned int ind;
246 void *wqe = NULL;
247 bool loopback;
248 int attr_mask;
249 u32 tmp_len;
250 int ret = 0;
251 u32 hr_op;
252 u8 *smac;
253 int nreq;
254 int i;
255
256 if (unlikely(ibqp->qp_type != IB_QPT_RC &&
257 ibqp->qp_type != IB_QPT_GSI &&
258 ibqp->qp_type != IB_QPT_UD)) {
259 dev_err(dev, "Not supported QP(0x%x)type!\n", ibqp->qp_type);
260 *bad_wr = wr;
261 return -EOPNOTSUPP;
262 }
263
264 if (unlikely(qp->state == IB_QPS_RESET || qp->state == IB_QPS_INIT ||
265 qp->state == IB_QPS_RTR)) {
266 dev_err(dev, "Post WQE fail, QP state %d err!\n", qp->state);
267 *bad_wr = wr;
268 return -EINVAL;
269 }
270
271 spin_lock_irqsave(&qp->sq.lock, flags);
272 ind = qp->sq_next_wqe;
273 sge_ind = qp->next_sge;
274
275 for (nreq = 0; wr; ++nreq, wr = wr->next) {
276 if (hns_roce_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) {
277 ret = -ENOMEM;
278 *bad_wr = wr;
279 goto out;
280 }
281
282 if (unlikely(wr->num_sge > qp->sq.max_gs)) {
283 dev_err(dev, "num_sge=%d > qp->sq.max_gs=%d\n",
284 wr->num_sge, qp->sq.max_gs);
285 ret = -EINVAL;
286 *bad_wr = wr;
287 goto out;
288 }
289
290 wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1));
291 qp->sq.wrid[(qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1)] =
292 wr->wr_id;
293
294 owner_bit =
295 ~(((qp->sq.head + nreq) >> ilog2(qp->sq.wqe_cnt)) & 0x1);
296 tmp_len = 0;
297
298 /* Corresponding to the QP type, wqe process separately */
299 if (ibqp->qp_type == IB_QPT_GSI) {
300 ud_sq_wqe = wqe;
301 memset(ud_sq_wqe, 0, sizeof(*ud_sq_wqe));
302
303 roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_0_M,
304 V2_UD_SEND_WQE_DMAC_0_S, ah->av.mac[0]);
305 roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_1_M,
306 V2_UD_SEND_WQE_DMAC_1_S, ah->av.mac[1]);
307 roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_2_M,
308 V2_UD_SEND_WQE_DMAC_2_S, ah->av.mac[2]);
309 roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_3_M,
310 V2_UD_SEND_WQE_DMAC_3_S, ah->av.mac[3]);
311 roce_set_field(ud_sq_wqe->byte_48,
312 V2_UD_SEND_WQE_BYTE_48_DMAC_4_M,
313 V2_UD_SEND_WQE_BYTE_48_DMAC_4_S,
314 ah->av.mac[4]);
315 roce_set_field(ud_sq_wqe->byte_48,
316 V2_UD_SEND_WQE_BYTE_48_DMAC_5_M,
317 V2_UD_SEND_WQE_BYTE_48_DMAC_5_S,
318 ah->av.mac[5]);
319
320 /* MAC loopback */
321 smac = (u8 *)hr_dev->dev_addr[qp->port];
322 loopback = ether_addr_equal_unaligned(ah->av.mac,
323 smac) ? 1 : 0;
324
325 roce_set_bit(ud_sq_wqe->byte_40,
326 V2_UD_SEND_WQE_BYTE_40_LBI_S, loopback);
327
328 roce_set_field(ud_sq_wqe->byte_4,
329 V2_UD_SEND_WQE_BYTE_4_OPCODE_M,
330 V2_UD_SEND_WQE_BYTE_4_OPCODE_S,
331 HNS_ROCE_V2_WQE_OP_SEND);
332
333 for (i = 0; i < wr->num_sge; i++)
334 tmp_len += wr->sg_list[i].length;
335
336 ud_sq_wqe->msg_len =
337 cpu_to_le32(le32_to_cpu(ud_sq_wqe->msg_len) + tmp_len);
338
339 switch (wr->opcode) {
340 case IB_WR_SEND_WITH_IMM:
341 case IB_WR_RDMA_WRITE_WITH_IMM:
342 ud_sq_wqe->immtdata =
343 cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
344 break;
345 default:
346 ud_sq_wqe->immtdata = 0;
347 break;
348 }
349
350 /* Set sig attr */
351 roce_set_bit(ud_sq_wqe->byte_4,
352 V2_UD_SEND_WQE_BYTE_4_CQE_S,
353 (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0);
354
355 /* Set se attr */
356 roce_set_bit(ud_sq_wqe->byte_4,
357 V2_UD_SEND_WQE_BYTE_4_SE_S,
358 (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0);
359
360 roce_set_bit(ud_sq_wqe->byte_4,
361 V2_UD_SEND_WQE_BYTE_4_OWNER_S, owner_bit);
362
363 roce_set_field(ud_sq_wqe->byte_16,
364 V2_UD_SEND_WQE_BYTE_16_PD_M,
365 V2_UD_SEND_WQE_BYTE_16_PD_S,
366 to_hr_pd(ibqp->pd)->pdn);
367
368 roce_set_field(ud_sq_wqe->byte_16,
369 V2_UD_SEND_WQE_BYTE_16_SGE_NUM_M,
370 V2_UD_SEND_WQE_BYTE_16_SGE_NUM_S,
371 wr->num_sge);
372
373 roce_set_field(ud_sq_wqe->byte_20,
374 V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M,
375 V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S,
376 sge_ind & (qp->sge.sge_cnt - 1));
377
378 roce_set_field(ud_sq_wqe->byte_24,
379 V2_UD_SEND_WQE_BYTE_24_UDPSPN_M,
380 V2_UD_SEND_WQE_BYTE_24_UDPSPN_S, 0);
381 ud_sq_wqe->qkey =
382 cpu_to_le32(ud_wr(wr)->remote_qkey & 0x80000000 ?
383 qp->qkey : ud_wr(wr)->remote_qkey);
384 roce_set_field(ud_sq_wqe->byte_32,
385 V2_UD_SEND_WQE_BYTE_32_DQPN_M,
386 V2_UD_SEND_WQE_BYTE_32_DQPN_S,
387 ud_wr(wr)->remote_qpn);
388
389 roce_set_field(ud_sq_wqe->byte_36,
390 V2_UD_SEND_WQE_BYTE_36_VLAN_M,
391 V2_UD_SEND_WQE_BYTE_36_VLAN_S,
392 ah->av.vlan_id);
393 roce_set_field(ud_sq_wqe->byte_36,
394 V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_M,
395 V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_S,
396 ah->av.hop_limit);
397 roce_set_field(ud_sq_wqe->byte_36,
398 V2_UD_SEND_WQE_BYTE_36_TCLASS_M,
399 V2_UD_SEND_WQE_BYTE_36_TCLASS_S,
400 ah->av.tclass);
401 roce_set_field(ud_sq_wqe->byte_40,
402 V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_M,
403 V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_S,
404 ah->av.flowlabel);
405 roce_set_field(ud_sq_wqe->byte_40,
406 V2_UD_SEND_WQE_BYTE_40_SL_M,
407 V2_UD_SEND_WQE_BYTE_40_SL_S,
408 ah->av.sl);
409 roce_set_field(ud_sq_wqe->byte_40,
410 V2_UD_SEND_WQE_BYTE_40_PORTN_M,
411 V2_UD_SEND_WQE_BYTE_40_PORTN_S,
412 qp->port);
413
414 roce_set_bit(ud_sq_wqe->byte_40,
415 V2_UD_SEND_WQE_BYTE_40_UD_VLAN_EN_S,
416 ah->av.vlan_en ? 1 : 0);
417 roce_set_field(ud_sq_wqe->byte_48,
418 V2_UD_SEND_WQE_BYTE_48_SGID_INDX_M,
419 V2_UD_SEND_WQE_BYTE_48_SGID_INDX_S,
420 hns_get_gid_index(hr_dev, qp->phy_port,
421 ah->av.gid_index));
422
423 memcpy(&ud_sq_wqe->dgid[0], &ah->av.dgid[0],
424 GID_LEN_V2);
425
426 set_extend_sge(qp, wr, &sge_ind);
427 ind++;
428 } else if (ibqp->qp_type == IB_QPT_RC) {
429 rc_sq_wqe = wqe;
430 memset(rc_sq_wqe, 0, sizeof(*rc_sq_wqe));
431 for (i = 0; i < wr->num_sge; i++)
432 tmp_len += wr->sg_list[i].length;
433
434 rc_sq_wqe->msg_len =
435 cpu_to_le32(le32_to_cpu(rc_sq_wqe->msg_len) + tmp_len);
436
437 switch (wr->opcode) {
438 case IB_WR_SEND_WITH_IMM:
439 case IB_WR_RDMA_WRITE_WITH_IMM:
440 rc_sq_wqe->immtdata =
441 cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
442 break;
443 case IB_WR_SEND_WITH_INV:
444 rc_sq_wqe->inv_key =
445 cpu_to_le32(wr->ex.invalidate_rkey);
446 break;
447 default:
448 rc_sq_wqe->immtdata = 0;
449 break;
450 }
451
452 roce_set_bit(rc_sq_wqe->byte_4,
453 V2_RC_SEND_WQE_BYTE_4_FENCE_S,
454 (wr->send_flags & IB_SEND_FENCE) ? 1 : 0);
455
456 roce_set_bit(rc_sq_wqe->byte_4,
457 V2_RC_SEND_WQE_BYTE_4_SE_S,
458 (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0);
459
460 roce_set_bit(rc_sq_wqe->byte_4,
461 V2_RC_SEND_WQE_BYTE_4_CQE_S,
462 (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0);
463
464 roce_set_bit(rc_sq_wqe->byte_4,
465 V2_RC_SEND_WQE_BYTE_4_OWNER_S, owner_bit);
466
467 wqe += sizeof(struct hns_roce_v2_rc_send_wqe);
468 switch (wr->opcode) {
469 case IB_WR_RDMA_READ:
470 hr_op = HNS_ROCE_V2_WQE_OP_RDMA_READ;
471 rc_sq_wqe->rkey =
472 cpu_to_le32(rdma_wr(wr)->rkey);
473 rc_sq_wqe->va =
474 cpu_to_le64(rdma_wr(wr)->remote_addr);
475 break;
476 case IB_WR_RDMA_WRITE:
477 hr_op = HNS_ROCE_V2_WQE_OP_RDMA_WRITE;
478 rc_sq_wqe->rkey =
479 cpu_to_le32(rdma_wr(wr)->rkey);
480 rc_sq_wqe->va =
481 cpu_to_le64(rdma_wr(wr)->remote_addr);
482 break;
483 case IB_WR_RDMA_WRITE_WITH_IMM:
484 hr_op = HNS_ROCE_V2_WQE_OP_RDMA_WRITE_WITH_IMM;
485 rc_sq_wqe->rkey =
486 cpu_to_le32(rdma_wr(wr)->rkey);
487 rc_sq_wqe->va =
488 cpu_to_le64(rdma_wr(wr)->remote_addr);
489 break;
490 case IB_WR_SEND:
491 hr_op = HNS_ROCE_V2_WQE_OP_SEND;
492 break;
493 case IB_WR_SEND_WITH_INV:
494 hr_op = HNS_ROCE_V2_WQE_OP_SEND_WITH_INV;
495 break;
496 case IB_WR_SEND_WITH_IMM:
497 hr_op = HNS_ROCE_V2_WQE_OP_SEND_WITH_IMM;
498 break;
499 case IB_WR_LOCAL_INV:
500 hr_op = HNS_ROCE_V2_WQE_OP_LOCAL_INV;
501 roce_set_bit(rc_sq_wqe->byte_4,
502 V2_RC_SEND_WQE_BYTE_4_SO_S, 1);
503 rc_sq_wqe->inv_key =
504 cpu_to_le32(wr->ex.invalidate_rkey);
505 break;
506 case IB_WR_REG_MR:
507 hr_op = HNS_ROCE_V2_WQE_OP_FAST_REG_PMR;
508 fseg = wqe;
509 set_frmr_seg(rc_sq_wqe, fseg, reg_wr(wr));
510 break;
511 case IB_WR_ATOMIC_CMP_AND_SWP:
512 hr_op = HNS_ROCE_V2_WQE_OP_ATOM_CMP_AND_SWAP;
513 rc_sq_wqe->rkey =
514 cpu_to_le32(atomic_wr(wr)->rkey);
515 rc_sq_wqe->va =
516 cpu_to_le64(atomic_wr(wr)->remote_addr);
517 break;
518 case IB_WR_ATOMIC_FETCH_AND_ADD:
519 hr_op = HNS_ROCE_V2_WQE_OP_ATOM_FETCH_AND_ADD;
520 rc_sq_wqe->rkey =
521 cpu_to_le32(atomic_wr(wr)->rkey);
522 rc_sq_wqe->va =
523 cpu_to_le64(atomic_wr(wr)->remote_addr);
524 break;
525 case IB_WR_MASKED_ATOMIC_CMP_AND_SWP:
526 hr_op =
527 HNS_ROCE_V2_WQE_OP_ATOM_MSK_CMP_AND_SWAP;
528 break;
529 case IB_WR_MASKED_ATOMIC_FETCH_AND_ADD:
530 hr_op =
531 HNS_ROCE_V2_WQE_OP_ATOM_MSK_FETCH_AND_ADD;
532 break;
533 default:
534 hr_op = HNS_ROCE_V2_WQE_OP_MASK;
535 break;
536 }
537
538 roce_set_field(rc_sq_wqe->byte_4,
539 V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
540 V2_RC_SEND_WQE_BYTE_4_OPCODE_S, hr_op);
541
542 if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
543 wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
544 struct hns_roce_v2_wqe_data_seg *dseg;
545
546 dseg = wqe;
547 set_data_seg_v2(dseg, wr->sg_list);
548 wqe += sizeof(struct hns_roce_v2_wqe_data_seg);
549 set_atomic_seg(wqe, atomic_wr(wr));
550 roce_set_field(rc_sq_wqe->byte_16,
551 V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M,
552 V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S,
553 wr->num_sge);
554 } else if (wr->opcode != IB_WR_REG_MR) {
555 ret = set_rwqe_data_seg(ibqp, wr, rc_sq_wqe,
556 wqe, &sge_ind, bad_wr);
557 if (ret)
558 goto out;
559 }
560
561 ind++;
562 } else {
563 dev_err(dev, "Illegal qp_type(0x%x)\n", ibqp->qp_type);
564 spin_unlock_irqrestore(&qp->sq.lock, flags);
565 *bad_wr = wr;
566 return -EOPNOTSUPP;
567 }
568 }
569
570 out:
571 if (likely(nreq)) {
572 qp->sq.head += nreq;
573 /* Memory barrier */
574 wmb();
575
576 sq_db.byte_4 = 0;
577 sq_db.parameter = 0;
578
579 roce_set_field(sq_db.byte_4, V2_DB_BYTE_4_TAG_M,
580 V2_DB_BYTE_4_TAG_S, qp->doorbell_qpn);
581 roce_set_field(sq_db.byte_4, V2_DB_BYTE_4_CMD_M,
582 V2_DB_BYTE_4_CMD_S, HNS_ROCE_V2_SQ_DB);
583 roce_set_field(sq_db.parameter, V2_DB_PARAMETER_IDX_M,
584 V2_DB_PARAMETER_IDX_S,
585 qp->sq.head & ((qp->sq.wqe_cnt << 1) - 1));
586 roce_set_field(sq_db.parameter, V2_DB_PARAMETER_SL_M,
587 V2_DB_PARAMETER_SL_S, qp->sl);
588
589 hns_roce_write64(hr_dev, (__le32 *)&sq_db, qp->sq.db_reg_l);
590
591 qp->sq_next_wqe = ind;
592 qp->next_sge = sge_ind;
593
594 if (qp->state == IB_QPS_ERR) {
595 attr_mask = IB_QP_STATE;
596 attr.qp_state = IB_QPS_ERR;
597
598 ret = hns_roce_v2_modify_qp(&qp->ibqp, &attr, attr_mask,
599 qp->state, IB_QPS_ERR);
600 if (ret) {
601 spin_unlock_irqrestore(&qp->sq.lock, flags);
602 *bad_wr = wr;
603 return ret;
604 }
605 }
606 }
607
608 spin_unlock_irqrestore(&qp->sq.lock, flags);
609
610 return ret;
611 }
612
613 static int hns_roce_v2_post_recv(struct ib_qp *ibqp,
614 const struct ib_recv_wr *wr,
615 const struct ib_recv_wr **bad_wr)
616 {
617 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
618 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
619 struct hns_roce_v2_wqe_data_seg *dseg;
620 struct hns_roce_rinl_sge *sge_list;
621 struct device *dev = hr_dev->dev;
622 struct ib_qp_attr attr;
623 unsigned long flags;
624 void *wqe = NULL;
625 int attr_mask;
626 int ret = 0;
627 int nreq;
628 int ind;
629 int i;
630
631 spin_lock_irqsave(&hr_qp->rq.lock, flags);
632 ind = hr_qp->rq.head & (hr_qp->rq.wqe_cnt - 1);
633
634 if (hr_qp->state == IB_QPS_RESET) {
635 spin_unlock_irqrestore(&hr_qp->rq.lock, flags);
636 *bad_wr = wr;
637 return -EINVAL;
638 }
639
640 for (nreq = 0; wr; ++nreq, wr = wr->next) {
641 if (hns_roce_wq_overflow(&hr_qp->rq, nreq,
642 hr_qp->ibqp.recv_cq)) {
643 ret = -ENOMEM;
644 *bad_wr = wr;
645 goto out;
646 }
647
648 if (unlikely(wr->num_sge > hr_qp->rq.max_gs)) {
649 dev_err(dev, "rq:num_sge=%d > qp->sq.max_gs=%d\n",
650 wr->num_sge, hr_qp->rq.max_gs);
651 ret = -EINVAL;
652 *bad_wr = wr;
653 goto out;
654 }
655
656 wqe = get_recv_wqe(hr_qp, ind);
657 dseg = (struct hns_roce_v2_wqe_data_seg *)wqe;
658 for (i = 0; i < wr->num_sge; i++) {
659 if (!wr->sg_list[i].length)
660 continue;
661 set_data_seg_v2(dseg, wr->sg_list + i);
662 dseg++;
663 }
664
665 if (i < hr_qp->rq.max_gs) {
666 dseg->lkey = cpu_to_le32(HNS_ROCE_INVALID_LKEY);
667 dseg->addr = 0;
668 }
669
670 /* rq support inline data */
671 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) {
672 sge_list = hr_qp->rq_inl_buf.wqe_list[ind].sg_list;
673 hr_qp->rq_inl_buf.wqe_list[ind].sge_cnt =
674 (u32)wr->num_sge;
675 for (i = 0; i < wr->num_sge; i++) {
676 sge_list[i].addr =
677 (void *)(u64)wr->sg_list[i].addr;
678 sge_list[i].len = wr->sg_list[i].length;
679 }
680 }
681
682 hr_qp->rq.wrid[ind] = wr->wr_id;
683
684 ind = (ind + 1) & (hr_qp->rq.wqe_cnt - 1);
685 }
686
687 out:
688 if (likely(nreq)) {
689 hr_qp->rq.head += nreq;
690 /* Memory barrier */
691 wmb();
692
693 *hr_qp->rdb.db_record = hr_qp->rq.head & 0xffff;
694
695 if (hr_qp->state == IB_QPS_ERR) {
696 attr_mask = IB_QP_STATE;
697 attr.qp_state = IB_QPS_ERR;
698
699 ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, &attr,
700 attr_mask, hr_qp->state,
701 IB_QPS_ERR);
702 if (ret) {
703 spin_unlock_irqrestore(&hr_qp->rq.lock, flags);
704 *bad_wr = wr;
705 return ret;
706 }
707 }
708 }
709 spin_unlock_irqrestore(&hr_qp->rq.lock, flags);
710
711 return ret;
712 }
713
714 static int hns_roce_v2_cmd_hw_reseted(struct hns_roce_dev *hr_dev,
715 unsigned long instance_stage,
716 unsigned long reset_stage)
717 {
718 /* When hardware reset has been completed once or more, we should stop
719 * sending mailbox&cmq&doorbell to hardware. If now in .init_instance()
720 * function, we should exit with error. If now at HNAE3_INIT_CLIENT
721 * stage of soft reset process, we should exit with error, and then
722 * HNAE3_INIT_CLIENT related process can rollback the operation like
723 * notifing hardware to free resources, HNAE3_INIT_CLIENT related
724 * process will exit with error to notify NIC driver to reschedule soft
725 * reset process once again.
726 */
727 hr_dev->is_reset = true;
728 hr_dev->dis_db = true;
729
730 if (reset_stage == HNS_ROCE_STATE_RST_INIT ||
731 instance_stage == HNS_ROCE_STATE_INIT)
732 return CMD_RST_PRC_EBUSY;
733
734 return CMD_RST_PRC_SUCCESS;
735 }
736
737 static int hns_roce_v2_cmd_hw_resetting(struct hns_roce_dev *hr_dev,
738 unsigned long instance_stage,
739 unsigned long reset_stage)
740 {
741 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
742 struct hnae3_handle *handle = priv->handle;
743 const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
744
745 /* When hardware reset is detected, we should stop sending mailbox&cmq&
746 * doorbell to hardware. If now in .init_instance() function, we should
747 * exit with error. If now at HNAE3_INIT_CLIENT stage of soft reset
748 * process, we should exit with error, and then HNAE3_INIT_CLIENT
749 * related process can rollback the operation like notifing hardware to
750 * free resources, HNAE3_INIT_CLIENT related process will exit with
751 * error to notify NIC driver to reschedule soft reset process once
752 * again.
753 */
754 hr_dev->dis_db = true;
755 if (!ops->get_hw_reset_stat(handle))
756 hr_dev->is_reset = true;
757
758 if (!hr_dev->is_reset || reset_stage == HNS_ROCE_STATE_RST_INIT ||
759 instance_stage == HNS_ROCE_STATE_INIT)
760 return CMD_RST_PRC_EBUSY;
761
762 return CMD_RST_PRC_SUCCESS;
763 }
764
765 static int hns_roce_v2_cmd_sw_resetting(struct hns_roce_dev *hr_dev)
766 {
767 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
768 struct hnae3_handle *handle = priv->handle;
769 const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
770
771 /* When software reset is detected at .init_instance() function, we
772 * should stop sending mailbox&cmq&doorbell to hardware, and exit
773 * with error.
774 */
775 hr_dev->dis_db = true;
776 if (ops->ae_dev_reset_cnt(handle) != hr_dev->reset_cnt)
777 hr_dev->is_reset = true;
778
779 return CMD_RST_PRC_EBUSY;
780 }
781
782 static int hns_roce_v2_rst_process_cmd(struct hns_roce_dev *hr_dev)
783 {
784 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
785 struct hnae3_handle *handle = priv->handle;
786 const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
787 unsigned long instance_stage; /* the current instance stage */
788 unsigned long reset_stage; /* the current reset stage */
789 unsigned long reset_cnt;
790 bool sw_resetting;
791 bool hw_resetting;
792
793 if (hr_dev->is_reset)
794 return CMD_RST_PRC_SUCCESS;
795
796 /* Get information about reset from NIC driver or RoCE driver itself,
797 * the meaning of the following variables from NIC driver are described
798 * as below:
799 * reset_cnt -- The count value of completed hardware reset.
800 * hw_resetting -- Whether hardware device is resetting now.
801 * sw_resetting -- Whether NIC's software reset process is running now.
802 */
803 instance_stage = handle->rinfo.instance_state;
804 reset_stage = handle->rinfo.reset_state;
805 reset_cnt = ops->ae_dev_reset_cnt(handle);
806 hw_resetting = ops->get_hw_reset_stat(handle);
807 sw_resetting = ops->ae_dev_resetting(handle);
808
809 if (reset_cnt != hr_dev->reset_cnt)
810 return hns_roce_v2_cmd_hw_reseted(hr_dev, instance_stage,
811 reset_stage);
812 else if (hw_resetting)
813 return hns_roce_v2_cmd_hw_resetting(hr_dev, instance_stage,
814 reset_stage);
815 else if (sw_resetting && instance_stage == HNS_ROCE_STATE_INIT)
816 return hns_roce_v2_cmd_sw_resetting(hr_dev);
817
818 return 0;
819 }
820
821 static int hns_roce_cmq_space(struct hns_roce_v2_cmq_ring *ring)
822 {
823 int ntu = ring->next_to_use;
824 int ntc = ring->next_to_clean;
825 int used = (ntu - ntc + ring->desc_num) % ring->desc_num;
826
827 return ring->desc_num - used - 1;
828 }
829
830 static int hns_roce_alloc_cmq_desc(struct hns_roce_dev *hr_dev,
831 struct hns_roce_v2_cmq_ring *ring)
832 {
833 int size = ring->desc_num * sizeof(struct hns_roce_cmq_desc);
834
835 ring->desc = kzalloc(size, GFP_KERNEL);
836 if (!ring->desc)
837 return -ENOMEM;
838
839 ring->desc_dma_addr = dma_map_single(hr_dev->dev, ring->desc, size,
840 DMA_BIDIRECTIONAL);
841 if (dma_mapping_error(hr_dev->dev, ring->desc_dma_addr)) {
842 ring->desc_dma_addr = 0;
843 kfree(ring->desc);
844 ring->desc = NULL;
845 return -ENOMEM;
846 }
847
848 return 0;
849 }
850
851 static void hns_roce_free_cmq_desc(struct hns_roce_dev *hr_dev,
852 struct hns_roce_v2_cmq_ring *ring)
853 {
854 dma_unmap_single(hr_dev->dev, ring->desc_dma_addr,
855 ring->desc_num * sizeof(struct hns_roce_cmq_desc),
856 DMA_BIDIRECTIONAL);
857
858 ring->desc_dma_addr = 0;
859 kfree(ring->desc);
860 }
861
862 static int hns_roce_init_cmq_ring(struct hns_roce_dev *hr_dev, bool ring_type)
863 {
864 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
865 struct hns_roce_v2_cmq_ring *ring = (ring_type == TYPE_CSQ) ?
866 &priv->cmq.csq : &priv->cmq.crq;
867
868 ring->flag = ring_type;
869 ring->next_to_clean = 0;
870 ring->next_to_use = 0;
871
872 return hns_roce_alloc_cmq_desc(hr_dev, ring);
873 }
874
875 static void hns_roce_cmq_init_regs(struct hns_roce_dev *hr_dev, bool ring_type)
876 {
877 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
878 struct hns_roce_v2_cmq_ring *ring = (ring_type == TYPE_CSQ) ?
879 &priv->cmq.csq : &priv->cmq.crq;
880 dma_addr_t dma = ring->desc_dma_addr;
881
882 if (ring_type == TYPE_CSQ) {
883 roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_L_REG, (u32)dma);
884 roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_H_REG,
885 upper_32_bits(dma));
886 roce_write(hr_dev, ROCEE_TX_CMQ_DEPTH_REG,
887 ring->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S);
888 roce_write(hr_dev, ROCEE_TX_CMQ_HEAD_REG, 0);
889 roce_write(hr_dev, ROCEE_TX_CMQ_TAIL_REG, 0);
890 } else {
891 roce_write(hr_dev, ROCEE_RX_CMQ_BASEADDR_L_REG, (u32)dma);
892 roce_write(hr_dev, ROCEE_RX_CMQ_BASEADDR_H_REG,
893 upper_32_bits(dma));
894 roce_write(hr_dev, ROCEE_RX_CMQ_DEPTH_REG,
895 ring->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S);
896 roce_write(hr_dev, ROCEE_RX_CMQ_HEAD_REG, 0);
897 roce_write(hr_dev, ROCEE_RX_CMQ_TAIL_REG, 0);
898 }
899 }
900
901 static int hns_roce_v2_cmq_init(struct hns_roce_dev *hr_dev)
902 {
903 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
904 int ret;
905
906 /* Setup the queue entries for command queue */
907 priv->cmq.csq.desc_num = CMD_CSQ_DESC_NUM;
908 priv->cmq.crq.desc_num = CMD_CRQ_DESC_NUM;
909
910 /* Setup the lock for command queue */
911 spin_lock_init(&priv->cmq.csq.lock);
912 spin_lock_init(&priv->cmq.crq.lock);
913
914 /* Setup Tx write back timeout */
915 priv->cmq.tx_timeout = HNS_ROCE_CMQ_TX_TIMEOUT;
916
917 /* Init CSQ */
918 ret = hns_roce_init_cmq_ring(hr_dev, TYPE_CSQ);
919 if (ret) {
920 dev_err(hr_dev->dev, "Init CSQ error, ret = %d.\n", ret);
921 return ret;
922 }
923
924 /* Init CRQ */
925 ret = hns_roce_init_cmq_ring(hr_dev, TYPE_CRQ);
926 if (ret) {
927 dev_err(hr_dev->dev, "Init CRQ error, ret = %d.\n", ret);
928 goto err_crq;
929 }
930
931 /* Init CSQ REG */
932 hns_roce_cmq_init_regs(hr_dev, TYPE_CSQ);
933
934 /* Init CRQ REG */
935 hns_roce_cmq_init_regs(hr_dev, TYPE_CRQ);
936
937 return 0;
938
939 err_crq:
940 hns_roce_free_cmq_desc(hr_dev, &priv->cmq.csq);
941
942 return ret;
943 }
944
945 static void hns_roce_v2_cmq_exit(struct hns_roce_dev *hr_dev)
946 {
947 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
948
949 hns_roce_free_cmq_desc(hr_dev, &priv->cmq.csq);
950 hns_roce_free_cmq_desc(hr_dev, &priv->cmq.crq);
951 }
952
953 static void hns_roce_cmq_setup_basic_desc(struct hns_roce_cmq_desc *desc,
954 enum hns_roce_opcode_type opcode,
955 bool is_read)
956 {
957 memset((void *)desc, 0, sizeof(struct hns_roce_cmq_desc));
958 desc->opcode = cpu_to_le16(opcode);
959 desc->flag =
960 cpu_to_le16(HNS_ROCE_CMD_FLAG_NO_INTR | HNS_ROCE_CMD_FLAG_IN);
961 if (is_read)
962 desc->flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_WR);
963 else
964 desc->flag &= cpu_to_le16(~HNS_ROCE_CMD_FLAG_WR);
965 }
966
967 static int hns_roce_cmq_csq_done(struct hns_roce_dev *hr_dev)
968 {
969 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
970 u32 head = roce_read(hr_dev, ROCEE_TX_CMQ_HEAD_REG);
971
972 return head == priv->cmq.csq.next_to_use;
973 }
974
975 static int hns_roce_cmq_csq_clean(struct hns_roce_dev *hr_dev)
976 {
977 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
978 struct hns_roce_v2_cmq_ring *csq = &priv->cmq.csq;
979 struct hns_roce_cmq_desc *desc;
980 u16 ntc = csq->next_to_clean;
981 u32 head;
982 int clean = 0;
983
984 desc = &csq->desc[ntc];
985 head = roce_read(hr_dev, ROCEE_TX_CMQ_HEAD_REG);
986 while (head != ntc) {
987 memset(desc, 0, sizeof(*desc));
988 ntc++;
989 if (ntc == csq->desc_num)
990 ntc = 0;
991 desc = &csq->desc[ntc];
992 clean++;
993 }
994 csq->next_to_clean = ntc;
995
996 return clean;
997 }
998
999 static int __hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
1000 struct hns_roce_cmq_desc *desc, int num)
1001 {
1002 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
1003 struct hns_roce_v2_cmq_ring *csq = &priv->cmq.csq;
1004 struct hns_roce_cmq_desc *desc_to_use;
1005 bool complete = false;
1006 u32 timeout = 0;
1007 int handle = 0;
1008 u16 desc_ret;
1009 int ret = 0;
1010 int ntc;
1011
1012 spin_lock_bh(&csq->lock);
1013
1014 if (num > hns_roce_cmq_space(csq)) {
1015 spin_unlock_bh(&csq->lock);
1016 return -EBUSY;
1017 }
1018
1019 /*
1020 * Record the location of desc in the cmq for this time
1021 * which will be use for hardware to write back
1022 */
1023 ntc = csq->next_to_use;
1024
1025 while (handle < num) {
1026 desc_to_use = &csq->desc[csq->next_to_use];
1027 *desc_to_use = desc[handle];
1028 dev_dbg(hr_dev->dev, "set cmq desc:\n");
1029 csq->next_to_use++;
1030 if (csq->next_to_use == csq->desc_num)
1031 csq->next_to_use = 0;
1032 handle++;
1033 }
1034
1035 /* Write to hardware */
1036 roce_write(hr_dev, ROCEE_TX_CMQ_TAIL_REG, csq->next_to_use);
1037
1038 /*
1039 * If the command is sync, wait for the firmware to write back,
1040 * if multi descriptors to be sent, use the first one to check
1041 */
1042 if (le16_to_cpu(desc->flag) & HNS_ROCE_CMD_FLAG_NO_INTR) {
1043 do {
1044 if (hns_roce_cmq_csq_done(hr_dev))
1045 break;
1046 udelay(1);
1047 timeout++;
1048 } while (timeout < priv->cmq.tx_timeout);
1049 }
1050
1051 if (hns_roce_cmq_csq_done(hr_dev)) {
1052 complete = true;
1053 handle = 0;
1054 while (handle < num) {
1055 /* get the result of hardware write back */
1056 desc_to_use = &csq->desc[ntc];
1057 desc[handle] = *desc_to_use;
1058 dev_dbg(hr_dev->dev, "Get cmq desc:\n");
1059 desc_ret = le16_to_cpu(desc[handle].retval);
1060 if (desc_ret == CMD_EXEC_SUCCESS)
1061 ret = 0;
1062 else
1063 ret = -EIO;
1064 priv->cmq.last_status = desc_ret;
1065 ntc++;
1066 handle++;
1067 if (ntc == csq->desc_num)
1068 ntc = 0;
1069 }
1070 }
1071
1072 if (!complete)
1073 ret = -EAGAIN;
1074
1075 /* clean the command send queue */
1076 handle = hns_roce_cmq_csq_clean(hr_dev);
1077 if (handle != num)
1078 dev_warn(hr_dev->dev, "Cleaned %d, need to clean %d\n",
1079 handle, num);
1080
1081 spin_unlock_bh(&csq->lock);
1082
1083 return ret;
1084 }
1085
1086 static int hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
1087 struct hns_roce_cmq_desc *desc, int num)
1088 {
1089 int retval;
1090 int ret;
1091
1092 ret = hns_roce_v2_rst_process_cmd(hr_dev);
1093 if (ret == CMD_RST_PRC_SUCCESS)
1094 return 0;
1095 if (ret == CMD_RST_PRC_EBUSY)
1096 return -EBUSY;
1097
1098 ret = __hns_roce_cmq_send(hr_dev, desc, num);
1099 if (ret) {
1100 retval = hns_roce_v2_rst_process_cmd(hr_dev);
1101 if (retval == CMD_RST_PRC_SUCCESS)
1102 return 0;
1103 else if (retval == CMD_RST_PRC_EBUSY)
1104 return -EBUSY;
1105 }
1106
1107 return ret;
1108 }
1109
1110 static int hns_roce_cmq_query_hw_info(struct hns_roce_dev *hr_dev)
1111 {
1112 struct hns_roce_query_version *resp;
1113 struct hns_roce_cmq_desc desc;
1114 int ret;
1115
1116 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_HW_VER, true);
1117 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1118 if (ret)
1119 return ret;
1120
1121 resp = (struct hns_roce_query_version *)desc.data;
1122 hr_dev->hw_rev = le16_to_cpu(resp->rocee_hw_version);
1123 hr_dev->vendor_id = hr_dev->pci_dev->vendor;
1124
1125 return 0;
1126 }
1127
1128 static bool hns_roce_func_clr_chk_rst(struct hns_roce_dev *hr_dev)
1129 {
1130 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
1131 struct hnae3_handle *handle = priv->handle;
1132 const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
1133 unsigned long reset_cnt;
1134 bool sw_resetting;
1135 bool hw_resetting;
1136
1137 reset_cnt = ops->ae_dev_reset_cnt(handle);
1138 hw_resetting = ops->get_hw_reset_stat(handle);
1139 sw_resetting = ops->ae_dev_resetting(handle);
1140
1141 if (reset_cnt != hr_dev->reset_cnt || hw_resetting || sw_resetting)
1142 return true;
1143
1144 return false;
1145 }
1146
1147 static void hns_roce_func_clr_rst_prc(struct hns_roce_dev *hr_dev, int retval,
1148 int flag)
1149 {
1150 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
1151 struct hnae3_handle *handle = priv->handle;
1152 const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
1153 unsigned long instance_stage;
1154 unsigned long reset_cnt;
1155 unsigned long end;
1156 bool sw_resetting;
1157 bool hw_resetting;
1158
1159 instance_stage = handle->rinfo.instance_state;
1160 reset_cnt = ops->ae_dev_reset_cnt(handle);
1161 hw_resetting = ops->get_hw_reset_stat(handle);
1162 sw_resetting = ops->ae_dev_resetting(handle);
1163
1164 if (reset_cnt != hr_dev->reset_cnt) {
1165 hr_dev->dis_db = true;
1166 hr_dev->is_reset = true;
1167 dev_info(hr_dev->dev, "Func clear success after reset.\n");
1168 } else if (hw_resetting) {
1169 hr_dev->dis_db = true;
1170
1171 dev_warn(hr_dev->dev,
1172 "Func clear is pending, device in resetting state.\n");
1173 end = HNS_ROCE_V2_HW_RST_TIMEOUT;
1174 while (end) {
1175 if (!ops->get_hw_reset_stat(handle)) {
1176 hr_dev->is_reset = true;
1177 dev_info(hr_dev->dev,
1178 "Func clear success after reset.\n");
1179 return;
1180 }
1181 msleep(HNS_ROCE_V2_HW_RST_COMPLETION_WAIT);
1182 end -= HNS_ROCE_V2_HW_RST_COMPLETION_WAIT;
1183 }
1184
1185 dev_warn(hr_dev->dev, "Func clear failed.\n");
1186 } else if (sw_resetting && instance_stage == HNS_ROCE_STATE_INIT) {
1187 hr_dev->dis_db = true;
1188
1189 dev_warn(hr_dev->dev,
1190 "Func clear is pending, device in resetting state.\n");
1191 end = HNS_ROCE_V2_HW_RST_TIMEOUT;
1192 while (end) {
1193 if (ops->ae_dev_reset_cnt(handle) !=
1194 hr_dev->reset_cnt) {
1195 hr_dev->is_reset = true;
1196 dev_info(hr_dev->dev,
1197 "Func clear success after sw reset\n");
1198 return;
1199 }
1200 msleep(HNS_ROCE_V2_HW_RST_COMPLETION_WAIT);
1201 end -= HNS_ROCE_V2_HW_RST_COMPLETION_WAIT;
1202 }
1203
1204 dev_warn(hr_dev->dev, "Func clear failed because of unfinished sw reset\n");
1205 } else {
1206 if (retval && !flag)
1207 dev_warn(hr_dev->dev,
1208 "Func clear read failed, ret = %d.\n", retval);
1209
1210 dev_warn(hr_dev->dev, "Func clear failed.\n");
1211 }
1212 }
1213 static void hns_roce_function_clear(struct hns_roce_dev *hr_dev)
1214 {
1215 bool fclr_write_fail_flag = false;
1216 struct hns_roce_func_clear *resp;
1217 struct hns_roce_cmq_desc desc;
1218 unsigned long end;
1219 int ret = 0;
1220
1221 if (hns_roce_func_clr_chk_rst(hr_dev))
1222 goto out;
1223
1224 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR, false);
1225 resp = (struct hns_roce_func_clear *)desc.data;
1226
1227 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1228 if (ret) {
1229 fclr_write_fail_flag = true;
1230 dev_err(hr_dev->dev, "Func clear write failed, ret = %d.\n",
1231 ret);
1232 goto out;
1233 }
1234
1235 msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_INTERVAL);
1236 end = HNS_ROCE_V2_FUNC_CLEAR_TIMEOUT_MSECS;
1237 while (end) {
1238 if (hns_roce_func_clr_chk_rst(hr_dev))
1239 goto out;
1240 msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT);
1241 end -= HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT;
1242
1243 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR,
1244 true);
1245
1246 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1247 if (ret)
1248 continue;
1249
1250 if (roce_get_bit(resp->func_done, FUNC_CLEAR_RST_FUN_DONE_S)) {
1251 hr_dev->is_reset = true;
1252 return;
1253 }
1254 }
1255
1256 out:
1257 dev_err(hr_dev->dev, "Func clear fail.\n");
1258 hns_roce_func_clr_rst_prc(hr_dev, ret, fclr_write_fail_flag);
1259 }
1260
1261 static int hns_roce_query_fw_ver(struct hns_roce_dev *hr_dev)
1262 {
1263 struct hns_roce_query_fw_info *resp;
1264 struct hns_roce_cmq_desc desc;
1265 int ret;
1266
1267 hns_roce_cmq_setup_basic_desc(&desc, HNS_QUERY_FW_VER, true);
1268 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1269 if (ret)
1270 return ret;
1271
1272 resp = (struct hns_roce_query_fw_info *)desc.data;
1273 hr_dev->caps.fw_ver = (u64)(le32_to_cpu(resp->fw_ver));
1274
1275 return 0;
1276 }
1277
1278 static int hns_roce_config_global_param(struct hns_roce_dev *hr_dev)
1279 {
1280 struct hns_roce_cfg_global_param *req;
1281 struct hns_roce_cmq_desc desc;
1282
1283 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GLOBAL_PARAM,
1284 false);
1285
1286 req = (struct hns_roce_cfg_global_param *)desc.data;
1287 memset(req, 0, sizeof(*req));
1288 roce_set_field(req->time_cfg_udp_port,
1289 CFG_GLOBAL_PARAM_DATA_0_ROCEE_TIME_1US_CFG_M,
1290 CFG_GLOBAL_PARAM_DATA_0_ROCEE_TIME_1US_CFG_S, 0x3e8);
1291 roce_set_field(req->time_cfg_udp_port,
1292 CFG_GLOBAL_PARAM_DATA_0_ROCEE_UDP_PORT_M,
1293 CFG_GLOBAL_PARAM_DATA_0_ROCEE_UDP_PORT_S, 0x12b7);
1294
1295 return hns_roce_cmq_send(hr_dev, &desc, 1);
1296 }
1297
1298 static int hns_roce_query_pf_resource(struct hns_roce_dev *hr_dev)
1299 {
1300 struct hns_roce_cmq_desc desc[2];
1301 struct hns_roce_pf_res_a *req_a;
1302 struct hns_roce_pf_res_b *req_b;
1303 int ret;
1304 int i;
1305
1306 for (i = 0; i < 2; i++) {
1307 hns_roce_cmq_setup_basic_desc(&desc[i],
1308 HNS_ROCE_OPC_QUERY_PF_RES, true);
1309
1310 if (i == 0)
1311 desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1312 else
1313 desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1314 }
1315
1316 ret = hns_roce_cmq_send(hr_dev, desc, 2);
1317 if (ret)
1318 return ret;
1319
1320 req_a = (struct hns_roce_pf_res_a *)desc[0].data;
1321 req_b = (struct hns_roce_pf_res_b *)desc[1].data;
1322
1323 hr_dev->caps.qpc_bt_num = roce_get_field(req_a->qpc_bt_idx_num,
1324 PF_RES_DATA_1_PF_QPC_BT_NUM_M,
1325 PF_RES_DATA_1_PF_QPC_BT_NUM_S);
1326 hr_dev->caps.srqc_bt_num = roce_get_field(req_a->srqc_bt_idx_num,
1327 PF_RES_DATA_2_PF_SRQC_BT_NUM_M,
1328 PF_RES_DATA_2_PF_SRQC_BT_NUM_S);
1329 hr_dev->caps.cqc_bt_num = roce_get_field(req_a->cqc_bt_idx_num,
1330 PF_RES_DATA_3_PF_CQC_BT_NUM_M,
1331 PF_RES_DATA_3_PF_CQC_BT_NUM_S);
1332 hr_dev->caps.mpt_bt_num = roce_get_field(req_a->mpt_bt_idx_num,
1333 PF_RES_DATA_4_PF_MPT_BT_NUM_M,
1334 PF_RES_DATA_4_PF_MPT_BT_NUM_S);
1335
1336 hr_dev->caps.sl_num = roce_get_field(req_b->qid_idx_sl_num,
1337 PF_RES_DATA_3_PF_SL_NUM_M,
1338 PF_RES_DATA_3_PF_SL_NUM_S);
1339 hr_dev->caps.sccc_bt_num = roce_get_field(req_b->sccc_bt_idx_num,
1340 PF_RES_DATA_4_PF_SCCC_BT_NUM_M,
1341 PF_RES_DATA_4_PF_SCCC_BT_NUM_S);
1342
1343 return 0;
1344 }
1345
1346 static int hns_roce_query_pf_timer_resource(struct hns_roce_dev *hr_dev)
1347 {
1348 struct hns_roce_pf_timer_res_a *req_a;
1349 struct hns_roce_cmq_desc desc[2];
1350 int ret, i;
1351
1352 for (i = 0; i < 2; i++) {
1353 hns_roce_cmq_setup_basic_desc(&desc[i],
1354 HNS_ROCE_OPC_QUERY_PF_TIMER_RES,
1355 true);
1356
1357 if (i == 0)
1358 desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1359 else
1360 desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1361 }
1362
1363 ret = hns_roce_cmq_send(hr_dev, desc, 2);
1364 if (ret)
1365 return ret;
1366
1367 req_a = (struct hns_roce_pf_timer_res_a *)desc[0].data;
1368
1369 hr_dev->caps.qpc_timer_bt_num =
1370 roce_get_field(req_a->qpc_timer_bt_idx_num,
1371 PF_RES_DATA_1_PF_QPC_TIMER_BT_NUM_M,
1372 PF_RES_DATA_1_PF_QPC_TIMER_BT_NUM_S);
1373 hr_dev->caps.cqc_timer_bt_num =
1374 roce_get_field(req_a->cqc_timer_bt_idx_num,
1375 PF_RES_DATA_2_PF_CQC_TIMER_BT_NUM_M,
1376 PF_RES_DATA_2_PF_CQC_TIMER_BT_NUM_S);
1377
1378 return 0;
1379 }
1380
1381 static int hns_roce_set_vf_switch_param(struct hns_roce_dev *hr_dev,
1382 int vf_id)
1383 {
1384 struct hns_roce_cmq_desc desc;
1385 struct hns_roce_vf_switch *swt;
1386 int ret;
1387
1388 swt = (struct hns_roce_vf_switch *)desc.data;
1389 hns_roce_cmq_setup_basic_desc(&desc, HNS_SWITCH_PARAMETER_CFG, true);
1390 swt->rocee_sel |= cpu_to_le32(HNS_ICL_SWITCH_CMD_ROCEE_SEL);
1391 roce_set_field(swt->fun_id,
1392 VF_SWITCH_DATA_FUN_ID_VF_ID_M,
1393 VF_SWITCH_DATA_FUN_ID_VF_ID_S,
1394 vf_id);
1395 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1396 if (ret)
1397 return ret;
1398 desc.flag =
1399 cpu_to_le16(HNS_ROCE_CMD_FLAG_NO_INTR | HNS_ROCE_CMD_FLAG_IN);
1400 desc.flag &= cpu_to_le16(~HNS_ROCE_CMD_FLAG_WR);
1401 roce_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_LPBK_S, 1);
1402 roce_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_LCL_LPBK_S, 0);
1403 roce_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_DST_OVRD_S, 1);
1404
1405 return hns_roce_cmq_send(hr_dev, &desc, 1);
1406 }
1407
1408 static int hns_roce_alloc_vf_resource(struct hns_roce_dev *hr_dev)
1409 {
1410 struct hns_roce_cmq_desc desc[2];
1411 struct hns_roce_vf_res_a *req_a;
1412 struct hns_roce_vf_res_b *req_b;
1413 int i;
1414
1415 req_a = (struct hns_roce_vf_res_a *)desc[0].data;
1416 req_b = (struct hns_roce_vf_res_b *)desc[1].data;
1417 memset(req_a, 0, sizeof(*req_a));
1418 memset(req_b, 0, sizeof(*req_b));
1419 for (i = 0; i < 2; i++) {
1420 hns_roce_cmq_setup_basic_desc(&desc[i],
1421 HNS_ROCE_OPC_ALLOC_VF_RES, false);
1422
1423 if (i == 0)
1424 desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1425 else
1426 desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1427
1428 if (i == 0) {
1429 roce_set_field(req_a->vf_qpc_bt_idx_num,
1430 VF_RES_A_DATA_1_VF_QPC_BT_IDX_M,
1431 VF_RES_A_DATA_1_VF_QPC_BT_IDX_S, 0);
1432 roce_set_field(req_a->vf_qpc_bt_idx_num,
1433 VF_RES_A_DATA_1_VF_QPC_BT_NUM_M,
1434 VF_RES_A_DATA_1_VF_QPC_BT_NUM_S,
1435 HNS_ROCE_VF_QPC_BT_NUM);
1436
1437 roce_set_field(req_a->vf_srqc_bt_idx_num,
1438 VF_RES_A_DATA_2_VF_SRQC_BT_IDX_M,
1439 VF_RES_A_DATA_2_VF_SRQC_BT_IDX_S, 0);
1440 roce_set_field(req_a->vf_srqc_bt_idx_num,
1441 VF_RES_A_DATA_2_VF_SRQC_BT_NUM_M,
1442 VF_RES_A_DATA_2_VF_SRQC_BT_NUM_S,
1443 HNS_ROCE_VF_SRQC_BT_NUM);
1444
1445 roce_set_field(req_a->vf_cqc_bt_idx_num,
1446 VF_RES_A_DATA_3_VF_CQC_BT_IDX_M,
1447 VF_RES_A_DATA_3_VF_CQC_BT_IDX_S, 0);
1448 roce_set_field(req_a->vf_cqc_bt_idx_num,
1449 VF_RES_A_DATA_3_VF_CQC_BT_NUM_M,
1450 VF_RES_A_DATA_3_VF_CQC_BT_NUM_S,
1451 HNS_ROCE_VF_CQC_BT_NUM);
1452
1453 roce_set_field(req_a->vf_mpt_bt_idx_num,
1454 VF_RES_A_DATA_4_VF_MPT_BT_IDX_M,
1455 VF_RES_A_DATA_4_VF_MPT_BT_IDX_S, 0);
1456 roce_set_field(req_a->vf_mpt_bt_idx_num,
1457 VF_RES_A_DATA_4_VF_MPT_BT_NUM_M,
1458 VF_RES_A_DATA_4_VF_MPT_BT_NUM_S,
1459 HNS_ROCE_VF_MPT_BT_NUM);
1460
1461 roce_set_field(req_a->vf_eqc_bt_idx_num,
1462 VF_RES_A_DATA_5_VF_EQC_IDX_M,
1463 VF_RES_A_DATA_5_VF_EQC_IDX_S, 0);
1464 roce_set_field(req_a->vf_eqc_bt_idx_num,
1465 VF_RES_A_DATA_5_VF_EQC_NUM_M,
1466 VF_RES_A_DATA_5_VF_EQC_NUM_S,
1467 HNS_ROCE_VF_EQC_NUM);
1468 } else {
1469 roce_set_field(req_b->vf_smac_idx_num,
1470 VF_RES_B_DATA_1_VF_SMAC_IDX_M,
1471 VF_RES_B_DATA_1_VF_SMAC_IDX_S, 0);
1472 roce_set_field(req_b->vf_smac_idx_num,
1473 VF_RES_B_DATA_1_VF_SMAC_NUM_M,
1474 VF_RES_B_DATA_1_VF_SMAC_NUM_S,
1475 HNS_ROCE_VF_SMAC_NUM);
1476
1477 roce_set_field(req_b->vf_sgid_idx_num,
1478 VF_RES_B_DATA_2_VF_SGID_IDX_M,
1479 VF_RES_B_DATA_2_VF_SGID_IDX_S, 0);
1480 roce_set_field(req_b->vf_sgid_idx_num,
1481 VF_RES_B_DATA_2_VF_SGID_NUM_M,
1482 VF_RES_B_DATA_2_VF_SGID_NUM_S,
1483 HNS_ROCE_VF_SGID_NUM);
1484
1485 roce_set_field(req_b->vf_qid_idx_sl_num,
1486 VF_RES_B_DATA_3_VF_QID_IDX_M,
1487 VF_RES_B_DATA_3_VF_QID_IDX_S, 0);
1488 roce_set_field(req_b->vf_qid_idx_sl_num,
1489 VF_RES_B_DATA_3_VF_SL_NUM_M,
1490 VF_RES_B_DATA_3_VF_SL_NUM_S,
1491 HNS_ROCE_VF_SL_NUM);
1492
1493 roce_set_field(req_b->vf_sccc_idx_num,
1494 VF_RES_B_DATA_4_VF_SCCC_BT_IDX_M,
1495 VF_RES_B_DATA_4_VF_SCCC_BT_IDX_S, 0);
1496 roce_set_field(req_b->vf_sccc_idx_num,
1497 VF_RES_B_DATA_4_VF_SCCC_BT_NUM_M,
1498 VF_RES_B_DATA_4_VF_SCCC_BT_NUM_S,
1499 HNS_ROCE_VF_SCCC_BT_NUM);
1500 }
1501 }
1502
1503 return hns_roce_cmq_send(hr_dev, desc, 2);
1504 }
1505
1506 static int hns_roce_v2_set_bt(struct hns_roce_dev *hr_dev)
1507 {
1508 u8 srqc_hop_num = hr_dev->caps.srqc_hop_num;
1509 u8 qpc_hop_num = hr_dev->caps.qpc_hop_num;
1510 u8 cqc_hop_num = hr_dev->caps.cqc_hop_num;
1511 u8 mpt_hop_num = hr_dev->caps.mpt_hop_num;
1512 u8 sccc_hop_num = hr_dev->caps.sccc_hop_num;
1513 struct hns_roce_cfg_bt_attr *req;
1514 struct hns_roce_cmq_desc desc;
1515
1516 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_BT_ATTR, false);
1517 req = (struct hns_roce_cfg_bt_attr *)desc.data;
1518 memset(req, 0, sizeof(*req));
1519
1520 roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_M,
1521 CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_S,
1522 hr_dev->caps.qpc_ba_pg_sz + PG_SHIFT_OFFSET);
1523 roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_M,
1524 CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_S,
1525 hr_dev->caps.qpc_buf_pg_sz + PG_SHIFT_OFFSET);
1526 roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_M,
1527 CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_S,
1528 qpc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : qpc_hop_num);
1529
1530 roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_M,
1531 CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_S,
1532 hr_dev->caps.srqc_ba_pg_sz + PG_SHIFT_OFFSET);
1533 roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_M,
1534 CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_S,
1535 hr_dev->caps.srqc_buf_pg_sz + PG_SHIFT_OFFSET);
1536 roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_M,
1537 CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_S,
1538 srqc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : srqc_hop_num);
1539
1540 roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_M,
1541 CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_S,
1542 hr_dev->caps.cqc_ba_pg_sz + PG_SHIFT_OFFSET);
1543 roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_M,
1544 CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_S,
1545 hr_dev->caps.cqc_buf_pg_sz + PG_SHIFT_OFFSET);
1546 roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_M,
1547 CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_S,
1548 cqc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : cqc_hop_num);
1549
1550 roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_M,
1551 CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_S,
1552 hr_dev->caps.mpt_ba_pg_sz + PG_SHIFT_OFFSET);
1553 roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_M,
1554 CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_S,
1555 hr_dev->caps.mpt_buf_pg_sz + PG_SHIFT_OFFSET);
1556 roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_M,
1557 CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_S,
1558 mpt_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : mpt_hop_num);
1559
1560 roce_set_field(req->vf_sccc_cfg,
1561 CFG_BT_ATTR_DATA_4_VF_SCCC_BA_PGSZ_M,
1562 CFG_BT_ATTR_DATA_4_VF_SCCC_BA_PGSZ_S,
1563 hr_dev->caps.sccc_ba_pg_sz + PG_SHIFT_OFFSET);
1564 roce_set_field(req->vf_sccc_cfg,
1565 CFG_BT_ATTR_DATA_4_VF_SCCC_BUF_PGSZ_M,
1566 CFG_BT_ATTR_DATA_4_VF_SCCC_BUF_PGSZ_S,
1567 hr_dev->caps.sccc_buf_pg_sz + PG_SHIFT_OFFSET);
1568 roce_set_field(req->vf_sccc_cfg,
1569 CFG_BT_ATTR_DATA_4_VF_SCCC_HOPNUM_M,
1570 CFG_BT_ATTR_DATA_4_VF_SCCC_HOPNUM_S,
1571 sccc_hop_num ==
1572 HNS_ROCE_HOP_NUM_0 ? 0 : sccc_hop_num);
1573
1574 return hns_roce_cmq_send(hr_dev, &desc, 1);
1575 }
1576
1577 static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev)
1578 {
1579 struct hns_roce_caps *caps = &hr_dev->caps;
1580 int ret;
1581
1582 ret = hns_roce_cmq_query_hw_info(hr_dev);
1583 if (ret) {
1584 dev_err(hr_dev->dev, "Query hardware version fail, ret = %d.\n",
1585 ret);
1586 return ret;
1587 }
1588
1589 ret = hns_roce_query_fw_ver(hr_dev);
1590 if (ret) {
1591 dev_err(hr_dev->dev, "Query firmware version fail, ret = %d.\n",
1592 ret);
1593 return ret;
1594 }
1595
1596 ret = hns_roce_config_global_param(hr_dev);
1597 if (ret) {
1598 dev_err(hr_dev->dev, "Configure global param fail, ret = %d.\n",
1599 ret);
1600 return ret;
1601 }
1602
1603 /* Get pf resource owned by every pf */
1604 ret = hns_roce_query_pf_resource(hr_dev);
1605 if (ret) {
1606 dev_err(hr_dev->dev, "Query pf resource fail, ret = %d.\n",
1607 ret);
1608 return ret;
1609 }
1610
1611 if (hr_dev->pci_dev->revision == 0x21) {
1612 ret = hns_roce_query_pf_timer_resource(hr_dev);
1613 if (ret) {
1614 dev_err(hr_dev->dev,
1615 "Query pf timer resource fail, ret = %d.\n",
1616 ret);
1617 return ret;
1618 }
1619 }
1620
1621 ret = hns_roce_alloc_vf_resource(hr_dev);
1622 if (ret) {
1623 dev_err(hr_dev->dev, "Allocate vf resource fail, ret = %d.\n",
1624 ret);
1625 return ret;
1626 }
1627
1628 if (hr_dev->pci_dev->revision == 0x21) {
1629 ret = hns_roce_set_vf_switch_param(hr_dev, 0);
1630 if (ret) {
1631 dev_err(hr_dev->dev,
1632 "Set function switch param fail, ret = %d.\n",
1633 ret);
1634 return ret;
1635 }
1636 }
1637
1638 hr_dev->vendor_part_id = hr_dev->pci_dev->device;
1639 hr_dev->sys_image_guid = be64_to_cpu(hr_dev->ib_dev.node_guid);
1640
1641 caps->num_qps = HNS_ROCE_V2_MAX_QP_NUM;
1642 caps->max_wqes = HNS_ROCE_V2_MAX_WQE_NUM;
1643 caps->num_cqs = HNS_ROCE_V2_MAX_CQ_NUM;
1644 caps->num_srqs = HNS_ROCE_V2_MAX_SRQ_NUM;
1645 caps->min_cqes = HNS_ROCE_MIN_CQE_NUM;
1646 caps->max_cqes = HNS_ROCE_V2_MAX_CQE_NUM;
1647 caps->max_srqwqes = HNS_ROCE_V2_MAX_SRQWQE_NUM;
1648 caps->max_sq_sg = HNS_ROCE_V2_MAX_SQ_SGE_NUM;
1649 caps->max_extend_sg = HNS_ROCE_V2_MAX_EXTEND_SGE_NUM;
1650 caps->max_rq_sg = HNS_ROCE_V2_MAX_RQ_SGE_NUM;
1651 caps->max_sq_inline = HNS_ROCE_V2_MAX_SQ_INLINE;
1652 caps->max_srq_sg = HNS_ROCE_V2_MAX_SRQ_SGE_NUM;
1653 caps->num_uars = HNS_ROCE_V2_UAR_NUM;
1654 caps->phy_num_uars = HNS_ROCE_V2_PHY_UAR_NUM;
1655 caps->num_aeq_vectors = HNS_ROCE_V2_AEQE_VEC_NUM;
1656 caps->num_comp_vectors = HNS_ROCE_V2_COMP_VEC_NUM;
1657 caps->num_other_vectors = HNS_ROCE_V2_ABNORMAL_VEC_NUM;
1658 caps->num_mtpts = HNS_ROCE_V2_MAX_MTPT_NUM;
1659 caps->num_mtt_segs = HNS_ROCE_V2_MAX_MTT_SEGS;
1660 caps->num_cqe_segs = HNS_ROCE_V2_MAX_CQE_SEGS;
1661 caps->num_srqwqe_segs = HNS_ROCE_V2_MAX_SRQWQE_SEGS;
1662 caps->num_idx_segs = HNS_ROCE_V2_MAX_IDX_SEGS;
1663 caps->num_pds = HNS_ROCE_V2_MAX_PD_NUM;
1664 caps->max_qp_init_rdma = HNS_ROCE_V2_MAX_QP_INIT_RDMA;
1665 caps->max_qp_dest_rdma = HNS_ROCE_V2_MAX_QP_DEST_RDMA;
1666 caps->max_sq_desc_sz = HNS_ROCE_V2_MAX_SQ_DESC_SZ;
1667 caps->max_rq_desc_sz = HNS_ROCE_V2_MAX_RQ_DESC_SZ;
1668 caps->max_srq_desc_sz = HNS_ROCE_V2_MAX_SRQ_DESC_SZ;
1669 caps->qpc_entry_sz = HNS_ROCE_V2_QPC_ENTRY_SZ;
1670 caps->irrl_entry_sz = HNS_ROCE_V2_IRRL_ENTRY_SZ;
1671 caps->trrl_entry_sz = HNS_ROCE_V2_TRRL_ENTRY_SZ;
1672 caps->cqc_entry_sz = HNS_ROCE_V2_CQC_ENTRY_SZ;
1673 caps->srqc_entry_sz = HNS_ROCE_V2_SRQC_ENTRY_SZ;
1674 caps->mtpt_entry_sz = HNS_ROCE_V2_MTPT_ENTRY_SZ;
1675 caps->mtt_entry_sz = HNS_ROCE_V2_MTT_ENTRY_SZ;
1676 caps->idx_entry_sz = 4;
1677 caps->cq_entry_sz = HNS_ROCE_V2_CQE_ENTRY_SIZE;
1678 caps->page_size_cap = HNS_ROCE_V2_PAGE_SIZE_SUPPORTED;
1679 caps->reserved_lkey = 0;
1680 caps->reserved_pds = 0;
1681 caps->reserved_mrws = 1;
1682 caps->reserved_uars = 0;
1683 caps->reserved_cqs = 0;
1684 caps->reserved_srqs = 0;
1685 caps->reserved_qps = HNS_ROCE_V2_RSV_QPS;
1686
1687 caps->qpc_ba_pg_sz = 0;
1688 caps->qpc_buf_pg_sz = 0;
1689 caps->qpc_hop_num = HNS_ROCE_CONTEXT_HOP_NUM;
1690 caps->srqc_ba_pg_sz = 0;
1691 caps->srqc_buf_pg_sz = 0;
1692 caps->srqc_hop_num = HNS_ROCE_CONTEXT_HOP_NUM;
1693 caps->cqc_ba_pg_sz = 0;
1694 caps->cqc_buf_pg_sz = 0;
1695 caps->cqc_hop_num = HNS_ROCE_CONTEXT_HOP_NUM;
1696 caps->mpt_ba_pg_sz = 0;
1697 caps->mpt_buf_pg_sz = 0;
1698 caps->mpt_hop_num = HNS_ROCE_CONTEXT_HOP_NUM;
1699 caps->pbl_ba_pg_sz = 2;
1700 caps->pbl_buf_pg_sz = 0;
1701 caps->pbl_hop_num = HNS_ROCE_PBL_HOP_NUM;
1702 caps->mtt_ba_pg_sz = 0;
1703 caps->mtt_buf_pg_sz = 0;
1704 caps->mtt_hop_num = HNS_ROCE_MTT_HOP_NUM;
1705 caps->wqe_sq_hop_num = 2;
1706 caps->wqe_sge_hop_num = 1;
1707 caps->wqe_rq_hop_num = 2;
1708 caps->cqe_ba_pg_sz = 6;
1709 caps->cqe_buf_pg_sz = 0;
1710 caps->cqe_hop_num = HNS_ROCE_CQE_HOP_NUM;
1711 caps->srqwqe_ba_pg_sz = 0;
1712 caps->srqwqe_buf_pg_sz = 0;
1713 caps->srqwqe_hop_num = HNS_ROCE_SRQWQE_HOP_NUM;
1714 caps->idx_ba_pg_sz = 0;
1715 caps->idx_buf_pg_sz = 0;
1716 caps->idx_hop_num = HNS_ROCE_IDX_HOP_NUM;
1717 caps->eqe_ba_pg_sz = 0;
1718 caps->eqe_buf_pg_sz = 0;
1719 caps->eqe_hop_num = HNS_ROCE_EQE_HOP_NUM;
1720 caps->tsq_buf_pg_sz = 0;
1721 caps->chunk_sz = HNS_ROCE_V2_TABLE_CHUNK_SIZE;
1722
1723 caps->flags = HNS_ROCE_CAP_FLAG_REREG_MR |
1724 HNS_ROCE_CAP_FLAG_ROCE_V1_V2 |
1725 HNS_ROCE_CAP_FLAG_RQ_INLINE |
1726 HNS_ROCE_CAP_FLAG_RECORD_DB |
1727 HNS_ROCE_CAP_FLAG_SQ_RECORD_DB;
1728
1729 if (hr_dev->pci_dev->revision == 0x21)
1730 caps->flags |= HNS_ROCE_CAP_FLAG_MW |
1731 HNS_ROCE_CAP_FLAG_FRMR;
1732
1733 caps->pkey_table_len[0] = 1;
1734 caps->gid_table_len[0] = HNS_ROCE_V2_GID_INDEX_NUM;
1735 caps->ceqe_depth = HNS_ROCE_V2_COMP_EQE_NUM;
1736 caps->aeqe_depth = HNS_ROCE_V2_ASYNC_EQE_NUM;
1737 caps->local_ca_ack_delay = 0;
1738 caps->max_mtu = IB_MTU_4096;
1739
1740 caps->max_srqs = HNS_ROCE_V2_MAX_SRQ;
1741 caps->max_srq_wrs = HNS_ROCE_V2_MAX_SRQ_WR;
1742 caps->max_srq_sges = HNS_ROCE_V2_MAX_SRQ_SGE;
1743
1744 if (hr_dev->pci_dev->revision == 0x21) {
1745 caps->flags |= HNS_ROCE_CAP_FLAG_ATOMIC |
1746 HNS_ROCE_CAP_FLAG_SRQ |
1747 HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL;
1748
1749 caps->num_qpc_timer = HNS_ROCE_V2_MAX_QPC_TIMER_NUM;
1750 caps->qpc_timer_entry_sz = HNS_ROCE_V2_QPC_TIMER_ENTRY_SZ;
1751 caps->qpc_timer_ba_pg_sz = 0;
1752 caps->qpc_timer_buf_pg_sz = 0;
1753 caps->qpc_timer_hop_num = HNS_ROCE_HOP_NUM_0;
1754 caps->num_cqc_timer = HNS_ROCE_V2_MAX_CQC_TIMER_NUM;
1755 caps->cqc_timer_entry_sz = HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ;
1756 caps->cqc_timer_ba_pg_sz = 0;
1757 caps->cqc_timer_buf_pg_sz = 0;
1758 caps->cqc_timer_hop_num = HNS_ROCE_HOP_NUM_0;
1759
1760 caps->sccc_entry_sz = HNS_ROCE_V2_SCCC_ENTRY_SZ;
1761 caps->sccc_ba_pg_sz = 0;
1762 caps->sccc_buf_pg_sz = 0;
1763 caps->sccc_hop_num = HNS_ROCE_SCCC_HOP_NUM;
1764 }
1765
1766 ret = hns_roce_v2_set_bt(hr_dev);
1767 if (ret)
1768 dev_err(hr_dev->dev, "Configure bt attribute fail, ret = %d.\n",
1769 ret);
1770
1771 return ret;
1772 }
1773
1774 static int hns_roce_config_link_table(struct hns_roce_dev *hr_dev,
1775 enum hns_roce_link_table_type type)
1776 {
1777 struct hns_roce_cmq_desc desc[2];
1778 struct hns_roce_cfg_llm_a *req_a =
1779 (struct hns_roce_cfg_llm_a *)desc[0].data;
1780 struct hns_roce_cfg_llm_b *req_b =
1781 (struct hns_roce_cfg_llm_b *)desc[1].data;
1782 struct hns_roce_v2_priv *priv = hr_dev->priv;
1783 struct hns_roce_link_table *link_tbl;
1784 struct hns_roce_link_table_entry *entry;
1785 enum hns_roce_opcode_type opcode;
1786 u32 page_num;
1787 int i;
1788
1789 switch (type) {
1790 case TSQ_LINK_TABLE:
1791 link_tbl = &priv->tsq;
1792 opcode = HNS_ROCE_OPC_CFG_EXT_LLM;
1793 break;
1794 case TPQ_LINK_TABLE:
1795 link_tbl = &priv->tpq;
1796 opcode = HNS_ROCE_OPC_CFG_TMOUT_LLM;
1797 break;
1798 default:
1799 return -EINVAL;
1800 }
1801
1802 page_num = link_tbl->npages;
1803 entry = link_tbl->table.buf;
1804 memset(req_a, 0, sizeof(*req_a));
1805 memset(req_b, 0, sizeof(*req_b));
1806
1807 for (i = 0; i < 2; i++) {
1808 hns_roce_cmq_setup_basic_desc(&desc[i], opcode, false);
1809
1810 if (i == 0)
1811 desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1812 else
1813 desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1814
1815 if (i == 0) {
1816 req_a->base_addr_l =
1817 cpu_to_le32(link_tbl->table.map & 0xffffffff);
1818 req_a->base_addr_h =
1819 cpu_to_le32(link_tbl->table.map >> 32);
1820 roce_set_field(req_a->depth_pgsz_init_en,
1821 CFG_LLM_QUE_DEPTH_M,
1822 CFG_LLM_QUE_DEPTH_S,
1823 link_tbl->npages);
1824 roce_set_field(req_a->depth_pgsz_init_en,
1825 CFG_LLM_QUE_PGSZ_M,
1826 CFG_LLM_QUE_PGSZ_S,
1827 link_tbl->pg_sz);
1828 req_a->head_ba_l = cpu_to_le32(entry[0].blk_ba0);
1829 req_a->head_ba_h_nxtptr =
1830 cpu_to_le32(entry[0].blk_ba1_nxt_ptr);
1831 roce_set_field(req_a->head_ptr,
1832 CFG_LLM_HEAD_PTR_M,
1833 CFG_LLM_HEAD_PTR_S, 0);
1834 } else {
1835 req_b->tail_ba_l =
1836 cpu_to_le32(entry[page_num - 1].blk_ba0);
1837 roce_set_field(req_b->tail_ba_h,
1838 CFG_LLM_TAIL_BA_H_M,
1839 CFG_LLM_TAIL_BA_H_S,
1840 entry[page_num - 1].blk_ba1_nxt_ptr &
1841 HNS_ROCE_LINK_TABLE_BA1_M);
1842 roce_set_field(req_b->tail_ptr,
1843 CFG_LLM_TAIL_PTR_M,
1844 CFG_LLM_TAIL_PTR_S,
1845 (entry[page_num - 2].blk_ba1_nxt_ptr &
1846 HNS_ROCE_LINK_TABLE_NXT_PTR_M) >>
1847 HNS_ROCE_LINK_TABLE_NXT_PTR_S);
1848 }
1849 }
1850 roce_set_field(req_a->depth_pgsz_init_en,
1851 CFG_LLM_INIT_EN_M, CFG_LLM_INIT_EN_S, 1);
1852
1853 return hns_roce_cmq_send(hr_dev, desc, 2);
1854 }
1855
1856 static int hns_roce_init_link_table(struct hns_roce_dev *hr_dev,
1857 enum hns_roce_link_table_type type)
1858 {
1859 struct hns_roce_v2_priv *priv = hr_dev->priv;
1860 struct hns_roce_link_table *link_tbl;
1861 struct hns_roce_link_table_entry *entry;
1862 struct device *dev = hr_dev->dev;
1863 u32 buf_chk_sz;
1864 dma_addr_t t;
1865 int func_num = 1;
1866 int pg_num_a;
1867 int pg_num_b;
1868 int pg_num;
1869 int size;
1870 int i;
1871
1872 switch (type) {
1873 case TSQ_LINK_TABLE:
1874 link_tbl = &priv->tsq;
1875 buf_chk_sz = 1 << (hr_dev->caps.tsq_buf_pg_sz + PAGE_SHIFT);
1876 pg_num_a = hr_dev->caps.num_qps * 8 / buf_chk_sz;
1877 pg_num_b = hr_dev->caps.sl_num * 4 + 2;
1878 break;
1879 case TPQ_LINK_TABLE:
1880 link_tbl = &priv->tpq;
1881 buf_chk_sz = 1 << (hr_dev->caps.tpq_buf_pg_sz + PAGE_SHIFT);
1882 pg_num_a = hr_dev->caps.num_cqs * 4 / buf_chk_sz;
1883 pg_num_b = 2 * 4 * func_num + 2;
1884 break;
1885 default:
1886 return -EINVAL;
1887 }
1888
1889 pg_num = max(pg_num_a, pg_num_b);
1890 size = pg_num * sizeof(struct hns_roce_link_table_entry);
1891
1892 link_tbl->table.buf = dma_alloc_coherent(dev, size,
1893 &link_tbl->table.map,
1894 GFP_KERNEL);
1895 if (!link_tbl->table.buf)
1896 goto out;
1897
1898 link_tbl->pg_list = kcalloc(pg_num, sizeof(*link_tbl->pg_list),
1899 GFP_KERNEL);
1900 if (!link_tbl->pg_list)
1901 goto err_kcalloc_failed;
1902
1903 entry = link_tbl->table.buf;
1904 for (i = 0; i < pg_num; ++i) {
1905 link_tbl->pg_list[i].buf = dma_alloc_coherent(dev, buf_chk_sz,
1906 &t, GFP_KERNEL);
1907 if (!link_tbl->pg_list[i].buf)
1908 goto err_alloc_buf_failed;
1909
1910 link_tbl->pg_list[i].map = t;
1911
1912 entry[i].blk_ba0 = (u32)(t >> 12);
1913 entry[i].blk_ba1_nxt_ptr = (u32)(t >> 44);
1914
1915 if (i < (pg_num - 1))
1916 entry[i].blk_ba1_nxt_ptr |=
1917 (i + 1) << HNS_ROCE_LINK_TABLE_NXT_PTR_S;
1918
1919 }
1920 link_tbl->npages = pg_num;
1921 link_tbl->pg_sz = buf_chk_sz;
1922
1923 return hns_roce_config_link_table(hr_dev, type);
1924
1925 err_alloc_buf_failed:
1926 for (i -= 1; i >= 0; i--)
1927 dma_free_coherent(dev, buf_chk_sz,
1928 link_tbl->pg_list[i].buf,
1929 link_tbl->pg_list[i].map);
1930 kfree(link_tbl->pg_list);
1931
1932 err_kcalloc_failed:
1933 dma_free_coherent(dev, size, link_tbl->table.buf,
1934 link_tbl->table.map);
1935
1936 out:
1937 return -ENOMEM;
1938 }
1939
1940 static void hns_roce_free_link_table(struct hns_roce_dev *hr_dev,
1941 struct hns_roce_link_table *link_tbl)
1942 {
1943 struct device *dev = hr_dev->dev;
1944 int size;
1945 int i;
1946
1947 size = link_tbl->npages * sizeof(struct hns_roce_link_table_entry);
1948
1949 for (i = 0; i < link_tbl->npages; ++i)
1950 if (link_tbl->pg_list[i].buf)
1951 dma_free_coherent(dev, link_tbl->pg_sz,
1952 link_tbl->pg_list[i].buf,
1953 link_tbl->pg_list[i].map);
1954 kfree(link_tbl->pg_list);
1955
1956 dma_free_coherent(dev, size, link_tbl->table.buf,
1957 link_tbl->table.map);
1958 }
1959
1960 static int hns_roce_v2_init(struct hns_roce_dev *hr_dev)
1961 {
1962 struct hns_roce_v2_priv *priv = hr_dev->priv;
1963 int qpc_count, cqc_count;
1964 int ret, i;
1965
1966 /* TSQ includes SQ doorbell and ack doorbell */
1967 ret = hns_roce_init_link_table(hr_dev, TSQ_LINK_TABLE);
1968 if (ret) {
1969 dev_err(hr_dev->dev, "TSQ init failed, ret = %d.\n", ret);
1970 return ret;
1971 }
1972
1973 ret = hns_roce_init_link_table(hr_dev, TPQ_LINK_TABLE);
1974 if (ret) {
1975 dev_err(hr_dev->dev, "TPQ init failed, ret = %d.\n", ret);
1976 goto err_tpq_init_failed;
1977 }
1978
1979 /* Alloc memory for QPC Timer buffer space chunk */
1980 for (qpc_count = 0; qpc_count < hr_dev->caps.qpc_timer_bt_num;
1981 qpc_count++) {
1982 ret = hns_roce_table_get(hr_dev, &hr_dev->qpc_timer_table,
1983 qpc_count);
1984 if (ret) {
1985 dev_err(hr_dev->dev, "QPC Timer get failed\n");
1986 goto err_qpc_timer_failed;
1987 }
1988 }
1989
1990 /* Alloc memory for CQC Timer buffer space chunk */
1991 for (cqc_count = 0; cqc_count < hr_dev->caps.cqc_timer_bt_num;
1992 cqc_count++) {
1993 ret = hns_roce_table_get(hr_dev, &hr_dev->cqc_timer_table,
1994 cqc_count);
1995 if (ret) {
1996 dev_err(hr_dev->dev, "CQC Timer get failed\n");
1997 goto err_cqc_timer_failed;
1998 }
1999 }
2000
2001 return 0;
2002
2003 err_cqc_timer_failed:
2004 for (i = 0; i < cqc_count; i++)
2005 hns_roce_table_put(hr_dev, &hr_dev->cqc_timer_table, i);
2006
2007 err_qpc_timer_failed:
2008 for (i = 0; i < qpc_count; i++)
2009 hns_roce_table_put(hr_dev, &hr_dev->qpc_timer_table, i);
2010
2011 hns_roce_free_link_table(hr_dev, &priv->tpq);
2012
2013 err_tpq_init_failed:
2014 hns_roce_free_link_table(hr_dev, &priv->tsq);
2015
2016 return ret;
2017 }
2018
2019 static void hns_roce_v2_exit(struct hns_roce_dev *hr_dev)
2020 {
2021 struct hns_roce_v2_priv *priv = hr_dev->priv;
2022
2023 if (hr_dev->pci_dev->revision == 0x21)
2024 hns_roce_function_clear(hr_dev);
2025
2026 hns_roce_free_link_table(hr_dev, &priv->tpq);
2027 hns_roce_free_link_table(hr_dev, &priv->tsq);
2028 }
2029
2030 static int hns_roce_query_mbox_status(struct hns_roce_dev *hr_dev)
2031 {
2032 struct hns_roce_cmq_desc desc;
2033 struct hns_roce_mbox_status *mb_st =
2034 (struct hns_roce_mbox_status *)desc.data;
2035 enum hns_roce_cmd_return_status status;
2036
2037 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_MB_ST, true);
2038
2039 status = hns_roce_cmq_send(hr_dev, &desc, 1);
2040 if (status)
2041 return status;
2042
2043 return le32_to_cpu(mb_st->mb_status_hw_run);
2044 }
2045
2046 static int hns_roce_v2_cmd_pending(struct hns_roce_dev *hr_dev)
2047 {
2048 u32 status = hns_roce_query_mbox_status(hr_dev);
2049
2050 return status >> HNS_ROCE_HW_RUN_BIT_SHIFT;
2051 }
2052
2053 static int hns_roce_v2_cmd_complete(struct hns_roce_dev *hr_dev)
2054 {
2055 u32 status = hns_roce_query_mbox_status(hr_dev);
2056
2057 return status & HNS_ROCE_HW_MB_STATUS_MASK;
2058 }
2059
2060 static int hns_roce_mbox_post(struct hns_roce_dev *hr_dev, u64 in_param,
2061 u64 out_param, u32 in_modifier, u8 op_modifier,
2062 u16 op, u16 token, int event)
2063 {
2064 struct hns_roce_cmq_desc desc;
2065 struct hns_roce_post_mbox *mb = (struct hns_roce_post_mbox *)desc.data;
2066
2067 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_POST_MB, false);
2068
2069 mb->in_param_l = cpu_to_le32(in_param);
2070 mb->in_param_h = cpu_to_le32(in_param >> 32);
2071 mb->out_param_l = cpu_to_le32(out_param);
2072 mb->out_param_h = cpu_to_le32(out_param >> 32);
2073 mb->cmd_tag = cpu_to_le32(in_modifier << 8 | op);
2074 mb->token_event_en = cpu_to_le32(event << 16 | token);
2075
2076 return hns_roce_cmq_send(hr_dev, &desc, 1);
2077 }
2078
2079 static int hns_roce_v2_post_mbox(struct hns_roce_dev *hr_dev, u64 in_param,
2080 u64 out_param, u32 in_modifier, u8 op_modifier,
2081 u16 op, u16 token, int event)
2082 {
2083 struct device *dev = hr_dev->dev;
2084 unsigned long end;
2085 int ret;
2086
2087 end = msecs_to_jiffies(HNS_ROCE_V2_GO_BIT_TIMEOUT_MSECS) + jiffies;
2088 while (hns_roce_v2_cmd_pending(hr_dev)) {
2089 if (time_after(jiffies, end)) {
2090 dev_dbg(dev, "jiffies=%d end=%d\n", (int)jiffies,
2091 (int)end);
2092 return -EAGAIN;
2093 }
2094 cond_resched();
2095 }
2096
2097 ret = hns_roce_mbox_post(hr_dev, in_param, out_param, in_modifier,
2098 op_modifier, op, token, event);
2099 if (ret)
2100 dev_err(dev, "Post mailbox fail(%d)\n", ret);
2101
2102 return ret;
2103 }
2104
2105 static int hns_roce_v2_chk_mbox(struct hns_roce_dev *hr_dev,
2106 unsigned long timeout)
2107 {
2108 struct device *dev = hr_dev->dev;
2109 unsigned long end;
2110 u32 status;
2111
2112 end = msecs_to_jiffies(timeout) + jiffies;
2113 while (hns_roce_v2_cmd_pending(hr_dev) && time_before(jiffies, end))
2114 cond_resched();
2115
2116 if (hns_roce_v2_cmd_pending(hr_dev)) {
2117 dev_err(dev, "[cmd_poll]hw run cmd TIMEDOUT!\n");
2118 return -ETIMEDOUT;
2119 }
2120
2121 status = hns_roce_v2_cmd_complete(hr_dev);
2122 if (status != 0x1) {
2123 if (status == CMD_RST_PRC_EBUSY)
2124 return status;
2125
2126 dev_err(dev, "mailbox status 0x%x!\n", status);
2127 return -EBUSY;
2128 }
2129
2130 return 0;
2131 }
2132
2133 static int hns_roce_config_sgid_table(struct hns_roce_dev *hr_dev,
2134 int gid_index, const union ib_gid *gid,
2135 enum hns_roce_sgid_type sgid_type)
2136 {
2137 struct hns_roce_cmq_desc desc;
2138 struct hns_roce_cfg_sgid_tb *sgid_tb =
2139 (struct hns_roce_cfg_sgid_tb *)desc.data;
2140 u32 *p;
2141
2142 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_SGID_TB, false);
2143
2144 roce_set_field(sgid_tb->table_idx_rsv,
2145 CFG_SGID_TB_TABLE_IDX_M,
2146 CFG_SGID_TB_TABLE_IDX_S, gid_index);
2147 roce_set_field(sgid_tb->vf_sgid_type_rsv,
2148 CFG_SGID_TB_VF_SGID_TYPE_M,
2149 CFG_SGID_TB_VF_SGID_TYPE_S, sgid_type);
2150
2151 p = (u32 *)&gid->raw[0];
2152 sgid_tb->vf_sgid_l = cpu_to_le32(*p);
2153
2154 p = (u32 *)&gid->raw[4];
2155 sgid_tb->vf_sgid_ml = cpu_to_le32(*p);
2156
2157 p = (u32 *)&gid->raw[8];
2158 sgid_tb->vf_sgid_mh = cpu_to_le32(*p);
2159
2160 p = (u32 *)&gid->raw[0xc];
2161 sgid_tb->vf_sgid_h = cpu_to_le32(*p);
2162
2163 return hns_roce_cmq_send(hr_dev, &desc, 1);
2164 }
2165
2166 static int hns_roce_v2_set_gid(struct hns_roce_dev *hr_dev, u8 port,
2167 int gid_index, const union ib_gid *gid,
2168 const struct ib_gid_attr *attr)
2169 {
2170 enum hns_roce_sgid_type sgid_type = GID_TYPE_FLAG_ROCE_V1;
2171 int ret;
2172
2173 if (!gid || !attr)
2174 return -EINVAL;
2175
2176 if (attr->gid_type == IB_GID_TYPE_ROCE)
2177 sgid_type = GID_TYPE_FLAG_ROCE_V1;
2178
2179 if (attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) {
2180 if (ipv6_addr_v4mapped((void *)gid))
2181 sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV4;
2182 else
2183 sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV6;
2184 }
2185
2186 ret = hns_roce_config_sgid_table(hr_dev, gid_index, gid, sgid_type);
2187 if (ret)
2188 dev_err(hr_dev->dev, "Configure sgid table failed(%d)!\n", ret);
2189
2190 return ret;
2191 }
2192
2193 static int hns_roce_v2_set_mac(struct hns_roce_dev *hr_dev, u8 phy_port,
2194 u8 *addr)
2195 {
2196 struct hns_roce_cmq_desc desc;
2197 struct hns_roce_cfg_smac_tb *smac_tb =
2198 (struct hns_roce_cfg_smac_tb *)desc.data;
2199 u16 reg_smac_h;
2200 u32 reg_smac_l;
2201
2202 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_SMAC_TB, false);
2203
2204 reg_smac_l = *(u32 *)(&addr[0]);
2205 reg_smac_h = *(u16 *)(&addr[4]);
2206
2207 memset(smac_tb, 0, sizeof(*smac_tb));
2208 roce_set_field(smac_tb->tb_idx_rsv,
2209 CFG_SMAC_TB_IDX_M,
2210 CFG_SMAC_TB_IDX_S, phy_port);
2211 roce_set_field(smac_tb->vf_smac_h_rsv,
2212 CFG_SMAC_TB_VF_SMAC_H_M,
2213 CFG_SMAC_TB_VF_SMAC_H_S, reg_smac_h);
2214 smac_tb->vf_smac_l = cpu_to_le32(reg_smac_l);
2215
2216 return hns_roce_cmq_send(hr_dev, &desc, 1);
2217 }
2218
2219 static int set_mtpt_pbl(struct hns_roce_v2_mpt_entry *mpt_entry,
2220 struct hns_roce_mr *mr)
2221 {
2222 struct sg_dma_page_iter sg_iter;
2223 u64 page_addr;
2224 u64 *pages;
2225 int i;
2226
2227 mpt_entry->pbl_size = cpu_to_le32(mr->pbl_size);
2228 mpt_entry->pbl_ba_l = cpu_to_le32(lower_32_bits(mr->pbl_ba >> 3));
2229 roce_set_field(mpt_entry->byte_48_mode_ba,
2230 V2_MPT_BYTE_48_PBL_BA_H_M, V2_MPT_BYTE_48_PBL_BA_H_S,
2231 upper_32_bits(mr->pbl_ba >> 3));
2232
2233 pages = (u64 *)__get_free_page(GFP_KERNEL);
2234 if (!pages)
2235 return -ENOMEM;
2236
2237 i = 0;
2238 for_each_sg_dma_page(mr->umem->sg_head.sgl, &sg_iter, mr->umem->nmap, 0) {
2239 page_addr = sg_page_iter_dma_address(&sg_iter);
2240 pages[i] = page_addr >> 6;
2241
2242 /* Record the first 2 entry directly to MTPT table */
2243 if (i >= HNS_ROCE_V2_MAX_INNER_MTPT_NUM - 1)
2244 goto found;
2245 i++;
2246 }
2247 found:
2248 mpt_entry->pa0_l = cpu_to_le32(lower_32_bits(pages[0]));
2249 roce_set_field(mpt_entry->byte_56_pa0_h, V2_MPT_BYTE_56_PA0_H_M,
2250 V2_MPT_BYTE_56_PA0_H_S, upper_32_bits(pages[0]));
2251
2252 mpt_entry->pa1_l = cpu_to_le32(lower_32_bits(pages[1]));
2253 roce_set_field(mpt_entry->byte_64_buf_pa1, V2_MPT_BYTE_64_PA1_H_M,
2254 V2_MPT_BYTE_64_PA1_H_S, upper_32_bits(pages[1]));
2255 roce_set_field(mpt_entry->byte_64_buf_pa1,
2256 V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M,
2257 V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S,
2258 mr->pbl_buf_pg_sz + PG_SHIFT_OFFSET);
2259
2260 free_page((unsigned long)pages);
2261
2262 return 0;
2263 }
2264
2265 static int hns_roce_v2_write_mtpt(void *mb_buf, struct hns_roce_mr *mr,
2266 unsigned long mtpt_idx)
2267 {
2268 struct hns_roce_v2_mpt_entry *mpt_entry;
2269 int ret;
2270
2271 mpt_entry = mb_buf;
2272 memset(mpt_entry, 0, sizeof(*mpt_entry));
2273
2274 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,
2275 V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_VALID);
2276 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PBL_HOP_NUM_M,
2277 V2_MPT_BYTE_4_PBL_HOP_NUM_S, mr->pbl_hop_num ==
2278 HNS_ROCE_HOP_NUM_0 ? 0 : mr->pbl_hop_num);
2279 roce_set_field(mpt_entry->byte_4_pd_hop_st,
2280 V2_MPT_BYTE_4_PBL_BA_PG_SZ_M,
2281 V2_MPT_BYTE_4_PBL_BA_PG_SZ_S,
2282 mr->pbl_ba_pg_sz + PG_SHIFT_OFFSET);
2283 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
2284 V2_MPT_BYTE_4_PD_S, mr->pd);
2285
2286 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RA_EN_S, 0);
2287 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 0);
2288 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1);
2289 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_BIND_EN_S,
2290 (mr->access & IB_ACCESS_MW_BIND ? 1 : 0));
2291 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_ATOMIC_EN_S,
2292 mr->access & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0);
2293 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RR_EN_S,
2294 (mr->access & IB_ACCESS_REMOTE_READ ? 1 : 0));
2295 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RW_EN_S,
2296 (mr->access & IB_ACCESS_REMOTE_WRITE ? 1 : 0));
2297 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_LW_EN_S,
2298 (mr->access & IB_ACCESS_LOCAL_WRITE ? 1 : 0));
2299
2300 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_PA_S,
2301 mr->type == MR_TYPE_MR ? 0 : 1);
2302 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_INNER_PA_VLD_S,
2303 1);
2304
2305 mpt_entry->len_l = cpu_to_le32(lower_32_bits(mr->size));
2306 mpt_entry->len_h = cpu_to_le32(upper_32_bits(mr->size));
2307 mpt_entry->lkey = cpu_to_le32(mr->key);
2308 mpt_entry->va_l = cpu_to_le32(lower_32_bits(mr->iova));
2309 mpt_entry->va_h = cpu_to_le32(upper_32_bits(mr->iova));
2310
2311 if (mr->type == MR_TYPE_DMA)
2312 return 0;
2313
2314 ret = set_mtpt_pbl(mpt_entry, mr);
2315
2316 return ret;
2317 }
2318
2319 static int hns_roce_v2_rereg_write_mtpt(struct hns_roce_dev *hr_dev,
2320 struct hns_roce_mr *mr, int flags,
2321 u32 pdn, int mr_access_flags, u64 iova,
2322 u64 size, void *mb_buf)
2323 {
2324 struct hns_roce_v2_mpt_entry *mpt_entry = mb_buf;
2325 int ret = 0;
2326
2327 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,
2328 V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_VALID);
2329
2330 if (flags & IB_MR_REREG_PD) {
2331 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
2332 V2_MPT_BYTE_4_PD_S, pdn);
2333 mr->pd = pdn;
2334 }
2335
2336 if (flags & IB_MR_REREG_ACCESS) {
2337 roce_set_bit(mpt_entry->byte_8_mw_cnt_en,
2338 V2_MPT_BYTE_8_BIND_EN_S,
2339 (mr_access_flags & IB_ACCESS_MW_BIND ? 1 : 0));
2340 roce_set_bit(mpt_entry->byte_8_mw_cnt_en,
2341 V2_MPT_BYTE_8_ATOMIC_EN_S,
2342 mr_access_flags & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0);
2343 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RR_EN_S,
2344 mr_access_flags & IB_ACCESS_REMOTE_READ ? 1 : 0);
2345 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RW_EN_S,
2346 mr_access_flags & IB_ACCESS_REMOTE_WRITE ? 1 : 0);
2347 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_LW_EN_S,
2348 mr_access_flags & IB_ACCESS_LOCAL_WRITE ? 1 : 0);
2349 }
2350
2351 if (flags & IB_MR_REREG_TRANS) {
2352 mpt_entry->va_l = cpu_to_le32(lower_32_bits(iova));
2353 mpt_entry->va_h = cpu_to_le32(upper_32_bits(iova));
2354 mpt_entry->len_l = cpu_to_le32(lower_32_bits(size));
2355 mpt_entry->len_h = cpu_to_le32(upper_32_bits(size));
2356
2357 mr->iova = iova;
2358 mr->size = size;
2359
2360 ret = set_mtpt_pbl(mpt_entry, mr);
2361 }
2362
2363 return ret;
2364 }
2365
2366 static int hns_roce_v2_frmr_write_mtpt(void *mb_buf, struct hns_roce_mr *mr)
2367 {
2368 struct hns_roce_v2_mpt_entry *mpt_entry;
2369
2370 mpt_entry = mb_buf;
2371 memset(mpt_entry, 0, sizeof(*mpt_entry));
2372
2373 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,
2374 V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_FREE);
2375 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PBL_HOP_NUM_M,
2376 V2_MPT_BYTE_4_PBL_HOP_NUM_S, 1);
2377 roce_set_field(mpt_entry->byte_4_pd_hop_st,
2378 V2_MPT_BYTE_4_PBL_BA_PG_SZ_M,
2379 V2_MPT_BYTE_4_PBL_BA_PG_SZ_S,
2380 mr->pbl_ba_pg_sz + PG_SHIFT_OFFSET);
2381 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
2382 V2_MPT_BYTE_4_PD_S, mr->pd);
2383
2384 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RA_EN_S, 1);
2385 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 1);
2386 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1);
2387
2388 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_FRE_S, 1);
2389 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_PA_S, 0);
2390 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_MR_MW_S, 0);
2391 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_BPD_S, 1);
2392
2393 mpt_entry->pbl_size = cpu_to_le32(mr->pbl_size);
2394
2395 mpt_entry->pbl_ba_l = cpu_to_le32(lower_32_bits(mr->pbl_ba >> 3));
2396 roce_set_field(mpt_entry->byte_48_mode_ba, V2_MPT_BYTE_48_PBL_BA_H_M,
2397 V2_MPT_BYTE_48_PBL_BA_H_S,
2398 upper_32_bits(mr->pbl_ba >> 3));
2399
2400 roce_set_field(mpt_entry->byte_64_buf_pa1,
2401 V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M,
2402 V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S,
2403 mr->pbl_buf_pg_sz + PG_SHIFT_OFFSET);
2404
2405 return 0;
2406 }
2407
2408 static int hns_roce_v2_mw_write_mtpt(void *mb_buf, struct hns_roce_mw *mw)
2409 {
2410 struct hns_roce_v2_mpt_entry *mpt_entry;
2411
2412 mpt_entry = mb_buf;
2413 memset(mpt_entry, 0, sizeof(*mpt_entry));
2414
2415 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,
2416 V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_FREE);
2417 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
2418 V2_MPT_BYTE_4_PD_S, mw->pdn);
2419 roce_set_field(mpt_entry->byte_4_pd_hop_st,
2420 V2_MPT_BYTE_4_PBL_HOP_NUM_M,
2421 V2_MPT_BYTE_4_PBL_HOP_NUM_S,
2422 mw->pbl_hop_num == HNS_ROCE_HOP_NUM_0 ?
2423 0 : mw->pbl_hop_num);
2424 roce_set_field(mpt_entry->byte_4_pd_hop_st,
2425 V2_MPT_BYTE_4_PBL_BA_PG_SZ_M,
2426 V2_MPT_BYTE_4_PBL_BA_PG_SZ_S,
2427 mw->pbl_ba_pg_sz + PG_SHIFT_OFFSET);
2428
2429 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 1);
2430 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1);
2431
2432 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_PA_S, 0);
2433 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_MR_MW_S, 1);
2434 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_BPD_S, 1);
2435 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_BQP_S,
2436 mw->ibmw.type == IB_MW_TYPE_1 ? 0 : 1);
2437
2438 roce_set_field(mpt_entry->byte_64_buf_pa1,
2439 V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M,
2440 V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S,
2441 mw->pbl_buf_pg_sz + PG_SHIFT_OFFSET);
2442
2443 mpt_entry->lkey = cpu_to_le32(mw->rkey);
2444
2445 return 0;
2446 }
2447
2448 static void *get_cqe_v2(struct hns_roce_cq *hr_cq, int n)
2449 {
2450 return hns_roce_buf_offset(&hr_cq->buf, n * HNS_ROCE_V2_CQE_ENTRY_SIZE);
2451 }
2452
2453 static void *get_sw_cqe_v2(struct hns_roce_cq *hr_cq, int n)
2454 {
2455 struct hns_roce_v2_cqe *cqe = get_cqe_v2(hr_cq, n & hr_cq->ib_cq.cqe);
2456
2457 /* Get cqe when Owner bit is Conversely with the MSB of cons_idx */
2458 return (roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_OWNER_S) ^
2459 !!(n & hr_cq->cq_depth)) ? cqe : NULL;
2460 }
2461
2462 static struct hns_roce_v2_cqe *next_cqe_sw_v2(struct hns_roce_cq *hr_cq)
2463 {
2464 return get_sw_cqe_v2(hr_cq, hr_cq->cons_index);
2465 }
2466
2467 static void *get_srq_wqe(struct hns_roce_srq *srq, int n)
2468 {
2469 return hns_roce_buf_offset(&srq->buf, n << srq->wqe_shift);
2470 }
2471
2472 static void hns_roce_free_srq_wqe(struct hns_roce_srq *srq, int wqe_index)
2473 {
2474 /* always called with interrupts disabled. */
2475 spin_lock(&srq->lock);
2476
2477 bitmap_clear(srq->idx_que.bitmap, wqe_index, 1);
2478 srq->tail++;
2479
2480 spin_unlock(&srq->lock);
2481 }
2482
2483 static void hns_roce_v2_cq_set_ci(struct hns_roce_cq *hr_cq, u32 cons_index)
2484 {
2485 *hr_cq->set_ci_db = cons_index & 0xffffff;
2486 }
2487
2488 static void __hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
2489 struct hns_roce_srq *srq)
2490 {
2491 struct hns_roce_v2_cqe *cqe, *dest;
2492 u32 prod_index;
2493 int nfreed = 0;
2494 int wqe_index;
2495 u8 owner_bit;
2496
2497 for (prod_index = hr_cq->cons_index; get_sw_cqe_v2(hr_cq, prod_index);
2498 ++prod_index) {
2499 if (prod_index > hr_cq->cons_index + hr_cq->ib_cq.cqe)
2500 break;
2501 }
2502
2503 /*
2504 * Now backwards through the CQ, removing CQ entries
2505 * that match our QP by overwriting them with next entries.
2506 */
2507 while ((int) --prod_index - (int) hr_cq->cons_index >= 0) {
2508 cqe = get_cqe_v2(hr_cq, prod_index & hr_cq->ib_cq.cqe);
2509 if ((roce_get_field(cqe->byte_16, V2_CQE_BYTE_16_LCL_QPN_M,
2510 V2_CQE_BYTE_16_LCL_QPN_S) &
2511 HNS_ROCE_V2_CQE_QPN_MASK) == qpn) {
2512 if (srq &&
2513 roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_S_R_S)) {
2514 wqe_index = roce_get_field(cqe->byte_4,
2515 V2_CQE_BYTE_4_WQE_INDX_M,
2516 V2_CQE_BYTE_4_WQE_INDX_S);
2517 hns_roce_free_srq_wqe(srq, wqe_index);
2518 }
2519 ++nfreed;
2520 } else if (nfreed) {
2521 dest = get_cqe_v2(hr_cq, (prod_index + nfreed) &
2522 hr_cq->ib_cq.cqe);
2523 owner_bit = roce_get_bit(dest->byte_4,
2524 V2_CQE_BYTE_4_OWNER_S);
2525 memcpy(dest, cqe, sizeof(*cqe));
2526 roce_set_bit(dest->byte_4, V2_CQE_BYTE_4_OWNER_S,
2527 owner_bit);
2528 }
2529 }
2530
2531 if (nfreed) {
2532 hr_cq->cons_index += nfreed;
2533 /*
2534 * Make sure update of buffer contents is done before
2535 * updating consumer index.
2536 */
2537 wmb();
2538 hns_roce_v2_cq_set_ci(hr_cq, hr_cq->cons_index);
2539 }
2540 }
2541
2542 static void hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
2543 struct hns_roce_srq *srq)
2544 {
2545 spin_lock_irq(&hr_cq->lock);
2546 __hns_roce_v2_cq_clean(hr_cq, qpn, srq);
2547 spin_unlock_irq(&hr_cq->lock);
2548 }
2549
2550 static void hns_roce_v2_write_cqc(struct hns_roce_dev *hr_dev,
2551 struct hns_roce_cq *hr_cq, void *mb_buf,
2552 u64 *mtts, dma_addr_t dma_handle)
2553 {
2554 struct hns_roce_v2_cq_context *cq_context;
2555
2556 cq_context = mb_buf;
2557 memset(cq_context, 0, sizeof(*cq_context));
2558
2559 roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_CQ_ST_M,
2560 V2_CQC_BYTE_4_CQ_ST_S, V2_CQ_STATE_VALID);
2561 roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_ARM_ST_M,
2562 V2_CQC_BYTE_4_ARM_ST_S, REG_NXT_CEQE);
2563 roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_SHIFT_M,
2564 V2_CQC_BYTE_4_SHIFT_S,
2565 ilog2(hr_cq->cq_depth));
2566 roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_CEQN_M,
2567 V2_CQC_BYTE_4_CEQN_S, hr_cq->vector);
2568
2569 roce_set_field(cq_context->byte_8_cqn, V2_CQC_BYTE_8_CQN_M,
2570 V2_CQC_BYTE_8_CQN_S, hr_cq->cqn);
2571
2572 cq_context->cqe_cur_blk_addr = cpu_to_le32(mtts[0] >> PAGE_ADDR_SHIFT);
2573
2574 roce_set_field(cq_context->byte_16_hop_addr,
2575 V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_M,
2576 V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_S,
2577 mtts[0] >> (32 + PAGE_ADDR_SHIFT));
2578 roce_set_field(cq_context->byte_16_hop_addr,
2579 V2_CQC_BYTE_16_CQE_HOP_NUM_M,
2580 V2_CQC_BYTE_16_CQE_HOP_NUM_S, hr_dev->caps.cqe_hop_num ==
2581 HNS_ROCE_HOP_NUM_0 ? 0 : hr_dev->caps.cqe_hop_num);
2582
2583 cq_context->cqe_nxt_blk_addr = cpu_to_le32(mtts[1] >> PAGE_ADDR_SHIFT);
2584 roce_set_field(cq_context->byte_24_pgsz_addr,
2585 V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_M,
2586 V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_S,
2587 mtts[1] >> (32 + PAGE_ADDR_SHIFT));
2588 roce_set_field(cq_context->byte_24_pgsz_addr,
2589 V2_CQC_BYTE_24_CQE_BA_PG_SZ_M,
2590 V2_CQC_BYTE_24_CQE_BA_PG_SZ_S,
2591 hr_dev->caps.cqe_ba_pg_sz + PG_SHIFT_OFFSET);
2592 roce_set_field(cq_context->byte_24_pgsz_addr,
2593 V2_CQC_BYTE_24_CQE_BUF_PG_SZ_M,
2594 V2_CQC_BYTE_24_CQE_BUF_PG_SZ_S,
2595 hr_dev->caps.cqe_buf_pg_sz + PG_SHIFT_OFFSET);
2596
2597 cq_context->cqe_ba = cpu_to_le32(dma_handle >> 3);
2598
2599 roce_set_field(cq_context->byte_40_cqe_ba, V2_CQC_BYTE_40_CQE_BA_M,
2600 V2_CQC_BYTE_40_CQE_BA_S, (dma_handle >> (32 + 3)));
2601
2602 if (hr_cq->db_en)
2603 roce_set_bit(cq_context->byte_44_db_record,
2604 V2_CQC_BYTE_44_DB_RECORD_EN_S, 1);
2605
2606 roce_set_field(cq_context->byte_44_db_record,
2607 V2_CQC_BYTE_44_DB_RECORD_ADDR_M,
2608 V2_CQC_BYTE_44_DB_RECORD_ADDR_S,
2609 ((u32)hr_cq->db.dma) >> 1);
2610 cq_context->db_record_addr = cpu_to_le32(hr_cq->db.dma >> 32);
2611
2612 roce_set_field(cq_context->byte_56_cqe_period_maxcnt,
2613 V2_CQC_BYTE_56_CQ_MAX_CNT_M,
2614 V2_CQC_BYTE_56_CQ_MAX_CNT_S,
2615 HNS_ROCE_V2_CQ_DEFAULT_BURST_NUM);
2616 roce_set_field(cq_context->byte_56_cqe_period_maxcnt,
2617 V2_CQC_BYTE_56_CQ_PERIOD_M,
2618 V2_CQC_BYTE_56_CQ_PERIOD_S,
2619 HNS_ROCE_V2_CQ_DEFAULT_INTERVAL);
2620 }
2621
2622 static int hns_roce_v2_req_notify_cq(struct ib_cq *ibcq,
2623 enum ib_cq_notify_flags flags)
2624 {
2625 struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device);
2626 struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
2627 u32 notification_flag;
2628 __le32 doorbell[2];
2629
2630 doorbell[0] = 0;
2631 doorbell[1] = 0;
2632
2633 notification_flag = (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ?
2634 V2_CQ_DB_REQ_NOT : V2_CQ_DB_REQ_NOT_SOL;
2635 /*
2636 * flags = 0; Notification Flag = 1, next
2637 * flags = 1; Notification Flag = 0, solocited
2638 */
2639 roce_set_field(doorbell[0], V2_CQ_DB_BYTE_4_TAG_M, V2_DB_BYTE_4_TAG_S,
2640 hr_cq->cqn);
2641 roce_set_field(doorbell[0], V2_CQ_DB_BYTE_4_CMD_M, V2_DB_BYTE_4_CMD_S,
2642 HNS_ROCE_V2_CQ_DB_NTR);
2643 roce_set_field(doorbell[1], V2_CQ_DB_PARAMETER_CONS_IDX_M,
2644 V2_CQ_DB_PARAMETER_CONS_IDX_S,
2645 hr_cq->cons_index & ((hr_cq->cq_depth << 1) - 1));
2646 roce_set_field(doorbell[1], V2_CQ_DB_PARAMETER_CMD_SN_M,
2647 V2_CQ_DB_PARAMETER_CMD_SN_S, hr_cq->arm_sn & 0x3);
2648 roce_set_bit(doorbell[1], V2_CQ_DB_PARAMETER_NOTIFY_S,
2649 notification_flag);
2650
2651 hns_roce_write64(hr_dev, doorbell, hr_cq->cq_db_l);
2652
2653 return 0;
2654 }
2655
2656 static int hns_roce_handle_recv_inl_wqe(struct hns_roce_v2_cqe *cqe,
2657 struct hns_roce_qp **cur_qp,
2658 struct ib_wc *wc)
2659 {
2660 struct hns_roce_rinl_sge *sge_list;
2661 u32 wr_num, wr_cnt, sge_num;
2662 u32 sge_cnt, data_len, size;
2663 void *wqe_buf;
2664
2665 wr_num = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_WQE_INDX_M,
2666 V2_CQE_BYTE_4_WQE_INDX_S) & 0xffff;
2667 wr_cnt = wr_num & ((*cur_qp)->rq.wqe_cnt - 1);
2668
2669 sge_list = (*cur_qp)->rq_inl_buf.wqe_list[wr_cnt].sg_list;
2670 sge_num = (*cur_qp)->rq_inl_buf.wqe_list[wr_cnt].sge_cnt;
2671 wqe_buf = get_recv_wqe(*cur_qp, wr_cnt);
2672 data_len = wc->byte_len;
2673
2674 for (sge_cnt = 0; (sge_cnt < sge_num) && (data_len); sge_cnt++) {
2675 size = min(sge_list[sge_cnt].len, data_len);
2676 memcpy((void *)sge_list[sge_cnt].addr, wqe_buf, size);
2677
2678 data_len -= size;
2679 wqe_buf += size;
2680 }
2681
2682 if (data_len) {
2683 wc->status = IB_WC_LOC_LEN_ERR;
2684 return -EAGAIN;
2685 }
2686
2687 return 0;
2688 }
2689
2690 static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq,
2691 struct hns_roce_qp **cur_qp, struct ib_wc *wc)
2692 {
2693 struct hns_roce_srq *srq = NULL;
2694 struct hns_roce_dev *hr_dev;
2695 struct hns_roce_v2_cqe *cqe;
2696 struct hns_roce_qp *hr_qp;
2697 struct hns_roce_wq *wq;
2698 struct ib_qp_attr attr;
2699 int attr_mask;
2700 int is_send;
2701 u16 wqe_ctr;
2702 u32 opcode;
2703 u32 status;
2704 int qpn;
2705 int ret;
2706
2707 /* Find cqe according to consumer index */
2708 cqe = next_cqe_sw_v2(hr_cq);
2709 if (!cqe)
2710 return -EAGAIN;
2711
2712 ++hr_cq->cons_index;
2713 /* Memory barrier */
2714 rmb();
2715
2716 /* 0->SQ, 1->RQ */
2717 is_send = !roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_S_R_S);
2718
2719 qpn = roce_get_field(cqe->byte_16, V2_CQE_BYTE_16_LCL_QPN_M,
2720 V2_CQE_BYTE_16_LCL_QPN_S);
2721
2722 if (!*cur_qp || (qpn & HNS_ROCE_V2_CQE_QPN_MASK) != (*cur_qp)->qpn) {
2723 hr_dev = to_hr_dev(hr_cq->ib_cq.device);
2724 hr_qp = __hns_roce_qp_lookup(hr_dev, qpn);
2725 if (unlikely(!hr_qp)) {
2726 dev_err(hr_dev->dev, "CQ %06lx with entry for unknown QPN %06x\n",
2727 hr_cq->cqn, (qpn & HNS_ROCE_V2_CQE_QPN_MASK));
2728 return -EINVAL;
2729 }
2730 *cur_qp = hr_qp;
2731 }
2732
2733 wc->qp = &(*cur_qp)->ibqp;
2734 wc->vendor_err = 0;
2735
2736 if (is_send) {
2737 wq = &(*cur_qp)->sq;
2738 if ((*cur_qp)->sq_signal_bits) {
2739 /*
2740 * If sg_signal_bit is 1,
2741 * firstly tail pointer updated to wqe
2742 * which current cqe correspond to
2743 */
2744 wqe_ctr = (u16)roce_get_field(cqe->byte_4,
2745 V2_CQE_BYTE_4_WQE_INDX_M,
2746 V2_CQE_BYTE_4_WQE_INDX_S);
2747 wq->tail += (wqe_ctr - (u16)wq->tail) &
2748 (wq->wqe_cnt - 1);
2749 }
2750
2751 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
2752 ++wq->tail;
2753 } else if ((*cur_qp)->ibqp.srq) {
2754 srq = to_hr_srq((*cur_qp)->ibqp.srq);
2755 wqe_ctr = (u16)roce_get_field(cqe->byte_4,
2756 V2_CQE_BYTE_4_WQE_INDX_M,
2757 V2_CQE_BYTE_4_WQE_INDX_S);
2758 wc->wr_id = srq->wrid[wqe_ctr];
2759 hns_roce_free_srq_wqe(srq, wqe_ctr);
2760 } else {
2761 /* Update tail pointer, record wr_id */
2762 wq = &(*cur_qp)->rq;
2763 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
2764 ++wq->tail;
2765 }
2766
2767 status = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_STATUS_M,
2768 V2_CQE_BYTE_4_STATUS_S);
2769 switch (status & HNS_ROCE_V2_CQE_STATUS_MASK) {
2770 case HNS_ROCE_CQE_V2_SUCCESS:
2771 wc->status = IB_WC_SUCCESS;
2772 break;
2773 case HNS_ROCE_CQE_V2_LOCAL_LENGTH_ERR:
2774 wc->status = IB_WC_LOC_LEN_ERR;
2775 break;
2776 case HNS_ROCE_CQE_V2_LOCAL_QP_OP_ERR:
2777 wc->status = IB_WC_LOC_QP_OP_ERR;
2778 break;
2779 case HNS_ROCE_CQE_V2_LOCAL_PROT_ERR:
2780 wc->status = IB_WC_LOC_PROT_ERR;
2781 break;
2782 case HNS_ROCE_CQE_V2_WR_FLUSH_ERR:
2783 wc->status = IB_WC_WR_FLUSH_ERR;
2784 break;
2785 case HNS_ROCE_CQE_V2_MW_BIND_ERR:
2786 wc->status = IB_WC_MW_BIND_ERR;
2787 break;
2788 case HNS_ROCE_CQE_V2_BAD_RESP_ERR:
2789 wc->status = IB_WC_BAD_RESP_ERR;
2790 break;
2791 case HNS_ROCE_CQE_V2_LOCAL_ACCESS_ERR:
2792 wc->status = IB_WC_LOC_ACCESS_ERR;
2793 break;
2794 case HNS_ROCE_CQE_V2_REMOTE_INVAL_REQ_ERR:
2795 wc->status = IB_WC_REM_INV_REQ_ERR;
2796 break;
2797 case HNS_ROCE_CQE_V2_REMOTE_ACCESS_ERR:
2798 wc->status = IB_WC_REM_ACCESS_ERR;
2799 break;
2800 case HNS_ROCE_CQE_V2_REMOTE_OP_ERR:
2801 wc->status = IB_WC_REM_OP_ERR;
2802 break;
2803 case HNS_ROCE_CQE_V2_TRANSPORT_RETRY_EXC_ERR:
2804 wc->status = IB_WC_RETRY_EXC_ERR;
2805 break;
2806 case HNS_ROCE_CQE_V2_RNR_RETRY_EXC_ERR:
2807 wc->status = IB_WC_RNR_RETRY_EXC_ERR;
2808 break;
2809 case HNS_ROCE_CQE_V2_REMOTE_ABORT_ERR:
2810 wc->status = IB_WC_REM_ABORT_ERR;
2811 break;
2812 default:
2813 wc->status = IB_WC_GENERAL_ERR;
2814 break;
2815 }
2816
2817 /* flush cqe if wc status is error, excluding flush error */
2818 if ((wc->status != IB_WC_SUCCESS) &&
2819 (wc->status != IB_WC_WR_FLUSH_ERR)) {
2820 attr_mask = IB_QP_STATE;
2821 attr.qp_state = IB_QPS_ERR;
2822 return hns_roce_v2_modify_qp(&(*cur_qp)->ibqp,
2823 &attr, attr_mask,
2824 (*cur_qp)->state, IB_QPS_ERR);
2825 }
2826
2827 if (wc->status == IB_WC_WR_FLUSH_ERR)
2828 return 0;
2829
2830 if (is_send) {
2831 wc->wc_flags = 0;
2832 /* SQ corresponding to CQE */
2833 switch (roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_OPCODE_M,
2834 V2_CQE_BYTE_4_OPCODE_S) & 0x1f) {
2835 case HNS_ROCE_SQ_OPCODE_SEND:
2836 wc->opcode = IB_WC_SEND;
2837 break;
2838 case HNS_ROCE_SQ_OPCODE_SEND_WITH_INV:
2839 wc->opcode = IB_WC_SEND;
2840 break;
2841 case HNS_ROCE_SQ_OPCODE_SEND_WITH_IMM:
2842 wc->opcode = IB_WC_SEND;
2843 wc->wc_flags |= IB_WC_WITH_IMM;
2844 break;
2845 case HNS_ROCE_SQ_OPCODE_RDMA_READ:
2846 wc->opcode = IB_WC_RDMA_READ;
2847 wc->byte_len = le32_to_cpu(cqe->byte_cnt);
2848 break;
2849 case HNS_ROCE_SQ_OPCODE_RDMA_WRITE:
2850 wc->opcode = IB_WC_RDMA_WRITE;
2851 break;
2852 case HNS_ROCE_SQ_OPCODE_RDMA_WRITE_WITH_IMM:
2853 wc->opcode = IB_WC_RDMA_WRITE;
2854 wc->wc_flags |= IB_WC_WITH_IMM;
2855 break;
2856 case HNS_ROCE_SQ_OPCODE_LOCAL_INV:
2857 wc->opcode = IB_WC_LOCAL_INV;
2858 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
2859 break;
2860 case HNS_ROCE_SQ_OPCODE_ATOMIC_COMP_AND_SWAP:
2861 wc->opcode = IB_WC_COMP_SWAP;
2862 wc->byte_len = 8;
2863 break;
2864 case HNS_ROCE_SQ_OPCODE_ATOMIC_FETCH_AND_ADD:
2865 wc->opcode = IB_WC_FETCH_ADD;
2866 wc->byte_len = 8;
2867 break;
2868 case HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_COMP_AND_SWAP:
2869 wc->opcode = IB_WC_MASKED_COMP_SWAP;
2870 wc->byte_len = 8;
2871 break;
2872 case HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_FETCH_AND_ADD:
2873 wc->opcode = IB_WC_MASKED_FETCH_ADD;
2874 wc->byte_len = 8;
2875 break;
2876 case HNS_ROCE_SQ_OPCODE_FAST_REG_WR:
2877 wc->opcode = IB_WC_REG_MR;
2878 break;
2879 case HNS_ROCE_SQ_OPCODE_BIND_MW:
2880 wc->opcode = IB_WC_REG_MR;
2881 break;
2882 default:
2883 wc->status = IB_WC_GENERAL_ERR;
2884 break;
2885 }
2886 } else {
2887 /* RQ correspond to CQE */
2888 wc->byte_len = le32_to_cpu(cqe->byte_cnt);
2889
2890 opcode = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_OPCODE_M,
2891 V2_CQE_BYTE_4_OPCODE_S);
2892 switch (opcode & 0x1f) {
2893 case HNS_ROCE_V2_OPCODE_RDMA_WRITE_IMM:
2894 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
2895 wc->wc_flags = IB_WC_WITH_IMM;
2896 wc->ex.imm_data =
2897 cpu_to_be32(le32_to_cpu(cqe->immtdata));
2898 break;
2899 case HNS_ROCE_V2_OPCODE_SEND:
2900 wc->opcode = IB_WC_RECV;
2901 wc->wc_flags = 0;
2902 break;
2903 case HNS_ROCE_V2_OPCODE_SEND_WITH_IMM:
2904 wc->opcode = IB_WC_RECV;
2905 wc->wc_flags = IB_WC_WITH_IMM;
2906 wc->ex.imm_data =
2907 cpu_to_be32(le32_to_cpu(cqe->immtdata));
2908 break;
2909 case HNS_ROCE_V2_OPCODE_SEND_WITH_INV:
2910 wc->opcode = IB_WC_RECV;
2911 wc->wc_flags = IB_WC_WITH_INVALIDATE;
2912 wc->ex.invalidate_rkey = le32_to_cpu(cqe->rkey);
2913 break;
2914 default:
2915 wc->status = IB_WC_GENERAL_ERR;
2916 break;
2917 }
2918
2919 if ((wc->qp->qp_type == IB_QPT_RC ||
2920 wc->qp->qp_type == IB_QPT_UC) &&
2921 (opcode == HNS_ROCE_V2_OPCODE_SEND ||
2922 opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_IMM ||
2923 opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_INV) &&
2924 (roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_RQ_INLINE_S))) {
2925 ret = hns_roce_handle_recv_inl_wqe(cqe, cur_qp, wc);
2926 if (ret)
2927 return -EAGAIN;
2928 }
2929
2930 wc->sl = (u8)roce_get_field(cqe->byte_32, V2_CQE_BYTE_32_SL_M,
2931 V2_CQE_BYTE_32_SL_S);
2932 wc->src_qp = (u8)roce_get_field(cqe->byte_32,
2933 V2_CQE_BYTE_32_RMT_QPN_M,
2934 V2_CQE_BYTE_32_RMT_QPN_S);
2935 wc->slid = 0;
2936 wc->wc_flags |= (roce_get_bit(cqe->byte_32,
2937 V2_CQE_BYTE_32_GRH_S) ?
2938 IB_WC_GRH : 0);
2939 wc->port_num = roce_get_field(cqe->byte_32,
2940 V2_CQE_BYTE_32_PORTN_M, V2_CQE_BYTE_32_PORTN_S);
2941 wc->pkey_index = 0;
2942 memcpy(wc->smac, cqe->smac, 4);
2943 wc->smac[4] = roce_get_field(cqe->byte_28,
2944 V2_CQE_BYTE_28_SMAC_4_M,
2945 V2_CQE_BYTE_28_SMAC_4_S);
2946 wc->smac[5] = roce_get_field(cqe->byte_28,
2947 V2_CQE_BYTE_28_SMAC_5_M,
2948 V2_CQE_BYTE_28_SMAC_5_S);
2949 wc->wc_flags |= IB_WC_WITH_SMAC;
2950 if (roce_get_bit(cqe->byte_28, V2_CQE_BYTE_28_VID_VLD_S)) {
2951 wc->vlan_id = (u16)roce_get_field(cqe->byte_28,
2952 V2_CQE_BYTE_28_VID_M,
2953 V2_CQE_BYTE_28_VID_S);
2954 wc->wc_flags |= IB_WC_WITH_VLAN;
2955 } else {
2956 wc->vlan_id = 0xffff;
2957 }
2958
2959 wc->network_hdr_type = roce_get_field(cqe->byte_28,
2960 V2_CQE_BYTE_28_PORT_TYPE_M,
2961 V2_CQE_BYTE_28_PORT_TYPE_S);
2962 }
2963
2964 return 0;
2965 }
2966
2967 static int hns_roce_v2_poll_cq(struct ib_cq *ibcq, int num_entries,
2968 struct ib_wc *wc)
2969 {
2970 struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
2971 struct hns_roce_qp *cur_qp = NULL;
2972 unsigned long flags;
2973 int npolled;
2974
2975 spin_lock_irqsave(&hr_cq->lock, flags);
2976
2977 for (npolled = 0; npolled < num_entries; ++npolled) {
2978 if (hns_roce_v2_poll_one(hr_cq, &cur_qp, wc + npolled))
2979 break;
2980 }
2981
2982 if (npolled) {
2983 /* Memory barrier */
2984 wmb();
2985 hns_roce_v2_cq_set_ci(hr_cq, hr_cq->cons_index);
2986 }
2987
2988 spin_unlock_irqrestore(&hr_cq->lock, flags);
2989
2990 return npolled;
2991 }
2992
2993 static int get_op_for_set_hem(struct hns_roce_dev *hr_dev, u32 type,
2994 int step_idx)
2995 {
2996 int op;
2997
2998 if (type == HEM_TYPE_SCCC && step_idx)
2999 return -EINVAL;
3000
3001 switch (type) {
3002 case HEM_TYPE_QPC:
3003 op = HNS_ROCE_CMD_WRITE_QPC_BT0;
3004 break;
3005 case HEM_TYPE_MTPT:
3006 op = HNS_ROCE_CMD_WRITE_MPT_BT0;
3007 break;
3008 case HEM_TYPE_CQC:
3009 op = HNS_ROCE_CMD_WRITE_CQC_BT0;
3010 break;
3011 case HEM_TYPE_SRQC:
3012 op = HNS_ROCE_CMD_WRITE_SRQC_BT0;
3013 break;
3014 case HEM_TYPE_SCCC:
3015 op = HNS_ROCE_CMD_WRITE_SCCC_BT0;
3016 break;
3017 case HEM_TYPE_QPC_TIMER:
3018 op = HNS_ROCE_CMD_WRITE_QPC_TIMER_BT0;
3019 break;
3020 case HEM_TYPE_CQC_TIMER:
3021 op = HNS_ROCE_CMD_WRITE_CQC_TIMER_BT0;
3022 break;
3023 default:
3024 dev_warn(hr_dev->dev,
3025 "Table %d not to be written by mailbox!\n", type);
3026 return -EINVAL;
3027 }
3028
3029 return op + step_idx;
3030 }
3031
3032 static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev,
3033 struct hns_roce_hem_table *table, int obj,
3034 int step_idx)
3035 {
3036 struct hns_roce_cmd_mailbox *mailbox;
3037 struct hns_roce_hem_iter iter;
3038 struct hns_roce_hem_mhop mhop;
3039 struct hns_roce_hem *hem;
3040 unsigned long mhop_obj = obj;
3041 int i, j, k;
3042 int ret = 0;
3043 u64 hem_idx = 0;
3044 u64 l1_idx = 0;
3045 u64 bt_ba = 0;
3046 u32 chunk_ba_num;
3047 u32 hop_num;
3048 int op;
3049
3050 if (!hns_roce_check_whether_mhop(hr_dev, table->type))
3051 return 0;
3052
3053 hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop);
3054 i = mhop.l0_idx;
3055 j = mhop.l1_idx;
3056 k = mhop.l2_idx;
3057 hop_num = mhop.hop_num;
3058 chunk_ba_num = mhop.bt_chunk_size / 8;
3059
3060 if (hop_num == 2) {
3061 hem_idx = i * chunk_ba_num * chunk_ba_num + j * chunk_ba_num +
3062 k;
3063 l1_idx = i * chunk_ba_num + j;
3064 } else if (hop_num == 1) {
3065 hem_idx = i * chunk_ba_num + j;
3066 } else if (hop_num == HNS_ROCE_HOP_NUM_0) {
3067 hem_idx = i;
3068 }
3069
3070 op = get_op_for_set_hem(hr_dev, table->type, step_idx);
3071 if (op == -EINVAL)
3072 return 0;
3073
3074 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
3075 if (IS_ERR(mailbox))
3076 return PTR_ERR(mailbox);
3077
3078 if (table->type == HEM_TYPE_SCCC)
3079 obj = mhop.l0_idx;
3080
3081 if (check_whether_last_step(hop_num, step_idx)) {
3082 hem = table->hem[hem_idx];
3083 for (hns_roce_hem_first(hem, &iter);
3084 !hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) {
3085 bt_ba = hns_roce_hem_addr(&iter);
3086
3087 /* configure the ba, tag, and op */
3088 ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox->dma,
3089 obj, 0, op,
3090 HNS_ROCE_CMD_TIMEOUT_MSECS);
3091 }
3092 } else {
3093 if (step_idx == 0)
3094 bt_ba = table->bt_l0_dma_addr[i];
3095 else if (step_idx == 1 && hop_num == 2)
3096 bt_ba = table->bt_l1_dma_addr[l1_idx];
3097
3098 /* configure the ba, tag, and op */
3099 ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox->dma, obj,
3100 0, op, HNS_ROCE_CMD_TIMEOUT_MSECS);
3101 }
3102
3103 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
3104 return ret;
3105 }
3106
3107 static int hns_roce_v2_clear_hem(struct hns_roce_dev *hr_dev,
3108 struct hns_roce_hem_table *table, int obj,
3109 int step_idx)
3110 {
3111 struct device *dev = hr_dev->dev;
3112 struct hns_roce_cmd_mailbox *mailbox;
3113 int ret;
3114 u16 op = 0xff;
3115
3116 if (!hns_roce_check_whether_mhop(hr_dev, table->type))
3117 return 0;
3118
3119 switch (table->type) {
3120 case HEM_TYPE_QPC:
3121 op = HNS_ROCE_CMD_DESTROY_QPC_BT0;
3122 break;
3123 case HEM_TYPE_MTPT:
3124 op = HNS_ROCE_CMD_DESTROY_MPT_BT0;
3125 break;
3126 case HEM_TYPE_CQC:
3127 op = HNS_ROCE_CMD_DESTROY_CQC_BT0;
3128 break;
3129 case HEM_TYPE_SCCC:
3130 case HEM_TYPE_QPC_TIMER:
3131 case HEM_TYPE_CQC_TIMER:
3132 break;
3133 case HEM_TYPE_SRQC:
3134 op = HNS_ROCE_CMD_DESTROY_SRQC_BT0;
3135 break;
3136 default:
3137 dev_warn(dev, "Table %d not to be destroyed by mailbox!\n",
3138 table->type);
3139 return 0;
3140 }
3141
3142 if (table->type == HEM_TYPE_SCCC ||
3143 table->type == HEM_TYPE_QPC_TIMER ||
3144 table->type == HEM_TYPE_CQC_TIMER)
3145 return 0;
3146
3147 op += step_idx;
3148
3149 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
3150 if (IS_ERR(mailbox))
3151 return PTR_ERR(mailbox);
3152
3153 /* configure the tag and op */
3154 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, obj, 0, op,
3155 HNS_ROCE_CMD_TIMEOUT_MSECS);
3156
3157 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
3158 return ret;
3159 }
3160
3161 static int hns_roce_v2_qp_modify(struct hns_roce_dev *hr_dev,
3162 enum ib_qp_state cur_state,
3163 enum ib_qp_state new_state,
3164 struct hns_roce_v2_qp_context *context,
3165 struct hns_roce_qp *hr_qp)
3166 {
3167 struct hns_roce_cmd_mailbox *mailbox;
3168 int ret;
3169
3170 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
3171 if (IS_ERR(mailbox))
3172 return PTR_ERR(mailbox);
3173
3174 memcpy(mailbox->buf, context, sizeof(*context) * 2);
3175
3176 ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_qp->qpn, 0,
3177 HNS_ROCE_CMD_MODIFY_QPC,
3178 HNS_ROCE_CMD_TIMEOUT_MSECS);
3179
3180 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
3181
3182 return ret;
3183 }
3184
3185 static void set_access_flags(struct hns_roce_qp *hr_qp,
3186 struct hns_roce_v2_qp_context *context,
3187 struct hns_roce_v2_qp_context *qpc_mask,
3188 const struct ib_qp_attr *attr, int attr_mask)
3189 {
3190 u8 dest_rd_atomic;
3191 u32 access_flags;
3192
3193 dest_rd_atomic = (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) ?
3194 attr->max_dest_rd_atomic : hr_qp->resp_depth;
3195
3196 access_flags = (attr_mask & IB_QP_ACCESS_FLAGS) ?
3197 attr->qp_access_flags : hr_qp->atomic_rd_en;
3198
3199 if (!dest_rd_atomic)
3200 access_flags &= IB_ACCESS_REMOTE_WRITE;
3201
3202 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
3203 !!(access_flags & IB_ACCESS_REMOTE_READ));
3204 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S, 0);
3205
3206 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
3207 !!(access_flags & IB_ACCESS_REMOTE_WRITE));
3208 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S, 0);
3209
3210 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
3211 !!(access_flags & IB_ACCESS_REMOTE_ATOMIC));
3212 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S, 0);
3213 }
3214
3215 static void set_qpc_wqe_cnt(struct hns_roce_qp *hr_qp,
3216 struct hns_roce_v2_qp_context *context,
3217 struct hns_roce_v2_qp_context *qpc_mask)
3218 {
3219 if (hr_qp->ibqp.qp_type == IB_QPT_GSI)
3220 roce_set_field(context->byte_4_sqpn_tst,
3221 V2_QPC_BYTE_4_SGE_SHIFT_M,
3222 V2_QPC_BYTE_4_SGE_SHIFT_S,
3223 ilog2((unsigned int)hr_qp->sge.sge_cnt));
3224 else
3225 roce_set_field(context->byte_4_sqpn_tst,
3226 V2_QPC_BYTE_4_SGE_SHIFT_M,
3227 V2_QPC_BYTE_4_SGE_SHIFT_S,
3228 hr_qp->sq.max_gs >
3229 HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE ?
3230 ilog2((unsigned int)hr_qp->sge.sge_cnt) : 0);
3231
3232 roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SGE_SHIFT_M,
3233 V2_QPC_BYTE_4_SGE_SHIFT_S, 0);
3234
3235 roce_set_field(context->byte_20_smac_sgid_idx,
3236 V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S,
3237 ilog2((unsigned int)hr_qp->sq.wqe_cnt));
3238 roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
3239 V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S, 0);
3240
3241 roce_set_field(context->byte_20_smac_sgid_idx,
3242 V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S,
3243 (hr_qp->ibqp.qp_type == IB_QPT_XRC_INI ||
3244 hr_qp->ibqp.qp_type == IB_QPT_XRC_TGT ||
3245 hr_qp->ibqp.srq) ? 0 :
3246 ilog2((unsigned int)hr_qp->rq.wqe_cnt));
3247
3248 roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
3249 V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S, 0);
3250 }
3251
3252 static void modify_qp_reset_to_init(struct ib_qp *ibqp,
3253 const struct ib_qp_attr *attr,
3254 int attr_mask,
3255 struct hns_roce_v2_qp_context *context,
3256 struct hns_roce_v2_qp_context *qpc_mask)
3257 {
3258 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
3259 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3260
3261 /*
3262 * In v2 engine, software pass context and context mask to hardware
3263 * when modifying qp. If software need modify some fields in context,
3264 * we should set all bits of the relevant fields in context mask to
3265 * 0 at the same time, else set them to 0x1.
3266 */
3267 roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
3268 V2_QPC_BYTE_4_TST_S, to_hr_qp_type(hr_qp->ibqp.qp_type));
3269 roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
3270 V2_QPC_BYTE_4_TST_S, 0);
3271
3272 roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
3273 V2_QPC_BYTE_4_SQPN_S, hr_qp->qpn);
3274 roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
3275 V2_QPC_BYTE_4_SQPN_S, 0);
3276
3277 roce_set_field(context->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,
3278 V2_QPC_BYTE_16_PD_S, to_hr_pd(ibqp->pd)->pdn);
3279 roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,
3280 V2_QPC_BYTE_16_PD_S, 0);
3281
3282 roce_set_field(context->byte_20_smac_sgid_idx, V2_QPC_BYTE_20_RQWS_M,
3283 V2_QPC_BYTE_20_RQWS_S, ilog2(hr_qp->rq.max_gs));
3284 roce_set_field(qpc_mask->byte_20_smac_sgid_idx, V2_QPC_BYTE_20_RQWS_M,
3285 V2_QPC_BYTE_20_RQWS_S, 0);
3286
3287 set_qpc_wqe_cnt(hr_qp, context, qpc_mask);
3288
3289 /* No VLAN need to set 0xFFF */
3290 roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_ID_M,
3291 V2_QPC_BYTE_24_VLAN_ID_S, 0xfff);
3292 roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_ID_M,
3293 V2_QPC_BYTE_24_VLAN_ID_S, 0);
3294
3295 /*
3296 * Set some fields in context to zero, Because the default values
3297 * of all fields in context are zero, we need not set them to 0 again.
3298 * but we should set the relevant fields of context mask to 0.
3299 */
3300 roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_SQ_TX_ERR_S, 0);
3301 roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_SQ_RX_ERR_S, 0);
3302 roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_RQ_TX_ERR_S, 0);
3303 roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_RQ_RX_ERR_S, 0);
3304
3305 roce_set_field(qpc_mask->byte_60_qpst_tempid, V2_QPC_BYTE_60_TEMPID_M,
3306 V2_QPC_BYTE_60_TEMPID_S, 0);
3307
3308 roce_set_field(qpc_mask->byte_60_qpst_tempid,
3309 V2_QPC_BYTE_60_SCC_TOKEN_M, V2_QPC_BYTE_60_SCC_TOKEN_S,
3310 0);
3311 roce_set_bit(qpc_mask->byte_60_qpst_tempid,
3312 V2_QPC_BYTE_60_SQ_DB_DOING_S, 0);
3313 roce_set_bit(qpc_mask->byte_60_qpst_tempid,
3314 V2_QPC_BYTE_60_RQ_DB_DOING_S, 0);
3315 roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_CNP_TX_FLAG_S, 0);
3316 roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_CE_FLAG_S, 0);
3317
3318 if (hr_qp->rdb_en) {
3319 roce_set_bit(context->byte_68_rq_db,
3320 V2_QPC_BYTE_68_RQ_RECORD_EN_S, 1);
3321 roce_set_bit(qpc_mask->byte_68_rq_db,
3322 V2_QPC_BYTE_68_RQ_RECORD_EN_S, 0);
3323 }
3324
3325 roce_set_field(context->byte_68_rq_db,
3326 V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_M,
3327 V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_S,
3328 ((u32)hr_qp->rdb.dma) >> 1);
3329 roce_set_field(qpc_mask->byte_68_rq_db,
3330 V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_M,
3331 V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_S, 0);
3332 context->rq_db_record_addr = cpu_to_le32(hr_qp->rdb.dma >> 32);
3333 qpc_mask->rq_db_record_addr = 0;
3334
3335 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RQIE_S,
3336 (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) ? 1 : 0);
3337 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RQIE_S, 0);
3338
3339 roce_set_field(context->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,
3340 V2_QPC_BYTE_80_RX_CQN_S, to_hr_cq(ibqp->recv_cq)->cqn);
3341 roce_set_field(qpc_mask->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,
3342 V2_QPC_BYTE_80_RX_CQN_S, 0);
3343 if (ibqp->srq) {
3344 roce_set_field(context->byte_76_srqn_op_en,
3345 V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S,
3346 to_hr_srq(ibqp->srq)->srqn);
3347 roce_set_field(qpc_mask->byte_76_srqn_op_en,
3348 V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S, 0);
3349 roce_set_bit(context->byte_76_srqn_op_en,
3350 V2_QPC_BYTE_76_SRQ_EN_S, 1);
3351 roce_set_bit(qpc_mask->byte_76_srqn_op_en,
3352 V2_QPC_BYTE_76_SRQ_EN_S, 0);
3353 }
3354
3355 roce_set_field(qpc_mask->byte_84_rq_ci_pi,
3356 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
3357 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0);
3358 roce_set_field(qpc_mask->byte_84_rq_ci_pi,
3359 V2_QPC_BYTE_84_RQ_CONSUMER_IDX_M,
3360 V2_QPC_BYTE_84_RQ_CONSUMER_IDX_S, 0);
3361
3362 roce_set_field(qpc_mask->byte_92_srq_info, V2_QPC_BYTE_92_SRQ_INFO_M,
3363 V2_QPC_BYTE_92_SRQ_INFO_S, 0);
3364
3365 roce_set_field(qpc_mask->byte_96_rx_reqmsn, V2_QPC_BYTE_96_RX_REQ_MSN_M,
3366 V2_QPC_BYTE_96_RX_REQ_MSN_S, 0);
3367
3368 roce_set_field(qpc_mask->byte_104_rq_sge,
3369 V2_QPC_BYTE_104_RQ_CUR_WQE_SGE_NUM_M,
3370 V2_QPC_BYTE_104_RQ_CUR_WQE_SGE_NUM_S, 0);
3371
3372 roce_set_bit(qpc_mask->byte_108_rx_reqepsn,
3373 V2_QPC_BYTE_108_RX_REQ_PSN_ERR_S, 0);
3374 roce_set_field(qpc_mask->byte_108_rx_reqepsn,
3375 V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_M,
3376 V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_S, 0);
3377 roce_set_bit(qpc_mask->byte_108_rx_reqepsn,
3378 V2_QPC_BYTE_108_RX_REQ_RNR_S, 0);
3379
3380 qpc_mask->rq_rnr_timer = 0;
3381 qpc_mask->rx_msg_len = 0;
3382 qpc_mask->rx_rkey_pkt_info = 0;
3383 qpc_mask->rx_va = 0;
3384
3385 roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_HEAD_MAX_M,
3386 V2_QPC_BYTE_132_TRRL_HEAD_MAX_S, 0);
3387 roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_TAIL_MAX_M,
3388 V2_QPC_BYTE_132_TRRL_TAIL_MAX_S, 0);
3389
3390 roce_set_bit(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RQ_RTY_WAIT_DO_S,
3391 0);
3392 roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RAQ_TRRL_HEAD_M,
3393 V2_QPC_BYTE_140_RAQ_TRRL_HEAD_S, 0);
3394 roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RAQ_TRRL_TAIL_M,
3395 V2_QPC_BYTE_140_RAQ_TRRL_TAIL_S, 0);
3396
3397 roce_set_field(qpc_mask->byte_144_raq,
3398 V2_QPC_BYTE_144_RAQ_RTY_INI_PSN_M,
3399 V2_QPC_BYTE_144_RAQ_RTY_INI_PSN_S, 0);
3400 roce_set_field(qpc_mask->byte_144_raq, V2_QPC_BYTE_144_RAQ_CREDIT_M,
3401 V2_QPC_BYTE_144_RAQ_CREDIT_S, 0);
3402 roce_set_bit(qpc_mask->byte_144_raq, V2_QPC_BYTE_144_RESP_RTY_FLG_S, 0);
3403
3404 roce_set_field(qpc_mask->byte_148_raq, V2_QPC_BYTE_148_RQ_MSN_M,
3405 V2_QPC_BYTE_148_RQ_MSN_S, 0);
3406 roce_set_field(qpc_mask->byte_148_raq, V2_QPC_BYTE_148_RAQ_SYNDROME_M,
3407 V2_QPC_BYTE_148_RAQ_SYNDROME_S, 0);
3408
3409 roce_set_field(qpc_mask->byte_152_raq, V2_QPC_BYTE_152_RAQ_PSN_M,
3410 V2_QPC_BYTE_152_RAQ_PSN_S, 0);
3411 roce_set_field(qpc_mask->byte_152_raq,
3412 V2_QPC_BYTE_152_RAQ_TRRL_RTY_HEAD_M,
3413 V2_QPC_BYTE_152_RAQ_TRRL_RTY_HEAD_S, 0);
3414
3415 roce_set_field(qpc_mask->byte_156_raq, V2_QPC_BYTE_156_RAQ_USE_PKTN_M,
3416 V2_QPC_BYTE_156_RAQ_USE_PKTN_S, 0);
3417
3418 roce_set_field(qpc_mask->byte_160_sq_ci_pi,
3419 V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M,
3420 V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S, 0);
3421 roce_set_field(qpc_mask->byte_160_sq_ci_pi,
3422 V2_QPC_BYTE_160_SQ_CONSUMER_IDX_M,
3423 V2_QPC_BYTE_160_SQ_CONSUMER_IDX_S, 0);
3424
3425 roce_set_bit(qpc_mask->byte_168_irrl_idx,
3426 V2_QPC_BYTE_168_POLL_DB_WAIT_DO_S, 0);
3427 roce_set_bit(qpc_mask->byte_168_irrl_idx,
3428 V2_QPC_BYTE_168_SCC_TOKEN_FORBID_SQ_DEQ_S, 0);
3429 roce_set_bit(qpc_mask->byte_168_irrl_idx,
3430 V2_QPC_BYTE_168_WAIT_ACK_TIMEOUT_S, 0);
3431 roce_set_bit(qpc_mask->byte_168_irrl_idx,
3432 V2_QPC_BYTE_168_MSG_RTY_LP_FLG_S, 0);
3433 roce_set_bit(qpc_mask->byte_168_irrl_idx,
3434 V2_QPC_BYTE_168_SQ_INVLD_FLG_S, 0);
3435 roce_set_field(qpc_mask->byte_168_irrl_idx,
3436 V2_QPC_BYTE_168_IRRL_IDX_LSB_M,
3437 V2_QPC_BYTE_168_IRRL_IDX_LSB_S, 0);
3438
3439 roce_set_field(context->byte_172_sq_psn, V2_QPC_BYTE_172_ACK_REQ_FREQ_M,
3440 V2_QPC_BYTE_172_ACK_REQ_FREQ_S, 4);
3441 roce_set_field(qpc_mask->byte_172_sq_psn,
3442 V2_QPC_BYTE_172_ACK_REQ_FREQ_M,
3443 V2_QPC_BYTE_172_ACK_REQ_FREQ_S, 0);
3444
3445 roce_set_bit(qpc_mask->byte_172_sq_psn, V2_QPC_BYTE_172_MSG_RNR_FLG_S,
3446 0);
3447
3448 roce_set_bit(context->byte_172_sq_psn, V2_QPC_BYTE_172_FRE_S, 1);
3449 roce_set_bit(qpc_mask->byte_172_sq_psn, V2_QPC_BYTE_172_FRE_S, 0);
3450
3451 roce_set_field(qpc_mask->byte_176_msg_pktn,
3452 V2_QPC_BYTE_176_MSG_USE_PKTN_M,
3453 V2_QPC_BYTE_176_MSG_USE_PKTN_S, 0);
3454 roce_set_field(qpc_mask->byte_176_msg_pktn,
3455 V2_QPC_BYTE_176_IRRL_HEAD_PRE_M,
3456 V2_QPC_BYTE_176_IRRL_HEAD_PRE_S, 0);
3457
3458 roce_set_field(qpc_mask->byte_184_irrl_idx,
3459 V2_QPC_BYTE_184_IRRL_IDX_MSB_M,
3460 V2_QPC_BYTE_184_IRRL_IDX_MSB_S, 0);
3461
3462 qpc_mask->cur_sge_offset = 0;
3463
3464 roce_set_field(qpc_mask->byte_192_ext_sge,
3465 V2_QPC_BYTE_192_CUR_SGE_IDX_M,
3466 V2_QPC_BYTE_192_CUR_SGE_IDX_S, 0);
3467 roce_set_field(qpc_mask->byte_192_ext_sge,
3468 V2_QPC_BYTE_192_EXT_SGE_NUM_LEFT_M,
3469 V2_QPC_BYTE_192_EXT_SGE_NUM_LEFT_S, 0);
3470
3471 roce_set_field(qpc_mask->byte_196_sq_psn, V2_QPC_BYTE_196_IRRL_HEAD_M,
3472 V2_QPC_BYTE_196_IRRL_HEAD_S, 0);
3473
3474 roce_set_field(qpc_mask->byte_200_sq_max, V2_QPC_BYTE_200_SQ_MAX_IDX_M,
3475 V2_QPC_BYTE_200_SQ_MAX_IDX_S, 0);
3476 roce_set_field(qpc_mask->byte_200_sq_max,
3477 V2_QPC_BYTE_200_LCL_OPERATED_CNT_M,
3478 V2_QPC_BYTE_200_LCL_OPERATED_CNT_S, 0);
3479
3480 roce_set_bit(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_PKT_RNR_FLG_S, 0);
3481 roce_set_bit(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_PKT_RTY_FLG_S, 0);
3482
3483 roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_CHECK_FLG_M,
3484 V2_QPC_BYTE_212_CHECK_FLG_S, 0);
3485
3486 qpc_mask->sq_timer = 0;
3487
3488 roce_set_field(qpc_mask->byte_220_retry_psn_msn,
3489 V2_QPC_BYTE_220_RETRY_MSG_MSN_M,
3490 V2_QPC_BYTE_220_RETRY_MSG_MSN_S, 0);
3491 roce_set_field(qpc_mask->byte_232_irrl_sge,
3492 V2_QPC_BYTE_232_IRRL_SGE_IDX_M,
3493 V2_QPC_BYTE_232_IRRL_SGE_IDX_S, 0);
3494
3495 roce_set_bit(qpc_mask->byte_232_irrl_sge, V2_QPC_BYTE_232_SO_LP_VLD_S,
3496 0);
3497 roce_set_bit(qpc_mask->byte_232_irrl_sge,
3498 V2_QPC_BYTE_232_FENCE_LP_VLD_S, 0);
3499 roce_set_bit(qpc_mask->byte_232_irrl_sge, V2_QPC_BYTE_232_IRRL_LP_VLD_S,
3500 0);
3501
3502 qpc_mask->irrl_cur_sge_offset = 0;
3503
3504 roce_set_field(qpc_mask->byte_240_irrl_tail,
3505 V2_QPC_BYTE_240_IRRL_TAIL_REAL_M,
3506 V2_QPC_BYTE_240_IRRL_TAIL_REAL_S, 0);
3507 roce_set_field(qpc_mask->byte_240_irrl_tail,
3508 V2_QPC_BYTE_240_IRRL_TAIL_RD_M,
3509 V2_QPC_BYTE_240_IRRL_TAIL_RD_S, 0);
3510 roce_set_field(qpc_mask->byte_240_irrl_tail,
3511 V2_QPC_BYTE_240_RX_ACK_MSN_M,
3512 V2_QPC_BYTE_240_RX_ACK_MSN_S, 0);
3513
3514 roce_set_field(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_IRRL_PSN_M,
3515 V2_QPC_BYTE_248_IRRL_PSN_S, 0);
3516 roce_set_bit(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_ACK_PSN_ERR_S,
3517 0);
3518 roce_set_field(qpc_mask->byte_248_ack_psn,
3519 V2_QPC_BYTE_248_ACK_LAST_OPTYPE_M,
3520 V2_QPC_BYTE_248_ACK_LAST_OPTYPE_S, 0);
3521 roce_set_bit(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_IRRL_PSN_VLD_S,
3522 0);
3523 roce_set_bit(qpc_mask->byte_248_ack_psn,
3524 V2_QPC_BYTE_248_RNR_RETRY_FLAG_S, 0);
3525 roce_set_bit(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_CQ_ERR_IND_S,
3526 0);
3527
3528 hr_qp->access_flags = attr->qp_access_flags;
3529 roce_set_field(context->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
3530 V2_QPC_BYTE_252_TX_CQN_S, to_hr_cq(ibqp->send_cq)->cqn);
3531 roce_set_field(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
3532 V2_QPC_BYTE_252_TX_CQN_S, 0);
3533
3534 roce_set_field(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_ERR_TYPE_M,
3535 V2_QPC_BYTE_252_ERR_TYPE_S, 0);
3536
3537 roce_set_field(qpc_mask->byte_256_sqflush_rqcqe,
3538 V2_QPC_BYTE_256_RQ_CQE_IDX_M,
3539 V2_QPC_BYTE_256_RQ_CQE_IDX_S, 0);
3540 roce_set_field(qpc_mask->byte_256_sqflush_rqcqe,
3541 V2_QPC_BYTE_256_SQ_FLUSH_IDX_M,
3542 V2_QPC_BYTE_256_SQ_FLUSH_IDX_S, 0);
3543 }
3544
3545 static void modify_qp_init_to_init(struct ib_qp *ibqp,
3546 const struct ib_qp_attr *attr, int attr_mask,
3547 struct hns_roce_v2_qp_context *context,
3548 struct hns_roce_v2_qp_context *qpc_mask)
3549 {
3550 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3551
3552 /*
3553 * In v2 engine, software pass context and context mask to hardware
3554 * when modifying qp. If software need modify some fields in context,
3555 * we should set all bits of the relevant fields in context mask to
3556 * 0 at the same time, else set them to 0x1.
3557 */
3558 roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
3559 V2_QPC_BYTE_4_TST_S, to_hr_qp_type(hr_qp->ibqp.qp_type));
3560 roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
3561 V2_QPC_BYTE_4_TST_S, 0);
3562
3563 if (attr_mask & IB_QP_ACCESS_FLAGS) {
3564 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
3565 !!(attr->qp_access_flags & IB_ACCESS_REMOTE_READ));
3566 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
3567 0);
3568
3569 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
3570 !!(attr->qp_access_flags &
3571 IB_ACCESS_REMOTE_WRITE));
3572 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
3573 0);
3574
3575 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
3576 !!(attr->qp_access_flags &
3577 IB_ACCESS_REMOTE_ATOMIC));
3578 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
3579 0);
3580 } else {
3581 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
3582 !!(hr_qp->access_flags & IB_ACCESS_REMOTE_READ));
3583 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
3584 0);
3585
3586 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
3587 !!(hr_qp->access_flags & IB_ACCESS_REMOTE_WRITE));
3588 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
3589 0);
3590
3591 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
3592 !!(hr_qp->access_flags & IB_ACCESS_REMOTE_ATOMIC));
3593 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
3594 0);
3595 }
3596
3597 roce_set_field(context->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,
3598 V2_QPC_BYTE_16_PD_S, to_hr_pd(ibqp->pd)->pdn);
3599 roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,
3600 V2_QPC_BYTE_16_PD_S, 0);
3601
3602 roce_set_field(context->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,
3603 V2_QPC_BYTE_80_RX_CQN_S, to_hr_cq(ibqp->recv_cq)->cqn);
3604 roce_set_field(qpc_mask->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,
3605 V2_QPC_BYTE_80_RX_CQN_S, 0);
3606
3607 roce_set_field(context->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
3608 V2_QPC_BYTE_252_TX_CQN_S, to_hr_cq(ibqp->send_cq)->cqn);
3609 roce_set_field(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
3610 V2_QPC_BYTE_252_TX_CQN_S, 0);
3611
3612 if (ibqp->srq) {
3613 roce_set_bit(context->byte_76_srqn_op_en,
3614 V2_QPC_BYTE_76_SRQ_EN_S, 1);
3615 roce_set_bit(qpc_mask->byte_76_srqn_op_en,
3616 V2_QPC_BYTE_76_SRQ_EN_S, 0);
3617 roce_set_field(context->byte_76_srqn_op_en,
3618 V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S,
3619 to_hr_srq(ibqp->srq)->srqn);
3620 roce_set_field(qpc_mask->byte_76_srqn_op_en,
3621 V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S, 0);
3622 }
3623
3624 roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
3625 V2_QPC_BYTE_4_SQPN_S, hr_qp->qpn);
3626 roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
3627 V2_QPC_BYTE_4_SQPN_S, 0);
3628
3629 if (attr_mask & IB_QP_DEST_QPN) {
3630 roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_DQPN_M,
3631 V2_QPC_BYTE_56_DQPN_S, hr_qp->qpn);
3632 roce_set_field(qpc_mask->byte_56_dqpn_err,
3633 V2_QPC_BYTE_56_DQPN_M, V2_QPC_BYTE_56_DQPN_S, 0);
3634 }
3635 }
3636
3637 static bool check_wqe_rq_mtt_count(struct hns_roce_dev *hr_dev,
3638 struct hns_roce_qp *hr_qp, int mtt_cnt,
3639 u32 page_size)
3640 {
3641 struct device *dev = hr_dev->dev;
3642
3643 if (hr_qp->rq.wqe_cnt < 1)
3644 return true;
3645
3646 if (mtt_cnt < 1) {
3647 dev_err(dev, "qp(0x%lx) rqwqe buf ba find failed\n",
3648 hr_qp->qpn);
3649 return false;
3650 }
3651
3652 if (mtt_cnt < MTT_MIN_COUNT &&
3653 (hr_qp->rq.offset + page_size) < hr_qp->buff_size) {
3654 dev_err(dev, "qp(0x%lx) next rqwqe buf ba find failed\n",
3655 hr_qp->qpn);
3656 return false;
3657 }
3658
3659 return true;
3660 }
3661
3662 static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
3663 const struct ib_qp_attr *attr, int attr_mask,
3664 struct hns_roce_v2_qp_context *context,
3665 struct hns_roce_v2_qp_context *qpc_mask)
3666 {
3667 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
3668 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
3669 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3670 struct device *dev = hr_dev->dev;
3671 u64 mtts[MTT_MIN_COUNT] = { 0 };
3672 dma_addr_t dma_handle_3;
3673 dma_addr_t dma_handle_2;
3674 u64 wqe_sge_ba;
3675 u32 page_size;
3676 u8 port_num;
3677 u64 *mtts_3;
3678 u64 *mtts_2;
3679 int count;
3680 u8 *dmac;
3681 u8 *smac;
3682 int port;
3683
3684 /* Search qp buf's mtts */
3685 page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT);
3686 count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr,
3687 hr_qp->rq.offset / page_size, mtts,
3688 MTT_MIN_COUNT, &wqe_sge_ba);
3689 if (!ibqp->srq)
3690 if (!check_wqe_rq_mtt_count(hr_dev, hr_qp, count, page_size))
3691 return -EINVAL;
3692
3693 /* Search IRRL's mtts */
3694 mtts_2 = hns_roce_table_find(hr_dev, &hr_dev->qp_table.irrl_table,
3695 hr_qp->qpn, &dma_handle_2);
3696 if (!mtts_2) {
3697 dev_err(dev, "qp irrl_table find failed\n");
3698 return -EINVAL;
3699 }
3700
3701 /* Search TRRL's mtts */
3702 mtts_3 = hns_roce_table_find(hr_dev, &hr_dev->qp_table.trrl_table,
3703 hr_qp->qpn, &dma_handle_3);
3704 if (!mtts_3) {
3705 dev_err(dev, "qp trrl_table find failed\n");
3706 return -EINVAL;
3707 }
3708
3709 if (attr_mask & IB_QP_ALT_PATH) {
3710 dev_err(dev, "INIT2RTR attr_mask (0x%x) error\n", attr_mask);
3711 return -EINVAL;
3712 }
3713
3714 dmac = (u8 *)attr->ah_attr.roce.dmac;
3715 context->wqe_sge_ba = cpu_to_le32(wqe_sge_ba >> 3);
3716 qpc_mask->wqe_sge_ba = 0;
3717
3718 /*
3719 * In v2 engine, software pass context and context mask to hardware
3720 * when modifying qp. If software need modify some fields in context,
3721 * we should set all bits of the relevant fields in context mask to
3722 * 0 at the same time, else set them to 0x1.
3723 */
3724 roce_set_field(context->byte_12_sq_hop, V2_QPC_BYTE_12_WQE_SGE_BA_M,
3725 V2_QPC_BYTE_12_WQE_SGE_BA_S, wqe_sge_ba >> (32 + 3));
3726 roce_set_field(qpc_mask->byte_12_sq_hop, V2_QPC_BYTE_12_WQE_SGE_BA_M,
3727 V2_QPC_BYTE_12_WQE_SGE_BA_S, 0);
3728
3729 roce_set_field(context->byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M,
3730 V2_QPC_BYTE_12_SQ_HOP_NUM_S,
3731 hr_dev->caps.wqe_sq_hop_num == HNS_ROCE_HOP_NUM_0 ?
3732 0 : hr_dev->caps.wqe_sq_hop_num);
3733 roce_set_field(qpc_mask->byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M,
3734 V2_QPC_BYTE_12_SQ_HOP_NUM_S, 0);
3735
3736 roce_set_field(context->byte_20_smac_sgid_idx,
3737 V2_QPC_BYTE_20_SGE_HOP_NUM_M,
3738 V2_QPC_BYTE_20_SGE_HOP_NUM_S,
3739 ((ibqp->qp_type == IB_QPT_GSI) ||
3740 hr_qp->sq.max_gs > HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) ?
3741 hr_dev->caps.wqe_sge_hop_num : 0);
3742 roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
3743 V2_QPC_BYTE_20_SGE_HOP_NUM_M,
3744 V2_QPC_BYTE_20_SGE_HOP_NUM_S, 0);
3745
3746 roce_set_field(context->byte_20_smac_sgid_idx,
3747 V2_QPC_BYTE_20_RQ_HOP_NUM_M,
3748 V2_QPC_BYTE_20_RQ_HOP_NUM_S,
3749 hr_dev->caps.wqe_rq_hop_num == HNS_ROCE_HOP_NUM_0 ?
3750 0 : hr_dev->caps.wqe_rq_hop_num);
3751 roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
3752 V2_QPC_BYTE_20_RQ_HOP_NUM_M,
3753 V2_QPC_BYTE_20_RQ_HOP_NUM_S, 0);
3754
3755 roce_set_field(context->byte_16_buf_ba_pg_sz,
3756 V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M,
3757 V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S,
3758 hr_qp->wqe_bt_pg_shift + PG_SHIFT_OFFSET);
3759 roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz,
3760 V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M,
3761 V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S, 0);
3762
3763 roce_set_field(context->byte_16_buf_ba_pg_sz,
3764 V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M,
3765 V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S,
3766 hr_dev->caps.mtt_buf_pg_sz + PG_SHIFT_OFFSET);
3767 roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz,
3768 V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M,
3769 V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S, 0);
3770
3771 context->rq_cur_blk_addr = cpu_to_le32(mtts[0] >> PAGE_ADDR_SHIFT);
3772 qpc_mask->rq_cur_blk_addr = 0;
3773
3774 roce_set_field(context->byte_92_srq_info,
3775 V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M,
3776 V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S,
3777 mtts[0] >> (32 + PAGE_ADDR_SHIFT));
3778 roce_set_field(qpc_mask->byte_92_srq_info,
3779 V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M,
3780 V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S, 0);
3781
3782 context->rq_nxt_blk_addr = cpu_to_le32(mtts[1] >> PAGE_ADDR_SHIFT);
3783 qpc_mask->rq_nxt_blk_addr = 0;
3784
3785 roce_set_field(context->byte_104_rq_sge,
3786 V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M,
3787 V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S,
3788 mtts[1] >> (32 + PAGE_ADDR_SHIFT));
3789 roce_set_field(qpc_mask->byte_104_rq_sge,
3790 V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M,
3791 V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S, 0);
3792
3793 roce_set_field(context->byte_132_trrl, V2_QPC_BYTE_132_TRRL_BA_M,
3794 V2_QPC_BYTE_132_TRRL_BA_S, dma_handle_3 >> 4);
3795 roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_BA_M,
3796 V2_QPC_BYTE_132_TRRL_BA_S, 0);
3797 context->trrl_ba = cpu_to_le32(dma_handle_3 >> (16 + 4));
3798 qpc_mask->trrl_ba = 0;
3799 roce_set_field(context->byte_140_raq, V2_QPC_BYTE_140_TRRL_BA_M,
3800 V2_QPC_BYTE_140_TRRL_BA_S,
3801 (u32)(dma_handle_3 >> (32 + 16 + 4)));
3802 roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_TRRL_BA_M,
3803 V2_QPC_BYTE_140_TRRL_BA_S, 0);
3804
3805 context->irrl_ba = cpu_to_le32(dma_handle_2 >> 6);
3806 qpc_mask->irrl_ba = 0;
3807 roce_set_field(context->byte_208_irrl, V2_QPC_BYTE_208_IRRL_BA_M,
3808 V2_QPC_BYTE_208_IRRL_BA_S,
3809 dma_handle_2 >> (32 + 6));
3810 roce_set_field(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_IRRL_BA_M,
3811 V2_QPC_BYTE_208_IRRL_BA_S, 0);
3812
3813 roce_set_bit(context->byte_208_irrl, V2_QPC_BYTE_208_RMT_E2E_S, 1);
3814 roce_set_bit(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_RMT_E2E_S, 0);
3815
3816 roce_set_bit(context->byte_252_err_txcqn, V2_QPC_BYTE_252_SIG_TYPE_S,
3817 hr_qp->sq_signal_bits);
3818 roce_set_bit(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_SIG_TYPE_S,
3819 0);
3820
3821 port = (attr_mask & IB_QP_PORT) ? (attr->port_num - 1) : hr_qp->port;
3822
3823 smac = (u8 *)hr_dev->dev_addr[port];
3824 /* when dmac equals smac or loop_idc is 1, it should loopback */
3825 if (ether_addr_equal_unaligned(dmac, smac) ||
3826 hr_dev->loop_idc == 0x1) {
3827 roce_set_bit(context->byte_28_at_fl, V2_QPC_BYTE_28_LBI_S, 1);
3828 roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_LBI_S, 0);
3829 }
3830
3831 if (attr_mask & IB_QP_DEST_QPN) {
3832 roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_DQPN_M,
3833 V2_QPC_BYTE_56_DQPN_S, attr->dest_qp_num);
3834 roce_set_field(qpc_mask->byte_56_dqpn_err,
3835 V2_QPC_BYTE_56_DQPN_M, V2_QPC_BYTE_56_DQPN_S, 0);
3836 }
3837
3838 /* Configure GID index */
3839 port_num = rdma_ah_get_port_num(&attr->ah_attr);
3840 roce_set_field(context->byte_20_smac_sgid_idx,
3841 V2_QPC_BYTE_20_SGID_IDX_M,
3842 V2_QPC_BYTE_20_SGID_IDX_S,
3843 hns_get_gid_index(hr_dev, port_num - 1,
3844 grh->sgid_index));
3845 roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
3846 V2_QPC_BYTE_20_SGID_IDX_M,
3847 V2_QPC_BYTE_20_SGID_IDX_S, 0);
3848 memcpy(&(context->dmac), dmac, sizeof(u32));
3849 roce_set_field(context->byte_52_udpspn_dmac, V2_QPC_BYTE_52_DMAC_M,
3850 V2_QPC_BYTE_52_DMAC_S, *((u16 *)(&dmac[4])));
3851 qpc_mask->dmac = 0;
3852 roce_set_field(qpc_mask->byte_52_udpspn_dmac, V2_QPC_BYTE_52_DMAC_M,
3853 V2_QPC_BYTE_52_DMAC_S, 0);
3854
3855 /* mtu*(2^LP_PKTN_INI) should not bigger than 1 message length 64kb */
3856 roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_LP_PKTN_INI_M,
3857 V2_QPC_BYTE_56_LP_PKTN_INI_S, 4);
3858 roce_set_field(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_LP_PKTN_INI_M,
3859 V2_QPC_BYTE_56_LP_PKTN_INI_S, 0);
3860
3861 if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_UD)
3862 roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,
3863 V2_QPC_BYTE_24_MTU_S, IB_MTU_4096);
3864 else if (attr_mask & IB_QP_PATH_MTU)
3865 roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,
3866 V2_QPC_BYTE_24_MTU_S, attr->path_mtu);
3867
3868 roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,
3869 V2_QPC_BYTE_24_MTU_S, 0);
3870
3871 roce_set_field(context->byte_84_rq_ci_pi,
3872 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
3873 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, hr_qp->rq.head);
3874 roce_set_field(qpc_mask->byte_84_rq_ci_pi,
3875 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
3876 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0);
3877
3878 roce_set_field(qpc_mask->byte_84_rq_ci_pi,
3879 V2_QPC_BYTE_84_RQ_CONSUMER_IDX_M,
3880 V2_QPC_BYTE_84_RQ_CONSUMER_IDX_S, 0);
3881 roce_set_bit(qpc_mask->byte_108_rx_reqepsn,
3882 V2_QPC_BYTE_108_RX_REQ_PSN_ERR_S, 0);
3883 roce_set_field(qpc_mask->byte_96_rx_reqmsn, V2_QPC_BYTE_96_RX_REQ_MSN_M,
3884 V2_QPC_BYTE_96_RX_REQ_MSN_S, 0);
3885 roce_set_field(qpc_mask->byte_108_rx_reqepsn,
3886 V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_M,
3887 V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_S, 0);
3888
3889 context->rq_rnr_timer = 0;
3890 qpc_mask->rq_rnr_timer = 0;
3891
3892 roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_HEAD_MAX_M,
3893 V2_QPC_BYTE_132_TRRL_HEAD_MAX_S, 0);
3894 roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_TAIL_MAX_M,
3895 V2_QPC_BYTE_132_TRRL_TAIL_MAX_S, 0);
3896
3897 /* rocee send 2^lp_sgen_ini segs every time */
3898 roce_set_field(context->byte_168_irrl_idx,
3899 V2_QPC_BYTE_168_LP_SGEN_INI_M,
3900 V2_QPC_BYTE_168_LP_SGEN_INI_S, 3);
3901 roce_set_field(qpc_mask->byte_168_irrl_idx,
3902 V2_QPC_BYTE_168_LP_SGEN_INI_M,
3903 V2_QPC_BYTE_168_LP_SGEN_INI_S, 0);
3904
3905 return 0;
3906 }
3907
3908 static int modify_qp_rtr_to_rts(struct ib_qp *ibqp,
3909 const struct ib_qp_attr *attr, int attr_mask,
3910 struct hns_roce_v2_qp_context *context,
3911 struct hns_roce_v2_qp_context *qpc_mask)
3912 {
3913 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
3914 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3915 struct device *dev = hr_dev->dev;
3916 u64 sge_cur_blk = 0;
3917 u64 sq_cur_blk = 0;
3918 u32 page_size;
3919 int count;
3920
3921 /* Search qp buf's mtts */
3922 count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, 0, &sq_cur_blk, 1, NULL);
3923 if (count < 1) {
3924 dev_err(dev, "qp(0x%lx) buf pa find failed\n", hr_qp->qpn);
3925 return -EINVAL;
3926 }
3927
3928 if (hr_qp->sge.offset) {
3929 page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT);
3930 count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr,
3931 hr_qp->sge.offset / page_size,
3932 &sge_cur_blk, 1, NULL);
3933 if (count < 1) {
3934 dev_err(dev, "qp(0x%lx) sge pa find failed\n",
3935 hr_qp->qpn);
3936 return -EINVAL;
3937 }
3938 }
3939
3940 /* Not support alternate path and path migration */
3941 if ((attr_mask & IB_QP_ALT_PATH) ||
3942 (attr_mask & IB_QP_PATH_MIG_STATE)) {
3943 dev_err(dev, "RTR2RTS attr_mask (0x%x)error\n", attr_mask);
3944 return -EINVAL;
3945 }
3946
3947 /*
3948 * In v2 engine, software pass context and context mask to hardware
3949 * when modifying qp. If software need modify some fields in context,
3950 * we should set all bits of the relevant fields in context mask to
3951 * 0 at the same time, else set them to 0x1.
3952 */
3953 context->sq_cur_blk_addr = cpu_to_le32(sq_cur_blk >> PAGE_ADDR_SHIFT);
3954 roce_set_field(context->byte_168_irrl_idx,
3955 V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M,
3956 V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S,
3957 sq_cur_blk >> (32 + PAGE_ADDR_SHIFT));
3958 qpc_mask->sq_cur_blk_addr = 0;
3959 roce_set_field(qpc_mask->byte_168_irrl_idx,
3960 V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M,
3961 V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S, 0);
3962
3963 context->sq_cur_sge_blk_addr = ((ibqp->qp_type == IB_QPT_GSI) ||
3964 hr_qp->sq.max_gs > HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) ?
3965 cpu_to_le32(sge_cur_blk >>
3966 PAGE_ADDR_SHIFT) : 0;
3967 roce_set_field(context->byte_184_irrl_idx,
3968 V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M,
3969 V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S,
3970 ((ibqp->qp_type == IB_QPT_GSI) || hr_qp->sq.max_gs >
3971 HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) ?
3972 (sge_cur_blk >>
3973 (32 + PAGE_ADDR_SHIFT)) : 0);
3974 qpc_mask->sq_cur_sge_blk_addr = 0;
3975 roce_set_field(qpc_mask->byte_184_irrl_idx,
3976 V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M,
3977 V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S, 0);
3978
3979 context->rx_sq_cur_blk_addr =
3980 cpu_to_le32(sq_cur_blk >> PAGE_ADDR_SHIFT);
3981 roce_set_field(context->byte_232_irrl_sge,
3982 V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M,
3983 V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S,
3984 sq_cur_blk >> (32 + PAGE_ADDR_SHIFT));
3985 qpc_mask->rx_sq_cur_blk_addr = 0;
3986 roce_set_field(qpc_mask->byte_232_irrl_sge,
3987 V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M,
3988 V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S, 0);
3989
3990 /*
3991 * Set some fields in context to zero, Because the default values
3992 * of all fields in context are zero, we need not set them to 0 again.
3993 * but we should set the relevant fields of context mask to 0.
3994 */
3995 roce_set_field(qpc_mask->byte_232_irrl_sge,
3996 V2_QPC_BYTE_232_IRRL_SGE_IDX_M,
3997 V2_QPC_BYTE_232_IRRL_SGE_IDX_S, 0);
3998
3999 roce_set_field(qpc_mask->byte_240_irrl_tail,
4000 V2_QPC_BYTE_240_RX_ACK_MSN_M,
4001 V2_QPC_BYTE_240_RX_ACK_MSN_S, 0);
4002
4003 roce_set_field(qpc_mask->byte_248_ack_psn,
4004 V2_QPC_BYTE_248_ACK_LAST_OPTYPE_M,
4005 V2_QPC_BYTE_248_ACK_LAST_OPTYPE_S, 0);
4006 roce_set_bit(qpc_mask->byte_248_ack_psn,
4007 V2_QPC_BYTE_248_IRRL_PSN_VLD_S, 0);
4008 roce_set_field(qpc_mask->byte_248_ack_psn,
4009 V2_QPC_BYTE_248_IRRL_PSN_M,
4010 V2_QPC_BYTE_248_IRRL_PSN_S, 0);
4011
4012 roce_set_field(qpc_mask->byte_240_irrl_tail,
4013 V2_QPC_BYTE_240_IRRL_TAIL_REAL_M,
4014 V2_QPC_BYTE_240_IRRL_TAIL_REAL_S, 0);
4015
4016 roce_set_field(qpc_mask->byte_220_retry_psn_msn,
4017 V2_QPC_BYTE_220_RETRY_MSG_MSN_M,
4018 V2_QPC_BYTE_220_RETRY_MSG_MSN_S, 0);
4019
4020 roce_set_bit(qpc_mask->byte_248_ack_psn,
4021 V2_QPC_BYTE_248_RNR_RETRY_FLAG_S, 0);
4022
4023 roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_CHECK_FLG_M,
4024 V2_QPC_BYTE_212_CHECK_FLG_S, 0);
4025
4026 roce_set_field(context->byte_212_lsn, V2_QPC_BYTE_212_LSN_M,
4027 V2_QPC_BYTE_212_LSN_S, 0x100);
4028 roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_LSN_M,
4029 V2_QPC_BYTE_212_LSN_S, 0);
4030
4031 roce_set_field(qpc_mask->byte_196_sq_psn, V2_QPC_BYTE_196_IRRL_HEAD_M,
4032 V2_QPC_BYTE_196_IRRL_HEAD_S, 0);
4033
4034 return 0;
4035 }
4036
4037 static inline bool hns_roce_v2_check_qp_stat(enum ib_qp_state cur_state,
4038 enum ib_qp_state new_state)
4039 {
4040
4041 if ((cur_state != IB_QPS_RESET &&
4042 (new_state == IB_QPS_ERR || new_state == IB_QPS_RESET)) ||
4043 ((cur_state == IB_QPS_RTS || cur_state == IB_QPS_SQD) &&
4044 (new_state == IB_QPS_RTS || new_state == IB_QPS_SQD)) ||
4045 (cur_state == IB_QPS_SQE && new_state == IB_QPS_RTS))
4046 return true;
4047
4048 return false;
4049
4050 }
4051
4052 static int hns_roce_v2_set_path(struct ib_qp *ibqp,
4053 const struct ib_qp_attr *attr,
4054 int attr_mask,
4055 struct hns_roce_v2_qp_context *context,
4056 struct hns_roce_v2_qp_context *qpc_mask)
4057 {
4058 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
4059 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4060 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4061 const struct ib_gid_attr *gid_attr = NULL;
4062 int is_roce_protocol;
4063 u16 vlan_id = 0xffff;
4064 bool is_udp = false;
4065 u8 ib_port;
4066 u8 hr_port;
4067 int ret;
4068
4069 ib_port = (attr_mask & IB_QP_PORT) ? attr->port_num : hr_qp->port + 1;
4070 hr_port = ib_port - 1;
4071 is_roce_protocol = rdma_cap_eth_ah(&hr_dev->ib_dev, ib_port) &&
4072 rdma_ah_get_ah_flags(&attr->ah_attr) & IB_AH_GRH;
4073
4074 if (is_roce_protocol) {
4075 gid_attr = attr->ah_attr.grh.sgid_attr;
4076 ret = rdma_read_gid_l2_fields(gid_attr, &vlan_id, NULL);
4077 if (ret)
4078 return ret;
4079
4080 if (gid_attr)
4081 is_udp = (gid_attr->gid_type ==
4082 IB_GID_TYPE_ROCE_UDP_ENCAP);
4083 }
4084
4085 if (vlan_id < VLAN_N_VID) {
4086 roce_set_bit(context->byte_76_srqn_op_en,
4087 V2_QPC_BYTE_76_RQ_VLAN_EN_S, 1);
4088 roce_set_bit(qpc_mask->byte_76_srqn_op_en,
4089 V2_QPC_BYTE_76_RQ_VLAN_EN_S, 0);
4090 roce_set_bit(context->byte_168_irrl_idx,
4091 V2_QPC_BYTE_168_SQ_VLAN_EN_S, 1);
4092 roce_set_bit(qpc_mask->byte_168_irrl_idx,
4093 V2_QPC_BYTE_168_SQ_VLAN_EN_S, 0);
4094 }
4095
4096 roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_ID_M,
4097 V2_QPC_BYTE_24_VLAN_ID_S, vlan_id);
4098 roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_ID_M,
4099 V2_QPC_BYTE_24_VLAN_ID_S, 0);
4100
4101 if (grh->sgid_index >= hr_dev->caps.gid_table_len[hr_port]) {
4102 dev_err(hr_dev->dev, "sgid_index(%u) too large. max is %d\n",
4103 grh->sgid_index, hr_dev->caps.gid_table_len[hr_port]);
4104 return -EINVAL;
4105 }
4106
4107 if (attr->ah_attr.type != RDMA_AH_ATTR_TYPE_ROCE) {
4108 dev_err(hr_dev->dev, "ah attr is not RDMA roce type\n");
4109 return -EINVAL;
4110 }
4111
4112 roce_set_field(context->byte_52_udpspn_dmac, V2_QPC_BYTE_52_UDPSPN_M,
4113 V2_QPC_BYTE_52_UDPSPN_S,
4114 is_udp ? 0x12b7 : 0);
4115
4116 roce_set_field(qpc_mask->byte_52_udpspn_dmac, V2_QPC_BYTE_52_UDPSPN_M,
4117 V2_QPC_BYTE_52_UDPSPN_S, 0);
4118
4119 roce_set_field(context->byte_20_smac_sgid_idx,
4120 V2_QPC_BYTE_20_SGID_IDX_M, V2_QPC_BYTE_20_SGID_IDX_S,
4121 grh->sgid_index);
4122
4123 roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
4124 V2_QPC_BYTE_20_SGID_IDX_M, V2_QPC_BYTE_20_SGID_IDX_S, 0);
4125
4126 roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_HOP_LIMIT_M,
4127 V2_QPC_BYTE_24_HOP_LIMIT_S, grh->hop_limit);
4128 roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_HOP_LIMIT_M,
4129 V2_QPC_BYTE_24_HOP_LIMIT_S, 0);
4130
4131 if (hr_dev->pci_dev->revision == 0x21 && is_udp)
4132 roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M,
4133 V2_QPC_BYTE_24_TC_S, grh->traffic_class >> 2);
4134 else
4135 roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M,
4136 V2_QPC_BYTE_24_TC_S, grh->traffic_class);
4137 roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M,
4138 V2_QPC_BYTE_24_TC_S, 0);
4139 roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_FL_M,
4140 V2_QPC_BYTE_28_FL_S, grh->flow_label);
4141 roce_set_field(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_FL_M,
4142 V2_QPC_BYTE_28_FL_S, 0);
4143 memcpy(context->dgid, grh->dgid.raw, sizeof(grh->dgid.raw));
4144 memset(qpc_mask->dgid, 0, sizeof(grh->dgid.raw));
4145 roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_SL_M,
4146 V2_QPC_BYTE_28_SL_S, rdma_ah_get_sl(&attr->ah_attr));
4147 roce_set_field(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_SL_M,
4148 V2_QPC_BYTE_28_SL_S, 0);
4149 hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr);
4150
4151 return 0;
4152 }
4153
4154 static int hns_roce_v2_set_abs_fields(struct ib_qp *ibqp,
4155 const struct ib_qp_attr *attr,
4156 int attr_mask,
4157 enum ib_qp_state cur_state,
4158 enum ib_qp_state new_state,
4159 struct hns_roce_v2_qp_context *context,
4160 struct hns_roce_v2_qp_context *qpc_mask)
4161 {
4162 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4163 int ret = 0;
4164
4165 if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
4166 memset(qpc_mask, 0, sizeof(*qpc_mask));
4167 modify_qp_reset_to_init(ibqp, attr, attr_mask, context,
4168 qpc_mask);
4169 } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) {
4170 modify_qp_init_to_init(ibqp, attr, attr_mask, context,
4171 qpc_mask);
4172 } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
4173 ret = modify_qp_init_to_rtr(ibqp, attr, attr_mask, context,
4174 qpc_mask);
4175 if (ret)
4176 goto out;
4177 } else if (cur_state == IB_QPS_RTR && new_state == IB_QPS_RTS) {
4178 ret = modify_qp_rtr_to_rts(ibqp, attr, attr_mask, context,
4179 qpc_mask);
4180 if (ret)
4181 goto out;
4182 } else if (hns_roce_v2_check_qp_stat(cur_state, new_state)) {
4183 /* Nothing */
4184 ;
4185 } else {
4186 dev_err(hr_dev->dev, "Illegal state for QP!\n");
4187 ret = -EINVAL;
4188 goto out;
4189 }
4190
4191 out:
4192 return ret;
4193 }
4194
4195 static int hns_roce_v2_set_opt_fields(struct ib_qp *ibqp,
4196 const struct ib_qp_attr *attr,
4197 int attr_mask,
4198 struct hns_roce_v2_qp_context *context,
4199 struct hns_roce_v2_qp_context *qpc_mask)
4200 {
4201 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4202 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4203 int ret = 0;
4204
4205 if (attr_mask & IB_QP_AV) {
4206 ret = hns_roce_v2_set_path(ibqp, attr, attr_mask, context,
4207 qpc_mask);
4208 if (ret)
4209 return ret;
4210 }
4211
4212 if (attr_mask & IB_QP_TIMEOUT) {
4213 if (attr->timeout < 31) {
4214 roce_set_field(context->byte_28_at_fl,
4215 V2_QPC_BYTE_28_AT_M, V2_QPC_BYTE_28_AT_S,
4216 attr->timeout);
4217 roce_set_field(qpc_mask->byte_28_at_fl,
4218 V2_QPC_BYTE_28_AT_M, V2_QPC_BYTE_28_AT_S,
4219 0);
4220 } else {
4221 dev_warn(hr_dev->dev,
4222 "Local ACK timeout shall be 0 to 30.\n");
4223 }
4224 }
4225
4226 if (attr_mask & IB_QP_RETRY_CNT) {
4227 roce_set_field(context->byte_212_lsn,
4228 V2_QPC_BYTE_212_RETRY_NUM_INIT_M,
4229 V2_QPC_BYTE_212_RETRY_NUM_INIT_S,
4230 attr->retry_cnt);
4231 roce_set_field(qpc_mask->byte_212_lsn,
4232 V2_QPC_BYTE_212_RETRY_NUM_INIT_M,
4233 V2_QPC_BYTE_212_RETRY_NUM_INIT_S, 0);
4234
4235 roce_set_field(context->byte_212_lsn,
4236 V2_QPC_BYTE_212_RETRY_CNT_M,
4237 V2_QPC_BYTE_212_RETRY_CNT_S,
4238 attr->retry_cnt);
4239 roce_set_field(qpc_mask->byte_212_lsn,
4240 V2_QPC_BYTE_212_RETRY_CNT_M,
4241 V2_QPC_BYTE_212_RETRY_CNT_S, 0);
4242 }
4243
4244 if (attr_mask & IB_QP_RNR_RETRY) {
4245 roce_set_field(context->byte_244_rnr_rxack,
4246 V2_QPC_BYTE_244_RNR_NUM_INIT_M,
4247 V2_QPC_BYTE_244_RNR_NUM_INIT_S, attr->rnr_retry);
4248 roce_set_field(qpc_mask->byte_244_rnr_rxack,
4249 V2_QPC_BYTE_244_RNR_NUM_INIT_M,
4250 V2_QPC_BYTE_244_RNR_NUM_INIT_S, 0);
4251
4252 roce_set_field(context->byte_244_rnr_rxack,
4253 V2_QPC_BYTE_244_RNR_CNT_M,
4254 V2_QPC_BYTE_244_RNR_CNT_S, attr->rnr_retry);
4255 roce_set_field(qpc_mask->byte_244_rnr_rxack,
4256 V2_QPC_BYTE_244_RNR_CNT_M,
4257 V2_QPC_BYTE_244_RNR_CNT_S, 0);
4258 }
4259
4260 /* RC&UC&UD required attr */
4261 if (attr_mask & IB_QP_SQ_PSN) {
4262 roce_set_field(context->byte_172_sq_psn,
4263 V2_QPC_BYTE_172_SQ_CUR_PSN_M,
4264 V2_QPC_BYTE_172_SQ_CUR_PSN_S, attr->sq_psn);
4265 roce_set_field(qpc_mask->byte_172_sq_psn,
4266 V2_QPC_BYTE_172_SQ_CUR_PSN_M,
4267 V2_QPC_BYTE_172_SQ_CUR_PSN_S, 0);
4268
4269 roce_set_field(context->byte_196_sq_psn,
4270 V2_QPC_BYTE_196_SQ_MAX_PSN_M,
4271 V2_QPC_BYTE_196_SQ_MAX_PSN_S, attr->sq_psn);
4272 roce_set_field(qpc_mask->byte_196_sq_psn,
4273 V2_QPC_BYTE_196_SQ_MAX_PSN_M,
4274 V2_QPC_BYTE_196_SQ_MAX_PSN_S, 0);
4275
4276 roce_set_field(context->byte_220_retry_psn_msn,
4277 V2_QPC_BYTE_220_RETRY_MSG_PSN_M,
4278 V2_QPC_BYTE_220_RETRY_MSG_PSN_S, attr->sq_psn);
4279 roce_set_field(qpc_mask->byte_220_retry_psn_msn,
4280 V2_QPC_BYTE_220_RETRY_MSG_PSN_M,
4281 V2_QPC_BYTE_220_RETRY_MSG_PSN_S, 0);
4282
4283 roce_set_field(context->byte_224_retry_msg,
4284 V2_QPC_BYTE_224_RETRY_MSG_PSN_M,
4285 V2_QPC_BYTE_224_RETRY_MSG_PSN_S,
4286 attr->sq_psn >> V2_QPC_BYTE_220_RETRY_MSG_PSN_S);
4287 roce_set_field(qpc_mask->byte_224_retry_msg,
4288 V2_QPC_BYTE_224_RETRY_MSG_PSN_M,
4289 V2_QPC_BYTE_224_RETRY_MSG_PSN_S, 0);
4290
4291 roce_set_field(context->byte_224_retry_msg,
4292 V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_M,
4293 V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_S,
4294 attr->sq_psn);
4295 roce_set_field(qpc_mask->byte_224_retry_msg,
4296 V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_M,
4297 V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_S, 0);
4298
4299 roce_set_field(context->byte_244_rnr_rxack,
4300 V2_QPC_BYTE_244_RX_ACK_EPSN_M,
4301 V2_QPC_BYTE_244_RX_ACK_EPSN_S, attr->sq_psn);
4302 roce_set_field(qpc_mask->byte_244_rnr_rxack,
4303 V2_QPC_BYTE_244_RX_ACK_EPSN_M,
4304 V2_QPC_BYTE_244_RX_ACK_EPSN_S, 0);
4305 }
4306
4307 if ((attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) &&
4308 attr->max_dest_rd_atomic) {
4309 roce_set_field(context->byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M,
4310 V2_QPC_BYTE_140_RR_MAX_S,
4311 fls(attr->max_dest_rd_atomic - 1));
4312 roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M,
4313 V2_QPC_BYTE_140_RR_MAX_S, 0);
4314 }
4315
4316 if ((attr_mask & IB_QP_MAX_QP_RD_ATOMIC) && attr->max_rd_atomic) {
4317 roce_set_field(context->byte_208_irrl, V2_QPC_BYTE_208_SR_MAX_M,
4318 V2_QPC_BYTE_208_SR_MAX_S,
4319 fls(attr->max_rd_atomic - 1));
4320 roce_set_field(qpc_mask->byte_208_irrl,
4321 V2_QPC_BYTE_208_SR_MAX_M,
4322 V2_QPC_BYTE_208_SR_MAX_S, 0);
4323 }
4324
4325 if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC))
4326 set_access_flags(hr_qp, context, qpc_mask, attr, attr_mask);
4327
4328 if (attr_mask & IB_QP_MIN_RNR_TIMER) {
4329 roce_set_field(context->byte_80_rnr_rx_cqn,
4330 V2_QPC_BYTE_80_MIN_RNR_TIME_M,
4331 V2_QPC_BYTE_80_MIN_RNR_TIME_S,
4332 attr->min_rnr_timer);
4333 roce_set_field(qpc_mask->byte_80_rnr_rx_cqn,
4334 V2_QPC_BYTE_80_MIN_RNR_TIME_M,
4335 V2_QPC_BYTE_80_MIN_RNR_TIME_S, 0);
4336 }
4337
4338 /* RC&UC required attr */
4339 if (attr_mask & IB_QP_RQ_PSN) {
4340 roce_set_field(context->byte_108_rx_reqepsn,
4341 V2_QPC_BYTE_108_RX_REQ_EPSN_M,
4342 V2_QPC_BYTE_108_RX_REQ_EPSN_S, attr->rq_psn);
4343 roce_set_field(qpc_mask->byte_108_rx_reqepsn,
4344 V2_QPC_BYTE_108_RX_REQ_EPSN_M,
4345 V2_QPC_BYTE_108_RX_REQ_EPSN_S, 0);
4346
4347 roce_set_field(context->byte_152_raq, V2_QPC_BYTE_152_RAQ_PSN_M,
4348 V2_QPC_BYTE_152_RAQ_PSN_S, attr->rq_psn - 1);
4349 roce_set_field(qpc_mask->byte_152_raq,
4350 V2_QPC_BYTE_152_RAQ_PSN_M,
4351 V2_QPC_BYTE_152_RAQ_PSN_S, 0);
4352 }
4353
4354 if (attr_mask & IB_QP_QKEY) {
4355 context->qkey_xrcd = cpu_to_le32(attr->qkey);
4356 qpc_mask->qkey_xrcd = 0;
4357 hr_qp->qkey = attr->qkey;
4358 }
4359
4360 return ret;
4361 }
4362
4363 static void hns_roce_v2_record_opt_fields(struct ib_qp *ibqp,
4364 const struct ib_qp_attr *attr,
4365 int attr_mask)
4366 {
4367 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4368 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4369
4370 if (attr_mask & IB_QP_ACCESS_FLAGS)
4371 hr_qp->atomic_rd_en = attr->qp_access_flags;
4372
4373 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
4374 hr_qp->resp_depth = attr->max_dest_rd_atomic;
4375 if (attr_mask & IB_QP_PORT) {
4376 hr_qp->port = attr->port_num - 1;
4377 hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port];
4378 }
4379 }
4380
4381 static int hns_roce_v2_modify_qp(struct ib_qp *ibqp,
4382 const struct ib_qp_attr *attr,
4383 int attr_mask, enum ib_qp_state cur_state,
4384 enum ib_qp_state new_state)
4385 {
4386 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4387 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4388 struct hns_roce_v2_qp_context ctx[2];
4389 struct hns_roce_v2_qp_context *context = ctx;
4390 struct hns_roce_v2_qp_context *qpc_mask = ctx + 1;
4391 struct device *dev = hr_dev->dev;
4392 int ret;
4393
4394 /*
4395 * In v2 engine, software pass context and context mask to hardware
4396 * when modifying qp. If software need modify some fields in context,
4397 * we should set all bits of the relevant fields in context mask to
4398 * 0 at the same time, else set them to 0x1.
4399 */
4400 memset(context, 0, sizeof(*context));
4401 memset(qpc_mask, 0xff, sizeof(*qpc_mask));
4402 ret = hns_roce_v2_set_abs_fields(ibqp, attr, attr_mask, cur_state,
4403 new_state, context, qpc_mask);
4404 if (ret)
4405 goto out;
4406
4407 /* When QP state is err, SQ and RQ WQE should be flushed */
4408 if (new_state == IB_QPS_ERR) {
4409 roce_set_field(context->byte_160_sq_ci_pi,
4410 V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M,
4411 V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S,
4412 hr_qp->sq.head);
4413 roce_set_field(qpc_mask->byte_160_sq_ci_pi,
4414 V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M,
4415 V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S, 0);
4416
4417 if (!ibqp->srq) {
4418 roce_set_field(context->byte_84_rq_ci_pi,
4419 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
4420 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S,
4421 hr_qp->rq.head);
4422 roce_set_field(qpc_mask->byte_84_rq_ci_pi,
4423 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
4424 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0);
4425 }
4426 }
4427
4428 /* Configure the optional fields */
4429 ret = hns_roce_v2_set_opt_fields(ibqp, attr, attr_mask, context,
4430 qpc_mask);
4431 if (ret)
4432 goto out;
4433
4434 roce_set_bit(context->byte_108_rx_reqepsn, V2_QPC_BYTE_108_INV_CREDIT_S,
4435 ibqp->srq ? 1 : 0);
4436 roce_set_bit(qpc_mask->byte_108_rx_reqepsn,
4437 V2_QPC_BYTE_108_INV_CREDIT_S, 0);
4438
4439 /* Every status migrate must change state */
4440 roce_set_field(context->byte_60_qpst_tempid, V2_QPC_BYTE_60_QP_ST_M,
4441 V2_QPC_BYTE_60_QP_ST_S, new_state);
4442 roce_set_field(qpc_mask->byte_60_qpst_tempid, V2_QPC_BYTE_60_QP_ST_M,
4443 V2_QPC_BYTE_60_QP_ST_S, 0);
4444
4445 /* SW pass context to HW */
4446 ret = hns_roce_v2_qp_modify(hr_dev, cur_state, new_state, ctx, hr_qp);
4447 if (ret) {
4448 dev_err(dev, "hns_roce_qp_modify failed(%d)\n", ret);
4449 goto out;
4450 }
4451
4452 hr_qp->state = new_state;
4453
4454 hns_roce_v2_record_opt_fields(ibqp, attr, attr_mask);
4455
4456 if (new_state == IB_QPS_RESET && !ibqp->uobject) {
4457 hns_roce_v2_cq_clean(to_hr_cq(ibqp->recv_cq), hr_qp->qpn,
4458 ibqp->srq ? to_hr_srq(ibqp->srq) : NULL);
4459 if (ibqp->send_cq != ibqp->recv_cq)
4460 hns_roce_v2_cq_clean(to_hr_cq(ibqp->send_cq),
4461 hr_qp->qpn, NULL);
4462
4463 hr_qp->rq.head = 0;
4464 hr_qp->rq.tail = 0;
4465 hr_qp->sq.head = 0;
4466 hr_qp->sq.tail = 0;
4467 hr_qp->sq_next_wqe = 0;
4468 hr_qp->next_sge = 0;
4469 if (hr_qp->rq.wqe_cnt)
4470 *hr_qp->rdb.db_record = 0;
4471 }
4472
4473 out:
4474 return ret;
4475 }
4476
4477 static inline enum ib_qp_state to_ib_qp_st(enum hns_roce_v2_qp_state state)
4478 {
4479 switch (state) {
4480 case HNS_ROCE_QP_ST_RST: return IB_QPS_RESET;
4481 case HNS_ROCE_QP_ST_INIT: return IB_QPS_INIT;
4482 case HNS_ROCE_QP_ST_RTR: return IB_QPS_RTR;
4483 case HNS_ROCE_QP_ST_RTS: return IB_QPS_RTS;
4484 case HNS_ROCE_QP_ST_SQ_DRAINING:
4485 case HNS_ROCE_QP_ST_SQD: return IB_QPS_SQD;
4486 case HNS_ROCE_QP_ST_SQER: return IB_QPS_SQE;
4487 case HNS_ROCE_QP_ST_ERR: return IB_QPS_ERR;
4488 default: return -1;
4489 }
4490 }
4491
4492 static int hns_roce_v2_query_qpc(struct hns_roce_dev *hr_dev,
4493 struct hns_roce_qp *hr_qp,
4494 struct hns_roce_v2_qp_context *hr_context)
4495 {
4496 struct hns_roce_cmd_mailbox *mailbox;
4497 int ret;
4498
4499 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
4500 if (IS_ERR(mailbox))
4501 return PTR_ERR(mailbox);
4502
4503 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, hr_qp->qpn, 0,
4504 HNS_ROCE_CMD_QUERY_QPC,
4505 HNS_ROCE_CMD_TIMEOUT_MSECS);
4506 if (ret) {
4507 dev_err(hr_dev->dev, "QUERY QP cmd process error\n");
4508 goto out;
4509 }
4510
4511 memcpy(hr_context, mailbox->buf, sizeof(*hr_context));
4512
4513 out:
4514 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
4515 return ret;
4516 }
4517
4518 static int hns_roce_v2_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
4519 int qp_attr_mask,
4520 struct ib_qp_init_attr *qp_init_attr)
4521 {
4522 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4523 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4524 struct hns_roce_v2_qp_context context = {};
4525 struct device *dev = hr_dev->dev;
4526 int tmp_qp_state;
4527 int state;
4528 int ret;
4529
4530 memset(qp_attr, 0, sizeof(*qp_attr));
4531 memset(qp_init_attr, 0, sizeof(*qp_init_attr));
4532
4533 mutex_lock(&hr_qp->mutex);
4534
4535 if (hr_qp->state == IB_QPS_RESET) {
4536 qp_attr->qp_state = IB_QPS_RESET;
4537 ret = 0;
4538 goto done;
4539 }
4540
4541 ret = hns_roce_v2_query_qpc(hr_dev, hr_qp, &context);
4542 if (ret) {
4543 dev_err(dev, "query qpc error\n");
4544 ret = -EINVAL;
4545 goto out;
4546 }
4547
4548 state = roce_get_field(context.byte_60_qpst_tempid,
4549 V2_QPC_BYTE_60_QP_ST_M, V2_QPC_BYTE_60_QP_ST_S);
4550 tmp_qp_state = to_ib_qp_st((enum hns_roce_v2_qp_state)state);
4551 if (tmp_qp_state == -1) {
4552 dev_err(dev, "Illegal ib_qp_state\n");
4553 ret = -EINVAL;
4554 goto out;
4555 }
4556 hr_qp->state = (u8)tmp_qp_state;
4557 qp_attr->qp_state = (enum ib_qp_state)hr_qp->state;
4558 qp_attr->path_mtu = (enum ib_mtu)roce_get_field(context.byte_24_mtu_tc,
4559 V2_QPC_BYTE_24_MTU_M,
4560 V2_QPC_BYTE_24_MTU_S);
4561 qp_attr->path_mig_state = IB_MIG_ARMED;
4562 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
4563 if (hr_qp->ibqp.qp_type == IB_QPT_UD)
4564 qp_attr->qkey = V2_QKEY_VAL;
4565
4566 qp_attr->rq_psn = roce_get_field(context.byte_108_rx_reqepsn,
4567 V2_QPC_BYTE_108_RX_REQ_EPSN_M,
4568 V2_QPC_BYTE_108_RX_REQ_EPSN_S);
4569 qp_attr->sq_psn = (u32)roce_get_field(context.byte_172_sq_psn,
4570 V2_QPC_BYTE_172_SQ_CUR_PSN_M,
4571 V2_QPC_BYTE_172_SQ_CUR_PSN_S);
4572 qp_attr->dest_qp_num = (u8)roce_get_field(context.byte_56_dqpn_err,
4573 V2_QPC_BYTE_56_DQPN_M,
4574 V2_QPC_BYTE_56_DQPN_S);
4575 qp_attr->qp_access_flags = ((roce_get_bit(context.byte_76_srqn_op_en,
4576 V2_QPC_BYTE_76_RRE_S)) << V2_QP_RRE_S) |
4577 ((roce_get_bit(context.byte_76_srqn_op_en,
4578 V2_QPC_BYTE_76_RWE_S)) << V2_QP_RWE_S) |
4579 ((roce_get_bit(context.byte_76_srqn_op_en,
4580 V2_QPC_BYTE_76_ATE_S)) << V2_QP_ATE_S);
4581
4582 if (hr_qp->ibqp.qp_type == IB_QPT_RC ||
4583 hr_qp->ibqp.qp_type == IB_QPT_UC) {
4584 struct ib_global_route *grh =
4585 rdma_ah_retrieve_grh(&qp_attr->ah_attr);
4586
4587 rdma_ah_set_sl(&qp_attr->ah_attr,
4588 roce_get_field(context.byte_28_at_fl,
4589 V2_QPC_BYTE_28_SL_M,
4590 V2_QPC_BYTE_28_SL_S));
4591 grh->flow_label = roce_get_field(context.byte_28_at_fl,
4592 V2_QPC_BYTE_28_FL_M,
4593 V2_QPC_BYTE_28_FL_S);
4594 grh->sgid_index = roce_get_field(context.byte_20_smac_sgid_idx,
4595 V2_QPC_BYTE_20_SGID_IDX_M,
4596 V2_QPC_BYTE_20_SGID_IDX_S);
4597 grh->hop_limit = roce_get_field(context.byte_24_mtu_tc,
4598 V2_QPC_BYTE_24_HOP_LIMIT_M,
4599 V2_QPC_BYTE_24_HOP_LIMIT_S);
4600 grh->traffic_class = roce_get_field(context.byte_24_mtu_tc,
4601 V2_QPC_BYTE_24_TC_M,
4602 V2_QPC_BYTE_24_TC_S);
4603
4604 memcpy(grh->dgid.raw, context.dgid, sizeof(grh->dgid.raw));
4605 }
4606
4607 qp_attr->port_num = hr_qp->port + 1;
4608 qp_attr->sq_draining = 0;
4609 qp_attr->max_rd_atomic = 1 << roce_get_field(context.byte_208_irrl,
4610 V2_QPC_BYTE_208_SR_MAX_M,
4611 V2_QPC_BYTE_208_SR_MAX_S);
4612 qp_attr->max_dest_rd_atomic = 1 << roce_get_field(context.byte_140_raq,
4613 V2_QPC_BYTE_140_RR_MAX_M,
4614 V2_QPC_BYTE_140_RR_MAX_S);
4615 qp_attr->min_rnr_timer = (u8)roce_get_field(context.byte_80_rnr_rx_cqn,
4616 V2_QPC_BYTE_80_MIN_RNR_TIME_M,
4617 V2_QPC_BYTE_80_MIN_RNR_TIME_S);
4618 qp_attr->timeout = (u8)roce_get_field(context.byte_28_at_fl,
4619 V2_QPC_BYTE_28_AT_M,
4620 V2_QPC_BYTE_28_AT_S);
4621 qp_attr->retry_cnt = roce_get_field(context.byte_212_lsn,
4622 V2_QPC_BYTE_212_RETRY_CNT_M,
4623 V2_QPC_BYTE_212_RETRY_CNT_S);
4624 qp_attr->rnr_retry = le32_to_cpu(context.rq_rnr_timer);
4625
4626 done:
4627 qp_attr->cur_qp_state = qp_attr->qp_state;
4628 qp_attr->cap.max_recv_wr = hr_qp->rq.wqe_cnt;
4629 qp_attr->cap.max_recv_sge = hr_qp->rq.max_gs;
4630
4631 if (!ibqp->uobject) {
4632 qp_attr->cap.max_send_wr = hr_qp->sq.wqe_cnt;
4633 qp_attr->cap.max_send_sge = hr_qp->sq.max_gs;
4634 } else {
4635 qp_attr->cap.max_send_wr = 0;
4636 qp_attr->cap.max_send_sge = 0;
4637 }
4638
4639 qp_init_attr->cap = qp_attr->cap;
4640
4641 out:
4642 mutex_unlock(&hr_qp->mutex);
4643 return ret;
4644 }
4645
4646 static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev,
4647 struct hns_roce_qp *hr_qp,
4648 struct ib_udata *udata)
4649 {
4650 struct hns_roce_cq *send_cq, *recv_cq;
4651 struct ib_device *ibdev = &hr_dev->ib_dev;
4652 int ret = 0;
4653
4654 if (hr_qp->ibqp.qp_type == IB_QPT_RC && hr_qp->state != IB_QPS_RESET) {
4655 /* Modify qp to reset before destroying qp */
4656 ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, NULL, 0,
4657 hr_qp->state, IB_QPS_RESET);
4658 if (ret)
4659 ibdev_err(ibdev, "modify QP to Reset failed.\n");
4660 }
4661
4662 send_cq = to_hr_cq(hr_qp->ibqp.send_cq);
4663 recv_cq = to_hr_cq(hr_qp->ibqp.recv_cq);
4664
4665 hns_roce_lock_cqs(send_cq, recv_cq);
4666
4667 if (!udata) {
4668 __hns_roce_v2_cq_clean(recv_cq, hr_qp->qpn, hr_qp->ibqp.srq ?
4669 to_hr_srq(hr_qp->ibqp.srq) : NULL);
4670 if (send_cq != recv_cq)
4671 __hns_roce_v2_cq_clean(send_cq, hr_qp->qpn, NULL);
4672 }
4673
4674 hns_roce_qp_remove(hr_dev, hr_qp);
4675
4676 hns_roce_unlock_cqs(send_cq, recv_cq);
4677
4678 hns_roce_qp_free(hr_dev, hr_qp);
4679
4680 /* Not special_QP, free their QPN */
4681 if ((hr_qp->ibqp.qp_type == IB_QPT_RC) ||
4682 (hr_qp->ibqp.qp_type == IB_QPT_UC) ||
4683 (hr_qp->ibqp.qp_type == IB_QPT_UD))
4684 hns_roce_release_range_qp(hr_dev, hr_qp->qpn, 1);
4685
4686 hns_roce_mtr_cleanup(hr_dev, &hr_qp->mtr);
4687
4688 if (udata) {
4689 struct hns_roce_ucontext *context =
4690 rdma_udata_to_drv_context(
4691 udata,
4692 struct hns_roce_ucontext,
4693 ibucontext);
4694
4695 if (hr_qp->sq.wqe_cnt && (hr_qp->sdb_en == 1))
4696 hns_roce_db_unmap_user(context, &hr_qp->sdb);
4697
4698 if (hr_qp->rq.wqe_cnt && (hr_qp->rdb_en == 1))
4699 hns_roce_db_unmap_user(context, &hr_qp->rdb);
4700 } else {
4701 kfree(hr_qp->sq.wrid);
4702 kfree(hr_qp->rq.wrid);
4703 hns_roce_buf_free(hr_dev, hr_qp->buff_size, &hr_qp->hr_buf);
4704 if (hr_qp->rq.wqe_cnt)
4705 hns_roce_free_db(hr_dev, &hr_qp->rdb);
4706 }
4707 ib_umem_release(hr_qp->umem);
4708
4709 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) &&
4710 hr_qp->rq.wqe_cnt) {
4711 kfree(hr_qp->rq_inl_buf.wqe_list[0].sg_list);
4712 kfree(hr_qp->rq_inl_buf.wqe_list);
4713 }
4714
4715 return ret;
4716 }
4717
4718 static int hns_roce_v2_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
4719 {
4720 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4721 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4722 int ret;
4723
4724 ret = hns_roce_v2_destroy_qp_common(hr_dev, hr_qp, udata);
4725 if (ret)
4726 ibdev_err(&hr_dev->ib_dev, "Destroy qp 0x%06lx failed(%d)\n",
4727 hr_qp->qpn, ret);
4728
4729 kfree(hr_qp);
4730
4731 return 0;
4732 }
4733
4734 static int hns_roce_v2_qp_flow_control_init(struct hns_roce_dev *hr_dev,
4735 struct hns_roce_qp *hr_qp)
4736 {
4737 struct hns_roce_sccc_clr_done *resp;
4738 struct hns_roce_sccc_clr *clr;
4739 struct hns_roce_cmq_desc desc;
4740 int ret, i;
4741
4742 mutex_lock(&hr_dev->qp_table.scc_mutex);
4743
4744 /* set scc ctx clear done flag */
4745 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_RESET_SCCC, false);
4746 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
4747 if (ret) {
4748 dev_err(hr_dev->dev, "Reset SCC ctx failed(%d)\n", ret);
4749 goto out;
4750 }
4751
4752 /* clear scc context */
4753 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CLR_SCCC, false);
4754 clr = (struct hns_roce_sccc_clr *)desc.data;
4755 clr->qpn = cpu_to_le32(hr_qp->qpn);
4756 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
4757 if (ret) {
4758 dev_err(hr_dev->dev, "Clear SCC ctx failed(%d)\n", ret);
4759 goto out;
4760 }
4761
4762 /* query scc context clear is done or not */
4763 resp = (struct hns_roce_sccc_clr_done *)desc.data;
4764 for (i = 0; i <= HNS_ROCE_CMQ_SCC_CLR_DONE_CNT; i++) {
4765 hns_roce_cmq_setup_basic_desc(&desc,
4766 HNS_ROCE_OPC_QUERY_SCCC, true);
4767 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
4768 if (ret) {
4769 dev_err(hr_dev->dev, "Query clr cmq failed(%d)\n", ret);
4770 goto out;
4771 }
4772
4773 if (resp->clr_done)
4774 goto out;
4775
4776 msleep(20);
4777 }
4778
4779 dev_err(hr_dev->dev, "Query SCC clr done flag overtime.\n");
4780 ret = -ETIMEDOUT;
4781
4782 out:
4783 mutex_unlock(&hr_dev->qp_table.scc_mutex);
4784 return ret;
4785 }
4786
4787 static int hns_roce_v2_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)
4788 {
4789 struct hns_roce_dev *hr_dev = to_hr_dev(cq->device);
4790 struct hns_roce_v2_cq_context *cq_context;
4791 struct hns_roce_cq *hr_cq = to_hr_cq(cq);
4792 struct hns_roce_v2_cq_context *cqc_mask;
4793 struct hns_roce_cmd_mailbox *mailbox;
4794 int ret;
4795
4796 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
4797 if (IS_ERR(mailbox))
4798 return PTR_ERR(mailbox);
4799
4800 cq_context = mailbox->buf;
4801 cqc_mask = (struct hns_roce_v2_cq_context *)mailbox->buf + 1;
4802
4803 memset(cqc_mask, 0xff, sizeof(*cqc_mask));
4804
4805 roce_set_field(cq_context->byte_56_cqe_period_maxcnt,
4806 V2_CQC_BYTE_56_CQ_MAX_CNT_M, V2_CQC_BYTE_56_CQ_MAX_CNT_S,
4807 cq_count);
4808 roce_set_field(cqc_mask->byte_56_cqe_period_maxcnt,
4809 V2_CQC_BYTE_56_CQ_MAX_CNT_M, V2_CQC_BYTE_56_CQ_MAX_CNT_S,
4810 0);
4811 roce_set_field(cq_context->byte_56_cqe_period_maxcnt,
4812 V2_CQC_BYTE_56_CQ_PERIOD_M, V2_CQC_BYTE_56_CQ_PERIOD_S,
4813 cq_period);
4814 roce_set_field(cqc_mask->byte_56_cqe_period_maxcnt,
4815 V2_CQC_BYTE_56_CQ_PERIOD_M, V2_CQC_BYTE_56_CQ_PERIOD_S,
4816 0);
4817
4818 ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_cq->cqn, 1,
4819 HNS_ROCE_CMD_MODIFY_CQC,
4820 HNS_ROCE_CMD_TIMEOUT_MSECS);
4821 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
4822 if (ret)
4823 dev_err(hr_dev->dev, "MODIFY CQ Failed to cmd mailbox.\n");
4824
4825 return ret;
4826 }
4827
4828 static void hns_roce_set_qps_to_err(struct hns_roce_dev *hr_dev, u32 qpn)
4829 {
4830 struct hns_roce_qp *hr_qp;
4831 struct ib_qp_attr attr;
4832 int attr_mask;
4833 int ret;
4834
4835 hr_qp = __hns_roce_qp_lookup(hr_dev, qpn);
4836 if (!hr_qp) {
4837 dev_warn(hr_dev->dev, "no hr_qp can be found!\n");
4838 return;
4839 }
4840
4841 if (hr_qp->ibqp.uobject) {
4842 if (hr_qp->sdb_en == 1) {
4843 hr_qp->sq.head = *(int *)(hr_qp->sdb.virt_addr);
4844 if (hr_qp->rdb_en == 1)
4845 hr_qp->rq.head = *(int *)(hr_qp->rdb.virt_addr);
4846 } else {
4847 dev_warn(hr_dev->dev, "flush cqe is unsupported in userspace!\n");
4848 return;
4849 }
4850 }
4851
4852 attr_mask = IB_QP_STATE;
4853 attr.qp_state = IB_QPS_ERR;
4854 ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, &attr, attr_mask,
4855 hr_qp->state, IB_QPS_ERR);
4856 if (ret)
4857 dev_err(hr_dev->dev, "failed to modify qp %d to err state.\n",
4858 qpn);
4859 }
4860
4861 static void hns_roce_irq_work_handle(struct work_struct *work)
4862 {
4863 struct hns_roce_work *irq_work =
4864 container_of(work, struct hns_roce_work, work);
4865 struct device *dev = irq_work->hr_dev->dev;
4866 u32 qpn = irq_work->qpn;
4867 u32 cqn = irq_work->cqn;
4868
4869 switch (irq_work->event_type) {
4870 case HNS_ROCE_EVENT_TYPE_PATH_MIG:
4871 dev_info(dev, "Path migrated succeeded.\n");
4872 break;
4873 case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:
4874 dev_warn(dev, "Path migration failed.\n");
4875 break;
4876 case HNS_ROCE_EVENT_TYPE_COMM_EST:
4877 break;
4878 case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
4879 dev_warn(dev, "Send queue drained.\n");
4880 break;
4881 case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
4882 dev_err(dev, "Local work queue 0x%x catas error, sub_type:%d\n",
4883 qpn, irq_work->sub_type);
4884 hns_roce_set_qps_to_err(irq_work->hr_dev, qpn);
4885 break;
4886 case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
4887 dev_err(dev, "Invalid request local work queue 0x%x error.\n",
4888 qpn);
4889 hns_roce_set_qps_to_err(irq_work->hr_dev, qpn);
4890 break;
4891 case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
4892 dev_err(dev, "Local access violation work queue 0x%x error, sub_type:%d\n",
4893 qpn, irq_work->sub_type);
4894 hns_roce_set_qps_to_err(irq_work->hr_dev, qpn);
4895 break;
4896 case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:
4897 dev_warn(dev, "SRQ limit reach.\n");
4898 break;
4899 case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
4900 dev_warn(dev, "SRQ last wqe reach.\n");
4901 break;
4902 case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:
4903 dev_err(dev, "SRQ catas error.\n");
4904 break;
4905 case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:
4906 dev_err(dev, "CQ 0x%x access err.\n", cqn);
4907 break;
4908 case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:
4909 dev_warn(dev, "CQ 0x%x overflow\n", cqn);
4910 break;
4911 case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW:
4912 dev_warn(dev, "DB overflow.\n");
4913 break;
4914 case HNS_ROCE_EVENT_TYPE_FLR:
4915 dev_warn(dev, "Function level reset.\n");
4916 break;
4917 default:
4918 break;
4919 }
4920
4921 kfree(irq_work);
4922 }
4923
4924 static void hns_roce_v2_init_irq_work(struct hns_roce_dev *hr_dev,
4925 struct hns_roce_eq *eq,
4926 u32 qpn, u32 cqn)
4927 {
4928 struct hns_roce_work *irq_work;
4929
4930 irq_work = kzalloc(sizeof(struct hns_roce_work), GFP_ATOMIC);
4931 if (!irq_work)
4932 return;
4933
4934 INIT_WORK(&(irq_work->work), hns_roce_irq_work_handle);
4935 irq_work->hr_dev = hr_dev;
4936 irq_work->qpn = qpn;
4937 irq_work->cqn = cqn;
4938 irq_work->event_type = eq->event_type;
4939 irq_work->sub_type = eq->sub_type;
4940 queue_work(hr_dev->irq_workq, &(irq_work->work));
4941 }
4942
4943 static void set_eq_cons_index_v2(struct hns_roce_eq *eq)
4944 {
4945 struct hns_roce_dev *hr_dev = eq->hr_dev;
4946 __le32 doorbell[2] = {};
4947
4948 if (eq->type_flag == HNS_ROCE_AEQ) {
4949 roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_CMD_M,
4950 HNS_ROCE_V2_EQ_DB_CMD_S,
4951 eq->arm_st == HNS_ROCE_V2_EQ_ALWAYS_ARMED ?
4952 HNS_ROCE_EQ_DB_CMD_AEQ :
4953 HNS_ROCE_EQ_DB_CMD_AEQ_ARMED);
4954 } else {
4955 roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_TAG_M,
4956 HNS_ROCE_V2_EQ_DB_TAG_S, eq->eqn);
4957
4958 roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_CMD_M,
4959 HNS_ROCE_V2_EQ_DB_CMD_S,
4960 eq->arm_st == HNS_ROCE_V2_EQ_ALWAYS_ARMED ?
4961 HNS_ROCE_EQ_DB_CMD_CEQ :
4962 HNS_ROCE_EQ_DB_CMD_CEQ_ARMED);
4963 }
4964
4965 roce_set_field(doorbell[1], HNS_ROCE_V2_EQ_DB_PARA_M,
4966 HNS_ROCE_V2_EQ_DB_PARA_S,
4967 (eq->cons_index & HNS_ROCE_V2_CONS_IDX_M));
4968
4969 hns_roce_write64(hr_dev, doorbell, eq->doorbell);
4970 }
4971
4972 static struct hns_roce_aeqe *get_aeqe_v2(struct hns_roce_eq *eq, u32 entry)
4973 {
4974 u32 buf_chk_sz;
4975 unsigned long off;
4976
4977 buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
4978 off = (entry & (eq->entries - 1)) * HNS_ROCE_AEQ_ENTRY_SIZE;
4979
4980 return (struct hns_roce_aeqe *)((char *)(eq->buf_list->buf) +
4981 off % buf_chk_sz);
4982 }
4983
4984 static struct hns_roce_aeqe *mhop_get_aeqe(struct hns_roce_eq *eq, u32 entry)
4985 {
4986 u32 buf_chk_sz;
4987 unsigned long off;
4988
4989 buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
4990
4991 off = (entry & (eq->entries - 1)) * HNS_ROCE_AEQ_ENTRY_SIZE;
4992
4993 if (eq->hop_num == HNS_ROCE_HOP_NUM_0)
4994 return (struct hns_roce_aeqe *)((u8 *)(eq->bt_l0) +
4995 off % buf_chk_sz);
4996 else
4997 return (struct hns_roce_aeqe *)((u8 *)
4998 (eq->buf[off / buf_chk_sz]) + off % buf_chk_sz);
4999 }
5000
5001 static struct hns_roce_aeqe *next_aeqe_sw_v2(struct hns_roce_eq *eq)
5002 {
5003 struct hns_roce_aeqe *aeqe;
5004
5005 if (!eq->hop_num)
5006 aeqe = get_aeqe_v2(eq, eq->cons_index);
5007 else
5008 aeqe = mhop_get_aeqe(eq, eq->cons_index);
5009
5010 return (roce_get_bit(aeqe->asyn, HNS_ROCE_V2_AEQ_AEQE_OWNER_S) ^
5011 !!(eq->cons_index & eq->entries)) ? aeqe : NULL;
5012 }
5013
5014 static int hns_roce_v2_aeq_int(struct hns_roce_dev *hr_dev,
5015 struct hns_roce_eq *eq)
5016 {
5017 struct device *dev = hr_dev->dev;
5018 struct hns_roce_aeqe *aeqe = next_aeqe_sw_v2(eq);
5019 int aeqe_found = 0;
5020 int event_type;
5021 int sub_type;
5022 u32 srqn;
5023 u32 qpn;
5024 u32 cqn;
5025
5026 while (aeqe) {
5027 /* Make sure we read AEQ entry after we have checked the
5028 * ownership bit
5029 */
5030 dma_rmb();
5031
5032 event_type = roce_get_field(aeqe->asyn,
5033 HNS_ROCE_V2_AEQE_EVENT_TYPE_M,
5034 HNS_ROCE_V2_AEQE_EVENT_TYPE_S);
5035 sub_type = roce_get_field(aeqe->asyn,
5036 HNS_ROCE_V2_AEQE_SUB_TYPE_M,
5037 HNS_ROCE_V2_AEQE_SUB_TYPE_S);
5038 qpn = roce_get_field(aeqe->event.qp_event.qp,
5039 HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M,
5040 HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S);
5041 cqn = roce_get_field(aeqe->event.cq_event.cq,
5042 HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M,
5043 HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S);
5044 srqn = roce_get_field(aeqe->event.srq_event.srq,
5045 HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M,
5046 HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S);
5047
5048 switch (event_type) {
5049 case HNS_ROCE_EVENT_TYPE_PATH_MIG:
5050 case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:
5051 case HNS_ROCE_EVENT_TYPE_COMM_EST:
5052 case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
5053 case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
5054 case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
5055 case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
5056 case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
5057 hns_roce_qp_event(hr_dev, qpn, event_type);
5058 break;
5059 case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:
5060 case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:
5061 hns_roce_srq_event(hr_dev, srqn, event_type);
5062 break;
5063 case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:
5064 case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:
5065 hns_roce_cq_event(hr_dev, cqn, event_type);
5066 break;
5067 case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW:
5068 break;
5069 case HNS_ROCE_EVENT_TYPE_MB:
5070 hns_roce_cmd_event(hr_dev,
5071 le16_to_cpu(aeqe->event.cmd.token),
5072 aeqe->event.cmd.status,
5073 le64_to_cpu(aeqe->event.cmd.out_param));
5074 break;
5075 case HNS_ROCE_EVENT_TYPE_CEQ_OVERFLOW:
5076 break;
5077 case HNS_ROCE_EVENT_TYPE_FLR:
5078 break;
5079 default:
5080 dev_err(dev, "Unhandled event %d on EQ %d at idx %u.\n",
5081 event_type, eq->eqn, eq->cons_index);
5082 break;
5083 }
5084
5085 eq->event_type = event_type;
5086 eq->sub_type = sub_type;
5087 ++eq->cons_index;
5088 aeqe_found = 1;
5089
5090 if (eq->cons_index > (2 * eq->entries - 1))
5091 eq->cons_index = 0;
5092
5093 hns_roce_v2_init_irq_work(hr_dev, eq, qpn, cqn);
5094
5095 aeqe = next_aeqe_sw_v2(eq);
5096 }
5097
5098 set_eq_cons_index_v2(eq);
5099 return aeqe_found;
5100 }
5101
5102 static struct hns_roce_ceqe *get_ceqe_v2(struct hns_roce_eq *eq, u32 entry)
5103 {
5104 u32 buf_chk_sz;
5105 unsigned long off;
5106
5107 buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
5108 off = (entry & (eq->entries - 1)) * HNS_ROCE_CEQ_ENTRY_SIZE;
5109
5110 return (struct hns_roce_ceqe *)((char *)(eq->buf_list->buf) +
5111 off % buf_chk_sz);
5112 }
5113
5114 static struct hns_roce_ceqe *mhop_get_ceqe(struct hns_roce_eq *eq, u32 entry)
5115 {
5116 u32 buf_chk_sz;
5117 unsigned long off;
5118
5119 buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
5120
5121 off = (entry & (eq->entries - 1)) * HNS_ROCE_CEQ_ENTRY_SIZE;
5122
5123 if (eq->hop_num == HNS_ROCE_HOP_NUM_0)
5124 return (struct hns_roce_ceqe *)((u8 *)(eq->bt_l0) +
5125 off % buf_chk_sz);
5126 else
5127 return (struct hns_roce_ceqe *)((u8 *)(eq->buf[off /
5128 buf_chk_sz]) + off % buf_chk_sz);
5129 }
5130
5131 static struct hns_roce_ceqe *next_ceqe_sw_v2(struct hns_roce_eq *eq)
5132 {
5133 struct hns_roce_ceqe *ceqe;
5134
5135 if (!eq->hop_num)
5136 ceqe = get_ceqe_v2(eq, eq->cons_index);
5137 else
5138 ceqe = mhop_get_ceqe(eq, eq->cons_index);
5139
5140 return (!!(roce_get_bit(ceqe->comp, HNS_ROCE_V2_CEQ_CEQE_OWNER_S))) ^
5141 (!!(eq->cons_index & eq->entries)) ? ceqe : NULL;
5142 }
5143
5144 static int hns_roce_v2_ceq_int(struct hns_roce_dev *hr_dev,
5145 struct hns_roce_eq *eq)
5146 {
5147 struct device *dev = hr_dev->dev;
5148 struct hns_roce_ceqe *ceqe = next_ceqe_sw_v2(eq);
5149 int ceqe_found = 0;
5150 u32 cqn;
5151
5152 while (ceqe) {
5153 /* Make sure we read CEQ entry after we have checked the
5154 * ownership bit
5155 */
5156 dma_rmb();
5157
5158 cqn = roce_get_field(ceqe->comp,
5159 HNS_ROCE_V2_CEQE_COMP_CQN_M,
5160 HNS_ROCE_V2_CEQE_COMP_CQN_S);
5161
5162 hns_roce_cq_completion(hr_dev, cqn);
5163
5164 ++eq->cons_index;
5165 ceqe_found = 1;
5166
5167 if (eq->cons_index > (EQ_DEPTH_COEFF * eq->entries - 1)) {
5168 dev_warn(dev, "cons_index overflow, set back to 0.\n");
5169 eq->cons_index = 0;
5170 }
5171
5172 ceqe = next_ceqe_sw_v2(eq);
5173 }
5174
5175 set_eq_cons_index_v2(eq);
5176
5177 return ceqe_found;
5178 }
5179
5180 static irqreturn_t hns_roce_v2_msix_interrupt_eq(int irq, void *eq_ptr)
5181 {
5182 struct hns_roce_eq *eq = eq_ptr;
5183 struct hns_roce_dev *hr_dev = eq->hr_dev;
5184 int int_work = 0;
5185
5186 if (eq->type_flag == HNS_ROCE_CEQ)
5187 /* Completion event interrupt */
5188 int_work = hns_roce_v2_ceq_int(hr_dev, eq);
5189 else
5190 /* Asychronous event interrupt */
5191 int_work = hns_roce_v2_aeq_int(hr_dev, eq);
5192
5193 return IRQ_RETVAL(int_work);
5194 }
5195
5196 static irqreturn_t hns_roce_v2_msix_interrupt_abn(int irq, void *dev_id)
5197 {
5198 struct hns_roce_dev *hr_dev = dev_id;
5199 struct device *dev = hr_dev->dev;
5200 int int_work = 0;
5201 u32 int_st;
5202 u32 int_en;
5203
5204 /* Abnormal interrupt */
5205 int_st = roce_read(hr_dev, ROCEE_VF_ABN_INT_ST_REG);
5206 int_en = roce_read(hr_dev, ROCEE_VF_ABN_INT_EN_REG);
5207
5208 if (int_st & BIT(HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S)) {
5209 struct pci_dev *pdev = hr_dev->pci_dev;
5210 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
5211 const struct hnae3_ae_ops *ops = ae_dev->ops;
5212
5213 dev_err(dev, "AEQ overflow!\n");
5214
5215 int_st |= 1 << HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S;
5216 roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st);
5217
5218 /* Set reset level for reset_event() */
5219 if (ops->set_default_reset_request)
5220 ops->set_default_reset_request(ae_dev,
5221 HNAE3_FUNC_RESET);
5222 if (ops->reset_event)
5223 ops->reset_event(pdev, NULL);
5224
5225 int_en |= 1 << HNS_ROCE_V2_VF_ABN_INT_EN_S;
5226 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en);
5227
5228 int_work = 1;
5229 } else if (int_st & BIT(HNS_ROCE_V2_VF_INT_ST_BUS_ERR_S)) {
5230 dev_err(dev, "BUS ERR!\n");
5231
5232 int_st |= 1 << HNS_ROCE_V2_VF_INT_ST_BUS_ERR_S;
5233 roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st);
5234
5235 int_en |= 1 << HNS_ROCE_V2_VF_ABN_INT_EN_S;
5236 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en);
5237
5238 int_work = 1;
5239 } else if (int_st & BIT(HNS_ROCE_V2_VF_INT_ST_OTHER_ERR_S)) {
5240 dev_err(dev, "OTHER ERR!\n");
5241
5242 int_st |= 1 << HNS_ROCE_V2_VF_INT_ST_OTHER_ERR_S;
5243 roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st);
5244
5245 int_en |= 1 << HNS_ROCE_V2_VF_ABN_INT_EN_S;
5246 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en);
5247
5248 int_work = 1;
5249 } else
5250 dev_err(dev, "There is no abnormal irq found!\n");
5251
5252 return IRQ_RETVAL(int_work);
5253 }
5254
5255 static void hns_roce_v2_int_mask_enable(struct hns_roce_dev *hr_dev,
5256 int eq_num, int enable_flag)
5257 {
5258 int i;
5259
5260 if (enable_flag == EQ_ENABLE) {
5261 for (i = 0; i < eq_num; i++)
5262 roce_write(hr_dev, ROCEE_VF_EVENT_INT_EN_REG +
5263 i * EQ_REG_OFFSET,
5264 HNS_ROCE_V2_VF_EVENT_INT_EN_M);
5265
5266 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG,
5267 HNS_ROCE_V2_VF_ABN_INT_EN_M);
5268 roce_write(hr_dev, ROCEE_VF_ABN_INT_CFG_REG,
5269 HNS_ROCE_V2_VF_ABN_INT_CFG_M);
5270 } else {
5271 for (i = 0; i < eq_num; i++)
5272 roce_write(hr_dev, ROCEE_VF_EVENT_INT_EN_REG +
5273 i * EQ_REG_OFFSET,
5274 HNS_ROCE_V2_VF_EVENT_INT_EN_M & 0x0);
5275
5276 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG,
5277 HNS_ROCE_V2_VF_ABN_INT_EN_M & 0x0);
5278 roce_write(hr_dev, ROCEE_VF_ABN_INT_CFG_REG,
5279 HNS_ROCE_V2_VF_ABN_INT_CFG_M & 0x0);
5280 }
5281 }
5282
5283 static void hns_roce_v2_destroy_eqc(struct hns_roce_dev *hr_dev, int eqn)
5284 {
5285 struct device *dev = hr_dev->dev;
5286 int ret;
5287
5288 if (eqn < hr_dev->caps.num_comp_vectors)
5289 ret = hns_roce_cmd_mbox(hr_dev, 0, 0, eqn & HNS_ROCE_V2_EQN_M,
5290 0, HNS_ROCE_CMD_DESTROY_CEQC,
5291 HNS_ROCE_CMD_TIMEOUT_MSECS);
5292 else
5293 ret = hns_roce_cmd_mbox(hr_dev, 0, 0, eqn & HNS_ROCE_V2_EQN_M,
5294 0, HNS_ROCE_CMD_DESTROY_AEQC,
5295 HNS_ROCE_CMD_TIMEOUT_MSECS);
5296 if (ret)
5297 dev_err(dev, "[mailbox cmd] destroy eqc(%d) failed.\n", eqn);
5298 }
5299
5300 static void hns_roce_mhop_free_eq(struct hns_roce_dev *hr_dev,
5301 struct hns_roce_eq *eq)
5302 {
5303 struct device *dev = hr_dev->dev;
5304 u64 idx;
5305 u64 size;
5306 u32 buf_chk_sz;
5307 u32 bt_chk_sz;
5308 u32 mhop_num;
5309 int eqe_alloc;
5310 int i = 0;
5311 int j = 0;
5312
5313 mhop_num = hr_dev->caps.eqe_hop_num;
5314 buf_chk_sz = 1 << (hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT);
5315 bt_chk_sz = 1 << (hr_dev->caps.eqe_ba_pg_sz + PAGE_SHIFT);
5316
5317 if (mhop_num == HNS_ROCE_HOP_NUM_0) {
5318 dma_free_coherent(dev, (unsigned int)(eq->entries *
5319 eq->eqe_size), eq->bt_l0, eq->l0_dma);
5320 return;
5321 }
5322
5323 dma_free_coherent(dev, bt_chk_sz, eq->bt_l0, eq->l0_dma);
5324 if (mhop_num == 1) {
5325 for (i = 0; i < eq->l0_last_num; i++) {
5326 if (i == eq->l0_last_num - 1) {
5327 eqe_alloc = i * (buf_chk_sz / eq->eqe_size);
5328 size = (eq->entries - eqe_alloc) * eq->eqe_size;
5329 dma_free_coherent(dev, size, eq->buf[i],
5330 eq->buf_dma[i]);
5331 break;
5332 }
5333 dma_free_coherent(dev, buf_chk_sz, eq->buf[i],
5334 eq->buf_dma[i]);
5335 }
5336 } else if (mhop_num == 2) {
5337 for (i = 0; i < eq->l0_last_num; i++) {
5338 dma_free_coherent(dev, bt_chk_sz, eq->bt_l1[i],
5339 eq->l1_dma[i]);
5340
5341 for (j = 0; j < bt_chk_sz / BA_BYTE_LEN; j++) {
5342 idx = i * (bt_chk_sz / BA_BYTE_LEN) + j;
5343 if ((i == eq->l0_last_num - 1)
5344 && j == eq->l1_last_num - 1) {
5345 eqe_alloc = (buf_chk_sz / eq->eqe_size)
5346 * idx;
5347 size = (eq->entries - eqe_alloc)
5348 * eq->eqe_size;
5349 dma_free_coherent(dev, size,
5350 eq->buf[idx],
5351 eq->buf_dma[idx]);
5352 break;
5353 }
5354 dma_free_coherent(dev, buf_chk_sz, eq->buf[idx],
5355 eq->buf_dma[idx]);
5356 }
5357 }
5358 }
5359 kfree(eq->buf_dma);
5360 kfree(eq->buf);
5361 kfree(eq->l1_dma);
5362 kfree(eq->bt_l1);
5363 eq->buf_dma = NULL;
5364 eq->buf = NULL;
5365 eq->l1_dma = NULL;
5366 eq->bt_l1 = NULL;
5367 }
5368
5369 static void hns_roce_v2_free_eq(struct hns_roce_dev *hr_dev,
5370 struct hns_roce_eq *eq)
5371 {
5372 u32 buf_chk_sz;
5373
5374 buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
5375
5376 if (hr_dev->caps.eqe_hop_num) {
5377 hns_roce_mhop_free_eq(hr_dev, eq);
5378 return;
5379 }
5380
5381 dma_free_coherent(hr_dev->dev, buf_chk_sz, eq->buf_list->buf,
5382 eq->buf_list->map);
5383 kfree(eq->buf_list);
5384 }
5385
5386 static void hns_roce_config_eqc(struct hns_roce_dev *hr_dev,
5387 struct hns_roce_eq *eq,
5388 void *mb_buf)
5389 {
5390 struct hns_roce_eq_context *eqc;
5391
5392 eqc = mb_buf;
5393 memset(eqc, 0, sizeof(struct hns_roce_eq_context));
5394
5395 /* init eqc */
5396 eq->doorbell = hr_dev->reg_base + ROCEE_VF_EQ_DB_CFG0_REG;
5397 eq->hop_num = hr_dev->caps.eqe_hop_num;
5398 eq->cons_index = 0;
5399 eq->over_ignore = HNS_ROCE_V2_EQ_OVER_IGNORE_0;
5400 eq->coalesce = HNS_ROCE_V2_EQ_COALESCE_0;
5401 eq->arm_st = HNS_ROCE_V2_EQ_ALWAYS_ARMED;
5402 eq->eqe_ba_pg_sz = hr_dev->caps.eqe_ba_pg_sz;
5403 eq->eqe_buf_pg_sz = hr_dev->caps.eqe_buf_pg_sz;
5404 eq->shift = ilog2((unsigned int)eq->entries);
5405
5406 if (!eq->hop_num)
5407 eq->eqe_ba = eq->buf_list->map;
5408 else
5409 eq->eqe_ba = eq->l0_dma;
5410
5411 /* set eqc state */
5412 roce_set_field(eqc->byte_4,
5413 HNS_ROCE_EQC_EQ_ST_M,
5414 HNS_ROCE_EQC_EQ_ST_S,
5415 HNS_ROCE_V2_EQ_STATE_VALID);
5416
5417 /* set eqe hop num */
5418 roce_set_field(eqc->byte_4,
5419 HNS_ROCE_EQC_HOP_NUM_M,
5420 HNS_ROCE_EQC_HOP_NUM_S, eq->hop_num);
5421
5422 /* set eqc over_ignore */
5423 roce_set_field(eqc->byte_4,
5424 HNS_ROCE_EQC_OVER_IGNORE_M,
5425 HNS_ROCE_EQC_OVER_IGNORE_S, eq->over_ignore);
5426
5427 /* set eqc coalesce */
5428 roce_set_field(eqc->byte_4,
5429 HNS_ROCE_EQC_COALESCE_M,
5430 HNS_ROCE_EQC_COALESCE_S, eq->coalesce);
5431
5432 /* set eqc arm_state */
5433 roce_set_field(eqc->byte_4,
5434 HNS_ROCE_EQC_ARM_ST_M,
5435 HNS_ROCE_EQC_ARM_ST_S, eq->arm_st);
5436
5437 /* set eqn */
5438 roce_set_field(eqc->byte_4,
5439 HNS_ROCE_EQC_EQN_M,
5440 HNS_ROCE_EQC_EQN_S, eq->eqn);
5441
5442 /* set eqe_cnt */
5443 roce_set_field(eqc->byte_4,
5444 HNS_ROCE_EQC_EQE_CNT_M,
5445 HNS_ROCE_EQC_EQE_CNT_S,
5446 HNS_ROCE_EQ_INIT_EQE_CNT);
5447
5448 /* set eqe_ba_pg_sz */
5449 roce_set_field(eqc->byte_8,
5450 HNS_ROCE_EQC_BA_PG_SZ_M,
5451 HNS_ROCE_EQC_BA_PG_SZ_S,
5452 eq->eqe_ba_pg_sz + PG_SHIFT_OFFSET);
5453
5454 /* set eqe_buf_pg_sz */
5455 roce_set_field(eqc->byte_8,
5456 HNS_ROCE_EQC_BUF_PG_SZ_M,
5457 HNS_ROCE_EQC_BUF_PG_SZ_S,
5458 eq->eqe_buf_pg_sz + PG_SHIFT_OFFSET);
5459
5460 /* set eq_producer_idx */
5461 roce_set_field(eqc->byte_8,
5462 HNS_ROCE_EQC_PROD_INDX_M,
5463 HNS_ROCE_EQC_PROD_INDX_S,
5464 HNS_ROCE_EQ_INIT_PROD_IDX);
5465
5466 /* set eq_max_cnt */
5467 roce_set_field(eqc->byte_12,
5468 HNS_ROCE_EQC_MAX_CNT_M,
5469 HNS_ROCE_EQC_MAX_CNT_S, eq->eq_max_cnt);
5470
5471 /* set eq_period */
5472 roce_set_field(eqc->byte_12,
5473 HNS_ROCE_EQC_PERIOD_M,
5474 HNS_ROCE_EQC_PERIOD_S, eq->eq_period);
5475
5476 /* set eqe_report_timer */
5477 roce_set_field(eqc->eqe_report_timer,
5478 HNS_ROCE_EQC_REPORT_TIMER_M,
5479 HNS_ROCE_EQC_REPORT_TIMER_S,
5480 HNS_ROCE_EQ_INIT_REPORT_TIMER);
5481
5482 /* set eqe_ba [34:3] */
5483 roce_set_field(eqc->eqe_ba0,
5484 HNS_ROCE_EQC_EQE_BA_L_M,
5485 HNS_ROCE_EQC_EQE_BA_L_S, eq->eqe_ba >> 3);
5486
5487 /* set eqe_ba [64:35] */
5488 roce_set_field(eqc->eqe_ba1,
5489 HNS_ROCE_EQC_EQE_BA_H_M,
5490 HNS_ROCE_EQC_EQE_BA_H_S, eq->eqe_ba >> 35);
5491
5492 /* set eq shift */
5493 roce_set_field(eqc->byte_28,
5494 HNS_ROCE_EQC_SHIFT_M,
5495 HNS_ROCE_EQC_SHIFT_S, eq->shift);
5496
5497 /* set eq MSI_IDX */
5498 roce_set_field(eqc->byte_28,
5499 HNS_ROCE_EQC_MSI_INDX_M,
5500 HNS_ROCE_EQC_MSI_INDX_S,
5501 HNS_ROCE_EQ_INIT_MSI_IDX);
5502
5503 /* set cur_eqe_ba [27:12] */
5504 roce_set_field(eqc->byte_28,
5505 HNS_ROCE_EQC_CUR_EQE_BA_L_M,
5506 HNS_ROCE_EQC_CUR_EQE_BA_L_S, eq->cur_eqe_ba >> 12);
5507
5508 /* set cur_eqe_ba [59:28] */
5509 roce_set_field(eqc->byte_32,
5510 HNS_ROCE_EQC_CUR_EQE_BA_M_M,
5511 HNS_ROCE_EQC_CUR_EQE_BA_M_S, eq->cur_eqe_ba >> 28);
5512
5513 /* set cur_eqe_ba [63:60] */
5514 roce_set_field(eqc->byte_36,
5515 HNS_ROCE_EQC_CUR_EQE_BA_H_M,
5516 HNS_ROCE_EQC_CUR_EQE_BA_H_S, eq->cur_eqe_ba >> 60);
5517
5518 /* set eq consumer idx */
5519 roce_set_field(eqc->byte_36,
5520 HNS_ROCE_EQC_CONS_INDX_M,
5521 HNS_ROCE_EQC_CONS_INDX_S,
5522 HNS_ROCE_EQ_INIT_CONS_IDX);
5523
5524 /* set nex_eqe_ba[43:12] */
5525 roce_set_field(eqc->nxt_eqe_ba0,
5526 HNS_ROCE_EQC_NXT_EQE_BA_L_M,
5527 HNS_ROCE_EQC_NXT_EQE_BA_L_S, eq->nxt_eqe_ba >> 12);
5528
5529 /* set nex_eqe_ba[63:44] */
5530 roce_set_field(eqc->nxt_eqe_ba1,
5531 HNS_ROCE_EQC_NXT_EQE_BA_H_M,
5532 HNS_ROCE_EQC_NXT_EQE_BA_H_S, eq->nxt_eqe_ba >> 44);
5533 }
5534
5535 static int hns_roce_mhop_alloc_eq(struct hns_roce_dev *hr_dev,
5536 struct hns_roce_eq *eq)
5537 {
5538 struct device *dev = hr_dev->dev;
5539 int eq_alloc_done = 0;
5540 int eq_buf_cnt = 0;
5541 int eqe_alloc;
5542 u32 buf_chk_sz;
5543 u32 bt_chk_sz;
5544 u32 mhop_num;
5545 u64 size;
5546 u64 idx;
5547 int ba_num;
5548 int bt_num;
5549 int record_i;
5550 int record_j;
5551 int i = 0;
5552 int j = 0;
5553
5554 mhop_num = hr_dev->caps.eqe_hop_num;
5555 buf_chk_sz = 1 << (hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT);
5556 bt_chk_sz = 1 << (hr_dev->caps.eqe_ba_pg_sz + PAGE_SHIFT);
5557
5558 ba_num = DIV_ROUND_UP(PAGE_ALIGN(eq->entries * eq->eqe_size),
5559 buf_chk_sz);
5560 bt_num = DIV_ROUND_UP(ba_num, bt_chk_sz / BA_BYTE_LEN);
5561
5562 if (mhop_num == HNS_ROCE_HOP_NUM_0) {
5563 if (eq->entries > buf_chk_sz / eq->eqe_size) {
5564 dev_err(dev, "eq entries %d is larger than buf_pg_sz!",
5565 eq->entries);
5566 return -EINVAL;
5567 }
5568 eq->bt_l0 = dma_alloc_coherent(dev, eq->entries * eq->eqe_size,
5569 &(eq->l0_dma), GFP_KERNEL);
5570 if (!eq->bt_l0)
5571 return -ENOMEM;
5572
5573 eq->cur_eqe_ba = eq->l0_dma;
5574 eq->nxt_eqe_ba = 0;
5575
5576 return 0;
5577 }
5578
5579 eq->buf_dma = kcalloc(ba_num, sizeof(*eq->buf_dma), GFP_KERNEL);
5580 if (!eq->buf_dma)
5581 return -ENOMEM;
5582 eq->buf = kcalloc(ba_num, sizeof(*eq->buf), GFP_KERNEL);
5583 if (!eq->buf)
5584 goto err_kcalloc_buf;
5585
5586 if (mhop_num == 2) {
5587 eq->l1_dma = kcalloc(bt_num, sizeof(*eq->l1_dma), GFP_KERNEL);
5588 if (!eq->l1_dma)
5589 goto err_kcalloc_l1_dma;
5590
5591 eq->bt_l1 = kcalloc(bt_num, sizeof(*eq->bt_l1), GFP_KERNEL);
5592 if (!eq->bt_l1)
5593 goto err_kcalloc_bt_l1;
5594 }
5595
5596 /* alloc L0 BT */
5597 eq->bt_l0 = dma_alloc_coherent(dev, bt_chk_sz, &eq->l0_dma, GFP_KERNEL);
5598 if (!eq->bt_l0)
5599 goto err_dma_alloc_l0;
5600
5601 if (mhop_num == 1) {
5602 if (ba_num > (bt_chk_sz / BA_BYTE_LEN))
5603 dev_err(dev, "ba_num %d is too large for 1 hop\n",
5604 ba_num);
5605
5606 /* alloc buf */
5607 for (i = 0; i < bt_chk_sz / BA_BYTE_LEN; i++) {
5608 if (eq_buf_cnt + 1 < ba_num) {
5609 size = buf_chk_sz;
5610 } else {
5611 eqe_alloc = i * (buf_chk_sz / eq->eqe_size);
5612 size = (eq->entries - eqe_alloc) * eq->eqe_size;
5613 }
5614 eq->buf[i] = dma_alloc_coherent(dev, size,
5615 &(eq->buf_dma[i]),
5616 GFP_KERNEL);
5617 if (!eq->buf[i])
5618 goto err_dma_alloc_buf;
5619
5620 *(eq->bt_l0 + i) = eq->buf_dma[i];
5621
5622 eq_buf_cnt++;
5623 if (eq_buf_cnt >= ba_num)
5624 break;
5625 }
5626 eq->cur_eqe_ba = eq->buf_dma[0];
5627 if (ba_num > 1)
5628 eq->nxt_eqe_ba = eq->buf_dma[1];
5629
5630 } else if (mhop_num == 2) {
5631 /* alloc L1 BT and buf */
5632 for (i = 0; i < bt_chk_sz / BA_BYTE_LEN; i++) {
5633 eq->bt_l1[i] = dma_alloc_coherent(dev, bt_chk_sz,
5634 &(eq->l1_dma[i]),
5635 GFP_KERNEL);
5636 if (!eq->bt_l1[i])
5637 goto err_dma_alloc_l1;
5638 *(eq->bt_l0 + i) = eq->l1_dma[i];
5639
5640 for (j = 0; j < bt_chk_sz / BA_BYTE_LEN; j++) {
5641 idx = i * bt_chk_sz / BA_BYTE_LEN + j;
5642 if (eq_buf_cnt + 1 < ba_num) {
5643 size = buf_chk_sz;
5644 } else {
5645 eqe_alloc = (buf_chk_sz / eq->eqe_size)
5646 * idx;
5647 size = (eq->entries - eqe_alloc)
5648 * eq->eqe_size;
5649 }
5650 eq->buf[idx] = dma_alloc_coherent(dev, size,
5651 &(eq->buf_dma[idx]),
5652 GFP_KERNEL);
5653 if (!eq->buf[idx])
5654 goto err_dma_alloc_buf;
5655
5656 *(eq->bt_l1[i] + j) = eq->buf_dma[idx];
5657
5658 eq_buf_cnt++;
5659 if (eq_buf_cnt >= ba_num) {
5660 eq_alloc_done = 1;
5661 break;
5662 }
5663 }
5664
5665 if (eq_alloc_done)
5666 break;
5667 }
5668 eq->cur_eqe_ba = eq->buf_dma[0];
5669 if (ba_num > 1)
5670 eq->nxt_eqe_ba = eq->buf_dma[1];
5671 }
5672
5673 eq->l0_last_num = i + 1;
5674 if (mhop_num == 2)
5675 eq->l1_last_num = j + 1;
5676
5677 return 0;
5678
5679 err_dma_alloc_l1:
5680 dma_free_coherent(dev, bt_chk_sz, eq->bt_l0, eq->l0_dma);
5681 eq->bt_l0 = NULL;
5682 eq->l0_dma = 0;
5683 for (i -= 1; i >= 0; i--) {
5684 dma_free_coherent(dev, bt_chk_sz, eq->bt_l1[i],
5685 eq->l1_dma[i]);
5686
5687 for (j = 0; j < bt_chk_sz / BA_BYTE_LEN; j++) {
5688 idx = i * bt_chk_sz / BA_BYTE_LEN + j;
5689 dma_free_coherent(dev, buf_chk_sz, eq->buf[idx],
5690 eq->buf_dma[idx]);
5691 }
5692 }
5693 goto err_dma_alloc_l0;
5694
5695 err_dma_alloc_buf:
5696 dma_free_coherent(dev, bt_chk_sz, eq->bt_l0, eq->l0_dma);
5697 eq->bt_l0 = NULL;
5698 eq->l0_dma = 0;
5699
5700 if (mhop_num == 1)
5701 for (i -= 1; i >= 0; i--)
5702 dma_free_coherent(dev, buf_chk_sz, eq->buf[i],
5703 eq->buf_dma[i]);
5704 else if (mhop_num == 2) {
5705 record_i = i;
5706 record_j = j;
5707 for (; i >= 0; i--) {
5708 dma_free_coherent(dev, bt_chk_sz, eq->bt_l1[i],
5709 eq->l1_dma[i]);
5710
5711 for (j = 0; j < bt_chk_sz / BA_BYTE_LEN; j++) {
5712 if (i == record_i && j >= record_j)
5713 break;
5714
5715 idx = i * bt_chk_sz / BA_BYTE_LEN + j;
5716 dma_free_coherent(dev, buf_chk_sz,
5717 eq->buf[idx],
5718 eq->buf_dma[idx]);
5719 }
5720 }
5721 }
5722
5723 err_dma_alloc_l0:
5724 kfree(eq->bt_l1);
5725 eq->bt_l1 = NULL;
5726
5727 err_kcalloc_bt_l1:
5728 kfree(eq->l1_dma);
5729 eq->l1_dma = NULL;
5730
5731 err_kcalloc_l1_dma:
5732 kfree(eq->buf);
5733 eq->buf = NULL;
5734
5735 err_kcalloc_buf:
5736 kfree(eq->buf_dma);
5737 eq->buf_dma = NULL;
5738
5739 return -ENOMEM;
5740 }
5741
5742 static int hns_roce_v2_create_eq(struct hns_roce_dev *hr_dev,
5743 struct hns_roce_eq *eq,
5744 unsigned int eq_cmd)
5745 {
5746 struct device *dev = hr_dev->dev;
5747 struct hns_roce_cmd_mailbox *mailbox;
5748 u32 buf_chk_sz = 0;
5749 int ret;
5750
5751 /* Allocate mailbox memory */
5752 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
5753 if (IS_ERR(mailbox))
5754 return PTR_ERR(mailbox);
5755
5756 if (!hr_dev->caps.eqe_hop_num) {
5757 buf_chk_sz = 1 << (hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT);
5758
5759 eq->buf_list = kzalloc(sizeof(struct hns_roce_buf_list),
5760 GFP_KERNEL);
5761 if (!eq->buf_list) {
5762 ret = -ENOMEM;
5763 goto free_cmd_mbox;
5764 }
5765
5766 eq->buf_list->buf = dma_alloc_coherent(dev, buf_chk_sz,
5767 &(eq->buf_list->map),
5768 GFP_KERNEL);
5769 if (!eq->buf_list->buf) {
5770 ret = -ENOMEM;
5771 goto err_alloc_buf;
5772 }
5773
5774 } else {
5775 ret = hns_roce_mhop_alloc_eq(hr_dev, eq);
5776 if (ret) {
5777 ret = -ENOMEM;
5778 goto free_cmd_mbox;
5779 }
5780 }
5781
5782 hns_roce_config_eqc(hr_dev, eq, mailbox->buf);
5783
5784 ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, eq->eqn, 0,
5785 eq_cmd, HNS_ROCE_CMD_TIMEOUT_MSECS);
5786 if (ret) {
5787 dev_err(dev, "[mailbox cmd] create eqc failed.\n");
5788 goto err_cmd_mbox;
5789 }
5790
5791 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
5792
5793 return 0;
5794
5795 err_cmd_mbox:
5796 if (!hr_dev->caps.eqe_hop_num)
5797 dma_free_coherent(dev, buf_chk_sz, eq->buf_list->buf,
5798 eq->buf_list->map);
5799 else {
5800 hns_roce_mhop_free_eq(hr_dev, eq);
5801 goto free_cmd_mbox;
5802 }
5803
5804 err_alloc_buf:
5805 kfree(eq->buf_list);
5806
5807 free_cmd_mbox:
5808 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
5809
5810 return ret;
5811 }
5812
5813 static int __hns_roce_request_irq(struct hns_roce_dev *hr_dev, int irq_num,
5814 int comp_num, int aeq_num, int other_num)
5815 {
5816 struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
5817 int i, j;
5818 int ret;
5819
5820 for (i = 0; i < irq_num; i++) {
5821 hr_dev->irq_names[i] = kzalloc(HNS_ROCE_INT_NAME_LEN,
5822 GFP_KERNEL);
5823 if (!hr_dev->irq_names[i]) {
5824 ret = -ENOMEM;
5825 goto err_kzalloc_failed;
5826 }
5827 }
5828
5829 /* irq contains: abnormal + AEQ + CEQ */
5830 for (j = 0; j < other_num; j++)
5831 snprintf((char *)hr_dev->irq_names[j],
5832 HNS_ROCE_INT_NAME_LEN, "hns-abn-%d", j);
5833
5834 for (j = other_num; j < (other_num + aeq_num); j++)
5835 snprintf((char *)hr_dev->irq_names[j],
5836 HNS_ROCE_INT_NAME_LEN, "hns-aeq-%d",
5837 j - other_num);
5838
5839 for (j = (other_num + aeq_num); j < irq_num; j++)
5840 snprintf((char *)hr_dev->irq_names[j],
5841 HNS_ROCE_INT_NAME_LEN, "hns-ceq-%d",
5842 j - other_num - aeq_num);
5843
5844 for (j = 0; j < irq_num; j++) {
5845 if (j < other_num)
5846 ret = request_irq(hr_dev->irq[j],
5847 hns_roce_v2_msix_interrupt_abn,
5848 0, hr_dev->irq_names[j], hr_dev);
5849
5850 else if (j < (other_num + comp_num))
5851 ret = request_irq(eq_table->eq[j - other_num].irq,
5852 hns_roce_v2_msix_interrupt_eq,
5853 0, hr_dev->irq_names[j + aeq_num],
5854 &eq_table->eq[j - other_num]);
5855 else
5856 ret = request_irq(eq_table->eq[j - other_num].irq,
5857 hns_roce_v2_msix_interrupt_eq,
5858 0, hr_dev->irq_names[j - comp_num],
5859 &eq_table->eq[j - other_num]);
5860 if (ret) {
5861 dev_err(hr_dev->dev, "Request irq error!\n");
5862 goto err_request_failed;
5863 }
5864 }
5865
5866 return 0;
5867
5868 err_request_failed:
5869 for (j -= 1; j >= 0; j--)
5870 if (j < other_num)
5871 free_irq(hr_dev->irq[j], hr_dev);
5872 else
5873 free_irq(eq_table->eq[j - other_num].irq,
5874 &eq_table->eq[j - other_num]);
5875
5876 err_kzalloc_failed:
5877 for (i -= 1; i >= 0; i--)
5878 kfree(hr_dev->irq_names[i]);
5879
5880 return ret;
5881 }
5882
5883 static void __hns_roce_free_irq(struct hns_roce_dev *hr_dev)
5884 {
5885 int irq_num;
5886 int eq_num;
5887 int i;
5888
5889 eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors;
5890 irq_num = eq_num + hr_dev->caps.num_other_vectors;
5891
5892 for (i = 0; i < hr_dev->caps.num_other_vectors; i++)
5893 free_irq(hr_dev->irq[i], hr_dev);
5894
5895 for (i = 0; i < eq_num; i++)
5896 free_irq(hr_dev->eq_table.eq[i].irq, &hr_dev->eq_table.eq[i]);
5897
5898 for (i = 0; i < irq_num; i++)
5899 kfree(hr_dev->irq_names[i]);
5900 }
5901
5902 static int hns_roce_v2_init_eq_table(struct hns_roce_dev *hr_dev)
5903 {
5904 struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
5905 struct device *dev = hr_dev->dev;
5906 struct hns_roce_eq *eq;
5907 unsigned int eq_cmd;
5908 int irq_num;
5909 int eq_num;
5910 int other_num;
5911 int comp_num;
5912 int aeq_num;
5913 int i;
5914 int ret;
5915
5916 other_num = hr_dev->caps.num_other_vectors;
5917 comp_num = hr_dev->caps.num_comp_vectors;
5918 aeq_num = hr_dev->caps.num_aeq_vectors;
5919
5920 eq_num = comp_num + aeq_num;
5921 irq_num = eq_num + other_num;
5922
5923 eq_table->eq = kcalloc(eq_num, sizeof(*eq_table->eq), GFP_KERNEL);
5924 if (!eq_table->eq)
5925 return -ENOMEM;
5926
5927 /* create eq */
5928 for (i = 0; i < eq_num; i++) {
5929 eq = &eq_table->eq[i];
5930 eq->hr_dev = hr_dev;
5931 eq->eqn = i;
5932 if (i < comp_num) {
5933 /* CEQ */
5934 eq_cmd = HNS_ROCE_CMD_CREATE_CEQC;
5935 eq->type_flag = HNS_ROCE_CEQ;
5936 eq->entries = hr_dev->caps.ceqe_depth;
5937 eq->eqe_size = HNS_ROCE_CEQ_ENTRY_SIZE;
5938 eq->irq = hr_dev->irq[i + other_num + aeq_num];
5939 eq->eq_max_cnt = HNS_ROCE_CEQ_DEFAULT_BURST_NUM;
5940 eq->eq_period = HNS_ROCE_CEQ_DEFAULT_INTERVAL;
5941 } else {
5942 /* AEQ */
5943 eq_cmd = HNS_ROCE_CMD_CREATE_AEQC;
5944 eq->type_flag = HNS_ROCE_AEQ;
5945 eq->entries = hr_dev->caps.aeqe_depth;
5946 eq->eqe_size = HNS_ROCE_AEQ_ENTRY_SIZE;
5947 eq->irq = hr_dev->irq[i - comp_num + other_num];
5948 eq->eq_max_cnt = HNS_ROCE_AEQ_DEFAULT_BURST_NUM;
5949 eq->eq_period = HNS_ROCE_AEQ_DEFAULT_INTERVAL;
5950 }
5951
5952 ret = hns_roce_v2_create_eq(hr_dev, eq, eq_cmd);
5953 if (ret) {
5954 dev_err(dev, "eq create failed.\n");
5955 goto err_create_eq_fail;
5956 }
5957 }
5958
5959 /* enable irq */
5960 hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_ENABLE);
5961
5962 ret = __hns_roce_request_irq(hr_dev, irq_num, comp_num,
5963 aeq_num, other_num);
5964 if (ret) {
5965 dev_err(dev, "Request irq failed.\n");
5966 goto err_request_irq_fail;
5967 }
5968
5969 hr_dev->irq_workq =
5970 create_singlethread_workqueue("hns_roce_irq_workqueue");
5971 if (!hr_dev->irq_workq) {
5972 dev_err(dev, "Create irq workqueue failed!\n");
5973 ret = -ENOMEM;
5974 goto err_create_wq_fail;
5975 }
5976
5977 return 0;
5978
5979 err_create_wq_fail:
5980 __hns_roce_free_irq(hr_dev);
5981
5982 err_request_irq_fail:
5983 hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_DISABLE);
5984
5985 err_create_eq_fail:
5986 for (i -= 1; i >= 0; i--)
5987 hns_roce_v2_free_eq(hr_dev, &eq_table->eq[i]);
5988 kfree(eq_table->eq);
5989
5990 return ret;
5991 }
5992
5993 static void hns_roce_v2_cleanup_eq_table(struct hns_roce_dev *hr_dev)
5994 {
5995 struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
5996 int eq_num;
5997 int i;
5998
5999 eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors;
6000
6001 /* Disable irq */
6002 hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_DISABLE);
6003
6004 __hns_roce_free_irq(hr_dev);
6005
6006 for (i = 0; i < eq_num; i++) {
6007 hns_roce_v2_destroy_eqc(hr_dev, i);
6008
6009 hns_roce_v2_free_eq(hr_dev, &eq_table->eq[i]);
6010 }
6011
6012 kfree(eq_table->eq);
6013
6014 flush_workqueue(hr_dev->irq_workq);
6015 destroy_workqueue(hr_dev->irq_workq);
6016 }
6017
6018 static void hns_roce_v2_write_srqc(struct hns_roce_dev *hr_dev,
6019 struct hns_roce_srq *srq, u32 pdn, u16 xrcd,
6020 u32 cqn, void *mb_buf, u64 *mtts_wqe,
6021 u64 *mtts_idx, dma_addr_t dma_handle_wqe,
6022 dma_addr_t dma_handle_idx)
6023 {
6024 struct hns_roce_srq_context *srq_context;
6025
6026 srq_context = mb_buf;
6027 memset(srq_context, 0, sizeof(*srq_context));
6028
6029 roce_set_field(srq_context->byte_4_srqn_srqst, SRQC_BYTE_4_SRQ_ST_M,
6030 SRQC_BYTE_4_SRQ_ST_S, 1);
6031
6032 roce_set_field(srq_context->byte_4_srqn_srqst,
6033 SRQC_BYTE_4_SRQ_WQE_HOP_NUM_M,
6034 SRQC_BYTE_4_SRQ_WQE_HOP_NUM_S,
6035 (hr_dev->caps.srqwqe_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 :
6036 hr_dev->caps.srqwqe_hop_num));
6037 roce_set_field(srq_context->byte_4_srqn_srqst,
6038 SRQC_BYTE_4_SRQ_SHIFT_M, SRQC_BYTE_4_SRQ_SHIFT_S,
6039 ilog2(srq->wqe_cnt));
6040
6041 roce_set_field(srq_context->byte_4_srqn_srqst, SRQC_BYTE_4_SRQN_M,
6042 SRQC_BYTE_4_SRQN_S, srq->srqn);
6043
6044 roce_set_field(srq_context->byte_8_limit_wl, SRQC_BYTE_8_SRQ_LIMIT_WL_M,
6045 SRQC_BYTE_8_SRQ_LIMIT_WL_S, 0);
6046
6047 roce_set_field(srq_context->byte_12_xrcd, SRQC_BYTE_12_SRQ_XRCD_M,
6048 SRQC_BYTE_12_SRQ_XRCD_S, xrcd);
6049
6050 srq_context->wqe_bt_ba = cpu_to_le32((u32)(dma_handle_wqe >> 3));
6051
6052 roce_set_field(srq_context->byte_24_wqe_bt_ba,
6053 SRQC_BYTE_24_SRQ_WQE_BT_BA_M,
6054 SRQC_BYTE_24_SRQ_WQE_BT_BA_S,
6055 dma_handle_wqe >> 35);
6056
6057 roce_set_field(srq_context->byte_28_rqws_pd, SRQC_BYTE_28_PD_M,
6058 SRQC_BYTE_28_PD_S, pdn);
6059 roce_set_field(srq_context->byte_28_rqws_pd, SRQC_BYTE_28_RQWS_M,
6060 SRQC_BYTE_28_RQWS_S, srq->max_gs <= 0 ? 0 :
6061 fls(srq->max_gs - 1));
6062
6063 srq_context->idx_bt_ba = cpu_to_le32(dma_handle_idx >> 3);
6064 roce_set_field(srq_context->rsv_idx_bt_ba,
6065 SRQC_BYTE_36_SRQ_IDX_BT_BA_M,
6066 SRQC_BYTE_36_SRQ_IDX_BT_BA_S,
6067 dma_handle_idx >> 35);
6068
6069 srq_context->idx_cur_blk_addr =
6070 cpu_to_le32(mtts_idx[0] >> PAGE_ADDR_SHIFT);
6071 roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
6072 SRQC_BYTE_44_SRQ_IDX_CUR_BLK_ADDR_M,
6073 SRQC_BYTE_44_SRQ_IDX_CUR_BLK_ADDR_S,
6074 mtts_idx[0] >> (32 + PAGE_ADDR_SHIFT));
6075 roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
6076 SRQC_BYTE_44_SRQ_IDX_HOP_NUM_M,
6077 SRQC_BYTE_44_SRQ_IDX_HOP_NUM_S,
6078 hr_dev->caps.idx_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 :
6079 hr_dev->caps.idx_hop_num);
6080
6081 roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
6082 SRQC_BYTE_44_SRQ_IDX_BA_PG_SZ_M,
6083 SRQC_BYTE_44_SRQ_IDX_BA_PG_SZ_S,
6084 hr_dev->caps.idx_ba_pg_sz + PG_SHIFT_OFFSET);
6085 roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
6086 SRQC_BYTE_44_SRQ_IDX_BUF_PG_SZ_M,
6087 SRQC_BYTE_44_SRQ_IDX_BUF_PG_SZ_S,
6088 hr_dev->caps.idx_buf_pg_sz + PG_SHIFT_OFFSET);
6089
6090 srq_context->idx_nxt_blk_addr =
6091 cpu_to_le32(mtts_idx[1] >> PAGE_ADDR_SHIFT);
6092 roce_set_field(srq_context->rsv_idxnxtblkaddr,
6093 SRQC_BYTE_52_SRQ_IDX_NXT_BLK_ADDR_M,
6094 SRQC_BYTE_52_SRQ_IDX_NXT_BLK_ADDR_S,
6095 mtts_idx[1] >> (32 + PAGE_ADDR_SHIFT));
6096 roce_set_field(srq_context->byte_56_xrc_cqn,
6097 SRQC_BYTE_56_SRQ_XRC_CQN_M, SRQC_BYTE_56_SRQ_XRC_CQN_S,
6098 cqn);
6099 roce_set_field(srq_context->byte_56_xrc_cqn,
6100 SRQC_BYTE_56_SRQ_WQE_BA_PG_SZ_M,
6101 SRQC_BYTE_56_SRQ_WQE_BA_PG_SZ_S,
6102 hr_dev->caps.srqwqe_ba_pg_sz + PG_SHIFT_OFFSET);
6103 roce_set_field(srq_context->byte_56_xrc_cqn,
6104 SRQC_BYTE_56_SRQ_WQE_BUF_PG_SZ_M,
6105 SRQC_BYTE_56_SRQ_WQE_BUF_PG_SZ_S,
6106 hr_dev->caps.srqwqe_buf_pg_sz + PG_SHIFT_OFFSET);
6107
6108 roce_set_bit(srq_context->db_record_addr_record_en,
6109 SRQC_BYTE_60_SRQ_RECORD_EN_S, 0);
6110 }
6111
6112 static int hns_roce_v2_modify_srq(struct ib_srq *ibsrq,
6113 struct ib_srq_attr *srq_attr,
6114 enum ib_srq_attr_mask srq_attr_mask,
6115 struct ib_udata *udata)
6116 {
6117 struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
6118 struct hns_roce_srq *srq = to_hr_srq(ibsrq);
6119 struct hns_roce_srq_context *srq_context;
6120 struct hns_roce_srq_context *srqc_mask;
6121 struct hns_roce_cmd_mailbox *mailbox;
6122 int ret;
6123
6124 if (srq_attr_mask & IB_SRQ_LIMIT) {
6125 if (srq_attr->srq_limit >= srq->wqe_cnt)
6126 return -EINVAL;
6127
6128 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
6129 if (IS_ERR(mailbox))
6130 return PTR_ERR(mailbox);
6131
6132 srq_context = mailbox->buf;
6133 srqc_mask = (struct hns_roce_srq_context *)mailbox->buf + 1;
6134
6135 memset(srqc_mask, 0xff, sizeof(*srqc_mask));
6136
6137 roce_set_field(srq_context->byte_8_limit_wl,
6138 SRQC_BYTE_8_SRQ_LIMIT_WL_M,
6139 SRQC_BYTE_8_SRQ_LIMIT_WL_S, srq_attr->srq_limit);
6140 roce_set_field(srqc_mask->byte_8_limit_wl,
6141 SRQC_BYTE_8_SRQ_LIMIT_WL_M,
6142 SRQC_BYTE_8_SRQ_LIMIT_WL_S, 0);
6143
6144 ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, srq->srqn, 0,
6145 HNS_ROCE_CMD_MODIFY_SRQC,
6146 HNS_ROCE_CMD_TIMEOUT_MSECS);
6147 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
6148 if (ret) {
6149 dev_err(hr_dev->dev,
6150 "MODIFY SRQ Failed to cmd mailbox.\n");
6151 return ret;
6152 }
6153 }
6154
6155 return 0;
6156 }
6157
6158 static int hns_roce_v2_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr)
6159 {
6160 struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
6161 struct hns_roce_srq *srq = to_hr_srq(ibsrq);
6162 struct hns_roce_srq_context *srq_context;
6163 struct hns_roce_cmd_mailbox *mailbox;
6164 int limit_wl;
6165 int ret;
6166
6167 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
6168 if (IS_ERR(mailbox))
6169 return PTR_ERR(mailbox);
6170
6171 srq_context = mailbox->buf;
6172 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, srq->srqn, 0,
6173 HNS_ROCE_CMD_QUERY_SRQC,
6174 HNS_ROCE_CMD_TIMEOUT_MSECS);
6175 if (ret) {
6176 dev_err(hr_dev->dev, "QUERY SRQ cmd process error\n");
6177 goto out;
6178 }
6179
6180 limit_wl = roce_get_field(srq_context->byte_8_limit_wl,
6181 SRQC_BYTE_8_SRQ_LIMIT_WL_M,
6182 SRQC_BYTE_8_SRQ_LIMIT_WL_S);
6183
6184 attr->srq_limit = limit_wl;
6185 attr->max_wr = srq->wqe_cnt - 1;
6186 attr->max_sge = srq->max_gs;
6187
6188 memcpy(srq_context, mailbox->buf, sizeof(*srq_context));
6189
6190 out:
6191 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
6192 return ret;
6193 }
6194
6195 static int find_empty_entry(struct hns_roce_idx_que *idx_que,
6196 unsigned long size)
6197 {
6198 int wqe_idx;
6199
6200 if (unlikely(bitmap_full(idx_que->bitmap, size)))
6201 return -ENOSPC;
6202
6203 wqe_idx = find_first_zero_bit(idx_que->bitmap, size);
6204
6205 bitmap_set(idx_que->bitmap, wqe_idx, 1);
6206
6207 return wqe_idx;
6208 }
6209
6210 static void fill_idx_queue(struct hns_roce_idx_que *idx_que,
6211 int cur_idx, int wqe_idx)
6212 {
6213 unsigned int *addr;
6214
6215 addr = (unsigned int *)hns_roce_buf_offset(&idx_que->idx_buf,
6216 cur_idx * idx_que->entry_sz);
6217 *addr = wqe_idx;
6218 }
6219
6220 static int hns_roce_v2_post_srq_recv(struct ib_srq *ibsrq,
6221 const struct ib_recv_wr *wr,
6222 const struct ib_recv_wr **bad_wr)
6223 {
6224 struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
6225 struct hns_roce_srq *srq = to_hr_srq(ibsrq);
6226 struct hns_roce_v2_wqe_data_seg *dseg;
6227 struct hns_roce_v2_db srq_db;
6228 unsigned long flags;
6229 int ret = 0;
6230 int wqe_idx;
6231 void *wqe;
6232 int nreq;
6233 int ind;
6234 int i;
6235
6236 spin_lock_irqsave(&srq->lock, flags);
6237
6238 ind = srq->head & (srq->wqe_cnt - 1);
6239
6240 for (nreq = 0; wr; ++nreq, wr = wr->next) {
6241 if (unlikely(wr->num_sge > srq->max_gs)) {
6242 ret = -EINVAL;
6243 *bad_wr = wr;
6244 break;
6245 }
6246
6247 if (unlikely(srq->head == srq->tail)) {
6248 ret = -ENOMEM;
6249 *bad_wr = wr;
6250 break;
6251 }
6252
6253 wqe_idx = find_empty_entry(&srq->idx_que, srq->wqe_cnt);
6254 if (wqe_idx < 0) {
6255 ret = -ENOMEM;
6256 *bad_wr = wr;
6257 break;
6258 }
6259
6260 fill_idx_queue(&srq->idx_que, ind, wqe_idx);
6261 wqe = get_srq_wqe(srq, wqe_idx);
6262 dseg = (struct hns_roce_v2_wqe_data_seg *)wqe;
6263
6264 for (i = 0; i < wr->num_sge; ++i) {
6265 dseg[i].len = cpu_to_le32(wr->sg_list[i].length);
6266 dseg[i].lkey = cpu_to_le32(wr->sg_list[i].lkey);
6267 dseg[i].addr = cpu_to_le64(wr->sg_list[i].addr);
6268 }
6269
6270 if (i < srq->max_gs) {
6271 dseg[i].len = 0;
6272 dseg[i].lkey = cpu_to_le32(0x100);
6273 dseg[i].addr = 0;
6274 }
6275
6276 srq->wrid[wqe_idx] = wr->wr_id;
6277 ind = (ind + 1) & (srq->wqe_cnt - 1);
6278 }
6279
6280 if (likely(nreq)) {
6281 srq->head += nreq;
6282
6283 /*
6284 * Make sure that descriptors are written before
6285 * doorbell record.
6286 */
6287 wmb();
6288
6289 srq_db.byte_4 =
6290 cpu_to_le32(HNS_ROCE_V2_SRQ_DB << V2_DB_BYTE_4_CMD_S |
6291 (srq->srqn & V2_DB_BYTE_4_TAG_M));
6292 srq_db.parameter = cpu_to_le32(srq->head);
6293
6294 hns_roce_write64(hr_dev, (__le32 *)&srq_db, srq->db_reg_l);
6295
6296 }
6297
6298 spin_unlock_irqrestore(&srq->lock, flags);
6299
6300 return ret;
6301 }
6302
6303 static const struct hns_roce_dfx_hw hns_roce_dfx_hw_v2 = {
6304 .query_cqc_info = hns_roce_v2_query_cqc_info,
6305 };
6306
6307 static const struct ib_device_ops hns_roce_v2_dev_ops = {
6308 .destroy_qp = hns_roce_v2_destroy_qp,
6309 .modify_cq = hns_roce_v2_modify_cq,
6310 .poll_cq = hns_roce_v2_poll_cq,
6311 .post_recv = hns_roce_v2_post_recv,
6312 .post_send = hns_roce_v2_post_send,
6313 .query_qp = hns_roce_v2_query_qp,
6314 .req_notify_cq = hns_roce_v2_req_notify_cq,
6315 };
6316
6317 static const struct ib_device_ops hns_roce_v2_dev_srq_ops = {
6318 .modify_srq = hns_roce_v2_modify_srq,
6319 .post_srq_recv = hns_roce_v2_post_srq_recv,
6320 .query_srq = hns_roce_v2_query_srq,
6321 };
6322
6323 static const struct hns_roce_hw hns_roce_hw_v2 = {
6324 .cmq_init = hns_roce_v2_cmq_init,
6325 .cmq_exit = hns_roce_v2_cmq_exit,
6326 .hw_profile = hns_roce_v2_profile,
6327 .hw_init = hns_roce_v2_init,
6328 .hw_exit = hns_roce_v2_exit,
6329 .post_mbox = hns_roce_v2_post_mbox,
6330 .chk_mbox = hns_roce_v2_chk_mbox,
6331 .rst_prc_mbox = hns_roce_v2_rst_process_cmd,
6332 .set_gid = hns_roce_v2_set_gid,
6333 .set_mac = hns_roce_v2_set_mac,
6334 .write_mtpt = hns_roce_v2_write_mtpt,
6335 .rereg_write_mtpt = hns_roce_v2_rereg_write_mtpt,
6336 .frmr_write_mtpt = hns_roce_v2_frmr_write_mtpt,
6337 .mw_write_mtpt = hns_roce_v2_mw_write_mtpt,
6338 .write_cqc = hns_roce_v2_write_cqc,
6339 .set_hem = hns_roce_v2_set_hem,
6340 .clear_hem = hns_roce_v2_clear_hem,
6341 .modify_qp = hns_roce_v2_modify_qp,
6342 .query_qp = hns_roce_v2_query_qp,
6343 .destroy_qp = hns_roce_v2_destroy_qp,
6344 .qp_flow_control_init = hns_roce_v2_qp_flow_control_init,
6345 .modify_cq = hns_roce_v2_modify_cq,
6346 .post_send = hns_roce_v2_post_send,
6347 .post_recv = hns_roce_v2_post_recv,
6348 .req_notify_cq = hns_roce_v2_req_notify_cq,
6349 .poll_cq = hns_roce_v2_poll_cq,
6350 .init_eq = hns_roce_v2_init_eq_table,
6351 .cleanup_eq = hns_roce_v2_cleanup_eq_table,
6352 .write_srqc = hns_roce_v2_write_srqc,
6353 .modify_srq = hns_roce_v2_modify_srq,
6354 .query_srq = hns_roce_v2_query_srq,
6355 .post_srq_recv = hns_roce_v2_post_srq_recv,
6356 .hns_roce_dev_ops = &hns_roce_v2_dev_ops,
6357 .hns_roce_dev_srq_ops = &hns_roce_v2_dev_srq_ops,
6358 };
6359
6360 static const struct pci_device_id hns_roce_hw_v2_pci_tbl[] = {
6361 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA), 0},
6362 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC), 0},
6363 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA), 0},
6364 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC), 0},
6365 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC), 0},
6366 /* required last entry */
6367 {0, }
6368 };
6369
6370 MODULE_DEVICE_TABLE(pci, hns_roce_hw_v2_pci_tbl);
6371
6372 static void hns_roce_hw_v2_get_cfg(struct hns_roce_dev *hr_dev,
6373 struct hnae3_handle *handle)
6374 {
6375 struct hns_roce_v2_priv *priv = hr_dev->priv;
6376 int i;
6377
6378 hr_dev->pci_dev = handle->pdev;
6379 hr_dev->dev = &handle->pdev->dev;
6380 hr_dev->hw = &hns_roce_hw_v2;
6381 hr_dev->dfx = &hns_roce_dfx_hw_v2;
6382 hr_dev->sdb_offset = ROCEE_DB_SQ_L_0_REG;
6383 hr_dev->odb_offset = hr_dev->sdb_offset;
6384
6385 /* Get info from NIC driver. */
6386 hr_dev->reg_base = handle->rinfo.roce_io_base;
6387 hr_dev->caps.num_ports = 1;
6388 hr_dev->iboe.netdevs[0] = handle->rinfo.netdev;
6389 hr_dev->iboe.phy_port[0] = 0;
6390
6391 addrconf_addr_eui48((u8 *)&hr_dev->ib_dev.node_guid,
6392 hr_dev->iboe.netdevs[0]->dev_addr);
6393
6394 for (i = 0; i < HNS_ROCE_V2_MAX_IRQ_NUM; i++)
6395 hr_dev->irq[i] = pci_irq_vector(handle->pdev,
6396 i + handle->rinfo.base_vector);
6397
6398 /* cmd issue mode: 0 is poll, 1 is event */
6399 hr_dev->cmd_mod = 1;
6400 hr_dev->loop_idc = 0;
6401
6402 hr_dev->reset_cnt = handle->ae_algo->ops->ae_dev_reset_cnt(handle);
6403 priv->handle = handle;
6404 }
6405
6406 static int __hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)
6407 {
6408 struct hns_roce_dev *hr_dev;
6409 int ret;
6410
6411 hr_dev = ib_alloc_device(hns_roce_dev, ib_dev);
6412 if (!hr_dev)
6413 return -ENOMEM;
6414
6415 hr_dev->priv = kzalloc(sizeof(struct hns_roce_v2_priv), GFP_KERNEL);
6416 if (!hr_dev->priv) {
6417 ret = -ENOMEM;
6418 goto error_failed_kzalloc;
6419 }
6420
6421 hns_roce_hw_v2_get_cfg(hr_dev, handle);
6422
6423 ret = hns_roce_init(hr_dev);
6424 if (ret) {
6425 dev_err(hr_dev->dev, "RoCE Engine init failed!\n");
6426 goto error_failed_get_cfg;
6427 }
6428
6429 handle->priv = hr_dev;
6430
6431 return 0;
6432
6433 error_failed_get_cfg:
6434 kfree(hr_dev->priv);
6435
6436 error_failed_kzalloc:
6437 ib_dealloc_device(&hr_dev->ib_dev);
6438
6439 return ret;
6440 }
6441
6442 static void __hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle,
6443 bool reset)
6444 {
6445 struct hns_roce_dev *hr_dev = (struct hns_roce_dev *)handle->priv;
6446
6447 if (!hr_dev)
6448 return;
6449
6450 handle->priv = NULL;
6451 hns_roce_exit(hr_dev);
6452 kfree(hr_dev->priv);
6453 ib_dealloc_device(&hr_dev->ib_dev);
6454 }
6455
6456 static int hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)
6457 {
6458 const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
6459 const struct pci_device_id *id;
6460 struct device *dev = &handle->pdev->dev;
6461 int ret;
6462
6463 handle->rinfo.instance_state = HNS_ROCE_STATE_INIT;
6464
6465 if (ops->ae_dev_resetting(handle) || ops->get_hw_reset_stat(handle)) {
6466 handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;
6467 goto reset_chk_err;
6468 }
6469
6470 id = pci_match_id(hns_roce_hw_v2_pci_tbl, handle->pdev);
6471 if (!id)
6472 return 0;
6473
6474 ret = __hns_roce_hw_v2_init_instance(handle);
6475 if (ret) {
6476 handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;
6477 dev_err(dev, "RoCE instance init failed! ret = %d\n", ret);
6478 if (ops->ae_dev_resetting(handle) ||
6479 ops->get_hw_reset_stat(handle))
6480 goto reset_chk_err;
6481 else
6482 return ret;
6483 }
6484
6485 handle->rinfo.instance_state = HNS_ROCE_STATE_INITED;
6486
6487
6488 return 0;
6489
6490 reset_chk_err:
6491 dev_err(dev, "Device is busy in resetting state.\n"
6492 "please retry later.\n");
6493
6494 return -EBUSY;
6495 }
6496
6497 static void hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle,
6498 bool reset)
6499 {
6500 if (handle->rinfo.instance_state != HNS_ROCE_STATE_INITED)
6501 return;
6502
6503 handle->rinfo.instance_state = HNS_ROCE_STATE_UNINIT;
6504
6505 __hns_roce_hw_v2_uninit_instance(handle, reset);
6506
6507 handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;
6508 }
6509 static int hns_roce_hw_v2_reset_notify_down(struct hnae3_handle *handle)
6510 {
6511 struct hns_roce_dev *hr_dev;
6512 struct ib_event event;
6513
6514 if (handle->rinfo.instance_state != HNS_ROCE_STATE_INITED) {
6515 set_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state);
6516 return 0;
6517 }
6518
6519 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_DOWN;
6520 clear_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state);
6521
6522 hr_dev = (struct hns_roce_dev *)handle->priv;
6523 if (!hr_dev)
6524 return 0;
6525
6526 hr_dev->is_reset = true;
6527 hr_dev->active = false;
6528 hr_dev->dis_db = true;
6529
6530 event.event = IB_EVENT_DEVICE_FATAL;
6531 event.device = &hr_dev->ib_dev;
6532 event.element.port_num = 1;
6533 ib_dispatch_event(&event);
6534
6535 return 0;
6536 }
6537
6538 static int hns_roce_hw_v2_reset_notify_init(struct hnae3_handle *handle)
6539 {
6540 struct device *dev = &handle->pdev->dev;
6541 int ret;
6542
6543 if (test_and_clear_bit(HNS_ROCE_RST_DIRECT_RETURN,
6544 &handle->rinfo.state)) {
6545 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INITED;
6546 return 0;
6547 }
6548
6549 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INIT;
6550
6551 dev_info(&handle->pdev->dev, "In reset process RoCE client reinit.\n");
6552 ret = __hns_roce_hw_v2_init_instance(handle);
6553 if (ret) {
6554 /* when reset notify type is HNAE3_INIT_CLIENT In reset notify
6555 * callback function, RoCE Engine reinitialize. If RoCE reinit
6556 * failed, we should inform NIC driver.
6557 */
6558 handle->priv = NULL;
6559 dev_err(dev, "In reset process RoCE reinit failed %d.\n", ret);
6560 } else {
6561 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INITED;
6562 dev_info(dev, "Reset done, RoCE client reinit finished.\n");
6563 }
6564
6565 return ret;
6566 }
6567
6568 static int hns_roce_hw_v2_reset_notify_uninit(struct hnae3_handle *handle)
6569 {
6570 if (test_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state))
6571 return 0;
6572
6573 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_UNINIT;
6574 dev_info(&handle->pdev->dev, "In reset process RoCE client uninit.\n");
6575 msleep(HNS_ROCE_V2_HW_RST_UNINT_DELAY);
6576 __hns_roce_hw_v2_uninit_instance(handle, false);
6577
6578 return 0;
6579 }
6580
6581 static int hns_roce_hw_v2_reset_notify(struct hnae3_handle *handle,
6582 enum hnae3_reset_notify_type type)
6583 {
6584 int ret = 0;
6585
6586 switch (type) {
6587 case HNAE3_DOWN_CLIENT:
6588 ret = hns_roce_hw_v2_reset_notify_down(handle);
6589 break;
6590 case HNAE3_INIT_CLIENT:
6591 ret = hns_roce_hw_v2_reset_notify_init(handle);
6592 break;
6593 case HNAE3_UNINIT_CLIENT:
6594 ret = hns_roce_hw_v2_reset_notify_uninit(handle);
6595 break;
6596 default:
6597 break;
6598 }
6599
6600 return ret;
6601 }
6602
6603 static const struct hnae3_client_ops hns_roce_hw_v2_ops = {
6604 .init_instance = hns_roce_hw_v2_init_instance,
6605 .uninit_instance = hns_roce_hw_v2_uninit_instance,
6606 .reset_notify = hns_roce_hw_v2_reset_notify,
6607 };
6608
6609 static struct hnae3_client hns_roce_hw_v2_client = {
6610 .name = "hns_roce_hw_v2",
6611 .type = HNAE3_CLIENT_ROCE,
6612 .ops = &hns_roce_hw_v2_ops,
6613 };
6614
6615 static int __init hns_roce_hw_v2_init(void)
6616 {
6617 return hnae3_register_client(&hns_roce_hw_v2_client);
6618 }
6619
6620 static void __exit hns_roce_hw_v2_exit(void)
6621 {
6622 hnae3_unregister_client(&hns_roce_hw_v2_client);
6623 }
6624
6625 module_init(hns_roce_hw_v2_init);
6626 module_exit(hns_roce_hw_v2_exit);
6627
6628 MODULE_LICENSE("Dual BSD/GPL");
6629 MODULE_AUTHOR("Wei Hu <xavier.huwei@huawei.com>");
6630 MODULE_AUTHOR("Lijun Ou <oulijun@huawei.com>");
6631 MODULE_AUTHOR("Shaobo Xu <xushaobo2@huawei.com>");
6632 MODULE_DESCRIPTION("Hisilicon Hip08 Family RoCE Driver");
Linux with added FPC-III support