?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
powerpc/mm/4k: don't allocate larger pmd page table for 4k
[linux/fpc-iii.git] / drivers / infiniband / hw / hns / hns_roce_hw_v1.c
blobb8111b0c8877b5f4b285f75fc3a0b4880b05653f
1 /*
2 * Copyright (c) 2016 Hisilicon Limited.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33 #include <linux/platform_device.h>
34 #include <linux/acpi.h>
35 #include <linux/etherdevice.h>
36 #include <rdma/ib_umem.h>
37 #include "hns_roce_common.h"
38 #include "hns_roce_device.h"
39 #include "hns_roce_cmd.h"
40 #include "hns_roce_hem.h"
41 #include "hns_roce_hw_v1.h"
42
43 static void set_data_seg(struct hns_roce_wqe_data_seg *dseg, struct ib_sge *sg)
44 {
45 dseg->lkey = cpu_to_le32(sg->lkey);
46 dseg->addr = cpu_to_le64(sg->addr);
47 dseg->len = cpu_to_le32(sg->length);
48 }
49
50 static void set_raddr_seg(struct hns_roce_wqe_raddr_seg *rseg, u64 remote_addr,
51 u32 rkey)
52 {
53 rseg->raddr = cpu_to_le64(remote_addr);
54 rseg->rkey = cpu_to_le32(rkey);
55 rseg->len = 0;
56 }
57
58 int hns_roce_v1_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
59 struct ib_send_wr **bad_wr)
60 {
61 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
62 struct hns_roce_ah *ah = to_hr_ah(ud_wr(wr)->ah);
63 struct hns_roce_ud_send_wqe *ud_sq_wqe = NULL;
64 struct hns_roce_wqe_ctrl_seg *ctrl = NULL;
65 struct hns_roce_wqe_data_seg *dseg = NULL;
66 struct hns_roce_qp *qp = to_hr_qp(ibqp);
67 struct device *dev = &hr_dev->pdev->dev;
68 struct hns_roce_sq_db sq_db;
69 int ps_opcode = 0, i = 0;
70 unsigned long flags = 0;
71 void *wqe = NULL;
72 u32 doorbell[2];
73 int nreq = 0;
74 u32 ind = 0;
75 int ret = 0;
76 u8 *smac;
77 int loopback;
78
79 if (unlikely(ibqp->qp_type != IB_QPT_GSI &&
80 ibqp->qp_type != IB_QPT_RC)) {
81 dev_err(dev, "un-supported QP type\n");
82 *bad_wr = NULL;
83 return -EOPNOTSUPP;
84 }
85
86 spin_lock_irqsave(&qp->sq.lock, flags);
87 ind = qp->sq_next_wqe;
88 for (nreq = 0; wr; ++nreq, wr = wr->next) {
89 if (hns_roce_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) {
90 ret = -ENOMEM;
91 *bad_wr = wr;
92 goto out;
93 }
94
95 if (unlikely(wr->num_sge > qp->sq.max_gs)) {
96 dev_err(dev, "num_sge=%d > qp->sq.max_gs=%d\n",
97 wr->num_sge, qp->sq.max_gs);
98 ret = -EINVAL;
99 *bad_wr = wr;
100 goto out;
101 }
102
103 wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1));
104 qp->sq.wrid[(qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1)] =
105 wr->wr_id;
106
107 /* Corresponding to the RC and RD type wqe process separately */
108 if (ibqp->qp_type == IB_QPT_GSI) {
109 ud_sq_wqe = wqe;
110 roce_set_field(ud_sq_wqe->dmac_h,
111 UD_SEND_WQE_U32_4_DMAC_0_M,
112 UD_SEND_WQE_U32_4_DMAC_0_S,
113 ah->av.mac[0]);
114 roce_set_field(ud_sq_wqe->dmac_h,
115 UD_SEND_WQE_U32_4_DMAC_1_M,
116 UD_SEND_WQE_U32_4_DMAC_1_S,
117 ah->av.mac[1]);
118 roce_set_field(ud_sq_wqe->dmac_h,
119 UD_SEND_WQE_U32_4_DMAC_2_M,
120 UD_SEND_WQE_U32_4_DMAC_2_S,
121 ah->av.mac[2]);
122 roce_set_field(ud_sq_wqe->dmac_h,
123 UD_SEND_WQE_U32_4_DMAC_3_M,
124 UD_SEND_WQE_U32_4_DMAC_3_S,
125 ah->av.mac[3]);
126
127 roce_set_field(ud_sq_wqe->u32_8,
128 UD_SEND_WQE_U32_8_DMAC_4_M,
129 UD_SEND_WQE_U32_8_DMAC_4_S,
130 ah->av.mac[4]);
131 roce_set_field(ud_sq_wqe->u32_8,
132 UD_SEND_WQE_U32_8_DMAC_5_M,
133 UD_SEND_WQE_U32_8_DMAC_5_S,
134 ah->av.mac[5]);
135
136 smac = (u8 *)hr_dev->dev_addr[qp->port];
137 loopback = ether_addr_equal_unaligned(ah->av.mac,
138 smac) ? 1 : 0;
139 roce_set_bit(ud_sq_wqe->u32_8,
140 UD_SEND_WQE_U32_8_LOOPBACK_INDICATOR_S,
141 loopback);
142
143 roce_set_field(ud_sq_wqe->u32_8,
144 UD_SEND_WQE_U32_8_OPERATION_TYPE_M,
145 UD_SEND_WQE_U32_8_OPERATION_TYPE_S,
146 HNS_ROCE_WQE_OPCODE_SEND);
147 roce_set_field(ud_sq_wqe->u32_8,
148 UD_SEND_WQE_U32_8_NUMBER_OF_DATA_SEG_M,
149 UD_SEND_WQE_U32_8_NUMBER_OF_DATA_SEG_S,
150 2);
151 roce_set_bit(ud_sq_wqe->u32_8,
152 UD_SEND_WQE_U32_8_SEND_GL_ROUTING_HDR_FLAG_S,
153 1);
154
155 ud_sq_wqe->u32_8 |= (wr->send_flags & IB_SEND_SIGNALED ?
156 cpu_to_le32(HNS_ROCE_WQE_CQ_NOTIFY) : 0) |
157 (wr->send_flags & IB_SEND_SOLICITED ?
158 cpu_to_le32(HNS_ROCE_WQE_SE) : 0) |
159 ((wr->opcode == IB_WR_SEND_WITH_IMM) ?
160 cpu_to_le32(HNS_ROCE_WQE_IMM) : 0);
161
162 roce_set_field(ud_sq_wqe->u32_16,
163 UD_SEND_WQE_U32_16_DEST_QP_M,
164 UD_SEND_WQE_U32_16_DEST_QP_S,
165 ud_wr(wr)->remote_qpn);
166 roce_set_field(ud_sq_wqe->u32_16,
167 UD_SEND_WQE_U32_16_MAX_STATIC_RATE_M,
168 UD_SEND_WQE_U32_16_MAX_STATIC_RATE_S,
169 ah->av.stat_rate);
170
171 roce_set_field(ud_sq_wqe->u32_36,
172 UD_SEND_WQE_U32_36_FLOW_LABEL_M,
173 UD_SEND_WQE_U32_36_FLOW_LABEL_S, 0);
174 roce_set_field(ud_sq_wqe->u32_36,
175 UD_SEND_WQE_U32_36_PRIORITY_M,
176 UD_SEND_WQE_U32_36_PRIORITY_S,
177 ah->av.sl_tclass_flowlabel >>
178 HNS_ROCE_SL_SHIFT);
179 roce_set_field(ud_sq_wqe->u32_36,
180 UD_SEND_WQE_U32_36_SGID_INDEX_M,
181 UD_SEND_WQE_U32_36_SGID_INDEX_S,
182 hns_get_gid_index(hr_dev, qp->phy_port,
183 ah->av.gid_index));
184
185 roce_set_field(ud_sq_wqe->u32_40,
186 UD_SEND_WQE_U32_40_HOP_LIMIT_M,
187 UD_SEND_WQE_U32_40_HOP_LIMIT_S,
188 ah->av.hop_limit);
189 roce_set_field(ud_sq_wqe->u32_40,
190 UD_SEND_WQE_U32_40_TRAFFIC_CLASS_M,
191 UD_SEND_WQE_U32_40_TRAFFIC_CLASS_S, 0);
192
193 memcpy(&ud_sq_wqe->dgid[0], &ah->av.dgid[0], GID_LEN);
194
195 ud_sq_wqe->va0_l = (u32)wr->sg_list[0].addr;
196 ud_sq_wqe->va0_h = (wr->sg_list[0].addr) >> 32;
197 ud_sq_wqe->l_key0 = wr->sg_list[0].lkey;
198
199 ud_sq_wqe->va1_l = (u32)wr->sg_list[1].addr;
200 ud_sq_wqe->va1_h = (wr->sg_list[1].addr) >> 32;
201 ud_sq_wqe->l_key1 = wr->sg_list[1].lkey;
202 ind++;
203 } else if (ibqp->qp_type == IB_QPT_RC) {
204 ctrl = wqe;
205 memset(ctrl, 0, sizeof(struct hns_roce_wqe_ctrl_seg));
206 for (i = 0; i < wr->num_sge; i++)
207 ctrl->msg_length += wr->sg_list[i].length;
208
209 ctrl->sgl_pa_h = 0;
210 ctrl->flag = 0;
211 ctrl->imm_data = send_ieth(wr);
212
213 /*Ctrl field, ctrl set type: sig, solic, imm, fence */
214 /* SO wait for conforming application scenarios */
215 ctrl->flag |= (wr->send_flags & IB_SEND_SIGNALED ?
216 cpu_to_le32(HNS_ROCE_WQE_CQ_NOTIFY) : 0) |
217 (wr->send_flags & IB_SEND_SOLICITED ?
218 cpu_to_le32(HNS_ROCE_WQE_SE) : 0) |
219 ((wr->opcode == IB_WR_SEND_WITH_IMM ||
220 wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) ?
221 cpu_to_le32(HNS_ROCE_WQE_IMM) : 0) |
222 (wr->send_flags & IB_SEND_FENCE ?
223 (cpu_to_le32(HNS_ROCE_WQE_FENCE)) : 0);
224
225 wqe += sizeof(struct hns_roce_wqe_ctrl_seg);
226
227 switch (wr->opcode) {
228 case IB_WR_RDMA_READ:
229 ps_opcode = HNS_ROCE_WQE_OPCODE_RDMA_READ;
230 set_raddr_seg(wqe, atomic_wr(wr)->remote_addr,
231 atomic_wr(wr)->rkey);
232 break;
233 case IB_WR_RDMA_WRITE:
234 case IB_WR_RDMA_WRITE_WITH_IMM:
235 ps_opcode = HNS_ROCE_WQE_OPCODE_RDMA_WRITE;
236 set_raddr_seg(wqe, atomic_wr(wr)->remote_addr,
237 atomic_wr(wr)->rkey);
238 break;
239 case IB_WR_SEND:
240 case IB_WR_SEND_WITH_INV:
241 case IB_WR_SEND_WITH_IMM:
242 ps_opcode = HNS_ROCE_WQE_OPCODE_SEND;
243 break;
244 case IB_WR_LOCAL_INV:
245 break;
246 case IB_WR_ATOMIC_CMP_AND_SWP:
247 case IB_WR_ATOMIC_FETCH_AND_ADD:
248 case IB_WR_LSO:
249 default:
250 ps_opcode = HNS_ROCE_WQE_OPCODE_MASK;
251 break;
252 }
253 ctrl->flag |= cpu_to_le32(ps_opcode);
254 wqe += sizeof(struct hns_roce_wqe_raddr_seg);
255
256 dseg = wqe;
257 if (wr->send_flags & IB_SEND_INLINE && wr->num_sge) {
258 if (ctrl->msg_length >
259 hr_dev->caps.max_sq_inline) {
260 ret = -EINVAL;
261 *bad_wr = wr;
262 dev_err(dev, "inline len(1-%d)=%d, illegal",
263 ctrl->msg_length,
264 hr_dev->caps.max_sq_inline);
265 goto out;
266 }
267 for (i = 0; i < wr->num_sge; i++) {
268 memcpy(wqe, ((void *) (uintptr_t)
269 wr->sg_list[i].addr),
270 wr->sg_list[i].length);
271 wqe += wr->sg_list[i].length;
272 }
273 ctrl->flag |= HNS_ROCE_WQE_INLINE;
274 } else {
275 /*sqe num is two */
276 for (i = 0; i < wr->num_sge; i++)
277 set_data_seg(dseg + i, wr->sg_list + i);
278
279 ctrl->flag |= cpu_to_le32(wr->num_sge <<
280 HNS_ROCE_WQE_SGE_NUM_BIT);
281 }
282 ind++;
283 }
284 }
285
286 out:
287 /* Set DB return */
288 if (likely(nreq)) {
289 qp->sq.head += nreq;
290 /* Memory barrier */
291 wmb();
292
293 sq_db.u32_4 = 0;
294 sq_db.u32_8 = 0;
295 roce_set_field(sq_db.u32_4, SQ_DOORBELL_U32_4_SQ_HEAD_M,
296 SQ_DOORBELL_U32_4_SQ_HEAD_S,
297 (qp->sq.head & ((qp->sq.wqe_cnt << 1) - 1)));
298 roce_set_field(sq_db.u32_4, SQ_DOORBELL_U32_4_SL_M,
299 SQ_DOORBELL_U32_4_SL_S, qp->sl);
300 roce_set_field(sq_db.u32_4, SQ_DOORBELL_U32_4_PORT_M,
301 SQ_DOORBELL_U32_4_PORT_S, qp->phy_port);
302 roce_set_field(sq_db.u32_8, SQ_DOORBELL_U32_8_QPN_M,
303 SQ_DOORBELL_U32_8_QPN_S, qp->doorbell_qpn);
304 roce_set_bit(sq_db.u32_8, SQ_DOORBELL_HW_SYNC_S, 1);
305
306 doorbell[0] = sq_db.u32_4;
307 doorbell[1] = sq_db.u32_8;
308
309 hns_roce_write64_k(doorbell, qp->sq.db_reg_l);
310 qp->sq_next_wqe = ind;
311 }
312
313 spin_unlock_irqrestore(&qp->sq.lock, flags);
314
315 return ret;
316 }
317
318 int hns_roce_v1_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
319 struct ib_recv_wr **bad_wr)
320 {
321 int ret = 0;
322 int nreq = 0;
323 int ind = 0;
324 int i = 0;
325 u32 reg_val = 0;
326 unsigned long flags = 0;
327 struct hns_roce_rq_wqe_ctrl *ctrl = NULL;
328 struct hns_roce_wqe_data_seg *scat = NULL;
329 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
330 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
331 struct device *dev = &hr_dev->pdev->dev;
332 struct hns_roce_rq_db rq_db;
333 uint32_t doorbell[2] = {0};
334
335 spin_lock_irqsave(&hr_qp->rq.lock, flags);
336 ind = hr_qp->rq.head & (hr_qp->rq.wqe_cnt - 1);
337
338 for (nreq = 0; wr; ++nreq, wr = wr->next) {
339 if (hns_roce_wq_overflow(&hr_qp->rq, nreq,
340 hr_qp->ibqp.recv_cq)) {
341 ret = -ENOMEM;
342 *bad_wr = wr;
343 goto out;
344 }
345
346 if (unlikely(wr->num_sge > hr_qp->rq.max_gs)) {
347 dev_err(dev, "rq:num_sge=%d > qp->sq.max_gs=%d\n",
348 wr->num_sge, hr_qp->rq.max_gs);
349 ret = -EINVAL;
350 *bad_wr = wr;
351 goto out;
352 }
353
354 ctrl = get_recv_wqe(hr_qp, ind);
355
356 roce_set_field(ctrl->rwqe_byte_12,
357 RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_M,
358 RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_S,
359 wr->num_sge);
360
361 scat = (struct hns_roce_wqe_data_seg *)(ctrl + 1);
362
363 for (i = 0; i < wr->num_sge; i++)
364 set_data_seg(scat + i, wr->sg_list + i);
365
366 hr_qp->rq.wrid[ind] = wr->wr_id;
367
368 ind = (ind + 1) & (hr_qp->rq.wqe_cnt - 1);
369 }
370
371 out:
372 if (likely(nreq)) {
373 hr_qp->rq.head += nreq;
374 /* Memory barrier */
375 wmb();
376
377 if (ibqp->qp_type == IB_QPT_GSI) {
378 /* SW update GSI rq header */
379 reg_val = roce_read(to_hr_dev(ibqp->device),
380 ROCEE_QP1C_CFG3_0_REG +
381 QP1C_CFGN_OFFSET * hr_qp->phy_port);
382 roce_set_field(reg_val,
383 ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_M,
384 ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_S,
385 hr_qp->rq.head);
386 roce_write(to_hr_dev(ibqp->device),
387 ROCEE_QP1C_CFG3_0_REG +
388 QP1C_CFGN_OFFSET * hr_qp->phy_port, reg_val);
389 } else {
390 rq_db.u32_4 = 0;
391 rq_db.u32_8 = 0;
392
393 roce_set_field(rq_db.u32_4, RQ_DOORBELL_U32_4_RQ_HEAD_M,
394 RQ_DOORBELL_U32_4_RQ_HEAD_S,
395 hr_qp->rq.head);
396 roce_set_field(rq_db.u32_8, RQ_DOORBELL_U32_8_QPN_M,
397 RQ_DOORBELL_U32_8_QPN_S, hr_qp->qpn);
398 roce_set_field(rq_db.u32_8, RQ_DOORBELL_U32_8_CMD_M,
399 RQ_DOORBELL_U32_8_CMD_S, 1);
400 roce_set_bit(rq_db.u32_8, RQ_DOORBELL_U32_8_HW_SYNC_S,
401 1);
402
403 doorbell[0] = rq_db.u32_4;
404 doorbell[1] = rq_db.u32_8;
405
406 hns_roce_write64_k(doorbell, hr_qp->rq.db_reg_l);
407 }
408 }
409 spin_unlock_irqrestore(&hr_qp->rq.lock, flags);
410
411 return ret;
412 }
413
414 static void hns_roce_set_db_event_mode(struct hns_roce_dev *hr_dev,
415 int sdb_mode, int odb_mode)
416 {
417 u32 val;
418
419 val = roce_read(hr_dev, ROCEE_GLB_CFG_REG);
420 roce_set_bit(val, ROCEE_GLB_CFG_ROCEE_DB_SQ_MODE_S, sdb_mode);
421 roce_set_bit(val, ROCEE_GLB_CFG_ROCEE_DB_OTH_MODE_S, odb_mode);
422 roce_write(hr_dev, ROCEE_GLB_CFG_REG, val);
423 }
424
425 static void hns_roce_set_db_ext_mode(struct hns_roce_dev *hr_dev, u32 sdb_mode,
426 u32 odb_mode)
427 {
428 u32 val;
429
430 /* Configure SDB/ODB extend mode */
431 val = roce_read(hr_dev, ROCEE_GLB_CFG_REG);
432 roce_set_bit(val, ROCEE_GLB_CFG_SQ_EXT_DB_MODE_S, sdb_mode);
433 roce_set_bit(val, ROCEE_GLB_CFG_OTH_EXT_DB_MODE_S, odb_mode);
434 roce_write(hr_dev, ROCEE_GLB_CFG_REG, val);
435 }
436
437 static void hns_roce_set_sdb(struct hns_roce_dev *hr_dev, u32 sdb_alept,
438 u32 sdb_alful)
439 {
440 u32 val;
441
442 /* Configure SDB */
443 val = roce_read(hr_dev, ROCEE_DB_SQ_WL_REG);
444 roce_set_field(val, ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_M,
445 ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_S, sdb_alful);
446 roce_set_field(val, ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_M,
447 ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_S, sdb_alept);
448 roce_write(hr_dev, ROCEE_DB_SQ_WL_REG, val);
449 }
450
451 static void hns_roce_set_odb(struct hns_roce_dev *hr_dev, u32 odb_alept,
452 u32 odb_alful)
453 {
454 u32 val;
455
456 /* Configure ODB */
457 val = roce_read(hr_dev, ROCEE_DB_OTHERS_WL_REG);
458 roce_set_field(val, ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_M,
459 ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_S, odb_alful);
460 roce_set_field(val, ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_M,
461 ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_S, odb_alept);
462 roce_write(hr_dev, ROCEE_DB_OTHERS_WL_REG, val);
463 }
464
465 static void hns_roce_set_sdb_ext(struct hns_roce_dev *hr_dev, u32 ext_sdb_alept,
466 u32 ext_sdb_alful)
467 {
468 struct device *dev = &hr_dev->pdev->dev;
469 struct hns_roce_v1_priv *priv;
470 struct hns_roce_db_table *db;
471 dma_addr_t sdb_dma_addr;
472 u32 val;
473
474 priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
475 db = &priv->db_table;
476
477 /* Configure extend SDB threshold */
478 roce_write(hr_dev, ROCEE_EXT_DB_SQ_WL_EMPTY_REG, ext_sdb_alept);
479 roce_write(hr_dev, ROCEE_EXT_DB_SQ_WL_REG, ext_sdb_alful);
480
481 /* Configure extend SDB base addr */
482 sdb_dma_addr = db->ext_db->sdb_buf_list->map;
483 roce_write(hr_dev, ROCEE_EXT_DB_SQ_REG, (u32)(sdb_dma_addr >> 12));
484
485 /* Configure extend SDB depth */
486 val = roce_read(hr_dev, ROCEE_EXT_DB_SQ_H_REG);
487 roce_set_field(val, ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_M,
488 ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_S,
489 db->ext_db->esdb_dep);
490 /*
491 * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of
492 * using 4K page, and shift more 32 because of
493 * caculating the high 32 bit value evaluated to hardware.
494 */
495 roce_set_field(val, ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_M,
496 ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_S, sdb_dma_addr >> 44);
497 roce_write(hr_dev, ROCEE_EXT_DB_SQ_H_REG, val);
498
499 dev_dbg(dev, "ext SDB depth: 0x%x\n", db->ext_db->esdb_dep);
500 dev_dbg(dev, "ext SDB threshold: epmty: 0x%x, ful: 0x%x\n",
501 ext_sdb_alept, ext_sdb_alful);
502 }
503
504 static void hns_roce_set_odb_ext(struct hns_roce_dev *hr_dev, u32 ext_odb_alept,
505 u32 ext_odb_alful)
506 {
507 struct device *dev = &hr_dev->pdev->dev;
508 struct hns_roce_v1_priv *priv;
509 struct hns_roce_db_table *db;
510 dma_addr_t odb_dma_addr;
511 u32 val;
512
513 priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
514 db = &priv->db_table;
515
516 /* Configure extend ODB threshold */
517 roce_write(hr_dev, ROCEE_EXT_DB_OTHERS_WL_EMPTY_REG, ext_odb_alept);
518 roce_write(hr_dev, ROCEE_EXT_DB_OTHERS_WL_REG, ext_odb_alful);
519
520 /* Configure extend ODB base addr */
521 odb_dma_addr = db->ext_db->odb_buf_list->map;
522 roce_write(hr_dev, ROCEE_EXT_DB_OTH_REG, (u32)(odb_dma_addr >> 12));
523
524 /* Configure extend ODB depth */
525 val = roce_read(hr_dev, ROCEE_EXT_DB_OTH_H_REG);
526 roce_set_field(val, ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_M,
527 ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_S,
528 db->ext_db->eodb_dep);
529 roce_set_field(val, ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_M,
530 ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_S,
531 db->ext_db->eodb_dep);
532 roce_write(hr_dev, ROCEE_EXT_DB_OTH_H_REG, val);
533
534 dev_dbg(dev, "ext ODB depth: 0x%x\n", db->ext_db->eodb_dep);
535 dev_dbg(dev, "ext ODB threshold: empty: 0x%x, ful: 0x%x\n",
536 ext_odb_alept, ext_odb_alful);
537 }
538
539 static int hns_roce_db_ext_init(struct hns_roce_dev *hr_dev, u32 sdb_ext_mod,
540 u32 odb_ext_mod)
541 {
542 struct device *dev = &hr_dev->pdev->dev;
543 struct hns_roce_v1_priv *priv;
544 struct hns_roce_db_table *db;
545 dma_addr_t sdb_dma_addr;
546 dma_addr_t odb_dma_addr;
547 int ret = 0;
548
549 priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
550 db = &priv->db_table;
551
552 db->ext_db = kmalloc(sizeof(*db->ext_db), GFP_KERNEL);
553 if (!db->ext_db)
554 return -ENOMEM;
555
556 if (sdb_ext_mod) {
557 db->ext_db->sdb_buf_list = kmalloc(
558 sizeof(*db->ext_db->sdb_buf_list), GFP_KERNEL);
559 if (!db->ext_db->sdb_buf_list) {
560 ret = -ENOMEM;
561 goto ext_sdb_buf_fail_out;
562 }
563
564 db->ext_db->sdb_buf_list->buf = dma_alloc_coherent(dev,
565 HNS_ROCE_V1_EXT_SDB_SIZE,
566 &sdb_dma_addr, GFP_KERNEL);
567 if (!db->ext_db->sdb_buf_list->buf) {
568 ret = -ENOMEM;
569 goto alloc_sq_db_buf_fail;
570 }
571 db->ext_db->sdb_buf_list->map = sdb_dma_addr;
572
573 db->ext_db->esdb_dep = ilog2(HNS_ROCE_V1_EXT_SDB_DEPTH);
574 hns_roce_set_sdb_ext(hr_dev, HNS_ROCE_V1_EXT_SDB_ALEPT,
575 HNS_ROCE_V1_EXT_SDB_ALFUL);
576 } else
577 hns_roce_set_sdb(hr_dev, HNS_ROCE_V1_SDB_ALEPT,
578 HNS_ROCE_V1_SDB_ALFUL);
579
580 if (odb_ext_mod) {
581 db->ext_db->odb_buf_list = kmalloc(
582 sizeof(*db->ext_db->odb_buf_list), GFP_KERNEL);
583 if (!db->ext_db->odb_buf_list) {
584 ret = -ENOMEM;
585 goto ext_odb_buf_fail_out;
586 }
587
588 db->ext_db->odb_buf_list->buf = dma_alloc_coherent(dev,
589 HNS_ROCE_V1_EXT_ODB_SIZE,
590 &odb_dma_addr, GFP_KERNEL);
591 if (!db->ext_db->odb_buf_list->buf) {
592 ret = -ENOMEM;
593 goto alloc_otr_db_buf_fail;
594 }
595 db->ext_db->odb_buf_list->map = odb_dma_addr;
596
597 db->ext_db->eodb_dep = ilog2(HNS_ROCE_V1_EXT_ODB_DEPTH);
598 hns_roce_set_odb_ext(hr_dev, HNS_ROCE_V1_EXT_ODB_ALEPT,
599 HNS_ROCE_V1_EXT_ODB_ALFUL);
600 } else
601 hns_roce_set_odb(hr_dev, HNS_ROCE_V1_ODB_ALEPT,
602 HNS_ROCE_V1_ODB_ALFUL);
603
604 hns_roce_set_db_ext_mode(hr_dev, sdb_ext_mod, odb_ext_mod);
605
606 return 0;
607
608 alloc_otr_db_buf_fail:
609 kfree(db->ext_db->odb_buf_list);
610
611 ext_odb_buf_fail_out:
612 if (sdb_ext_mod) {
613 dma_free_coherent(dev, HNS_ROCE_V1_EXT_SDB_SIZE,
614 db->ext_db->sdb_buf_list->buf,
615 db->ext_db->sdb_buf_list->map);
616 }
617
618 alloc_sq_db_buf_fail:
619 if (sdb_ext_mod)
620 kfree(db->ext_db->sdb_buf_list);
621
622 ext_sdb_buf_fail_out:
623 kfree(db->ext_db);
624 return ret;
625 }
626
627 static struct hns_roce_qp *hns_roce_v1_create_lp_qp(struct hns_roce_dev *hr_dev,
628 struct ib_pd *pd)
629 {
630 struct device *dev = &hr_dev->pdev->dev;
631 struct ib_qp_init_attr init_attr;
632 struct ib_qp *qp;
633
634 memset(&init_attr, 0, sizeof(struct ib_qp_init_attr));
635 init_attr.qp_type = IB_QPT_RC;
636 init_attr.sq_sig_type = IB_SIGNAL_ALL_WR;
637 init_attr.cap.max_recv_wr = HNS_ROCE_MIN_WQE_NUM;
638 init_attr.cap.max_send_wr = HNS_ROCE_MIN_WQE_NUM;
639
640 qp = hns_roce_create_qp(pd, &init_attr, NULL);
641 if (IS_ERR(qp)) {
642 dev_err(dev, "Create loop qp for mr free failed!");
643 return NULL;
644 }
645
646 return to_hr_qp(qp);
647 }
648
649 static int hns_roce_v1_rsv_lp_qp(struct hns_roce_dev *hr_dev)
650 {
651 struct hns_roce_caps *caps = &hr_dev->caps;
652 struct device *dev = &hr_dev->pdev->dev;
653 struct ib_cq_init_attr cq_init_attr;
654 struct hns_roce_free_mr *free_mr;
655 struct ib_qp_attr attr = { 0 };
656 struct hns_roce_v1_priv *priv;
657 struct hns_roce_qp *hr_qp;
658 struct ib_cq *cq;
659 struct ib_pd *pd;
660 u64 subnet_prefix;
661 int attr_mask = 0;
662 int i;
663 int ret;
664 u8 phy_port;
665 u8 sl;
666
667 priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
668 free_mr = &priv->free_mr;
669
670 /* Reserved cq for loop qp */
671 cq_init_attr.cqe = HNS_ROCE_MIN_WQE_NUM * 2;
672 cq_init_attr.comp_vector = 0;
673 cq = hns_roce_ib_create_cq(&hr_dev->ib_dev, &cq_init_attr, NULL, NULL);
674 if (IS_ERR(cq)) {
675 dev_err(dev, "Create cq for reseved loop qp failed!");
676 return -ENOMEM;
677 }
678 free_mr->mr_free_cq = to_hr_cq(cq);
679 free_mr->mr_free_cq->ib_cq.device = &hr_dev->ib_dev;
680 free_mr->mr_free_cq->ib_cq.uobject = NULL;
681 free_mr->mr_free_cq->ib_cq.comp_handler = NULL;
682 free_mr->mr_free_cq->ib_cq.event_handler = NULL;
683 free_mr->mr_free_cq->ib_cq.cq_context = NULL;
684 atomic_set(&free_mr->mr_free_cq->ib_cq.usecnt, 0);
685
686 pd = hns_roce_alloc_pd(&hr_dev->ib_dev, NULL, NULL);
687 if (IS_ERR(pd)) {
688 dev_err(dev, "Create pd for reseved loop qp failed!");
689 ret = -ENOMEM;
690 goto alloc_pd_failed;
691 }
692 free_mr->mr_free_pd = to_hr_pd(pd);
693 free_mr->mr_free_pd->ibpd.device = &hr_dev->ib_dev;
694 free_mr->mr_free_pd->ibpd.uobject = NULL;
695 atomic_set(&free_mr->mr_free_pd->ibpd.usecnt, 0);
696
697 attr.qp_access_flags = IB_ACCESS_REMOTE_WRITE;
698 attr.pkey_index = 0;
699 attr.min_rnr_timer = 0;
700 /* Disable read ability */
701 attr.max_dest_rd_atomic = 0;
702 attr.max_rd_atomic = 0;
703 /* Use arbitrary values as rq_psn and sq_psn */
704 attr.rq_psn = 0x0808;
705 attr.sq_psn = 0x0808;
706 attr.retry_cnt = 7;
707 attr.rnr_retry = 7;
708 attr.timeout = 0x12;
709 attr.path_mtu = IB_MTU_256;
710 attr.ah_attr.ah_flags = 1;
711 attr.ah_attr.static_rate = 3;
712 attr.ah_attr.grh.sgid_index = 0;
713 attr.ah_attr.grh.hop_limit = 1;
714 attr.ah_attr.grh.flow_label = 0;
715 attr.ah_attr.grh.traffic_class = 0;
716
717 subnet_prefix = cpu_to_be64(0xfe80000000000000LL);
718 for (i = 0; i < HNS_ROCE_V1_RESV_QP; i++) {
719 free_mr->mr_free_qp[i] = hns_roce_v1_create_lp_qp(hr_dev, pd);
720 if (IS_ERR(free_mr->mr_free_qp[i])) {
721 dev_err(dev, "Create loop qp failed!\n");
722 goto create_lp_qp_failed;
723 }
724 hr_qp = free_mr->mr_free_qp[i];
725
726 sl = i / caps->num_ports;
727
728 if (caps->num_ports == HNS_ROCE_MAX_PORTS)
729 phy_port = (i >= HNS_ROCE_MAX_PORTS) ? (i - 2) :
730 (i % caps->num_ports);
731 else
732 phy_port = i % caps->num_ports;
733
734 hr_qp->port = phy_port + 1;
735 hr_qp->phy_port = phy_port;
736 hr_qp->ibqp.qp_type = IB_QPT_RC;
737 hr_qp->ibqp.device = &hr_dev->ib_dev;
738 hr_qp->ibqp.uobject = NULL;
739 atomic_set(&hr_qp->ibqp.usecnt, 0);
740 hr_qp->ibqp.pd = pd;
741 hr_qp->ibqp.recv_cq = cq;
742 hr_qp->ibqp.send_cq = cq;
743
744 attr.ah_attr.port_num = phy_port + 1;
745 attr.ah_attr.sl = sl;
746 attr.port_num = phy_port + 1;
747
748 attr.dest_qp_num = hr_qp->qpn;
749 memcpy(attr.ah_attr.dmac, hr_dev->dev_addr[phy_port],
750 MAC_ADDR_OCTET_NUM);
751
752 memcpy(attr.ah_attr.grh.dgid.raw,
753 &subnet_prefix, sizeof(u64));
754 memcpy(&attr.ah_attr.grh.dgid.raw[8],
755 hr_dev->dev_addr[phy_port], 3);
756 memcpy(&attr.ah_attr.grh.dgid.raw[13],
757 hr_dev->dev_addr[phy_port] + 3, 3);
758 attr.ah_attr.grh.dgid.raw[11] = 0xff;
759 attr.ah_attr.grh.dgid.raw[12] = 0xfe;
760 attr.ah_attr.grh.dgid.raw[8] ^= 2;
761
762 attr_mask |= IB_QP_PORT;
763
764 ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, &attr, attr_mask,
765 IB_QPS_RESET, IB_QPS_INIT);
766 if (ret) {
767 dev_err(dev, "modify qp failed(%d)!\n", ret);
768 goto create_lp_qp_failed;
769 }
770
771 ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, &attr, attr_mask,
772 IB_QPS_INIT, IB_QPS_RTR);
773 if (ret) {
774 dev_err(dev, "modify qp failed(%d)!\n", ret);
775 goto create_lp_qp_failed;
776 }
777
778 ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, &attr, attr_mask,
779 IB_QPS_RTR, IB_QPS_RTS);
780 if (ret) {
781 dev_err(dev, "modify qp failed(%d)!\n", ret);
782 goto create_lp_qp_failed;
783 }
784 }
785
786 return 0;
787
788 create_lp_qp_failed:
789 for (i -= 1; i >= 0; i--) {
790 hr_qp = free_mr->mr_free_qp[i];
791 if (hns_roce_v1_destroy_qp(&hr_qp->ibqp))
792 dev_err(dev, "Destroy qp %d for mr free failed!\n", i);
793 }
794
795 if (hns_roce_dealloc_pd(pd))
796 dev_err(dev, "Destroy pd for create_lp_qp failed!\n");
797
798 alloc_pd_failed:
799 if (hns_roce_ib_destroy_cq(cq))
800 dev_err(dev, "Destroy cq for create_lp_qp failed!\n");
801
802 return -EINVAL;
803 }
804
805 static void hns_roce_v1_release_lp_qp(struct hns_roce_dev *hr_dev)
806 {
807 struct device *dev = &hr_dev->pdev->dev;
808 struct hns_roce_free_mr *free_mr;
809 struct hns_roce_v1_priv *priv;
810 struct hns_roce_qp *hr_qp;
811 int ret;
812 int i;
813
814 priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
815 free_mr = &priv->free_mr;
816
817 for (i = 0; i < HNS_ROCE_V1_RESV_QP; i++) {
818 hr_qp = free_mr->mr_free_qp[i];
819 ret = hns_roce_v1_destroy_qp(&hr_qp->ibqp);
820 if (ret)
821 dev_err(dev, "Destroy qp %d for mr free failed(%d)!\n",
822 i, ret);
823 }
824
825 ret = hns_roce_ib_destroy_cq(&free_mr->mr_free_cq->ib_cq);
826 if (ret)
827 dev_err(dev, "Destroy cq for mr_free failed(%d)!\n", ret);
828
829 ret = hns_roce_dealloc_pd(&free_mr->mr_free_pd->ibpd);
830 if (ret)
831 dev_err(dev, "Destroy pd for mr_free failed(%d)!\n", ret);
832 }
833
834 static int hns_roce_db_init(struct hns_roce_dev *hr_dev)
835 {
836 struct device *dev = &hr_dev->pdev->dev;
837 struct hns_roce_v1_priv *priv;
838 struct hns_roce_db_table *db;
839 u32 sdb_ext_mod;
840 u32 odb_ext_mod;
841 u32 sdb_evt_mod;
842 u32 odb_evt_mod;
843 int ret = 0;
844
845 priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
846 db = &priv->db_table;
847
848 memset(db, 0, sizeof(*db));
849
850 /* Default DB mode */
851 sdb_ext_mod = HNS_ROCE_SDB_EXTEND_MODE;
852 odb_ext_mod = HNS_ROCE_ODB_EXTEND_MODE;
853 sdb_evt_mod = HNS_ROCE_SDB_NORMAL_MODE;
854 odb_evt_mod = HNS_ROCE_ODB_POLL_MODE;
855
856 db->sdb_ext_mod = sdb_ext_mod;
857 db->odb_ext_mod = odb_ext_mod;
858
859 /* Init extend DB */
860 ret = hns_roce_db_ext_init(hr_dev, sdb_ext_mod, odb_ext_mod);
861 if (ret) {
862 dev_err(dev, "Failed in extend DB configuration.\n");
863 return ret;
864 }
865
866 hns_roce_set_db_event_mode(hr_dev, sdb_evt_mod, odb_evt_mod);
867
868 return 0;
869 }
870
871 void hns_roce_v1_recreate_lp_qp_work_fn(struct work_struct *work)
872 {
873 struct hns_roce_recreate_lp_qp_work *lp_qp_work;
874 struct hns_roce_dev *hr_dev;
875
876 lp_qp_work = container_of(work, struct hns_roce_recreate_lp_qp_work,
877 work);
878 hr_dev = to_hr_dev(lp_qp_work->ib_dev);
879
880 hns_roce_v1_release_lp_qp(hr_dev);
881
882 if (hns_roce_v1_rsv_lp_qp(hr_dev))
883 dev_err(&hr_dev->pdev->dev, "create reserver qp failed\n");
884
885 if (lp_qp_work->comp_flag)
886 complete(lp_qp_work->comp);
887
888 kfree(lp_qp_work);
889 }
890
891 static int hns_roce_v1_recreate_lp_qp(struct hns_roce_dev *hr_dev)
892 {
893 struct device *dev = &hr_dev->pdev->dev;
894 struct hns_roce_recreate_lp_qp_work *lp_qp_work;
895 struct hns_roce_free_mr *free_mr;
896 struct hns_roce_v1_priv *priv;
897 struct completion comp;
898 unsigned long end =
899 msecs_to_jiffies(HNS_ROCE_V1_RECREATE_LP_QP_TIMEOUT_MSECS) + jiffies;
900
901 priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
902 free_mr = &priv->free_mr;
903
904 lp_qp_work = kzalloc(sizeof(struct hns_roce_recreate_lp_qp_work),
905 GFP_KERNEL);
906
907 INIT_WORK(&(lp_qp_work->work), hns_roce_v1_recreate_lp_qp_work_fn);
908
909 lp_qp_work->ib_dev = &(hr_dev->ib_dev);
910 lp_qp_work->comp = &comp;
911 lp_qp_work->comp_flag = 1;
912
913 init_completion(lp_qp_work->comp);
914
915 queue_work(free_mr->free_mr_wq, &(lp_qp_work->work));
916
917 while (time_before_eq(jiffies, end)) {
918 if (try_wait_for_completion(&comp))
919 return 0;
920 msleep(HNS_ROCE_V1_RECREATE_LP_QP_WAIT_VALUE);
921 }
922
923 lp_qp_work->comp_flag = 0;
924 if (try_wait_for_completion(&comp))
925 return 0;
926
927 dev_warn(dev, "recreate lp qp failed 20s timeout and return failed!\n");
928 return -ETIMEDOUT;
929 }
930
931 static int hns_roce_v1_send_lp_wqe(struct hns_roce_qp *hr_qp)
932 {
933 struct hns_roce_dev *hr_dev = to_hr_dev(hr_qp->ibqp.device);
934 struct device *dev = &hr_dev->pdev->dev;
935 struct ib_send_wr send_wr, *bad_wr;
936 int ret;
937
938 memset(&send_wr, 0, sizeof(send_wr));
939 send_wr.next = NULL;
940 send_wr.num_sge = 0;
941 send_wr.send_flags = 0;
942 send_wr.sg_list = NULL;
943 send_wr.wr_id = (unsigned long long)&send_wr;
944 send_wr.opcode = IB_WR_RDMA_WRITE;
945
946 ret = hns_roce_v1_post_send(&hr_qp->ibqp, &send_wr, &bad_wr);
947 if (ret) {
948 dev_err(dev, "Post write wqe for mr free failed(%d)!", ret);
949 return ret;
950 }
951
952 return 0;
953 }
954
955 static void hns_roce_v1_mr_free_work_fn(struct work_struct *work)
956 {
957 struct hns_roce_mr_free_work *mr_work;
958 struct ib_wc wc[HNS_ROCE_V1_RESV_QP];
959 struct hns_roce_free_mr *free_mr;
960 struct hns_roce_cq *mr_free_cq;
961 struct hns_roce_v1_priv *priv;
962 struct hns_roce_dev *hr_dev;
963 struct hns_roce_mr *hr_mr;
964 struct hns_roce_qp *hr_qp;
965 struct device *dev;
966 unsigned long end =
967 msecs_to_jiffies(HNS_ROCE_V1_FREE_MR_TIMEOUT_MSECS) + jiffies;
968 int i;
969 int ret;
970 int ne;
971
972 mr_work = container_of(work, struct hns_roce_mr_free_work, work);
973 hr_mr = (struct hns_roce_mr *)mr_work->mr;
974 hr_dev = to_hr_dev(mr_work->ib_dev);
975 dev = &hr_dev->pdev->dev;
976
977 priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
978 free_mr = &priv->free_mr;
979 mr_free_cq = free_mr->mr_free_cq;
980
981 for (i = 0; i < HNS_ROCE_V1_RESV_QP; i++) {
982 hr_qp = free_mr->mr_free_qp[i];
983 ret = hns_roce_v1_send_lp_wqe(hr_qp);
984 if (ret) {
985 dev_err(dev,
986 "Send wqe (qp:0x%lx) for mr free failed(%d)!\n",
987 hr_qp->qpn, ret);
988 goto free_work;
989 }
990 }
991
992 ne = HNS_ROCE_V1_RESV_QP;
993 do {
994 ret = hns_roce_v1_poll_cq(&mr_free_cq->ib_cq, ne, wc);
995 if (ret < 0) {
996 dev_err(dev,
997 "(qp:0x%lx) starts, Poll cqe failed(%d) for mr 0x%x free! Remain %d cqe\n",
998 hr_qp->qpn, ret, hr_mr->key, ne);
999 goto free_work;
1000 }
1001 ne -= ret;
1002 msleep(HNS_ROCE_V1_FREE_MR_WAIT_VALUE);
1003 } while (ne && time_before_eq(jiffies, end));
1004
1005 if (ne != 0)
1006 dev_err(dev,
1007 "Poll cqe for mr 0x%x free timeout! Remain %d cqe\n",
1008 hr_mr->key, ne);
1009
1010 free_work:
1011 if (mr_work->comp_flag)
1012 complete(mr_work->comp);
1013 kfree(mr_work);
1014 }
1015
1016 int hns_roce_v1_dereg_mr(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr)
1017 {
1018 struct device *dev = &hr_dev->pdev->dev;
1019 struct hns_roce_mr_free_work *mr_work;
1020 struct hns_roce_free_mr *free_mr;
1021 struct hns_roce_v1_priv *priv;
1022 struct completion comp;
1023 unsigned long end =
1024 msecs_to_jiffies(HNS_ROCE_V1_FREE_MR_TIMEOUT_MSECS) + jiffies;
1025 unsigned long start = jiffies;
1026 int npages;
1027 int ret = 0;
1028
1029 priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
1030 free_mr = &priv->free_mr;
1031
1032 if (mr->enabled) {
1033 if (hns_roce_hw2sw_mpt(hr_dev, NULL, key_to_hw_index(mr->key)
1034 & (hr_dev->caps.num_mtpts - 1)))
1035 dev_warn(dev, "HW2SW_MPT failed!\n");
1036 }
1037
1038 mr_work = kzalloc(sizeof(*mr_work), GFP_KERNEL);
1039 if (!mr_work) {
1040 ret = -ENOMEM;
1041 goto free_mr;
1042 }
1043
1044 INIT_WORK(&(mr_work->work), hns_roce_v1_mr_free_work_fn);
1045
1046 mr_work->ib_dev = &(hr_dev->ib_dev);
1047 mr_work->comp = &comp;
1048 mr_work->comp_flag = 1;
1049 mr_work->mr = (void *)mr;
1050 init_completion(mr_work->comp);
1051
1052 queue_work(free_mr->free_mr_wq, &(mr_work->work));
1053
1054 while (time_before_eq(jiffies, end)) {
1055 if (try_wait_for_completion(&comp))
1056 goto free_mr;
1057 msleep(HNS_ROCE_V1_FREE_MR_WAIT_VALUE);
1058 }
1059
1060 mr_work->comp_flag = 0;
1061 if (try_wait_for_completion(&comp))
1062 goto free_mr;
1063
1064 dev_warn(dev, "Free mr work 0x%x over 50s and failed!\n", mr->key);
1065 ret = -ETIMEDOUT;
1066
1067 free_mr:
1068 dev_dbg(dev, "Free mr 0x%x use 0x%x us.\n",
1069 mr->key, jiffies_to_usecs(jiffies) - jiffies_to_usecs(start));
1070
1071 if (mr->size != ~0ULL) {
1072 npages = ib_umem_page_count(mr->umem);
1073 dma_free_coherent(dev, npages * 8, mr->pbl_buf,
1074 mr->pbl_dma_addr);
1075 }
1076
1077 hns_roce_bitmap_free(&hr_dev->mr_table.mtpt_bitmap,
1078 key_to_hw_index(mr->key), 0);
1079
1080 if (mr->umem)
1081 ib_umem_release(mr->umem);
1082
1083 kfree(mr);
1084
1085 return ret;
1086 }
1087
1088 static void hns_roce_db_free(struct hns_roce_dev *hr_dev)
1089 {
1090 struct device *dev = &hr_dev->pdev->dev;
1091 struct hns_roce_v1_priv *priv;
1092 struct hns_roce_db_table *db;
1093
1094 priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
1095 db = &priv->db_table;
1096
1097 if (db->sdb_ext_mod) {
1098 dma_free_coherent(dev, HNS_ROCE_V1_EXT_SDB_SIZE,
1099 db->ext_db->sdb_buf_list->buf,
1100 db->ext_db->sdb_buf_list->map);
1101 kfree(db->ext_db->sdb_buf_list);
1102 }
1103
1104 if (db->odb_ext_mod) {
1105 dma_free_coherent(dev, HNS_ROCE_V1_EXT_ODB_SIZE,
1106 db->ext_db->odb_buf_list->buf,
1107 db->ext_db->odb_buf_list->map);
1108 kfree(db->ext_db->odb_buf_list);
1109 }
1110
1111 kfree(db->ext_db);
1112 }
1113
1114 static int hns_roce_raq_init(struct hns_roce_dev *hr_dev)
1115 {
1116 int ret;
1117 int raq_shift = 0;
1118 dma_addr_t addr;
1119 u32 val;
1120 struct hns_roce_v1_priv *priv;
1121 struct hns_roce_raq_table *raq;
1122 struct device *dev = &hr_dev->pdev->dev;
1123
1124 priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
1125 raq = &priv->raq_table;
1126
1127 raq->e_raq_buf = kzalloc(sizeof(*(raq->e_raq_buf)), GFP_KERNEL);
1128 if (!raq->e_raq_buf)
1129 return -ENOMEM;
1130
1131 raq->e_raq_buf->buf = dma_alloc_coherent(dev, HNS_ROCE_V1_RAQ_SIZE,
1132 &addr, GFP_KERNEL);
1133 if (!raq->e_raq_buf->buf) {
1134 ret = -ENOMEM;
1135 goto err_dma_alloc_raq;
1136 }
1137 raq->e_raq_buf->map = addr;
1138
1139 /* Configure raq extended address. 48bit 4K align*/
1140 roce_write(hr_dev, ROCEE_EXT_RAQ_REG, raq->e_raq_buf->map >> 12);
1141
1142 /* Configure raq_shift */
1143 raq_shift = ilog2(HNS_ROCE_V1_RAQ_SIZE / HNS_ROCE_V1_RAQ_ENTRY);
1144 val = roce_read(hr_dev, ROCEE_EXT_RAQ_H_REG);
1145 roce_set_field(val, ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_M,
1146 ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_S, raq_shift);
1147 /*
1148 * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of
1149 * using 4K page, and shift more 32 because of
1150 * caculating the high 32 bit value evaluated to hardware.
1151 */
1152 roce_set_field(val, ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_M,
1153 ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_S,
1154 raq->e_raq_buf->map >> 44);
1155 roce_write(hr_dev, ROCEE_EXT_RAQ_H_REG, val);
1156 dev_dbg(dev, "Configure raq_shift 0x%x.\n", val);
1157
1158 /* Configure raq threshold */
1159 val = roce_read(hr_dev, ROCEE_RAQ_WL_REG);
1160 roce_set_field(val, ROCEE_RAQ_WL_ROCEE_RAQ_WL_M,
1161 ROCEE_RAQ_WL_ROCEE_RAQ_WL_S,
1162 HNS_ROCE_V1_EXT_RAQ_WF);
1163 roce_write(hr_dev, ROCEE_RAQ_WL_REG, val);
1164 dev_dbg(dev, "Configure raq_wl 0x%x.\n", val);
1165
1166 /* Enable extend raq */
1167 val = roce_read(hr_dev, ROCEE_WRMS_POL_TIME_INTERVAL_REG);
1168 roce_set_field(val,
1169 ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_M,
1170 ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_S,
1171 POL_TIME_INTERVAL_VAL);
1172 roce_set_bit(val, ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_EXT_RAQ_MODE, 1);
1173 roce_set_field(val,
1174 ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_M,
1175 ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_S,
1176 2);
1177 roce_set_bit(val,
1178 ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_EN_S, 1);
1179 roce_write(hr_dev, ROCEE_WRMS_POL_TIME_INTERVAL_REG, val);
1180 dev_dbg(dev, "Configure WrmsPolTimeInterval 0x%x.\n", val);
1181
1182 /* Enable raq drop */
1183 val = roce_read(hr_dev, ROCEE_GLB_CFG_REG);
1184 roce_set_bit(val, ROCEE_GLB_CFG_TRP_RAQ_DROP_EN_S, 1);
1185 roce_write(hr_dev, ROCEE_GLB_CFG_REG, val);
1186 dev_dbg(dev, "Configure GlbCfg = 0x%x.\n", val);
1187
1188 return 0;
1189
1190 err_dma_alloc_raq:
1191 kfree(raq->e_raq_buf);
1192 return ret;
1193 }
1194
1195 static void hns_roce_raq_free(struct hns_roce_dev *hr_dev)
1196 {
1197 struct device *dev = &hr_dev->pdev->dev;
1198 struct hns_roce_v1_priv *priv;
1199 struct hns_roce_raq_table *raq;
1200
1201 priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
1202 raq = &priv->raq_table;
1203
1204 dma_free_coherent(dev, HNS_ROCE_V1_RAQ_SIZE, raq->e_raq_buf->buf,
1205 raq->e_raq_buf->map);
1206 kfree(raq->e_raq_buf);
1207 }
1208
1209 static void hns_roce_port_enable(struct hns_roce_dev *hr_dev, int enable_flag)
1210 {
1211 u32 val;
1212
1213 if (enable_flag) {
1214 val = roce_read(hr_dev, ROCEE_GLB_CFG_REG);
1215 /* Open all ports */
1216 roce_set_field(val, ROCEE_GLB_CFG_ROCEE_PORT_ST_M,
1217 ROCEE_GLB_CFG_ROCEE_PORT_ST_S,
1218 ALL_PORT_VAL_OPEN);
1219 roce_write(hr_dev, ROCEE_GLB_CFG_REG, val);
1220 } else {
1221 val = roce_read(hr_dev, ROCEE_GLB_CFG_REG);
1222 /* Close all ports */
1223 roce_set_field(val, ROCEE_GLB_CFG_ROCEE_PORT_ST_M,
1224 ROCEE_GLB_CFG_ROCEE_PORT_ST_S, 0x0);
1225 roce_write(hr_dev, ROCEE_GLB_CFG_REG, val);
1226 }
1227 }
1228
1229 static int hns_roce_bt_init(struct hns_roce_dev *hr_dev)
1230 {
1231 struct device *dev = &hr_dev->pdev->dev;
1232 struct hns_roce_v1_priv *priv;
1233 int ret;
1234
1235 priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
1236
1237 priv->bt_table.qpc_buf.buf = dma_alloc_coherent(dev,
1238 HNS_ROCE_BT_RSV_BUF_SIZE, &priv->bt_table.qpc_buf.map,
1239 GFP_KERNEL);
1240 if (!priv->bt_table.qpc_buf.buf)
1241 return -ENOMEM;
1242
1243 priv->bt_table.mtpt_buf.buf = dma_alloc_coherent(dev,
1244 HNS_ROCE_BT_RSV_BUF_SIZE, &priv->bt_table.mtpt_buf.map,
1245 GFP_KERNEL);
1246 if (!priv->bt_table.mtpt_buf.buf) {
1247 ret = -ENOMEM;
1248 goto err_failed_alloc_mtpt_buf;
1249 }
1250
1251 priv->bt_table.cqc_buf.buf = dma_alloc_coherent(dev,
1252 HNS_ROCE_BT_RSV_BUF_SIZE, &priv->bt_table.cqc_buf.map,
1253 GFP_KERNEL);
1254 if (!priv->bt_table.cqc_buf.buf) {
1255 ret = -ENOMEM;
1256 goto err_failed_alloc_cqc_buf;
1257 }
1258
1259 return 0;
1260
1261 err_failed_alloc_cqc_buf:
1262 dma_free_coherent(dev, HNS_ROCE_BT_RSV_BUF_SIZE,
1263 priv->bt_table.mtpt_buf.buf, priv->bt_table.mtpt_buf.map);
1264
1265 err_failed_alloc_mtpt_buf:
1266 dma_free_coherent(dev, HNS_ROCE_BT_RSV_BUF_SIZE,
1267 priv->bt_table.qpc_buf.buf, priv->bt_table.qpc_buf.map);
1268
1269 return ret;
1270 }
1271
1272 static void hns_roce_bt_free(struct hns_roce_dev *hr_dev)
1273 {
1274 struct device *dev = &hr_dev->pdev->dev;
1275 struct hns_roce_v1_priv *priv;
1276
1277 priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
1278
1279 dma_free_coherent(dev, HNS_ROCE_BT_RSV_BUF_SIZE,
1280 priv->bt_table.cqc_buf.buf, priv->bt_table.cqc_buf.map);
1281
1282 dma_free_coherent(dev, HNS_ROCE_BT_RSV_BUF_SIZE,
1283 priv->bt_table.mtpt_buf.buf, priv->bt_table.mtpt_buf.map);
1284
1285 dma_free_coherent(dev, HNS_ROCE_BT_RSV_BUF_SIZE,
1286 priv->bt_table.qpc_buf.buf, priv->bt_table.qpc_buf.map);
1287 }
1288
1289 static int hns_roce_tptr_init(struct hns_roce_dev *hr_dev)
1290 {
1291 struct device *dev = &hr_dev->pdev->dev;
1292 struct hns_roce_buf_list *tptr_buf;
1293 struct hns_roce_v1_priv *priv;
1294
1295 priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
1296 tptr_buf = &priv->tptr_table.tptr_buf;
1297
1298 /*
1299 * This buffer will be used for CQ's tptr(tail pointer), also
1300 * named ci(customer index). Every CQ will use 2 bytes to save
1301 * cqe ci in hip06. Hardware will read this area to get new ci
1302 * when the queue is almost full.
1303 */
1304 tptr_buf->buf = dma_alloc_coherent(dev, HNS_ROCE_V1_TPTR_BUF_SIZE,
1305 &tptr_buf->map, GFP_KERNEL);
1306 if (!tptr_buf->buf)
1307 return -ENOMEM;
1308
1309 hr_dev->tptr_dma_addr = tptr_buf->map;
1310 hr_dev->tptr_size = HNS_ROCE_V1_TPTR_BUF_SIZE;
1311
1312 return 0;
1313 }
1314
1315 static void hns_roce_tptr_free(struct hns_roce_dev *hr_dev)
1316 {
1317 struct device *dev = &hr_dev->pdev->dev;
1318 struct hns_roce_buf_list *tptr_buf;
1319 struct hns_roce_v1_priv *priv;
1320
1321 priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
1322 tptr_buf = &priv->tptr_table.tptr_buf;
1323
1324 dma_free_coherent(dev, HNS_ROCE_V1_TPTR_BUF_SIZE,
1325 tptr_buf->buf, tptr_buf->map);
1326 }
1327
1328 static int hns_roce_free_mr_init(struct hns_roce_dev *hr_dev)
1329 {
1330 struct device *dev = &hr_dev->pdev->dev;
1331 struct hns_roce_free_mr *free_mr;
1332 struct hns_roce_v1_priv *priv;
1333 int ret = 0;
1334
1335 priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
1336 free_mr = &priv->free_mr;
1337
1338 free_mr->free_mr_wq = create_singlethread_workqueue("hns_roce_free_mr");
1339 if (!free_mr->free_mr_wq) {
1340 dev_err(dev, "Create free mr workqueue failed!\n");
1341 return -ENOMEM;
1342 }
1343
1344 ret = hns_roce_v1_rsv_lp_qp(hr_dev);
1345 if (ret) {
1346 dev_err(dev, "Reserved loop qp failed(%d)!\n", ret);
1347 flush_workqueue(free_mr->free_mr_wq);
1348 destroy_workqueue(free_mr->free_mr_wq);
1349 }
1350
1351 return ret;
1352 }
1353
1354 static void hns_roce_free_mr_free(struct hns_roce_dev *hr_dev)
1355 {
1356 struct hns_roce_free_mr *free_mr;
1357 struct hns_roce_v1_priv *priv;
1358
1359 priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
1360 free_mr = &priv->free_mr;
1361
1362 flush_workqueue(free_mr->free_mr_wq);
1363 destroy_workqueue(free_mr->free_mr_wq);
1364
1365 hns_roce_v1_release_lp_qp(hr_dev);
1366 }
1367
1368 /**
1369 * hns_roce_v1_reset - reset RoCE
1370 * @hr_dev: RoCE device struct pointer
1371 * @enable: true -- drop reset, false -- reset
1372 * return 0 - success , negative --fail
1373 */
1374 int hns_roce_v1_reset(struct hns_roce_dev *hr_dev, bool dereset)
1375 {
1376 struct device_node *dsaf_node;
1377 struct device *dev = &hr_dev->pdev->dev;
1378 struct device_node *np = dev->of_node;
1379 struct fwnode_handle *fwnode;
1380 int ret;
1381
1382 /* check if this is DT/ACPI case */
1383 if (dev_of_node(dev)) {
1384 dsaf_node = of_parse_phandle(np, "dsaf-handle", 0);
1385 if (!dsaf_node) {
1386 dev_err(dev, "could not find dsaf-handle\n");
1387 return -EINVAL;
1388 }
1389 fwnode = &dsaf_node->fwnode;
1390 } else if (is_acpi_device_node(dev->fwnode)) {
1391 struct acpi_reference_args args;
1392
1393 ret = acpi_node_get_property_reference(dev->fwnode,
1394 "dsaf-handle", 0, &args);
1395 if (ret) {
1396 dev_err(dev, "could not find dsaf-handle\n");
1397 return ret;
1398 }
1399 fwnode = acpi_fwnode_handle(args.adev);
1400 } else {
1401 dev_err(dev, "cannot read data from DT or ACPI\n");
1402 return -ENXIO;
1403 }
1404
1405 ret = hns_dsaf_roce_reset(fwnode, false);
1406 if (ret)
1407 return ret;
1408
1409 if (dereset) {
1410 msleep(SLEEP_TIME_INTERVAL);
1411 ret = hns_dsaf_roce_reset(fwnode, true);
1412 }
1413
1414 return ret;
1415 }
1416
1417 static int hns_roce_des_qp_init(struct hns_roce_dev *hr_dev)
1418 {
1419 struct device *dev = &hr_dev->pdev->dev;
1420 struct hns_roce_v1_priv *priv;
1421 struct hns_roce_des_qp *des_qp;
1422
1423 priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
1424 des_qp = &priv->des_qp;
1425
1426 des_qp->requeue_flag = 1;
1427 des_qp->qp_wq = create_singlethread_workqueue("hns_roce_destroy_qp");
1428 if (!des_qp->qp_wq) {
1429 dev_err(dev, "Create destroy qp workqueue failed!\n");
1430 return -ENOMEM;
1431 }
1432
1433 return 0;
1434 }
1435
1436 static void hns_roce_des_qp_free(struct hns_roce_dev *hr_dev)
1437 {
1438 struct hns_roce_v1_priv *priv;
1439 struct hns_roce_des_qp *des_qp;
1440
1441 priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
1442 des_qp = &priv->des_qp;
1443
1444 des_qp->requeue_flag = 0;
1445 flush_workqueue(des_qp->qp_wq);
1446 destroy_workqueue(des_qp->qp_wq);
1447 }
1448
1449 void hns_roce_v1_profile(struct hns_roce_dev *hr_dev)
1450 {
1451 int i = 0;
1452 struct hns_roce_caps *caps = &hr_dev->caps;
1453
1454 hr_dev->vendor_id = le32_to_cpu(roce_read(hr_dev, ROCEE_VENDOR_ID_REG));
1455 hr_dev->vendor_part_id = le32_to_cpu(roce_read(hr_dev,
1456 ROCEE_VENDOR_PART_ID_REG));
1457 hr_dev->sys_image_guid = le32_to_cpu(roce_read(hr_dev,
1458 ROCEE_SYS_IMAGE_GUID_L_REG)) |
1459 ((u64)le32_to_cpu(roce_read(hr_dev,
1460 ROCEE_SYS_IMAGE_GUID_H_REG)) << 32);
1461 hr_dev->hw_rev = HNS_ROCE_HW_VER1;
1462
1463 caps->num_qps = HNS_ROCE_V1_MAX_QP_NUM;
1464 caps->max_wqes = HNS_ROCE_V1_MAX_WQE_NUM;
1465 caps->num_cqs = HNS_ROCE_V1_MAX_CQ_NUM;
1466 caps->max_cqes = HNS_ROCE_V1_MAX_CQE_NUM;
1467 caps->max_sq_sg = HNS_ROCE_V1_SG_NUM;
1468 caps->max_rq_sg = HNS_ROCE_V1_SG_NUM;
1469 caps->max_sq_inline = HNS_ROCE_V1_INLINE_SIZE;
1470 caps->num_uars = HNS_ROCE_V1_UAR_NUM;
1471 caps->phy_num_uars = HNS_ROCE_V1_PHY_UAR_NUM;
1472 caps->num_aeq_vectors = HNS_ROCE_AEQE_VEC_NUM;
1473 caps->num_comp_vectors = HNS_ROCE_COMP_VEC_NUM;
1474 caps->num_other_vectors = HNS_ROCE_AEQE_OF_VEC_NUM;
1475 caps->num_mtpts = HNS_ROCE_V1_MAX_MTPT_NUM;
1476 caps->num_mtt_segs = HNS_ROCE_V1_MAX_MTT_SEGS;
1477 caps->num_pds = HNS_ROCE_V1_MAX_PD_NUM;
1478 caps->max_qp_init_rdma = HNS_ROCE_V1_MAX_QP_INIT_RDMA;
1479 caps->max_qp_dest_rdma = HNS_ROCE_V1_MAX_QP_DEST_RDMA;
1480 caps->max_sq_desc_sz = HNS_ROCE_V1_MAX_SQ_DESC_SZ;
1481 caps->max_rq_desc_sz = HNS_ROCE_V1_MAX_RQ_DESC_SZ;
1482 caps->qpc_entry_sz = HNS_ROCE_V1_QPC_ENTRY_SIZE;
1483 caps->irrl_entry_sz = HNS_ROCE_V1_IRRL_ENTRY_SIZE;
1484 caps->cqc_entry_sz = HNS_ROCE_V1_CQC_ENTRY_SIZE;
1485 caps->mtpt_entry_sz = HNS_ROCE_V1_MTPT_ENTRY_SIZE;
1486 caps->mtt_entry_sz = HNS_ROCE_V1_MTT_ENTRY_SIZE;
1487 caps->cq_entry_sz = HNS_ROCE_V1_CQE_ENTRY_SIZE;
1488 caps->page_size_cap = HNS_ROCE_V1_PAGE_SIZE_SUPPORT;
1489 caps->reserved_lkey = 0;
1490 caps->reserved_pds = 0;
1491 caps->reserved_mrws = 1;
1492 caps->reserved_uars = 0;
1493 caps->reserved_cqs = 0;
1494
1495 for (i = 0; i < caps->num_ports; i++)
1496 caps->pkey_table_len[i] = 1;
1497
1498 for (i = 0; i < caps->num_ports; i++) {
1499 /* Six ports shared 16 GID in v1 engine */
1500 if (i >= (HNS_ROCE_V1_GID_NUM % caps->num_ports))
1501 caps->gid_table_len[i] = HNS_ROCE_V1_GID_NUM /
1502 caps->num_ports;
1503 else
1504 caps->gid_table_len[i] = HNS_ROCE_V1_GID_NUM /
1505 caps->num_ports + 1;
1506 }
1507
1508 for (i = 0; i < caps->num_comp_vectors; i++)
1509 caps->ceqe_depth[i] = HNS_ROCE_V1_NUM_COMP_EQE;
1510
1511 caps->aeqe_depth = HNS_ROCE_V1_NUM_ASYNC_EQE;
1512 caps->local_ca_ack_delay = le32_to_cpu(roce_read(hr_dev,
1513 ROCEE_ACK_DELAY_REG));
1514 caps->max_mtu = IB_MTU_2048;
1515 }
1516
1517 int hns_roce_v1_init(struct hns_roce_dev *hr_dev)
1518 {
1519 int ret;
1520 u32 val;
1521 struct device *dev = &hr_dev->pdev->dev;
1522
1523 /* DMAE user config */
1524 val = roce_read(hr_dev, ROCEE_DMAE_USER_CFG1_REG);
1525 roce_set_field(val, ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_M,
1526 ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_S, 0xf);
1527 roce_set_field(val, ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_M,
1528 ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_S,
1529 1 << PAGES_SHIFT_16);
1530 roce_write(hr_dev, ROCEE_DMAE_USER_CFG1_REG, val);
1531
1532 val = roce_read(hr_dev, ROCEE_DMAE_USER_CFG2_REG);
1533 roce_set_field(val, ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_M,
1534 ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_S, 0xf);
1535 roce_set_field(val, ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_M,
1536 ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_S,
1537 1 << PAGES_SHIFT_16);
1538
1539 ret = hns_roce_db_init(hr_dev);
1540 if (ret) {
1541 dev_err(dev, "doorbell init failed!\n");
1542 return ret;
1543 }
1544
1545 ret = hns_roce_raq_init(hr_dev);
1546 if (ret) {
1547 dev_err(dev, "raq init failed!\n");
1548 goto error_failed_raq_init;
1549 }
1550
1551 ret = hns_roce_bt_init(hr_dev);
1552 if (ret) {
1553 dev_err(dev, "bt init failed!\n");
1554 goto error_failed_bt_init;
1555 }
1556
1557 ret = hns_roce_tptr_init(hr_dev);
1558 if (ret) {
1559 dev_err(dev, "tptr init failed!\n");
1560 goto error_failed_tptr_init;
1561 }
1562
1563 ret = hns_roce_des_qp_init(hr_dev);
1564 if (ret) {
1565 dev_err(dev, "des qp init failed!\n");
1566 goto error_failed_des_qp_init;
1567 }
1568
1569 ret = hns_roce_free_mr_init(hr_dev);
1570 if (ret) {
1571 dev_err(dev, "free mr init failed!\n");
1572 goto error_failed_free_mr_init;
1573 }
1574
1575 hns_roce_port_enable(hr_dev, HNS_ROCE_PORT_UP);
1576
1577 return 0;
1578
1579 error_failed_free_mr_init:
1580 hns_roce_des_qp_free(hr_dev);
1581
1582 error_failed_des_qp_init:
1583 hns_roce_tptr_free(hr_dev);
1584
1585 error_failed_tptr_init:
1586 hns_roce_bt_free(hr_dev);
1587
1588 error_failed_bt_init:
1589 hns_roce_raq_free(hr_dev);
1590
1591 error_failed_raq_init:
1592 hns_roce_db_free(hr_dev);
1593 return ret;
1594 }
1595
1596 void hns_roce_v1_exit(struct hns_roce_dev *hr_dev)
1597 {
1598 hns_roce_port_enable(hr_dev, HNS_ROCE_PORT_DOWN);
1599 hns_roce_free_mr_free(hr_dev);
1600 hns_roce_des_qp_free(hr_dev);
1601 hns_roce_tptr_free(hr_dev);
1602 hns_roce_bt_free(hr_dev);
1603 hns_roce_raq_free(hr_dev);
1604 hns_roce_db_free(hr_dev);
1605 }
1606
1607 void hns_roce_v1_set_gid(struct hns_roce_dev *hr_dev, u8 port, int gid_index,
1608 union ib_gid *gid)
1609 {
1610 u32 *p = NULL;
1611 u8 gid_idx = 0;
1612
1613 gid_idx = hns_get_gid_index(hr_dev, port, gid_index);
1614
1615 p = (u32 *)&gid->raw[0];
1616 roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_L_0_REG +
1617 (HNS_ROCE_V1_GID_NUM * gid_idx));
1618
1619 p = (u32 *)&gid->raw[4];
1620 roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_ML_0_REG +
1621 (HNS_ROCE_V1_GID_NUM * gid_idx));
1622
1623 p = (u32 *)&gid->raw[8];
1624 roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_MH_0_REG +
1625 (HNS_ROCE_V1_GID_NUM * gid_idx));
1626
1627 p = (u32 *)&gid->raw[0xc];
1628 roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_H_0_REG +
1629 (HNS_ROCE_V1_GID_NUM * gid_idx));
1630 }
1631
1632 void hns_roce_v1_set_mac(struct hns_roce_dev *hr_dev, u8 phy_port, u8 *addr)
1633 {
1634 u32 reg_smac_l;
1635 u16 reg_smac_h;
1636 u16 *p_h;
1637 u32 *p;
1638 u32 val;
1639
1640 /*
1641 * When mac changed, loopback may fail
1642 * because of smac not equal to dmac.
1643 * We Need to release and create reserved qp again.
1644 */
1645 if (hr_dev->hw->dereg_mr && hns_roce_v1_recreate_lp_qp(hr_dev))
1646 dev_warn(&hr_dev->pdev->dev, "recreate lp qp timeout!\n");
1647
1648 p = (u32 *)(&addr[0]);
1649 reg_smac_l = *p;
1650 roce_raw_write(reg_smac_l, hr_dev->reg_base + ROCEE_SMAC_L_0_REG +
1651 PHY_PORT_OFFSET * phy_port);
1652
1653 val = roce_read(hr_dev,
1654 ROCEE_SMAC_H_0_REG + phy_port * PHY_PORT_OFFSET);
1655 p_h = (u16 *)(&addr[4]);
1656 reg_smac_h = *p_h;
1657 roce_set_field(val, ROCEE_SMAC_H_ROCEE_SMAC_H_M,
1658 ROCEE_SMAC_H_ROCEE_SMAC_H_S, reg_smac_h);
1659 roce_write(hr_dev, ROCEE_SMAC_H_0_REG + phy_port * PHY_PORT_OFFSET,
1660 val);
1661 }
1662
1663 void hns_roce_v1_set_mtu(struct hns_roce_dev *hr_dev, u8 phy_port,
1664 enum ib_mtu mtu)
1665 {
1666 u32 val;
1667
1668 val = roce_read(hr_dev,
1669 ROCEE_SMAC_H_0_REG + phy_port * PHY_PORT_OFFSET);
1670 roce_set_field(val, ROCEE_SMAC_H_ROCEE_PORT_MTU_M,
1671 ROCEE_SMAC_H_ROCEE_PORT_MTU_S, mtu);
1672 roce_write(hr_dev, ROCEE_SMAC_H_0_REG + phy_port * PHY_PORT_OFFSET,
1673 val);
1674 }
1675
1676 int hns_roce_v1_write_mtpt(void *mb_buf, struct hns_roce_mr *mr,
1677 unsigned long mtpt_idx)
1678 {
1679 struct hns_roce_v1_mpt_entry *mpt_entry;
1680 struct scatterlist *sg;
1681 u64 *pages;
1682 int entry;
1683 int i;
1684
1685 /* MPT filled into mailbox buf */
1686 mpt_entry = (struct hns_roce_v1_mpt_entry *)mb_buf;
1687 memset(mpt_entry, 0, sizeof(*mpt_entry));
1688
1689 roce_set_field(mpt_entry->mpt_byte_4, MPT_BYTE_4_KEY_STATE_M,
1690 MPT_BYTE_4_KEY_STATE_S, KEY_VALID);
1691 roce_set_field(mpt_entry->mpt_byte_4, MPT_BYTE_4_KEY_M,
1692 MPT_BYTE_4_KEY_S, mr->key);
1693 roce_set_field(mpt_entry->mpt_byte_4, MPT_BYTE_4_PAGE_SIZE_M,
1694 MPT_BYTE_4_PAGE_SIZE_S, MR_SIZE_4K);
1695 roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_MW_TYPE_S, 0);
1696 roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_MW_BIND_ENABLE_S,
1697 (mr->access & IB_ACCESS_MW_BIND ? 1 : 0));
1698 roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_OWN_S, 0);
1699 roce_set_field(mpt_entry->mpt_byte_4, MPT_BYTE_4_MEMORY_LOCATION_TYPE_M,
1700 MPT_BYTE_4_MEMORY_LOCATION_TYPE_S, mr->type);
1701 roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_REMOTE_ATOMIC_S, 0);
1702 roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_LOCAL_WRITE_S,
1703 (mr->access & IB_ACCESS_LOCAL_WRITE ? 1 : 0));
1704 roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_REMOTE_WRITE_S,
1705 (mr->access & IB_ACCESS_REMOTE_WRITE ? 1 : 0));
1706 roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_REMOTE_READ_S,
1707 (mr->access & IB_ACCESS_REMOTE_READ ? 1 : 0));
1708 roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_REMOTE_INVAL_ENABLE_S,
1709 0);
1710 roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_ADDRESS_TYPE_S, 0);
1711
1712 roce_set_field(mpt_entry->mpt_byte_12, MPT_BYTE_12_PBL_ADDR_H_M,
1713 MPT_BYTE_12_PBL_ADDR_H_S, 0);
1714 roce_set_field(mpt_entry->mpt_byte_12, MPT_BYTE_12_MW_BIND_COUNTER_M,
1715 MPT_BYTE_12_MW_BIND_COUNTER_S, 0);
1716
1717 mpt_entry->virt_addr_l = (u32)mr->iova;
1718 mpt_entry->virt_addr_h = (u32)(mr->iova >> 32);
1719 mpt_entry->length = (u32)mr->size;
1720
1721 roce_set_field(mpt_entry->mpt_byte_28, MPT_BYTE_28_PD_M,
1722 MPT_BYTE_28_PD_S, mr->pd);
1723 roce_set_field(mpt_entry->mpt_byte_28, MPT_BYTE_28_L_KEY_IDX_L_M,
1724 MPT_BYTE_28_L_KEY_IDX_L_S, mtpt_idx);
1725 roce_set_field(mpt_entry->mpt_byte_64, MPT_BYTE_64_L_KEY_IDX_H_M,
1726 MPT_BYTE_64_L_KEY_IDX_H_S, mtpt_idx >> MTPT_IDX_SHIFT);
1727
1728 /* DMA momery regsiter */
1729 if (mr->type == MR_TYPE_DMA)
1730 return 0;
1731
1732 pages = (u64 *) __get_free_page(GFP_KERNEL);
1733 if (!pages)
1734 return -ENOMEM;
1735
1736 i = 0;
1737 for_each_sg(mr->umem->sg_head.sgl, sg, mr->umem->nmap, entry) {
1738 pages[i] = ((u64)sg_dma_address(sg)) >> 12;
1739
1740 /* Directly record to MTPT table firstly 7 entry */
1741 if (i >= HNS_ROCE_MAX_INNER_MTPT_NUM)
1742 break;
1743 i++;
1744 }
1745
1746 /* Register user mr */
1747 for (i = 0; i < HNS_ROCE_MAX_INNER_MTPT_NUM; i++) {
1748 switch (i) {
1749 case 0:
1750 mpt_entry->pa0_l = cpu_to_le32((u32)(pages[i]));
1751 roce_set_field(mpt_entry->mpt_byte_36,
1752 MPT_BYTE_36_PA0_H_M,
1753 MPT_BYTE_36_PA0_H_S,
1754 cpu_to_le32((u32)(pages[i] >> PAGES_SHIFT_32)));
1755 break;
1756 case 1:
1757 roce_set_field(mpt_entry->mpt_byte_36,
1758 MPT_BYTE_36_PA1_L_M,
1759 MPT_BYTE_36_PA1_L_S,
1760 cpu_to_le32((u32)(pages[i])));
1761 roce_set_field(mpt_entry->mpt_byte_40,
1762 MPT_BYTE_40_PA1_H_M,
1763 MPT_BYTE_40_PA1_H_S,
1764 cpu_to_le32((u32)(pages[i] >> PAGES_SHIFT_24)));
1765 break;
1766 case 2:
1767 roce_set_field(mpt_entry->mpt_byte_40,
1768 MPT_BYTE_40_PA2_L_M,
1769 MPT_BYTE_40_PA2_L_S,
1770 cpu_to_le32((u32)(pages[i])));
1771 roce_set_field(mpt_entry->mpt_byte_44,
1772 MPT_BYTE_44_PA2_H_M,
1773 MPT_BYTE_44_PA2_H_S,
1774 cpu_to_le32((u32)(pages[i] >> PAGES_SHIFT_16)));
1775 break;
1776 case 3:
1777 roce_set_field(mpt_entry->mpt_byte_44,
1778 MPT_BYTE_44_PA3_L_M,
1779 MPT_BYTE_44_PA3_L_S,
1780 cpu_to_le32((u32)(pages[i])));
1781 roce_set_field(mpt_entry->mpt_byte_48,
1782 MPT_BYTE_48_PA3_H_M,
1783 MPT_BYTE_48_PA3_H_S,
1784 cpu_to_le32((u32)(pages[i] >> PAGES_SHIFT_8)));
1785 break;
1786 case 4:
1787 mpt_entry->pa4_l = cpu_to_le32((u32)(pages[i]));
1788 roce_set_field(mpt_entry->mpt_byte_56,
1789 MPT_BYTE_56_PA4_H_M,
1790 MPT_BYTE_56_PA4_H_S,
1791 cpu_to_le32((u32)(pages[i] >> PAGES_SHIFT_32)));
1792 break;
1793 case 5:
1794 roce_set_field(mpt_entry->mpt_byte_56,
1795 MPT_BYTE_56_PA5_L_M,
1796 MPT_BYTE_56_PA5_L_S,
1797 cpu_to_le32((u32)(pages[i])));
1798 roce_set_field(mpt_entry->mpt_byte_60,
1799 MPT_BYTE_60_PA5_H_M,
1800 MPT_BYTE_60_PA5_H_S,
1801 cpu_to_le32((u32)(pages[i] >> PAGES_SHIFT_24)));
1802 break;
1803 case 6:
1804 roce_set_field(mpt_entry->mpt_byte_60,
1805 MPT_BYTE_60_PA6_L_M,
1806 MPT_BYTE_60_PA6_L_S,
1807 cpu_to_le32((u32)(pages[i])));
1808 roce_set_field(mpt_entry->mpt_byte_64,
1809 MPT_BYTE_64_PA6_H_M,
1810 MPT_BYTE_64_PA6_H_S,
1811 cpu_to_le32((u32)(pages[i] >> PAGES_SHIFT_16)));
1812 break;
1813 default:
1814 break;
1815 }
1816 }
1817
1818 free_page((unsigned long) pages);
1819
1820 mpt_entry->pbl_addr_l = (u32)(mr->pbl_dma_addr);
1821
1822 roce_set_field(mpt_entry->mpt_byte_12, MPT_BYTE_12_PBL_ADDR_H_M,
1823 MPT_BYTE_12_PBL_ADDR_H_S,
1824 ((u32)(mr->pbl_dma_addr >> 32)));
1825
1826 return 0;
1827 }
1828
1829 static void *get_cqe(struct hns_roce_cq *hr_cq, int n)
1830 {
1831 return hns_roce_buf_offset(&hr_cq->hr_buf.hr_buf,
1832 n * HNS_ROCE_V1_CQE_ENTRY_SIZE);
1833 }
1834
1835 static void *get_sw_cqe(struct hns_roce_cq *hr_cq, int n)
1836 {
1837 struct hns_roce_cqe *hr_cqe = get_cqe(hr_cq, n & hr_cq->ib_cq.cqe);
1838
1839 /* Get cqe when Owner bit is Conversely with the MSB of cons_idx */
1840 return (roce_get_bit(hr_cqe->cqe_byte_4, CQE_BYTE_4_OWNER_S) ^
1841 !!(n & (hr_cq->ib_cq.cqe + 1))) ? hr_cqe : NULL;
1842 }
1843
1844 static struct hns_roce_cqe *next_cqe_sw(struct hns_roce_cq *hr_cq)
1845 {
1846 return get_sw_cqe(hr_cq, hr_cq->cons_index);
1847 }
1848
1849 void hns_roce_v1_cq_set_ci(struct hns_roce_cq *hr_cq, u32 cons_index)
1850 {
1851 u32 doorbell[2];
1852
1853 doorbell[0] = cons_index & ((hr_cq->cq_depth << 1) - 1);
1854 roce_set_bit(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_HW_SYNS_S, 1);
1855 roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_M,
1856 ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_S, 3);
1857 roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_M,
1858 ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_S, 0);
1859 roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_M,
1860 ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_S, hr_cq->cqn);
1861
1862 hns_roce_write64_k(doorbell, hr_cq->cq_db_l);
1863 }
1864
1865 static void __hns_roce_v1_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
1866 struct hns_roce_srq *srq)
1867 {
1868 struct hns_roce_cqe *cqe, *dest;
1869 u32 prod_index;
1870 int nfreed = 0;
1871 u8 owner_bit;
1872
1873 for (prod_index = hr_cq->cons_index; get_sw_cqe(hr_cq, prod_index);
1874 ++prod_index) {
1875 if (prod_index == hr_cq->cons_index + hr_cq->ib_cq.cqe)
1876 break;
1877 }
1878
1879 /*
1880 * Now backwards through the CQ, removing CQ entries
1881 * that match our QP by overwriting them with next entries.
1882 */
1883 while ((int) --prod_index - (int) hr_cq->cons_index >= 0) {
1884 cqe = get_cqe(hr_cq, prod_index & hr_cq->ib_cq.cqe);
1885 if ((roce_get_field(cqe->cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M,
1886 CQE_BYTE_16_LOCAL_QPN_S) &
1887 HNS_ROCE_CQE_QPN_MASK) == qpn) {
1888 /* In v1 engine, not support SRQ */
1889 ++nfreed;
1890 } else if (nfreed) {
1891 dest = get_cqe(hr_cq, (prod_index + nfreed) &
1892 hr_cq->ib_cq.cqe);
1893 owner_bit = roce_get_bit(dest->cqe_byte_4,
1894 CQE_BYTE_4_OWNER_S);
1895 memcpy(dest, cqe, sizeof(*cqe));
1896 roce_set_bit(dest->cqe_byte_4, CQE_BYTE_4_OWNER_S,
1897 owner_bit);
1898 }
1899 }
1900
1901 if (nfreed) {
1902 hr_cq->cons_index += nfreed;
1903 /*
1904 * Make sure update of buffer contents is done before
1905 * updating consumer index.
1906 */
1907 wmb();
1908
1909 hns_roce_v1_cq_set_ci(hr_cq, hr_cq->cons_index);
1910 }
1911 }
1912
1913 static void hns_roce_v1_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
1914 struct hns_roce_srq *srq)
1915 {
1916 spin_lock_irq(&hr_cq->lock);
1917 __hns_roce_v1_cq_clean(hr_cq, qpn, srq);
1918 spin_unlock_irq(&hr_cq->lock);
1919 }
1920
1921 void hns_roce_v1_write_cqc(struct hns_roce_dev *hr_dev,
1922 struct hns_roce_cq *hr_cq, void *mb_buf, u64 *mtts,
1923 dma_addr_t dma_handle, int nent, u32 vector)
1924 {
1925 struct hns_roce_cq_context *cq_context = NULL;
1926 struct hns_roce_buf_list *tptr_buf;
1927 struct hns_roce_v1_priv *priv;
1928 dma_addr_t tptr_dma_addr;
1929 int offset;
1930
1931 priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
1932 tptr_buf = &priv->tptr_table.tptr_buf;
1933
1934 cq_context = mb_buf;
1935 memset(cq_context, 0, sizeof(*cq_context));
1936
1937 /* Get the tptr for this CQ. */
1938 offset = hr_cq->cqn * HNS_ROCE_V1_TPTR_ENTRY_SIZE;
1939 tptr_dma_addr = tptr_buf->map + offset;
1940 hr_cq->tptr_addr = (u16 *)(tptr_buf->buf + offset);
1941
1942 /* Register cq_context members */
1943 roce_set_field(cq_context->cqc_byte_4,
1944 CQ_CONTEXT_CQC_BYTE_4_CQC_STATE_M,
1945 CQ_CONTEXT_CQC_BYTE_4_CQC_STATE_S, CQ_STATE_VALID);
1946 roce_set_field(cq_context->cqc_byte_4, CQ_CONTEXT_CQC_BYTE_4_CQN_M,
1947 CQ_CONTEXT_CQC_BYTE_4_CQN_S, hr_cq->cqn);
1948 cq_context->cqc_byte_4 = cpu_to_le32(cq_context->cqc_byte_4);
1949
1950 cq_context->cq_bt_l = (u32)dma_handle;
1951 cq_context->cq_bt_l = cpu_to_le32(cq_context->cq_bt_l);
1952
1953 roce_set_field(cq_context->cqc_byte_12,
1954 CQ_CONTEXT_CQC_BYTE_12_CQ_BT_H_M,
1955 CQ_CONTEXT_CQC_BYTE_12_CQ_BT_H_S,
1956 ((u64)dma_handle >> 32));
1957 roce_set_field(cq_context->cqc_byte_12,
1958 CQ_CONTEXT_CQC_BYTE_12_CQ_CQE_SHIFT_M,
1959 CQ_CONTEXT_CQC_BYTE_12_CQ_CQE_SHIFT_S,
1960 ilog2((unsigned int)nent));
1961 roce_set_field(cq_context->cqc_byte_12, CQ_CONTEXT_CQC_BYTE_12_CEQN_M,
1962 CQ_CONTEXT_CQC_BYTE_12_CEQN_S, vector);
1963 cq_context->cqc_byte_12 = cpu_to_le32(cq_context->cqc_byte_12);
1964
1965 cq_context->cur_cqe_ba0_l = (u32)(mtts[0]);
1966 cq_context->cur_cqe_ba0_l = cpu_to_le32(cq_context->cur_cqe_ba0_l);
1967
1968 roce_set_field(cq_context->cqc_byte_20,
1969 CQ_CONTEXT_CQC_BYTE_20_CUR_CQE_BA0_H_M,
1970 CQ_CONTEXT_CQC_BYTE_20_CUR_CQE_BA0_H_S,
1971 cpu_to_le32((mtts[0]) >> 32));
1972 /* Dedicated hardware, directly set 0 */
1973 roce_set_field(cq_context->cqc_byte_20,
1974 CQ_CONTEXT_CQC_BYTE_20_CQ_CUR_INDEX_M,
1975 CQ_CONTEXT_CQC_BYTE_20_CQ_CUR_INDEX_S, 0);
1976 /**
1977 * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of
1978 * using 4K page, and shift more 32 because of
1979 * caculating the high 32 bit value evaluated to hardware.
1980 */
1981 roce_set_field(cq_context->cqc_byte_20,
1982 CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_M,
1983 CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_S,
1984 tptr_dma_addr >> 44);
1985 cq_context->cqc_byte_20 = cpu_to_le32(cq_context->cqc_byte_20);
1986
1987 cq_context->cqe_tptr_addr_l = (u32)(tptr_dma_addr >> 12);
1988
1989 roce_set_field(cq_context->cqc_byte_32,
1990 CQ_CONTEXT_CQC_BYTE_32_CUR_CQE_BA1_H_M,
1991 CQ_CONTEXT_CQC_BYTE_32_CUR_CQE_BA1_H_S, 0);
1992 roce_set_bit(cq_context->cqc_byte_32,
1993 CQ_CONTEXT_CQC_BYTE_32_SE_FLAG_S, 0);
1994 roce_set_bit(cq_context->cqc_byte_32,
1995 CQ_CONTEXT_CQC_BYTE_32_CE_FLAG_S, 0);
1996 roce_set_bit(cq_context->cqc_byte_32,
1997 CQ_CONTEXT_CQC_BYTE_32_NOTIFICATION_FLAG_S, 0);
1998 roce_set_bit(cq_context->cqc_byte_32,
1999 CQ_CQNTEXT_CQC_BYTE_32_TYPE_OF_COMPLETION_NOTIFICATION_S,
2000 0);
2001 /* The initial value of cq's ci is 0 */
2002 roce_set_field(cq_context->cqc_byte_32,
2003 CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_M,
2004 CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_S, 0);
2005 cq_context->cqc_byte_32 = cpu_to_le32(cq_context->cqc_byte_32);
2006 }
2007
2008 int hns_roce_v1_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
2009 {
2010 struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
2011 u32 notification_flag;
2012 u32 doorbell[2];
2013 int ret = 0;
2014
2015 notification_flag = (flags & IB_CQ_SOLICITED_MASK) ==
2016 IB_CQ_SOLICITED ? CQ_DB_REQ_NOT : CQ_DB_REQ_NOT_SOL;
2017 /*
2018 * flags = 0; Notification Flag = 1, next
2019 * flags = 1; Notification Flag = 0, solocited
2020 */
2021 doorbell[0] = hr_cq->cons_index & ((hr_cq->cq_depth << 1) - 1);
2022 roce_set_bit(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_HW_SYNS_S, 1);
2023 roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_M,
2024 ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_S, 3);
2025 roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_M,
2026 ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_S, 1);
2027 roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_M,
2028 ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_S,
2029 hr_cq->cqn | notification_flag);
2030
2031 hns_roce_write64_k(doorbell, hr_cq->cq_db_l);
2032
2033 return ret;
2034 }
2035
2036 static int hns_roce_v1_poll_one(struct hns_roce_cq *hr_cq,
2037 struct hns_roce_qp **cur_qp, struct ib_wc *wc)
2038 {
2039 int qpn;
2040 int is_send;
2041 u16 wqe_ctr;
2042 u32 status;
2043 u32 opcode;
2044 struct hns_roce_cqe *cqe;
2045 struct hns_roce_qp *hr_qp;
2046 struct hns_roce_wq *wq;
2047 struct hns_roce_wqe_ctrl_seg *sq_wqe;
2048 struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device);
2049 struct device *dev = &hr_dev->pdev->dev;
2050
2051 /* Find cqe according consumer index */
2052 cqe = next_cqe_sw(hr_cq);
2053 if (!cqe)
2054 return -EAGAIN;
2055
2056 ++hr_cq->cons_index;
2057 /* Memory barrier */
2058 rmb();
2059 /* 0->SQ, 1->RQ */
2060 is_send = !(roce_get_bit(cqe->cqe_byte_4, CQE_BYTE_4_SQ_RQ_FLAG_S));
2061
2062 /* Local_qpn in UD cqe is always 1, so it needs to compute new qpn */
2063 if (roce_get_field(cqe->cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M,
2064 CQE_BYTE_16_LOCAL_QPN_S) <= 1) {
2065 qpn = roce_get_field(cqe->cqe_byte_20, CQE_BYTE_20_PORT_NUM_M,
2066 CQE_BYTE_20_PORT_NUM_S) +
2067 roce_get_field(cqe->cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M,
2068 CQE_BYTE_16_LOCAL_QPN_S) *
2069 HNS_ROCE_MAX_PORTS;
2070 } else {
2071 qpn = roce_get_field(cqe->cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M,
2072 CQE_BYTE_16_LOCAL_QPN_S);
2073 }
2074
2075 if (!*cur_qp || (qpn & HNS_ROCE_CQE_QPN_MASK) != (*cur_qp)->qpn) {
2076 hr_qp = __hns_roce_qp_lookup(hr_dev, qpn);
2077 if (unlikely(!hr_qp)) {
2078 dev_err(dev, "CQ %06lx with entry for unknown QPN %06x\n",
2079 hr_cq->cqn, (qpn & HNS_ROCE_CQE_QPN_MASK));
2080 return -EINVAL;
2081 }
2082
2083 *cur_qp = hr_qp;
2084 }
2085
2086 wc->qp = &(*cur_qp)->ibqp;
2087 wc->vendor_err = 0;
2088
2089 status = roce_get_field(cqe->cqe_byte_4,
2090 CQE_BYTE_4_STATUS_OF_THE_OPERATION_M,
2091 CQE_BYTE_4_STATUS_OF_THE_OPERATION_S) &
2092 HNS_ROCE_CQE_STATUS_MASK;
2093 switch (status) {
2094 case HNS_ROCE_CQE_SUCCESS:
2095 wc->status = IB_WC_SUCCESS;
2096 break;
2097 case HNS_ROCE_CQE_SYNDROME_LOCAL_LENGTH_ERR:
2098 wc->status = IB_WC_LOC_LEN_ERR;
2099 break;
2100 case HNS_ROCE_CQE_SYNDROME_LOCAL_QP_OP_ERR:
2101 wc->status = IB_WC_LOC_QP_OP_ERR;
2102 break;
2103 case HNS_ROCE_CQE_SYNDROME_LOCAL_PROT_ERR:
2104 wc->status = IB_WC_LOC_PROT_ERR;
2105 break;
2106 case HNS_ROCE_CQE_SYNDROME_WR_FLUSH_ERR:
2107 wc->status = IB_WC_WR_FLUSH_ERR;
2108 break;
2109 case HNS_ROCE_CQE_SYNDROME_MEM_MANAGE_OPERATE_ERR:
2110 wc->status = IB_WC_MW_BIND_ERR;
2111 break;
2112 case HNS_ROCE_CQE_SYNDROME_BAD_RESP_ERR:
2113 wc->status = IB_WC_BAD_RESP_ERR;
2114 break;
2115 case HNS_ROCE_CQE_SYNDROME_LOCAL_ACCESS_ERR:
2116 wc->status = IB_WC_LOC_ACCESS_ERR;
2117 break;
2118 case HNS_ROCE_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR:
2119 wc->status = IB_WC_REM_INV_REQ_ERR;
2120 break;
2121 case HNS_ROCE_CQE_SYNDROME_REMOTE_ACCESS_ERR:
2122 wc->status = IB_WC_REM_ACCESS_ERR;
2123 break;
2124 case HNS_ROCE_CQE_SYNDROME_REMOTE_OP_ERR:
2125 wc->status = IB_WC_REM_OP_ERR;
2126 break;
2127 case HNS_ROCE_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR:
2128 wc->status = IB_WC_RETRY_EXC_ERR;
2129 break;
2130 case HNS_ROCE_CQE_SYNDROME_RNR_RETRY_EXC_ERR:
2131 wc->status = IB_WC_RNR_RETRY_EXC_ERR;
2132 break;
2133 default:
2134 wc->status = IB_WC_GENERAL_ERR;
2135 break;
2136 }
2137
2138 /* CQE status error, directly return */
2139 if (wc->status != IB_WC_SUCCESS)
2140 return 0;
2141
2142 if (is_send) {
2143 /* SQ conrespond to CQE */
2144 sq_wqe = get_send_wqe(*cur_qp, roce_get_field(cqe->cqe_byte_4,
2145 CQE_BYTE_4_WQE_INDEX_M,
2146 CQE_BYTE_4_WQE_INDEX_S)&
2147 ((*cur_qp)->sq.wqe_cnt-1));
2148 switch (sq_wqe->flag & HNS_ROCE_WQE_OPCODE_MASK) {
2149 case HNS_ROCE_WQE_OPCODE_SEND:
2150 wc->opcode = IB_WC_SEND;
2151 break;
2152 case HNS_ROCE_WQE_OPCODE_RDMA_READ:
2153 wc->opcode = IB_WC_RDMA_READ;
2154 wc->byte_len = le32_to_cpu(cqe->byte_cnt);
2155 break;
2156 case HNS_ROCE_WQE_OPCODE_RDMA_WRITE:
2157 wc->opcode = IB_WC_RDMA_WRITE;
2158 break;
2159 case HNS_ROCE_WQE_OPCODE_LOCAL_INV:
2160 wc->opcode = IB_WC_LOCAL_INV;
2161 break;
2162 case HNS_ROCE_WQE_OPCODE_UD_SEND:
2163 wc->opcode = IB_WC_SEND;
2164 break;
2165 default:
2166 wc->status = IB_WC_GENERAL_ERR;
2167 break;
2168 }
2169 wc->wc_flags = (sq_wqe->flag & HNS_ROCE_WQE_IMM ?
2170 IB_WC_WITH_IMM : 0);
2171
2172 wq = &(*cur_qp)->sq;
2173 if ((*cur_qp)->sq_signal_bits) {
2174 /*
2175 * If sg_signal_bit is 1,
2176 * firstly tail pointer updated to wqe
2177 * which current cqe correspond to
2178 */
2179 wqe_ctr = (u16)roce_get_field(cqe->cqe_byte_4,
2180 CQE_BYTE_4_WQE_INDEX_M,
2181 CQE_BYTE_4_WQE_INDEX_S);
2182 wq->tail += (wqe_ctr - (u16)wq->tail) &
2183 (wq->wqe_cnt - 1);
2184 }
2185 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
2186 ++wq->tail;
2187 } else {
2188 /* RQ conrespond to CQE */
2189 wc->byte_len = le32_to_cpu(cqe->byte_cnt);
2190 opcode = roce_get_field(cqe->cqe_byte_4,
2191 CQE_BYTE_4_OPERATION_TYPE_M,
2192 CQE_BYTE_4_OPERATION_TYPE_S) &
2193 HNS_ROCE_CQE_OPCODE_MASK;
2194 switch (opcode) {
2195 case HNS_ROCE_OPCODE_RDMA_WITH_IMM_RECEIVE:
2196 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
2197 wc->wc_flags = IB_WC_WITH_IMM;
2198 wc->ex.imm_data = le32_to_cpu(cqe->immediate_data);
2199 break;
2200 case HNS_ROCE_OPCODE_SEND_DATA_RECEIVE:
2201 if (roce_get_bit(cqe->cqe_byte_4,
2202 CQE_BYTE_4_IMM_INDICATOR_S)) {
2203 wc->opcode = IB_WC_RECV;
2204 wc->wc_flags = IB_WC_WITH_IMM;
2205 wc->ex.imm_data = le32_to_cpu(
2206 cqe->immediate_data);
2207 } else {
2208 wc->opcode = IB_WC_RECV;
2209 wc->wc_flags = 0;
2210 }
2211 break;
2212 default:
2213 wc->status = IB_WC_GENERAL_ERR;
2214 break;
2215 }
2216
2217 /* Update tail pointer, record wr_id */
2218 wq = &(*cur_qp)->rq;
2219 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
2220 ++wq->tail;
2221 wc->sl = (u8)roce_get_field(cqe->cqe_byte_20, CQE_BYTE_20_SL_M,
2222 CQE_BYTE_20_SL_S);
2223 wc->src_qp = (u8)roce_get_field(cqe->cqe_byte_20,
2224 CQE_BYTE_20_REMOTE_QPN_M,
2225 CQE_BYTE_20_REMOTE_QPN_S);
2226 wc->wc_flags |= (roce_get_bit(cqe->cqe_byte_20,
2227 CQE_BYTE_20_GRH_PRESENT_S) ?
2228 IB_WC_GRH : 0);
2229 wc->pkey_index = (u16)roce_get_field(cqe->cqe_byte_28,
2230 CQE_BYTE_28_P_KEY_IDX_M,
2231 CQE_BYTE_28_P_KEY_IDX_S);
2232 }
2233
2234 return 0;
2235 }
2236
2237 int hns_roce_v1_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
2238 {
2239 struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
2240 struct hns_roce_qp *cur_qp = NULL;
2241 unsigned long flags;
2242 int npolled;
2243 int ret = 0;
2244
2245 spin_lock_irqsave(&hr_cq->lock, flags);
2246
2247 for (npolled = 0; npolled < num_entries; ++npolled) {
2248 ret = hns_roce_v1_poll_one(hr_cq, &cur_qp, wc + npolled);
2249 if (ret)
2250 break;
2251 }
2252
2253 if (npolled) {
2254 *hr_cq->tptr_addr = hr_cq->cons_index &
2255 ((hr_cq->cq_depth << 1) - 1);
2256
2257 /* Memroy barrier */
2258 wmb();
2259 hns_roce_v1_cq_set_ci(hr_cq, hr_cq->cons_index);
2260 }
2261
2262 spin_unlock_irqrestore(&hr_cq->lock, flags);
2263
2264 if (ret == 0 || ret == -EAGAIN)
2265 return npolled;
2266 else
2267 return ret;
2268 }
2269
2270 int hns_roce_v1_clear_hem(struct hns_roce_dev *hr_dev,
2271 struct hns_roce_hem_table *table, int obj)
2272 {
2273 struct device *dev = &hr_dev->pdev->dev;
2274 struct hns_roce_v1_priv *priv;
2275 unsigned long end = 0, flags = 0;
2276 uint32_t bt_cmd_val[2] = {0};
2277 void __iomem *bt_cmd;
2278 u64 bt_ba = 0;
2279
2280 priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
2281
2282 switch (table->type) {
2283 case HEM_TYPE_QPC:
2284 roce_set_field(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
2285 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_QPC);
2286 bt_ba = priv->bt_table.qpc_buf.map >> 12;
2287 break;
2288 case HEM_TYPE_MTPT:
2289 roce_set_field(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
2290 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_MTPT);
2291 bt_ba = priv->bt_table.mtpt_buf.map >> 12;
2292 break;
2293 case HEM_TYPE_CQC:
2294 roce_set_field(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
2295 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_CQC);
2296 bt_ba = priv->bt_table.cqc_buf.map >> 12;
2297 break;
2298 case HEM_TYPE_SRQC:
2299 dev_dbg(dev, "HEM_TYPE_SRQC not support.\n");
2300 return -EINVAL;
2301 default:
2302 return 0;
2303 }
2304 roce_set_field(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M,
2305 ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S, obj);
2306 roce_set_bit(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_S, 0);
2307 roce_set_bit(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S, 1);
2308
2309 spin_lock_irqsave(&hr_dev->bt_cmd_lock, flags);
2310
2311 bt_cmd = hr_dev->reg_base + ROCEE_BT_CMD_H_REG;
2312
2313 end = msecs_to_jiffies(HW_SYNC_TIMEOUT_MSECS) + jiffies;
2314 while (1) {
2315 if (readl(bt_cmd) >> BT_CMD_SYNC_SHIFT) {
2316 if (!(time_before(jiffies, end))) {
2317 dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n");
2318 spin_unlock_irqrestore(&hr_dev->bt_cmd_lock,
2319 flags);
2320 return -EBUSY;
2321 }
2322 } else {
2323 break;
2324 }
2325 msleep(HW_SYNC_SLEEP_TIME_INTERVAL);
2326 }
2327
2328 bt_cmd_val[0] = (uint32_t)bt_ba;
2329 roce_set_field(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M,
2330 ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S, bt_ba >> 32);
2331 hns_roce_write64_k(bt_cmd_val, hr_dev->reg_base + ROCEE_BT_CMD_L_REG);
2332
2333 spin_unlock_irqrestore(&hr_dev->bt_cmd_lock, flags);
2334
2335 return 0;
2336 }
2337
2338 static int hns_roce_v1_qp_modify(struct hns_roce_dev *hr_dev,
2339 struct hns_roce_mtt *mtt,
2340 enum hns_roce_qp_state cur_state,
2341 enum hns_roce_qp_state new_state,
2342 struct hns_roce_qp_context *context,
2343 struct hns_roce_qp *hr_qp)
2344 {
2345 static const u16
2346 op[HNS_ROCE_QP_NUM_STATE][HNS_ROCE_QP_NUM_STATE] = {
2347 [HNS_ROCE_QP_STATE_RST] = {
2348 [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,
2349 [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,
2350 [HNS_ROCE_QP_STATE_INIT] = HNS_ROCE_CMD_RST2INIT_QP,
2351 },
2352 [HNS_ROCE_QP_STATE_INIT] = {
2353 [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,
2354 [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,
2355 /* Note: In v1 engine, HW doesn't support RST2INIT.
2356 * We use RST2INIT cmd instead of INIT2INIT.
2357 */
2358 [HNS_ROCE_QP_STATE_INIT] = HNS_ROCE_CMD_RST2INIT_QP,
2359 [HNS_ROCE_QP_STATE_RTR] = HNS_ROCE_CMD_INIT2RTR_QP,
2360 },
2361 [HNS_ROCE_QP_STATE_RTR] = {
2362 [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,
2363 [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,
2364 [HNS_ROCE_QP_STATE_RTS] = HNS_ROCE_CMD_RTR2RTS_QP,
2365 },
2366 [HNS_ROCE_QP_STATE_RTS] = {
2367 [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,
2368 [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,
2369 [HNS_ROCE_QP_STATE_RTS] = HNS_ROCE_CMD_RTS2RTS_QP,
2370 [HNS_ROCE_QP_STATE_SQD] = HNS_ROCE_CMD_RTS2SQD_QP,
2371 },
2372 [HNS_ROCE_QP_STATE_SQD] = {
2373 [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,
2374 [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,
2375 [HNS_ROCE_QP_STATE_RTS] = HNS_ROCE_CMD_SQD2RTS_QP,
2376 [HNS_ROCE_QP_STATE_SQD] = HNS_ROCE_CMD_SQD2SQD_QP,
2377 },
2378 [HNS_ROCE_QP_STATE_ERR] = {
2379 [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,
2380 [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,
2381 }
2382 };
2383
2384 struct hns_roce_cmd_mailbox *mailbox;
2385 struct device *dev = &hr_dev->pdev->dev;
2386 int ret = 0;
2387
2388 if (cur_state >= HNS_ROCE_QP_NUM_STATE ||
2389 new_state >= HNS_ROCE_QP_NUM_STATE ||
2390 !op[cur_state][new_state]) {
2391 dev_err(dev, "[modify_qp]not support state %d to %d\n",
2392 cur_state, new_state);
2393 return -EINVAL;
2394 }
2395
2396 if (op[cur_state][new_state] == HNS_ROCE_CMD_2RST_QP)
2397 return hns_roce_cmd_mbox(hr_dev, 0, 0, hr_qp->qpn, 2,
2398 HNS_ROCE_CMD_2RST_QP,
2399 HNS_ROCE_CMD_TIMEOUT_MSECS);
2400
2401 if (op[cur_state][new_state] == HNS_ROCE_CMD_2ERR_QP)
2402 return hns_roce_cmd_mbox(hr_dev, 0, 0, hr_qp->qpn, 2,
2403 HNS_ROCE_CMD_2ERR_QP,
2404 HNS_ROCE_CMD_TIMEOUT_MSECS);
2405
2406 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
2407 if (IS_ERR(mailbox))
2408 return PTR_ERR(mailbox);
2409
2410 memcpy(mailbox->buf, context, sizeof(*context));
2411
2412 ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_qp->qpn, 0,
2413 op[cur_state][new_state],
2414 HNS_ROCE_CMD_TIMEOUT_MSECS);
2415
2416 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
2417 return ret;
2418 }
2419
2420 static int hns_roce_v1_m_sqp(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
2421 int attr_mask, enum ib_qp_state cur_state,
2422 enum ib_qp_state new_state)
2423 {
2424 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
2425 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
2426 struct hns_roce_sqp_context *context;
2427 struct device *dev = &hr_dev->pdev->dev;
2428 dma_addr_t dma_handle = 0;
2429 int rq_pa_start;
2430 u32 reg_val;
2431 u64 *mtts;
2432 u32 *addr;
2433
2434 context = kzalloc(sizeof(*context), GFP_KERNEL);
2435 if (!context)
2436 return -ENOMEM;
2437
2438 /* Search QP buf's MTTs */
2439 mtts = hns_roce_table_find(&hr_dev->mr_table.mtt_table,
2440 hr_qp->mtt.first_seg, &dma_handle);
2441 if (!mtts) {
2442 dev_err(dev, "qp buf pa find failed\n");
2443 goto out;
2444 }
2445
2446 if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
2447 roce_set_field(context->qp1c_bytes_4,
2448 QP1C_BYTES_4_SQ_WQE_SHIFT_M,
2449 QP1C_BYTES_4_SQ_WQE_SHIFT_S,
2450 ilog2((unsigned int)hr_qp->sq.wqe_cnt));
2451 roce_set_field(context->qp1c_bytes_4,
2452 QP1C_BYTES_4_RQ_WQE_SHIFT_M,
2453 QP1C_BYTES_4_RQ_WQE_SHIFT_S,
2454 ilog2((unsigned int)hr_qp->rq.wqe_cnt));
2455 roce_set_field(context->qp1c_bytes_4, QP1C_BYTES_4_PD_M,
2456 QP1C_BYTES_4_PD_S, to_hr_pd(ibqp->pd)->pdn);
2457
2458 context->sq_rq_bt_l = (u32)(dma_handle);
2459 roce_set_field(context->qp1c_bytes_12,
2460 QP1C_BYTES_12_SQ_RQ_BT_H_M,
2461 QP1C_BYTES_12_SQ_RQ_BT_H_S,
2462 ((u32)(dma_handle >> 32)));
2463
2464 roce_set_field(context->qp1c_bytes_16, QP1C_BYTES_16_RQ_HEAD_M,
2465 QP1C_BYTES_16_RQ_HEAD_S, hr_qp->rq.head);
2466 roce_set_field(context->qp1c_bytes_16, QP1C_BYTES_16_PORT_NUM_M,
2467 QP1C_BYTES_16_PORT_NUM_S, hr_qp->phy_port);
2468 roce_set_bit(context->qp1c_bytes_16,
2469 QP1C_BYTES_16_SIGNALING_TYPE_S,
2470 hr_qp->sq_signal_bits);
2471 roce_set_bit(context->qp1c_bytes_16, QP1C_BYTES_16_RQ_BA_FLG_S,
2472 1);
2473 roce_set_bit(context->qp1c_bytes_16, QP1C_BYTES_16_SQ_BA_FLG_S,
2474 1);
2475 roce_set_bit(context->qp1c_bytes_16, QP1C_BYTES_16_QP1_ERR_S,
2476 0);
2477
2478 roce_set_field(context->qp1c_bytes_20, QP1C_BYTES_20_SQ_HEAD_M,
2479 QP1C_BYTES_20_SQ_HEAD_S, hr_qp->sq.head);
2480 roce_set_field(context->qp1c_bytes_20, QP1C_BYTES_20_PKEY_IDX_M,
2481 QP1C_BYTES_20_PKEY_IDX_S, attr->pkey_index);
2482
2483 rq_pa_start = (u32)hr_qp->rq.offset / PAGE_SIZE;
2484 context->cur_rq_wqe_ba_l = (u32)(mtts[rq_pa_start]);
2485
2486 roce_set_field(context->qp1c_bytes_28,
2487 QP1C_BYTES_28_CUR_RQ_WQE_BA_H_M,
2488 QP1C_BYTES_28_CUR_RQ_WQE_BA_H_S,
2489 (mtts[rq_pa_start]) >> 32);
2490 roce_set_field(context->qp1c_bytes_28,
2491 QP1C_BYTES_28_RQ_CUR_IDX_M,
2492 QP1C_BYTES_28_RQ_CUR_IDX_S, 0);
2493
2494 roce_set_field(context->qp1c_bytes_32,
2495 QP1C_BYTES_32_RX_CQ_NUM_M,
2496 QP1C_BYTES_32_RX_CQ_NUM_S,
2497 to_hr_cq(ibqp->recv_cq)->cqn);
2498 roce_set_field(context->qp1c_bytes_32,
2499 QP1C_BYTES_32_TX_CQ_NUM_M,
2500 QP1C_BYTES_32_TX_CQ_NUM_S,
2501 to_hr_cq(ibqp->send_cq)->cqn);
2502
2503 context->cur_sq_wqe_ba_l = (u32)mtts[0];
2504
2505 roce_set_field(context->qp1c_bytes_40,
2506 QP1C_BYTES_40_CUR_SQ_WQE_BA_H_M,
2507 QP1C_BYTES_40_CUR_SQ_WQE_BA_H_S,
2508 (mtts[0]) >> 32);
2509 roce_set_field(context->qp1c_bytes_40,
2510 QP1C_BYTES_40_SQ_CUR_IDX_M,
2511 QP1C_BYTES_40_SQ_CUR_IDX_S, 0);
2512
2513 /* Copy context to QP1C register */
2514 addr = (u32 *)(hr_dev->reg_base + ROCEE_QP1C_CFG0_0_REG +
2515 hr_qp->phy_port * sizeof(*context));
2516
2517 writel(context->qp1c_bytes_4, addr);
2518 writel(context->sq_rq_bt_l, addr + 1);
2519 writel(context->qp1c_bytes_12, addr + 2);
2520 writel(context->qp1c_bytes_16, addr + 3);
2521 writel(context->qp1c_bytes_20, addr + 4);
2522 writel(context->cur_rq_wqe_ba_l, addr + 5);
2523 writel(context->qp1c_bytes_28, addr + 6);
2524 writel(context->qp1c_bytes_32, addr + 7);
2525 writel(context->cur_sq_wqe_ba_l, addr + 8);
2526 writel(context->qp1c_bytes_40, addr + 9);
2527 }
2528
2529 /* Modify QP1C status */
2530 reg_val = roce_read(hr_dev, ROCEE_QP1C_CFG0_0_REG +
2531 hr_qp->phy_port * sizeof(*context));
2532 roce_set_field(reg_val, ROCEE_QP1C_CFG0_0_ROCEE_QP1C_QP_ST_M,
2533 ROCEE_QP1C_CFG0_0_ROCEE_QP1C_QP_ST_S, new_state);
2534 roce_write(hr_dev, ROCEE_QP1C_CFG0_0_REG +
2535 hr_qp->phy_port * sizeof(*context), reg_val);
2536
2537 hr_qp->state = new_state;
2538 if (new_state == IB_QPS_RESET) {
2539 hns_roce_v1_cq_clean(to_hr_cq(ibqp->recv_cq), hr_qp->qpn,
2540 ibqp->srq ? to_hr_srq(ibqp->srq) : NULL);
2541 if (ibqp->send_cq != ibqp->recv_cq)
2542 hns_roce_v1_cq_clean(to_hr_cq(ibqp->send_cq),
2543 hr_qp->qpn, NULL);
2544
2545 hr_qp->rq.head = 0;
2546 hr_qp->rq.tail = 0;
2547 hr_qp->sq.head = 0;
2548 hr_qp->sq.tail = 0;
2549 hr_qp->sq_next_wqe = 0;
2550 }
2551
2552 kfree(context);
2553 return 0;
2554
2555 out:
2556 kfree(context);
2557 return -EINVAL;
2558 }
2559
2560 static int hns_roce_v1_m_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
2561 int attr_mask, enum ib_qp_state cur_state,
2562 enum ib_qp_state new_state)
2563 {
2564 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
2565 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
2566 struct device *dev = &hr_dev->pdev->dev;
2567 struct hns_roce_qp_context *context;
2568 dma_addr_t dma_handle_2 = 0;
2569 dma_addr_t dma_handle = 0;
2570 uint32_t doorbell[2] = {0};
2571 int rq_pa_start = 0;
2572 u64 *mtts_2 = NULL;
2573 int ret = -EINVAL;
2574 u64 *mtts = NULL;
2575 int port;
2576 u8 *dmac;
2577 u8 *smac;
2578
2579 context = kzalloc(sizeof(*context), GFP_KERNEL);
2580 if (!context)
2581 return -ENOMEM;
2582
2583 /* Search qp buf's mtts */
2584 mtts = hns_roce_table_find(&hr_dev->mr_table.mtt_table,
2585 hr_qp->mtt.first_seg, &dma_handle);
2586 if (mtts == NULL) {
2587 dev_err(dev, "qp buf pa find failed\n");
2588 goto out;
2589 }
2590
2591 /* Search IRRL's mtts */
2592 mtts_2 = hns_roce_table_find(&hr_dev->qp_table.irrl_table, hr_qp->qpn,
2593 &dma_handle_2);
2594 if (mtts_2 == NULL) {
2595 dev_err(dev, "qp irrl_table find failed\n");
2596 goto out;
2597 }
2598
2599 /*
2600 * Reset to init
2601 * Mandatory param:
2602 * IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT | IB_QP_ACCESS_FLAGS
2603 * Optional param: NA
2604 */
2605 if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
2606 roce_set_field(context->qpc_bytes_4,
2607 QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_M,
2608 QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_S,
2609 to_hr_qp_type(hr_qp->ibqp.qp_type));
2610
2611 roce_set_bit(context->qpc_bytes_4,
2612 QP_CONTEXT_QPC_BYTE_4_ENABLE_FPMR_S, 0);
2613 roce_set_bit(context->qpc_bytes_4,
2614 QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S,
2615 !!(attr->qp_access_flags & IB_ACCESS_REMOTE_READ));
2616 roce_set_bit(context->qpc_bytes_4,
2617 QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S,
2618 !!(attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE)
2619 );
2620 roce_set_bit(context->qpc_bytes_4,
2621 QP_CONTEXT_QPC_BYTE_4_ATOMIC_OPERATION_ENABLE_S,
2622 !!(attr->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC)
2623 );
2624 roce_set_bit(context->qpc_bytes_4,
2625 QP_CONTEXT_QPC_BYTE_4_RDMAR_USE_S, 1);
2626 roce_set_field(context->qpc_bytes_4,
2627 QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_M,
2628 QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_S,
2629 ilog2((unsigned int)hr_qp->sq.wqe_cnt));
2630 roce_set_field(context->qpc_bytes_4,
2631 QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_M,
2632 QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_S,
2633 ilog2((unsigned int)hr_qp->rq.wqe_cnt));
2634 roce_set_field(context->qpc_bytes_4,
2635 QP_CONTEXT_QPC_BYTES_4_PD_M,
2636 QP_CONTEXT_QPC_BYTES_4_PD_S,
2637 to_hr_pd(ibqp->pd)->pdn);
2638 hr_qp->access_flags = attr->qp_access_flags;
2639 roce_set_field(context->qpc_bytes_8,
2640 QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_M,
2641 QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_S,
2642 to_hr_cq(ibqp->send_cq)->cqn);
2643 roce_set_field(context->qpc_bytes_8,
2644 QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_M,
2645 QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_S,
2646 to_hr_cq(ibqp->recv_cq)->cqn);
2647
2648 if (ibqp->srq)
2649 roce_set_field(context->qpc_bytes_12,
2650 QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_M,
2651 QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_S,
2652 to_hr_srq(ibqp->srq)->srqn);
2653
2654 roce_set_field(context->qpc_bytes_12,
2655 QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M,
2656 QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S,
2657 attr->pkey_index);
2658 hr_qp->pkey_index = attr->pkey_index;
2659 roce_set_field(context->qpc_bytes_16,
2660 QP_CONTEXT_QPC_BYTES_16_QP_NUM_M,
2661 QP_CONTEXT_QPC_BYTES_16_QP_NUM_S, hr_qp->qpn);
2662
2663 } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) {
2664 roce_set_field(context->qpc_bytes_4,
2665 QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_M,
2666 QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_S,
2667 to_hr_qp_type(hr_qp->ibqp.qp_type));
2668 roce_set_bit(context->qpc_bytes_4,
2669 QP_CONTEXT_QPC_BYTE_4_ENABLE_FPMR_S, 0);
2670 if (attr_mask & IB_QP_ACCESS_FLAGS) {
2671 roce_set_bit(context->qpc_bytes_4,
2672 QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S,
2673 !!(attr->qp_access_flags &
2674 IB_ACCESS_REMOTE_READ));
2675 roce_set_bit(context->qpc_bytes_4,
2676 QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S,
2677 !!(attr->qp_access_flags &
2678 IB_ACCESS_REMOTE_WRITE));
2679 } else {
2680 roce_set_bit(context->qpc_bytes_4,
2681 QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S,
2682 !!(hr_qp->access_flags &
2683 IB_ACCESS_REMOTE_READ));
2684 roce_set_bit(context->qpc_bytes_4,
2685 QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S,
2686 !!(hr_qp->access_flags &
2687 IB_ACCESS_REMOTE_WRITE));
2688 }
2689
2690 roce_set_bit(context->qpc_bytes_4,
2691 QP_CONTEXT_QPC_BYTE_4_RDMAR_USE_S, 1);
2692 roce_set_field(context->qpc_bytes_4,
2693 QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_M,
2694 QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_S,
2695 ilog2((unsigned int)hr_qp->sq.wqe_cnt));
2696 roce_set_field(context->qpc_bytes_4,
2697 QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_M,
2698 QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_S,
2699 ilog2((unsigned int)hr_qp->rq.wqe_cnt));
2700 roce_set_field(context->qpc_bytes_4,
2701 QP_CONTEXT_QPC_BYTES_4_PD_M,
2702 QP_CONTEXT_QPC_BYTES_4_PD_S,
2703 to_hr_pd(ibqp->pd)->pdn);
2704
2705 roce_set_field(context->qpc_bytes_8,
2706 QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_M,
2707 QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_S,
2708 to_hr_cq(ibqp->send_cq)->cqn);
2709 roce_set_field(context->qpc_bytes_8,
2710 QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_M,
2711 QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_S,
2712 to_hr_cq(ibqp->recv_cq)->cqn);
2713
2714 if (ibqp->srq)
2715 roce_set_field(context->qpc_bytes_12,
2716 QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_M,
2717 QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_S,
2718 to_hr_srq(ibqp->srq)->srqn);
2719 if (attr_mask & IB_QP_PKEY_INDEX)
2720 roce_set_field(context->qpc_bytes_12,
2721 QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M,
2722 QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S,
2723 attr->pkey_index);
2724 else
2725 roce_set_field(context->qpc_bytes_12,
2726 QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M,
2727 QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S,
2728 hr_qp->pkey_index);
2729
2730 roce_set_field(context->qpc_bytes_16,
2731 QP_CONTEXT_QPC_BYTES_16_QP_NUM_M,
2732 QP_CONTEXT_QPC_BYTES_16_QP_NUM_S, hr_qp->qpn);
2733 } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
2734 if ((attr_mask & IB_QP_ALT_PATH) ||
2735 (attr_mask & IB_QP_ACCESS_FLAGS) ||
2736 (attr_mask & IB_QP_PKEY_INDEX) ||
2737 (attr_mask & IB_QP_QKEY)) {
2738 dev_err(dev, "INIT2RTR attr_mask error\n");
2739 goto out;
2740 }
2741
2742 dmac = (u8 *)attr->ah_attr.dmac;
2743
2744 context->sq_rq_bt_l = (u32)(dma_handle);
2745 roce_set_field(context->qpc_bytes_24,
2746 QP_CONTEXT_QPC_BYTES_24_SQ_RQ_BT_H_M,
2747 QP_CONTEXT_QPC_BYTES_24_SQ_RQ_BT_H_S,
2748 ((u32)(dma_handle >> 32)));
2749 roce_set_bit(context->qpc_bytes_24,
2750 QP_CONTEXT_QPC_BYTE_24_REMOTE_ENABLE_E2E_CREDITS_S,
2751 1);
2752 roce_set_field(context->qpc_bytes_24,
2753 QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_M,
2754 QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_S,
2755 attr->min_rnr_timer);
2756 context->irrl_ba_l = (u32)(dma_handle_2);
2757 roce_set_field(context->qpc_bytes_32,
2758 QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_M,
2759 QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_S,
2760 ((u32)(dma_handle_2 >> 32)) &
2761 QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_M);
2762 roce_set_field(context->qpc_bytes_32,
2763 QP_CONTEXT_QPC_BYTES_32_MIG_STATE_M,
2764 QP_CONTEXT_QPC_BYTES_32_MIG_STATE_S, 0);
2765 roce_set_bit(context->qpc_bytes_32,
2766 QP_CONTEXT_QPC_BYTE_32_LOCAL_ENABLE_E2E_CREDITS_S,
2767 1);
2768 roce_set_bit(context->qpc_bytes_32,
2769 QP_CONTEXT_QPC_BYTE_32_SIGNALING_TYPE_S,
2770 hr_qp->sq_signal_bits);
2771
2772 port = (attr_mask & IB_QP_PORT) ? (attr->port_num - 1) :
2773 hr_qp->port;
2774 smac = (u8 *)hr_dev->dev_addr[port];
2775 /* when dmac equals smac or loop_idc is 1, it should loopback */
2776 if (ether_addr_equal_unaligned(dmac, smac) ||
2777 hr_dev->loop_idc == 0x1)
2778 roce_set_bit(context->qpc_bytes_32,
2779 QP_CONTEXT_QPC_BYTE_32_LOOPBACK_INDICATOR_S, 1);
2780
2781 roce_set_bit(context->qpc_bytes_32,
2782 QP_CONTEXT_QPC_BYTE_32_GLOBAL_HEADER_S,
2783 attr->ah_attr.ah_flags);
2784 roce_set_field(context->qpc_bytes_32,
2785 QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_M,
2786 QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_S,
2787 ilog2((unsigned int)attr->max_dest_rd_atomic));
2788
2789 roce_set_field(context->qpc_bytes_36,
2790 QP_CONTEXT_QPC_BYTES_36_DEST_QP_M,
2791 QP_CONTEXT_QPC_BYTES_36_DEST_QP_S,
2792 attr->dest_qp_num);
2793
2794 /* Configure GID index */
2795 roce_set_field(context->qpc_bytes_36,
2796 QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_M,
2797 QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_S,
2798 hns_get_gid_index(hr_dev,
2799 attr->ah_attr.port_num - 1,
2800 attr->ah_attr.grh.sgid_index));
2801
2802 memcpy(&(context->dmac_l), dmac, 4);
2803
2804 roce_set_field(context->qpc_bytes_44,
2805 QP_CONTEXT_QPC_BYTES_44_DMAC_H_M,
2806 QP_CONTEXT_QPC_BYTES_44_DMAC_H_S,
2807 *((u16 *)(&dmac[4])));
2808 roce_set_field(context->qpc_bytes_44,
2809 QP_CONTEXT_QPC_BYTES_44_MAXIMUM_STATIC_RATE_M,
2810 QP_CONTEXT_QPC_BYTES_44_MAXIMUM_STATIC_RATE_S,
2811 attr->ah_attr.static_rate);
2812 roce_set_field(context->qpc_bytes_44,
2813 QP_CONTEXT_QPC_BYTES_44_HOPLMT_M,
2814 QP_CONTEXT_QPC_BYTES_44_HOPLMT_S,
2815 attr->ah_attr.grh.hop_limit);
2816
2817 roce_set_field(context->qpc_bytes_48,
2818 QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_M,
2819 QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_S,
2820 attr->ah_attr.grh.flow_label);
2821 roce_set_field(context->qpc_bytes_48,
2822 QP_CONTEXT_QPC_BYTES_48_TCLASS_M,
2823 QP_CONTEXT_QPC_BYTES_48_TCLASS_S,
2824 attr->ah_attr.grh.traffic_class);
2825 roce_set_field(context->qpc_bytes_48,
2826 QP_CONTEXT_QPC_BYTES_48_MTU_M,
2827 QP_CONTEXT_QPC_BYTES_48_MTU_S, attr->path_mtu);
2828
2829 memcpy(context->dgid, attr->ah_attr.grh.dgid.raw,
2830 sizeof(attr->ah_attr.grh.dgid.raw));
2831
2832 dev_dbg(dev, "dmac:%x :%lx\n", context->dmac_l,
2833 roce_get_field(context->qpc_bytes_44,
2834 QP_CONTEXT_QPC_BYTES_44_DMAC_H_M,
2835 QP_CONTEXT_QPC_BYTES_44_DMAC_H_S));
2836
2837 roce_set_field(context->qpc_bytes_68,
2838 QP_CONTEXT_QPC_BYTES_68_RQ_HEAD_M,
2839 QP_CONTEXT_QPC_BYTES_68_RQ_HEAD_S,
2840 hr_qp->rq.head);
2841 roce_set_field(context->qpc_bytes_68,
2842 QP_CONTEXT_QPC_BYTES_68_RQ_CUR_INDEX_M,
2843 QP_CONTEXT_QPC_BYTES_68_RQ_CUR_INDEX_S, 0);
2844
2845 rq_pa_start = (u32)hr_qp->rq.offset / PAGE_SIZE;
2846 context->cur_rq_wqe_ba_l = (u32)(mtts[rq_pa_start]);
2847
2848 roce_set_field(context->qpc_bytes_76,
2849 QP_CONTEXT_QPC_BYTES_76_CUR_RQ_WQE_BA_H_M,
2850 QP_CONTEXT_QPC_BYTES_76_CUR_RQ_WQE_BA_H_S,
2851 mtts[rq_pa_start] >> 32);
2852 roce_set_field(context->qpc_bytes_76,
2853 QP_CONTEXT_QPC_BYTES_76_RX_REQ_MSN_M,
2854 QP_CONTEXT_QPC_BYTES_76_RX_REQ_MSN_S, 0);
2855
2856 context->rx_rnr_time = 0;
2857
2858 roce_set_field(context->qpc_bytes_84,
2859 QP_CONTEXT_QPC_BYTES_84_LAST_ACK_PSN_M,
2860 QP_CONTEXT_QPC_BYTES_84_LAST_ACK_PSN_S,
2861 attr->rq_psn - 1);
2862 roce_set_field(context->qpc_bytes_84,
2863 QP_CONTEXT_QPC_BYTES_84_TRRL_HEAD_M,
2864 QP_CONTEXT_QPC_BYTES_84_TRRL_HEAD_S, 0);
2865
2866 roce_set_field(context->qpc_bytes_88,
2867 QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_M,
2868 QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_S,
2869 attr->rq_psn);
2870 roce_set_bit(context->qpc_bytes_88,
2871 QP_CONTEXT_QPC_BYTES_88_RX_REQ_PSN_ERR_FLAG_S, 0);
2872 roce_set_bit(context->qpc_bytes_88,
2873 QP_CONTEXT_QPC_BYTES_88_RX_LAST_OPCODE_FLG_S, 0);
2874 roce_set_field(context->qpc_bytes_88,
2875 QP_CONTEXT_QPC_BYTES_88_RQ_REQ_LAST_OPERATION_TYPE_M,
2876 QP_CONTEXT_QPC_BYTES_88_RQ_REQ_LAST_OPERATION_TYPE_S,
2877 0);
2878 roce_set_field(context->qpc_bytes_88,
2879 QP_CONTEXT_QPC_BYTES_88_RQ_REQ_RDMA_WR_FLAG_M,
2880 QP_CONTEXT_QPC_BYTES_88_RQ_REQ_RDMA_WR_FLAG_S,
2881 0);
2882
2883 context->dma_length = 0;
2884 context->r_key = 0;
2885 context->va_l = 0;
2886 context->va_h = 0;
2887
2888 roce_set_field(context->qpc_bytes_108,
2889 QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_M,
2890 QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_S, 0);
2891 roce_set_bit(context->qpc_bytes_108,
2892 QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_FLG_S, 0);
2893 roce_set_bit(context->qpc_bytes_108,
2894 QP_CONTEXT_QPC_BYTES_108_TRRL_TDB_PSN_FLG_S, 0);
2895
2896 roce_set_field(context->qpc_bytes_112,
2897 QP_CONTEXT_QPC_BYTES_112_TRRL_TDB_PSN_M,
2898 QP_CONTEXT_QPC_BYTES_112_TRRL_TDB_PSN_S, 0);
2899 roce_set_field(context->qpc_bytes_112,
2900 QP_CONTEXT_QPC_BYTES_112_TRRL_TAIL_M,
2901 QP_CONTEXT_QPC_BYTES_112_TRRL_TAIL_S, 0);
2902
2903 /* For chip resp ack */
2904 roce_set_field(context->qpc_bytes_156,
2905 QP_CONTEXT_QPC_BYTES_156_PORT_NUM_M,
2906 QP_CONTEXT_QPC_BYTES_156_PORT_NUM_S,
2907 hr_qp->phy_port);
2908 roce_set_field(context->qpc_bytes_156,
2909 QP_CONTEXT_QPC_BYTES_156_SL_M,
2910 QP_CONTEXT_QPC_BYTES_156_SL_S, attr->ah_attr.sl);
2911 hr_qp->sl = attr->ah_attr.sl;
2912 } else if (cur_state == IB_QPS_RTR &&
2913 new_state == IB_QPS_RTS) {
2914 /* If exist optional param, return error */
2915 if ((attr_mask & IB_QP_ALT_PATH) ||
2916 (attr_mask & IB_QP_ACCESS_FLAGS) ||
2917 (attr_mask & IB_QP_QKEY) ||
2918 (attr_mask & IB_QP_PATH_MIG_STATE) ||
2919 (attr_mask & IB_QP_CUR_STATE) ||
2920 (attr_mask & IB_QP_MIN_RNR_TIMER)) {
2921 dev_err(dev, "RTR2RTS attr_mask error\n");
2922 goto out;
2923 }
2924
2925 context->rx_cur_sq_wqe_ba_l = (u32)(mtts[0]);
2926
2927 roce_set_field(context->qpc_bytes_120,
2928 QP_CONTEXT_QPC_BYTES_120_RX_CUR_SQ_WQE_BA_H_M,
2929 QP_CONTEXT_QPC_BYTES_120_RX_CUR_SQ_WQE_BA_H_S,
2930 (mtts[0]) >> 32);
2931
2932 roce_set_field(context->qpc_bytes_124,
2933 QP_CONTEXT_QPC_BYTES_124_RX_ACK_MSN_M,
2934 QP_CONTEXT_QPC_BYTES_124_RX_ACK_MSN_S, 0);
2935 roce_set_field(context->qpc_bytes_124,
2936 QP_CONTEXT_QPC_BYTES_124_IRRL_MSG_IDX_M,
2937 QP_CONTEXT_QPC_BYTES_124_IRRL_MSG_IDX_S, 0);
2938
2939 roce_set_field(context->qpc_bytes_128,
2940 QP_CONTEXT_QPC_BYTES_128_RX_ACK_EPSN_M,
2941 QP_CONTEXT_QPC_BYTES_128_RX_ACK_EPSN_S,
2942 attr->sq_psn);
2943 roce_set_bit(context->qpc_bytes_128,
2944 QP_CONTEXT_QPC_BYTES_128_RX_ACK_PSN_ERR_FLG_S, 0);
2945 roce_set_field(context->qpc_bytes_128,
2946 QP_CONTEXT_QPC_BYTES_128_ACK_LAST_OPERATION_TYPE_M,
2947 QP_CONTEXT_QPC_BYTES_128_ACK_LAST_OPERATION_TYPE_S,
2948 0);
2949 roce_set_bit(context->qpc_bytes_128,
2950 QP_CONTEXT_QPC_BYTES_128_IRRL_PSN_VLD_FLG_S, 0);
2951
2952 roce_set_field(context->qpc_bytes_132,
2953 QP_CONTEXT_QPC_BYTES_132_IRRL_PSN_M,
2954 QP_CONTEXT_QPC_BYTES_132_IRRL_PSN_S, 0);
2955 roce_set_field(context->qpc_bytes_132,
2956 QP_CONTEXT_QPC_BYTES_132_IRRL_TAIL_M,
2957 QP_CONTEXT_QPC_BYTES_132_IRRL_TAIL_S, 0);
2958
2959 roce_set_field(context->qpc_bytes_136,
2960 QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_PSN_M,
2961 QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_PSN_S,
2962 attr->sq_psn);
2963 roce_set_field(context->qpc_bytes_136,
2964 QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_FPKT_PSN_L_M,
2965 QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_FPKT_PSN_L_S,
2966 attr->sq_psn);
2967
2968 roce_set_field(context->qpc_bytes_140,
2969 QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_FPKT_PSN_H_M,
2970 QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_FPKT_PSN_H_S,
2971 (attr->sq_psn >> SQ_PSN_SHIFT));
2972 roce_set_field(context->qpc_bytes_140,
2973 QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_MSN_M,
2974 QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_MSN_S, 0);
2975 roce_set_bit(context->qpc_bytes_140,
2976 QP_CONTEXT_QPC_BYTES_140_RNR_RETRY_FLG_S, 0);
2977
2978 roce_set_field(context->qpc_bytes_148,
2979 QP_CONTEXT_QPC_BYTES_148_CHECK_FLAG_M,
2980 QP_CONTEXT_QPC_BYTES_148_CHECK_FLAG_S, 0);
2981 roce_set_field(context->qpc_bytes_148,
2982 QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_M,
2983 QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_S,
2984 attr->retry_cnt);
2985 roce_set_field(context->qpc_bytes_148,
2986 QP_CONTEXT_QPC_BYTES_148_RNR_RETRY_COUNT_M,
2987 QP_CONTEXT_QPC_BYTES_148_RNR_RETRY_COUNT_S,
2988 attr->rnr_retry);
2989 roce_set_field(context->qpc_bytes_148,
2990 QP_CONTEXT_QPC_BYTES_148_LSN_M,
2991 QP_CONTEXT_QPC_BYTES_148_LSN_S, 0x100);
2992
2993 context->rnr_retry = 0;
2994
2995 roce_set_field(context->qpc_bytes_156,
2996 QP_CONTEXT_QPC_BYTES_156_RETRY_COUNT_INIT_M,
2997 QP_CONTEXT_QPC_BYTES_156_RETRY_COUNT_INIT_S,
2998 attr->retry_cnt);
2999 if (attr->timeout < 0x12) {
3000 dev_info(dev, "ack timeout value(0x%x) must bigger than 0x12.\n",
3001 attr->timeout);
3002 roce_set_field(context->qpc_bytes_156,
3003 QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_M,
3004 QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_S,
3005 0x12);
3006 } else {
3007 roce_set_field(context->qpc_bytes_156,
3008 QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_M,
3009 QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_S,
3010 attr->timeout);
3011 }
3012 roce_set_field(context->qpc_bytes_156,
3013 QP_CONTEXT_QPC_BYTES_156_RNR_RETRY_COUNT_INIT_M,
3014 QP_CONTEXT_QPC_BYTES_156_RNR_RETRY_COUNT_INIT_S,
3015 attr->rnr_retry);
3016 roce_set_field(context->qpc_bytes_156,
3017 QP_CONTEXT_QPC_BYTES_156_PORT_NUM_M,
3018 QP_CONTEXT_QPC_BYTES_156_PORT_NUM_S,
3019 hr_qp->phy_port);
3020 roce_set_field(context->qpc_bytes_156,
3021 QP_CONTEXT_QPC_BYTES_156_SL_M,
3022 QP_CONTEXT_QPC_BYTES_156_SL_S, attr->ah_attr.sl);
3023 hr_qp->sl = attr->ah_attr.sl;
3024 roce_set_field(context->qpc_bytes_156,
3025 QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_M,
3026 QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_S,
3027 ilog2((unsigned int)attr->max_rd_atomic));
3028 roce_set_field(context->qpc_bytes_156,
3029 QP_CONTEXT_QPC_BYTES_156_ACK_REQ_IND_M,
3030 QP_CONTEXT_QPC_BYTES_156_ACK_REQ_IND_S, 0);
3031 context->pkt_use_len = 0;
3032
3033 roce_set_field(context->qpc_bytes_164,
3034 QP_CONTEXT_QPC_BYTES_164_SQ_PSN_M,
3035 QP_CONTEXT_QPC_BYTES_164_SQ_PSN_S, attr->sq_psn);
3036 roce_set_field(context->qpc_bytes_164,
3037 QP_CONTEXT_QPC_BYTES_164_IRRL_HEAD_M,
3038 QP_CONTEXT_QPC_BYTES_164_IRRL_HEAD_S, 0);
3039
3040 roce_set_field(context->qpc_bytes_168,
3041 QP_CONTEXT_QPC_BYTES_168_RETRY_SQ_PSN_M,
3042 QP_CONTEXT_QPC_BYTES_168_RETRY_SQ_PSN_S,
3043 attr->sq_psn);
3044 roce_set_field(context->qpc_bytes_168,
3045 QP_CONTEXT_QPC_BYTES_168_SGE_USE_FLA_M,
3046 QP_CONTEXT_QPC_BYTES_168_SGE_USE_FLA_S, 0);
3047 roce_set_field(context->qpc_bytes_168,
3048 QP_CONTEXT_QPC_BYTES_168_DB_TYPE_M,
3049 QP_CONTEXT_QPC_BYTES_168_DB_TYPE_S, 0);
3050 roce_set_bit(context->qpc_bytes_168,
3051 QP_CONTEXT_QPC_BYTES_168_MSG_LP_IND_S, 0);
3052 roce_set_bit(context->qpc_bytes_168,
3053 QP_CONTEXT_QPC_BYTES_168_CSDB_LP_IND_S, 0);
3054 roce_set_bit(context->qpc_bytes_168,
3055 QP_CONTEXT_QPC_BYTES_168_QP_ERR_FLG_S, 0);
3056 context->sge_use_len = 0;
3057
3058 roce_set_field(context->qpc_bytes_176,
3059 QP_CONTEXT_QPC_BYTES_176_DB_CUR_INDEX_M,
3060 QP_CONTEXT_QPC_BYTES_176_DB_CUR_INDEX_S, 0);
3061 roce_set_field(context->qpc_bytes_176,
3062 QP_CONTEXT_QPC_BYTES_176_RETRY_DB_CUR_INDEX_M,
3063 QP_CONTEXT_QPC_BYTES_176_RETRY_DB_CUR_INDEX_S,
3064 0);
3065 roce_set_field(context->qpc_bytes_180,
3066 QP_CONTEXT_QPC_BYTES_180_SQ_CUR_INDEX_M,
3067 QP_CONTEXT_QPC_BYTES_180_SQ_CUR_INDEX_S, 0);
3068 roce_set_field(context->qpc_bytes_180,
3069 QP_CONTEXT_QPC_BYTES_180_SQ_HEAD_M,
3070 QP_CONTEXT_QPC_BYTES_180_SQ_HEAD_S, 0);
3071
3072 context->tx_cur_sq_wqe_ba_l = (u32)(mtts[0]);
3073
3074 roce_set_field(context->qpc_bytes_188,
3075 QP_CONTEXT_QPC_BYTES_188_TX_CUR_SQ_WQE_BA_H_M,
3076 QP_CONTEXT_QPC_BYTES_188_TX_CUR_SQ_WQE_BA_H_S,
3077 (mtts[0]) >> 32);
3078 roce_set_bit(context->qpc_bytes_188,
3079 QP_CONTEXT_QPC_BYTES_188_PKT_RETRY_FLG_S, 0);
3080 roce_set_field(context->qpc_bytes_188,
3081 QP_CONTEXT_QPC_BYTES_188_TX_RETRY_CUR_INDEX_M,
3082 QP_CONTEXT_QPC_BYTES_188_TX_RETRY_CUR_INDEX_S,
3083 0);
3084 } else if (!((cur_state == IB_QPS_INIT && new_state == IB_QPS_RESET) ||
3085 (cur_state == IB_QPS_INIT && new_state == IB_QPS_ERR) ||
3086 (cur_state == IB_QPS_RTR && new_state == IB_QPS_RESET) ||
3087 (cur_state == IB_QPS_RTR && new_state == IB_QPS_ERR) ||
3088 (cur_state == IB_QPS_RTS && new_state == IB_QPS_RESET) ||
3089 (cur_state == IB_QPS_RTS && new_state == IB_QPS_ERR) ||
3090 (cur_state == IB_QPS_ERR && new_state == IB_QPS_RESET) ||
3091 (cur_state == IB_QPS_ERR && new_state == IB_QPS_ERR))) {
3092 dev_err(dev, "not support this status migration\n");
3093 goto out;
3094 }
3095
3096 /* Every status migrate must change state */
3097 roce_set_field(context->qpc_bytes_144,
3098 QP_CONTEXT_QPC_BYTES_144_QP_STATE_M,
3099 QP_CONTEXT_QPC_BYTES_144_QP_STATE_S, new_state);
3100
3101 /* SW pass context to HW */
3102 ret = hns_roce_v1_qp_modify(hr_dev, &hr_qp->mtt,
3103 to_hns_roce_state(cur_state),
3104 to_hns_roce_state(new_state), context,
3105 hr_qp);
3106 if (ret) {
3107 dev_err(dev, "hns_roce_qp_modify failed\n");
3108 goto out;
3109 }
3110
3111 /*
3112 * Use rst2init to instead of init2init with drv,
3113 * need to hw to flash RQ HEAD by DB again
3114 */
3115 if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) {
3116 /* Memory barrier */
3117 wmb();
3118
3119 roce_set_field(doorbell[0], RQ_DOORBELL_U32_4_RQ_HEAD_M,
3120 RQ_DOORBELL_U32_4_RQ_HEAD_S, hr_qp->rq.head);
3121 roce_set_field(doorbell[1], RQ_DOORBELL_U32_8_QPN_M,
3122 RQ_DOORBELL_U32_8_QPN_S, hr_qp->qpn);
3123 roce_set_field(doorbell[1], RQ_DOORBELL_U32_8_CMD_M,
3124 RQ_DOORBELL_U32_8_CMD_S, 1);
3125 roce_set_bit(doorbell[1], RQ_DOORBELL_U32_8_HW_SYNC_S, 1);
3126
3127 if (ibqp->uobject) {
3128 hr_qp->rq.db_reg_l = hr_dev->reg_base +
3129 ROCEE_DB_OTHERS_L_0_REG +
3130 DB_REG_OFFSET * hr_dev->priv_uar.index;
3131 }
3132
3133 hns_roce_write64_k(doorbell, hr_qp->rq.db_reg_l);
3134 }
3135
3136 hr_qp->state = new_state;
3137
3138 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
3139 hr_qp->resp_depth = attr->max_dest_rd_atomic;
3140 if (attr_mask & IB_QP_PORT) {
3141 hr_qp->port = attr->port_num - 1;
3142 hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port];
3143 }
3144
3145 if (new_state == IB_QPS_RESET && !ibqp->uobject) {
3146 hns_roce_v1_cq_clean(to_hr_cq(ibqp->recv_cq), hr_qp->qpn,
3147 ibqp->srq ? to_hr_srq(ibqp->srq) : NULL);
3148 if (ibqp->send_cq != ibqp->recv_cq)
3149 hns_roce_v1_cq_clean(to_hr_cq(ibqp->send_cq),
3150 hr_qp->qpn, NULL);
3151
3152 hr_qp->rq.head = 0;
3153 hr_qp->rq.tail = 0;
3154 hr_qp->sq.head = 0;
3155 hr_qp->sq.tail = 0;
3156 hr_qp->sq_next_wqe = 0;
3157 }
3158 out:
3159 kfree(context);
3160 return ret;
3161 }
3162
3163 int hns_roce_v1_modify_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
3164 int attr_mask, enum ib_qp_state cur_state,
3165 enum ib_qp_state new_state)
3166 {
3167
3168 if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI)
3169 return hns_roce_v1_m_sqp(ibqp, attr, attr_mask, cur_state,
3170 new_state);
3171 else
3172 return hns_roce_v1_m_qp(ibqp, attr, attr_mask, cur_state,
3173 new_state);
3174 }
3175
3176 static enum ib_qp_state to_ib_qp_state(enum hns_roce_qp_state state)
3177 {
3178 switch (state) {
3179 case HNS_ROCE_QP_STATE_RST:
3180 return IB_QPS_RESET;
3181 case HNS_ROCE_QP_STATE_INIT:
3182 return IB_QPS_INIT;
3183 case HNS_ROCE_QP_STATE_RTR:
3184 return IB_QPS_RTR;
3185 case HNS_ROCE_QP_STATE_RTS:
3186 return IB_QPS_RTS;
3187 case HNS_ROCE_QP_STATE_SQD:
3188 return IB_QPS_SQD;
3189 case HNS_ROCE_QP_STATE_ERR:
3190 return IB_QPS_ERR;
3191 default:
3192 return IB_QPS_ERR;
3193 }
3194 }
3195
3196 static int hns_roce_v1_query_qpc(struct hns_roce_dev *hr_dev,
3197 struct hns_roce_qp *hr_qp,
3198 struct hns_roce_qp_context *hr_context)
3199 {
3200 struct hns_roce_cmd_mailbox *mailbox;
3201 int ret;
3202
3203 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
3204 if (IS_ERR(mailbox))
3205 return PTR_ERR(mailbox);
3206
3207 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, hr_qp->qpn, 0,
3208 HNS_ROCE_CMD_QUERY_QP,
3209 HNS_ROCE_CMD_TIMEOUT_MSECS);
3210 if (!ret)
3211 memcpy(hr_context, mailbox->buf, sizeof(*hr_context));
3212 else
3213 dev_err(&hr_dev->pdev->dev, "QUERY QP cmd process error\n");
3214
3215 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
3216
3217 return ret;
3218 }
3219
3220 static int hns_roce_v1_q_sqp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
3221 int qp_attr_mask,
3222 struct ib_qp_init_attr *qp_init_attr)
3223 {
3224 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
3225 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3226 struct hns_roce_sqp_context context;
3227 u32 addr;
3228
3229 mutex_lock(&hr_qp->mutex);
3230
3231 if (hr_qp->state == IB_QPS_RESET) {
3232 qp_attr->qp_state = IB_QPS_RESET;
3233 goto done;
3234 }
3235
3236 addr = ROCEE_QP1C_CFG0_0_REG +
3237 hr_qp->port * sizeof(struct hns_roce_sqp_context);
3238 context.qp1c_bytes_4 = roce_read(hr_dev, addr);
3239 context.sq_rq_bt_l = roce_read(hr_dev, addr + 1);
3240 context.qp1c_bytes_12 = roce_read(hr_dev, addr + 2);
3241 context.qp1c_bytes_16 = roce_read(hr_dev, addr + 3);
3242 context.qp1c_bytes_20 = roce_read(hr_dev, addr + 4);
3243 context.cur_rq_wqe_ba_l = roce_read(hr_dev, addr + 5);
3244 context.qp1c_bytes_28 = roce_read(hr_dev, addr + 6);
3245 context.qp1c_bytes_32 = roce_read(hr_dev, addr + 7);
3246 context.cur_sq_wqe_ba_l = roce_read(hr_dev, addr + 8);
3247 context.qp1c_bytes_40 = roce_read(hr_dev, addr + 9);
3248
3249 hr_qp->state = roce_get_field(context.qp1c_bytes_4,
3250 QP1C_BYTES_4_QP_STATE_M,
3251 QP1C_BYTES_4_QP_STATE_S);
3252 qp_attr->qp_state = hr_qp->state;
3253 qp_attr->path_mtu = IB_MTU_256;
3254 qp_attr->path_mig_state = IB_MIG_ARMED;
3255 qp_attr->qkey = QKEY_VAL;
3256 qp_attr->rq_psn = 0;
3257 qp_attr->sq_psn = 0;
3258 qp_attr->dest_qp_num = 1;
3259 qp_attr->qp_access_flags = 6;
3260
3261 qp_attr->pkey_index = roce_get_field(context.qp1c_bytes_20,
3262 QP1C_BYTES_20_PKEY_IDX_M,
3263 QP1C_BYTES_20_PKEY_IDX_S);
3264 qp_attr->port_num = hr_qp->port + 1;
3265 qp_attr->sq_draining = 0;
3266 qp_attr->max_rd_atomic = 0;
3267 qp_attr->max_dest_rd_atomic = 0;
3268 qp_attr->min_rnr_timer = 0;
3269 qp_attr->timeout = 0;
3270 qp_attr->retry_cnt = 0;
3271 qp_attr->rnr_retry = 0;
3272 qp_attr->alt_timeout = 0;
3273
3274 done:
3275 qp_attr->cur_qp_state = qp_attr->qp_state;
3276 qp_attr->cap.max_recv_wr = hr_qp->rq.wqe_cnt;
3277 qp_attr->cap.max_recv_sge = hr_qp->rq.max_gs;
3278 qp_attr->cap.max_send_wr = hr_qp->sq.wqe_cnt;
3279 qp_attr->cap.max_send_sge = hr_qp->sq.max_gs;
3280 qp_attr->cap.max_inline_data = 0;
3281 qp_init_attr->cap = qp_attr->cap;
3282 qp_init_attr->create_flags = 0;
3283
3284 mutex_unlock(&hr_qp->mutex);
3285
3286 return 0;
3287 }
3288
3289 static int hns_roce_v1_q_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
3290 int qp_attr_mask,
3291 struct ib_qp_init_attr *qp_init_attr)
3292 {
3293 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
3294 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3295 struct device *dev = &hr_dev->pdev->dev;
3296 struct hns_roce_qp_context *context;
3297 int tmp_qp_state = 0;
3298 int ret = 0;
3299 int state;
3300
3301 context = kzalloc(sizeof(*context), GFP_KERNEL);
3302 if (!context)
3303 return -ENOMEM;
3304
3305 memset(qp_attr, 0, sizeof(*qp_attr));
3306 memset(qp_init_attr, 0, sizeof(*qp_init_attr));
3307
3308 mutex_lock(&hr_qp->mutex);
3309
3310 if (hr_qp->state == IB_QPS_RESET) {
3311 qp_attr->qp_state = IB_QPS_RESET;
3312 goto done;
3313 }
3314
3315 ret = hns_roce_v1_query_qpc(hr_dev, hr_qp, context);
3316 if (ret) {
3317 dev_err(dev, "query qpc error\n");
3318 ret = -EINVAL;
3319 goto out;
3320 }
3321
3322 state = roce_get_field(context->qpc_bytes_144,
3323 QP_CONTEXT_QPC_BYTES_144_QP_STATE_M,
3324 QP_CONTEXT_QPC_BYTES_144_QP_STATE_S);
3325 tmp_qp_state = (int)to_ib_qp_state((enum hns_roce_qp_state)state);
3326 if (tmp_qp_state == -1) {
3327 dev_err(dev, "to_ib_qp_state error\n");
3328 ret = -EINVAL;
3329 goto out;
3330 }
3331 hr_qp->state = (u8)tmp_qp_state;
3332 qp_attr->qp_state = (enum ib_qp_state)hr_qp->state;
3333 qp_attr->path_mtu = (enum ib_mtu)roce_get_field(context->qpc_bytes_48,
3334 QP_CONTEXT_QPC_BYTES_48_MTU_M,
3335 QP_CONTEXT_QPC_BYTES_48_MTU_S);
3336 qp_attr->path_mig_state = IB_MIG_ARMED;
3337 if (hr_qp->ibqp.qp_type == IB_QPT_UD)
3338 qp_attr->qkey = QKEY_VAL;
3339
3340 qp_attr->rq_psn = roce_get_field(context->qpc_bytes_88,
3341 QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_M,
3342 QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_S);
3343 qp_attr->sq_psn = (u32)roce_get_field(context->qpc_bytes_164,
3344 QP_CONTEXT_QPC_BYTES_164_SQ_PSN_M,
3345 QP_CONTEXT_QPC_BYTES_164_SQ_PSN_S);
3346 qp_attr->dest_qp_num = (u8)roce_get_field(context->qpc_bytes_36,
3347 QP_CONTEXT_QPC_BYTES_36_DEST_QP_M,
3348 QP_CONTEXT_QPC_BYTES_36_DEST_QP_S);
3349 qp_attr->qp_access_flags = ((roce_get_bit(context->qpc_bytes_4,
3350 QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S)) << 2) |
3351 ((roce_get_bit(context->qpc_bytes_4,
3352 QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S)) << 1) |
3353 ((roce_get_bit(context->qpc_bytes_4,
3354 QP_CONTEXT_QPC_BYTE_4_ATOMIC_OPERATION_ENABLE_S)) << 3);
3355
3356 if (hr_qp->ibqp.qp_type == IB_QPT_RC ||
3357 hr_qp->ibqp.qp_type == IB_QPT_UC) {
3358 qp_attr->ah_attr.sl = roce_get_field(context->qpc_bytes_156,
3359 QP_CONTEXT_QPC_BYTES_156_SL_M,
3360 QP_CONTEXT_QPC_BYTES_156_SL_S);
3361 qp_attr->ah_attr.grh.flow_label = roce_get_field(
3362 context->qpc_bytes_48,
3363 QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_M,
3364 QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_S);
3365 qp_attr->ah_attr.grh.sgid_index = roce_get_field(
3366 context->qpc_bytes_36,
3367 QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_M,
3368 QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_S);
3369 qp_attr->ah_attr.grh.hop_limit = roce_get_field(
3370 context->qpc_bytes_44,
3371 QP_CONTEXT_QPC_BYTES_44_HOPLMT_M,
3372 QP_CONTEXT_QPC_BYTES_44_HOPLMT_S);
3373 qp_attr->ah_attr.grh.traffic_class = roce_get_field(
3374 context->qpc_bytes_48,
3375 QP_CONTEXT_QPC_BYTES_48_TCLASS_M,
3376 QP_CONTEXT_QPC_BYTES_48_TCLASS_S);
3377
3378 memcpy(qp_attr->ah_attr.grh.dgid.raw, context->dgid,
3379 sizeof(qp_attr->ah_attr.grh.dgid.raw));
3380 }
3381
3382 qp_attr->pkey_index = roce_get_field(context->qpc_bytes_12,
3383 QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M,
3384 QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S);
3385 qp_attr->port_num = hr_qp->port + 1;
3386 qp_attr->sq_draining = 0;
3387 qp_attr->max_rd_atomic = roce_get_field(context->qpc_bytes_156,
3388 QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_M,
3389 QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_S);
3390 qp_attr->max_dest_rd_atomic = roce_get_field(context->qpc_bytes_32,
3391 QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_M,
3392 QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_S);
3393 qp_attr->min_rnr_timer = (u8)(roce_get_field(context->qpc_bytes_24,
3394 QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_M,
3395 QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_S));
3396 qp_attr->timeout = (u8)(roce_get_field(context->qpc_bytes_156,
3397 QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_M,
3398 QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_S));
3399 qp_attr->retry_cnt = roce_get_field(context->qpc_bytes_148,
3400 QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_M,
3401 QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_S);
3402 qp_attr->rnr_retry = context->rnr_retry;
3403
3404 done:
3405 qp_attr->cur_qp_state = qp_attr->qp_state;
3406 qp_attr->cap.max_recv_wr = hr_qp->rq.wqe_cnt;
3407 qp_attr->cap.max_recv_sge = hr_qp->rq.max_gs;
3408
3409 if (!ibqp->uobject) {
3410 qp_attr->cap.max_send_wr = hr_qp->sq.wqe_cnt;
3411 qp_attr->cap.max_send_sge = hr_qp->sq.max_gs;
3412 } else {
3413 qp_attr->cap.max_send_wr = 0;
3414 qp_attr->cap.max_send_sge = 0;
3415 }
3416
3417 qp_init_attr->cap = qp_attr->cap;
3418
3419 out:
3420 mutex_unlock(&hr_qp->mutex);
3421 kfree(context);
3422 return ret;
3423 }
3424
3425 int hns_roce_v1_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
3426 int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
3427 {
3428 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3429
3430 return hr_qp->doorbell_qpn <= 1 ?
3431 hns_roce_v1_q_sqp(ibqp, qp_attr, qp_attr_mask, qp_init_attr) :
3432 hns_roce_v1_q_qp(ibqp, qp_attr, qp_attr_mask, qp_init_attr);
3433 }
3434
3435 static int check_qp_db_process_status(struct hns_roce_dev *hr_dev,
3436 struct hns_roce_qp *hr_qp,
3437 u32 sdb_issue_ptr,
3438 u32 *sdb_inv_cnt,
3439 u32 *wait_stage)
3440 {
3441 struct device *dev = &hr_dev->pdev->dev;
3442 u32 sdb_retry_cnt, old_retry;
3443 u32 sdb_send_ptr, old_send;
3444 u32 success_flags = 0;
3445 u32 cur_cnt, old_cnt;
3446 unsigned long end;
3447 u32 send_ptr;
3448 u32 inv_cnt;
3449 u32 tsp_st;
3450
3451 if (*wait_stage > HNS_ROCE_V1_DB_STAGE2 ||
3452 *wait_stage < HNS_ROCE_V1_DB_STAGE1) {
3453 dev_err(dev, "QP(0x%lx) db status wait stage(%d) error!\n",
3454 hr_qp->qpn, *wait_stage);
3455 return -EINVAL;
3456 }
3457
3458 /* Calculate the total timeout for the entire verification process */
3459 end = msecs_to_jiffies(HNS_ROCE_V1_CHECK_DB_TIMEOUT_MSECS) + jiffies;
3460
3461 if (*wait_stage == HNS_ROCE_V1_DB_STAGE1) {
3462 /* Query db process status, until hw process completely */
3463 sdb_send_ptr = roce_read(hr_dev, ROCEE_SDB_SEND_PTR_REG);
3464 while (roce_hw_index_cmp_lt(sdb_send_ptr, sdb_issue_ptr,
3465 ROCEE_SDB_PTR_CMP_BITS)) {
3466 if (!time_before(jiffies, end)) {
3467 dev_dbg(dev, "QP(0x%lx) db process stage1 timeout. issue 0x%x send 0x%x.\n",
3468 hr_qp->qpn, sdb_issue_ptr,
3469 sdb_send_ptr);
3470 return 0;
3471 }
3472
3473 msleep(HNS_ROCE_V1_CHECK_DB_SLEEP_MSECS);
3474 sdb_send_ptr = roce_read(hr_dev,
3475 ROCEE_SDB_SEND_PTR_REG);
3476 }
3477
3478 if (roce_get_field(sdb_issue_ptr,
3479 ROCEE_SDB_ISSUE_PTR_SDB_ISSUE_PTR_M,
3480 ROCEE_SDB_ISSUE_PTR_SDB_ISSUE_PTR_S) ==
3481 roce_get_field(sdb_send_ptr,
3482 ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M,
3483 ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S)) {
3484 old_send = roce_read(hr_dev, ROCEE_SDB_SEND_PTR_REG);
3485 old_retry = roce_read(hr_dev, ROCEE_SDB_RETRY_CNT_REG);
3486
3487 do {
3488 tsp_st = roce_read(hr_dev, ROCEE_TSP_BP_ST_REG);
3489 if (roce_get_bit(tsp_st,
3490 ROCEE_TSP_BP_ST_QH_FIFO_ENTRY_S) == 1) {
3491 *wait_stage = HNS_ROCE_V1_DB_WAIT_OK;
3492 return 0;
3493 }
3494
3495 if (!time_before(jiffies, end)) {
3496 dev_dbg(dev, "QP(0x%lx) db process stage1 timeout when send ptr equals issue ptr.\n"
3497 "issue 0x%x send 0x%x.\n",
3498 hr_qp->qpn, sdb_issue_ptr,
3499 sdb_send_ptr);
3500 return 0;
3501 }
3502
3503 msleep(HNS_ROCE_V1_CHECK_DB_SLEEP_MSECS);
3504
3505 sdb_send_ptr = roce_read(hr_dev,
3506 ROCEE_SDB_SEND_PTR_REG);
3507 sdb_retry_cnt = roce_read(hr_dev,
3508 ROCEE_SDB_RETRY_CNT_REG);
3509 cur_cnt = roce_get_field(sdb_send_ptr,
3510 ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M,
3511 ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S) +
3512 roce_get_field(sdb_retry_cnt,
3513 ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_M,
3514 ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_S);
3515 if (!roce_get_bit(tsp_st,
3516 ROCEE_CNT_CLR_CE_CNT_CLR_CE_S)) {
3517 old_cnt = roce_get_field(old_send,
3518 ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M,
3519 ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S) +
3520 roce_get_field(old_retry,
3521 ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_M,
3522 ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_S);
3523 if (cur_cnt - old_cnt > SDB_ST_CMP_VAL)
3524 success_flags = 1;
3525 } else {
3526 old_cnt = roce_get_field(old_send,
3527 ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M,
3528 ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S);
3529 if (cur_cnt - old_cnt > SDB_ST_CMP_VAL)
3530 success_flags = 1;
3531 else {
3532 send_ptr = roce_get_field(old_send,
3533 ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M,
3534 ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S) +
3535 roce_get_field(sdb_retry_cnt,
3536 ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_M,
3537 ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_S);
3538 roce_set_field(old_send,
3539 ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M,
3540 ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S,
3541 send_ptr);
3542 }
3543 }
3544 } while (!success_flags);
3545 }
3546
3547 *wait_stage = HNS_ROCE_V1_DB_STAGE2;
3548
3549 /* Get list pointer */
3550 *sdb_inv_cnt = roce_read(hr_dev, ROCEE_SDB_INV_CNT_REG);
3551 dev_dbg(dev, "QP(0x%lx) db process stage2. inv cnt = 0x%x.\n",
3552 hr_qp->qpn, *sdb_inv_cnt);
3553 }
3554
3555 if (*wait_stage == HNS_ROCE_V1_DB_STAGE2) {
3556 /* Query db's list status, until hw reversal */
3557 inv_cnt = roce_read(hr_dev, ROCEE_SDB_INV_CNT_REG);
3558 while (roce_hw_index_cmp_lt(inv_cnt,
3559 *sdb_inv_cnt + SDB_INV_CNT_OFFSET,
3560 ROCEE_SDB_CNT_CMP_BITS)) {
3561 if (!time_before(jiffies, end)) {
3562 dev_dbg(dev, "QP(0x%lx) db process stage2 timeout. inv cnt 0x%x.\n",
3563 hr_qp->qpn, inv_cnt);
3564 return 0;
3565 }
3566
3567 msleep(HNS_ROCE_V1_CHECK_DB_SLEEP_MSECS);
3568 inv_cnt = roce_read(hr_dev, ROCEE_SDB_INV_CNT_REG);
3569 }
3570
3571 *wait_stage = HNS_ROCE_V1_DB_WAIT_OK;
3572 }
3573
3574 return 0;
3575 }
3576
3577 static int check_qp_reset_state(struct hns_roce_dev *hr_dev,
3578 struct hns_roce_qp *hr_qp,
3579 struct hns_roce_qp_work *qp_work_entry,
3580 int *is_timeout)
3581 {
3582 struct device *dev = &hr_dev->pdev->dev;
3583 u32 sdb_issue_ptr;
3584 int ret;
3585
3586 if (hr_qp->state != IB_QPS_RESET) {
3587 /* Set qp to ERR, waiting for hw complete processing all dbs */
3588 ret = hns_roce_v1_modify_qp(&hr_qp->ibqp, NULL, 0, hr_qp->state,
3589 IB_QPS_ERR);
3590 if (ret) {
3591 dev_err(dev, "Modify QP(0x%lx) to ERR failed!\n",
3592 hr_qp->qpn);
3593 return ret;
3594 }
3595
3596 /* Record issued doorbell */
3597 sdb_issue_ptr = roce_read(hr_dev, ROCEE_SDB_ISSUE_PTR_REG);
3598 qp_work_entry->sdb_issue_ptr = sdb_issue_ptr;
3599 qp_work_entry->db_wait_stage = HNS_ROCE_V1_DB_STAGE1;
3600
3601 /* Query db process status, until hw process completely */
3602 ret = check_qp_db_process_status(hr_dev, hr_qp, sdb_issue_ptr,
3603 &qp_work_entry->sdb_inv_cnt,
3604 &qp_work_entry->db_wait_stage);
3605 if (ret) {
3606 dev_err(dev, "Check QP(0x%lx) db process status failed!\n",
3607 hr_qp->qpn);
3608 return ret;
3609 }
3610
3611 if (qp_work_entry->db_wait_stage != HNS_ROCE_V1_DB_WAIT_OK) {
3612 qp_work_entry->sche_cnt = 0;
3613 *is_timeout = 1;
3614 return 0;
3615 }
3616
3617 /* Modify qp to reset before destroying qp */
3618 ret = hns_roce_v1_modify_qp(&hr_qp->ibqp, NULL, 0, hr_qp->state,
3619 IB_QPS_RESET);
3620 if (ret) {
3621 dev_err(dev, "Modify QP(0x%lx) to RST failed!\n",
3622 hr_qp->qpn);
3623 return ret;
3624 }
3625 }
3626
3627 return 0;
3628 }
3629
3630 static void hns_roce_v1_destroy_qp_work_fn(struct work_struct *work)
3631 {
3632 struct hns_roce_qp_work *qp_work_entry;
3633 struct hns_roce_v1_priv *priv;
3634 struct hns_roce_dev *hr_dev;
3635 struct hns_roce_qp *hr_qp;
3636 struct device *dev;
3637 int ret;
3638
3639 qp_work_entry = container_of(work, struct hns_roce_qp_work, work);
3640 hr_dev = to_hr_dev(qp_work_entry->ib_dev);
3641 dev = &hr_dev->pdev->dev;
3642 priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
3643 hr_qp = qp_work_entry->qp;
3644
3645 dev_dbg(dev, "Schedule destroy QP(0x%lx) work.\n", hr_qp->qpn);
3646
3647 qp_work_entry->sche_cnt++;
3648
3649 /* Query db process status, until hw process completely */
3650 ret = check_qp_db_process_status(hr_dev, hr_qp,
3651 qp_work_entry->sdb_issue_ptr,
3652 &qp_work_entry->sdb_inv_cnt,
3653 &qp_work_entry->db_wait_stage);
3654 if (ret) {
3655 dev_err(dev, "Check QP(0x%lx) db process status failed!\n",
3656 hr_qp->qpn);
3657 return;
3658 }
3659
3660 if (qp_work_entry->db_wait_stage != HNS_ROCE_V1_DB_WAIT_OK &&
3661 priv->des_qp.requeue_flag) {
3662 queue_work(priv->des_qp.qp_wq, work);
3663 return;
3664 }
3665
3666 /* Modify qp to reset before destroying qp */
3667 ret = hns_roce_v1_modify_qp(&hr_qp->ibqp, NULL, 0, hr_qp->state,
3668 IB_QPS_RESET);
3669 if (ret) {
3670 dev_err(dev, "Modify QP(0x%lx) to RST failed!\n", hr_qp->qpn);
3671 return;
3672 }
3673
3674 hns_roce_qp_remove(hr_dev, hr_qp);
3675 hns_roce_qp_free(hr_dev, hr_qp);
3676
3677 if (hr_qp->ibqp.qp_type == IB_QPT_RC) {
3678 /* RC QP, release QPN */
3679 hns_roce_release_range_qp(hr_dev, hr_qp->qpn, 1);
3680 kfree(hr_qp);
3681 } else
3682 kfree(hr_to_hr_sqp(hr_qp));
3683
3684 kfree(qp_work_entry);
3685
3686 dev_dbg(dev, "Accomplished destroy QP(0x%lx) work.\n", hr_qp->qpn);
3687 }
3688
3689 int hns_roce_v1_destroy_qp(struct ib_qp *ibqp)
3690 {
3691 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
3692 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3693 struct device *dev = &hr_dev->pdev->dev;
3694 struct hns_roce_qp_work qp_work_entry;
3695 struct hns_roce_qp_work *qp_work;
3696 struct hns_roce_v1_priv *priv;
3697 struct hns_roce_cq *send_cq, *recv_cq;
3698 int is_user = !!ibqp->pd->uobject;
3699 int is_timeout = 0;
3700 int ret;
3701
3702 ret = check_qp_reset_state(hr_dev, hr_qp, &qp_work_entry, &is_timeout);
3703 if (ret) {
3704 dev_err(dev, "QP reset state check failed(%d)!\n", ret);
3705 return ret;
3706 }
3707
3708 send_cq = to_hr_cq(hr_qp->ibqp.send_cq);
3709 recv_cq = to_hr_cq(hr_qp->ibqp.recv_cq);
3710
3711 hns_roce_lock_cqs(send_cq, recv_cq);
3712 if (!is_user) {
3713 __hns_roce_v1_cq_clean(recv_cq, hr_qp->qpn, hr_qp->ibqp.srq ?
3714 to_hr_srq(hr_qp->ibqp.srq) : NULL);
3715 if (send_cq != recv_cq)
3716 __hns_roce_v1_cq_clean(send_cq, hr_qp->qpn, NULL);
3717 }
3718 hns_roce_unlock_cqs(send_cq, recv_cq);
3719
3720 if (!is_timeout) {
3721 hns_roce_qp_remove(hr_dev, hr_qp);
3722 hns_roce_qp_free(hr_dev, hr_qp);
3723
3724 /* RC QP, release QPN */
3725 if (hr_qp->ibqp.qp_type == IB_QPT_RC)
3726 hns_roce_release_range_qp(hr_dev, hr_qp->qpn, 1);
3727 }
3728
3729 hns_roce_mtt_cleanup(hr_dev, &hr_qp->mtt);
3730
3731 if (is_user)
3732 ib_umem_release(hr_qp->umem);
3733 else {
3734 kfree(hr_qp->sq.wrid);
3735 kfree(hr_qp->rq.wrid);
3736
3737 hns_roce_buf_free(hr_dev, hr_qp->buff_size, &hr_qp->hr_buf);
3738 }
3739
3740 if (!is_timeout) {
3741 if (hr_qp->ibqp.qp_type == IB_QPT_RC)
3742 kfree(hr_qp);
3743 else
3744 kfree(hr_to_hr_sqp(hr_qp));
3745 } else {
3746 qp_work = kzalloc(sizeof(*qp_work), GFP_KERNEL);
3747 if (!qp_work)
3748 return -ENOMEM;
3749
3750 INIT_WORK(&qp_work->work, hns_roce_v1_destroy_qp_work_fn);
3751 qp_work->ib_dev = &hr_dev->ib_dev;
3752 qp_work->qp = hr_qp;
3753 qp_work->db_wait_stage = qp_work_entry.db_wait_stage;
3754 qp_work->sdb_issue_ptr = qp_work_entry.sdb_issue_ptr;
3755 qp_work->sdb_inv_cnt = qp_work_entry.sdb_inv_cnt;
3756 qp_work->sche_cnt = qp_work_entry.sche_cnt;
3757
3758 priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
3759 queue_work(priv->des_qp.qp_wq, &qp_work->work);
3760 dev_dbg(dev, "Begin destroy QP(0x%lx) work.\n", hr_qp->qpn);
3761 }
3762
3763 return 0;
3764 }
3765
3766 int hns_roce_v1_destroy_cq(struct ib_cq *ibcq)
3767 {
3768 struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device);
3769 struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
3770 struct device *dev = &hr_dev->pdev->dev;
3771 u32 cqe_cnt_ori;
3772 u32 cqe_cnt_cur;
3773 u32 cq_buf_size;
3774 int wait_time = 0;
3775 int ret = 0;
3776
3777 hns_roce_free_cq(hr_dev, hr_cq);
3778
3779 /*
3780 * Before freeing cq buffer, we need to ensure that the outstanding CQE
3781 * have been written by checking the CQE counter.
3782 */
3783 cqe_cnt_ori = roce_read(hr_dev, ROCEE_SCAEP_WR_CQE_CNT);
3784 while (1) {
3785 if (roce_read(hr_dev, ROCEE_CAEP_CQE_WCMD_EMPTY) &
3786 HNS_ROCE_CQE_WCMD_EMPTY_BIT)
3787 break;
3788
3789 cqe_cnt_cur = roce_read(hr_dev, ROCEE_SCAEP_WR_CQE_CNT);
3790 if ((cqe_cnt_cur - cqe_cnt_ori) >= HNS_ROCE_MIN_CQE_CNT)
3791 break;
3792
3793 msleep(HNS_ROCE_EACH_FREE_CQ_WAIT_MSECS);
3794 if (wait_time > HNS_ROCE_MAX_FREE_CQ_WAIT_CNT) {
3795 dev_warn(dev, "Destroy cq 0x%lx timeout!\n",
3796 hr_cq->cqn);
3797 ret = -ETIMEDOUT;
3798 break;
3799 }
3800 wait_time++;
3801 }
3802
3803 hns_roce_mtt_cleanup(hr_dev, &hr_cq->hr_buf.hr_mtt);
3804
3805 if (ibcq->uobject)
3806 ib_umem_release(hr_cq->umem);
3807 else {
3808 /* Free the buff of stored cq */
3809 cq_buf_size = (ibcq->cqe + 1) * hr_dev->caps.cq_entry_sz;
3810 hns_roce_buf_free(hr_dev, cq_buf_size, &hr_cq->hr_buf.hr_buf);
3811 }
3812
3813 kfree(hr_cq);
3814
3815 return ret;
3816 }
3817
3818 struct hns_roce_v1_priv hr_v1_priv;
3819
3820 struct hns_roce_hw hns_roce_hw_v1 = {
3821 .reset = hns_roce_v1_reset,
3822 .hw_profile = hns_roce_v1_profile,
3823 .hw_init = hns_roce_v1_init,
3824 .hw_exit = hns_roce_v1_exit,
3825 .set_gid = hns_roce_v1_set_gid,
3826 .set_mac = hns_roce_v1_set_mac,
3827 .set_mtu = hns_roce_v1_set_mtu,
3828 .write_mtpt = hns_roce_v1_write_mtpt,
3829 .write_cqc = hns_roce_v1_write_cqc,
3830 .clear_hem = hns_roce_v1_clear_hem,
3831 .modify_qp = hns_roce_v1_modify_qp,
3832 .query_qp = hns_roce_v1_query_qp,
3833 .destroy_qp = hns_roce_v1_destroy_qp,
3834 .post_send = hns_roce_v1_post_send,
3835 .post_recv = hns_roce_v1_post_recv,
3836 .req_notify_cq = hns_roce_v1_req_notify_cq,
3837 .poll_cq = hns_roce_v1_poll_cq,
3838 .dereg_mr = hns_roce_v1_dereg_mr,
3839 .destroy_cq = hns_roce_v1_destroy_cq,
3840 .priv = &hr_v1_priv,
3841 };
Linux with added FPC-III support