?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Linux 5.9.7
[linux/fpc-iii.git] / drivers / crypto / hisilicon / qm.c
blob6527c53b073fe17a4296801e1d052aa4d6d3285e
1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2019 HiSilicon Limited. */
3 #include <asm/page.h>
4 #include <linux/acpi.h>
5 #include <linux/aer.h>
6 #include <linux/bitmap.h>
7 #include <linux/debugfs.h>
8 #include <linux/dma-mapping.h>
9 #include <linux/idr.h>
10 #include <linux/io.h>
11 #include <linux/irqreturn.h>
12 #include <linux/log2.h>
13 #include <linux/seq_file.h>
14 #include <linux/slab.h>
15 #include <linux/uacce.h>
16 #include <linux/uaccess.h>
17 #include <uapi/misc/uacce/hisi_qm.h>
18 #include "qm.h"
19
20 /* eq/aeq irq enable */
21 #define QM_VF_AEQ_INT_SOURCE 0x0
22 #define QM_VF_AEQ_INT_MASK 0x4
23 #define QM_VF_EQ_INT_SOURCE 0x8
24 #define QM_VF_EQ_INT_MASK 0xc
25 #define QM_IRQ_NUM_V1 1
26 #define QM_IRQ_NUM_PF_V2 4
27 #define QM_IRQ_NUM_VF_V2 2
28
29 #define QM_EQ_EVENT_IRQ_VECTOR 0
30 #define QM_AEQ_EVENT_IRQ_VECTOR 1
31 #define QM_ABNORMAL_EVENT_IRQ_VECTOR 3
32
33 /* mailbox */
34 #define QM_MB_CMD_SQC 0x0
35 #define QM_MB_CMD_CQC 0x1
36 #define QM_MB_CMD_EQC 0x2
37 #define QM_MB_CMD_AEQC 0x3
38 #define QM_MB_CMD_SQC_BT 0x4
39 #define QM_MB_CMD_CQC_BT 0x5
40 #define QM_MB_CMD_SQC_VFT_V2 0x6
41
42 #define QM_MB_CMD_SEND_BASE 0x300
43 #define QM_MB_EVENT_SHIFT 8
44 #define QM_MB_BUSY_SHIFT 13
45 #define QM_MB_OP_SHIFT 14
46 #define QM_MB_CMD_DATA_ADDR_L 0x304
47 #define QM_MB_CMD_DATA_ADDR_H 0x308
48
49 /* sqc shift */
50 #define QM_SQ_HOP_NUM_SHIFT 0
51 #define QM_SQ_PAGE_SIZE_SHIFT 4
52 #define QM_SQ_BUF_SIZE_SHIFT 8
53 #define QM_SQ_SQE_SIZE_SHIFT 12
54 #define QM_SQ_PRIORITY_SHIFT 0
55 #define QM_SQ_ORDERS_SHIFT 4
56 #define QM_SQ_TYPE_SHIFT 8
57
58 #define QM_SQ_TYPE_MASK GENMASK(3, 0)
59 #define QM_SQ_TAIL_IDX(sqc) ((le16_to_cpu((sqc)->w11) >> 6) & 0x1)
60
61 /* cqc shift */
62 #define QM_CQ_HOP_NUM_SHIFT 0
63 #define QM_CQ_PAGE_SIZE_SHIFT 4
64 #define QM_CQ_BUF_SIZE_SHIFT 8
65 #define QM_CQ_CQE_SIZE_SHIFT 12
66 #define QM_CQ_PHASE_SHIFT 0
67 #define QM_CQ_FLAG_SHIFT 1
68
69 #define QM_CQE_PHASE(cqe) (le16_to_cpu((cqe)->w7) & 0x1)
70 #define QM_QC_CQE_SIZE 4
71 #define QM_CQ_TAIL_IDX(cqc) ((le16_to_cpu((cqc)->w11) >> 6) & 0x1)
72
73 /* eqc shift */
74 #define QM_EQE_AEQE_SIZE (2UL << 12)
75 #define QM_EQC_PHASE_SHIFT 16
76
77 #define QM_EQE_PHASE(eqe) ((le32_to_cpu((eqe)->dw0) >> 16) & 0x1)
78 #define QM_EQE_CQN_MASK GENMASK(15, 0)
79
80 #define QM_AEQE_PHASE(aeqe) ((le32_to_cpu((aeqe)->dw0) >> 16) & 0x1)
81 #define QM_AEQE_TYPE_SHIFT 17
82
83 #define QM_DOORBELL_CMD_SQ 0
84 #define QM_DOORBELL_CMD_CQ 1
85 #define QM_DOORBELL_CMD_EQ 2
86 #define QM_DOORBELL_CMD_AEQ 3
87
88 #define QM_DOORBELL_BASE_V1 0x340
89 #define QM_DB_CMD_SHIFT_V1 16
90 #define QM_DB_INDEX_SHIFT_V1 32
91 #define QM_DB_PRIORITY_SHIFT_V1 48
92 #define QM_DOORBELL_SQ_CQ_BASE_V2 0x1000
93 #define QM_DOORBELL_EQ_AEQ_BASE_V2 0x2000
94 #define QM_DB_CMD_SHIFT_V2 12
95 #define QM_DB_RAND_SHIFT_V2 16
96 #define QM_DB_INDEX_SHIFT_V2 32
97 #define QM_DB_PRIORITY_SHIFT_V2 48
98
99 #define QM_MEM_START_INIT 0x100040
100 #define QM_MEM_INIT_DONE 0x100044
101 #define QM_VFT_CFG_RDY 0x10006c
102 #define QM_VFT_CFG_OP_WR 0x100058
103 #define QM_VFT_CFG_TYPE 0x10005c
104 #define QM_SQC_VFT 0x0
105 #define QM_CQC_VFT 0x1
106 #define QM_VFT_CFG 0x100060
107 #define QM_VFT_CFG_OP_ENABLE 0x100054
108
109 #define QM_VFT_CFG_DATA_L 0x100064
110 #define QM_VFT_CFG_DATA_H 0x100068
111 #define QM_SQC_VFT_BUF_SIZE (7ULL << 8)
112 #define QM_SQC_VFT_SQC_SIZE (5ULL << 12)
113 #define QM_SQC_VFT_INDEX_NUMBER (1ULL << 16)
114 #define QM_SQC_VFT_START_SQN_SHIFT 28
115 #define QM_SQC_VFT_VALID (1ULL << 44)
116 #define QM_SQC_VFT_SQN_SHIFT 45
117 #define QM_CQC_VFT_BUF_SIZE (7ULL << 8)
118 #define QM_CQC_VFT_SQC_SIZE (5ULL << 12)
119 #define QM_CQC_VFT_INDEX_NUMBER (1ULL << 16)
120 #define QM_CQC_VFT_VALID (1ULL << 28)
121
122 #define QM_SQC_VFT_BASE_SHIFT_V2 28
123 #define QM_SQC_VFT_BASE_MASK_V2 GENMASK(5, 0)
124 #define QM_SQC_VFT_NUM_SHIFT_V2 45
125 #define QM_SQC_VFT_NUM_MASK_v2 GENMASK(9, 0)
126
127 #define QM_DFX_CNT_CLR_CE 0x100118
128
129 #define QM_ABNORMAL_INT_SOURCE 0x100000
130 #define QM_ABNORMAL_INT_SOURCE_CLR GENMASK(12, 0)
131 #define QM_ABNORMAL_INT_MASK 0x100004
132 #define QM_ABNORMAL_INT_MASK_VALUE 0x1fff
133 #define QM_ABNORMAL_INT_STATUS 0x100008
134 #define QM_ABNORMAL_INT_SET 0x10000c
135 #define QM_ABNORMAL_INF00 0x100010
136 #define QM_FIFO_OVERFLOW_TYPE 0xc0
137 #define QM_FIFO_OVERFLOW_TYPE_SHIFT 6
138 #define QM_FIFO_OVERFLOW_VF 0x3f
139 #define QM_ABNORMAL_INF01 0x100014
140 #define QM_DB_TIMEOUT_TYPE 0xc0
141 #define QM_DB_TIMEOUT_TYPE_SHIFT 6
142 #define QM_DB_TIMEOUT_VF 0x3f
143 #define QM_RAS_CE_ENABLE 0x1000ec
144 #define QM_RAS_FE_ENABLE 0x1000f0
145 #define QM_RAS_NFE_ENABLE 0x1000f4
146 #define QM_RAS_CE_THRESHOLD 0x1000f8
147 #define QM_RAS_CE_TIMES_PER_IRQ 1
148 #define QM_RAS_MSI_INT_SEL 0x1040f4
149
150 #define QM_DEV_RESET_FLAG 0
151 #define QM_RESET_WAIT_TIMEOUT 400
152 #define QM_PEH_VENDOR_ID 0x1000d8
153 #define ACC_VENDOR_ID_VALUE 0x5a5a
154 #define QM_PEH_DFX_INFO0 0x1000fc
155 #define ACC_PEH_SRIOV_CTRL_VF_MSE_SHIFT 3
156 #define ACC_PEH_MSI_DISABLE GENMASK(31, 0)
157 #define ACC_MASTER_GLOBAL_CTRL_SHUTDOWN 0x1
158 #define ACC_MASTER_TRANS_RETURN_RW 3
159 #define ACC_MASTER_TRANS_RETURN 0x300150
160 #define ACC_MASTER_GLOBAL_CTRL 0x300000
161 #define ACC_AM_CFG_PORT_WR_EN 0x30001c
162 #define QM_RAS_NFE_MBIT_DISABLE ~QM_ECC_MBIT
163 #define ACC_AM_ROB_ECC_INT_STS 0x300104
164 #define ACC_ROB_ECC_ERR_MULTPL BIT(1)
165
166 #define POLL_PERIOD 10
167 #define POLL_TIMEOUT 1000
168 #define WAIT_PERIOD_US_MAX 200
169 #define WAIT_PERIOD_US_MIN 100
170 #define MAX_WAIT_COUNTS 1000
171 #define QM_CACHE_WB_START 0x204
172 #define QM_CACHE_WB_DONE 0x208
173
174 #define PCI_BAR_2 2
175 #define QM_SQE_DATA_ALIGN_MASK GENMASK(6, 0)
176 #define QMC_ALIGN(sz) ALIGN(sz, 32)
177
178 #define QM_DBG_READ_LEN 256
179 #define QM_DBG_WRITE_LEN 1024
180 #define QM_DBG_TMP_BUF_LEN 22
181 #define QM_PCI_COMMAND_INVALID ~0
182
183 #define QM_SQE_ADDR_MASK GENMASK(7, 0)
184
185 #define QM_MK_CQC_DW3_V1(hop_num, pg_sz, buf_sz, cqe_sz) \
186 (((hop_num) << QM_CQ_HOP_NUM_SHIFT) | \
187 ((pg_sz) << QM_CQ_PAGE_SIZE_SHIFT) | \
188 ((buf_sz) << QM_CQ_BUF_SIZE_SHIFT) | \
189 ((cqe_sz) << QM_CQ_CQE_SIZE_SHIFT))
190
191 #define QM_MK_CQC_DW3_V2(cqe_sz) \
192 ((QM_Q_DEPTH - 1) | ((cqe_sz) << QM_CQ_CQE_SIZE_SHIFT))
193
194 #define QM_MK_SQC_W13(priority, orders, alg_type) \
195 (((priority) << QM_SQ_PRIORITY_SHIFT) | \
196 ((orders) << QM_SQ_ORDERS_SHIFT) | \
197 (((alg_type) & QM_SQ_TYPE_MASK) << QM_SQ_TYPE_SHIFT))
198
199 #define QM_MK_SQC_DW3_V1(hop_num, pg_sz, buf_sz, sqe_sz) \
200 (((hop_num) << QM_SQ_HOP_NUM_SHIFT) | \
201 ((pg_sz) << QM_SQ_PAGE_SIZE_SHIFT) | \
202 ((buf_sz) << QM_SQ_BUF_SIZE_SHIFT) | \
203 ((u32)ilog2(sqe_sz) << QM_SQ_SQE_SIZE_SHIFT))
204
205 #define QM_MK_SQC_DW3_V2(sqe_sz) \
206 ((QM_Q_DEPTH - 1) | ((u32)ilog2(sqe_sz) << QM_SQ_SQE_SIZE_SHIFT))
207
208 #define INIT_QC_COMMON(qc, base, pasid) do { \
209 (qc)->head = 0; \
210 (qc)->tail = 0; \
211 (qc)->base_l = cpu_to_le32(lower_32_bits(base)); \
212 (qc)->base_h = cpu_to_le32(upper_32_bits(base)); \
213 (qc)->dw3 = 0; \
214 (qc)->w8 = 0; \
215 (qc)->rsvd0 = 0; \
216 (qc)->pasid = cpu_to_le16(pasid); \
217 (qc)->w11 = 0; \
218 (qc)->rsvd1 = 0; \
219 } while (0)
220
221 enum vft_type {
222 SQC_VFT = 0,
223 CQC_VFT,
224 };
225
226 enum acc_err_result {
227 ACC_ERR_NONE,
228 ACC_ERR_NEED_RESET,
229 ACC_ERR_RECOVERED,
230 };
231
232 struct qm_cqe {
233 __le32 rsvd0;
234 __le16 cmd_id;
235 __le16 rsvd1;
236 __le16 sq_head;
237 __le16 sq_num;
238 __le16 rsvd2;
239 __le16 w7;
240 };
241
242 struct qm_eqe {
243 __le32 dw0;
244 };
245
246 struct qm_aeqe {
247 __le32 dw0;
248 };
249
250 struct qm_sqc {
251 __le16 head;
252 __le16 tail;
253 __le32 base_l;
254 __le32 base_h;
255 __le32 dw3;
256 __le16 w8;
257 __le16 rsvd0;
258 __le16 pasid;
259 __le16 w11;
260 __le16 cq_num;
261 __le16 w13;
262 __le32 rsvd1;
263 };
264
265 struct qm_cqc {
266 __le16 head;
267 __le16 tail;
268 __le32 base_l;
269 __le32 base_h;
270 __le32 dw3;
271 __le16 w8;
272 __le16 rsvd0;
273 __le16 pasid;
274 __le16 w11;
275 __le32 dw6;
276 __le32 rsvd1;
277 };
278
279 struct qm_eqc {
280 __le16 head;
281 __le16 tail;
282 __le32 base_l;
283 __le32 base_h;
284 __le32 dw3;
285 __le32 rsvd[2];
286 __le32 dw6;
287 };
288
289 struct qm_aeqc {
290 __le16 head;
291 __le16 tail;
292 __le32 base_l;
293 __le32 base_h;
294 __le32 dw3;
295 __le32 rsvd[2];
296 __le32 dw6;
297 };
298
299 struct qm_mailbox {
300 __le16 w0;
301 __le16 queue_num;
302 __le32 base_l;
303 __le32 base_h;
304 __le32 rsvd;
305 };
306
307 struct qm_doorbell {
308 __le16 queue_num;
309 __le16 cmd;
310 __le16 index;
311 __le16 priority;
312 };
313
314 struct hisi_qm_resource {
315 struct hisi_qm *qm;
316 int distance;
317 struct list_head list;
318 };
319
320 struct hisi_qm_hw_ops {
321 int (*get_vft)(struct hisi_qm *qm, u32 *base, u32 *number);
322 void (*qm_db)(struct hisi_qm *qm, u16 qn,
323 u8 cmd, u16 index, u8 priority);
324 u32 (*get_irq_num)(struct hisi_qm *qm);
325 int (*debug_init)(struct hisi_qm *qm);
326 void (*hw_error_init)(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe);
327 void (*hw_error_uninit)(struct hisi_qm *qm);
328 enum acc_err_result (*hw_error_handle)(struct hisi_qm *qm);
329 };
330
331 struct qm_dfx_item {
332 const char *name;
333 u32 offset;
334 };
335
336 static struct qm_dfx_item qm_dfx_files[] = {
337 {"err_irq", offsetof(struct qm_dfx, err_irq_cnt)},
338 {"aeq_irq", offsetof(struct qm_dfx, aeq_irq_cnt)},
339 {"abnormal_irq", offsetof(struct qm_dfx, abnormal_irq_cnt)},
340 {"create_qp_err", offsetof(struct qm_dfx, create_qp_err_cnt)},
341 {"mb_err", offsetof(struct qm_dfx, mb_err_cnt)},
342 };
343
344 static const char * const qm_debug_file_name[] = {
345 [CURRENT_Q] = "current_q",
346 [CLEAR_ENABLE] = "clear_enable",
347 };
348
349 struct hisi_qm_hw_error {
350 u32 int_msk;
351 const char *msg;
352 };
353
354 static const struct hisi_qm_hw_error qm_hw_error[] = {
355 { .int_msk = BIT(0), .msg = "qm_axi_rresp" },
356 { .int_msk = BIT(1), .msg = "qm_axi_bresp" },
357 { .int_msk = BIT(2), .msg = "qm_ecc_mbit" },
358 { .int_msk = BIT(3), .msg = "qm_ecc_1bit" },
359 { .int_msk = BIT(4), .msg = "qm_acc_get_task_timeout" },
360 { .int_msk = BIT(5), .msg = "qm_acc_do_task_timeout" },
361 { .int_msk = BIT(6), .msg = "qm_acc_wb_not_ready_timeout" },
362 { .int_msk = BIT(7), .msg = "qm_sq_cq_vf_invalid" },
363 { .int_msk = BIT(8), .msg = "qm_cq_vf_invalid" },
364 { .int_msk = BIT(9), .msg = "qm_sq_vf_invalid" },
365 { .int_msk = BIT(10), .msg = "qm_db_timeout" },
366 { .int_msk = BIT(11), .msg = "qm_of_fifo_of" },
367 { .int_msk = BIT(12), .msg = "qm_db_random_invalid" },
368 { /* sentinel */ }
369 };
370
371 static const char * const qm_db_timeout[] = {
372 "sq", "cq", "eq", "aeq",
373 };
374
375 static const char * const qm_fifo_overflow[] = {
376 "cq", "eq", "aeq",
377 };
378
379 static const char * const qm_s[] = {
380 "init", "start", "close", "stop",
381 };
382
383 static const char * const qp_s[] = {
384 "none", "init", "start", "stop", "close",
385 };
386
387 static bool qm_avail_state(struct hisi_qm *qm, enum qm_state new)
388 {
389 enum qm_state curr = atomic_read(&qm->status.flags);
390 bool avail = false;
391
392 switch (curr) {
393 case QM_INIT:
394 if (new == QM_START || new == QM_CLOSE)
395 avail = true;
396 break;
397 case QM_START:
398 if (new == QM_STOP)
399 avail = true;
400 break;
401 case QM_STOP:
402 if (new == QM_CLOSE || new == QM_START)
403 avail = true;
404 break;
405 default:
406 break;
407 }
408
409 dev_dbg(&qm->pdev->dev, "change qm state from %s to %s\n",
410 qm_s[curr], qm_s[new]);
411
412 if (!avail)
413 dev_warn(&qm->pdev->dev, "Can not change qm state from %s to %s\n",
414 qm_s[curr], qm_s[new]);
415
416 return avail;
417 }
418
419 static bool qm_qp_avail_state(struct hisi_qm *qm, struct hisi_qp *qp,
420 enum qp_state new)
421 {
422 enum qm_state qm_curr = atomic_read(&qm->status.flags);
423 enum qp_state qp_curr = 0;
424 bool avail = false;
425
426 if (qp)
427 qp_curr = atomic_read(&qp->qp_status.flags);
428
429 switch (new) {
430 case QP_INIT:
431 if (qm_curr == QM_START || qm_curr == QM_INIT)
432 avail = true;
433 break;
434 case QP_START:
435 if ((qm_curr == QM_START && qp_curr == QP_INIT) ||
436 (qm_curr == QM_START && qp_curr == QP_STOP))
437 avail = true;
438 break;
439 case QP_STOP:
440 if ((qm_curr == QM_START && qp_curr == QP_START) ||
441 (qp_curr == QP_INIT))
442 avail = true;
443 break;
444 case QP_CLOSE:
445 if ((qm_curr == QM_START && qp_curr == QP_INIT) ||
446 (qm_curr == QM_START && qp_curr == QP_STOP) ||
447 (qm_curr == QM_STOP && qp_curr == QP_STOP) ||
448 (qm_curr == QM_STOP && qp_curr == QP_INIT))
449 avail = true;
450 break;
451 default:
452 break;
453 }
454
455 dev_dbg(&qm->pdev->dev, "change qp state from %s to %s in QM %s\n",
456 qp_s[qp_curr], qp_s[new], qm_s[qm_curr]);
457
458 if (!avail)
459 dev_warn(&qm->pdev->dev,
460 "Can not change qp state from %s to %s in QM %s\n",
461 qp_s[qp_curr], qp_s[new], qm_s[qm_curr]);
462
463 return avail;
464 }
465
466 /* return 0 mailbox ready, -ETIMEDOUT hardware timeout */
467 static int qm_wait_mb_ready(struct hisi_qm *qm)
468 {
469 u32 val;
470
471 return readl_relaxed_poll_timeout(qm->io_base + QM_MB_CMD_SEND_BASE,
472 val, !((val >> QM_MB_BUSY_SHIFT) &
473 0x1), 10, 1000);
474 }
475
476 /* 128 bit should be written to hardware at one time to trigger a mailbox */
477 static void qm_mb_write(struct hisi_qm *qm, const void *src)
478 {
479 void __iomem *fun_base = qm->io_base + QM_MB_CMD_SEND_BASE;
480 unsigned long tmp0 = 0, tmp1 = 0;
481
482 if (!IS_ENABLED(CONFIG_ARM64)) {
483 memcpy_toio(fun_base, src, 16);
484 wmb();
485 return;
486 }
487
488 asm volatile("ldp %0, %1, %3\n"
489 "stp %0, %1, %2\n"
490 "dsb sy\n"
491 : "=&r" (tmp0),
492 "=&r" (tmp1),
493 "+Q" (*((char __iomem *)fun_base))
494 : "Q" (*((char *)src))
495 : "memory");
496 }
497
498 static int qm_mb(struct hisi_qm *qm, u8 cmd, dma_addr_t dma_addr, u16 queue,
499 bool op)
500 {
501 struct qm_mailbox mailbox;
502 int ret = 0;
503
504 dev_dbg(&qm->pdev->dev, "QM mailbox request to q%u: %u-%llx\n",
505 queue, cmd, (unsigned long long)dma_addr);
506
507 mailbox.w0 = cpu_to_le16(cmd |
508 (op ? 0x1 << QM_MB_OP_SHIFT : 0) |
509 (0x1 << QM_MB_BUSY_SHIFT));
510 mailbox.queue_num = cpu_to_le16(queue);
511 mailbox.base_l = cpu_to_le32(lower_32_bits(dma_addr));
512 mailbox.base_h = cpu_to_le32(upper_32_bits(dma_addr));
513 mailbox.rsvd = 0;
514
515 mutex_lock(&qm->mailbox_lock);
516
517 if (unlikely(qm_wait_mb_ready(qm))) {
518 ret = -EBUSY;
519 dev_err(&qm->pdev->dev, "QM mailbox is busy to start!\n");
520 goto busy_unlock;
521 }
522
523 qm_mb_write(qm, &mailbox);
524
525 if (unlikely(qm_wait_mb_ready(qm))) {
526 ret = -EBUSY;
527 dev_err(&qm->pdev->dev, "QM mailbox operation timeout!\n");
528 goto busy_unlock;
529 }
530
531 busy_unlock:
532 mutex_unlock(&qm->mailbox_lock);
533
534 if (ret)
535 atomic64_inc(&qm->debug.dfx.mb_err_cnt);
536 return ret;
537 }
538
539 static void qm_db_v1(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority)
540 {
541 u64 doorbell;
542
543 doorbell = qn | ((u64)cmd << QM_DB_CMD_SHIFT_V1) |
544 ((u64)index << QM_DB_INDEX_SHIFT_V1) |
545 ((u64)priority << QM_DB_PRIORITY_SHIFT_V1);
546
547 writeq(doorbell, qm->io_base + QM_DOORBELL_BASE_V1);
548 }
549
550 static void qm_db_v2(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority)
551 {
552 u64 doorbell;
553 u64 dbase;
554 u16 randata = 0;
555
556 if (cmd == QM_DOORBELL_CMD_SQ || cmd == QM_DOORBELL_CMD_CQ)
557 dbase = QM_DOORBELL_SQ_CQ_BASE_V2;
558 else
559 dbase = QM_DOORBELL_EQ_AEQ_BASE_V2;
560
561 doorbell = qn | ((u64)cmd << QM_DB_CMD_SHIFT_V2) |
562 ((u64)randata << QM_DB_RAND_SHIFT_V2) |
563 ((u64)index << QM_DB_INDEX_SHIFT_V2) |
564 ((u64)priority << QM_DB_PRIORITY_SHIFT_V2);
565
566 writeq(doorbell, qm->io_base + dbase);
567 }
568
569 static void qm_db(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority)
570 {
571 dev_dbg(&qm->pdev->dev, "QM doorbell request: qn=%u, cmd=%u, index=%u\n",
572 qn, cmd, index);
573
574 qm->ops->qm_db(qm, qn, cmd, index, priority);
575 }
576
577 static int qm_dev_mem_reset(struct hisi_qm *qm)
578 {
579 u32 val;
580
581 writel(0x1, qm->io_base + QM_MEM_START_INIT);
582 return readl_relaxed_poll_timeout(qm->io_base + QM_MEM_INIT_DONE, val,
583 val & BIT(0), 10, 1000);
584 }
585
586 static u32 qm_get_irq_num_v1(struct hisi_qm *qm)
587 {
588 return QM_IRQ_NUM_V1;
589 }
590
591 static u32 qm_get_irq_num_v2(struct hisi_qm *qm)
592 {
593 if (qm->fun_type == QM_HW_PF)
594 return QM_IRQ_NUM_PF_V2;
595 else
596 return QM_IRQ_NUM_VF_V2;
597 }
598
599 static struct hisi_qp *qm_to_hisi_qp(struct hisi_qm *qm, struct qm_eqe *eqe)
600 {
601 u16 cqn = le32_to_cpu(eqe->dw0) & QM_EQE_CQN_MASK;
602
603 return &qm->qp_array[cqn];
604 }
605
606 static void qm_cq_head_update(struct hisi_qp *qp)
607 {
608 if (qp->qp_status.cq_head == QM_Q_DEPTH - 1) {
609 qp->qp_status.cqc_phase = !qp->qp_status.cqc_phase;
610 qp->qp_status.cq_head = 0;
611 } else {
612 qp->qp_status.cq_head++;
613 }
614 }
615
616 static void qm_poll_qp(struct hisi_qp *qp, struct hisi_qm *qm)
617 {
618 if (qp->event_cb) {
619 qp->event_cb(qp);
620 return;
621 }
622
623 if (qp->req_cb) {
624 struct qm_cqe *cqe = qp->cqe + qp->qp_status.cq_head;
625
626 while (QM_CQE_PHASE(cqe) == qp->qp_status.cqc_phase) {
627 dma_rmb();
628 qp->req_cb(qp, qp->sqe + qm->sqe_size *
629 le16_to_cpu(cqe->sq_head));
630 qm_cq_head_update(qp);
631 cqe = qp->cqe + qp->qp_status.cq_head;
632 qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_CQ,
633 qp->qp_status.cq_head, 0);
634 atomic_dec(&qp->qp_status.used);
635 }
636
637 /* set c_flag */
638 qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_CQ,
639 qp->qp_status.cq_head, 1);
640 }
641 }
642
643 static void qm_work_process(struct work_struct *work)
644 {
645 struct hisi_qm *qm = container_of(work, struct hisi_qm, work);
646 struct qm_eqe *eqe = qm->eqe + qm->status.eq_head;
647 struct hisi_qp *qp;
648 int eqe_num = 0;
649
650 while (QM_EQE_PHASE(eqe) == qm->status.eqc_phase) {
651 eqe_num++;
652 qp = qm_to_hisi_qp(qm, eqe);
653 qm_poll_qp(qp, qm);
654
655 if (qm->status.eq_head == QM_Q_DEPTH - 1) {
656 qm->status.eqc_phase = !qm->status.eqc_phase;
657 eqe = qm->eqe;
658 qm->status.eq_head = 0;
659 } else {
660 eqe++;
661 qm->status.eq_head++;
662 }
663
664 if (eqe_num == QM_Q_DEPTH / 2 - 1) {
665 eqe_num = 0;
666 qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0);
667 }
668 }
669
670 qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0);
671 }
672
673 static irqreturn_t do_qm_irq(int irq, void *data)
674 {
675 struct hisi_qm *qm = (struct hisi_qm *)data;
676
677 /* the workqueue created by device driver of QM */
678 if (qm->wq)
679 queue_work(qm->wq, &qm->work);
680 else
681 schedule_work(&qm->work);
682
683 return IRQ_HANDLED;
684 }
685
686 static irqreturn_t qm_irq(int irq, void *data)
687 {
688 struct hisi_qm *qm = data;
689
690 if (readl(qm->io_base + QM_VF_EQ_INT_SOURCE))
691 return do_qm_irq(irq, data);
692
693 atomic64_inc(&qm->debug.dfx.err_irq_cnt);
694 dev_err(&qm->pdev->dev, "invalid int source\n");
695 qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0);
696
697 return IRQ_NONE;
698 }
699
700 static irqreturn_t qm_aeq_irq(int irq, void *data)
701 {
702 struct hisi_qm *qm = data;
703 struct qm_aeqe *aeqe = qm->aeqe + qm->status.aeq_head;
704 u32 type;
705
706 atomic64_inc(&qm->debug.dfx.aeq_irq_cnt);
707 if (!readl(qm->io_base + QM_VF_AEQ_INT_SOURCE))
708 return IRQ_NONE;
709
710 while (QM_AEQE_PHASE(aeqe) == qm->status.aeqc_phase) {
711 type = le32_to_cpu(aeqe->dw0) >> QM_AEQE_TYPE_SHIFT;
712 if (type < ARRAY_SIZE(qm_fifo_overflow))
713 dev_err(&qm->pdev->dev, "%s overflow\n",
714 qm_fifo_overflow[type]);
715 else
716 dev_err(&qm->pdev->dev, "unknown error type %d\n",
717 type);
718
719 if (qm->status.aeq_head == QM_Q_DEPTH - 1) {
720 qm->status.aeqc_phase = !qm->status.aeqc_phase;
721 aeqe = qm->aeqe;
722 qm->status.aeq_head = 0;
723 } else {
724 aeqe++;
725 qm->status.aeq_head++;
726 }
727
728 qm_db(qm, 0, QM_DOORBELL_CMD_AEQ, qm->status.aeq_head, 0);
729 }
730
731 return IRQ_HANDLED;
732 }
733
734 static void qm_irq_unregister(struct hisi_qm *qm)
735 {
736 struct pci_dev *pdev = qm->pdev;
737
738 free_irq(pci_irq_vector(pdev, QM_EQ_EVENT_IRQ_VECTOR), qm);
739
740 if (qm->ver == QM_HW_V1)
741 return;
742
743 free_irq(pci_irq_vector(pdev, QM_AEQ_EVENT_IRQ_VECTOR), qm);
744
745 if (qm->fun_type == QM_HW_PF)
746 free_irq(pci_irq_vector(pdev,
747 QM_ABNORMAL_EVENT_IRQ_VECTOR), qm);
748 }
749
750 static void qm_init_qp_status(struct hisi_qp *qp)
751 {
752 struct hisi_qp_status *qp_status = &qp->qp_status;
753
754 qp_status->sq_tail = 0;
755 qp_status->cq_head = 0;
756 qp_status->cqc_phase = true;
757 atomic_set(&qp_status->flags, 0);
758 }
759
760 static void qm_vft_data_cfg(struct hisi_qm *qm, enum vft_type type, u32 base,
761 u32 number)
762 {
763 u64 tmp = 0;
764
765 if (number > 0) {
766 switch (type) {
767 case SQC_VFT:
768 if (qm->ver == QM_HW_V1) {
769 tmp = QM_SQC_VFT_BUF_SIZE |
770 QM_SQC_VFT_SQC_SIZE |
771 QM_SQC_VFT_INDEX_NUMBER |
772 QM_SQC_VFT_VALID |
773 (u64)base << QM_SQC_VFT_START_SQN_SHIFT;
774 } else {
775 tmp = (u64)base << QM_SQC_VFT_START_SQN_SHIFT |
776 QM_SQC_VFT_VALID |
777 (u64)(number - 1) << QM_SQC_VFT_SQN_SHIFT;
778 }
779 break;
780 case CQC_VFT:
781 if (qm->ver == QM_HW_V1) {
782 tmp = QM_CQC_VFT_BUF_SIZE |
783 QM_CQC_VFT_SQC_SIZE |
784 QM_CQC_VFT_INDEX_NUMBER |
785 QM_CQC_VFT_VALID;
786 } else {
787 tmp = QM_CQC_VFT_VALID;
788 }
789 break;
790 }
791 }
792
793 writel(lower_32_bits(tmp), qm->io_base + QM_VFT_CFG_DATA_L);
794 writel(upper_32_bits(tmp), qm->io_base + QM_VFT_CFG_DATA_H);
795 }
796
797 static int qm_set_vft_common(struct hisi_qm *qm, enum vft_type type,
798 u32 fun_num, u32 base, u32 number)
799 {
800 unsigned int val;
801 int ret;
802
803 ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val,
804 val & BIT(0), 10, 1000);
805 if (ret)
806 return ret;
807
808 writel(0x0, qm->io_base + QM_VFT_CFG_OP_WR);
809 writel(type, qm->io_base + QM_VFT_CFG_TYPE);
810 writel(fun_num, qm->io_base + QM_VFT_CFG);
811
812 qm_vft_data_cfg(qm, type, base, number);
813
814 writel(0x0, qm->io_base + QM_VFT_CFG_RDY);
815 writel(0x1, qm->io_base + QM_VFT_CFG_OP_ENABLE);
816
817 return readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val,
818 val & BIT(0), 10, 1000);
819 }
820
821 /* The config should be conducted after qm_dev_mem_reset() */
822 static int qm_set_sqc_cqc_vft(struct hisi_qm *qm, u32 fun_num, u32 base,
823 u32 number)
824 {
825 int ret, i;
826
827 for (i = SQC_VFT; i <= CQC_VFT; i++) {
828 ret = qm_set_vft_common(qm, i, fun_num, base, number);
829 if (ret)
830 return ret;
831 }
832
833 return 0;
834 }
835
836 static int qm_get_vft_v2(struct hisi_qm *qm, u32 *base, u32 *number)
837 {
838 u64 sqc_vft;
839 int ret;
840
841 ret = qm_mb(qm, QM_MB_CMD_SQC_VFT_V2, 0, 0, 1);
842 if (ret)
843 return ret;
844
845 sqc_vft = readl(qm->io_base + QM_MB_CMD_DATA_ADDR_L) |
846 ((u64)readl(qm->io_base + QM_MB_CMD_DATA_ADDR_H) << 32);
847 *base = QM_SQC_VFT_BASE_MASK_V2 & (sqc_vft >> QM_SQC_VFT_BASE_SHIFT_V2);
848 *number = (QM_SQC_VFT_NUM_MASK_v2 &
849 (sqc_vft >> QM_SQC_VFT_NUM_SHIFT_V2)) + 1;
850
851 return 0;
852 }
853
854 static struct hisi_qm *file_to_qm(struct debugfs_file *file)
855 {
856 struct qm_debug *debug = file->debug;
857
858 return container_of(debug, struct hisi_qm, debug);
859 }
860
861 static u32 current_q_read(struct debugfs_file *file)
862 {
863 struct hisi_qm *qm = file_to_qm(file);
864
865 return readl(qm->io_base + QM_DFX_SQE_CNT_VF_SQN) >> QM_DFX_QN_SHIFT;
866 }
867
868 static int current_q_write(struct debugfs_file *file, u32 val)
869 {
870 struct hisi_qm *qm = file_to_qm(file);
871 u32 tmp;
872
873 if (val >= qm->debug.curr_qm_qp_num)
874 return -EINVAL;
875
876 tmp = val << QM_DFX_QN_SHIFT |
877 (readl(qm->io_base + QM_DFX_SQE_CNT_VF_SQN) & CURRENT_FUN_MASK);
878 writel(tmp, qm->io_base + QM_DFX_SQE_CNT_VF_SQN);
879
880 tmp = val << QM_DFX_QN_SHIFT |
881 (readl(qm->io_base + QM_DFX_CQE_CNT_VF_CQN) & CURRENT_FUN_MASK);
882 writel(tmp, qm->io_base + QM_DFX_CQE_CNT_VF_CQN);
883
884 return 0;
885 }
886
887 static u32 clear_enable_read(struct debugfs_file *file)
888 {
889 struct hisi_qm *qm = file_to_qm(file);
890
891 return readl(qm->io_base + QM_DFX_CNT_CLR_CE);
892 }
893
894 /* rd_clr_ctrl 1 enable read clear, otherwise 0 disable it */
895 static int clear_enable_write(struct debugfs_file *file, u32 rd_clr_ctrl)
896 {
897 struct hisi_qm *qm = file_to_qm(file);
898
899 if (rd_clr_ctrl > 1)
900 return -EINVAL;
901
902 writel(rd_clr_ctrl, qm->io_base + QM_DFX_CNT_CLR_CE);
903
904 return 0;
905 }
906
907 static ssize_t qm_debug_read(struct file *filp, char __user *buf,
908 size_t count, loff_t *pos)
909 {
910 struct debugfs_file *file = filp->private_data;
911 enum qm_debug_file index = file->index;
912 char tbuf[QM_DBG_TMP_BUF_LEN];
913 u32 val;
914 int ret;
915
916 mutex_lock(&file->lock);
917 switch (index) {
918 case CURRENT_Q:
919 val = current_q_read(file);
920 break;
921 case CLEAR_ENABLE:
922 val = clear_enable_read(file);
923 break;
924 default:
925 mutex_unlock(&file->lock);
926 return -EINVAL;
927 }
928 mutex_unlock(&file->lock);
929 ret = sprintf(tbuf, "%u\n", val);
930 return simple_read_from_buffer(buf, count, pos, tbuf, ret);
931 }
932
933 static ssize_t qm_debug_write(struct file *filp, const char __user *buf,
934 size_t count, loff_t *pos)
935 {
936 struct debugfs_file *file = filp->private_data;
937 enum qm_debug_file index = file->index;
938 unsigned long val;
939 char tbuf[QM_DBG_TMP_BUF_LEN];
940 int len, ret;
941
942 if (*pos != 0)
943 return 0;
944
945 if (count >= QM_DBG_TMP_BUF_LEN)
946 return -ENOSPC;
947
948 len = simple_write_to_buffer(tbuf, QM_DBG_TMP_BUF_LEN - 1, pos, buf,
949 count);
950 if (len < 0)
951 return len;
952
953 tbuf[len] = '\0';
954 if (kstrtoul(tbuf, 0, &val))
955 return -EFAULT;
956
957 mutex_lock(&file->lock);
958 switch (index) {
959 case CURRENT_Q:
960 ret = current_q_write(file, val);
961 if (ret)
962 goto err_input;
963 break;
964 case CLEAR_ENABLE:
965 ret = clear_enable_write(file, val);
966 if (ret)
967 goto err_input;
968 break;
969 default:
970 ret = -EINVAL;
971 goto err_input;
972 }
973 mutex_unlock(&file->lock);
974
975 return count;
976
977 err_input:
978 mutex_unlock(&file->lock);
979 return ret;
980 }
981
982 static const struct file_operations qm_debug_fops = {
983 .owner = THIS_MODULE,
984 .open = simple_open,
985 .read = qm_debug_read,
986 .write = qm_debug_write,
987 };
988
989 struct qm_dfx_registers {
990 char *reg_name;
991 u64 reg_offset;
992 };
993
994 #define CNT_CYC_REGS_NUM 10
995 static struct qm_dfx_registers qm_dfx_regs[] = {
996 /* XXX_CNT are reading clear register */
997 {"QM_ECC_1BIT_CNT ", 0x104000ull},
998 {"QM_ECC_MBIT_CNT ", 0x104008ull},
999 {"QM_DFX_MB_CNT ", 0x104018ull},
1000 {"QM_DFX_DB_CNT ", 0x104028ull},
1001 {"QM_DFX_SQE_CNT ", 0x104038ull},
1002 {"QM_DFX_CQE_CNT ", 0x104048ull},
1003 {"QM_DFX_SEND_SQE_TO_ACC_CNT ", 0x104050ull},
1004 {"QM_DFX_WB_SQE_FROM_ACC_CNT ", 0x104058ull},
1005 {"QM_DFX_ACC_FINISH_CNT ", 0x104060ull},
1006 {"QM_DFX_CQE_ERR_CNT ", 0x1040b4ull},
1007 {"QM_DFX_FUNS_ACTIVE_ST ", 0x200ull},
1008 {"QM_ECC_1BIT_INF ", 0x104004ull},
1009 {"QM_ECC_MBIT_INF ", 0x10400cull},
1010 {"QM_DFX_ACC_RDY_VLD0 ", 0x1040a0ull},
1011 {"QM_DFX_ACC_RDY_VLD1 ", 0x1040a4ull},
1012 {"QM_DFX_AXI_RDY_VLD ", 0x1040a8ull},
1013 {"QM_DFX_FF_ST0 ", 0x1040c8ull},
1014 {"QM_DFX_FF_ST1 ", 0x1040ccull},
1015 {"QM_DFX_FF_ST2 ", 0x1040d0ull},
1016 {"QM_DFX_FF_ST3 ", 0x1040d4ull},
1017 {"QM_DFX_FF_ST4 ", 0x1040d8ull},
1018 {"QM_DFX_FF_ST5 ", 0x1040dcull},
1019 {"QM_DFX_FF_ST6 ", 0x1040e0ull},
1020 {"QM_IN_IDLE_ST ", 0x1040e4ull},
1021 { NULL, 0}
1022 };
1023
1024 static struct qm_dfx_registers qm_vf_dfx_regs[] = {
1025 {"QM_DFX_FUNS_ACTIVE_ST ", 0x200ull},
1026 { NULL, 0}
1027 };
1028
1029 static int qm_regs_show(struct seq_file *s, void *unused)
1030 {
1031 struct hisi_qm *qm = s->private;
1032 struct qm_dfx_registers *regs;
1033 u32 val;
1034
1035 if (qm->fun_type == QM_HW_PF)
1036 regs = qm_dfx_regs;
1037 else
1038 regs = qm_vf_dfx_regs;
1039
1040 while (regs->reg_name) {
1041 val = readl(qm->io_base + regs->reg_offset);
1042 seq_printf(s, "%s= 0x%08x\n", regs->reg_name, val);
1043 regs++;
1044 }
1045
1046 return 0;
1047 }
1048
1049 static int qm_regs_open(struct inode *inode, struct file *file)
1050 {
1051 return single_open(file, qm_regs_show, inode->i_private);
1052 }
1053
1054 static const struct file_operations qm_regs_fops = {
1055 .owner = THIS_MODULE,
1056 .open = qm_regs_open,
1057 .read = seq_read,
1058 .release = single_release,
1059 };
1060
1061 static ssize_t qm_cmd_read(struct file *filp, char __user *buffer,
1062 size_t count, loff_t *pos)
1063 {
1064 char buf[QM_DBG_READ_LEN];
1065 int len;
1066
1067 len = scnprintf(buf, QM_DBG_READ_LEN, "%s\n",
1068 "Please echo help to cmd to get help information");
1069
1070 return simple_read_from_buffer(buffer, count, pos, buf, len);
1071 }
1072
1073 static void *qm_ctx_alloc(struct hisi_qm *qm, size_t ctx_size,
1074 dma_addr_t *dma_addr)
1075 {
1076 struct device *dev = &qm->pdev->dev;
1077 void *ctx_addr;
1078
1079 ctx_addr = kzalloc(ctx_size, GFP_KERNEL);
1080 if (!ctx_addr)
1081 return ERR_PTR(-ENOMEM);
1082
1083 *dma_addr = dma_map_single(dev, ctx_addr, ctx_size, DMA_FROM_DEVICE);
1084 if (dma_mapping_error(dev, *dma_addr)) {
1085 dev_err(dev, "DMA mapping error!\n");
1086 kfree(ctx_addr);
1087 return ERR_PTR(-ENOMEM);
1088 }
1089
1090 return ctx_addr;
1091 }
1092
1093 static void qm_ctx_free(struct hisi_qm *qm, size_t ctx_size,
1094 const void *ctx_addr, dma_addr_t *dma_addr)
1095 {
1096 struct device *dev = &qm->pdev->dev;
1097
1098 dma_unmap_single(dev, *dma_addr, ctx_size, DMA_FROM_DEVICE);
1099 kfree(ctx_addr);
1100 }
1101
1102 static int dump_show(struct hisi_qm *qm, void *info,
1103 unsigned int info_size, char *info_name)
1104 {
1105 struct device *dev = &qm->pdev->dev;
1106 u8 *info_buf, *info_curr = info;
1107 u32 i;
1108 #define BYTE_PER_DW 4
1109
1110 info_buf = kzalloc(info_size, GFP_KERNEL);
1111 if (!info_buf)
1112 return -ENOMEM;
1113
1114 for (i = 0; i < info_size; i++, info_curr++) {
1115 if (i % BYTE_PER_DW == 0)
1116 info_buf[i + 3UL] = *info_curr;
1117 else if (i % BYTE_PER_DW == 1)
1118 info_buf[i + 1UL] = *info_curr;
1119 else if (i % BYTE_PER_DW == 2)
1120 info_buf[i - 1] = *info_curr;
1121 else if (i % BYTE_PER_DW == 3)
1122 info_buf[i - 3] = *info_curr;
1123 }
1124
1125 dev_info(dev, "%s DUMP\n", info_name);
1126 for (i = 0; i < info_size; i += BYTE_PER_DW) {
1127 pr_info("DW%d: %02X%02X %02X%02X\n", i / BYTE_PER_DW,
1128 info_buf[i], info_buf[i + 1UL],
1129 info_buf[i + 2UL], info_buf[i + 3UL]);
1130 }
1131
1132 kfree(info_buf);
1133
1134 return 0;
1135 }
1136
1137 static int qm_dump_sqc_raw(struct hisi_qm *qm, dma_addr_t dma_addr, u16 qp_id)
1138 {
1139 return qm_mb(qm, QM_MB_CMD_SQC, dma_addr, qp_id, 1);
1140 }
1141
1142 static int qm_dump_cqc_raw(struct hisi_qm *qm, dma_addr_t dma_addr, u16 qp_id)
1143 {
1144 return qm_mb(qm, QM_MB_CMD_CQC, dma_addr, qp_id, 1);
1145 }
1146
1147 static int qm_sqc_dump(struct hisi_qm *qm, const char *s)
1148 {
1149 struct device *dev = &qm->pdev->dev;
1150 struct qm_sqc *sqc, *sqc_curr;
1151 dma_addr_t sqc_dma;
1152 u32 qp_id;
1153 int ret;
1154
1155 if (!s)
1156 return -EINVAL;
1157
1158 ret = kstrtou32(s, 0, &qp_id);
1159 if (ret || qp_id >= qm->qp_num) {
1160 dev_err(dev, "Please input qp num (0-%d)", qm->qp_num - 1);
1161 return -EINVAL;
1162 }
1163
1164 sqc = qm_ctx_alloc(qm, sizeof(*sqc), &sqc_dma);
1165 if (IS_ERR(sqc))
1166 return PTR_ERR(sqc);
1167
1168 ret = qm_dump_sqc_raw(qm, sqc_dma, qp_id);
1169 if (ret) {
1170 down_read(&qm->qps_lock);
1171 if (qm->sqc) {
1172 sqc_curr = qm->sqc + qp_id;
1173
1174 ret = dump_show(qm, sqc_curr, sizeof(*sqc),
1175 "SOFT SQC");
1176 if (ret)
1177 dev_info(dev, "Show soft sqc failed!\n");
1178 }
1179 up_read(&qm->qps_lock);
1180
1181 goto err_free_ctx;
1182 }
1183
1184 ret = dump_show(qm, sqc, sizeof(*sqc), "SQC");
1185 if (ret)
1186 dev_info(dev, "Show hw sqc failed!\n");
1187
1188 err_free_ctx:
1189 qm_ctx_free(qm, sizeof(*sqc), sqc, &sqc_dma);
1190 return ret;
1191 }
1192
1193 static int qm_cqc_dump(struct hisi_qm *qm, const char *s)
1194 {
1195 struct device *dev = &qm->pdev->dev;
1196 struct qm_cqc *cqc, *cqc_curr;
1197 dma_addr_t cqc_dma;
1198 u32 qp_id;
1199 int ret;
1200
1201 if (!s)
1202 return -EINVAL;
1203
1204 ret = kstrtou32(s, 0, &qp_id);
1205 if (ret || qp_id >= qm->qp_num) {
1206 dev_err(dev, "Please input qp num (0-%d)", qm->qp_num - 1);
1207 return -EINVAL;
1208 }
1209
1210 cqc = qm_ctx_alloc(qm, sizeof(*cqc), &cqc_dma);
1211 if (IS_ERR(cqc))
1212 return PTR_ERR(cqc);
1213
1214 ret = qm_dump_cqc_raw(qm, cqc_dma, qp_id);
1215 if (ret) {
1216 down_read(&qm->qps_lock);
1217 if (qm->cqc) {
1218 cqc_curr = qm->cqc + qp_id;
1219
1220 ret = dump_show(qm, cqc_curr, sizeof(*cqc),
1221 "SOFT CQC");
1222 if (ret)
1223 dev_info(dev, "Show soft cqc failed!\n");
1224 }
1225 up_read(&qm->qps_lock);
1226
1227 goto err_free_ctx;
1228 }
1229
1230 ret = dump_show(qm, cqc, sizeof(*cqc), "CQC");
1231 if (ret)
1232 dev_info(dev, "Show hw cqc failed!\n");
1233
1234 err_free_ctx:
1235 qm_ctx_free(qm, sizeof(*cqc), cqc, &cqc_dma);
1236 return ret;
1237 }
1238
1239 static int qm_eqc_aeqc_dump(struct hisi_qm *qm, char *s, size_t size,
1240 int cmd, char *name)
1241 {
1242 struct device *dev = &qm->pdev->dev;
1243 dma_addr_t xeqc_dma;
1244 void *xeqc;
1245 int ret;
1246
1247 if (strsep(&s, " ")) {
1248 dev_err(dev, "Please do not input extra characters!\n");
1249 return -EINVAL;
1250 }
1251
1252 xeqc = qm_ctx_alloc(qm, size, &xeqc_dma);
1253 if (IS_ERR(xeqc))
1254 return PTR_ERR(xeqc);
1255
1256 ret = qm_mb(qm, cmd, xeqc_dma, 0, 1);
1257 if (ret)
1258 goto err_free_ctx;
1259
1260 ret = dump_show(qm, xeqc, size, name);
1261 if (ret)
1262 dev_info(dev, "Show hw %s failed!\n", name);
1263
1264 err_free_ctx:
1265 qm_ctx_free(qm, size, xeqc, &xeqc_dma);
1266 return ret;
1267 }
1268
1269 static int q_dump_param_parse(struct hisi_qm *qm, char *s,
1270 u32 *e_id, u32 *q_id)
1271 {
1272 struct device *dev = &qm->pdev->dev;
1273 unsigned int qp_num = qm->qp_num;
1274 char *presult;
1275 int ret;
1276
1277 presult = strsep(&s, " ");
1278 if (!presult) {
1279 dev_err(dev, "Please input qp number!\n");
1280 return -EINVAL;
1281 }
1282
1283 ret = kstrtou32(presult, 0, q_id);
1284 if (ret || *q_id >= qp_num) {
1285 dev_err(dev, "Please input qp num (0-%d)", qp_num - 1);
1286 return -EINVAL;
1287 }
1288
1289 presult = strsep(&s, " ");
1290 if (!presult) {
1291 dev_err(dev, "Please input sqe number!\n");
1292 return -EINVAL;
1293 }
1294
1295 ret = kstrtou32(presult, 0, e_id);
1296 if (ret || *e_id >= QM_Q_DEPTH) {
1297 dev_err(dev, "Please input sqe num (0-%d)", QM_Q_DEPTH - 1);
1298 return -EINVAL;
1299 }
1300
1301 if (strsep(&s, " ")) {
1302 dev_err(dev, "Please do not input extra characters!\n");
1303 return -EINVAL;
1304 }
1305
1306 return 0;
1307 }
1308
1309 static int qm_sq_dump(struct hisi_qm *qm, char *s)
1310 {
1311 struct device *dev = &qm->pdev->dev;
1312 void *sqe, *sqe_curr;
1313 struct hisi_qp *qp;
1314 u32 qp_id, sqe_id;
1315 int ret;
1316
1317 ret = q_dump_param_parse(qm, s, &sqe_id, &qp_id);
1318 if (ret)
1319 return ret;
1320
1321 sqe = kzalloc(qm->sqe_size * QM_Q_DEPTH, GFP_KERNEL);
1322 if (!sqe)
1323 return -ENOMEM;
1324
1325 qp = &qm->qp_array[qp_id];
1326 memcpy(sqe, qp->sqe, qm->sqe_size * QM_Q_DEPTH);
1327 sqe_curr = sqe + (u32)(sqe_id * qm->sqe_size);
1328 memset(sqe_curr + qm->debug.sqe_mask_offset, QM_SQE_ADDR_MASK,
1329 qm->debug.sqe_mask_len);
1330
1331 ret = dump_show(qm, sqe_curr, qm->sqe_size, "SQE");
1332 if (ret)
1333 dev_info(dev, "Show sqe failed!\n");
1334
1335 kfree(sqe);
1336
1337 return ret;
1338 }
1339
1340 static int qm_cq_dump(struct hisi_qm *qm, char *s)
1341 {
1342 struct device *dev = &qm->pdev->dev;
1343 struct qm_cqe *cqe_curr;
1344 struct hisi_qp *qp;
1345 u32 qp_id, cqe_id;
1346 int ret;
1347
1348 ret = q_dump_param_parse(qm, s, &cqe_id, &qp_id);
1349 if (ret)
1350 return ret;
1351
1352 qp = &qm->qp_array[qp_id];
1353 cqe_curr = qp->cqe + cqe_id;
1354 ret = dump_show(qm, cqe_curr, sizeof(struct qm_cqe), "CQE");
1355 if (ret)
1356 dev_info(dev, "Show cqe failed!\n");
1357
1358 return ret;
1359 }
1360
1361 static int qm_eq_aeq_dump(struct hisi_qm *qm, const char *s,
1362 size_t size, char *name)
1363 {
1364 struct device *dev = &qm->pdev->dev;
1365 void *xeqe;
1366 u32 xeqe_id;
1367 int ret;
1368
1369 if (!s)
1370 return -EINVAL;
1371
1372 ret = kstrtou32(s, 0, &xeqe_id);
1373 if (ret || xeqe_id >= QM_Q_DEPTH) {
1374 dev_err(dev, "Please input aeqe num (0-%d)", QM_Q_DEPTH - 1);
1375 return -EINVAL;
1376 }
1377
1378 down_read(&qm->qps_lock);
1379
1380 if (qm->eqe && !strcmp(name, "EQE")) {
1381 xeqe = qm->eqe + xeqe_id;
1382 } else if (qm->aeqe && !strcmp(name, "AEQE")) {
1383 xeqe = qm->aeqe + xeqe_id;
1384 } else {
1385 ret = -EINVAL;
1386 goto err_unlock;
1387 }
1388
1389 ret = dump_show(qm, xeqe, size, name);
1390 if (ret)
1391 dev_info(dev, "Show %s failed!\n", name);
1392
1393 err_unlock:
1394 up_read(&qm->qps_lock);
1395 return ret;
1396 }
1397
1398 static int qm_dbg_help(struct hisi_qm *qm, char *s)
1399 {
1400 struct device *dev = &qm->pdev->dev;
1401
1402 if (strsep(&s, " ")) {
1403 dev_err(dev, "Please do not input extra characters!\n");
1404 return -EINVAL;
1405 }
1406
1407 dev_info(dev, "available commands:\n");
1408 dev_info(dev, "sqc <num>\n");
1409 dev_info(dev, "cqc <num>\n");
1410 dev_info(dev, "eqc\n");
1411 dev_info(dev, "aeqc\n");
1412 dev_info(dev, "sq <num> <e>\n");
1413 dev_info(dev, "cq <num> <e>\n");
1414 dev_info(dev, "eq <e>\n");
1415 dev_info(dev, "aeq <e>\n");
1416
1417 return 0;
1418 }
1419
1420 static int qm_cmd_write_dump(struct hisi_qm *qm, const char *cmd_buf)
1421 {
1422 struct device *dev = &qm->pdev->dev;
1423 char *presult, *s;
1424 int ret;
1425
1426 s = kstrdup(cmd_buf, GFP_KERNEL);
1427 if (!s)
1428 return -ENOMEM;
1429
1430 presult = strsep(&s, " ");
1431 if (!presult) {
1432 kfree(s);
1433 return -EINVAL;
1434 }
1435
1436 if (!strcmp(presult, "sqc"))
1437 ret = qm_sqc_dump(qm, s);
1438 else if (!strcmp(presult, "cqc"))
1439 ret = qm_cqc_dump(qm, s);
1440 else if (!strcmp(presult, "eqc"))
1441 ret = qm_eqc_aeqc_dump(qm, s, sizeof(struct qm_eqc),
1442 QM_MB_CMD_EQC, "EQC");
1443 else if (!strcmp(presult, "aeqc"))
1444 ret = qm_eqc_aeqc_dump(qm, s, sizeof(struct qm_aeqc),
1445 QM_MB_CMD_AEQC, "AEQC");
1446 else if (!strcmp(presult, "sq"))
1447 ret = qm_sq_dump(qm, s);
1448 else if (!strcmp(presult, "cq"))
1449 ret = qm_cq_dump(qm, s);
1450 else if (!strcmp(presult, "eq"))
1451 ret = qm_eq_aeq_dump(qm, s, sizeof(struct qm_eqe), "EQE");
1452 else if (!strcmp(presult, "aeq"))
1453 ret = qm_eq_aeq_dump(qm, s, sizeof(struct qm_aeqe), "AEQE");
1454 else if (!strcmp(presult, "help"))
1455 ret = qm_dbg_help(qm, s);
1456 else
1457 ret = -EINVAL;
1458
1459 if (ret)
1460 dev_info(dev, "Please echo help\n");
1461
1462 kfree(s);
1463
1464 return ret;
1465 }
1466
1467 static ssize_t qm_cmd_write(struct file *filp, const char __user *buffer,
1468 size_t count, loff_t *pos)
1469 {
1470 struct hisi_qm *qm = filp->private_data;
1471 char *cmd_buf, *cmd_buf_tmp;
1472 int ret;
1473
1474 if (*pos)
1475 return 0;
1476
1477 /* Judge if the instance is being reset. */
1478 if (unlikely(atomic_read(&qm->status.flags) == QM_STOP))
1479 return 0;
1480
1481 if (count > QM_DBG_WRITE_LEN)
1482 return -ENOSPC;
1483
1484 cmd_buf = kzalloc(count + 1, GFP_KERNEL);
1485 if (!cmd_buf)
1486 return -ENOMEM;
1487
1488 if (copy_from_user(cmd_buf, buffer, count)) {
1489 kfree(cmd_buf);
1490 return -EFAULT;
1491 }
1492
1493 cmd_buf[count] = '\0';
1494
1495 cmd_buf_tmp = strchr(cmd_buf, '\n');
1496 if (cmd_buf_tmp) {
1497 *cmd_buf_tmp = '\0';
1498 count = cmd_buf_tmp - cmd_buf + 1;
1499 }
1500
1501 ret = qm_cmd_write_dump(qm, cmd_buf);
1502 if (ret) {
1503 kfree(cmd_buf);
1504 return ret;
1505 }
1506
1507 kfree(cmd_buf);
1508
1509 return count;
1510 }
1511
1512 static const struct file_operations qm_cmd_fops = {
1513 .owner = THIS_MODULE,
1514 .open = simple_open,
1515 .read = qm_cmd_read,
1516 .write = qm_cmd_write,
1517 };
1518
1519 static int qm_create_debugfs_file(struct hisi_qm *qm, enum qm_debug_file index)
1520 {
1521 struct dentry *qm_d = qm->debug.qm_d;
1522 struct debugfs_file *file = qm->debug.files + index;
1523
1524 debugfs_create_file(qm_debug_file_name[index], 0600, qm_d, file,
1525 &qm_debug_fops);
1526
1527 file->index = index;
1528 mutex_init(&file->lock);
1529 file->debug = &qm->debug;
1530
1531 return 0;
1532 }
1533
1534 static void qm_hw_error_init_v1(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe)
1535 {
1536 writel(QM_ABNORMAL_INT_MASK_VALUE, qm->io_base + QM_ABNORMAL_INT_MASK);
1537 }
1538
1539 static void qm_hw_error_init_v2(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe)
1540 {
1541 u32 irq_enable = ce | nfe | fe;
1542 u32 irq_unmask = ~irq_enable;
1543
1544 qm->error_mask = ce | nfe | fe;
1545
1546 /* clear QM hw residual error source */
1547 writel(QM_ABNORMAL_INT_SOURCE_CLR,
1548 qm->io_base + QM_ABNORMAL_INT_SOURCE);
1549
1550 /* configure error type */
1551 writel(ce, qm->io_base + QM_RAS_CE_ENABLE);
1552 writel(QM_RAS_CE_TIMES_PER_IRQ, qm->io_base + QM_RAS_CE_THRESHOLD);
1553 writel(nfe, qm->io_base + QM_RAS_NFE_ENABLE);
1554 writel(fe, qm->io_base + QM_RAS_FE_ENABLE);
1555
1556 irq_unmask &= readl(qm->io_base + QM_ABNORMAL_INT_MASK);
1557 writel(irq_unmask, qm->io_base + QM_ABNORMAL_INT_MASK);
1558 }
1559
1560 static void qm_hw_error_uninit_v2(struct hisi_qm *qm)
1561 {
1562 writel(QM_ABNORMAL_INT_MASK_VALUE, qm->io_base + QM_ABNORMAL_INT_MASK);
1563 }
1564
1565 static void qm_log_hw_error(struct hisi_qm *qm, u32 error_status)
1566 {
1567 const struct hisi_qm_hw_error *err;
1568 struct device *dev = &qm->pdev->dev;
1569 u32 reg_val, type, vf_num;
1570 int i;
1571
1572 for (i = 0; i < ARRAY_SIZE(qm_hw_error); i++) {
1573 err = &qm_hw_error[i];
1574 if (!(err->int_msk & error_status))
1575 continue;
1576
1577 dev_err(dev, "%s [error status=0x%x] found\n",
1578 err->msg, err->int_msk);
1579
1580 if (err->int_msk & QM_DB_TIMEOUT) {
1581 reg_val = readl(qm->io_base + QM_ABNORMAL_INF01);
1582 type = (reg_val & QM_DB_TIMEOUT_TYPE) >>
1583 QM_DB_TIMEOUT_TYPE_SHIFT;
1584 vf_num = reg_val & QM_DB_TIMEOUT_VF;
1585 dev_err(dev, "qm %s doorbell timeout in function %u\n",
1586 qm_db_timeout[type], vf_num);
1587 } else if (err->int_msk & QM_OF_FIFO_OF) {
1588 reg_val = readl(qm->io_base + QM_ABNORMAL_INF00);
1589 type = (reg_val & QM_FIFO_OVERFLOW_TYPE) >>
1590 QM_FIFO_OVERFLOW_TYPE_SHIFT;
1591 vf_num = reg_val & QM_FIFO_OVERFLOW_VF;
1592
1593 if (type < ARRAY_SIZE(qm_fifo_overflow))
1594 dev_err(dev, "qm %s fifo overflow in function %u\n",
1595 qm_fifo_overflow[type], vf_num);
1596 else
1597 dev_err(dev, "unknown error type\n");
1598 }
1599 }
1600 }
1601
1602 static enum acc_err_result qm_hw_error_handle_v2(struct hisi_qm *qm)
1603 {
1604 u32 error_status, tmp;
1605
1606 /* read err sts */
1607 tmp = readl(qm->io_base + QM_ABNORMAL_INT_STATUS);
1608 error_status = qm->error_mask & tmp;
1609
1610 if (error_status) {
1611 if (error_status & QM_ECC_MBIT)
1612 qm->err_status.is_qm_ecc_mbit = true;
1613
1614 qm_log_hw_error(qm, error_status);
1615 if (error_status == QM_DB_RANDOM_INVALID) {
1616 writel(error_status, qm->io_base +
1617 QM_ABNORMAL_INT_SOURCE);
1618 return ACC_ERR_RECOVERED;
1619 }
1620
1621 return ACC_ERR_NEED_RESET;
1622 }
1623
1624 return ACC_ERR_RECOVERED;
1625 }
1626
1627 static const struct hisi_qm_hw_ops qm_hw_ops_v1 = {
1628 .qm_db = qm_db_v1,
1629 .get_irq_num = qm_get_irq_num_v1,
1630 .hw_error_init = qm_hw_error_init_v1,
1631 };
1632
1633 static const struct hisi_qm_hw_ops qm_hw_ops_v2 = {
1634 .get_vft = qm_get_vft_v2,
1635 .qm_db = qm_db_v2,
1636 .get_irq_num = qm_get_irq_num_v2,
1637 .hw_error_init = qm_hw_error_init_v2,
1638 .hw_error_uninit = qm_hw_error_uninit_v2,
1639 .hw_error_handle = qm_hw_error_handle_v2,
1640 };
1641
1642 static void *qm_get_avail_sqe(struct hisi_qp *qp)
1643 {
1644 struct hisi_qp_status *qp_status = &qp->qp_status;
1645 u16 sq_tail = qp_status->sq_tail;
1646
1647 if (unlikely(atomic_read(&qp->qp_status.used) == QM_Q_DEPTH))
1648 return NULL;
1649
1650 return qp->sqe + sq_tail * qp->qm->sqe_size;
1651 }
1652
1653 static struct hisi_qp *qm_create_qp_nolock(struct hisi_qm *qm, u8 alg_type)
1654 {
1655 struct device *dev = &qm->pdev->dev;
1656 struct hisi_qp *qp;
1657 int qp_id;
1658
1659 if (!qm_qp_avail_state(qm, NULL, QP_INIT))
1660 return ERR_PTR(-EPERM);
1661
1662 if (qm->qp_in_used == qm->qp_num) {
1663 dev_info_ratelimited(dev, "All %u queues of QM are busy!\n",
1664 qm->qp_num);
1665 atomic64_inc(&qm->debug.dfx.create_qp_err_cnt);
1666 return ERR_PTR(-EBUSY);
1667 }
1668
1669 qp_id = idr_alloc_cyclic(&qm->qp_idr, NULL, 0, qm->qp_num, GFP_ATOMIC);
1670 if (qp_id < 0) {
1671 dev_info_ratelimited(dev, "All %u queues of QM are busy!\n",
1672 qm->qp_num);
1673 atomic64_inc(&qm->debug.dfx.create_qp_err_cnt);
1674 return ERR_PTR(-EBUSY);
1675 }
1676
1677 qp = &qm->qp_array[qp_id];
1678
1679 memset(qp->cqe, 0, sizeof(struct qm_cqe) * QM_Q_DEPTH);
1680
1681 qp->event_cb = NULL;
1682 qp->req_cb = NULL;
1683 qp->qp_id = qp_id;
1684 qp->alg_type = alg_type;
1685 qm->qp_in_used++;
1686 atomic_set(&qp->qp_status.flags, QP_INIT);
1687
1688 return qp;
1689 }
1690
1691 /**
1692 * hisi_qm_create_qp() - Create a queue pair from qm.
1693 * @qm: The qm we create a qp from.
1694 * @alg_type: Accelerator specific algorithm type in sqc.
1695 *
1696 * return created qp, -EBUSY if all qps in qm allocated, -ENOMEM if allocating
1697 * qp memory fails.
1698 */
1699 struct hisi_qp *hisi_qm_create_qp(struct hisi_qm *qm, u8 alg_type)
1700 {
1701 struct hisi_qp *qp;
1702
1703 down_write(&qm->qps_lock);
1704 qp = qm_create_qp_nolock(qm, alg_type);
1705 up_write(&qm->qps_lock);
1706
1707 return qp;
1708 }
1709 EXPORT_SYMBOL_GPL(hisi_qm_create_qp);
1710
1711 /**
1712 * hisi_qm_release_qp() - Release a qp back to its qm.
1713 * @qp: The qp we want to release.
1714 *
1715 * This function releases the resource of a qp.
1716 */
1717 void hisi_qm_release_qp(struct hisi_qp *qp)
1718 {
1719 struct hisi_qm *qm = qp->qm;
1720
1721 down_write(&qm->qps_lock);
1722
1723 if (!qm_qp_avail_state(qm, qp, QP_CLOSE)) {
1724 up_write(&qm->qps_lock);
1725 return;
1726 }
1727
1728 qm->qp_in_used--;
1729 idr_remove(&qm->qp_idr, qp->qp_id);
1730
1731 up_write(&qm->qps_lock);
1732 }
1733 EXPORT_SYMBOL_GPL(hisi_qm_release_qp);
1734
1735 static int qm_qp_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid)
1736 {
1737 struct hisi_qm *qm = qp->qm;
1738 struct device *dev = &qm->pdev->dev;
1739 enum qm_hw_ver ver = qm->ver;
1740 struct qm_sqc *sqc;
1741 struct qm_cqc *cqc;
1742 dma_addr_t sqc_dma;
1743 dma_addr_t cqc_dma;
1744 int ret;
1745
1746 qm_init_qp_status(qp);
1747
1748 sqc = kzalloc(sizeof(struct qm_sqc), GFP_KERNEL);
1749 if (!sqc)
1750 return -ENOMEM;
1751 sqc_dma = dma_map_single(dev, sqc, sizeof(struct qm_sqc),
1752 DMA_TO_DEVICE);
1753 if (dma_mapping_error(dev, sqc_dma)) {
1754 kfree(sqc);
1755 return -ENOMEM;
1756 }
1757
1758 INIT_QC_COMMON(sqc, qp->sqe_dma, pasid);
1759 if (ver == QM_HW_V1) {
1760 sqc->dw3 = cpu_to_le32(QM_MK_SQC_DW3_V1(0, 0, 0, qm->sqe_size));
1761 sqc->w8 = cpu_to_le16(QM_Q_DEPTH - 1);
1762 } else {
1763 sqc->dw3 = cpu_to_le32(QM_MK_SQC_DW3_V2(qm->sqe_size));
1764 sqc->w8 = 0; /* rand_qc */
1765 }
1766 sqc->cq_num = cpu_to_le16(qp_id);
1767 sqc->w13 = cpu_to_le16(QM_MK_SQC_W13(0, 1, qp->alg_type));
1768
1769 ret = qm_mb(qm, QM_MB_CMD_SQC, sqc_dma, qp_id, 0);
1770 dma_unmap_single(dev, sqc_dma, sizeof(struct qm_sqc), DMA_TO_DEVICE);
1771 kfree(sqc);
1772 if (ret)
1773 return ret;
1774
1775 cqc = kzalloc(sizeof(struct qm_cqc), GFP_KERNEL);
1776 if (!cqc)
1777 return -ENOMEM;
1778 cqc_dma = dma_map_single(dev, cqc, sizeof(struct qm_cqc),
1779 DMA_TO_DEVICE);
1780 if (dma_mapping_error(dev, cqc_dma)) {
1781 kfree(cqc);
1782 return -ENOMEM;
1783 }
1784
1785 INIT_QC_COMMON(cqc, qp->cqe_dma, pasid);
1786 if (ver == QM_HW_V1) {
1787 cqc->dw3 = cpu_to_le32(QM_MK_CQC_DW3_V1(0, 0, 0, 4));
1788 cqc->w8 = cpu_to_le16(QM_Q_DEPTH - 1);
1789 } else {
1790 cqc->dw3 = cpu_to_le32(QM_MK_CQC_DW3_V2(4));
1791 cqc->w8 = 0;
1792 }
1793 cqc->dw6 = cpu_to_le32(1 << QM_CQ_PHASE_SHIFT | 1 << QM_CQ_FLAG_SHIFT);
1794
1795 ret = qm_mb(qm, QM_MB_CMD_CQC, cqc_dma, qp_id, 0);
1796 dma_unmap_single(dev, cqc_dma, sizeof(struct qm_cqc), DMA_TO_DEVICE);
1797 kfree(cqc);
1798
1799 return ret;
1800 }
1801
1802 static int qm_start_qp_nolock(struct hisi_qp *qp, unsigned long arg)
1803 {
1804 struct hisi_qm *qm = qp->qm;
1805 struct device *dev = &qm->pdev->dev;
1806 int qp_id = qp->qp_id;
1807 u32 pasid = arg;
1808 int ret;
1809
1810 if (!qm_qp_avail_state(qm, qp, QP_START))
1811 return -EPERM;
1812
1813 ret = qm_qp_ctx_cfg(qp, qp_id, pasid);
1814 if (ret)
1815 return ret;
1816
1817 atomic_set(&qp->qp_status.flags, QP_START);
1818 dev_dbg(dev, "queue %d started\n", qp_id);
1819
1820 return 0;
1821 }
1822
1823 /**
1824 * hisi_qm_start_qp() - Start a qp into running.
1825 * @qp: The qp we want to start to run.
1826 * @arg: Accelerator specific argument.
1827 *
1828 * After this function, qp can receive request from user. Return 0 if
1829 * successful, Return -EBUSY if failed.
1830 */
1831 int hisi_qm_start_qp(struct hisi_qp *qp, unsigned long arg)
1832 {
1833 struct hisi_qm *qm = qp->qm;
1834 int ret;
1835
1836 down_write(&qm->qps_lock);
1837 ret = qm_start_qp_nolock(qp, arg);
1838 up_write(&qm->qps_lock);
1839
1840 return ret;
1841 }
1842 EXPORT_SYMBOL_GPL(hisi_qm_start_qp);
1843
1844 /**
1845 * Determine whether the queue is cleared by judging the tail pointers of
1846 * sq and cq.
1847 */
1848 static int qm_drain_qp(struct hisi_qp *qp)
1849 {
1850 size_t size = sizeof(struct qm_sqc) + sizeof(struct qm_cqc);
1851 struct hisi_qm *qm = qp->qm;
1852 struct device *dev = &qm->pdev->dev;
1853 struct qm_sqc *sqc;
1854 struct qm_cqc *cqc;
1855 dma_addr_t dma_addr;
1856 int ret = 0, i = 0;
1857 void *addr;
1858
1859 /*
1860 * No need to judge if ECC multi-bit error occurs because the
1861 * master OOO will be blocked.
1862 */
1863 if (qm->err_status.is_qm_ecc_mbit || qm->err_status.is_dev_ecc_mbit)
1864 return 0;
1865
1866 addr = qm_ctx_alloc(qm, size, &dma_addr);
1867 if (IS_ERR(addr)) {
1868 dev_err(dev, "Failed to alloc ctx for sqc and cqc!\n");
1869 return -ENOMEM;
1870 }
1871
1872 while (++i) {
1873 ret = qm_dump_sqc_raw(qm, dma_addr, qp->qp_id);
1874 if (ret) {
1875 dev_err_ratelimited(dev, "Failed to dump sqc!\n");
1876 break;
1877 }
1878 sqc = addr;
1879
1880 ret = qm_dump_cqc_raw(qm, (dma_addr + sizeof(struct qm_sqc)),
1881 qp->qp_id);
1882 if (ret) {
1883 dev_err_ratelimited(dev, "Failed to dump cqc!\n");
1884 break;
1885 }
1886 cqc = addr + sizeof(struct qm_sqc);
1887
1888 if ((sqc->tail == cqc->tail) &&
1889 (QM_SQ_TAIL_IDX(sqc) == QM_CQ_TAIL_IDX(cqc)))
1890 break;
1891
1892 if (i == MAX_WAIT_COUNTS) {
1893 dev_err(dev, "Fail to empty queue %u!\n", qp->qp_id);
1894 ret = -EBUSY;
1895 break;
1896 }
1897
1898 usleep_range(WAIT_PERIOD_US_MIN, WAIT_PERIOD_US_MAX);
1899 }
1900
1901 qm_ctx_free(qm, size, addr, &dma_addr);
1902
1903 return ret;
1904 }
1905
1906 static int qm_stop_qp_nolock(struct hisi_qp *qp)
1907 {
1908 struct device *dev = &qp->qm->pdev->dev;
1909 int ret;
1910
1911 /*
1912 * It is allowed to stop and release qp when reset, If the qp is
1913 * stopped when reset but still want to be released then, the
1914 * is_resetting flag should be set negative so that this qp will not
1915 * be restarted after reset.
1916 */
1917 if (atomic_read(&qp->qp_status.flags) == QP_STOP) {
1918 qp->is_resetting = false;
1919 return 0;
1920 }
1921
1922 if (!qm_qp_avail_state(qp->qm, qp, QP_STOP))
1923 return -EPERM;
1924
1925 atomic_set(&qp->qp_status.flags, QP_STOP);
1926
1927 ret = qm_drain_qp(qp);
1928 if (ret)
1929 dev_err(dev, "Failed to drain out data for stopping!\n");
1930
1931 if (qp->qm->wq)
1932 flush_workqueue(qp->qm->wq);
1933 else
1934 flush_work(&qp->qm->work);
1935
1936 dev_dbg(dev, "stop queue %u!", qp->qp_id);
1937
1938 return 0;
1939 }
1940
1941 /**
1942 * hisi_qm_stop_qp() - Stop a qp in qm.
1943 * @qp: The qp we want to stop.
1944 *
1945 * This function is reverse of hisi_qm_start_qp. Return 0 if successful.
1946 */
1947 int hisi_qm_stop_qp(struct hisi_qp *qp)
1948 {
1949 int ret;
1950
1951 down_write(&qp->qm->qps_lock);
1952 ret = qm_stop_qp_nolock(qp);
1953 up_write(&qp->qm->qps_lock);
1954
1955 return ret;
1956 }
1957 EXPORT_SYMBOL_GPL(hisi_qm_stop_qp);
1958
1959 /**
1960 * hisi_qp_send() - Queue up a task in the hardware queue.
1961 * @qp: The qp in which to put the message.
1962 * @msg: The message.
1963 *
1964 * This function will return -EBUSY if qp is currently full, and -EAGAIN
1965 * if qp related qm is resetting.
1966 *
1967 * Note: This function may run with qm_irq_thread and ACC reset at same time.
1968 * It has no race with qm_irq_thread. However, during hisi_qp_send, ACC
1969 * reset may happen, we have no lock here considering performance. This
1970 * causes current qm_db sending fail or can not receive sended sqe. QM
1971 * sync/async receive function should handle the error sqe. ACC reset
1972 * done function should clear used sqe to 0.
1973 */
1974 int hisi_qp_send(struct hisi_qp *qp, const void *msg)
1975 {
1976 struct hisi_qp_status *qp_status = &qp->qp_status;
1977 u16 sq_tail = qp_status->sq_tail;
1978 u16 sq_tail_next = (sq_tail + 1) % QM_Q_DEPTH;
1979 void *sqe = qm_get_avail_sqe(qp);
1980
1981 if (unlikely(atomic_read(&qp->qp_status.flags) == QP_STOP ||
1982 atomic_read(&qp->qm->status.flags) == QM_STOP ||
1983 qp->is_resetting)) {
1984 dev_info(&qp->qm->pdev->dev, "QP is stopped or resetting\n");
1985 return -EAGAIN;
1986 }
1987
1988 if (!sqe)
1989 return -EBUSY;
1990
1991 memcpy(sqe, msg, qp->qm->sqe_size);
1992
1993 qm_db(qp->qm, qp->qp_id, QM_DOORBELL_CMD_SQ, sq_tail_next, 0);
1994 atomic_inc(&qp->qp_status.used);
1995 qp_status->sq_tail = sq_tail_next;
1996
1997 return 0;
1998 }
1999 EXPORT_SYMBOL_GPL(hisi_qp_send);
2000
2001 static void hisi_qm_cache_wb(struct hisi_qm *qm)
2002 {
2003 unsigned int val;
2004
2005 if (qm->ver == QM_HW_V1)
2006 return;
2007
2008 writel(0x1, qm->io_base + QM_CACHE_WB_START);
2009 if (readl_relaxed_poll_timeout(qm->io_base + QM_CACHE_WB_DONE,
2010 val, val & BIT(0), 10, 1000))
2011 dev_err(&qm->pdev->dev, "QM writeback sqc cache fail!\n");
2012 }
2013
2014 static void qm_qp_event_notifier(struct hisi_qp *qp)
2015 {
2016 wake_up_interruptible(&qp->uacce_q->wait);
2017 }
2018
2019 static int hisi_qm_get_available_instances(struct uacce_device *uacce)
2020 {
2021 return hisi_qm_get_free_qp_num(uacce->priv);
2022 }
2023
2024 static int hisi_qm_uacce_get_queue(struct uacce_device *uacce,
2025 unsigned long arg,
2026 struct uacce_queue *q)
2027 {
2028 struct hisi_qm *qm = uacce->priv;
2029 struct hisi_qp *qp;
2030 u8 alg_type = 0;
2031
2032 qp = hisi_qm_create_qp(qm, alg_type);
2033 if (IS_ERR(qp))
2034 return PTR_ERR(qp);
2035
2036 q->priv = qp;
2037 q->uacce = uacce;
2038 qp->uacce_q = q;
2039 qp->event_cb = qm_qp_event_notifier;
2040 qp->pasid = arg;
2041
2042 return 0;
2043 }
2044
2045 static void hisi_qm_uacce_put_queue(struct uacce_queue *q)
2046 {
2047 struct hisi_qp *qp = q->priv;
2048
2049 hisi_qm_cache_wb(qp->qm);
2050 hisi_qm_release_qp(qp);
2051 }
2052
2053 /* map sq/cq/doorbell to user space */
2054 static int hisi_qm_uacce_mmap(struct uacce_queue *q,
2055 struct vm_area_struct *vma,
2056 struct uacce_qfile_region *qfr)
2057 {
2058 struct hisi_qp *qp = q->priv;
2059 struct hisi_qm *qm = qp->qm;
2060 size_t sz = vma->vm_end - vma->vm_start;
2061 struct pci_dev *pdev = qm->pdev;
2062 struct device *dev = &pdev->dev;
2063 unsigned long vm_pgoff;
2064 int ret;
2065
2066 switch (qfr->type) {
2067 case UACCE_QFRT_MMIO:
2068 if (qm->ver == QM_HW_V1) {
2069 if (sz > PAGE_SIZE * QM_DOORBELL_PAGE_NR)
2070 return -EINVAL;
2071 } else {
2072 if (sz > PAGE_SIZE * (QM_DOORBELL_PAGE_NR +
2073 QM_DOORBELL_SQ_CQ_BASE_V2 / PAGE_SIZE))
2074 return -EINVAL;
2075 }
2076
2077 vma->vm_flags |= VM_IO;
2078
2079 return remap_pfn_range(vma, vma->vm_start,
2080 qm->phys_base >> PAGE_SHIFT,
2081 sz, pgprot_noncached(vma->vm_page_prot));
2082 case UACCE_QFRT_DUS:
2083 if (sz != qp->qdma.size)
2084 return -EINVAL;
2085
2086 /*
2087 * dma_mmap_coherent() requires vm_pgoff as 0
2088 * restore vm_pfoff to initial value for mmap()
2089 */
2090 vm_pgoff = vma->vm_pgoff;
2091 vma->vm_pgoff = 0;
2092 ret = dma_mmap_coherent(dev, vma, qp->qdma.va,
2093 qp->qdma.dma, sz);
2094 vma->vm_pgoff = vm_pgoff;
2095 return ret;
2096
2097 default:
2098 return -EINVAL;
2099 }
2100 }
2101
2102 static int hisi_qm_uacce_start_queue(struct uacce_queue *q)
2103 {
2104 struct hisi_qp *qp = q->priv;
2105
2106 return hisi_qm_start_qp(qp, qp->pasid);
2107 }
2108
2109 static void hisi_qm_uacce_stop_queue(struct uacce_queue *q)
2110 {
2111 hisi_qm_stop_qp(q->priv);
2112 }
2113
2114 static int qm_set_sqctype(struct uacce_queue *q, u16 type)
2115 {
2116 struct hisi_qm *qm = q->uacce->priv;
2117 struct hisi_qp *qp = q->priv;
2118
2119 down_write(&qm->qps_lock);
2120 qp->alg_type = type;
2121 up_write(&qm->qps_lock);
2122
2123 return 0;
2124 }
2125
2126 static long hisi_qm_uacce_ioctl(struct uacce_queue *q, unsigned int cmd,
2127 unsigned long arg)
2128 {
2129 struct hisi_qp *qp = q->priv;
2130 struct hisi_qp_ctx qp_ctx;
2131
2132 if (cmd == UACCE_CMD_QM_SET_QP_CTX) {
2133 if (copy_from_user(&qp_ctx, (void __user *)arg,
2134 sizeof(struct hisi_qp_ctx)))
2135 return -EFAULT;
2136
2137 if (qp_ctx.qc_type != 0 && qp_ctx.qc_type != 1)
2138 return -EINVAL;
2139
2140 qm_set_sqctype(q, qp_ctx.qc_type);
2141 qp_ctx.id = qp->qp_id;
2142
2143 if (copy_to_user((void __user *)arg, &qp_ctx,
2144 sizeof(struct hisi_qp_ctx)))
2145 return -EFAULT;
2146 } else {
2147 return -EINVAL;
2148 }
2149
2150 return 0;
2151 }
2152
2153 static const struct uacce_ops uacce_qm_ops = {
2154 .get_available_instances = hisi_qm_get_available_instances,
2155 .get_queue = hisi_qm_uacce_get_queue,
2156 .put_queue = hisi_qm_uacce_put_queue,
2157 .start_queue = hisi_qm_uacce_start_queue,
2158 .stop_queue = hisi_qm_uacce_stop_queue,
2159 .mmap = hisi_qm_uacce_mmap,
2160 .ioctl = hisi_qm_uacce_ioctl,
2161 };
2162
2163 static int qm_alloc_uacce(struct hisi_qm *qm)
2164 {
2165 struct pci_dev *pdev = qm->pdev;
2166 struct uacce_device *uacce;
2167 unsigned long mmio_page_nr;
2168 unsigned long dus_page_nr;
2169 struct uacce_interface interface = {
2170 .flags = UACCE_DEV_SVA,
2171 .ops = &uacce_qm_ops,
2172 };
2173 int ret;
2174
2175 ret = strscpy(interface.name, pdev->driver->name,
2176 sizeof(interface.name));
2177 if (ret < 0)
2178 return -ENAMETOOLONG;
2179
2180 uacce = uacce_alloc(&pdev->dev, &interface);
2181 if (IS_ERR(uacce))
2182 return PTR_ERR(uacce);
2183
2184 if (uacce->flags & UACCE_DEV_SVA) {
2185 qm->use_sva = true;
2186 } else {
2187 /* only consider sva case */
2188 uacce_remove(uacce);
2189 qm->uacce = NULL;
2190 return -EINVAL;
2191 }
2192
2193 uacce->is_vf = pdev->is_virtfn;
2194 uacce->priv = qm;
2195 uacce->algs = qm->algs;
2196
2197 if (qm->ver == QM_HW_V1) {
2198 mmio_page_nr = QM_DOORBELL_PAGE_NR;
2199 uacce->api_ver = HISI_QM_API_VER_BASE;
2200 } else {
2201 mmio_page_nr = QM_DOORBELL_PAGE_NR +
2202 QM_DOORBELL_SQ_CQ_BASE_V2 / PAGE_SIZE;
2203 uacce->api_ver = HISI_QM_API_VER2_BASE;
2204 }
2205
2206 dus_page_nr = (PAGE_SIZE - 1 + qm->sqe_size * QM_Q_DEPTH +
2207 sizeof(struct qm_cqe) * QM_Q_DEPTH) >> PAGE_SHIFT;
2208
2209 uacce->qf_pg_num[UACCE_QFRT_MMIO] = mmio_page_nr;
2210 uacce->qf_pg_num[UACCE_QFRT_DUS] = dus_page_nr;
2211
2212 qm->uacce = uacce;
2213
2214 return 0;
2215 }
2216
2217 /**
2218 * hisi_qm_get_free_qp_num() - Get free number of qp in qm.
2219 * @qm: The qm which want to get free qp.
2220 *
2221 * This function return free number of qp in qm.
2222 */
2223 int hisi_qm_get_free_qp_num(struct hisi_qm *qm)
2224 {
2225 int ret;
2226
2227 down_read(&qm->qps_lock);
2228 ret = qm->qp_num - qm->qp_in_used;
2229 up_read(&qm->qps_lock);
2230
2231 return ret;
2232 }
2233 EXPORT_SYMBOL_GPL(hisi_qm_get_free_qp_num);
2234
2235 static void hisi_qp_memory_uninit(struct hisi_qm *qm, int num)
2236 {
2237 struct device *dev = &qm->pdev->dev;
2238 struct qm_dma *qdma;
2239 int i;
2240
2241 for (i = num - 1; i >= 0; i--) {
2242 qdma = &qm->qp_array[i].qdma;
2243 dma_free_coherent(dev, qdma->size, qdma->va, qdma->dma);
2244 }
2245
2246 kfree(qm->qp_array);
2247 }
2248
2249 static int hisi_qp_memory_init(struct hisi_qm *qm, size_t dma_size, int id)
2250 {
2251 struct device *dev = &qm->pdev->dev;
2252 size_t off = qm->sqe_size * QM_Q_DEPTH;
2253 struct hisi_qp *qp;
2254
2255 qp = &qm->qp_array[id];
2256 qp->qdma.va = dma_alloc_coherent(dev, dma_size, &qp->qdma.dma,
2257 GFP_KERNEL);
2258 if (!qp->qdma.va)
2259 return -ENOMEM;
2260
2261 qp->sqe = qp->qdma.va;
2262 qp->sqe_dma = qp->qdma.dma;
2263 qp->cqe = qp->qdma.va + off;
2264 qp->cqe_dma = qp->qdma.dma + off;
2265 qp->qdma.size = dma_size;
2266 qp->qm = qm;
2267 qp->qp_id = id;
2268
2269 return 0;
2270 }
2271
2272 static int hisi_qm_memory_init(struct hisi_qm *qm)
2273 {
2274 struct device *dev = &qm->pdev->dev;
2275 size_t qp_dma_size, off = 0;
2276 int i, ret = 0;
2277
2278 #define QM_INIT_BUF(qm, type, num) do { \
2279 (qm)->type = ((qm)->qdma.va + (off)); \
2280 (qm)->type##_dma = (qm)->qdma.dma + (off); \
2281 off += QMC_ALIGN(sizeof(struct qm_##type) * (num)); \
2282 } while (0)
2283
2284 idr_init(&qm->qp_idr);
2285 qm->qdma.size = QMC_ALIGN(sizeof(struct qm_eqe) * QM_Q_DEPTH) +
2286 QMC_ALIGN(sizeof(struct qm_aeqe) * QM_Q_DEPTH) +
2287 QMC_ALIGN(sizeof(struct qm_sqc) * qm->qp_num) +
2288 QMC_ALIGN(sizeof(struct qm_cqc) * qm->qp_num);
2289 qm->qdma.va = dma_alloc_coherent(dev, qm->qdma.size, &qm->qdma.dma,
2290 GFP_ATOMIC);
2291 dev_dbg(dev, "allocate qm dma buf size=%zx)\n", qm->qdma.size);
2292 if (!qm->qdma.va)
2293 return -ENOMEM;
2294
2295 QM_INIT_BUF(qm, eqe, QM_Q_DEPTH);
2296 QM_INIT_BUF(qm, aeqe, QM_Q_DEPTH);
2297 QM_INIT_BUF(qm, sqc, qm->qp_num);
2298 QM_INIT_BUF(qm, cqc, qm->qp_num);
2299
2300 qm->qp_array = kcalloc(qm->qp_num, sizeof(struct hisi_qp), GFP_KERNEL);
2301 if (!qm->qp_array) {
2302 ret = -ENOMEM;
2303 goto err_alloc_qp_array;
2304 }
2305
2306 /* one more page for device or qp statuses */
2307 qp_dma_size = qm->sqe_size * QM_Q_DEPTH +
2308 sizeof(struct qm_cqe) * QM_Q_DEPTH;
2309 qp_dma_size = PAGE_ALIGN(qp_dma_size);
2310 for (i = 0; i < qm->qp_num; i++) {
2311 ret = hisi_qp_memory_init(qm, qp_dma_size, i);
2312 if (ret)
2313 goto err_init_qp_mem;
2314
2315 dev_dbg(dev, "allocate qp dma buf size=%zx)\n", qp_dma_size);
2316 }
2317
2318 return ret;
2319
2320 err_init_qp_mem:
2321 hisi_qp_memory_uninit(qm, i);
2322 err_alloc_qp_array:
2323 dma_free_coherent(dev, qm->qdma.size, qm->qdma.va, qm->qdma.dma);
2324
2325 return ret;
2326 }
2327
2328 static void hisi_qm_pre_init(struct hisi_qm *qm)
2329 {
2330 struct pci_dev *pdev = qm->pdev;
2331
2332 if (qm->ver == QM_HW_V1)
2333 qm->ops = &qm_hw_ops_v1;
2334 else
2335 qm->ops = &qm_hw_ops_v2;
2336
2337 pci_set_drvdata(pdev, qm);
2338 mutex_init(&qm->mailbox_lock);
2339 init_rwsem(&qm->qps_lock);
2340 qm->qp_in_used = 0;
2341 }
2342
2343 /**
2344 * hisi_qm_uninit() - Uninitialize qm.
2345 * @qm: The qm needed uninit.
2346 *
2347 * This function uninits qm related device resources.
2348 */
2349 void hisi_qm_uninit(struct hisi_qm *qm)
2350 {
2351 struct pci_dev *pdev = qm->pdev;
2352 struct device *dev = &pdev->dev;
2353
2354 down_write(&qm->qps_lock);
2355
2356 if (!qm_avail_state(qm, QM_CLOSE)) {
2357 up_write(&qm->qps_lock);
2358 return;
2359 }
2360
2361 uacce_remove(qm->uacce);
2362 qm->uacce = NULL;
2363
2364 hisi_qp_memory_uninit(qm, qm->qp_num);
2365 idr_destroy(&qm->qp_idr);
2366
2367 if (qm->qdma.va) {
2368 hisi_qm_cache_wb(qm);
2369 dma_free_coherent(dev, qm->qdma.size,
2370 qm->qdma.va, qm->qdma.dma);
2371 memset(&qm->qdma, 0, sizeof(qm->qdma));
2372 }
2373
2374 qm_irq_unregister(qm);
2375 pci_free_irq_vectors(pdev);
2376 iounmap(qm->io_base);
2377 pci_release_mem_regions(pdev);
2378 pci_disable_device(pdev);
2379
2380 up_write(&qm->qps_lock);
2381 }
2382 EXPORT_SYMBOL_GPL(hisi_qm_uninit);
2383
2384 /**
2385 * hisi_qm_get_vft() - Get vft from a qm.
2386 * @qm: The qm we want to get its vft.
2387 * @base: The base number of queue in vft.
2388 * @number: The number of queues in vft.
2389 *
2390 * We can allocate multiple queues to a qm by configuring virtual function
2391 * table. We get related configures by this function. Normally, we call this
2392 * function in VF driver to get the queue information.
2393 *
2394 * qm hw v1 does not support this interface.
2395 */
2396 int hisi_qm_get_vft(struct hisi_qm *qm, u32 *base, u32 *number)
2397 {
2398 if (!base || !number)
2399 return -EINVAL;
2400
2401 if (!qm->ops->get_vft) {
2402 dev_err(&qm->pdev->dev, "Don't support vft read!\n");
2403 return -EINVAL;
2404 }
2405
2406 return qm->ops->get_vft(qm, base, number);
2407 }
2408 EXPORT_SYMBOL_GPL(hisi_qm_get_vft);
2409
2410 /**
2411 * This function is alway called in PF driver, it is used to assign queues
2412 * among PF and VFs.
2413 *
2414 * Assign queues A~B to PF: hisi_qm_set_vft(qm, 0, A, B - A + 1)
2415 * Assign queues A~B to VF: hisi_qm_set_vft(qm, 2, A, B - A + 1)
2416 * (VF function number 0x2)
2417 */
2418 static int hisi_qm_set_vft(struct hisi_qm *qm, u32 fun_num, u32 base,
2419 u32 number)
2420 {
2421 u32 max_q_num = qm->ctrl_qp_num;
2422
2423 if (base >= max_q_num || number > max_q_num ||
2424 (base + number) > max_q_num)
2425 return -EINVAL;
2426
2427 return qm_set_sqc_cqc_vft(qm, fun_num, base, number);
2428 }
2429
2430 static void qm_init_eq_aeq_status(struct hisi_qm *qm)
2431 {
2432 struct hisi_qm_status *status = &qm->status;
2433
2434 status->eq_head = 0;
2435 status->aeq_head = 0;
2436 status->eqc_phase = true;
2437 status->aeqc_phase = true;
2438 }
2439
2440 static int qm_eq_ctx_cfg(struct hisi_qm *qm)
2441 {
2442 struct device *dev = &qm->pdev->dev;
2443 struct qm_eqc *eqc;
2444 struct qm_aeqc *aeqc;
2445 dma_addr_t eqc_dma;
2446 dma_addr_t aeqc_dma;
2447 int ret;
2448
2449 qm_init_eq_aeq_status(qm);
2450
2451 eqc = kzalloc(sizeof(struct qm_eqc), GFP_KERNEL);
2452 if (!eqc)
2453 return -ENOMEM;
2454 eqc_dma = dma_map_single(dev, eqc, sizeof(struct qm_eqc),
2455 DMA_TO_DEVICE);
2456 if (dma_mapping_error(dev, eqc_dma)) {
2457 kfree(eqc);
2458 return -ENOMEM;
2459 }
2460
2461 eqc->base_l = cpu_to_le32(lower_32_bits(qm->eqe_dma));
2462 eqc->base_h = cpu_to_le32(upper_32_bits(qm->eqe_dma));
2463 if (qm->ver == QM_HW_V1)
2464 eqc->dw3 = cpu_to_le32(QM_EQE_AEQE_SIZE);
2465 eqc->dw6 = cpu_to_le32((QM_Q_DEPTH - 1) | (1 << QM_EQC_PHASE_SHIFT));
2466 ret = qm_mb(qm, QM_MB_CMD_EQC, eqc_dma, 0, 0);
2467 dma_unmap_single(dev, eqc_dma, sizeof(struct qm_eqc), DMA_TO_DEVICE);
2468 kfree(eqc);
2469 if (ret)
2470 return ret;
2471
2472 aeqc = kzalloc(sizeof(struct qm_aeqc), GFP_KERNEL);
2473 if (!aeqc)
2474 return -ENOMEM;
2475 aeqc_dma = dma_map_single(dev, aeqc, sizeof(struct qm_aeqc),
2476 DMA_TO_DEVICE);
2477 if (dma_mapping_error(dev, aeqc_dma)) {
2478 kfree(aeqc);
2479 return -ENOMEM;
2480 }
2481
2482 aeqc->base_l = cpu_to_le32(lower_32_bits(qm->aeqe_dma));
2483 aeqc->base_h = cpu_to_le32(upper_32_bits(qm->aeqe_dma));
2484 aeqc->dw6 = cpu_to_le32((QM_Q_DEPTH - 1) | (1 << QM_EQC_PHASE_SHIFT));
2485
2486 ret = qm_mb(qm, QM_MB_CMD_AEQC, aeqc_dma, 0, 0);
2487 dma_unmap_single(dev, aeqc_dma, sizeof(struct qm_aeqc), DMA_TO_DEVICE);
2488 kfree(aeqc);
2489
2490 return ret;
2491 }
2492
2493 static int __hisi_qm_start(struct hisi_qm *qm)
2494 {
2495 int ret;
2496
2497 WARN_ON(!qm->qdma.dma);
2498
2499 if (qm->fun_type == QM_HW_PF) {
2500 ret = qm_dev_mem_reset(qm);
2501 if (ret)
2502 return ret;
2503
2504 ret = hisi_qm_set_vft(qm, 0, qm->qp_base, qm->qp_num);
2505 if (ret)
2506 return ret;
2507 }
2508
2509 ret = qm_eq_ctx_cfg(qm);
2510 if (ret)
2511 return ret;
2512
2513 ret = qm_mb(qm, QM_MB_CMD_SQC_BT, qm->sqc_dma, 0, 0);
2514 if (ret)
2515 return ret;
2516
2517 ret = qm_mb(qm, QM_MB_CMD_CQC_BT, qm->cqc_dma, 0, 0);
2518 if (ret)
2519 return ret;
2520
2521 writel(0x0, qm->io_base + QM_VF_EQ_INT_MASK);
2522 writel(0x0, qm->io_base + QM_VF_AEQ_INT_MASK);
2523
2524 return 0;
2525 }
2526
2527 /**
2528 * hisi_qm_start() - start qm
2529 * @qm: The qm to be started.
2530 *
2531 * This function starts a qm, then we can allocate qp from this qm.
2532 */
2533 int hisi_qm_start(struct hisi_qm *qm)
2534 {
2535 struct device *dev = &qm->pdev->dev;
2536 int ret = 0;
2537
2538 down_write(&qm->qps_lock);
2539
2540 if (!qm_avail_state(qm, QM_START)) {
2541 up_write(&qm->qps_lock);
2542 return -EPERM;
2543 }
2544
2545 dev_dbg(dev, "qm start with %d queue pairs\n", qm->qp_num);
2546
2547 if (!qm->qp_num) {
2548 dev_err(dev, "qp_num should not be 0\n");
2549 ret = -EINVAL;
2550 goto err_unlock;
2551 }
2552
2553 ret = __hisi_qm_start(qm);
2554 if (!ret)
2555 atomic_set(&qm->status.flags, QM_START);
2556
2557 err_unlock:
2558 up_write(&qm->qps_lock);
2559 return ret;
2560 }
2561 EXPORT_SYMBOL_GPL(hisi_qm_start);
2562
2563 static int qm_restart(struct hisi_qm *qm)
2564 {
2565 struct device *dev = &qm->pdev->dev;
2566 struct hisi_qp *qp;
2567 int ret, i;
2568
2569 ret = hisi_qm_start(qm);
2570 if (ret < 0)
2571 return ret;
2572
2573 down_write(&qm->qps_lock);
2574 for (i = 0; i < qm->qp_num; i++) {
2575 qp = &qm->qp_array[i];
2576 if (atomic_read(&qp->qp_status.flags) == QP_STOP &&
2577 qp->is_resetting == true) {
2578 ret = qm_start_qp_nolock(qp, 0);
2579 if (ret < 0) {
2580 dev_err(dev, "Failed to start qp%d!\n", i);
2581
2582 up_write(&qm->qps_lock);
2583 return ret;
2584 }
2585 qp->is_resetting = false;
2586 }
2587 }
2588 up_write(&qm->qps_lock);
2589
2590 return 0;
2591 }
2592
2593 /* Stop started qps in reset flow */
2594 static int qm_stop_started_qp(struct hisi_qm *qm)
2595 {
2596 struct device *dev = &qm->pdev->dev;
2597 struct hisi_qp *qp;
2598 int i, ret;
2599
2600 for (i = 0; i < qm->qp_num; i++) {
2601 qp = &qm->qp_array[i];
2602 if (qp && atomic_read(&qp->qp_status.flags) == QP_START) {
2603 qp->is_resetting = true;
2604 ret = qm_stop_qp_nolock(qp);
2605 if (ret < 0) {
2606 dev_err(dev, "Failed to stop qp%d!\n", i);
2607 return ret;
2608 }
2609 }
2610 }
2611
2612 return 0;
2613 }
2614
2615 /**
2616 * This function clears all queues memory in a qm. Reset of accelerator can
2617 * use this to clear queues.
2618 */
2619 static void qm_clear_queues(struct hisi_qm *qm)
2620 {
2621 struct hisi_qp *qp;
2622 int i;
2623
2624 for (i = 0; i < qm->qp_num; i++) {
2625 qp = &qm->qp_array[i];
2626 if (qp->is_resetting)
2627 memset(qp->qdma.va, 0, qp->qdma.size);
2628 }
2629
2630 memset(qm->qdma.va, 0, qm->qdma.size);
2631 }
2632
2633 /**
2634 * hisi_qm_stop() - Stop a qm.
2635 * @qm: The qm which will be stopped.
2636 *
2637 * This function stops qm and its qps, then qm can not accept request.
2638 * Related resources are not released at this state, we can use hisi_qm_start
2639 * to let qm start again.
2640 */
2641 int hisi_qm_stop(struct hisi_qm *qm)
2642 {
2643 struct device *dev = &qm->pdev->dev;
2644 int ret = 0;
2645
2646 down_write(&qm->qps_lock);
2647
2648 if (!qm_avail_state(qm, QM_STOP)) {
2649 ret = -EPERM;
2650 goto err_unlock;
2651 }
2652
2653 if (qm->status.stop_reason == QM_SOFT_RESET ||
2654 qm->status.stop_reason == QM_FLR) {
2655 ret = qm_stop_started_qp(qm);
2656 if (ret < 0) {
2657 dev_err(dev, "Failed to stop started qp!\n");
2658 goto err_unlock;
2659 }
2660 }
2661
2662 /* Mask eq and aeq irq */
2663 writel(0x1, qm->io_base + QM_VF_EQ_INT_MASK);
2664 writel(0x1, qm->io_base + QM_VF_AEQ_INT_MASK);
2665
2666 if (qm->fun_type == QM_HW_PF) {
2667 ret = hisi_qm_set_vft(qm, 0, 0, 0);
2668 if (ret < 0) {
2669 dev_err(dev, "Failed to set vft!\n");
2670 ret = -EBUSY;
2671 goto err_unlock;
2672 }
2673 }
2674
2675 qm_clear_queues(qm);
2676 atomic_set(&qm->status.flags, QM_STOP);
2677
2678 err_unlock:
2679 up_write(&qm->qps_lock);
2680 return ret;
2681 }
2682 EXPORT_SYMBOL_GPL(hisi_qm_stop);
2683
2684 static ssize_t qm_status_read(struct file *filp, char __user *buffer,
2685 size_t count, loff_t *pos)
2686 {
2687 struct hisi_qm *qm = filp->private_data;
2688 char buf[QM_DBG_READ_LEN];
2689 int val, len;
2690
2691 val = atomic_read(&qm->status.flags);
2692 len = scnprintf(buf, QM_DBG_READ_LEN, "%s\n", qm_s[val]);
2693
2694 return simple_read_from_buffer(buffer, count, pos, buf, len);
2695 }
2696
2697 static const struct file_operations qm_status_fops = {
2698 .owner = THIS_MODULE,
2699 .open = simple_open,
2700 .read = qm_status_read,
2701 };
2702
2703 static int qm_debugfs_atomic64_set(void *data, u64 val)
2704 {
2705 if (val)
2706 return -EINVAL;
2707
2708 atomic64_set((atomic64_t *)data, 0);
2709
2710 return 0;
2711 }
2712
2713 static int qm_debugfs_atomic64_get(void *data, u64 *val)
2714 {
2715 *val = atomic64_read((atomic64_t *)data);
2716
2717 return 0;
2718 }
2719
2720 DEFINE_DEBUGFS_ATTRIBUTE(qm_atomic64_ops, qm_debugfs_atomic64_get,
2721 qm_debugfs_atomic64_set, "%llu\n");
2722
2723 /**
2724 * hisi_qm_debug_init() - Initialize qm related debugfs files.
2725 * @qm: The qm for which we want to add debugfs files.
2726 *
2727 * Create qm related debugfs files.
2728 */
2729 int hisi_qm_debug_init(struct hisi_qm *qm)
2730 {
2731 struct qm_dfx *dfx = &qm->debug.dfx;
2732 struct dentry *qm_d;
2733 void *data;
2734 int i, ret;
2735
2736 qm_d = debugfs_create_dir("qm", qm->debug.debug_root);
2737 qm->debug.qm_d = qm_d;
2738
2739 /* only show this in PF */
2740 if (qm->fun_type == QM_HW_PF)
2741 for (i = CURRENT_Q; i < DEBUG_FILE_NUM; i++)
2742 if (qm_create_debugfs_file(qm, i)) {
2743 ret = -ENOENT;
2744 goto failed_to_create;
2745 }
2746
2747 debugfs_create_file("regs", 0444, qm->debug.qm_d, qm, &qm_regs_fops);
2748
2749 debugfs_create_file("cmd", 0444, qm->debug.qm_d, qm, &qm_cmd_fops);
2750
2751 debugfs_create_file("status", 0444, qm->debug.qm_d, qm,
2752 &qm_status_fops);
2753 for (i = 0; i < ARRAY_SIZE(qm_dfx_files); i++) {
2754 data = (atomic64_t *)((uintptr_t)dfx + qm_dfx_files[i].offset);
2755 debugfs_create_file(qm_dfx_files[i].name,
2756 0644,
2757 qm_d,
2758 data,
2759 &qm_atomic64_ops);
2760 }
2761
2762 return 0;
2763
2764 failed_to_create:
2765 debugfs_remove_recursive(qm_d);
2766 return ret;
2767 }
2768 EXPORT_SYMBOL_GPL(hisi_qm_debug_init);
2769
2770 /**
2771 * hisi_qm_debug_regs_clear() - clear qm debug related registers.
2772 * @qm: The qm for which we want to clear its debug registers.
2773 */
2774 void hisi_qm_debug_regs_clear(struct hisi_qm *qm)
2775 {
2776 struct qm_dfx_registers *regs;
2777 int i;
2778
2779 /* clear current_q */
2780 writel(0x0, qm->io_base + QM_DFX_SQE_CNT_VF_SQN);
2781 writel(0x0, qm->io_base + QM_DFX_CQE_CNT_VF_CQN);
2782
2783 /*
2784 * these registers are reading and clearing, so clear them after
2785 * reading them.
2786 */
2787 writel(0x1, qm->io_base + QM_DFX_CNT_CLR_CE);
2788
2789 regs = qm_dfx_regs;
2790 for (i = 0; i < CNT_CYC_REGS_NUM; i++) {
2791 readl(qm->io_base + regs->reg_offset);
2792 regs++;
2793 }
2794
2795 writel(0x0, qm->io_base + QM_DFX_CNT_CLR_CE);
2796 }
2797 EXPORT_SYMBOL_GPL(hisi_qm_debug_regs_clear);
2798
2799 static void qm_hw_error_init(struct hisi_qm *qm)
2800 {
2801 const struct hisi_qm_err_info *err_info = &qm->err_ini->err_info;
2802
2803 if (!qm->ops->hw_error_init) {
2804 dev_err(&qm->pdev->dev, "QM doesn't support hw error handling!\n");
2805 return;
2806 }
2807
2808 qm->ops->hw_error_init(qm, err_info->ce, err_info->nfe, err_info->fe);
2809 }
2810
2811 static void qm_hw_error_uninit(struct hisi_qm *qm)
2812 {
2813 if (!qm->ops->hw_error_uninit) {
2814 dev_err(&qm->pdev->dev, "Unexpected QM hw error uninit!\n");
2815 return;
2816 }
2817
2818 qm->ops->hw_error_uninit(qm);
2819 }
2820
2821 static enum acc_err_result qm_hw_error_handle(struct hisi_qm *qm)
2822 {
2823 if (!qm->ops->hw_error_handle) {
2824 dev_err(&qm->pdev->dev, "QM doesn't support hw error report!\n");
2825 return ACC_ERR_NONE;
2826 }
2827
2828 return qm->ops->hw_error_handle(qm);
2829 }
2830
2831 /**
2832 * hisi_qm_dev_err_init() - Initialize device error configuration.
2833 * @qm: The qm for which we want to do error initialization.
2834 *
2835 * Initialize QM and device error related configuration.
2836 */
2837 void hisi_qm_dev_err_init(struct hisi_qm *qm)
2838 {
2839 if (qm->fun_type == QM_HW_VF)
2840 return;
2841
2842 qm_hw_error_init(qm);
2843
2844 if (!qm->err_ini->hw_err_enable) {
2845 dev_err(&qm->pdev->dev, "Device doesn't support hw error init!\n");
2846 return;
2847 }
2848 qm->err_ini->hw_err_enable(qm);
2849 }
2850 EXPORT_SYMBOL_GPL(hisi_qm_dev_err_init);
2851
2852 /**
2853 * hisi_qm_dev_err_uninit() - Uninitialize device error configuration.
2854 * @qm: The qm for which we want to do error uninitialization.
2855 *
2856 * Uninitialize QM and device error related configuration.
2857 */
2858 void hisi_qm_dev_err_uninit(struct hisi_qm *qm)
2859 {
2860 if (qm->fun_type == QM_HW_VF)
2861 return;
2862
2863 qm_hw_error_uninit(qm);
2864
2865 if (!qm->err_ini->hw_err_disable) {
2866 dev_err(&qm->pdev->dev, "Unexpected device hw error uninit!\n");
2867 return;
2868 }
2869 qm->err_ini->hw_err_disable(qm);
2870 }
2871 EXPORT_SYMBOL_GPL(hisi_qm_dev_err_uninit);
2872
2873 /**
2874 * hisi_qm_free_qps() - free multiple queue pairs.
2875 * @qps: The queue pairs need to be freed.
2876 * @qp_num: The num of queue pairs.
2877 */
2878 void hisi_qm_free_qps(struct hisi_qp **qps, int qp_num)
2879 {
2880 int i;
2881
2882 if (!qps || qp_num <= 0)
2883 return;
2884
2885 for (i = qp_num - 1; i >= 0; i--)
2886 hisi_qm_release_qp(qps[i]);
2887 }
2888 EXPORT_SYMBOL_GPL(hisi_qm_free_qps);
2889
2890 static void free_list(struct list_head *head)
2891 {
2892 struct hisi_qm_resource *res, *tmp;
2893
2894 list_for_each_entry_safe(res, tmp, head, list) {
2895 list_del(&res->list);
2896 kfree(res);
2897 }
2898 }
2899
2900 static int hisi_qm_sort_devices(int node, struct list_head *head,
2901 struct hisi_qm_list *qm_list)
2902 {
2903 struct hisi_qm_resource *res, *tmp;
2904 struct hisi_qm *qm;
2905 struct list_head *n;
2906 struct device *dev;
2907 int dev_node = 0;
2908
2909 list_for_each_entry(qm, &qm_list->list, list) {
2910 dev = &qm->pdev->dev;
2911
2912 if (IS_ENABLED(CONFIG_NUMA)) {
2913 dev_node = dev_to_node(dev);
2914 if (dev_node < 0)
2915 dev_node = 0;
2916 }
2917
2918 res = kzalloc(sizeof(*res), GFP_KERNEL);
2919 if (!res)
2920 return -ENOMEM;
2921
2922 res->qm = qm;
2923 res->distance = node_distance(dev_node, node);
2924 n = head;
2925 list_for_each_entry(tmp, head, list) {
2926 if (res->distance < tmp->distance) {
2927 n = &tmp->list;
2928 break;
2929 }
2930 }
2931 list_add_tail(&res->list, n);
2932 }
2933
2934 return 0;
2935 }
2936
2937 /**
2938 * hisi_qm_alloc_qps_node() - Create multiple queue pairs.
2939 * @qm_list: The list of all available devices.
2940 * @qp_num: The number of queue pairs need created.
2941 * @alg_type: The algorithm type.
2942 * @node: The numa node.
2943 * @qps: The queue pairs need created.
2944 *
2945 * This function will sort all available device according to numa distance.
2946 * Then try to create all queue pairs from one device, if all devices do
2947 * not meet the requirements will return error.
2948 */
2949 int hisi_qm_alloc_qps_node(struct hisi_qm_list *qm_list, int qp_num,
2950 u8 alg_type, int node, struct hisi_qp **qps)
2951 {
2952 struct hisi_qm_resource *tmp;
2953 int ret = -ENODEV;
2954 LIST_HEAD(head);
2955 int i;
2956
2957 if (!qps || !qm_list || qp_num <= 0)
2958 return -EINVAL;
2959
2960 mutex_lock(&qm_list->lock);
2961 if (hisi_qm_sort_devices(node, &head, qm_list)) {
2962 mutex_unlock(&qm_list->lock);
2963 goto err;
2964 }
2965
2966 list_for_each_entry(tmp, &head, list) {
2967 for (i = 0; i < qp_num; i++) {
2968 qps[i] = hisi_qm_create_qp(tmp->qm, alg_type);
2969 if (IS_ERR(qps[i])) {
2970 hisi_qm_free_qps(qps, i);
2971 break;
2972 }
2973 }
2974
2975 if (i == qp_num) {
2976 ret = 0;
2977 break;
2978 }
2979 }
2980
2981 mutex_unlock(&qm_list->lock);
2982 if (ret)
2983 pr_info("Failed to create qps, node[%d], alg[%d], qp[%d]!\n",
2984 node, alg_type, qp_num);
2985
2986 err:
2987 free_list(&head);
2988 return ret;
2989 }
2990 EXPORT_SYMBOL_GPL(hisi_qm_alloc_qps_node);
2991
2992 static int qm_vf_q_assign(struct hisi_qm *qm, u32 num_vfs)
2993 {
2994 u32 remain_q_num, q_num, i, j;
2995 u32 q_base = qm->qp_num;
2996 int ret;
2997
2998 if (!num_vfs)
2999 return -EINVAL;
3000
3001 remain_q_num = qm->ctrl_qp_num - qm->qp_num;
3002
3003 /* If remain queues not enough, return error. */
3004 if (qm->ctrl_qp_num < qm->qp_num || remain_q_num < num_vfs)
3005 return -EINVAL;
3006
3007 q_num = remain_q_num / num_vfs;
3008 for (i = 1; i <= num_vfs; i++) {
3009 if (i == num_vfs)
3010 q_num += remain_q_num % num_vfs;
3011 ret = hisi_qm_set_vft(qm, i, q_base, q_num);
3012 if (ret) {
3013 for (j = i; j > 0; j--)
3014 hisi_qm_set_vft(qm, j, 0, 0);
3015 return ret;
3016 }
3017 q_base += q_num;
3018 }
3019
3020 return 0;
3021 }
3022
3023 static int qm_clear_vft_config(struct hisi_qm *qm)
3024 {
3025 int ret;
3026 u32 i;
3027
3028 for (i = 1; i <= qm->vfs_num; i++) {
3029 ret = hisi_qm_set_vft(qm, i, 0, 0);
3030 if (ret)
3031 return ret;
3032 }
3033 qm->vfs_num = 0;
3034
3035 return 0;
3036 }
3037
3038 /**
3039 * hisi_qm_sriov_enable() - enable virtual functions
3040 * @pdev: the PCIe device
3041 * @max_vfs: the number of virtual functions to enable
3042 *
3043 * Returns the number of enabled VFs. If there are VFs enabled already or
3044 * max_vfs is more than the total number of device can be enabled, returns
3045 * failure.
3046 */
3047 int hisi_qm_sriov_enable(struct pci_dev *pdev, int max_vfs)
3048 {
3049 struct hisi_qm *qm = pci_get_drvdata(pdev);
3050 int pre_existing_vfs, num_vfs, total_vfs, ret;
3051
3052 total_vfs = pci_sriov_get_totalvfs(pdev);
3053 pre_existing_vfs = pci_num_vf(pdev);
3054 if (pre_existing_vfs) {
3055 pci_err(pdev, "%d VFs already enabled. Please disable pre-enabled VFs!\n",
3056 pre_existing_vfs);
3057 return 0;
3058 }
3059
3060 num_vfs = min_t(int, max_vfs, total_vfs);
3061 ret = qm_vf_q_assign(qm, num_vfs);
3062 if (ret) {
3063 pci_err(pdev, "Can't assign queues for VF!\n");
3064 return ret;
3065 }
3066
3067 qm->vfs_num = num_vfs;
3068
3069 ret = pci_enable_sriov(pdev, num_vfs);
3070 if (ret) {
3071 pci_err(pdev, "Can't enable VF!\n");
3072 qm_clear_vft_config(qm);
3073 return ret;
3074 }
3075
3076 pci_info(pdev, "VF enabled, vfs_num(=%d)!\n", num_vfs);
3077
3078 return num_vfs;
3079 }
3080 EXPORT_SYMBOL_GPL(hisi_qm_sriov_enable);
3081
3082 /**
3083 * hisi_qm_sriov_disable - disable virtual functions
3084 * @pdev: the PCI device
3085 *
3086 * Return failure if there are VFs assigned already.
3087 */
3088 int hisi_qm_sriov_disable(struct pci_dev *pdev)
3089 {
3090 struct hisi_qm *qm = pci_get_drvdata(pdev);
3091
3092 if (pci_vfs_assigned(pdev)) {
3093 pci_err(pdev, "Failed to disable VFs as VFs are assigned!\n");
3094 return -EPERM;
3095 }
3096
3097 /* remove in hpre_pci_driver will be called to free VF resources */
3098 pci_disable_sriov(pdev);
3099 return qm_clear_vft_config(qm);
3100 }
3101 EXPORT_SYMBOL_GPL(hisi_qm_sriov_disable);
3102
3103 /**
3104 * hisi_qm_sriov_configure - configure the number of VFs
3105 * @pdev: The PCI device
3106 * @num_vfs: The number of VFs need enabled
3107 *
3108 * Enable SR-IOV according to num_vfs, 0 means disable.
3109 */
3110 int hisi_qm_sriov_configure(struct pci_dev *pdev, int num_vfs)
3111 {
3112 if (num_vfs == 0)
3113 return hisi_qm_sriov_disable(pdev);
3114 else
3115 return hisi_qm_sriov_enable(pdev, num_vfs);
3116 }
3117 EXPORT_SYMBOL_GPL(hisi_qm_sriov_configure);
3118
3119 static enum acc_err_result qm_dev_err_handle(struct hisi_qm *qm)
3120 {
3121 u32 err_sts;
3122
3123 if (!qm->err_ini->get_dev_hw_err_status) {
3124 dev_err(&qm->pdev->dev, "Device doesn't support get hw error status!\n");
3125 return ACC_ERR_NONE;
3126 }
3127
3128 /* get device hardware error status */
3129 err_sts = qm->err_ini->get_dev_hw_err_status(qm);
3130 if (err_sts) {
3131 if (err_sts & qm->err_ini->err_info.ecc_2bits_mask)
3132 qm->err_status.is_dev_ecc_mbit = true;
3133
3134 if (!qm->err_ini->log_dev_hw_err) {
3135 dev_err(&qm->pdev->dev, "Device doesn't support log hw error!\n");
3136 return ACC_ERR_NEED_RESET;
3137 }
3138
3139 qm->err_ini->log_dev_hw_err(qm, err_sts);
3140 return ACC_ERR_NEED_RESET;
3141 }
3142
3143 return ACC_ERR_RECOVERED;
3144 }
3145
3146 static enum acc_err_result qm_process_dev_error(struct hisi_qm *qm)
3147 {
3148 enum acc_err_result qm_ret, dev_ret;
3149
3150 /* log qm error */
3151 qm_ret = qm_hw_error_handle(qm);
3152
3153 /* log device error */
3154 dev_ret = qm_dev_err_handle(qm);
3155
3156 return (qm_ret == ACC_ERR_NEED_RESET ||
3157 dev_ret == ACC_ERR_NEED_RESET) ?
3158 ACC_ERR_NEED_RESET : ACC_ERR_RECOVERED;
3159 }
3160
3161 /**
3162 * hisi_qm_dev_err_detected() - Get device and qm error status then log it.
3163 * @pdev: The PCI device which need report error.
3164 * @state: The connectivity between CPU and device.
3165 *
3166 * We register this function into PCIe AER handlers, It will report device or
3167 * qm hardware error status when error occur.
3168 */
3169 pci_ers_result_t hisi_qm_dev_err_detected(struct pci_dev *pdev,
3170 pci_channel_state_t state)
3171 {
3172 struct hisi_qm *qm = pci_get_drvdata(pdev);
3173 enum acc_err_result ret;
3174
3175 if (pdev->is_virtfn)
3176 return PCI_ERS_RESULT_NONE;
3177
3178 pci_info(pdev, "PCI error detected, state(=%d)!!\n", state);
3179 if (state == pci_channel_io_perm_failure)
3180 return PCI_ERS_RESULT_DISCONNECT;
3181
3182 ret = qm_process_dev_error(qm);
3183 if (ret == ACC_ERR_NEED_RESET)
3184 return PCI_ERS_RESULT_NEED_RESET;
3185
3186 return PCI_ERS_RESULT_RECOVERED;
3187 }
3188 EXPORT_SYMBOL_GPL(hisi_qm_dev_err_detected);
3189
3190 static int qm_get_hw_error_status(struct hisi_qm *qm)
3191 {
3192 return readl(qm->io_base + QM_ABNORMAL_INT_STATUS);
3193 }
3194
3195 static int qm_check_req_recv(struct hisi_qm *qm)
3196 {
3197 struct pci_dev *pdev = qm->pdev;
3198 int ret;
3199 u32 val;
3200
3201 writel(ACC_VENDOR_ID_VALUE, qm->io_base + QM_PEH_VENDOR_ID);
3202 ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_VENDOR_ID, val,
3203 (val == ACC_VENDOR_ID_VALUE),
3204 POLL_PERIOD, POLL_TIMEOUT);
3205 if (ret) {
3206 dev_err(&pdev->dev, "Fails to read QM reg!\n");
3207 return ret;
3208 }
3209
3210 writel(PCI_VENDOR_ID_HUAWEI, qm->io_base + QM_PEH_VENDOR_ID);
3211 ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_VENDOR_ID, val,
3212 (val == PCI_VENDOR_ID_HUAWEI),
3213 POLL_PERIOD, POLL_TIMEOUT);
3214 if (ret)
3215 dev_err(&pdev->dev, "Fails to read QM reg in the second time!\n");
3216
3217 return ret;
3218 }
3219
3220 static int qm_set_pf_mse(struct hisi_qm *qm, bool set)
3221 {
3222 struct pci_dev *pdev = qm->pdev;
3223 u16 cmd;
3224 int i;
3225
3226 pci_read_config_word(pdev, PCI_COMMAND, &cmd);
3227 if (set)
3228 cmd |= PCI_COMMAND_MEMORY;
3229 else
3230 cmd &= ~PCI_COMMAND_MEMORY;
3231
3232 pci_write_config_word(pdev, PCI_COMMAND, cmd);
3233 for (i = 0; i < MAX_WAIT_COUNTS; i++) {
3234 pci_read_config_word(pdev, PCI_COMMAND, &cmd);
3235 if (set == ((cmd & PCI_COMMAND_MEMORY) >> 1))
3236 return 0;
3237
3238 udelay(1);
3239 }
3240
3241 return -ETIMEDOUT;
3242 }
3243
3244 static int qm_set_vf_mse(struct hisi_qm *qm, bool set)
3245 {
3246 struct pci_dev *pdev = qm->pdev;
3247 u16 sriov_ctrl;
3248 int pos;
3249 int i;
3250
3251 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV);
3252 pci_read_config_word(pdev, pos + PCI_SRIOV_CTRL, &sriov_ctrl);
3253 if (set)
3254 sriov_ctrl |= PCI_SRIOV_CTRL_MSE;
3255 else
3256 sriov_ctrl &= ~PCI_SRIOV_CTRL_MSE;
3257 pci_write_config_word(pdev, pos + PCI_SRIOV_CTRL, sriov_ctrl);
3258
3259 for (i = 0; i < MAX_WAIT_COUNTS; i++) {
3260 pci_read_config_word(pdev, pos + PCI_SRIOV_CTRL, &sriov_ctrl);
3261 if (set == (sriov_ctrl & PCI_SRIOV_CTRL_MSE) >>
3262 ACC_PEH_SRIOV_CTRL_VF_MSE_SHIFT)
3263 return 0;
3264
3265 udelay(1);
3266 }
3267
3268 return -ETIMEDOUT;
3269 }
3270
3271 static int qm_set_msi(struct hisi_qm *qm, bool set)
3272 {
3273 struct pci_dev *pdev = qm->pdev;
3274
3275 if (set) {
3276 pci_write_config_dword(pdev, pdev->msi_cap + PCI_MSI_MASK_64,
3277 0);
3278 } else {
3279 pci_write_config_dword(pdev, pdev->msi_cap + PCI_MSI_MASK_64,
3280 ACC_PEH_MSI_DISABLE);
3281 if (qm->err_status.is_qm_ecc_mbit ||
3282 qm->err_status.is_dev_ecc_mbit)
3283 return 0;
3284
3285 mdelay(1);
3286 if (readl(qm->io_base + QM_PEH_DFX_INFO0))
3287 return -EFAULT;
3288 }
3289
3290 return 0;
3291 }
3292
3293 static int qm_vf_reset_prepare(struct hisi_qm *qm)
3294 {
3295 struct hisi_qm_list *qm_list = qm->qm_list;
3296 int stop_reason = qm->status.stop_reason;
3297 struct pci_dev *pdev = qm->pdev;
3298 struct pci_dev *virtfn;
3299 struct hisi_qm *vf_qm;
3300 int ret = 0;
3301
3302 mutex_lock(&qm_list->lock);
3303 list_for_each_entry(vf_qm, &qm_list->list, list) {
3304 virtfn = vf_qm->pdev;
3305 if (virtfn == pdev)
3306 continue;
3307
3308 if (pci_physfn(virtfn) == pdev) {
3309 vf_qm->status.stop_reason = stop_reason;
3310 ret = hisi_qm_stop(vf_qm);
3311 if (ret)
3312 goto stop_fail;
3313 }
3314 }
3315
3316 stop_fail:
3317 mutex_unlock(&qm_list->lock);
3318 return ret;
3319 }
3320
3321 static int qm_reset_prepare_ready(struct hisi_qm *qm)
3322 {
3323 struct pci_dev *pdev = qm->pdev;
3324 struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev));
3325 int delay = 0;
3326
3327 /* All reset requests need to be queued for processing */
3328 while (test_and_set_bit(QM_DEV_RESET_FLAG, &pf_qm->reset_flag)) {
3329 msleep(++delay);
3330 if (delay > QM_RESET_WAIT_TIMEOUT)
3331 return -EBUSY;
3332 }
3333
3334 return 0;
3335 }
3336
3337 static int qm_controller_reset_prepare(struct hisi_qm *qm)
3338 {
3339 struct pci_dev *pdev = qm->pdev;
3340 int ret;
3341
3342 ret = qm_reset_prepare_ready(qm);
3343 if (ret) {
3344 pci_err(pdev, "Controller reset not ready!\n");
3345 return ret;
3346 }
3347
3348 if (qm->vfs_num) {
3349 ret = qm_vf_reset_prepare(qm);
3350 if (ret) {
3351 pci_err(pdev, "Fails to stop VFs!\n");
3352 return ret;
3353 }
3354 }
3355
3356 qm->status.stop_reason = QM_SOFT_RESET;
3357 ret = hisi_qm_stop(qm);
3358 if (ret) {
3359 pci_err(pdev, "Fails to stop QM!\n");
3360 return ret;
3361 }
3362
3363 return 0;
3364 }
3365
3366 static void qm_dev_ecc_mbit_handle(struct hisi_qm *qm)
3367 {
3368 u32 nfe_enb = 0;
3369
3370 if (!qm->err_status.is_dev_ecc_mbit &&
3371 qm->err_status.is_qm_ecc_mbit &&
3372 qm->err_ini->close_axi_master_ooo) {
3373
3374 qm->err_ini->close_axi_master_ooo(qm);
3375
3376 } else if (qm->err_status.is_dev_ecc_mbit &&
3377 !qm->err_status.is_qm_ecc_mbit &&
3378 !qm->err_ini->close_axi_master_ooo) {
3379
3380 nfe_enb = readl(qm->io_base + QM_RAS_NFE_ENABLE);
3381 writel(nfe_enb & QM_RAS_NFE_MBIT_DISABLE,
3382 qm->io_base + QM_RAS_NFE_ENABLE);
3383 writel(QM_ECC_MBIT, qm->io_base + QM_ABNORMAL_INT_SET);
3384 }
3385 }
3386
3387 static int qm_soft_reset(struct hisi_qm *qm)
3388 {
3389 struct pci_dev *pdev = qm->pdev;
3390 int ret;
3391 u32 val;
3392
3393 /* Ensure all doorbells and mailboxes received by QM */
3394 ret = qm_check_req_recv(qm);
3395 if (ret)
3396 return ret;
3397
3398 if (qm->vfs_num) {
3399 ret = qm_set_vf_mse(qm, false);
3400 if (ret) {
3401 pci_err(pdev, "Fails to disable vf MSE bit.\n");
3402 return ret;
3403 }
3404 }
3405
3406 ret = qm_set_msi(qm, false);
3407 if (ret) {
3408 pci_err(pdev, "Fails to disable PEH MSI bit.\n");
3409 return ret;
3410 }
3411
3412 qm_dev_ecc_mbit_handle(qm);
3413
3414 /* OOO register set and check */
3415 writel(ACC_MASTER_GLOBAL_CTRL_SHUTDOWN,
3416 qm->io_base + ACC_MASTER_GLOBAL_CTRL);
3417
3418 /* If bus lock, reset chip */
3419 ret = readl_relaxed_poll_timeout(qm->io_base + ACC_MASTER_TRANS_RETURN,
3420 val,
3421 (val == ACC_MASTER_TRANS_RETURN_RW),
3422 POLL_PERIOD, POLL_TIMEOUT);
3423 if (ret) {
3424 pci_emerg(pdev, "Bus lock! Please reset system.\n");
3425 return ret;
3426 }
3427
3428 ret = qm_set_pf_mse(qm, false);
3429 if (ret) {
3430 pci_err(pdev, "Fails to disable pf MSE bit.\n");
3431 return ret;
3432 }
3433
3434 /* The reset related sub-control registers are not in PCI BAR */
3435 if (ACPI_HANDLE(&pdev->dev)) {
3436 unsigned long long value = 0;
3437 acpi_status s;
3438
3439 s = acpi_evaluate_integer(ACPI_HANDLE(&pdev->dev),
3440 qm->err_ini->err_info.acpi_rst,
3441 NULL, &value);
3442 if (ACPI_FAILURE(s)) {
3443 pci_err(pdev, "NO controller reset method!\n");
3444 return -EIO;
3445 }
3446
3447 if (value) {
3448 pci_err(pdev, "Reset step %llu failed!\n", value);
3449 return -EIO;
3450 }
3451 } else {
3452 pci_err(pdev, "No reset method!\n");
3453 return -EINVAL;
3454 }
3455
3456 return 0;
3457 }
3458
3459 static int qm_vf_reset_done(struct hisi_qm *qm)
3460 {
3461 struct hisi_qm_list *qm_list = qm->qm_list;
3462 struct pci_dev *pdev = qm->pdev;
3463 struct pci_dev *virtfn;
3464 struct hisi_qm *vf_qm;
3465 int ret = 0;
3466
3467 mutex_lock(&qm_list->lock);
3468 list_for_each_entry(vf_qm, &qm_list->list, list) {
3469 virtfn = vf_qm->pdev;
3470 if (virtfn == pdev)
3471 continue;
3472
3473 if (pci_physfn(virtfn) == pdev) {
3474 ret = qm_restart(vf_qm);
3475 if (ret)
3476 goto restart_fail;
3477 }
3478 }
3479
3480 restart_fail:
3481 mutex_unlock(&qm_list->lock);
3482 return ret;
3483 }
3484
3485 static int qm_get_dev_err_status(struct hisi_qm *qm)
3486 {
3487 return qm->err_ini->get_dev_hw_err_status(qm);
3488 }
3489
3490 static int qm_dev_hw_init(struct hisi_qm *qm)
3491 {
3492 return qm->err_ini->hw_init(qm);
3493 }
3494
3495 static void qm_restart_prepare(struct hisi_qm *qm)
3496 {
3497 u32 value;
3498
3499 if (!qm->err_status.is_qm_ecc_mbit &&
3500 !qm->err_status.is_dev_ecc_mbit)
3501 return;
3502
3503 /* temporarily close the OOO port used for PEH to write out MSI */
3504 value = readl(qm->io_base + ACC_AM_CFG_PORT_WR_EN);
3505 writel(value & ~qm->err_ini->err_info.msi_wr_port,
3506 qm->io_base + ACC_AM_CFG_PORT_WR_EN);
3507
3508 /* clear dev ecc 2bit error source if having */
3509 value = qm_get_dev_err_status(qm) &
3510 qm->err_ini->err_info.ecc_2bits_mask;
3511 if (value && qm->err_ini->clear_dev_hw_err_status)
3512 qm->err_ini->clear_dev_hw_err_status(qm, value);
3513
3514 /* clear QM ecc mbit error source */
3515 writel(QM_ECC_MBIT, qm->io_base + QM_ABNORMAL_INT_SOURCE);
3516
3517 /* clear AM Reorder Buffer ecc mbit source */
3518 writel(ACC_ROB_ECC_ERR_MULTPL, qm->io_base + ACC_AM_ROB_ECC_INT_STS);
3519
3520 if (qm->err_ini->open_axi_master_ooo)
3521 qm->err_ini->open_axi_master_ooo(qm);
3522 }
3523
3524 static void qm_restart_done(struct hisi_qm *qm)
3525 {
3526 u32 value;
3527
3528 if (!qm->err_status.is_qm_ecc_mbit &&
3529 !qm->err_status.is_dev_ecc_mbit)
3530 return;
3531
3532 /* open the OOO port for PEH to write out MSI */
3533 value = readl(qm->io_base + ACC_AM_CFG_PORT_WR_EN);
3534 value |= qm->err_ini->err_info.msi_wr_port;
3535 writel(value, qm->io_base + ACC_AM_CFG_PORT_WR_EN);
3536
3537 qm->err_status.is_qm_ecc_mbit = false;
3538 qm->err_status.is_dev_ecc_mbit = false;
3539 }
3540
3541 static int qm_controller_reset_done(struct hisi_qm *qm)
3542 {
3543 struct pci_dev *pdev = qm->pdev;
3544 int ret;
3545
3546 ret = qm_set_msi(qm, true);
3547 if (ret) {
3548 pci_err(pdev, "Fails to enable PEH MSI bit!\n");
3549 return ret;
3550 }
3551
3552 ret = qm_set_pf_mse(qm, true);
3553 if (ret) {
3554 pci_err(pdev, "Fails to enable pf MSE bit!\n");
3555 return ret;
3556 }
3557
3558 if (qm->vfs_num) {
3559 ret = qm_set_vf_mse(qm, true);
3560 if (ret) {
3561 pci_err(pdev, "Fails to enable vf MSE bit!\n");
3562 return ret;
3563 }
3564 }
3565
3566 ret = qm_dev_hw_init(qm);
3567 if (ret) {
3568 pci_err(pdev, "Failed to init device\n");
3569 return ret;
3570 }
3571
3572 qm_restart_prepare(qm);
3573
3574 ret = qm_restart(qm);
3575 if (ret) {
3576 pci_err(pdev, "Failed to start QM!\n");
3577 return ret;
3578 }
3579
3580 if (qm->vfs_num) {
3581 ret = qm_vf_q_assign(qm, qm->vfs_num);
3582 if (ret) {
3583 pci_err(pdev, "Failed to assign queue!\n");
3584 return ret;
3585 }
3586 }
3587
3588 ret = qm_vf_reset_done(qm);
3589 if (ret) {
3590 pci_err(pdev, "Failed to start VFs!\n");
3591 return -EPERM;
3592 }
3593
3594 hisi_qm_dev_err_init(qm);
3595 qm_restart_done(qm);
3596
3597 clear_bit(QM_DEV_RESET_FLAG, &qm->reset_flag);
3598
3599 return 0;
3600 }
3601
3602 static int qm_controller_reset(struct hisi_qm *qm)
3603 {
3604 struct pci_dev *pdev = qm->pdev;
3605 int ret;
3606
3607 pci_info(pdev, "Controller resetting...\n");
3608
3609 ret = qm_controller_reset_prepare(qm);
3610 if (ret)
3611 return ret;
3612
3613 ret = qm_soft_reset(qm);
3614 if (ret) {
3615 pci_err(pdev, "Controller reset failed (%d)\n", ret);
3616 return ret;
3617 }
3618
3619 ret = qm_controller_reset_done(qm);
3620 if (ret)
3621 return ret;
3622
3623 pci_info(pdev, "Controller reset complete\n");
3624
3625 return 0;
3626 }
3627
3628 /**
3629 * hisi_qm_dev_slot_reset() - slot reset
3630 * @pdev: the PCIe device
3631 *
3632 * This function offers QM relate PCIe device reset interface. Drivers which
3633 * use QM can use this function as slot_reset in its struct pci_error_handlers.
3634 */
3635 pci_ers_result_t hisi_qm_dev_slot_reset(struct pci_dev *pdev)
3636 {
3637 struct hisi_qm *qm = pci_get_drvdata(pdev);
3638 int ret;
3639
3640 if (pdev->is_virtfn)
3641 return PCI_ERS_RESULT_RECOVERED;
3642
3643 pci_aer_clear_nonfatal_status(pdev);
3644
3645 /* reset pcie device controller */
3646 ret = qm_controller_reset(qm);
3647 if (ret) {
3648 pci_err(pdev, "Controller reset failed (%d)\n", ret);
3649 return PCI_ERS_RESULT_DISCONNECT;
3650 }
3651
3652 return PCI_ERS_RESULT_RECOVERED;
3653 }
3654 EXPORT_SYMBOL_GPL(hisi_qm_dev_slot_reset);
3655
3656 /* check the interrupt is ecc-mbit error or not */
3657 static int qm_check_dev_error(struct hisi_qm *qm)
3658 {
3659 int ret;
3660
3661 if (qm->fun_type == QM_HW_VF)
3662 return 0;
3663
3664 ret = qm_get_hw_error_status(qm) & QM_ECC_MBIT;
3665 if (ret)
3666 return ret;
3667
3668 return (qm_get_dev_err_status(qm) &
3669 qm->err_ini->err_info.ecc_2bits_mask);
3670 }
3671
3672 void hisi_qm_reset_prepare(struct pci_dev *pdev)
3673 {
3674 struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev));
3675 struct hisi_qm *qm = pci_get_drvdata(pdev);
3676 u32 delay = 0;
3677 int ret;
3678
3679 hisi_qm_dev_err_uninit(pf_qm);
3680
3681 /*
3682 * Check whether there is an ECC mbit error, If it occurs, need to
3683 * wait for soft reset to fix it.
3684 */
3685 while (qm_check_dev_error(pf_qm)) {
3686 msleep(++delay);
3687 if (delay > QM_RESET_WAIT_TIMEOUT)
3688 return;
3689 }
3690
3691 ret = qm_reset_prepare_ready(qm);
3692 if (ret) {
3693 pci_err(pdev, "FLR not ready!\n");
3694 return;
3695 }
3696
3697 if (qm->vfs_num) {
3698 ret = qm_vf_reset_prepare(qm);
3699 if (ret) {
3700 pci_err(pdev, "Failed to prepare reset, ret = %d.\n",
3701 ret);
3702 return;
3703 }
3704 }
3705
3706 ret = hisi_qm_stop(qm);
3707 if (ret) {
3708 pci_err(pdev, "Failed to stop QM, ret = %d.\n", ret);
3709 return;
3710 }
3711
3712 pci_info(pdev, "FLR resetting...\n");
3713 }
3714 EXPORT_SYMBOL_GPL(hisi_qm_reset_prepare);
3715
3716 static bool qm_flr_reset_complete(struct pci_dev *pdev)
3717 {
3718 struct pci_dev *pf_pdev = pci_physfn(pdev);
3719 struct hisi_qm *qm = pci_get_drvdata(pf_pdev);
3720 u32 id;
3721
3722 pci_read_config_dword(qm->pdev, PCI_COMMAND, &id);
3723 if (id == QM_PCI_COMMAND_INVALID) {
3724 pci_err(pdev, "Device can not be used!\n");
3725 return false;
3726 }
3727
3728 clear_bit(QM_DEV_RESET_FLAG, &qm->reset_flag);
3729
3730 return true;
3731 }
3732
3733 void hisi_qm_reset_done(struct pci_dev *pdev)
3734 {
3735 struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev));
3736 struct hisi_qm *qm = pci_get_drvdata(pdev);
3737 int ret;
3738
3739 hisi_qm_dev_err_init(pf_qm);
3740
3741 ret = qm_restart(qm);
3742 if (ret) {
3743 pci_err(pdev, "Failed to start QM, ret = %d.\n", ret);
3744 goto flr_done;
3745 }
3746
3747 if (qm->fun_type == QM_HW_PF) {
3748 ret = qm_dev_hw_init(qm);
3749 if (ret) {
3750 pci_err(pdev, "Failed to init PF, ret = %d.\n", ret);
3751 goto flr_done;
3752 }
3753
3754 if (!qm->vfs_num)
3755 goto flr_done;
3756
3757 ret = qm_vf_q_assign(qm, qm->vfs_num);
3758 if (ret) {
3759 pci_err(pdev, "Failed to assign VFs, ret = %d.\n", ret);
3760 goto flr_done;
3761 }
3762
3763 ret = qm_vf_reset_done(qm);
3764 if (ret) {
3765 pci_err(pdev, "Failed to start VFs, ret = %d.\n", ret);
3766 goto flr_done;
3767 }
3768 }
3769
3770 flr_done:
3771 if (qm_flr_reset_complete(pdev))
3772 pci_info(pdev, "FLR reset complete\n");
3773 }
3774 EXPORT_SYMBOL_GPL(hisi_qm_reset_done);
3775
3776 static irqreturn_t qm_abnormal_irq(int irq, void *data)
3777 {
3778 struct hisi_qm *qm = data;
3779 enum acc_err_result ret;
3780
3781 atomic64_inc(&qm->debug.dfx.abnormal_irq_cnt);
3782 ret = qm_process_dev_error(qm);
3783 if (ret == ACC_ERR_NEED_RESET)
3784 schedule_work(&qm->rst_work);
3785
3786 return IRQ_HANDLED;
3787 }
3788
3789 static int qm_irq_register(struct hisi_qm *qm)
3790 {
3791 struct pci_dev *pdev = qm->pdev;
3792 int ret;
3793
3794 ret = request_irq(pci_irq_vector(pdev, QM_EQ_EVENT_IRQ_VECTOR),
3795 qm_irq, IRQF_SHARED, qm->dev_name, qm);
3796 if (ret)
3797 return ret;
3798
3799 if (qm->ver != QM_HW_V1) {
3800 ret = request_irq(pci_irq_vector(pdev, QM_AEQ_EVENT_IRQ_VECTOR),
3801 qm_aeq_irq, IRQF_SHARED, qm->dev_name, qm);
3802 if (ret)
3803 goto err_aeq_irq;
3804
3805 if (qm->fun_type == QM_HW_PF) {
3806 ret = request_irq(pci_irq_vector(pdev,
3807 QM_ABNORMAL_EVENT_IRQ_VECTOR),
3808 qm_abnormal_irq, IRQF_SHARED,
3809 qm->dev_name, qm);
3810 if (ret)
3811 goto err_abonormal_irq;
3812 }
3813 }
3814
3815 return 0;
3816
3817 err_abonormal_irq:
3818 free_irq(pci_irq_vector(pdev, QM_AEQ_EVENT_IRQ_VECTOR), qm);
3819 err_aeq_irq:
3820 free_irq(pci_irq_vector(pdev, QM_EQ_EVENT_IRQ_VECTOR), qm);
3821 return ret;
3822 }
3823
3824 static void hisi_qm_controller_reset(struct work_struct *rst_work)
3825 {
3826 struct hisi_qm *qm = container_of(rst_work, struct hisi_qm, rst_work);
3827 int ret;
3828
3829 /* reset pcie device controller */
3830 ret = qm_controller_reset(qm);
3831 if (ret)
3832 dev_err(&qm->pdev->dev, "controller reset failed (%d)\n", ret);
3833
3834 }
3835
3836 /**
3837 * hisi_qm_init() - Initialize configures about qm.
3838 * @qm: The qm needing init.
3839 *
3840 * This function init qm, then we can call hisi_qm_start to put qm into work.
3841 */
3842 int hisi_qm_init(struct hisi_qm *qm)
3843 {
3844 struct pci_dev *pdev = qm->pdev;
3845 struct device *dev = &pdev->dev;
3846 unsigned int num_vec;
3847 int ret;
3848
3849 hisi_qm_pre_init(qm);
3850
3851 ret = qm_alloc_uacce(qm);
3852 if (ret < 0)
3853 dev_warn(&pdev->dev, "fail to alloc uacce (%d)\n", ret);
3854
3855 ret = pci_enable_device_mem(pdev);
3856 if (ret < 0) {
3857 dev_err(&pdev->dev, "Failed to enable device mem!\n");
3858 goto err_remove_uacce;
3859 }
3860
3861 ret = pci_request_mem_regions(pdev, qm->dev_name);
3862 if (ret < 0) {
3863 dev_err(&pdev->dev, "Failed to request mem regions!\n");
3864 goto err_disable_pcidev;
3865 }
3866
3867 qm->phys_base = pci_resource_start(pdev, PCI_BAR_2);
3868 qm->phys_size = pci_resource_len(qm->pdev, PCI_BAR_2);
3869 qm->io_base = ioremap(qm->phys_base, qm->phys_size);
3870 if (!qm->io_base) {
3871 ret = -EIO;
3872 goto err_release_mem_regions;
3873 }
3874
3875 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
3876 if (ret < 0)
3877 goto err_iounmap;
3878 pci_set_master(pdev);
3879
3880 if (!qm->ops->get_irq_num) {
3881 ret = -EOPNOTSUPP;
3882 goto err_iounmap;
3883 }
3884 num_vec = qm->ops->get_irq_num(qm);
3885 ret = pci_alloc_irq_vectors(pdev, num_vec, num_vec, PCI_IRQ_MSI);
3886 if (ret < 0) {
3887 dev_err(dev, "Failed to enable MSI vectors!\n");
3888 goto err_iounmap;
3889 }
3890
3891 ret = qm_irq_register(qm);
3892 if (ret)
3893 goto err_free_irq_vectors;
3894
3895 if (qm->fun_type == QM_HW_VF && qm->ver != QM_HW_V1) {
3896 /* v2 starts to support get vft by mailbox */
3897 ret = hisi_qm_get_vft(qm, &qm->qp_base, &qm->qp_num);
3898 if (ret)
3899 goto err_irq_unregister;
3900 }
3901
3902 ret = hisi_qm_memory_init(qm);
3903 if (ret)
3904 goto err_irq_unregister;
3905
3906 INIT_WORK(&qm->work, qm_work_process);
3907 if (qm->fun_type == QM_HW_PF)
3908 INIT_WORK(&qm->rst_work, hisi_qm_controller_reset);
3909
3910 atomic_set(&qm->status.flags, QM_INIT);
3911
3912 return 0;
3913
3914 err_irq_unregister:
3915 qm_irq_unregister(qm);
3916 err_free_irq_vectors:
3917 pci_free_irq_vectors(pdev);
3918 err_iounmap:
3919 iounmap(qm->io_base);
3920 err_release_mem_regions:
3921 pci_release_mem_regions(pdev);
3922 err_disable_pcidev:
3923 pci_disable_device(pdev);
3924 err_remove_uacce:
3925 uacce_remove(qm->uacce);
3926 qm->uacce = NULL;
3927 return ret;
3928 }
3929 EXPORT_SYMBOL_GPL(hisi_qm_init);
3930
3931
3932 MODULE_LICENSE("GPL v2");
3933 MODULE_AUTHOR("Zhou Wang <wangzhou1@hisilicon.com>");
3934 MODULE_DESCRIPTION("HiSilicon Accelerator queue manager driver");
Linux with added FPC-III support