1 /* Copyright 2008 - 2016 Freescale Semiconductor, Inc.
3 * Redistribution and use in source and binary forms, with or without
4 * modification, are permitted provided that the following conditions are met:
5 * * Redistributions of source code must retain the above copyright
6 * notice, this list of conditions and the following disclaimer.
7 * * Redistributions in binary form must reproduce the above copyright
8 * notice, this list of conditions and the following disclaimer in the
9 * documentation and/or other materials provided with the distribution.
10 * * Neither the name of Freescale Semiconductor nor the
11 * names of its contributors may be used to endorse or promote products
12 * derived from this software without specific prior written permission.
14 * ALTERNATIVELY, this software may be distributed under the terms of the
15 * GNU General Public License ("GPL") as published by the Free Software
16 * Foundation, either version 2 of that License or (at your option) any
19 * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
20 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
21 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22 * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
23 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
24 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
25 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
26 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
28 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31 #include "qman_priv.h"
33 #define DQRR_MAXFILL 15
34 #define EQCR_ITHRESH 4 /* if EQCR congests, interrupt threshold */
35 #define IRQNAME "QMan portal %d"
36 #define MAX_IRQNAME 16 /* big enough for "QMan portal %d" */
37 #define QMAN_POLL_LIMIT 32
38 #define QMAN_PIRQ_DQRR_ITHRESH 12
39 #define QMAN_DQRR_IT_MAX 15
40 #define QMAN_ITP_MAX 0xFFF
41 #define QMAN_PIRQ_MR_ITHRESH 4
42 #define QMAN_PIRQ_IPERIOD 100
44 /* Portal register assists */
46 #if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
47 /* Cache-inhibited register offsets */
48 #define QM_REG_EQCR_PI_CINH 0x3000
49 #define QM_REG_EQCR_CI_CINH 0x3040
50 #define QM_REG_EQCR_ITR 0x3080
51 #define QM_REG_DQRR_PI_CINH 0x3100
52 #define QM_REG_DQRR_CI_CINH 0x3140
53 #define QM_REG_DQRR_ITR 0x3180
54 #define QM_REG_DQRR_DCAP 0x31C0
55 #define QM_REG_DQRR_SDQCR 0x3200
56 #define QM_REG_DQRR_VDQCR 0x3240
57 #define QM_REG_DQRR_PDQCR 0x3280
58 #define QM_REG_MR_PI_CINH 0x3300
59 #define QM_REG_MR_CI_CINH 0x3340
60 #define QM_REG_MR_ITR 0x3380
61 #define QM_REG_CFG 0x3500
62 #define QM_REG_ISR 0x3600
63 #define QM_REG_IER 0x3640
64 #define QM_REG_ISDR 0x3680
65 #define QM_REG_IIR 0x36C0
66 #define QM_REG_ITPR 0x3740
68 /* Cache-enabled register offsets */
69 #define QM_CL_EQCR 0x0000
70 #define QM_CL_DQRR 0x1000
71 #define QM_CL_MR 0x2000
72 #define QM_CL_EQCR_PI_CENA 0x3000
73 #define QM_CL_EQCR_CI_CENA 0x3040
74 #define QM_CL_DQRR_PI_CENA 0x3100
75 #define QM_CL_DQRR_CI_CENA 0x3140
76 #define QM_CL_MR_PI_CENA 0x3300
77 #define QM_CL_MR_CI_CENA 0x3340
78 #define QM_CL_CR 0x3800
79 #define QM_CL_RR0 0x3900
80 #define QM_CL_RR1 0x3940
83 /* Cache-inhibited register offsets */
84 #define QM_REG_EQCR_PI_CINH 0x0000
85 #define QM_REG_EQCR_CI_CINH 0x0004
86 #define QM_REG_EQCR_ITR 0x0008
87 #define QM_REG_DQRR_PI_CINH 0x0040
88 #define QM_REG_DQRR_CI_CINH 0x0044
89 #define QM_REG_DQRR_ITR 0x0048
90 #define QM_REG_DQRR_DCAP 0x0050
91 #define QM_REG_DQRR_SDQCR 0x0054
92 #define QM_REG_DQRR_VDQCR 0x0058
93 #define QM_REG_DQRR_PDQCR 0x005c
94 #define QM_REG_MR_PI_CINH 0x0080
95 #define QM_REG_MR_CI_CINH 0x0084
96 #define QM_REG_MR_ITR 0x0088
97 #define QM_REG_CFG 0x0100
98 #define QM_REG_ISR 0x0e00
99 #define QM_REG_IER 0x0e04
100 #define QM_REG_ISDR 0x0e08
101 #define QM_REG_IIR 0x0e0c
102 #define QM_REG_ITPR 0x0e14
104 /* Cache-enabled register offsets */
105 #define QM_CL_EQCR 0x0000
106 #define QM_CL_DQRR 0x1000
107 #define QM_CL_MR 0x2000
108 #define QM_CL_EQCR_PI_CENA 0x3000
109 #define QM_CL_EQCR_CI_CENA 0x3100
110 #define QM_CL_DQRR_PI_CENA 0x3200
111 #define QM_CL_DQRR_CI_CENA 0x3300
112 #define QM_CL_MR_PI_CENA 0x3400
113 #define QM_CL_MR_CI_CENA 0x3500
114 #define QM_CL_CR 0x3800
115 #define QM_CL_RR0 0x3900
116 #define QM_CL_RR1 0x3940
120 * BTW, the drivers (and h/w programming model) already obtain the required
121 * synchronisation for portal accesses and data-dependencies. Use of barrier()s
122 * or other order-preserving primitives simply degrade performance. Hence the
123 * use of the __raw_*() interfaces, which simply ensure that the compiler treats
124 * the portal registers as volatile
127 /* Cache-enabled ring access */
128 #define qm_cl(base, idx) ((void *)base + ((idx) << 6))
133 * pmode == production mode
134 * cmode == consumption mode,
135 * dmode == h/w dequeue mode.
136 * Enum values use 3 letter codes. First letter matches the portal mode,
137 * remaining two letters indicate;
138 * ci == cache-inhibited portal register
139 * ce == cache-enabled portal register
140 * vb == in-band valid-bit (cache-enabled)
141 * dc == DCA (Discrete Consumption Acknowledgment), DQRR-only
142 * As for "enum qm_dqrr_dmode", it should be self-explanatory.
144 enum qm_eqcr_pmode
{ /* matches QCSP_CFG::EPM */
145 qm_eqcr_pci
= 0, /* PI index, cache-inhibited */
146 qm_eqcr_pce
= 1, /* PI index, cache-enabled */
147 qm_eqcr_pvb
= 2 /* valid-bit */
149 enum qm_dqrr_dmode
{ /* matches QCSP_CFG::DP */
150 qm_dqrr_dpush
= 0, /* SDQCR + VDQCR */
151 qm_dqrr_dpull
= 1 /* PDQCR */
153 enum qm_dqrr_pmode
{ /* s/w-only */
154 qm_dqrr_pci
, /* reads DQRR_PI_CINH */
155 qm_dqrr_pce
, /* reads DQRR_PI_CENA */
156 qm_dqrr_pvb
/* reads valid-bit */
158 enum qm_dqrr_cmode
{ /* matches QCSP_CFG::DCM */
159 qm_dqrr_cci
= 0, /* CI index, cache-inhibited */
160 qm_dqrr_cce
= 1, /* CI index, cache-enabled */
161 qm_dqrr_cdc
= 2 /* Discrete Consumption Acknowledgment */
163 enum qm_mr_pmode
{ /* s/w-only */
164 qm_mr_pci
, /* reads MR_PI_CINH */
165 qm_mr_pce
, /* reads MR_PI_CENA */
166 qm_mr_pvb
/* reads valid-bit */
168 enum qm_mr_cmode
{ /* matches QCSP_CFG::MM */
169 qm_mr_cci
= 0, /* CI index, cache-inhibited */
170 qm_mr_cce
= 1 /* CI index, cache-enabled */
173 /* --- Portal structures --- */
175 #define QM_EQCR_SIZE 8
176 #define QM_DQRR_SIZE 16
179 /* "Enqueue Command" */
180 struct qm_eqcr_entry
{
181 u8 _ncw_verb
; /* writes to this are non-coherent */
185 __be32 fqid
; /* 24-bit */
190 #define QM_EQCR_VERB_VBIT 0x80
191 #define QM_EQCR_VERB_CMD_MASK 0x61 /* but only one value; */
192 #define QM_EQCR_VERB_CMD_ENQUEUE 0x01
193 #define QM_EQCR_SEQNUM_NESN 0x8000 /* Advance NESN */
194 #define QM_EQCR_SEQNUM_NLIS 0x4000 /* More fragments to come */
195 #define QM_EQCR_SEQNUM_SEQMASK 0x3fff /* sequence number goes here */
198 struct qm_eqcr_entry
*ring
, *cursor
;
199 u8 ci
, available
, ithresh
, vbit
;
200 #ifdef CONFIG_FSL_DPAA_CHECKING
202 enum qm_eqcr_pmode pmode
;
207 const struct qm_dqrr_entry
*ring
, *cursor
;
208 u8 pi
, ci
, fill
, ithresh
, vbit
;
209 #ifdef CONFIG_FSL_DPAA_CHECKING
210 enum qm_dqrr_dmode dmode
;
211 enum qm_dqrr_pmode pmode
;
212 enum qm_dqrr_cmode cmode
;
217 union qm_mr_entry
*ring
, *cursor
;
218 u8 pi
, ci
, fill
, ithresh
, vbit
;
219 #ifdef CONFIG_FSL_DPAA_CHECKING
220 enum qm_mr_pmode pmode
;
221 enum qm_mr_cmode cmode
;
225 /* MC (Management Command) command */
226 /* "FQ" command layout */
230 __be32 fqid
; /* 24-bit */
234 /* "CGR" command layout */
242 #define QM_MCC_VERB_VBIT 0x80
243 #define QM_MCC_VERB_MASK 0x7f /* where the verb contains; */
244 #define QM_MCC_VERB_INITFQ_PARKED 0x40
245 #define QM_MCC_VERB_INITFQ_SCHED 0x41
246 #define QM_MCC_VERB_QUERYFQ 0x44
247 #define QM_MCC_VERB_QUERYFQ_NP 0x45 /* "non-programmable" fields */
248 #define QM_MCC_VERB_QUERYWQ 0x46
249 #define QM_MCC_VERB_QUERYWQ_DEDICATED 0x47
250 #define QM_MCC_VERB_ALTER_SCHED 0x48 /* Schedule FQ */
251 #define QM_MCC_VERB_ALTER_FE 0x49 /* Force Eligible FQ */
252 #define QM_MCC_VERB_ALTER_RETIRE 0x4a /* Retire FQ */
253 #define QM_MCC_VERB_ALTER_OOS 0x4b /* Take FQ out of service */
254 #define QM_MCC_VERB_ALTER_FQXON 0x4d /* FQ XON */
255 #define QM_MCC_VERB_ALTER_FQXOFF 0x4e /* FQ XOFF */
256 #define QM_MCC_VERB_INITCGR 0x50
257 #define QM_MCC_VERB_MODIFYCGR 0x51
258 #define QM_MCC_VERB_CGRTESTWRITE 0x52
259 #define QM_MCC_VERB_QUERYCGR 0x58
260 #define QM_MCC_VERB_QUERYCONGESTION 0x59
261 union qm_mc_command
{
263 u8 _ncw_verb
; /* writes to this are non-coherent */
266 struct qm_mcc_initfq initfq
;
267 struct qm_mcc_initcgr initcgr
;
269 struct qm_mcc_cgr cgr
;
272 /* MC (Management Command) result */
274 struct qm_mcr_queryfq
{
278 struct qm_fqd fqd
; /* the FQD fields are here */
282 /* "Alter FQ State Commands" */
283 struct qm_mcr_alterfq
{
286 u8 fqs
; /* Frame Queue Status */
289 #define QM_MCR_VERB_RRID 0x80
290 #define QM_MCR_VERB_MASK QM_MCC_VERB_MASK
291 #define QM_MCR_VERB_INITFQ_PARKED QM_MCC_VERB_INITFQ_PARKED
292 #define QM_MCR_VERB_INITFQ_SCHED QM_MCC_VERB_INITFQ_SCHED
293 #define QM_MCR_VERB_QUERYFQ QM_MCC_VERB_QUERYFQ
294 #define QM_MCR_VERB_QUERYFQ_NP QM_MCC_VERB_QUERYFQ_NP
295 #define QM_MCR_VERB_QUERYWQ QM_MCC_VERB_QUERYWQ
296 #define QM_MCR_VERB_QUERYWQ_DEDICATED QM_MCC_VERB_QUERYWQ_DEDICATED
297 #define QM_MCR_VERB_ALTER_SCHED QM_MCC_VERB_ALTER_SCHED
298 #define QM_MCR_VERB_ALTER_FE QM_MCC_VERB_ALTER_FE
299 #define QM_MCR_VERB_ALTER_RETIRE QM_MCC_VERB_ALTER_RETIRE
300 #define QM_MCR_VERB_ALTER_OOS QM_MCC_VERB_ALTER_OOS
301 #define QM_MCR_RESULT_NULL 0x00
302 #define QM_MCR_RESULT_OK 0xf0
303 #define QM_MCR_RESULT_ERR_FQID 0xf1
304 #define QM_MCR_RESULT_ERR_FQSTATE 0xf2
305 #define QM_MCR_RESULT_ERR_NOTEMPTY 0xf3 /* OOS fails if FQ is !empty */
306 #define QM_MCR_RESULT_ERR_BADCHANNEL 0xf4
307 #define QM_MCR_RESULT_PENDING 0xf8
308 #define QM_MCR_RESULT_ERR_BADCOMMAND 0xff
309 #define QM_MCR_FQS_ORLPRESENT 0x02 /* ORL fragments to come */
310 #define QM_MCR_FQS_NOTEMPTY 0x01 /* FQ has enqueued frames */
311 #define QM_MCR_TIMEOUT 10000 /* us */
318 struct qm_mcr_queryfq queryfq
;
319 struct qm_mcr_alterfq alterfq
;
320 struct qm_mcr_querycgr querycgr
;
321 struct qm_mcr_querycongestion querycongestion
;
322 struct qm_mcr_querywq querywq
;
323 struct qm_mcr_queryfq_np queryfq_np
;
327 union qm_mc_command
*cr
;
328 union qm_mc_result
*rr
;
330 #ifdef CONFIG_FSL_DPAA_CHECKING
332 /* Can be _mc_start()ed */
334 /* Can be _mc_commit()ed or _mc_abort()ed */
336 /* Can only be _mc_retry()ed */
343 void *ce
; /* cache-enabled */
344 __be32
*ce_be
; /* same value as above but for direct access */
345 void __iomem
*ci
; /* cache-inhibited */
350 * In the non-CONFIG_FSL_DPAA_CHECKING case, the following stuff up to
351 * and including 'mc' fits within a cacheline (yay!). The 'config' part
352 * is setup-only, so isn't a cause for a concern. In other words, don't
353 * rearrange this structure on a whim, there be dragons ...
360 } ____cacheline_aligned
;
362 /* Cache-inhibited register access. */
363 static inline u32
qm_in(struct qm_portal
*p
, u32 offset
)
365 return ioread32be(p
->addr
.ci
+ offset
);
368 static inline void qm_out(struct qm_portal
*p
, u32 offset
, u32 val
)
370 iowrite32be(val
, p
->addr
.ci
+ offset
);
373 /* Cache Enabled Portal Access */
374 static inline void qm_cl_invalidate(struct qm_portal
*p
, u32 offset
)
376 dpaa_invalidate(p
->addr
.ce
+ offset
);
379 static inline void qm_cl_touch_ro(struct qm_portal
*p
, u32 offset
)
381 dpaa_touch_ro(p
->addr
.ce
+ offset
);
384 static inline u32
qm_ce_in(struct qm_portal
*p
, u32 offset
)
386 return be32_to_cpu(*(p
->addr
.ce_be
+ (offset
/4)));
389 /* --- EQCR API --- */
391 #define EQCR_SHIFT ilog2(sizeof(struct qm_eqcr_entry))
392 #define EQCR_CARRY (uintptr_t)(QM_EQCR_SIZE << EQCR_SHIFT)
394 /* Bit-wise logic to wrap a ring pointer by clearing the "carry bit" */
395 static struct qm_eqcr_entry
*eqcr_carryclear(struct qm_eqcr_entry
*p
)
397 uintptr_t addr
= (uintptr_t)p
;
401 return (struct qm_eqcr_entry
*)addr
;
404 /* Bit-wise logic to convert a ring pointer to a ring index */
405 static int eqcr_ptr2idx(struct qm_eqcr_entry
*e
)
407 return ((uintptr_t)e
>> EQCR_SHIFT
) & (QM_EQCR_SIZE
- 1);
410 /* Increment the 'cursor' ring pointer, taking 'vbit' into account */
411 static inline void eqcr_inc(struct qm_eqcr
*eqcr
)
413 /* increment to the next EQCR pointer and handle overflow and 'vbit' */
414 struct qm_eqcr_entry
*partial
= eqcr
->cursor
+ 1;
416 eqcr
->cursor
= eqcr_carryclear(partial
);
417 if (partial
!= eqcr
->cursor
)
418 eqcr
->vbit
^= QM_EQCR_VERB_VBIT
;
421 static inline int qm_eqcr_init(struct qm_portal
*portal
,
422 enum qm_eqcr_pmode pmode
,
423 unsigned int eq_stash_thresh
,
426 struct qm_eqcr
*eqcr
= &portal
->eqcr
;
430 eqcr
->ring
= portal
->addr
.ce
+ QM_CL_EQCR
;
431 eqcr
->ci
= qm_in(portal
, QM_REG_EQCR_CI_CINH
) & (QM_EQCR_SIZE
- 1);
432 qm_cl_invalidate(portal
, QM_CL_EQCR_CI_CENA
);
433 pi
= qm_in(portal
, QM_REG_EQCR_PI_CINH
) & (QM_EQCR_SIZE
- 1);
434 eqcr
->cursor
= eqcr
->ring
+ pi
;
435 eqcr
->vbit
= (qm_in(portal
, QM_REG_EQCR_PI_CINH
) & QM_EQCR_SIZE
) ?
436 QM_EQCR_VERB_VBIT
: 0;
437 eqcr
->available
= QM_EQCR_SIZE
- 1 -
438 dpaa_cyc_diff(QM_EQCR_SIZE
, eqcr
->ci
, pi
);
439 eqcr
->ithresh
= qm_in(portal
, QM_REG_EQCR_ITR
);
440 #ifdef CONFIG_FSL_DPAA_CHECKING
444 cfg
= (qm_in(portal
, QM_REG_CFG
) & 0x00ffffff) |
445 (eq_stash_thresh
<< 28) | /* QCSP_CFG: EST */
446 (eq_stash_prio
<< 26) | /* QCSP_CFG: EP */
447 ((pmode
& 0x3) << 24); /* QCSP_CFG::EPM */
448 qm_out(portal
, QM_REG_CFG
, cfg
);
452 static inline void qm_eqcr_finish(struct qm_portal
*portal
)
454 struct qm_eqcr
*eqcr
= &portal
->eqcr
;
455 u8 pi
= qm_in(portal
, QM_REG_EQCR_PI_CINH
) & (QM_EQCR_SIZE
- 1);
456 u8 ci
= qm_in(portal
, QM_REG_EQCR_CI_CINH
) & (QM_EQCR_SIZE
- 1);
458 DPAA_ASSERT(!eqcr
->busy
);
459 if (pi
!= eqcr_ptr2idx(eqcr
->cursor
))
460 pr_crit("losing uncommitted EQCR entries\n");
462 pr_crit("missing existing EQCR completions\n");
463 if (eqcr
->ci
!= eqcr_ptr2idx(eqcr
->cursor
))
464 pr_crit("EQCR destroyed unquiesced\n");
467 static inline struct qm_eqcr_entry
*qm_eqcr_start_no_stash(struct qm_portal
470 struct qm_eqcr
*eqcr
= &portal
->eqcr
;
472 DPAA_ASSERT(!eqcr
->busy
);
473 if (!eqcr
->available
)
476 #ifdef CONFIG_FSL_DPAA_CHECKING
479 dpaa_zero(eqcr
->cursor
);
483 static inline struct qm_eqcr_entry
*qm_eqcr_start_stash(struct qm_portal
486 struct qm_eqcr
*eqcr
= &portal
->eqcr
;
489 DPAA_ASSERT(!eqcr
->busy
);
490 if (!eqcr
->available
) {
492 eqcr
->ci
= qm_ce_in(portal
, QM_CL_EQCR_CI_CENA
) &
494 diff
= dpaa_cyc_diff(QM_EQCR_SIZE
, old_ci
, eqcr
->ci
);
495 eqcr
->available
+= diff
;
499 #ifdef CONFIG_FSL_DPAA_CHECKING
502 dpaa_zero(eqcr
->cursor
);
506 static inline void eqcr_commit_checks(struct qm_eqcr
*eqcr
)
508 DPAA_ASSERT(eqcr
->busy
);
509 DPAA_ASSERT(!(be32_to_cpu(eqcr
->cursor
->fqid
) & ~QM_FQID_MASK
));
510 DPAA_ASSERT(eqcr
->available
>= 1);
513 static inline void qm_eqcr_pvb_commit(struct qm_portal
*portal
, u8 myverb
)
515 struct qm_eqcr
*eqcr
= &portal
->eqcr
;
516 struct qm_eqcr_entry
*eqcursor
;
518 eqcr_commit_checks(eqcr
);
519 DPAA_ASSERT(eqcr
->pmode
== qm_eqcr_pvb
);
521 eqcursor
= eqcr
->cursor
;
522 eqcursor
->_ncw_verb
= myverb
| eqcr
->vbit
;
523 dpaa_flush(eqcursor
);
526 #ifdef CONFIG_FSL_DPAA_CHECKING
531 static inline void qm_eqcr_cce_prefetch(struct qm_portal
*portal
)
533 qm_cl_touch_ro(portal
, QM_CL_EQCR_CI_CENA
);
536 static inline u8
qm_eqcr_cce_update(struct qm_portal
*portal
)
538 struct qm_eqcr
*eqcr
= &portal
->eqcr
;
539 u8 diff
, old_ci
= eqcr
->ci
;
541 eqcr
->ci
= qm_ce_in(portal
, QM_CL_EQCR_CI_CENA
) & (QM_EQCR_SIZE
- 1);
542 qm_cl_invalidate(portal
, QM_CL_EQCR_CI_CENA
);
543 diff
= dpaa_cyc_diff(QM_EQCR_SIZE
, old_ci
, eqcr
->ci
);
544 eqcr
->available
+= diff
;
548 static inline void qm_eqcr_set_ithresh(struct qm_portal
*portal
, u8 ithresh
)
550 struct qm_eqcr
*eqcr
= &portal
->eqcr
;
552 eqcr
->ithresh
= ithresh
;
553 qm_out(portal
, QM_REG_EQCR_ITR
, ithresh
);
556 static inline u8
qm_eqcr_get_avail(struct qm_portal
*portal
)
558 struct qm_eqcr
*eqcr
= &portal
->eqcr
;
560 return eqcr
->available
;
563 static inline u8
qm_eqcr_get_fill(struct qm_portal
*portal
)
565 struct qm_eqcr
*eqcr
= &portal
->eqcr
;
567 return QM_EQCR_SIZE
- 1 - eqcr
->available
;
570 /* --- DQRR API --- */
572 #define DQRR_SHIFT ilog2(sizeof(struct qm_dqrr_entry))
573 #define DQRR_CARRY (uintptr_t)(QM_DQRR_SIZE << DQRR_SHIFT)
575 static const struct qm_dqrr_entry
*dqrr_carryclear(
576 const struct qm_dqrr_entry
*p
)
578 uintptr_t addr
= (uintptr_t)p
;
582 return (const struct qm_dqrr_entry
*)addr
;
585 static inline int dqrr_ptr2idx(const struct qm_dqrr_entry
*e
)
587 return ((uintptr_t)e
>> DQRR_SHIFT
) & (QM_DQRR_SIZE
- 1);
590 static const struct qm_dqrr_entry
*dqrr_inc(const struct qm_dqrr_entry
*e
)
592 return dqrr_carryclear(e
+ 1);
595 static inline void qm_dqrr_set_maxfill(struct qm_portal
*portal
, u8 mf
)
597 qm_out(portal
, QM_REG_CFG
, (qm_in(portal
, QM_REG_CFG
) & 0xff0fffff) |
598 ((mf
& (QM_DQRR_SIZE
- 1)) << 20));
601 static inline int qm_dqrr_init(struct qm_portal
*portal
,
602 const struct qm_portal_config
*config
,
603 enum qm_dqrr_dmode dmode
,
604 enum qm_dqrr_pmode pmode
,
605 enum qm_dqrr_cmode cmode
, u8 max_fill
)
607 struct qm_dqrr
*dqrr
= &portal
->dqrr
;
610 /* Make sure the DQRR will be idle when we enable */
611 qm_out(portal
, QM_REG_DQRR_SDQCR
, 0);
612 qm_out(portal
, QM_REG_DQRR_VDQCR
, 0);
613 qm_out(portal
, QM_REG_DQRR_PDQCR
, 0);
614 dqrr
->ring
= portal
->addr
.ce
+ QM_CL_DQRR
;
615 dqrr
->pi
= qm_in(portal
, QM_REG_DQRR_PI_CINH
) & (QM_DQRR_SIZE
- 1);
616 dqrr
->ci
= qm_in(portal
, QM_REG_DQRR_CI_CINH
) & (QM_DQRR_SIZE
- 1);
617 dqrr
->cursor
= dqrr
->ring
+ dqrr
->ci
;
618 dqrr
->fill
= dpaa_cyc_diff(QM_DQRR_SIZE
, dqrr
->ci
, dqrr
->pi
);
619 dqrr
->vbit
= (qm_in(portal
, QM_REG_DQRR_PI_CINH
) & QM_DQRR_SIZE
) ?
620 QM_DQRR_VERB_VBIT
: 0;
621 dqrr
->ithresh
= qm_in(portal
, QM_REG_DQRR_ITR
);
622 #ifdef CONFIG_FSL_DPAA_CHECKING
627 /* Invalidate every ring entry before beginning */
628 for (cfg
= 0; cfg
< QM_DQRR_SIZE
; cfg
++)
629 dpaa_invalidate(qm_cl(dqrr
->ring
, cfg
));
630 cfg
= (qm_in(portal
, QM_REG_CFG
) & 0xff000f00) |
631 ((max_fill
& (QM_DQRR_SIZE
- 1)) << 20) | /* DQRR_MF */
632 ((dmode
& 1) << 18) | /* DP */
633 ((cmode
& 3) << 16) | /* DCM */
635 (0 ? 0x40 : 0) | /* Ignore RP */
636 (0 ? 0x10 : 0); /* Ignore SP */
637 qm_out(portal
, QM_REG_CFG
, cfg
);
638 qm_dqrr_set_maxfill(portal
, max_fill
);
642 static inline void qm_dqrr_finish(struct qm_portal
*portal
)
644 #ifdef CONFIG_FSL_DPAA_CHECKING
645 struct qm_dqrr
*dqrr
= &portal
->dqrr
;
647 if (dqrr
->cmode
!= qm_dqrr_cdc
&&
648 dqrr
->ci
!= dqrr_ptr2idx(dqrr
->cursor
))
649 pr_crit("Ignoring completed DQRR entries\n");
653 static inline const struct qm_dqrr_entry
*qm_dqrr_current(
654 struct qm_portal
*portal
)
656 struct qm_dqrr
*dqrr
= &portal
->dqrr
;
663 static inline u8
qm_dqrr_next(struct qm_portal
*portal
)
665 struct qm_dqrr
*dqrr
= &portal
->dqrr
;
667 DPAA_ASSERT(dqrr
->fill
);
668 dqrr
->cursor
= dqrr_inc(dqrr
->cursor
);
672 static inline void qm_dqrr_pvb_update(struct qm_portal
*portal
)
674 struct qm_dqrr
*dqrr
= &portal
->dqrr
;
675 struct qm_dqrr_entry
*res
= qm_cl(dqrr
->ring
, dqrr
->pi
);
677 DPAA_ASSERT(dqrr
->pmode
== qm_dqrr_pvb
);
678 #ifndef CONFIG_FSL_PAMU
680 * If PAMU is not available we need to invalidate the cache.
681 * When PAMU is available the cache is updated by stash
683 dpaa_invalidate_touch_ro(res
);
685 if ((res
->verb
& QM_DQRR_VERB_VBIT
) == dqrr
->vbit
) {
686 dqrr
->pi
= (dqrr
->pi
+ 1) & (QM_DQRR_SIZE
- 1);
688 dqrr
->vbit
^= QM_DQRR_VERB_VBIT
;
693 static inline void qm_dqrr_cdc_consume_1ptr(struct qm_portal
*portal
,
694 const struct qm_dqrr_entry
*dq
,
697 __maybe_unused
struct qm_dqrr
*dqrr
= &portal
->dqrr
;
698 int idx
= dqrr_ptr2idx(dq
);
700 DPAA_ASSERT(dqrr
->cmode
== qm_dqrr_cdc
);
701 DPAA_ASSERT((dqrr
->ring
+ idx
) == dq
);
702 DPAA_ASSERT(idx
< QM_DQRR_SIZE
);
703 qm_out(portal
, QM_REG_DQRR_DCAP
, (0 << 8) | /* DQRR_DCAP::S */
704 ((park
? 1 : 0) << 6) | /* DQRR_DCAP::PK */
705 idx
); /* DQRR_DCAP::DCAP_CI */
708 static inline void qm_dqrr_cdc_consume_n(struct qm_portal
*portal
, u32 bitmask
)
710 __maybe_unused
struct qm_dqrr
*dqrr
= &portal
->dqrr
;
712 DPAA_ASSERT(dqrr
->cmode
== qm_dqrr_cdc
);
713 qm_out(portal
, QM_REG_DQRR_DCAP
, (1 << 8) | /* DQRR_DCAP::S */
714 (bitmask
<< 16)); /* DQRR_DCAP::DCAP_CI */
717 static inline void qm_dqrr_sdqcr_set(struct qm_portal
*portal
, u32 sdqcr
)
719 qm_out(portal
, QM_REG_DQRR_SDQCR
, sdqcr
);
722 static inline void qm_dqrr_vdqcr_set(struct qm_portal
*portal
, u32 vdqcr
)
724 qm_out(portal
, QM_REG_DQRR_VDQCR
, vdqcr
);
727 static inline int qm_dqrr_set_ithresh(struct qm_portal
*portal
, u8 ithresh
)
730 if (ithresh
> QMAN_DQRR_IT_MAX
)
733 qm_out(portal
, QM_REG_DQRR_ITR
, ithresh
);
740 #define MR_SHIFT ilog2(sizeof(union qm_mr_entry))
741 #define MR_CARRY (uintptr_t)(QM_MR_SIZE << MR_SHIFT)
743 static union qm_mr_entry
*mr_carryclear(union qm_mr_entry
*p
)
745 uintptr_t addr
= (uintptr_t)p
;
749 return (union qm_mr_entry
*)addr
;
752 static inline int mr_ptr2idx(const union qm_mr_entry
*e
)
754 return ((uintptr_t)e
>> MR_SHIFT
) & (QM_MR_SIZE
- 1);
757 static inline union qm_mr_entry
*mr_inc(union qm_mr_entry
*e
)
759 return mr_carryclear(e
+ 1);
762 static inline int qm_mr_init(struct qm_portal
*portal
, enum qm_mr_pmode pmode
,
763 enum qm_mr_cmode cmode
)
765 struct qm_mr
*mr
= &portal
->mr
;
768 mr
->ring
= portal
->addr
.ce
+ QM_CL_MR
;
769 mr
->pi
= qm_in(portal
, QM_REG_MR_PI_CINH
) & (QM_MR_SIZE
- 1);
770 mr
->ci
= qm_in(portal
, QM_REG_MR_CI_CINH
) & (QM_MR_SIZE
- 1);
771 mr
->cursor
= mr
->ring
+ mr
->ci
;
772 mr
->fill
= dpaa_cyc_diff(QM_MR_SIZE
, mr
->ci
, mr
->pi
);
773 mr
->vbit
= (qm_in(portal
, QM_REG_MR_PI_CINH
) & QM_MR_SIZE
)
774 ? QM_MR_VERB_VBIT
: 0;
775 mr
->ithresh
= qm_in(portal
, QM_REG_MR_ITR
);
776 #ifdef CONFIG_FSL_DPAA_CHECKING
780 cfg
= (qm_in(portal
, QM_REG_CFG
) & 0xfffff0ff) |
781 ((cmode
& 1) << 8); /* QCSP_CFG:MM */
782 qm_out(portal
, QM_REG_CFG
, cfg
);
786 static inline void qm_mr_finish(struct qm_portal
*portal
)
788 struct qm_mr
*mr
= &portal
->mr
;
790 if (mr
->ci
!= mr_ptr2idx(mr
->cursor
))
791 pr_crit("Ignoring completed MR entries\n");
794 static inline const union qm_mr_entry
*qm_mr_current(struct qm_portal
*portal
)
796 struct qm_mr
*mr
= &portal
->mr
;
803 static inline int qm_mr_next(struct qm_portal
*portal
)
805 struct qm_mr
*mr
= &portal
->mr
;
807 DPAA_ASSERT(mr
->fill
);
808 mr
->cursor
= mr_inc(mr
->cursor
);
812 static inline void qm_mr_pvb_update(struct qm_portal
*portal
)
814 struct qm_mr
*mr
= &portal
->mr
;
815 union qm_mr_entry
*res
= qm_cl(mr
->ring
, mr
->pi
);
817 DPAA_ASSERT(mr
->pmode
== qm_mr_pvb
);
819 if ((res
->verb
& QM_MR_VERB_VBIT
) == mr
->vbit
) {
820 mr
->pi
= (mr
->pi
+ 1) & (QM_MR_SIZE
- 1);
822 mr
->vbit
^= QM_MR_VERB_VBIT
;
826 dpaa_invalidate_touch_ro(res
);
829 static inline void qm_mr_cci_consume(struct qm_portal
*portal
, u8 num
)
831 struct qm_mr
*mr
= &portal
->mr
;
833 DPAA_ASSERT(mr
->cmode
== qm_mr_cci
);
834 mr
->ci
= (mr
->ci
+ num
) & (QM_MR_SIZE
- 1);
835 qm_out(portal
, QM_REG_MR_CI_CINH
, mr
->ci
);
838 static inline void qm_mr_cci_consume_to_current(struct qm_portal
*portal
)
840 struct qm_mr
*mr
= &portal
->mr
;
842 DPAA_ASSERT(mr
->cmode
== qm_mr_cci
);
843 mr
->ci
= mr_ptr2idx(mr
->cursor
);
844 qm_out(portal
, QM_REG_MR_CI_CINH
, mr
->ci
);
847 static inline void qm_mr_set_ithresh(struct qm_portal
*portal
, u8 ithresh
)
849 qm_out(portal
, QM_REG_MR_ITR
, ithresh
);
852 /* --- Management command API --- */
854 static inline int qm_mc_init(struct qm_portal
*portal
)
857 struct qm_mc
*mc
= &portal
->mc
;
859 mc
->cr
= portal
->addr
.ce
+ QM_CL_CR
;
860 mc
->rr
= portal
->addr
.ce
+ QM_CL_RR0
;
862 * The expected valid bit polarity for the next CR command is 0
863 * if RR1 contains a valid response, and is 1 if RR0 contains a
864 * valid response. If both RR contain all 0, this indicates either
865 * that no command has been executed since reset (in which case the
866 * expected valid bit polarity is 1)
869 rr1
= (mc
->rr
+1)->verb
;
870 if ((rr0
== 0 && rr1
== 0) || rr0
!= 0)
874 mc
->vbit
= mc
->rridx
? QM_MCC_VERB_VBIT
: 0;
875 #ifdef CONFIG_FSL_DPAA_CHECKING
876 mc
->state
= qman_mc_idle
;
881 static inline void qm_mc_finish(struct qm_portal
*portal
)
883 #ifdef CONFIG_FSL_DPAA_CHECKING
884 struct qm_mc
*mc
= &portal
->mc
;
886 DPAA_ASSERT(mc
->state
== qman_mc_idle
);
887 if (mc
->state
!= qman_mc_idle
)
888 pr_crit("Losing incomplete MC command\n");
892 static inline union qm_mc_command
*qm_mc_start(struct qm_portal
*portal
)
894 struct qm_mc
*mc
= &portal
->mc
;
896 DPAA_ASSERT(mc
->state
== qman_mc_idle
);
897 #ifdef CONFIG_FSL_DPAA_CHECKING
898 mc
->state
= qman_mc_user
;
904 static inline void qm_mc_commit(struct qm_portal
*portal
, u8 myverb
)
906 struct qm_mc
*mc
= &portal
->mc
;
907 union qm_mc_result
*rr
= mc
->rr
+ mc
->rridx
;
909 DPAA_ASSERT(mc
->state
== qman_mc_user
);
911 mc
->cr
->_ncw_verb
= myverb
| mc
->vbit
;
913 dpaa_invalidate_touch_ro(rr
);
914 #ifdef CONFIG_FSL_DPAA_CHECKING
915 mc
->state
= qman_mc_hw
;
919 static inline union qm_mc_result
*qm_mc_result(struct qm_portal
*portal
)
921 struct qm_mc
*mc
= &portal
->mc
;
922 union qm_mc_result
*rr
= mc
->rr
+ mc
->rridx
;
924 DPAA_ASSERT(mc
->state
== qman_mc_hw
);
926 * The inactive response register's verb byte always returns zero until
927 * its command is submitted and completed. This includes the valid-bit,
928 * in case you were wondering...
931 dpaa_invalidate_touch_ro(rr
);
935 mc
->vbit
^= QM_MCC_VERB_VBIT
;
936 #ifdef CONFIG_FSL_DPAA_CHECKING
937 mc
->state
= qman_mc_idle
;
942 static inline int qm_mc_result_timeout(struct qm_portal
*portal
,
943 union qm_mc_result
**mcr
)
945 int timeout
= QM_MCR_TIMEOUT
;
948 *mcr
= qm_mc_result(portal
);
957 static inline void fq_set(struct qman_fq
*fq
, u32 mask
)
962 static inline void fq_clear(struct qman_fq
*fq
, u32 mask
)
967 static inline int fq_isset(struct qman_fq
*fq
, u32 mask
)
969 return fq
->flags
& mask
;
972 static inline int fq_isclear(struct qman_fq
*fq
, u32 mask
)
974 return !(fq
->flags
& mask
);
979 /* PORTAL_BITS_*** - dynamic, strictly internal */
981 /* interrupt sources processed by portal_isr(), configurable */
982 unsigned long irq_sources
;
983 u32 use_eqcr_ci_stashing
;
984 /* only 1 volatile dequeue at a time */
985 struct qman_fq
*vdqcr_owned
;
987 /* probing time config params for cpu-affine portals */
988 const struct qm_portal_config
*config
;
989 /* 2-element array. cgrs[0] is mask, cgrs[1] is snapshot. */
990 struct qman_cgrs
*cgrs
;
991 /* linked-list of CSCN handlers. */
992 struct list_head cgr_cbs
;
995 struct work_struct congestion_work
;
996 struct work_struct mr_work
;
997 char irqname
[MAX_IRQNAME
];
1000 static cpumask_t affine_mask
;
1001 static DEFINE_SPINLOCK(affine_mask_lock
);
1002 static u16 affine_channels
[NR_CPUS
];
1003 static DEFINE_PER_CPU(struct qman_portal
, qman_affine_portal
);
1004 struct qman_portal
*affine_portals
[NR_CPUS
];
1006 static inline struct qman_portal
*get_affine_portal(void)
1008 return &get_cpu_var(qman_affine_portal
);
1011 static inline void put_affine_portal(void)
1013 put_cpu_var(qman_affine_portal
);
1017 static inline struct qman_portal
*get_portal_for_channel(u16 channel
)
1021 for (i
= 0; i
< num_possible_cpus(); i
++) {
1022 if (affine_portals
[i
] &&
1023 affine_portals
[i
]->config
->channel
== channel
)
1024 return affine_portals
[i
];
1030 static struct workqueue_struct
*qm_portal_wq
;
1032 int qman_dqrr_set_ithresh(struct qman_portal
*portal
, u8 ithresh
)
1039 res
= qm_dqrr_set_ithresh(&portal
->p
, ithresh
);
1043 portal
->p
.dqrr
.ithresh
= ithresh
;
1047 EXPORT_SYMBOL(qman_dqrr_set_ithresh
);
1049 void qman_dqrr_get_ithresh(struct qman_portal
*portal
, u8
*ithresh
)
1051 if (portal
&& ithresh
)
1052 *ithresh
= qm_in(&portal
->p
, QM_REG_DQRR_ITR
);
1054 EXPORT_SYMBOL(qman_dqrr_get_ithresh
);
1056 void qman_portal_get_iperiod(struct qman_portal
*portal
, u32
*iperiod
)
1058 if (portal
&& iperiod
)
1059 *iperiod
= qm_in(&portal
->p
, QM_REG_ITPR
);
1061 EXPORT_SYMBOL(qman_portal_get_iperiod
);
1063 int qman_portal_set_iperiod(struct qman_portal
*portal
, u32 iperiod
)
1065 if (!portal
|| iperiod
> QMAN_ITP_MAX
)
1068 qm_out(&portal
->p
, QM_REG_ITPR
, iperiod
);
1072 EXPORT_SYMBOL(qman_portal_set_iperiod
);
1074 int qman_wq_alloc(void)
1076 qm_portal_wq
= alloc_workqueue("qman_portal_wq", 0, 1);
1083 void qman_enable_irqs(void)
1087 for (i
= 0; i
< num_possible_cpus(); i
++) {
1088 if (affine_portals
[i
]) {
1089 qm_out(&affine_portals
[i
]->p
, QM_REG_ISR
, 0xffffffff);
1090 qm_out(&affine_portals
[i
]->p
, QM_REG_IIR
, 0);
1097 * This is what everything can wait on, even if it migrates to a different cpu
1098 * to the one whose affine portal it is waiting on.
1100 static DECLARE_WAIT_QUEUE_HEAD(affine_queue
);
1102 static struct qman_fq
**fq_table
;
1103 static u32 num_fqids
;
1105 int qman_alloc_fq_table(u32 _num_fqids
)
1107 num_fqids
= _num_fqids
;
1109 fq_table
= vzalloc(array3_size(sizeof(struct qman_fq
*),
1114 pr_debug("Allocated fq lookup table at %p, entry count %u\n",
1115 fq_table
, num_fqids
* 2);
1119 static struct qman_fq
*idx_to_fq(u32 idx
)
1123 #ifdef CONFIG_FSL_DPAA_CHECKING
1124 if (WARN_ON(idx
>= num_fqids
* 2))
1128 DPAA_ASSERT(!fq
|| idx
== fq
->idx
);
1134 * Only returns full-service fq objects, not enqueue-only
1135 * references (QMAN_FQ_FLAG_NO_MODIFY).
1137 static struct qman_fq
*fqid_to_fq(u32 fqid
)
1139 return idx_to_fq(fqid
* 2);
1142 static struct qman_fq
*tag_to_fq(u32 tag
)
1144 #if BITS_PER_LONG == 64
1145 return idx_to_fq(tag
);
1147 return (struct qman_fq
*)tag
;
1151 static u32
fq_to_tag(struct qman_fq
*fq
)
1153 #if BITS_PER_LONG == 64
1160 static u32
__poll_portal_slow(struct qman_portal
*p
, u32 is
);
1161 static inline unsigned int __poll_portal_fast(struct qman_portal
*p
,
1162 unsigned int poll_limit
, bool sched_napi
);
1163 static void qm_congestion_task(struct work_struct
*work
);
1164 static void qm_mr_process_task(struct work_struct
*work
);
1166 static irqreturn_t
portal_isr(int irq
, void *ptr
)
1168 struct qman_portal
*p
= ptr
;
1169 u32 is
= qm_in(&p
->p
, QM_REG_ISR
) & p
->irq_sources
;
1175 /* DQRR-handling if it's interrupt-driven */
1176 if (is
& QM_PIRQ_DQRI
) {
1177 __poll_portal_fast(p
, QMAN_POLL_LIMIT
, true);
1178 clear
= QM_DQAVAIL_MASK
| QM_PIRQ_DQRI
;
1180 /* Handling of anything else that's interrupt-driven */
1181 clear
|= __poll_portal_slow(p
, is
) & QM_PIRQ_SLOW
;
1182 qm_out(&p
->p
, QM_REG_ISR
, clear
);
1186 static int drain_mr_fqrni(struct qm_portal
*p
)
1188 const union qm_mr_entry
*msg
;
1190 qm_mr_pvb_update(p
);
1191 msg
= qm_mr_current(p
);
1194 * if MR was full and h/w had other FQRNI entries to produce, we
1195 * need to allow it time to produce those entries once the
1196 * existing entries are consumed. A worst-case situation
1197 * (fully-loaded system) means h/w sequencers may have to do 3-4
1198 * other things before servicing the portal's MR pump, each of
1199 * which (if slow) may take ~50 qman cycles (which is ~200
1200 * processor cycles). So rounding up and then multiplying this
1201 * worst-case estimate by a factor of 10, just to be
1202 * ultra-paranoid, goes as high as 10,000 cycles. NB, we consume
1203 * one entry at a time, so h/w has an opportunity to produce new
1204 * entries well before the ring has been fully consumed, so
1205 * we're being *really* paranoid here.
1208 qm_mr_pvb_update(p
);
1209 msg
= qm_mr_current(p
);
1213 if ((msg
->verb
& QM_MR_VERB_TYPE_MASK
) != QM_MR_VERB_FQRNI
) {
1214 /* We aren't draining anything but FQRNIs */
1215 pr_err("Found verb 0x%x in MR\n", msg
->verb
);
1219 qm_mr_cci_consume(p
, 1);
1223 static int qman_create_portal(struct qman_portal
*portal
,
1224 const struct qm_portal_config
*c
,
1225 const struct qman_cgrs
*cgrs
)
1227 struct qm_portal
*p
;
1233 #ifdef CONFIG_FSL_PAMU
1234 /* PAMU is required for stashing */
1235 portal
->use_eqcr_ci_stashing
= ((qman_ip_rev
>= QMAN_REV30
) ? 1 : 0);
1237 portal
->use_eqcr_ci_stashing
= 0;
1240 * prep the low-level portal struct with the mapped addresses from the
1241 * config, everything that follows depends on it and "config" is more
1244 p
->addr
.ce
= c
->addr_virt_ce
;
1245 p
->addr
.ce_be
= c
->addr_virt_ce
;
1246 p
->addr
.ci
= c
->addr_virt_ci
;
1248 * If CI-stashing is used, the current defaults use a threshold of 3,
1249 * and stash with high-than-DQRR priority.
1251 if (qm_eqcr_init(p
, qm_eqcr_pvb
,
1252 portal
->use_eqcr_ci_stashing
? 3 : 0, 1)) {
1253 dev_err(c
->dev
, "EQCR initialisation failed\n");
1256 if (qm_dqrr_init(p
, c
, qm_dqrr_dpush
, qm_dqrr_pvb
,
1257 qm_dqrr_cdc
, DQRR_MAXFILL
)) {
1258 dev_err(c
->dev
, "DQRR initialisation failed\n");
1261 if (qm_mr_init(p
, qm_mr_pvb
, qm_mr_cci
)) {
1262 dev_err(c
->dev
, "MR initialisation failed\n");
1265 if (qm_mc_init(p
)) {
1266 dev_err(c
->dev
, "MC initialisation failed\n");
1269 /* static interrupt-gating controls */
1270 qm_dqrr_set_ithresh(p
, QMAN_PIRQ_DQRR_ITHRESH
);
1271 qm_mr_set_ithresh(p
, QMAN_PIRQ_MR_ITHRESH
);
1272 qm_out(p
, QM_REG_ITPR
, QMAN_PIRQ_IPERIOD
);
1273 portal
->cgrs
= kmalloc_array(2, sizeof(*cgrs
), GFP_KERNEL
);
1276 /* initial snapshot is no-depletion */
1277 qman_cgrs_init(&portal
->cgrs
[1]);
1279 portal
->cgrs
[0] = *cgrs
;
1281 /* if the given mask is NULL, assume all CGRs can be seen */
1282 qman_cgrs_fill(&portal
->cgrs
[0]);
1283 INIT_LIST_HEAD(&portal
->cgr_cbs
);
1284 spin_lock_init(&portal
->cgr_lock
);
1285 INIT_WORK(&portal
->congestion_work
, qm_congestion_task
);
1286 INIT_WORK(&portal
->mr_work
, qm_mr_process_task
);
1288 portal
->sdqcr
= QM_SDQCR_SOURCE_CHANNELS
| QM_SDQCR_COUNT_UPTO3
|
1289 QM_SDQCR_DEDICATED_PRECEDENCE
| QM_SDQCR_TYPE_PRIO_QOS
|
1290 QM_SDQCR_TOKEN_SET(0xab) | QM_SDQCR_CHANNELS_DEDICATED
;
1292 qm_out(p
, QM_REG_ISDR
, isdr
);
1293 portal
->irq_sources
= 0;
1294 qm_out(p
, QM_REG_IER
, 0);
1295 snprintf(portal
->irqname
, MAX_IRQNAME
, IRQNAME
, c
->cpu
);
1296 qm_out(p
, QM_REG_IIR
, 1);
1297 if (request_irq(c
->irq
, portal_isr
, 0, portal
->irqname
, portal
)) {
1298 dev_err(c
->dev
, "request_irq() failed\n");
1302 if (dpaa_set_portal_irq_affinity(c
->dev
, c
->irq
, c
->cpu
))
1305 /* Need EQCR to be empty before continuing */
1306 isdr
&= ~QM_PIRQ_EQCI
;
1307 qm_out(p
, QM_REG_ISDR
, isdr
);
1308 ret
= qm_eqcr_get_fill(p
);
1310 dev_err(c
->dev
, "EQCR unclean\n");
1311 goto fail_eqcr_empty
;
1313 isdr
&= ~(QM_PIRQ_DQRI
| QM_PIRQ_MRI
);
1314 qm_out(p
, QM_REG_ISDR
, isdr
);
1315 if (qm_dqrr_current(p
)) {
1316 dev_dbg(c
->dev
, "DQRR unclean\n");
1317 qm_dqrr_cdc_consume_n(p
, 0xffff);
1319 if (qm_mr_current(p
) && drain_mr_fqrni(p
)) {
1320 /* special handling, drain just in case it's a few FQRNIs */
1321 const union qm_mr_entry
*e
= qm_mr_current(p
);
1323 dev_err(c
->dev
, "MR dirty, VB 0x%x, rc 0x%x, addr 0x%llx\n",
1324 e
->verb
, e
->ern
.rc
, qm_fd_addr_get64(&e
->ern
.fd
));
1325 goto fail_dqrr_mr_empty
;
1329 qm_out(p
, QM_REG_ISR
, 0xffffffff);
1330 qm_out(p
, QM_REG_ISDR
, 0);
1331 if (!qman_requires_cleanup())
1332 qm_out(p
, QM_REG_IIR
, 0);
1333 /* Write a sane SDQCR */
1334 qm_dqrr_sdqcr_set(p
, portal
->sdqcr
);
1340 free_irq(c
->irq
, portal
);
1342 kfree(portal
->cgrs
);
1355 struct qman_portal
*qman_create_affine_portal(const struct qm_portal_config
*c
,
1356 const struct qman_cgrs
*cgrs
)
1358 struct qman_portal
*portal
;
1361 portal
= &per_cpu(qman_affine_portal
, c
->cpu
);
1362 err
= qman_create_portal(portal
, c
, cgrs
);
1366 spin_lock(&affine_mask_lock
);
1367 cpumask_set_cpu(c
->cpu
, &affine_mask
);
1368 affine_channels
[c
->cpu
] = c
->channel
;
1369 affine_portals
[c
->cpu
] = portal
;
1370 spin_unlock(&affine_mask_lock
);
1375 static void qman_destroy_portal(struct qman_portal
*qm
)
1377 const struct qm_portal_config
*pcfg
;
1379 /* Stop dequeues on the portal */
1380 qm_dqrr_sdqcr_set(&qm
->p
, 0);
1383 * NB we do this to "quiesce" EQCR. If we add enqueue-completions or
1384 * something related to QM_PIRQ_EQCI, this may need fixing.
1385 * Also, due to the prefetching model used for CI updates in the enqueue
1386 * path, this update will only invalidate the CI cacheline *after*
1387 * working on it, so we need to call this twice to ensure a full update
1388 * irrespective of where the enqueue processing was at when the teardown
1391 qm_eqcr_cce_update(&qm
->p
);
1392 qm_eqcr_cce_update(&qm
->p
);
1395 free_irq(pcfg
->irq
, qm
);
1398 qm_mc_finish(&qm
->p
);
1399 qm_mr_finish(&qm
->p
);
1400 qm_dqrr_finish(&qm
->p
);
1401 qm_eqcr_finish(&qm
->p
);
1406 const struct qm_portal_config
*qman_destroy_affine_portal(void)
1408 struct qman_portal
*qm
= get_affine_portal();
1409 const struct qm_portal_config
*pcfg
;
1415 qman_destroy_portal(qm
);
1417 spin_lock(&affine_mask_lock
);
1418 cpumask_clear_cpu(cpu
, &affine_mask
);
1419 spin_unlock(&affine_mask_lock
);
1420 put_affine_portal();
1424 /* Inline helper to reduce nesting in __poll_portal_slow() */
1425 static inline void fq_state_change(struct qman_portal
*p
, struct qman_fq
*fq
,
1426 const union qm_mr_entry
*msg
, u8 verb
)
1429 case QM_MR_VERB_FQRL
:
1430 DPAA_ASSERT(fq_isset(fq
, QMAN_FQ_STATE_ORL
));
1431 fq_clear(fq
, QMAN_FQ_STATE_ORL
);
1433 case QM_MR_VERB_FQRN
:
1434 DPAA_ASSERT(fq
->state
== qman_fq_state_parked
||
1435 fq
->state
== qman_fq_state_sched
);
1436 DPAA_ASSERT(fq_isset(fq
, QMAN_FQ_STATE_CHANGING
));
1437 fq_clear(fq
, QMAN_FQ_STATE_CHANGING
);
1438 if (msg
->fq
.fqs
& QM_MR_FQS_NOTEMPTY
)
1439 fq_set(fq
, QMAN_FQ_STATE_NE
);
1440 if (msg
->fq
.fqs
& QM_MR_FQS_ORLPRESENT
)
1441 fq_set(fq
, QMAN_FQ_STATE_ORL
);
1442 fq
->state
= qman_fq_state_retired
;
1444 case QM_MR_VERB_FQPN
:
1445 DPAA_ASSERT(fq
->state
== qman_fq_state_sched
);
1446 DPAA_ASSERT(fq_isclear(fq
, QMAN_FQ_STATE_CHANGING
));
1447 fq
->state
= qman_fq_state_parked
;
1451 static void qm_congestion_task(struct work_struct
*work
)
1453 struct qman_portal
*p
= container_of(work
, struct qman_portal
,
1455 struct qman_cgrs rr
, c
;
1456 union qm_mc_result
*mcr
;
1457 struct qman_cgr
*cgr
;
1459 spin_lock(&p
->cgr_lock
);
1461 qm_mc_commit(&p
->p
, QM_MCC_VERB_QUERYCONGESTION
);
1462 if (!qm_mc_result_timeout(&p
->p
, &mcr
)) {
1463 spin_unlock(&p
->cgr_lock
);
1464 dev_crit(p
->config
->dev
, "QUERYCONGESTION timeout\n");
1465 qman_p_irqsource_add(p
, QM_PIRQ_CSCI
);
1468 /* mask out the ones I'm not interested in */
1469 qman_cgrs_and(&rr
, (struct qman_cgrs
*)&mcr
->querycongestion
.state
,
1471 /* check previous snapshot for delta, enter/exit congestion */
1472 qman_cgrs_xor(&c
, &rr
, &p
->cgrs
[1]);
1473 /* update snapshot */
1474 qman_cgrs_cp(&p
->cgrs
[1], &rr
);
1475 /* Invoke callback */
1476 list_for_each_entry(cgr
, &p
->cgr_cbs
, node
)
1477 if (cgr
->cb
&& qman_cgrs_get(&c
, cgr
->cgrid
))
1478 cgr
->cb(p
, cgr
, qman_cgrs_get(&rr
, cgr
->cgrid
));
1479 spin_unlock(&p
->cgr_lock
);
1480 qman_p_irqsource_add(p
, QM_PIRQ_CSCI
);
1483 static void qm_mr_process_task(struct work_struct
*work
)
1485 struct qman_portal
*p
= container_of(work
, struct qman_portal
,
1487 const union qm_mr_entry
*msg
;
1494 qm_mr_pvb_update(&p
->p
);
1495 msg
= qm_mr_current(&p
->p
);
1499 verb
= msg
->verb
& QM_MR_VERB_TYPE_MASK
;
1500 /* The message is a software ERN iff the 0x20 bit is clear */
1503 case QM_MR_VERB_FQRNI
:
1504 /* nada, we drop FQRNIs on the floor */
1506 case QM_MR_VERB_FQRN
:
1507 case QM_MR_VERB_FQRL
:
1508 /* Lookup in the retirement table */
1509 fq
= fqid_to_fq(qm_fqid_get(&msg
->fq
));
1512 fq_state_change(p
, fq
, msg
, verb
);
1514 fq
->cb
.fqs(p
, fq
, msg
);
1516 case QM_MR_VERB_FQPN
:
1518 fq
= tag_to_fq(be32_to_cpu(msg
->fq
.context_b
));
1519 fq_state_change(p
, fq
, msg
, verb
);
1521 fq
->cb
.fqs(p
, fq
, msg
);
1523 case QM_MR_VERB_DC_ERN
:
1525 pr_crit_once("Leaking DCP ERNs!\n");
1528 pr_crit("Invalid MR verb 0x%02x\n", verb
);
1531 /* Its a software ERN */
1532 fq
= tag_to_fq(be32_to_cpu(msg
->ern
.tag
));
1533 fq
->cb
.ern(p
, fq
, msg
);
1539 qm_mr_cci_consume(&p
->p
, num
);
1540 qman_p_irqsource_add(p
, QM_PIRQ_MRI
);
1544 static u32
__poll_portal_slow(struct qman_portal
*p
, u32 is
)
1546 if (is
& QM_PIRQ_CSCI
) {
1547 qman_p_irqsource_remove(p
, QM_PIRQ_CSCI
);
1548 queue_work_on(smp_processor_id(), qm_portal_wq
,
1549 &p
->congestion_work
);
1552 if (is
& QM_PIRQ_EQRI
) {
1553 qm_eqcr_cce_update(&p
->p
);
1554 qm_eqcr_set_ithresh(&p
->p
, 0);
1555 wake_up(&affine_queue
);
1558 if (is
& QM_PIRQ_MRI
) {
1559 qman_p_irqsource_remove(p
, QM_PIRQ_MRI
);
1560 queue_work_on(smp_processor_id(), qm_portal_wq
,
1568 * remove some slowish-path stuff from the "fast path" and make sure it isn't
1571 static noinline
void clear_vdqcr(struct qman_portal
*p
, struct qman_fq
*fq
)
1573 p
->vdqcr_owned
= NULL
;
1574 fq_clear(fq
, QMAN_FQ_STATE_VDQCR
);
1575 wake_up(&affine_queue
);
1579 * The only states that would conflict with other things if they ran at the
1580 * same time on the same cpu are:
1582 * (i) setting/clearing vdqcr_owned, and
1583 * (ii) clearing the NE (Not Empty) flag.
1585 * Both are safe. Because;
1587 * (i) this clearing can only occur after qman_volatile_dequeue() has set the
1588 * vdqcr_owned field (which it does before setting VDQCR), and
1589 * qman_volatile_dequeue() blocks interrupts and preemption while this is
1590 * done so that we can't interfere.
1591 * (ii) the NE flag is only cleared after qman_retire_fq() has set it, and as
1592 * with (i) that API prevents us from interfering until it's safe.
1594 * The good thing is that qman_volatile_dequeue() and qman_retire_fq() run far
1595 * less frequently (ie. per-FQ) than __poll_portal_fast() does, so the nett
1596 * advantage comes from this function not having to "lock" anything at all.
1598 * Note also that the callbacks are invoked at points which are safe against the
1599 * above potential conflicts, but that this function itself is not re-entrant
1600 * (this is because the function tracks one end of each FIFO in the portal and
1601 * we do *not* want to lock that). So the consequence is that it is safe for
1602 * user callbacks to call into any QMan API.
1604 static inline unsigned int __poll_portal_fast(struct qman_portal
*p
,
1605 unsigned int poll_limit
, bool sched_napi
)
1607 const struct qm_dqrr_entry
*dq
;
1609 enum qman_cb_dqrr_result res
;
1610 unsigned int limit
= 0;
1613 qm_dqrr_pvb_update(&p
->p
);
1614 dq
= qm_dqrr_current(&p
->p
);
1618 if (dq
->stat
& QM_DQRR_STAT_UNSCHEDULED
) {
1620 * VDQCR: don't trust context_b as the FQ may have
1621 * been configured for h/w consumption and we're
1622 * draining it post-retirement.
1624 fq
= p
->vdqcr_owned
;
1626 * We only set QMAN_FQ_STATE_NE when retiring, so we
1627 * only need to check for clearing it when doing
1628 * volatile dequeues. It's one less thing to check
1629 * in the critical path (SDQCR).
1631 if (dq
->stat
& QM_DQRR_STAT_FQ_EMPTY
)
1632 fq_clear(fq
, QMAN_FQ_STATE_NE
);
1634 * This is duplicated from the SDQCR code, but we
1635 * have stuff to do before *and* after this callback,
1636 * and we don't want multiple if()s in the critical
1639 res
= fq
->cb
.dqrr(p
, fq
, dq
, sched_napi
);
1640 if (res
== qman_cb_dqrr_stop
)
1642 /* Check for VDQCR completion */
1643 if (dq
->stat
& QM_DQRR_STAT_DQCR_EXPIRED
)
1646 /* SDQCR: context_b points to the FQ */
1647 fq
= tag_to_fq(be32_to_cpu(dq
->context_b
));
1648 /* Now let the callback do its stuff */
1649 res
= fq
->cb
.dqrr(p
, fq
, dq
, sched_napi
);
1651 * The callback can request that we exit without
1652 * consuming this entry nor advancing;
1654 if (res
== qman_cb_dqrr_stop
)
1657 /* Interpret 'dq' from a driver perspective. */
1659 * Parking isn't possible unless HELDACTIVE was set. NB,
1660 * FORCEELIGIBLE implies HELDACTIVE, so we only need to
1661 * check for HELDACTIVE to cover both.
1663 DPAA_ASSERT((dq
->stat
& QM_DQRR_STAT_FQ_HELDACTIVE
) ||
1664 (res
!= qman_cb_dqrr_park
));
1665 /* just means "skip it, I'll consume it myself later on" */
1666 if (res
!= qman_cb_dqrr_defer
)
1667 qm_dqrr_cdc_consume_1ptr(&p
->p
, dq
,
1668 res
== qman_cb_dqrr_park
);
1670 qm_dqrr_next(&p
->p
);
1672 * Entry processed and consumed, increment our counter. The
1673 * callback can request that we exit after consuming the
1674 * entry, and we also exit if we reach our processing limit,
1675 * so loop back only if neither of these conditions is met.
1677 } while (++limit
< poll_limit
&& res
!= qman_cb_dqrr_consume_stop
);
1682 void qman_p_irqsource_add(struct qman_portal
*p
, u32 bits
)
1684 unsigned long irqflags
;
1686 local_irq_save(irqflags
);
1687 p
->irq_sources
|= bits
& QM_PIRQ_VISIBLE
;
1688 qm_out(&p
->p
, QM_REG_IER
, p
->irq_sources
);
1689 local_irq_restore(irqflags
);
1691 EXPORT_SYMBOL(qman_p_irqsource_add
);
1693 void qman_p_irqsource_remove(struct qman_portal
*p
, u32 bits
)
1695 unsigned long irqflags
;
1699 * Our interrupt handler only processes+clears status register bits that
1700 * are in p->irq_sources. As we're trimming that mask, if one of them
1701 * were to assert in the status register just before we remove it from
1702 * the enable register, there would be an interrupt-storm when we
1703 * release the IRQ lock. So we wait for the enable register update to
1704 * take effect in h/w (by reading it back) and then clear all other bits
1705 * in the status register. Ie. we clear them from ISR once it's certain
1706 * IER won't allow them to reassert.
1708 local_irq_save(irqflags
);
1709 bits
&= QM_PIRQ_VISIBLE
;
1710 p
->irq_sources
&= ~bits
;
1711 qm_out(&p
->p
, QM_REG_IER
, p
->irq_sources
);
1712 ier
= qm_in(&p
->p
, QM_REG_IER
);
1714 * Using "~ier" (rather than "bits" or "~p->irq_sources") creates a
1715 * data-dependency, ie. to protect against re-ordering.
1717 qm_out(&p
->p
, QM_REG_ISR
, ~ier
);
1718 local_irq_restore(irqflags
);
1720 EXPORT_SYMBOL(qman_p_irqsource_remove
);
1722 const cpumask_t
*qman_affine_cpus(void)
1724 return &affine_mask
;
1726 EXPORT_SYMBOL(qman_affine_cpus
);
1728 u16
qman_affine_channel(int cpu
)
1731 struct qman_portal
*portal
= get_affine_portal();
1733 cpu
= portal
->config
->cpu
;
1734 put_affine_portal();
1736 WARN_ON(!cpumask_test_cpu(cpu
, &affine_mask
));
1737 return affine_channels
[cpu
];
1739 EXPORT_SYMBOL(qman_affine_channel
);
1741 struct qman_portal
*qman_get_affine_portal(int cpu
)
1743 return affine_portals
[cpu
];
1745 EXPORT_SYMBOL(qman_get_affine_portal
);
1747 int qman_start_using_portal(struct qman_portal
*p
, struct device
*dev
)
1749 return (!device_link_add(dev
, p
->config
->dev
,
1750 DL_FLAG_AUTOREMOVE_CONSUMER
)) ? -EINVAL
: 0;
1752 EXPORT_SYMBOL(qman_start_using_portal
);
1754 int qman_p_poll_dqrr(struct qman_portal
*p
, unsigned int limit
)
1756 return __poll_portal_fast(p
, limit
, false);
1758 EXPORT_SYMBOL(qman_p_poll_dqrr
);
1760 void qman_p_static_dequeue_add(struct qman_portal
*p
, u32 pools
)
1762 unsigned long irqflags
;
1764 local_irq_save(irqflags
);
1765 pools
&= p
->config
->pools
;
1767 qm_dqrr_sdqcr_set(&p
->p
, p
->sdqcr
);
1768 local_irq_restore(irqflags
);
1770 EXPORT_SYMBOL(qman_p_static_dequeue_add
);
1772 /* Frame queue API */
1774 static const char *mcr_result_str(u8 result
)
1777 case QM_MCR_RESULT_NULL
:
1778 return "QM_MCR_RESULT_NULL";
1779 case QM_MCR_RESULT_OK
:
1780 return "QM_MCR_RESULT_OK";
1781 case QM_MCR_RESULT_ERR_FQID
:
1782 return "QM_MCR_RESULT_ERR_FQID";
1783 case QM_MCR_RESULT_ERR_FQSTATE
:
1784 return "QM_MCR_RESULT_ERR_FQSTATE";
1785 case QM_MCR_RESULT_ERR_NOTEMPTY
:
1786 return "QM_MCR_RESULT_ERR_NOTEMPTY";
1787 case QM_MCR_RESULT_PENDING
:
1788 return "QM_MCR_RESULT_PENDING";
1789 case QM_MCR_RESULT_ERR_BADCOMMAND
:
1790 return "QM_MCR_RESULT_ERR_BADCOMMAND";
1792 return "<unknown MCR result>";
1795 int qman_create_fq(u32 fqid
, u32 flags
, struct qman_fq
*fq
)
1797 if (flags
& QMAN_FQ_FLAG_DYNAMIC_FQID
) {
1798 int ret
= qman_alloc_fqid(&fqid
);
1805 fq
->state
= qman_fq_state_oos
;
1806 fq
->cgr_groupid
= 0;
1808 /* A context_b of 0 is allegedly special, so don't use that fqid */
1809 if (fqid
== 0 || fqid
>= num_fqids
) {
1810 WARN(1, "bad fqid %d\n", fqid
);
1815 if (flags
& QMAN_FQ_FLAG_NO_MODIFY
)
1818 WARN_ON(fq_table
[fq
->idx
]);
1819 fq_table
[fq
->idx
] = fq
;
1823 EXPORT_SYMBOL(qman_create_fq
);
1825 void qman_destroy_fq(struct qman_fq
*fq
)
1828 * We don't need to lock the FQ as it is a pre-condition that the FQ be
1829 * quiesced. Instead, run some checks.
1831 switch (fq
->state
) {
1832 case qman_fq_state_parked
:
1833 case qman_fq_state_oos
:
1834 if (fq_isset(fq
, QMAN_FQ_FLAG_DYNAMIC_FQID
))
1835 qman_release_fqid(fq
->fqid
);
1837 DPAA_ASSERT(fq_table
[fq
->idx
]);
1838 fq_table
[fq
->idx
] = NULL
;
1843 DPAA_ASSERT(NULL
== "qman_free_fq() on unquiesced FQ!");
1845 EXPORT_SYMBOL(qman_destroy_fq
);
1847 u32
qman_fq_fqid(struct qman_fq
*fq
)
1851 EXPORT_SYMBOL(qman_fq_fqid
);
1853 int qman_init_fq(struct qman_fq
*fq
, u32 flags
, struct qm_mcc_initfq
*opts
)
1855 union qm_mc_command
*mcc
;
1856 union qm_mc_result
*mcr
;
1857 struct qman_portal
*p
;
1861 myverb
= (flags
& QMAN_INITFQ_FLAG_SCHED
)
1862 ? QM_MCC_VERB_INITFQ_SCHED
: QM_MCC_VERB_INITFQ_PARKED
;
1864 if (fq
->state
!= qman_fq_state_oos
&&
1865 fq
->state
!= qman_fq_state_parked
)
1867 #ifdef CONFIG_FSL_DPAA_CHECKING
1868 if (fq_isset(fq
, QMAN_FQ_FLAG_NO_MODIFY
))
1871 if (opts
&& (be16_to_cpu(opts
->we_mask
) & QM_INITFQ_WE_OAC
)) {
1872 /* And can't be set at the same time as TDTHRESH */
1873 if (be16_to_cpu(opts
->we_mask
) & QM_INITFQ_WE_TDTHRESH
)
1876 /* Issue an INITFQ_[PARKED|SCHED] management command */
1877 p
= get_affine_portal();
1878 if (fq_isset(fq
, QMAN_FQ_STATE_CHANGING
) ||
1879 (fq
->state
!= qman_fq_state_oos
&&
1880 fq
->state
!= qman_fq_state_parked
)) {
1884 mcc
= qm_mc_start(&p
->p
);
1886 mcc
->initfq
= *opts
;
1887 qm_fqid_set(&mcc
->fq
, fq
->fqid
);
1888 mcc
->initfq
.count
= 0;
1890 * If the FQ does *not* have the TO_DCPORTAL flag, context_b is set as a
1891 * demux pointer. Otherwise, the caller-provided value is allowed to
1892 * stand, don't overwrite it.
1894 if (fq_isclear(fq
, QMAN_FQ_FLAG_TO_DCPORTAL
)) {
1897 mcc
->initfq
.we_mask
|= cpu_to_be16(QM_INITFQ_WE_CONTEXTB
);
1898 mcc
->initfq
.fqd
.context_b
= cpu_to_be32(fq_to_tag(fq
));
1900 * and the physical address - NB, if the user wasn't trying to
1901 * set CONTEXTA, clear the stashing settings.
1903 if (!(be16_to_cpu(mcc
->initfq
.we_mask
) &
1904 QM_INITFQ_WE_CONTEXTA
)) {
1905 mcc
->initfq
.we_mask
|=
1906 cpu_to_be16(QM_INITFQ_WE_CONTEXTA
);
1907 memset(&mcc
->initfq
.fqd
.context_a
, 0,
1908 sizeof(mcc
->initfq
.fqd
.context_a
));
1910 struct qman_portal
*p
= qman_dma_portal
;
1912 phys_fq
= dma_map_single(p
->config
->dev
, fq
,
1913 sizeof(*fq
), DMA_TO_DEVICE
);
1914 if (dma_mapping_error(p
->config
->dev
, phys_fq
)) {
1915 dev_err(p
->config
->dev
, "dma_mapping failed\n");
1920 qm_fqd_stashing_set64(&mcc
->initfq
.fqd
, phys_fq
);
1923 if (flags
& QMAN_INITFQ_FLAG_LOCAL
) {
1926 if (!(be16_to_cpu(mcc
->initfq
.we_mask
) &
1927 QM_INITFQ_WE_DESTWQ
)) {
1928 mcc
->initfq
.we_mask
|=
1929 cpu_to_be16(QM_INITFQ_WE_DESTWQ
);
1932 qm_fqd_set_destwq(&mcc
->initfq
.fqd
, p
->config
->channel
, wq
);
1934 qm_mc_commit(&p
->p
, myverb
);
1935 if (!qm_mc_result_timeout(&p
->p
, &mcr
)) {
1936 dev_err(p
->config
->dev
, "MCR timeout\n");
1941 DPAA_ASSERT((mcr
->verb
& QM_MCR_VERB_MASK
) == myverb
);
1943 if (res
!= QM_MCR_RESULT_OK
) {
1948 if (be16_to_cpu(opts
->we_mask
) & QM_INITFQ_WE_FQCTRL
) {
1949 if (be16_to_cpu(opts
->fqd
.fq_ctrl
) & QM_FQCTRL_CGE
)
1950 fq_set(fq
, QMAN_FQ_STATE_CGR_EN
);
1952 fq_clear(fq
, QMAN_FQ_STATE_CGR_EN
);
1954 if (be16_to_cpu(opts
->we_mask
) & QM_INITFQ_WE_CGID
)
1955 fq
->cgr_groupid
= opts
->fqd
.cgid
;
1957 fq
->state
= (flags
& QMAN_INITFQ_FLAG_SCHED
) ?
1958 qman_fq_state_sched
: qman_fq_state_parked
;
1961 put_affine_portal();
1964 EXPORT_SYMBOL(qman_init_fq
);
1966 int qman_schedule_fq(struct qman_fq
*fq
)
1968 union qm_mc_command
*mcc
;
1969 union qm_mc_result
*mcr
;
1970 struct qman_portal
*p
;
1973 if (fq
->state
!= qman_fq_state_parked
)
1975 #ifdef CONFIG_FSL_DPAA_CHECKING
1976 if (fq_isset(fq
, QMAN_FQ_FLAG_NO_MODIFY
))
1979 /* Issue a ALTERFQ_SCHED management command */
1980 p
= get_affine_portal();
1981 if (fq_isset(fq
, QMAN_FQ_STATE_CHANGING
) ||
1982 fq
->state
!= qman_fq_state_parked
) {
1986 mcc
= qm_mc_start(&p
->p
);
1987 qm_fqid_set(&mcc
->fq
, fq
->fqid
);
1988 qm_mc_commit(&p
->p
, QM_MCC_VERB_ALTER_SCHED
);
1989 if (!qm_mc_result_timeout(&p
->p
, &mcr
)) {
1990 dev_err(p
->config
->dev
, "ALTER_SCHED timeout\n");
1995 DPAA_ASSERT((mcr
->verb
& QM_MCR_VERB_MASK
) == QM_MCR_VERB_ALTER_SCHED
);
1996 if (mcr
->result
!= QM_MCR_RESULT_OK
) {
2000 fq
->state
= qman_fq_state_sched
;
2002 put_affine_portal();
2005 EXPORT_SYMBOL(qman_schedule_fq
);
2007 int qman_retire_fq(struct qman_fq
*fq
, u32
*flags
)
2009 union qm_mc_command
*mcc
;
2010 union qm_mc_result
*mcr
;
2011 struct qman_portal
*p
;
2015 if (fq
->state
!= qman_fq_state_parked
&&
2016 fq
->state
!= qman_fq_state_sched
)
2018 #ifdef CONFIG_FSL_DPAA_CHECKING
2019 if (fq_isset(fq
, QMAN_FQ_FLAG_NO_MODIFY
))
2022 p
= get_affine_portal();
2023 if (fq_isset(fq
, QMAN_FQ_STATE_CHANGING
) ||
2024 fq
->state
== qman_fq_state_retired
||
2025 fq
->state
== qman_fq_state_oos
) {
2029 mcc
= qm_mc_start(&p
->p
);
2030 qm_fqid_set(&mcc
->fq
, fq
->fqid
);
2031 qm_mc_commit(&p
->p
, QM_MCC_VERB_ALTER_RETIRE
);
2032 if (!qm_mc_result_timeout(&p
->p
, &mcr
)) {
2033 dev_crit(p
->config
->dev
, "ALTER_RETIRE timeout\n");
2038 DPAA_ASSERT((mcr
->verb
& QM_MCR_VERB_MASK
) == QM_MCR_VERB_ALTER_RETIRE
);
2041 * "Elegant" would be to treat OK/PENDING the same way; set CHANGING,
2042 * and defer the flags until FQRNI or FQRN (respectively) show up. But
2043 * "Friendly" is to process OK immediately, and not set CHANGING. We do
2044 * friendly, otherwise the caller doesn't necessarily have a fully
2045 * "retired" FQ on return even if the retirement was immediate. However
2046 * this does mean some code duplication between here and
2047 * fq_state_change().
2049 if (res
== QM_MCR_RESULT_OK
) {
2051 /* Process 'fq' right away, we'll ignore FQRNI */
2052 if (mcr
->alterfq
.fqs
& QM_MCR_FQS_NOTEMPTY
)
2053 fq_set(fq
, QMAN_FQ_STATE_NE
);
2054 if (mcr
->alterfq
.fqs
& QM_MCR_FQS_ORLPRESENT
)
2055 fq_set(fq
, QMAN_FQ_STATE_ORL
);
2058 fq
->state
= qman_fq_state_retired
;
2061 * Another issue with supporting "immediate" retirement
2062 * is that we're forced to drop FQRNIs, because by the
2063 * time they're seen it may already be "too late" (the
2064 * fq may have been OOS'd and free()'d already). But if
2065 * the upper layer wants a callback whether it's
2066 * immediate or not, we have to fake a "MR" entry to
2067 * look like an FQRNI...
2069 union qm_mr_entry msg
;
2071 msg
.verb
= QM_MR_VERB_FQRNI
;
2072 msg
.fq
.fqs
= mcr
->alterfq
.fqs
;
2073 qm_fqid_set(&msg
.fq
, fq
->fqid
);
2074 msg
.fq
.context_b
= cpu_to_be32(fq_to_tag(fq
));
2075 fq
->cb
.fqs(p
, fq
, &msg
);
2077 } else if (res
== QM_MCR_RESULT_PENDING
) {
2079 fq_set(fq
, QMAN_FQ_STATE_CHANGING
);
2084 put_affine_portal();
2087 EXPORT_SYMBOL(qman_retire_fq
);
2089 int qman_oos_fq(struct qman_fq
*fq
)
2091 union qm_mc_command
*mcc
;
2092 union qm_mc_result
*mcr
;
2093 struct qman_portal
*p
;
2096 if (fq
->state
!= qman_fq_state_retired
)
2098 #ifdef CONFIG_FSL_DPAA_CHECKING
2099 if (fq_isset(fq
, QMAN_FQ_FLAG_NO_MODIFY
))
2102 p
= get_affine_portal();
2103 if (fq_isset(fq
, QMAN_FQ_STATE_BLOCKOOS
) ||
2104 fq
->state
!= qman_fq_state_retired
) {
2108 mcc
= qm_mc_start(&p
->p
);
2109 qm_fqid_set(&mcc
->fq
, fq
->fqid
);
2110 qm_mc_commit(&p
->p
, QM_MCC_VERB_ALTER_OOS
);
2111 if (!qm_mc_result_timeout(&p
->p
, &mcr
)) {
2115 DPAA_ASSERT((mcr
->verb
& QM_MCR_VERB_MASK
) == QM_MCR_VERB_ALTER_OOS
);
2116 if (mcr
->result
!= QM_MCR_RESULT_OK
) {
2120 fq
->state
= qman_fq_state_oos
;
2122 put_affine_portal();
2125 EXPORT_SYMBOL(qman_oos_fq
);
2127 int qman_query_fq(struct qman_fq
*fq
, struct qm_fqd
*fqd
)
2129 union qm_mc_command
*mcc
;
2130 union qm_mc_result
*mcr
;
2131 struct qman_portal
*p
= get_affine_portal();
2134 mcc
= qm_mc_start(&p
->p
);
2135 qm_fqid_set(&mcc
->fq
, fq
->fqid
);
2136 qm_mc_commit(&p
->p
, QM_MCC_VERB_QUERYFQ
);
2137 if (!qm_mc_result_timeout(&p
->p
, &mcr
)) {
2142 DPAA_ASSERT((mcr
->verb
& QM_MCR_VERB_MASK
) == QM_MCR_VERB_QUERYFQ
);
2143 if (mcr
->result
== QM_MCR_RESULT_OK
)
2144 *fqd
= mcr
->queryfq
.fqd
;
2148 put_affine_portal();
2152 int qman_query_fq_np(struct qman_fq
*fq
, struct qm_mcr_queryfq_np
*np
)
2154 union qm_mc_command
*mcc
;
2155 union qm_mc_result
*mcr
;
2156 struct qman_portal
*p
= get_affine_portal();
2159 mcc
= qm_mc_start(&p
->p
);
2160 qm_fqid_set(&mcc
->fq
, fq
->fqid
);
2161 qm_mc_commit(&p
->p
, QM_MCC_VERB_QUERYFQ_NP
);
2162 if (!qm_mc_result_timeout(&p
->p
, &mcr
)) {
2167 DPAA_ASSERT((mcr
->verb
& QM_MCR_VERB_MASK
) == QM_MCR_VERB_QUERYFQ_NP
);
2168 if (mcr
->result
== QM_MCR_RESULT_OK
)
2169 *np
= mcr
->queryfq_np
;
2170 else if (mcr
->result
== QM_MCR_RESULT_ERR_FQID
)
2175 put_affine_portal();
2178 EXPORT_SYMBOL(qman_query_fq_np
);
2180 static int qman_query_cgr(struct qman_cgr
*cgr
,
2181 struct qm_mcr_querycgr
*cgrd
)
2183 union qm_mc_command
*mcc
;
2184 union qm_mc_result
*mcr
;
2185 struct qman_portal
*p
= get_affine_portal();
2188 mcc
= qm_mc_start(&p
->p
);
2189 mcc
->cgr
.cgid
= cgr
->cgrid
;
2190 qm_mc_commit(&p
->p
, QM_MCC_VERB_QUERYCGR
);
2191 if (!qm_mc_result_timeout(&p
->p
, &mcr
)) {
2195 DPAA_ASSERT((mcr
->verb
& QM_MCR_VERB_MASK
) == QM_MCC_VERB_QUERYCGR
);
2196 if (mcr
->result
== QM_MCR_RESULT_OK
)
2197 *cgrd
= mcr
->querycgr
;
2199 dev_err(p
->config
->dev
, "QUERY_CGR failed: %s\n",
2200 mcr_result_str(mcr
->result
));
2204 put_affine_portal();
2208 int qman_query_cgr_congested(struct qman_cgr
*cgr
, bool *result
)
2210 struct qm_mcr_querycgr query_cgr
;
2213 err
= qman_query_cgr(cgr
, &query_cgr
);
2217 *result
= !!query_cgr
.cgr
.cs
;
2220 EXPORT_SYMBOL(qman_query_cgr_congested
);
2222 /* internal function used as a wait_event() expression */
2223 static int set_p_vdqcr(struct qman_portal
*p
, struct qman_fq
*fq
, u32 vdqcr
)
2225 unsigned long irqflags
;
2228 local_irq_save(irqflags
);
2231 if (fq_isset(fq
, QMAN_FQ_STATE_VDQCR
))
2234 fq_set(fq
, QMAN_FQ_STATE_VDQCR
);
2235 p
->vdqcr_owned
= fq
;
2236 qm_dqrr_vdqcr_set(&p
->p
, vdqcr
);
2239 local_irq_restore(irqflags
);
2243 static int set_vdqcr(struct qman_portal
**p
, struct qman_fq
*fq
, u32 vdqcr
)
2247 *p
= get_affine_portal();
2248 ret
= set_p_vdqcr(*p
, fq
, vdqcr
);
2249 put_affine_portal();
2253 static int wait_vdqcr_start(struct qman_portal
**p
, struct qman_fq
*fq
,
2254 u32 vdqcr
, u32 flags
)
2258 if (flags
& QMAN_VOLATILE_FLAG_WAIT_INT
)
2259 ret
= wait_event_interruptible(affine_queue
,
2260 !set_vdqcr(p
, fq
, vdqcr
));
2262 wait_event(affine_queue
, !set_vdqcr(p
, fq
, vdqcr
));
2266 int qman_volatile_dequeue(struct qman_fq
*fq
, u32 flags
, u32 vdqcr
)
2268 struct qman_portal
*p
;
2271 if (fq
->state
!= qman_fq_state_parked
&&
2272 fq
->state
!= qman_fq_state_retired
)
2274 if (vdqcr
& QM_VDQCR_FQID_MASK
)
2276 if (fq_isset(fq
, QMAN_FQ_STATE_VDQCR
))
2278 vdqcr
= (vdqcr
& ~QM_VDQCR_FQID_MASK
) | fq
->fqid
;
2279 if (flags
& QMAN_VOLATILE_FLAG_WAIT
)
2280 ret
= wait_vdqcr_start(&p
, fq
, vdqcr
, flags
);
2282 ret
= set_vdqcr(&p
, fq
, vdqcr
);
2286 if (flags
& QMAN_VOLATILE_FLAG_FINISH
) {
2287 if (flags
& QMAN_VOLATILE_FLAG_WAIT_INT
)
2289 * NB: don't propagate any error - the caller wouldn't
2290 * know whether the VDQCR was issued or not. A signal
2291 * could arrive after returning anyway, so the caller
2292 * can check signal_pending() if that's an issue.
2294 wait_event_interruptible(affine_queue
,
2295 !fq_isset(fq
, QMAN_FQ_STATE_VDQCR
));
2297 wait_event(affine_queue
,
2298 !fq_isset(fq
, QMAN_FQ_STATE_VDQCR
));
2302 EXPORT_SYMBOL(qman_volatile_dequeue
);
2304 static void update_eqcr_ci(struct qman_portal
*p
, u8 avail
)
2307 qm_eqcr_cce_prefetch(&p
->p
);
2309 qm_eqcr_cce_update(&p
->p
);
2312 int qman_enqueue(struct qman_fq
*fq
, const struct qm_fd
*fd
)
2314 struct qman_portal
*p
;
2315 struct qm_eqcr_entry
*eq
;
2316 unsigned long irqflags
;
2319 p
= get_affine_portal();
2320 local_irq_save(irqflags
);
2322 if (p
->use_eqcr_ci_stashing
) {
2324 * The stashing case is easy, only update if we need to in
2325 * order to try and liberate ring entries.
2327 eq
= qm_eqcr_start_stash(&p
->p
);
2330 * The non-stashing case is harder, need to prefetch ahead of
2333 avail
= qm_eqcr_get_avail(&p
->p
);
2335 update_eqcr_ci(p
, avail
);
2336 eq
= qm_eqcr_start_no_stash(&p
->p
);
2342 qm_fqid_set(eq
, fq
->fqid
);
2343 eq
->tag
= cpu_to_be32(fq_to_tag(fq
));
2346 qm_eqcr_pvb_commit(&p
->p
, QM_EQCR_VERB_CMD_ENQUEUE
);
2348 local_irq_restore(irqflags
);
2349 put_affine_portal();
2352 EXPORT_SYMBOL(qman_enqueue
);
2354 static int qm_modify_cgr(struct qman_cgr
*cgr
, u32 flags
,
2355 struct qm_mcc_initcgr
*opts
)
2357 union qm_mc_command
*mcc
;
2358 union qm_mc_result
*mcr
;
2359 struct qman_portal
*p
= get_affine_portal();
2360 u8 verb
= QM_MCC_VERB_MODIFYCGR
;
2363 mcc
= qm_mc_start(&p
->p
);
2365 mcc
->initcgr
= *opts
;
2366 mcc
->initcgr
.cgid
= cgr
->cgrid
;
2367 if (flags
& QMAN_CGR_FLAG_USE_INIT
)
2368 verb
= QM_MCC_VERB_INITCGR
;
2369 qm_mc_commit(&p
->p
, verb
);
2370 if (!qm_mc_result_timeout(&p
->p
, &mcr
)) {
2375 DPAA_ASSERT((mcr
->verb
& QM_MCR_VERB_MASK
) == verb
);
2376 if (mcr
->result
!= QM_MCR_RESULT_OK
)
2380 put_affine_portal();
2384 #define PORTAL_IDX(n) (n->config->channel - QM_CHANNEL_SWPORTAL0)
2386 /* congestion state change notification target update control */
2387 static void qm_cgr_cscn_targ_set(struct __qm_mc_cgr
*cgr
, int pi
, u32 val
)
2389 if (qman_ip_rev
>= QMAN_REV30
)
2390 cgr
->cscn_targ_upd_ctrl
= cpu_to_be16(pi
|
2391 QM_CGR_TARG_UDP_CTRL_WRITE_BIT
);
2393 cgr
->cscn_targ
= cpu_to_be32(val
| QM_CGR_TARG_PORTAL(pi
));
2396 static void qm_cgr_cscn_targ_clear(struct __qm_mc_cgr
*cgr
, int pi
, u32 val
)
2398 if (qman_ip_rev
>= QMAN_REV30
)
2399 cgr
->cscn_targ_upd_ctrl
= cpu_to_be16(pi
);
2401 cgr
->cscn_targ
= cpu_to_be32(val
& ~QM_CGR_TARG_PORTAL(pi
));
2404 static u8 qman_cgr_cpus
[CGR_NUM
];
2406 void qman_init_cgr_all(void)
2408 struct qman_cgr cgr
;
2411 for (cgr
.cgrid
= 0; cgr
.cgrid
< CGR_NUM
; cgr
.cgrid
++) {
2412 if (qm_modify_cgr(&cgr
, QMAN_CGR_FLAG_USE_INIT
, NULL
))
2417 pr_err("Warning: %d error%s while initialising CGR h/w\n",
2418 err_cnt
, (err_cnt
> 1) ? "s" : "");
2421 int qman_create_cgr(struct qman_cgr
*cgr
, u32 flags
,
2422 struct qm_mcc_initcgr
*opts
)
2424 struct qm_mcr_querycgr cgr_state
;
2426 struct qman_portal
*p
;
2429 * We have to check that the provided CGRID is within the limits of the
2430 * data-structures, for obvious reasons. However we'll let h/w take
2431 * care of determining whether it's within the limits of what exists on
2434 if (cgr
->cgrid
>= CGR_NUM
)
2438 p
= get_affine_portal();
2439 qman_cgr_cpus
[cgr
->cgrid
] = smp_processor_id();
2442 cgr
->chan
= p
->config
->channel
;
2443 spin_lock(&p
->cgr_lock
);
2446 struct qm_mcc_initcgr local_opts
= *opts
;
2448 ret
= qman_query_cgr(cgr
, &cgr_state
);
2452 qm_cgr_cscn_targ_set(&local_opts
.cgr
, PORTAL_IDX(p
),
2453 be32_to_cpu(cgr_state
.cgr
.cscn_targ
));
2454 local_opts
.we_mask
|= cpu_to_be16(QM_CGR_WE_CSCN_TARG
);
2456 /* send init if flags indicate so */
2457 if (flags
& QMAN_CGR_FLAG_USE_INIT
)
2458 ret
= qm_modify_cgr(cgr
, QMAN_CGR_FLAG_USE_INIT
,
2461 ret
= qm_modify_cgr(cgr
, 0, &local_opts
);
2466 list_add(&cgr
->node
, &p
->cgr_cbs
);
2468 /* Determine if newly added object requires its callback to be called */
2469 ret
= qman_query_cgr(cgr
, &cgr_state
);
2471 /* we can't go back, so proceed and return success */
2472 dev_err(p
->config
->dev
, "CGR HW state partially modified\n");
2476 if (cgr
->cb
&& cgr_state
.cgr
.cscn_en
&&
2477 qman_cgrs_get(&p
->cgrs
[1], cgr
->cgrid
))
2480 spin_unlock(&p
->cgr_lock
);
2481 put_affine_portal();
2484 EXPORT_SYMBOL(qman_create_cgr
);
2486 int qman_delete_cgr(struct qman_cgr
*cgr
)
2488 unsigned long irqflags
;
2489 struct qm_mcr_querycgr cgr_state
;
2490 struct qm_mcc_initcgr local_opts
;
2493 struct qman_portal
*p
= get_affine_portal();
2495 if (cgr
->chan
!= p
->config
->channel
) {
2496 /* attempt to delete from other portal than creator */
2497 dev_err(p
->config
->dev
, "CGR not owned by current portal");
2498 dev_dbg(p
->config
->dev
, " create 0x%x, delete 0x%x\n",
2499 cgr
->chan
, p
->config
->channel
);
2504 memset(&local_opts
, 0, sizeof(struct qm_mcc_initcgr
));
2505 spin_lock_irqsave(&p
->cgr_lock
, irqflags
);
2506 list_del(&cgr
->node
);
2508 * If there are no other CGR objects for this CGRID in the list,
2509 * update CSCN_TARG accordingly
2511 list_for_each_entry(i
, &p
->cgr_cbs
, node
)
2512 if (i
->cgrid
== cgr
->cgrid
&& i
->cb
)
2514 ret
= qman_query_cgr(cgr
, &cgr_state
);
2516 /* add back to the list */
2517 list_add(&cgr
->node
, &p
->cgr_cbs
);
2521 local_opts
.we_mask
= cpu_to_be16(QM_CGR_WE_CSCN_TARG
);
2522 qm_cgr_cscn_targ_clear(&local_opts
.cgr
, PORTAL_IDX(p
),
2523 be32_to_cpu(cgr_state
.cgr
.cscn_targ
));
2525 ret
= qm_modify_cgr(cgr
, 0, &local_opts
);
2527 /* add back to the list */
2528 list_add(&cgr
->node
, &p
->cgr_cbs
);
2530 spin_unlock_irqrestore(&p
->cgr_lock
, irqflags
);
2532 put_affine_portal();
2535 EXPORT_SYMBOL(qman_delete_cgr
);
2538 struct qman_cgr
*cgr
;
2539 struct completion completion
;
2542 static void qman_delete_cgr_smp_call(void *p
)
2544 qman_delete_cgr((struct qman_cgr
*)p
);
2547 void qman_delete_cgr_safe(struct qman_cgr
*cgr
)
2550 if (qman_cgr_cpus
[cgr
->cgrid
] != smp_processor_id()) {
2551 smp_call_function_single(qman_cgr_cpus
[cgr
->cgrid
],
2552 qman_delete_cgr_smp_call
, cgr
, true);
2557 qman_delete_cgr(cgr
);
2560 EXPORT_SYMBOL(qman_delete_cgr_safe
);
2564 static int _qm_mr_consume_and_match_verb(struct qm_portal
*p
, int v
)
2566 const union qm_mr_entry
*msg
;
2569 qm_mr_pvb_update(p
);
2570 msg
= qm_mr_current(p
);
2572 if ((msg
->verb
& QM_MR_VERB_TYPE_MASK
) == v
)
2575 qm_mr_cci_consume_to_current(p
);
2576 qm_mr_pvb_update(p
);
2577 msg
= qm_mr_current(p
);
2582 static int _qm_dqrr_consume_and_match(struct qm_portal
*p
, u32 fqid
, int s
,
2585 const struct qm_dqrr_entry
*dqrr
;
2589 qm_dqrr_pvb_update(p
);
2590 dqrr
= qm_dqrr_current(p
);
2593 } while (wait
&& !dqrr
);
2596 if (qm_fqid_get(dqrr
) == fqid
&& (dqrr
->stat
& s
))
2598 qm_dqrr_cdc_consume_1ptr(p
, dqrr
, 0);
2599 qm_dqrr_pvb_update(p
);
2601 dqrr
= qm_dqrr_current(p
);
2606 #define qm_mr_drain(p, V) \
2607 _qm_mr_consume_and_match_verb(p, QM_MR_VERB_##V)
2609 #define qm_dqrr_drain(p, f, S) \
2610 _qm_dqrr_consume_and_match(p, f, QM_DQRR_STAT_##S, false)
2612 #define qm_dqrr_drain_wait(p, f, S) \
2613 _qm_dqrr_consume_and_match(p, f, QM_DQRR_STAT_##S, true)
2615 #define qm_dqrr_drain_nomatch(p) \
2616 _qm_dqrr_consume_and_match(p, 0, 0, false)
2618 int qman_shutdown_fq(u32 fqid
)
2620 struct qman_portal
*p
, *channel_portal
;
2622 union qm_mc_command
*mcc
;
2623 union qm_mc_result
*mcr
;
2624 int orl_empty
, drain
= 0, ret
= 0;
2628 p
= get_affine_portal();
2629 dev
= p
->config
->dev
;
2630 /* Determine the state of the FQID */
2631 mcc
= qm_mc_start(&p
->p
);
2632 qm_fqid_set(&mcc
->fq
, fqid
);
2633 qm_mc_commit(&p
->p
, QM_MCC_VERB_QUERYFQ_NP
);
2634 if (!qm_mc_result_timeout(&p
->p
, &mcr
)) {
2635 dev_err(dev
, "QUERYFQ_NP timeout\n");
2640 DPAA_ASSERT((mcr
->verb
& QM_MCR_VERB_MASK
) == QM_MCR_VERB_QUERYFQ_NP
);
2641 state
= mcr
->queryfq_np
.state
& QM_MCR_NP_STATE_MASK
;
2642 if (state
== QM_MCR_NP_STATE_OOS
)
2643 goto out
; /* Already OOS, no need to do anymore checks */
2645 /* Query which channel the FQ is using */
2646 mcc
= qm_mc_start(&p
->p
);
2647 qm_fqid_set(&mcc
->fq
, fqid
);
2648 qm_mc_commit(&p
->p
, QM_MCC_VERB_QUERYFQ
);
2649 if (!qm_mc_result_timeout(&p
->p
, &mcr
)) {
2650 dev_err(dev
, "QUERYFQ timeout\n");
2655 DPAA_ASSERT((mcr
->verb
& QM_MCR_VERB_MASK
) == QM_MCR_VERB_QUERYFQ
);
2656 /* Need to store these since the MCR gets reused */
2657 channel
= qm_fqd_get_chan(&mcr
->queryfq
.fqd
);
2658 qm_fqd_get_wq(&mcr
->queryfq
.fqd
);
2660 if (channel
< qm_channel_pool1
) {
2661 channel_portal
= get_portal_for_channel(channel
);
2662 if (channel_portal
== NULL
) {
2663 dev_err(dev
, "Can't find portal for dedicated channel 0x%x\n",
2672 case QM_MCR_NP_STATE_TEN_SCHED
:
2673 case QM_MCR_NP_STATE_TRU_SCHED
:
2674 case QM_MCR_NP_STATE_ACTIVE
:
2675 case QM_MCR_NP_STATE_PARKED
:
2677 mcc
= qm_mc_start(&channel_portal
->p
);
2678 qm_fqid_set(&mcc
->fq
, fqid
);
2679 qm_mc_commit(&channel_portal
->p
, QM_MCC_VERB_ALTER_RETIRE
);
2680 if (!qm_mc_result_timeout(&channel_portal
->p
, &mcr
)) {
2681 dev_err(dev
, "ALTER_RETIRE timeout\n");
2685 DPAA_ASSERT((mcr
->verb
& QM_MCR_VERB_MASK
) ==
2686 QM_MCR_VERB_ALTER_RETIRE
);
2687 res
= mcr
->result
; /* Make a copy as we reuse MCR below */
2689 if (res
== QM_MCR_RESULT_OK
)
2690 drain_mr_fqrni(&channel_portal
->p
);
2692 if (res
== QM_MCR_RESULT_PENDING
) {
2694 * Need to wait for the FQRN in the message ring, which
2695 * will only occur once the FQ has been drained. In
2696 * order for the FQ to drain the portal needs to be set
2697 * to dequeue from the channel the FQ is scheduled on
2701 /* Flag that we need to drain FQ */
2704 if (channel
>= qm_channel_pool1
&&
2705 channel
< qm_channel_pool1
+ 15) {
2706 /* Pool channel, enable the bit in the portal */
2707 } else if (channel
< qm_channel_pool1
) {
2708 /* Dedicated channel */
2710 dev_err(dev
, "Can't recover FQ 0x%x, ch: 0x%x",
2715 /* Set the sdqcr to drain this channel */
2716 if (channel
< qm_channel_pool1
)
2717 qm_dqrr_sdqcr_set(&channel_portal
->p
,
2718 QM_SDQCR_TYPE_ACTIVE
|
2719 QM_SDQCR_CHANNELS_DEDICATED
);
2721 qm_dqrr_sdqcr_set(&channel_portal
->p
,
2722 QM_SDQCR_TYPE_ACTIVE
|
2723 QM_SDQCR_CHANNELS_POOL_CONV
2726 /* Keep draining DQRR while checking the MR*/
2727 qm_dqrr_drain_nomatch(&channel_portal
->p
);
2728 /* Process message ring too */
2729 found_fqrn
= qm_mr_drain(&channel_portal
->p
,
2732 } while (!found_fqrn
);
2734 qm_dqrr_sdqcr_set(&channel_portal
->p
,
2735 channel_portal
->sdqcr
);
2738 if (res
!= QM_MCR_RESULT_OK
&&
2739 res
!= QM_MCR_RESULT_PENDING
) {
2740 dev_err(dev
, "retire_fq failed: FQ 0x%x, res=0x%x\n",
2745 if (!(mcr
->alterfq
.fqs
& QM_MCR_FQS_ORLPRESENT
)) {
2747 * ORL had no entries, no need to wait until the
2753 * Retirement succeeded, check to see if FQ needs
2756 if (drain
|| mcr
->alterfq
.fqs
& QM_MCR_FQS_NOTEMPTY
) {
2757 /* FQ is Not Empty, drain using volatile DQ commands */
2759 u32 vdqcr
= fqid
| QM_VDQCR_NUMFRAMES_SET(3);
2761 qm_dqrr_vdqcr_set(&p
->p
, vdqcr
);
2763 * Wait for a dequeue and process the dequeues,
2764 * making sure to empty the ring completely
2766 } while (!qm_dqrr_drain_wait(&p
->p
, fqid
, FQ_EMPTY
));
2769 while (!orl_empty
) {
2770 /* Wait for the ORL to have been completely drained */
2771 orl_empty
= qm_mr_drain(&p
->p
, FQRL
);
2774 mcc
= qm_mc_start(&p
->p
);
2775 qm_fqid_set(&mcc
->fq
, fqid
);
2776 qm_mc_commit(&p
->p
, QM_MCC_VERB_ALTER_OOS
);
2777 if (!qm_mc_result_timeout(&p
->p
, &mcr
)) {
2782 DPAA_ASSERT((mcr
->verb
& QM_MCR_VERB_MASK
) ==
2783 QM_MCR_VERB_ALTER_OOS
);
2784 if (mcr
->result
!= QM_MCR_RESULT_OK
) {
2785 dev_err(dev
, "OOS after drain fail: FQ 0x%x (0x%x)\n",
2792 case QM_MCR_NP_STATE_RETIRED
:
2793 /* Send OOS Command */
2794 mcc
= qm_mc_start(&p
->p
);
2795 qm_fqid_set(&mcc
->fq
, fqid
);
2796 qm_mc_commit(&p
->p
, QM_MCC_VERB_ALTER_OOS
);
2797 if (!qm_mc_result_timeout(&p
->p
, &mcr
)) {
2802 DPAA_ASSERT((mcr
->verb
& QM_MCR_VERB_MASK
) ==
2803 QM_MCR_VERB_ALTER_OOS
);
2804 if (mcr
->result
!= QM_MCR_RESULT_OK
) {
2805 dev_err(dev
, "OOS fail: FQ 0x%x (0x%x)\n",
2812 case QM_MCR_NP_STATE_OOS
:
2821 put_affine_portal();
2825 const struct qm_portal_config
*qman_get_qm_portal_config(
2826 struct qman_portal
*portal
)
2828 return portal
->config
;
2830 EXPORT_SYMBOL(qman_get_qm_portal_config
);
2832 struct gen_pool
*qm_fqalloc
; /* FQID allocator */
2833 struct gen_pool
*qm_qpalloc
; /* pool-channel allocator */
2834 struct gen_pool
*qm_cgralloc
; /* CGR ID allocator */
2836 static int qman_alloc_range(struct gen_pool
*p
, u32
*result
, u32 cnt
)
2843 addr
= gen_pool_alloc(p
, cnt
);
2847 *result
= addr
& ~DPAA_GENALLOC_OFF
;
2852 int qman_alloc_fqid_range(u32
*result
, u32 count
)
2854 return qman_alloc_range(qm_fqalloc
, result
, count
);
2856 EXPORT_SYMBOL(qman_alloc_fqid_range
);
2858 int qman_alloc_pool_range(u32
*result
, u32 count
)
2860 return qman_alloc_range(qm_qpalloc
, result
, count
);
2862 EXPORT_SYMBOL(qman_alloc_pool_range
);
2864 int qman_alloc_cgrid_range(u32
*result
, u32 count
)
2866 return qman_alloc_range(qm_cgralloc
, result
, count
);
2868 EXPORT_SYMBOL(qman_alloc_cgrid_range
);
2870 int qman_release_fqid(u32 fqid
)
2872 int ret
= qman_shutdown_fq(fqid
);
2875 pr_debug("FQID %d leaked\n", fqid
);
2879 gen_pool_free(qm_fqalloc
, fqid
| DPAA_GENALLOC_OFF
, 1);
2882 EXPORT_SYMBOL(qman_release_fqid
);
2884 static int qpool_cleanup(u32 qp
)
2887 * We query all FQDs starting from
2888 * FQID 1 until we get an "invalid FQID" error, looking for non-OOS FQDs
2889 * whose destination channel is the pool-channel being released.
2890 * When a non-OOS FQD is found we attempt to clean it up
2892 struct qman_fq fq
= {
2893 .fqid
= QM_FQID_RANGE_START
2898 struct qm_mcr_queryfq_np np
;
2900 err
= qman_query_fq_np(&fq
, &np
);
2902 /* FQID range exceeded, found no problems */
2904 else if (WARN_ON(err
))
2907 if ((np
.state
& QM_MCR_NP_STATE_MASK
) != QM_MCR_NP_STATE_OOS
) {
2910 err
= qman_query_fq(&fq
, &fqd
);
2913 if (qm_fqd_get_chan(&fqd
) == qp
) {
2914 /* The channel is the FQ's target, clean it */
2915 err
= qman_shutdown_fq(fq
.fqid
);
2918 * Couldn't shut down the FQ
2919 * so the pool must be leaked
2924 /* Move to the next FQID */
2929 int qman_release_pool(u32 qp
)
2933 ret
= qpool_cleanup(qp
);
2935 pr_debug("CHID %d leaked\n", qp
);
2939 gen_pool_free(qm_qpalloc
, qp
| DPAA_GENALLOC_OFF
, 1);
2942 EXPORT_SYMBOL(qman_release_pool
);
2944 static int cgr_cleanup(u32 cgrid
)
2947 * query all FQDs starting from FQID 1 until we get an "invalid FQID"
2948 * error, looking for non-OOS FQDs whose CGR is the CGR being released
2950 struct qman_fq fq
= {
2951 .fqid
= QM_FQID_RANGE_START
2956 struct qm_mcr_queryfq_np np
;
2958 err
= qman_query_fq_np(&fq
, &np
);
2960 /* FQID range exceeded, found no problems */
2962 else if (WARN_ON(err
))
2965 if ((np
.state
& QM_MCR_NP_STATE_MASK
) != QM_MCR_NP_STATE_OOS
) {
2968 err
= qman_query_fq(&fq
, &fqd
);
2971 if (be16_to_cpu(fqd
.fq_ctrl
) & QM_FQCTRL_CGE
&&
2972 fqd
.cgid
== cgrid
) {
2973 pr_err("CRGID 0x%x is being used by FQID 0x%x, CGR will be leaked\n",
2978 /* Move to the next FQID */
2983 int qman_release_cgrid(u32 cgrid
)
2987 ret
= cgr_cleanup(cgrid
);
2989 pr_debug("CGRID %d leaked\n", cgrid
);
2993 gen_pool_free(qm_cgralloc
, cgrid
| DPAA_GENALLOC_OFF
, 1);
2996 EXPORT_SYMBOL(qman_release_cgrid
);