?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Expand PMF_FN_* macros.
[netbsd-mini2440.git] / sys / arch / powerpc / marvell / extintr.c
blobdf4f410f90a71addc313fb829ec5e6c07f755374
1 /* $NetBSD: extintr.c,v 1.19 2009/03/14 14:46:05 dsl Exp $ */
2
3 /*
4 * Copyright (c) 2002 Allegro Networks, Inc., Wasabi Systems, Inc.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 3. All advertising materials mentioning features or use of this software
16 * must display the following acknowledgement:
17 * This product includes software developed for the NetBSD Project by
18 * Allegro Networks, Inc., and Wasabi Systems, Inc.
19 * 4. The name of Allegro Networks, Inc. may not be used to endorse
20 * or promote products derived from this software without specific prior
21 * written permission.
22 * 5. The name of Wasabi Systems, Inc. may not be used to endorse
23 * or promote products derived from this software without specific prior
24 * written permission.
25 *
26 * THIS SOFTWARE IS PROVIDED BY ALLEGRO NETWORKS, INC. AND
27 * WASABI SYSTEMS, INC. ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
28 * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
29 * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
30 * IN NO EVENT SHALL EITHER ALLEGRO NETWORKS, INC. OR WASABI SYSTEMS, INC.
31 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
32 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
33 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
34 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
35 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
36 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
37 * POSSIBILITY OF SUCH DAMAGE.
38 */
39
40 /*
41 * extintr.c - external interrupt management for discovery
42 *
43 * Interrupts are software-prioritized and preempting,
44 * they are only actually masked when pending.
45 * this allows avoiding slow, off-CPU mask reprogramming for spl/splx.
46 * When a lower priority interrupt preempts a high one,
47 * it is pended and masked. Masks are re-enabled after service.
48 *
49 * `ci->ci_cpl' is a "priority level" not a bitmask.
50 * `irq' is a bit number in the 128 bit imask_t which reflects
51 * the GT-64260 Main Cause register pair (64 bits), and
52 * GPP Cause register (32 bits) interrupts.
53 *
54 * Intra IPL dispatch order is defined in cause_irq()
55 *
56 * Summary bits in cause registers are not valid IRQs
57 *
58 * Within a cause register bit vector ISRs are called in
59 * order of IRQ (descending).
60 *
61 * When IRQs are shared, ISRs are called in order on the queue
62 * (which is arbitrarily first-established-first-served).
63 *
64 * GT64260 GPP setup is for edge-triggered interrupts.
65 * We maintain a mask of level-triggered type IRQs
66 * and gather unlatched level from the GPP Value register.
67 *
68 * Software interrupts are just like regular IRQs,
69 * they are established, pended, and dispatched exactly the
70 * same as HW interrupts.
71 *
72 * 128 bit imask_t operations could be implemented with Altivec
73 * ("vand", "vor", etc however no "vcntlzw" or "vffs"....)
74 *
75 * creation Tue Feb 6 17:27:18 PST 2001 cliff
76 */
77
78 #include <sys/cdefs.h>
79 __KERNEL_RCSID(0, "$NetBSD: extintr.c,v 1.19 2009/03/14 14:46:05 dsl Exp $");
80
81 #include "opt_marvell.h"
82 #include "opt_kgdb.h"
83
84 #include <sys/param.h>
85 #include <sys/systm.h>
86 #include <sys/types.h>
87 #include <sys/malloc.h>
88 #include <sys/kernel.h>
89
90 #include <machine/psl.h>
91 #include <machine/bus.h>
92 #include <machine/cpu.h>
93 #include <machine/intr.h>
94 #ifdef KGDB
95 #include <machine/db_machdep.h>
96 #endif
97 #include <dev/marvell/gtreg.h>
98 #include <dev/marvell/gtvar.h>
99 #include <dev/marvell/gtintrreg.h>
100
101 #include <net/netisr.h>
102
103 #include <uvm/uvm_extern.h>
104
105 #if defined(DEBUG) && defined(DDB)
106 #endif
107
108 #ifdef DEBUG
109 # define STATIC
110 int intrdebug = 0;
111 # define DPRINTF(x) do { if (intrdebug) printf x ; } while (0)
112 # define DPRINTFN(n, x) do { if (intrdebug > (n)) printf x ; } while (0)
113 #else
114 # define STATIC static
115 # define DPRINTF(x)
116 # define DPRINTFN(n, x)
117 #endif
118
119 #ifdef DIAGNOSTIC
120 # define DIAGPRF(x) printf x
121 #else
122 # define DIAGPRF(x)
123 #endif
124
125 #define ILLEGAL_IRQ(x) (((x) < 0) || ((x) >= NIRQ) || \
126 ((1<<((x)&IMASK_BITMASK)) & imres.bits[(x)>>IMASK_WORDSHIFT]))
127
128 extern struct powerpc_bus_space gt_mem_bs_tag;
129 extern bus_space_handle_t gt_memh;
130 extern int safepri;
131
132 struct intrsource {
133 struct intrhand *is_hand;
134 struct evcnt is_evcnt;
135 int is_level;
136 int is_type;
137
138 const char *is_name;
139 };
140
141 /*
142 * Interrupt handler chains. intr_establish() inserts a handler into
143 * the list. The handler is called with its (single) argument.
144 */
145 struct intrhand {
146 int (*ih_fun)(void *);
147 void *ih_arg;
148 struct intrhand *ih_next;
149 struct intrsource *ih_source;
150 int ih_flags;
151 int ih_level; /* sanity only */
152 u_long ih_count;
153 int ih_softimask;
154 };
155 #define IH_ACTIVE 0x01
156
157 struct intrsource intr_sources[NIRQ];
158
159 struct intrhand *softnet_handlers[32];
160
161 const char intr_source_strings[NIRQ][16] = {
162 "unknown 0", "dev", "dma", "cpu",
163 "idma 01", "idma 23", "idma 45", "idma 67",
164 "timer 01", "timer 23", "timer 45", "timer 67",
165 "pci0-0", "pci0-1", "pci0-2", "pci0-3",
166 /*16*/ "pci1-0", "ecc", "pci1-1", "pci1-2",
167 "pci1-3", "pci0-outl", "pci0-outh", "pci1-outl",
168 "pci1-outh", "unknown 25", "pci0-inl", "pci0-inh",
169 "pci1-inl", "pci1-inh", "unknown 30", "unknown 31",
170 /*32*/ "ether 0", "ether 1", "ether 2", "unknown 35",
171 "sdma", "iic", "unknown 38", "brg",
172 "mpsc 0", "unknown 41", "mpsc 1", "comm",
173 "unknown 44", "unknown 45", "unknown 46", "unknown 47",
174 /*48*/ "unknown 48", "unknown 49", "unknown 50", "unknown 51",
175 "unknown 52", "unknown 53", "unknown 54", "unknown 55",
176 "gppsum 0", "gppsum 1", "gppsum 2", "gppsum 3",
177 "unknown 60", "unknown 61", "unknown 62", "unknown 63",
178 /*64*/ "gpp 0", "gpp 1", "gpp 2", "gpp 3",
179 "gpp 4", "gpp 5", "gpp 6", "gpp 7",
180 "gpp 8", "gpp 9", "gpp 10", "gpp 11",
181 "gpp 12", "gpp 13", "gpp 14", "gpp 15",
182 /*80*/ "gpp 16", "gpp 17", "gpp 18", "gpp 19",
183 "gpp 20", "gpp 21", "gpp 22", "gpp 23",
184 "gpp 24", "gpp 25", "gpp 26", "gpp 27",
185 "gpp 28", "gpp 29", "gpp 30", "gpp 31",
186 /*96*/ "soft 0", "soft 1", "soft 2", "soft 3",
187 "soft 4", "soft 5", "soft 6", "soft 7",
188 "soft 8", "soft 9", "soft 10", "soft 11",
189 "soft 12", "soft 13", "soft 14", "soft 15",
190 /*112*/ "soft 16", "soft 17", "soft 18", "soft 19",
191 "soft 20", "soft 21", "soft 22", "soft 23",
192 "soft 24", "soft 25", "soft 26", "soft clock",
193 "soft net", "soft iic", "soft serial", "hwclock",
194 };
195
196 #ifdef EXT_INTR_STATS
197 #define EXT_INTR_STAT_HISTSZ 4096
198 ext_intr_stat_t ext_intr_stats[NIRQ] = {{0},};
199 ext_intr_hist_t ext_intr_hist[NIRQ][EXT_INTR_STAT_HISTSZ];
200 #endif
201
202
203 #define GPP_RES ~GT_MPP_INTERRUPTS /* from config */
204
205 /*
206 * ipending and imen HW IRQs require MSR[EE]=0 protection
207 */
208 volatile imask_t ipending __attribute__ ((aligned(CACHELINESIZE)));
209 volatile imask_t imen __attribute__ ((aligned(CACHELINESIZE)));
210 imask_t imask[NIPL] __attribute__ ((aligned(CACHELINESIZE)));
211 const imask_t imres =
212 { { (IML_RES|IML_SUM), (IMH_RES|IMH_GPP_SUM), GPP_RES, SIR_RES } };
213
214 #ifdef DEBUG
215 struct intrframe *intrframe = 0;
216 static int intr_depth;
217 #define EXTINTR_SANITY_SZ 16
218 struct {
219 unsigned int ipl;
220 imask_t oimen;
221 } extintr_sanity[EXTINTR_SANITY_SZ] = {{ 0, }};
222
223 #define EXTINTR_SANITY() \
224 if (intr_depth < EXTINTR_SANITY_SZ) { \
225 extintr_sanity[intr_depth].ipl = ci->ci_cpl; \
226 imask_dup_v(&extintr_sanity[intr_depth].oimen, &imen); \
227 } \
228 intr_depth++
229 #else
230 #define EXTINTR_SANITY()
231 #endif /* DEBUG */
232
233 const char * const intr_typename_strings[NIST] = {
234 "(none)",
235 "pulse",
236 "edge",
237 "level",
238 "soft",
239 "clock"
240 };
241 u_int32_t gpp_intrtype_level_mask = 0;
242
243 STATIC void write_intr_mask(volatile imask_t *);
244 STATIC void intr_calculatemasks(void);
245 STATIC int ext_intr_cause(volatile imask_t *, volatile imask_t *,
246 const imask_t *);
247 STATIC int cause_irq(const imask_t *, const imask_t *);
248
249 static inline void
250 imask_print(char *str, volatile imask_t *imp)
251 {
252 DPRINTF(("%s: %#10x %#10x %#10x %#10x\n",
253 str,
254 imp->bits[IMASK_ICU_LO],
255 imp->bits[IMASK_ICU_HI],
256 imp->bits[IMASK_ICU_GPP],
257 imp->bits[IMASK_SOFTINT]));
258 }
259
260 /*
261 * initialize interrupt subsystem
262 */
263 void
264 init_interrupt(void)
265 {
266 safepri = IPL_NONE;
267 intr_calculatemasks();
268 EXT_INTR_STATS_INIT();
269 SPL_STATS_INIT();
270 }
271
272 const char *
273 intr_typename(int type)
274 {
275 if (type >= NIST)
276 return "illegal";
277 return intr_typename_strings[type];
278 }
279
280 /*
281 * intr_establish - register an interrupt handler.
282 */
283 void *
284 intr_establish(
285 int irq,
286 int type,
287 int level,
288 int (*ih_fun)(void *),
289 void *ih_arg)
290 {
291 struct intrsource *is = &intr_sources[irq];
292 struct intrhand **p, *ih;
293 register_t omsr;
294
295 if (ILLEGAL_IRQ(irq)) {
296 DIAGPRF(("intr_establish: illegal irq 0x%x", irq));
297 return NULL;
298 }
299 if (level <= IPL_NONE || level >= IPL_HIGH) {
300 DIAGPRF(("intr_establish: illegal level %d\n", level));
301 return NULL;
302 }
303 if (type <= IST_NONE || type >= NIST) {
304 DIAGPRF(("intr_establish: illegal type 0x%x", type));
305 return NULL;
306 }
307 if (((irq >= SIR_BASE) && (type != IST_SOFT))
308 || ((irq < SIR_BASE) && (type == IST_SOFT))) {
309 DIAGPRF(("intr_establish: irq %d out of range for type %s\n",
310 irq, intr_typename_strings[type]));
311 return NULL;
312 }
313 if (is->is_type != IST_NONE && is->is_type != type) {
314 DIAGPRF(("intr_establish: irq %d: can't share %s with %s",
315 irq, intr_typename_strings[is->is_type],
316 intr_typename_strings[type]));
317 return NULL;
318 }
319 if (is->is_level != IPL_NONE && is->is_level != level) {
320 DIAGPRF(("intr_establish: irq %d conflicting levels %d, %d\n",
321 irq, is->is_level, level));
322 return NULL;
323 }
324
325 ih = malloc(sizeof *ih, M_DEVBUF, cold ? M_NOWAIT : M_WAITOK);
326 if (ih == NULL) {
327 DIAGPRF(("intr_establish: can't malloc handler info"));
328 return NULL;
329 }
330
331 omsr = extintr_disable();
332
333 if (is->is_hand == NULL) {
334 is->is_evcnt.ev_count = 0;
335 evcnt_attach_dynamic(&is->is_evcnt, EVCNT_TYPE_INTR,
336 NULL, "discovery", intr_source_strings[irq]);
337 }
338
339 is->is_type = type;
340 is->is_level = level;
341
342 if ((IRQ_IS_GPP(irq)) && type == IST_LEVEL)
343 gpp_intrtype_level_mask |= (1 << (irq - IRQ_GPP_BASE));
344
345 for (p = &is->is_hand; *p != NULL; p = &(*p)->ih_next)
346 ;
347 ih->ih_fun = ih_fun;
348 ih->ih_arg = ih_arg;
349 ih->ih_level = level;
350 ih->ih_source = is;
351 ih->ih_next = NULL;
352 ih->ih_count = 0;
353 if (irq >= SIR_BASE) {
354 ih->ih_flags = 0;
355 ih->ih_softimask = SIBIT(irq);
356 } else {
357 ih->ih_flags = IH_ACTIVE;
358 ih->ih_softimask = 0;
359 }
360 *p = ih;
361
362 intr_calculatemasks();
363
364 extintr_restore(omsr);
365
366 return (ih);
367 }
368
369 /*
370 * intr_disestablish - deregister an interrupt handler.
371 */
372 void
373 intr_disestablish(void *arg)
374 {
375 struct intrhand *ih = arg;
376 struct intrsource *is = ih->ih_source;
377 int irq = is - intr_sources;
378 struct intrhand **p;
379 register_t omsr;
380
381 if (ILLEGAL_IRQ(irq))
382 panic("intr_disestablish: bogus irq %d", irq);
383
384 /*
385 * Remove the handler from the chain.
386 * This is O(n^2), too.
387 */
388 omsr = extintr_disable();
389 for (p = &is->is_hand; *p != NULL && (*p) != ih; p = &(*p)->ih_next)
390 ;
391 if (*p != NULL)
392 *p = (*p)->ih_next;
393 else
394 panic("intr_disestablish: handler not registered");
395 free((void *)ih, M_DEVBUF);
396
397 intr_calculatemasks();
398
399 if (is->is_hand == NULL) {
400 evcnt_detach(&is->is_evcnt);
401 if ((IRQ_IS_GPP(irq)) && (is->is_type == IST_LEVEL))
402 gpp_intrtype_level_mask &= ~(1 << (irq - IRQ_GPP_BASE));
403 is->is_level = IPL_NONE;
404 is->is_type = IST_NONE;
405 }
406 extintr_restore(omsr);
407 }
408
409 const char *
410 intr_string(int irq)
411 {
412 return intr_source_strings[irq];
413 }
414
415 const struct evcnt *
416 intr_evcnt(int irq)
417 {
418 return &intr_sources[irq].is_evcnt;
419 }
420
421 /*
422 * write_intr_mask - update GT-64260 ICU Mask registers
423 *
424 * - IRQs already pending are always disabled
425 * - ICU summary bits are always enabled
426 *
427 * always called with interrupts disabled
428 */
429 STATIC void
430 write_intr_mask(volatile imask_t *imp)
431 {
432 u_int32_t lo;
433 u_int32_t hi;
434 u_int32_t gpp;
435
436 imask_andnot_icu_vv(imp, &ipending);
437
438 lo = imp->bits[IMASK_ICU_LO];
439 hi = imp->bits[IMASK_ICU_HI];
440 gpp = imp->bits[IMASK_ICU_GPP];
441
442 lo |= IML_SUM;
443 hi |= IMH_GPP_SUM;
444
445 bus_space_write_4(&gt_mem_bs_tag, gt_memh, ICR_CIM_LO, lo);
446 bus_space_write_4(&gt_mem_bs_tag, gt_memh, ICR_CIM_HI, hi);
447 bus_space_write_4(&gt_mem_bs_tag, gt_memh, GT_GPP_Interrupt_Mask, gpp);
448 (void)bus_space_read_4(&gt_mem_bs_tag, gt_memh,
449 GT_GPP_Interrupt_Mask); /* R.A.W. */
450 }
451
452 /*
453 * intr_calculatemasks - generate dispatch tables based on established IRQs
454 *
455 * enforces uniform level and intrtype for all handlers of shared IRQ
456 * in case of conflict simply create a new IPL
457 *
458 * always called with interrupts disabled
459 */
460 STATIC void
461 intr_calculatemasks(void)
462 {
463 int irq, level;
464 struct intrhand *q;
465 struct intrsource *is;
466
467 /*
468 * level to use for each IRQ is defined by first to claim it
469 * if the IRQ is shared, level cannot vary
470 */
471 for (irq = 0, is = intr_sources; irq < NIRQ; irq++, is++) {
472 for (q = is->is_hand; q; q = q->ih_next)
473 if (q->ih_level != is->is_level)
474 panic("intr_calculatemasks: irq %d "
475 "conflicting levels %d, %d\n",
476 irq, q->ih_level, is->is_level);
477 }
478
479 /*
480 * imask[] indicates IRQs enabled at each level.
481 */
482 for (level = 0; level < NIPL; level++) {
483 imask_t *imp = &imask[level];
484 imask_zero(imp);
485 for (irq = 0, is = intr_sources; irq < NIRQ; irq++, is++) {
486 if (level < is->is_level) {
487 imask_orbit(imp, irq);
488 }
489 }
490 }
491
492 /* enable IRQs with handlers */
493 imask_zero_v(&imen);
494 for (irq = 0, is = intr_sources; irq < NIRQ; irq++, is++) {
495 if (is->is_hand) {
496 imask_orbit_v(&imen, irq);
497 }
498 }
499
500 /*
501 * enable all live interrupts
502 */
503 write_intr_mask(&imen);
504 }
505
506 /*
507 * ext_intr_cause - collect, ack (GPP), pend and mask ICU causes
508 *
509 * set (OR-in) current ICU causes in impendp
510 *
511 * read GT-64260 Main Interrupt Cause and
512 * GPP Interrupt Cause and Value registers
513 *
514 * always called with interrupts disabled
515 */
516 STATIC int
517 ext_intr_cause(
518 volatile imask_t *impendp,
519 volatile imask_t *imenp,
520 const imask_t *imresp)
521 {
522 u_int32_t lo;
523 u_int32_t hi;
524 u_int32_t gpp = 0;
525 u_int32_t gpp_v;
526
527 lo = bus_space_read_4(&gt_mem_bs_tag, gt_memh, ICR_MIC_LO);
528 lo &= ~(imresp->bits[IMASK_ICU_LO] | impendp->bits[IMASK_ICU_LO]);
529 imenp->bits[IMASK_ICU_LO] &= ~lo;
530
531 hi = bus_space_read_4(&gt_mem_bs_tag, gt_memh, ICR_MIC_HI);
532
533 if (hi & IMH_GPP_SUM) {
534 gpp = bus_space_read_4(&gt_mem_bs_tag, gt_memh, GT_GPP_Interrupt_Cause);
535 gpp &= ~imresp->bits[IMASK_ICU_GPP];
536 bus_space_write_4(&gt_mem_bs_tag, gt_memh, GT_GPP_Interrupt_Cause, ~gpp);
537 }
538
539 gpp_v = bus_space_read_4(&gt_mem_bs_tag, gt_memh, GT_GPP_Value);
540 KASSERT((gpp_intrtype_level_mask & imresp->bits[IMASK_ICU_GPP]) == 0);
541 gpp_v &= (gpp_intrtype_level_mask & imenp->bits[IMASK_ICU_GPP]);
542
543 gpp |= gpp_v;
544 gpp &= ~impendp->bits[IMASK_ICU_GPP];
545 imenp->bits[IMASK_ICU_GPP] &= ~gpp;
546
547 hi &= ~(imresp->bits[IMASK_ICU_HI] | impendp->bits[IMASK_ICU_HI]);
548 imenp->bits[IMASK_ICU_HI] &= ~hi;
549
550 write_intr_mask(imenp);
551
552 EXT_INTR_STATS_CAUSE(lo, hi, gpp, 0);
553
554 if ((lo | hi | gpp) == 0) {
555 return 0; /* nothing left */
556 }
557
558 /*
559 * post pending IRQs in caller's imask
560 */
561 impendp->bits[IMASK_ICU_LO] |= lo;
562 impendp->bits[IMASK_ICU_HI] |= hi;
563 impendp->bits[IMASK_ICU_GPP] |= gpp;
564 EXT_INTR_STATS_PEND(lo, hi, gpp, 0);
565
566 return 1;
567 }
568
569 /*
570 * cause_irq - find a qualified irq
571 *
572 * `cause' indicates pending interrupt causes
573 * `mask' indicates what interrupts are enabled
574 *
575 * dispatch order is: lo, hi, gpp, soft
576 *
577 * return -1 if no qualified interrupts are pending.
578 */
579 STATIC int
580 cause_irq(const imask_t *cause, const imask_t *mask)
581 {
582 u_int32_t clo = cause->bits[IMASK_ICU_LO];
583 u_int32_t chi = cause->bits[IMASK_ICU_HI];
584 u_int32_t cgpp = cause->bits[IMASK_ICU_GPP];
585 u_int32_t csft = cause->bits[IMASK_SOFTINT];
586 u_int32_t mlo = mask->bits[IMASK_ICU_LO];
587 u_int32_t mhi = mask->bits[IMASK_ICU_HI];
588 u_int32_t mgpp = mask->bits[IMASK_ICU_GPP];
589 u_int32_t msft = mask->bits[IMASK_SOFTINT];
590 int irq;
591
592 clo &= mlo;
593 chi &= mhi;
594 cgpp &= mgpp;
595 csft &= msft;
596 if ((clo | chi | cgpp | csft) == 0)
597 return -1;
598 if ((irq = 31 - cntlzw(clo)) >= 0)
599 return irq;
600 if ((irq = 31 - cntlzw(chi)) >= 0)
601 return (irq + 32);
602 if ((irq = 31 - cntlzw(cgpp)) >= 0)
603 return irq + IRQ_GPP_BASE;
604 if ((irq = 31 - cntlzw(csft)) >= 0)
605 return (irq + SIR_BASE);
606 return -1;
607 }
608
609 /*
610 * ext_intr - external interrupt service routine
611 *
612 * collect and post filtered causes, then have a go at dispatch
613 * always called (from locore.S) with interrupts disabled
614 */
615 void
616 ext_intr(struct intrframe *frame)
617 {
618 struct cpu_info * const ci = curcpu();
619 #ifdef DEBUG
620 struct intrframe *oframe;
621 #endif
622
623 #ifdef __HAVE_FAST_SOFTINTS
624 #error don't count soft interrupts here
625 #else
626 ci->ci_idepth++;
627 #endif
628 EXT_INTR_STATS_DECL(tstart);
629 EXT_INTR_STATS_DEPTH();
630
631 #ifdef DEBUG
632 if (extintr_disable() & PSL_EE)
633 panic("ext_intr: PSL_EE");
634 oframe = intrframe;
635 intrframe = frame;
636 EXTINTR_SANITY();
637 #endif
638
639 if (ext_intr_cause(&ipending, &imen, &imres) == 0) {
640 DIAGPRF(("ext_intr: stray interrupt\n"));
641 #ifdef DEBUG
642 DPRINTF(("cpl %d\n", ci->ci_cpl));
643 imask_print("imask[cpl]", (volatile imask_t *)&imask[ci->ci_cpl]);
644 imask_print("ipending ", &ipending);
645 imask_print("imen ", &imen);
646 #endif
647 } else {
648 do {
649 intr_dispatch();
650 } while (ext_intr_cause(&ipending, &imen, &imres) != 0);
651 }
652 #ifdef DEBUG
653 intrframe = oframe;
654 intr_depth--;
655 #endif
656 ci->ci_idepth--;
657 }
658
659 /*
660 * intr_dispatch - process pending and soft interrupts
661 *
662 * always called with interrupts disabled
663 * pending interrupts must be masked off in ICU
664 */
665 void
666 intr_dispatch(void)
667 {
668 struct cpu_info * const ci = curcpu();
669 struct intrhand *ih;
670 imask_t *imp;
671 int irq;
672 int ipl;
673 int ocpl;
674 imask_t imdisp;
675 EXT_INTR_STATS_DECL(tstart);
676
677 KASSERT((extintr_disable() & PSL_EE) == 0);
678 KASSERT(ci->ci_cpl < IPL_HIGH);
679 KASSERT(ci->ci_cpl >= IPL_NONE);
680
681 if (ci->ci_cpl >= IPL_HIGH)
682 return;
683 ocpl = ci->ci_cpl;
684
685 /*
686 * process pending irpts that are not masked at ocpl and above
687 * copy and clear these from ipending while irpts are disabled
688 */
689 loop:
690 imask_dup_v(&imdisp, &ipending);
691 imask_and(&imdisp, &imask[ocpl]);
692
693 imdisp.bits[IMASK_SOFTINT] &= imen.bits[IMASK_SOFTINT];
694 imen.bits[IMASK_SOFTINT] &= ~imdisp.bits[IMASK_SOFTINT];
695
696 if (imask_empty(&imdisp)) {
697 ci->ci_cpl = ocpl;
698 SPL_STATS_LOG(ocpl, 0);
699 return;
700 }
701
702 imask_andnot_v(&ipending, &imdisp);
703 EXT_INTR_STATS_COMMIT_M(&imdisp);
704
705 imp = &imask[IPL_HIGH-1];
706 for (ipl=IPL_HIGH; ipl > ocpl; ipl--) {
707 ci->ci_cpl = ipl;
708 SPL_STATS_LOG(ipl, 0);
709 while ((irq = cause_irq(&imdisp, imp)) >= 0) {
710 struct intrsource *is = &intr_sources[irq];
711 #ifdef DEBUG
712 if (imask_andbit_v(&imen, irq) != 0)
713 panic("intr_dispatch(a): irq %d enabled", irq);
714 #endif
715 is->is_evcnt.ev_count++;
716 EXT_INTR_STATS_PRE(irq, tstart);
717 for (ih = is->is_hand; ih != NULL; ih = ih->ih_next) {
718 if (ih->ih_flags & IH_ACTIVE) {
719 int rv;
720
721 if (irq >= SIR_BASE)
722 ih->ih_flags &= ~IH_ACTIVE;
723 (void)extintr_enable();
724 rv = (*ih->ih_fun)(ih->ih_arg);
725 (void)extintr_disable();
726 if (rv != 0)
727 ih->ih_count++;
728 }
729
730 KASSERT(ci->ci_cpl == ipl);
731 }
732 EXT_INTR_STATS_POST(irq, tstart);
733 if (irq >= ICU_LEN)
734 uvmexp.softs++;
735 #ifdef DEBUG
736 if (imask_andbit_v(&imen, irq) != 0)
737 panic("intr_dispatch(b): irq %d enabled", irq);
738 #endif
739 imask_clrbit(&imdisp, irq);
740 imask_orbit_v(&imen, irq);
741
742 if (irq < ICU_LEN)
743 write_intr_mask(&imen);
744 }
745 imp--;
746 if (imask_empty(&imdisp))
747 break;
748 }
749 goto loop;
750 }
751
752
753 #ifdef EXT_INTR_STATS
754 /*
755 * ISR counts, pended, prempted, timing stats
756 */
757
758 int intr_depth_max = 0;
759 int ext_intr_stats_enb = 1;
760 ext_intr_stat_t ext_intr_stats[NIRQ] = {{0},};
761 ext_intr_stat_t *ext_intr_statp = 0;
762
763 #define EXT_INTR_STAT_HISTSZ 4096
764 ext_intr_hist_t ext_intr_hist[NIRQ][EXT_INTR_STAT_HISTSZ];
765
766 static inline u_int64_t
767 _mftb(void)
768 {
769 u_long scratch;
770 u_int64_t tb;
771
772 __asm volatile ("1: mftbu %0; mftb %0+1; mftbu %1; cmpw 0,%0,%1; bne 1b"
773 : "=r"(tb), "=r"(scratch));
774 return tb;
775 }
776
777 STATIC void
778 _ext_intr_stats_cause(u_int64_t tcause, u_int32_t cause, u_int32_t irqbase)
779 {
780 int irq;
781 u_int32_t bit;
782 unsigned int ix;
783 ext_intr_stat_t *statp;
784 ext_intr_hist_t *histp;
785
786 if (cause == 0)
787 return;
788 while ((irq = 31 - cntlzw(cause)) >= 0) {
789 bit = 1 << irq;
790 cause &= ~bit;
791 statp = &ext_intr_stats[irq + irqbase];
792 ix = statp->histix;
793 ++ix;
794 if (ix >= EXT_INTR_STAT_HISTSZ)
795 ix = 0;
796 statp->histix = ix;
797 histp = &statp->histp[ix];
798 memset(histp, 0, sizeof(ext_intr_hist_t));
799 histp->tcause = tcause;
800 }
801
802 }
803
804 void
805 ext_intr_stats_cause(u_int32_t lo, u_int32_t hi, u_int32_t gpp, u_int32_t soft)
806 {
807 u_int64_t tcause;
808
809 tcause = _mftb();
810
811 _ext_intr_stats_cause(tcause, lo, 0);
812 _ext_intr_stats_cause(tcause, hi, 32);
813 _ext_intr_stats_cause(tcause, gpp, IRQ_GPP_BASE);
814 _ext_intr_stats_cause(tcause, soft, SIR_BASE);
815 }
816
817 STATIC void
818 _ext_intr_stats_pend(u_int64_t tpend, u_int32_t cause, u_int32_t irqbase)
819 {
820 int irq;
821 u_int32_t bit;
822 ext_intr_stat_t *statp;
823 ext_intr_hist_t *histp;
824
825 if (cause == 0)
826 return;
827 while ((irq = 31 - cntlzw(cause)) >= 0) {
828 bit = 1 << irq;
829 cause &= ~bit;
830 EXT_INTR_STATS_PEND_IRQ(irq);
831 }
832
833 }
834
835 void
836 ext_intr_stats_pend(u_int32_t lo, u_int32_t hi, u_int32_t gpp, u_int32_t soft)
837 {
838 u_int64_t tpend;
839 ext_intr_stat_t *statp;
840 ext_intr_hist_t *histp;
841 int irq;
842 u_int32_t bit;
843
844 tpend = _mftb();
845
846 _ext_intr_stats_pend(tpend, lo, 0);
847 _ext_intr_stats_pend(tpend, hi, 32);
848 _ext_intr_stats_pend(tpend, gpp, IRQ_GPP_BASE);
849 _ext_intr_stats_pend(tpend, soft, SIR_BASE);
850 }
851
852 STATIC void
853 _ext_intr_stats_commit(u_int64_t tcommit, u_int32_t cause, u_int32_t irqbase)
854 {
855 int irq;
856 u_int32_t bit;
857 ext_intr_stat_t *statp;
858 ext_intr_hist_t *histp;
859
860 if (cause == 0)
861 return;
862 while ((irq = 31 - cntlzw(cause)) >= 0) {
863 bit = 1 << irq;
864 cause &= ~bit;
865 statp = &ext_intr_stats[irq + irqbase];
866 histp = &statp->histp[statp->histix];
867 histp->tcommit = tcommit;
868 }
869
870 }
871
872 void
873 ext_intr_stats_commit_m(imask_t *imp)
874 {
875 u_int64_t tcommit;
876 ext_intr_stat_t *statp;
877 ext_intr_hist_t *histp;
878 int irq;
879 u_int32_t bit;
880
881 tcommit = _mftb();
882
883 _ext_intr_stats_commit(tcommit, (*imp)[IMASK_ICU_LO], 0);
884 _ext_intr_stats_commit(tcommit, (*imp)[IMASK_ICU_HI], 32);
885 _ext_intr_stats_commit(tcommit, (*imp)[IMASK_ICU_GPP], IRQ_GPP_BASE);
886 _ext_intr_stats_commit(tcommit, (*imp)[IMASK_SOFTINT], SIR_BASE);
887 }
888
889 void
890 ext_intr_stats_commit_irq(u_int irq)
891 {
892 u_int64_t tcommit;
893 ext_intr_stat_t *statp;
894 ext_intr_hist_t *histp;
895 u_int32_t bit;
896
897 tcommit = _mftb();
898
899 _ext_intr_stats_commit(tcommit, irq & (32-1), irq & ~(32-1));
900 }
901
902 u_int64_t
903 ext_intr_stats_pre(int irq)
904 {
905 ext_intr_stat_t *statp;
906
907 statp = &ext_intr_stats[irq];
908 #ifdef DEBUG
909 if (statp == ext_intr_statp)
910 panic("ext_intr_stats_pre: statp == ext_intr_statp");
911 #endif
912 statp->save = ext_intr_statp;
913 statp->borrowed = 0;
914 ext_intr_statp = statp;
915 return _mftb();
916 }
917
918 void
919 ext_intr_stats_post(int irq, u_int64_t tstart)
920 {
921 u_int64_t dt;
922 ext_intr_stat_t *sp;
923 ext_intr_stat_t *statp=NULL;
924 ext_intr_hist_t *histp=NULL;
925
926 dt = _mftb();
927
928 if (ext_intr_stats_enb <= 0) {
929 if (ext_intr_stats_enb == -1)
930 ext_intr_stats_enb = 1;
931 ext_intr_statp = statp->save;
932 statp->save = 0;
933 return;
934 }
935
936 statp = &ext_intr_stats[irq];
937 histp = &statp->histp[statp->histix];
938 histp->tstart = tstart;
939 histp->tfin = dt;
940 dt -= tstart;
941
942 sp = statp->save;
943 statp->save = 0;
944 if (sp) {
945 #ifdef DEBUG
946 if (statp == sp)
947 panic("ext_intr_stats_post: statp == sp");
948 #endif
949 sp->preempted[irq]++;
950 sp->borrowed += dt;
951 }
952 dt -= statp->borrowed;
953 statp->cnt++;
954 statp->sum += dt;
955
956 if ((dt < statp->min) || (statp->min == 0))
957 statp->min = dt;
958 if ((dt > statp->max) || (statp->max == 0))
959 statp->max = dt;
960
961 ext_intr_statp = sp;
962 }
963
964 void
965 ext_intr_stats_print(void)
966 {
967 unsigned int i;
968 ext_intr_stat_t *statp;
969 int irq;
970 unsigned int level;
971 u_int64_t avg;
972 struct intrhand *ih;
973 struct intrsource *is;
974
975 printf("intr_depth_max %d\n", intr_depth_max);
976 statp = ext_intr_stats;
977 for (irq = 0, is = intr_sources; irq < NIRQ; irq++, is++) {
978 ih = is->is_hand;
979 if (ih != 0) {
980 int once = 0;
981
982 avg = statp->sum / statp->cnt;
983 level = is->is_level;
984 printf("irq %d ", irq);
985 printf("lvl %d ", level);
986 printf("cnt %lld ", statp->cnt);
987 printf("avg %lld ", avg);
988 printf("min %lld ", statp->min);
989 printf("max %lld ", statp->max);
990 printf("pnd %lld\n", statp->pnd);
991
992 for (i=0; i < NIRQ; i++) {
993 if (statp->preempted[i]) {
994 if (!once) {
995 once = 1;
996 printf("preemption: ");
997 }
998 printf("%d:%ld, ", i,
999 statp->preempted[i]);
1000 }
1001 }
1002 if (once)
1003 printf("\n");
1004
1005 if (ih != 0) {
1006 printf("ISRs: ");
1007 do {
1008 printf("%p, ", ih->ih_fun);
1009 ih = ih->ih_next;
1010 } while (ih);
1011 printf("\n");
1012 }
1013 }
1014 statp++;
1015 }
1016 }
1017
1018 void
1019 ext_intr_hist_prsubr(int irq)
1020 {
1021 unsigned int cnt;
1022 ext_intr_stat_t *statp;
1023 ext_intr_hist_t *histp;
1024 struct intrhand *ih;
1025
1026 if (irq >= NIRQ)
1027 return;
1028 ih = intr_sources[irq].is_hand;
1029 statp = &ext_intr_stats[irq];
1030 histp = statp->histp;
1031 if (ih != 0) {
1032 cnt = (statp->cnt < EXT_INTR_STAT_HISTSZ) ?
1033 statp->cnt : EXT_INTR_STAT_HISTSZ;
1034 printf("irq %d: %d samples:\n", irq, cnt);
1035 while (cnt--) {
1036 printf("%llu %llu %llu %llu\n",
1037 histp->tcause, histp->tcommit,
1038 histp->tstart, histp->tfin);
1039 histp++;
1040 }
1041 }
1042 }
1043
1044 void
1045 ext_intr_hist_print(u_int32_t lo, u_int32_t hi, u_int32_t gpp, u_int32_t soft)
1046 {
1047 int irq;
1048 u_int32_t bit;
1049 unsigned int omsr;
1050
1051 omsr = extintr_disable();
1052
1053 while ((irq = 31 - cntlzw(lo)) >= 0) {
1054 bit = 1 << irq;
1055 lo &= ~bit;
1056 ext_intr_hist_prsubr(irq);
1057 }
1058
1059 while ((irq = 31 - cntlzw(hi)) >= 0) {
1060 bit = 1 << irq;
1061 hi &= ~bit;
1062 ext_intr_hist_prsubr(irq + 32);
1063 }
1064
1065 while ((irq = 31 - cntlzw(gpp)) >= 0) {
1066 bit = 1 << irq;
1067 gpp &= ~bit;
1068 ext_intr_hist_prsubr(irq + IRQ_GPP_BASE);
1069 }
1070
1071 while ((irq = 31 - cntlzw(soft)) >= 0) {
1072 bit = 1 << irq;
1073 soft &= ~bit;
1074 ext_intr_hist_prsubr(irq + SIR_BASE);
1075 }
1076
1077 extintr_restore(omsr);
1078 }
1079
1080 void
1081 ext_intr_stats_init(void)
1082 {
1083 int irq;
1084
1085 memset(ext_intr_stats, 0, sizeof(ext_intr_stats));
1086 memset(ext_intr_hist, 0, sizeof(ext_intr_hist));
1087 for (irq=0; irq < NIRQ; irq++) {
1088 ext_intr_stats[irq].histp = &ext_intr_hist[irq][0];
1089 ext_intr_stats[irq].histix = EXT_INTR_STAT_HISTSZ-1;
1090 }
1091 }
1092 #endif /* EXT_INTR_STATS */
1093
1094 #ifdef SPL_STATS
1095 #define SPL_STATS_HISTSZ EXT_INTR_STAT_HISTSZ
1096 spl_hist_t spl_stats_hist[SPL_STATS_HISTSZ];
1097 unsigned int spl_stats_histix = 0;
1098 unsigned int spl_stats_enb = 1;
1099
1100
1101 void
1102 spl_stats_init(void)
1103 {
1104 spl_stats_histix = 0;
1105 }
1106
1107 void
1108 spl_stats_log(int ipl, int cc)
1109 {
1110 unsigned int ix = spl_stats_histix;
1111 spl_hist_t *histp;
1112 register_t *fp;
1113 register_t *pc;
1114
1115 if (spl_stats_enb == 0)
1116 return;
1117 histp = &spl_stats_hist[ix];
1118
1119 if (cc == 0) {
1120 /* log our calling address */
1121 __asm volatile ("mflr %0;" : "=r"(pc));
1122 pc--;
1123 } else {
1124 /* log our caller's calling address */
1125 __asm volatile ("mr %0,1;" : "=r"(fp));
1126 fp = (register_t *)*fp;
1127 pc = (register_t *)(fp[1] - sizeof(register_t));
1128 }
1129
1130 histp->level = ipl;
1131 histp->addr = (void *)pc;
1132 histp->time = _mftb();
1133 if (++ix >= SPL_STATS_HISTSZ)
1134 ix = 0;
1135 spl_stats_histix = ix;
1136 }
1137
1138 void
1139 spl_stats_print(void)
1140 {
1141 spl_hist_t *histp;
1142 int i;
1143
1144 printf("spl stats:\n");
1145 histp = &spl_stats_hist[0];
1146 for (i=SPL_STATS_HISTSZ; --i; ) {
1147 printf("%llu %d %p\n", histp->time, histp->level, histp->addr);
1148 histp++;
1149 }
1150 }
1151
1152 #endif /* SPL_STATS */
1153
1154 #ifndef SPL_INLINE
1155 int
1156 splraise(int ncpl)
1157 {
1158 struct cpu_info * const ci = curcpu();
1159 register_t omsr;
1160 int ocpl;
1161
1162 omsr = extintr_disable();
1163 ocpl = ci->ci_cpl;
1164 if (ncpl > ci->ci_cpl) {
1165 SPL_STATS_LOG(ncpl, 1);
1166 ci->ci_cpl = ncpl;
1167 if ((ncpl == IPL_HIGH) && ((omsr & PSL_EE) != 0)) {
1168 /* leave external interrupts disabled */
1169 return (ocpl | IPL_EE);
1170 }
1171 }
1172 extintr_restore(omsr);
1173 return (ocpl);
1174 }
1175
1176 void
1177 splx(int xcpl)
1178 {
1179 struct cpu_info * const ci = curcpu();
1180 imask_t *ncplp;
1181 unsigned int omsr;
1182 int ncpl = xcpl & IPL_PRIMASK;
1183 #ifdef DIAGNOSTIC
1184 int ocpl = ci->ci_cpl;
1185 #endif
1186
1187 omsr = extintr_disable();
1188 if (ci->ci_cpl < ncpl) {
1189 printf("ci->ci_cpl = %d, ncpl = %d\n", ci->ci_cpl, ncpl);
1190 }
1191 KASSERT(ncpl <= ci->ci_cpl);
1192 if (ncpl < ci->ci_cpl) {
1193 ci->ci_cpl = ncpl;
1194 ncplp = &imask[ncpl];
1195 SPL_STATS_LOG(ncpl, 1);
1196 if (imask_test_v(&ipending, ncplp))
1197 intr_dispatch();
1198 }
1199 if (xcpl & IPL_EE) {
1200 KASSERT(ocpl == IPL_HIGH);
1201 KASSERT((omsr & PSL_EE) == 0);
1202 omsr |= PSL_EE;
1203 }
1204 extintr_restore(omsr);
1205 }
1206
1207 int
1208 spllower(int ncpl)
1209 {
1210 struct cpu_info * const ci = curcpu();
1211 int ocpl;
1212 imask_t *ncplp;
1213 unsigned int omsr;
1214
1215 ncpl &= IPL_PRIMASK;
1216 ncplp = &imask[ncpl];
1217
1218 omsr = extintr_disable();
1219 ocpl = ci->ci_cpl;
1220 ci->ci_cpl = ncpl;
1221 SPL_STATS_LOG(ncpl, 1);
1222 #ifdef EXT_INTR_STATS
1223 ext_intr_statp = 0;
1224 #endif
1225 if (imask_test_v(&ipending, ncplp))
1226 intr_dispatch();
1227
1228 if (ncpl < IPL_HIGH)
1229 omsr |= PSL_EE;
1230 extintr_restore(omsr);
1231
1232 return (ocpl);
1233 }
1234 #endif
NetBSD on the FriendlyARM MINI2440