1 // SPDX-License-Identifier: GPL-2.0
2 /* irq.c: UltraSparc IRQ handling/init/registry.
4 * Copyright (C) 1997, 2007, 2008 David S. Miller (davem@davemloft.net)
5 * Copyright (C) 1998 Eddie C. Dost (ecd@skynet.be)
6 * Copyright (C) 1998 Jakub Jelinek (jj@ultra.linux.cz)
9 #include <linux/sched.h>
10 #include <linux/linkage.h>
11 #include <linux/ptrace.h>
12 #include <linux/errno.h>
13 #include <linux/kernel_stat.h>
14 #include <linux/signal.h>
16 #include <linux/interrupt.h>
17 #include <linux/slab.h>
18 #include <linux/random.h>
19 #include <linux/init.h>
20 #include <linux/delay.h>
21 #include <linux/proc_fs.h>
22 #include <linux/seq_file.h>
23 #include <linux/ftrace.h>
24 #include <linux/irq.h>
26 #include <asm/ptrace.h>
27 #include <asm/processor.h>
28 #include <linux/atomic.h>
31 #include <asm/iommu.h>
33 #include <asm/oplib.h>
35 #include <asm/timer.h>
37 #include <asm/starfire.h>
38 #include <linux/uaccess.h>
39 #include <asm/cache.h>
40 #include <asm/cpudata.h>
41 #include <asm/auxio.h>
43 #include <asm/hypervisor.h>
44 #include <asm/cacheflush.h>
50 struct ino_bucket
*ivector_table
;
51 unsigned long ivector_table_pa
;
53 /* On several sun4u processors, it is illegal to mix bypass and
54 * non-bypass accesses. Therefore we access all INO buckets
55 * using bypass accesses only.
57 static unsigned long bucket_get_chain_pa(unsigned long bucket_pa
)
61 __asm__
__volatile__("ldxa [%1] %2, %0"
64 offsetof(struct ino_bucket
,
66 "i" (ASI_PHYS_USE_EC
));
71 static void bucket_clear_chain_pa(unsigned long bucket_pa
)
73 __asm__
__volatile__("stxa %%g0, [%0] %1"
76 offsetof(struct ino_bucket
,
78 "i" (ASI_PHYS_USE_EC
));
81 static unsigned int bucket_get_irq(unsigned long bucket_pa
)
85 __asm__
__volatile__("lduwa [%1] %2, %0"
88 offsetof(struct ino_bucket
,
90 "i" (ASI_PHYS_USE_EC
));
95 static void bucket_set_irq(unsigned long bucket_pa
, unsigned int irq
)
97 __asm__
__volatile__("stwa %0, [%1] %2"
101 offsetof(struct ino_bucket
,
103 "i" (ASI_PHYS_USE_EC
));
106 #define irq_work_pa(__cpu) &(trap_block[(__cpu)].irq_worklist_pa)
108 static unsigned long hvirq_major __initdata
;
109 static int __init
early_hvirq_major(char *p
)
111 int rc
= kstrtoul(p
, 10, &hvirq_major
);
115 early_param("hvirq", early_hvirq_major
);
117 static int hv_irq_version
;
119 /* Major version 2.0 of HV_GRP_INTR added support for the VIRQ cookie
120 * based interfaces, but:
122 * 1) Several OSs, Solaris and Linux included, use them even when only
123 * negotiating version 1.0 (or failing to negotiate at all). So the
124 * hypervisor has a workaround that provides the VIRQ interfaces even
125 * when only verion 1.0 of the API is in use.
127 * 2) Second, and more importantly, with major version 2.0 these VIRQ
128 * interfaces only were actually hooked up for LDC interrupts, even
129 * though the Hypervisor specification clearly stated:
131 * The new interrupt API functions will be available to a guest
132 * when it negotiates version 2.0 in the interrupt API group 0x2. When
133 * a guest negotiates version 2.0, all interrupt sources will only
134 * support using the cookie interface, and any attempt to use the
135 * version 1.0 interrupt APIs numbered 0xa0 to 0xa6 will result in the
136 * ENOTSUPPORTED error being returned.
138 * with an emphasis on "all interrupt sources".
140 * To correct this, major version 3.0 was created which does actually
141 * support VIRQs for all interrupt sources (not just LDC devices). So
142 * if we want to move completely over the cookie based VIRQs we must
143 * negotiate major version 3.0 or later of HV_GRP_INTR.
145 static bool sun4v_cookie_only_virqs(void)
147 if (hv_irq_version
>= 3)
152 static void __init
irq_init_hv(void)
154 unsigned long hv_error
, major
, minor
= 0;
156 if (tlb_type
!= hypervisor
)
164 hv_error
= sun4v_hvapi_register(HV_GRP_INTR
, major
, &minor
);
166 hv_irq_version
= major
;
170 pr_info("SUN4V: Using IRQ API major %d, cookie only virqs %s\n",
172 sun4v_cookie_only_virqs() ? "enabled" : "disabled");
175 /* This function is for the timer interrupt.*/
176 int __init
arch_probe_nr_irqs(void)
181 #define DEFAULT_NUM_IVECS (0xfffU)
182 static unsigned int nr_ivec
= DEFAULT_NUM_IVECS
;
183 #define NUM_IVECS (nr_ivec)
185 static unsigned int __init
size_nr_ivec(void)
187 if (tlb_type
== hypervisor
) {
188 switch (sun4v_chip_type
) {
189 /* Athena's devhandle|devino is large.*/
190 case SUN4V_CHIP_SPARC64X
:
198 struct irq_handler_data
{
201 unsigned int dev_handle
;
202 unsigned int dev_ino
;
204 unsigned long sysino
;
206 struct ino_bucket bucket
;
211 static inline unsigned int irq_data_to_handle(struct irq_data
*data
)
213 struct irq_handler_data
*ihd
= irq_data_get_irq_handler_data(data
);
215 return ihd
->dev_handle
;
218 static inline unsigned int irq_data_to_ino(struct irq_data
*data
)
220 struct irq_handler_data
*ihd
= irq_data_get_irq_handler_data(data
);
225 static inline unsigned long irq_data_to_sysino(struct irq_data
*data
)
227 struct irq_handler_data
*ihd
= irq_data_get_irq_handler_data(data
);
232 void irq_free(unsigned int irq
)
234 void *data
= irq_get_handler_data(irq
);
237 irq_set_handler_data(irq
, NULL
);
238 irq_free_descs(irq
, 1);
241 unsigned int irq_alloc(unsigned int dev_handle
, unsigned int dev_ino
)
245 irq
= __irq_alloc_descs(-1, 1, 1, numa_node_id(), NULL
, NULL
);
254 static unsigned int cookie_exists(u32 devhandle
, unsigned int devino
)
256 unsigned long hv_err
, cookie
;
257 struct ino_bucket
*bucket
;
258 unsigned int irq
= 0U;
260 hv_err
= sun4v_vintr_get_cookie(devhandle
, devino
, &cookie
);
262 pr_err("HV get cookie failed hv_err = %ld\n", hv_err
);
266 if (cookie
& ((1UL << 63UL))) {
268 bucket
= (struct ino_bucket
*) __va(cookie
);
275 static unsigned int sysino_exists(u32 devhandle
, unsigned int devino
)
277 unsigned long sysino
= sun4v_devino_to_sysino(devhandle
, devino
);
278 struct ino_bucket
*bucket
;
281 bucket
= &ivector_table
[sysino
];
282 irq
= bucket_get_irq(__pa(bucket
));
287 void ack_bad_irq(unsigned int irq
)
289 pr_crit("BAD IRQ ack %d\n", irq
);
292 void irq_install_pre_handler(int irq
,
293 void (*func
)(unsigned int, void *, void *),
294 void *arg1
, void *arg2
)
296 pr_warn("IRQ pre handler NOT supported.\n");
300 * /proc/interrupts printing:
302 int arch_show_interrupts(struct seq_file
*p
, int prec
)
306 seq_printf(p
, "NMI: ");
307 for_each_online_cpu(j
)
308 seq_printf(p
, "%10u ", cpu_data(j
).__nmi_count
);
309 seq_printf(p
, " Non-maskable interrupts\n");
313 static unsigned int sun4u_compute_tid(unsigned long imap
, unsigned long cpuid
)
317 if (this_is_starfire
) {
318 tid
= starfire_translate(imap
, cpuid
);
319 tid
<<= IMAP_TID_SHIFT
;
322 if (tlb_type
== cheetah
|| tlb_type
== cheetah_plus
) {
325 __asm__ ("rdpr %%ver, %0" : "=r" (ver
));
326 if ((ver
>> 32UL) == __JALAPENO_ID
||
327 (ver
>> 32UL) == __SERRANO_ID
) {
328 tid
= cpuid
<< IMAP_TID_SHIFT
;
329 tid
&= IMAP_TID_JBUS
;
331 unsigned int a
= cpuid
& 0x1f;
332 unsigned int n
= (cpuid
>> 5) & 0x1f;
334 tid
= ((a
<< IMAP_AID_SHIFT
) |
335 (n
<< IMAP_NID_SHIFT
));
336 tid
&= (IMAP_AID_SAFARI
|
340 tid
= cpuid
<< IMAP_TID_SHIFT
;
349 static int irq_choose_cpu(unsigned int irq
, const struct cpumask
*affinity
)
354 cpumask_copy(&mask
, affinity
);
355 if (cpumask_equal(&mask
, cpu_online_mask
)) {
356 cpuid
= map_to_cpu(irq
);
360 cpumask_and(&tmp
, cpu_online_mask
, &mask
);
361 cpuid
= cpumask_empty(&tmp
) ? map_to_cpu(irq
) : cpumask_first(&tmp
);
367 #define irq_choose_cpu(irq, affinity) \
368 real_hard_smp_processor_id()
371 static void sun4u_irq_enable(struct irq_data
*data
)
373 struct irq_handler_data
*handler_data
;
375 handler_data
= irq_data_get_irq_handler_data(data
);
376 if (likely(handler_data
)) {
377 unsigned long cpuid
, imap
, val
;
380 cpuid
= irq_choose_cpu(data
->irq
,
381 irq_data_get_affinity_mask(data
));
382 imap
= handler_data
->imap
;
384 tid
= sun4u_compute_tid(imap
, cpuid
);
386 val
= upa_readq(imap
);
387 val
&= ~(IMAP_TID_UPA
| IMAP_TID_JBUS
|
388 IMAP_AID_SAFARI
| IMAP_NID_SAFARI
);
389 val
|= tid
| IMAP_VALID
;
390 upa_writeq(val
, imap
);
391 upa_writeq(ICLR_IDLE
, handler_data
->iclr
);
395 static int sun4u_set_affinity(struct irq_data
*data
,
396 const struct cpumask
*mask
, bool force
)
398 struct irq_handler_data
*handler_data
;
400 handler_data
= irq_data_get_irq_handler_data(data
);
401 if (likely(handler_data
)) {
402 unsigned long cpuid
, imap
, val
;
405 cpuid
= irq_choose_cpu(data
->irq
, mask
);
406 imap
= handler_data
->imap
;
408 tid
= sun4u_compute_tid(imap
, cpuid
);
410 val
= upa_readq(imap
);
411 val
&= ~(IMAP_TID_UPA
| IMAP_TID_JBUS
|
412 IMAP_AID_SAFARI
| IMAP_NID_SAFARI
);
413 val
|= tid
| IMAP_VALID
;
414 upa_writeq(val
, imap
);
415 upa_writeq(ICLR_IDLE
, handler_data
->iclr
);
421 /* Don't do anything. The desc->status check for IRQ_DISABLED in
422 * handler_irq() will skip the handler call and that will leave the
423 * interrupt in the sent state. The next ->enable() call will hit the
424 * ICLR register to reset the state machine.
426 * This scheme is necessary, instead of clearing the Valid bit in the
427 * IMAP register, to handle the case of IMAP registers being shared by
428 * multiple INOs (and thus ICLR registers). Since we use a different
429 * virtual IRQ for each shared IMAP instance, the generic code thinks
430 * there is only one user so it prematurely calls ->disable() on
433 * We have to provide an explicit ->disable() method instead of using
434 * NULL to get the default. The reason is that if the generic code
435 * sees that, it also hooks up a default ->shutdown method which
436 * invokes ->mask() which we do not want. See irq_chip_set_defaults().
438 static void sun4u_irq_disable(struct irq_data
*data
)
442 static void sun4u_irq_eoi(struct irq_data
*data
)
444 struct irq_handler_data
*handler_data
;
446 handler_data
= irq_data_get_irq_handler_data(data
);
447 if (likely(handler_data
))
448 upa_writeq(ICLR_IDLE
, handler_data
->iclr
);
451 static void sun4v_irq_enable(struct irq_data
*data
)
453 unsigned long cpuid
= irq_choose_cpu(data
->irq
,
454 irq_data_get_affinity_mask(data
));
455 unsigned int ino
= irq_data_to_sysino(data
);
458 err
= sun4v_intr_settarget(ino
, cpuid
);
460 printk(KERN_ERR
"sun4v_intr_settarget(%x,%lu): "
461 "err(%d)\n", ino
, cpuid
, err
);
462 err
= sun4v_intr_setstate(ino
, HV_INTR_STATE_IDLE
);
464 printk(KERN_ERR
"sun4v_intr_setstate(%x): "
465 "err(%d)\n", ino
, err
);
466 err
= sun4v_intr_setenabled(ino
, HV_INTR_ENABLED
);
468 printk(KERN_ERR
"sun4v_intr_setenabled(%x): err(%d)\n",
472 static int sun4v_set_affinity(struct irq_data
*data
,
473 const struct cpumask
*mask
, bool force
)
475 unsigned long cpuid
= irq_choose_cpu(data
->irq
, mask
);
476 unsigned int ino
= irq_data_to_sysino(data
);
479 err
= sun4v_intr_settarget(ino
, cpuid
);
481 printk(KERN_ERR
"sun4v_intr_settarget(%x,%lu): "
482 "err(%d)\n", ino
, cpuid
, err
);
487 static void sun4v_irq_disable(struct irq_data
*data
)
489 unsigned int ino
= irq_data_to_sysino(data
);
492 err
= sun4v_intr_setenabled(ino
, HV_INTR_DISABLED
);
494 printk(KERN_ERR
"sun4v_intr_setenabled(%x): "
495 "err(%d)\n", ino
, err
);
498 static void sun4v_irq_eoi(struct irq_data
*data
)
500 unsigned int ino
= irq_data_to_sysino(data
);
503 err
= sun4v_intr_setstate(ino
, HV_INTR_STATE_IDLE
);
505 printk(KERN_ERR
"sun4v_intr_setstate(%x): "
506 "err(%d)\n", ino
, err
);
509 static void sun4v_virq_enable(struct irq_data
*data
)
511 unsigned long dev_handle
= irq_data_to_handle(data
);
512 unsigned long dev_ino
= irq_data_to_ino(data
);
516 cpuid
= irq_choose_cpu(data
->irq
, irq_data_get_affinity_mask(data
));
518 err
= sun4v_vintr_set_target(dev_handle
, dev_ino
, cpuid
);
520 printk(KERN_ERR
"sun4v_vintr_set_target(%lx,%lx,%lu): "
522 dev_handle
, dev_ino
, cpuid
, err
);
523 err
= sun4v_vintr_set_state(dev_handle
, dev_ino
,
526 printk(KERN_ERR
"sun4v_vintr_set_state(%lx,%lx,"
527 "HV_INTR_STATE_IDLE): err(%d)\n",
528 dev_handle
, dev_ino
, err
);
529 err
= sun4v_vintr_set_valid(dev_handle
, dev_ino
,
532 printk(KERN_ERR
"sun4v_vintr_set_state(%lx,%lx,"
533 "HV_INTR_ENABLED): err(%d)\n",
534 dev_handle
, dev_ino
, err
);
537 static int sun4v_virt_set_affinity(struct irq_data
*data
,
538 const struct cpumask
*mask
, bool force
)
540 unsigned long dev_handle
= irq_data_to_handle(data
);
541 unsigned long dev_ino
= irq_data_to_ino(data
);
545 cpuid
= irq_choose_cpu(data
->irq
, mask
);
547 err
= sun4v_vintr_set_target(dev_handle
, dev_ino
, cpuid
);
549 printk(KERN_ERR
"sun4v_vintr_set_target(%lx,%lx,%lu): "
551 dev_handle
, dev_ino
, cpuid
, err
);
556 static void sun4v_virq_disable(struct irq_data
*data
)
558 unsigned long dev_handle
= irq_data_to_handle(data
);
559 unsigned long dev_ino
= irq_data_to_ino(data
);
563 err
= sun4v_vintr_set_valid(dev_handle
, dev_ino
,
566 printk(KERN_ERR
"sun4v_vintr_set_state(%lx,%lx,"
567 "HV_INTR_DISABLED): err(%d)\n",
568 dev_handle
, dev_ino
, err
);
571 static void sun4v_virq_eoi(struct irq_data
*data
)
573 unsigned long dev_handle
= irq_data_to_handle(data
);
574 unsigned long dev_ino
= irq_data_to_ino(data
);
577 err
= sun4v_vintr_set_state(dev_handle
, dev_ino
,
580 printk(KERN_ERR
"sun4v_vintr_set_state(%lx,%lx,"
581 "HV_INTR_STATE_IDLE): err(%d)\n",
582 dev_handle
, dev_ino
, err
);
585 static struct irq_chip sun4u_irq
= {
587 .irq_enable
= sun4u_irq_enable
,
588 .irq_disable
= sun4u_irq_disable
,
589 .irq_eoi
= sun4u_irq_eoi
,
590 .irq_set_affinity
= sun4u_set_affinity
,
591 .flags
= IRQCHIP_EOI_IF_HANDLED
,
594 static struct irq_chip sun4v_irq
= {
596 .irq_enable
= sun4v_irq_enable
,
597 .irq_disable
= sun4v_irq_disable
,
598 .irq_eoi
= sun4v_irq_eoi
,
599 .irq_set_affinity
= sun4v_set_affinity
,
600 .flags
= IRQCHIP_EOI_IF_HANDLED
,
603 static struct irq_chip sun4v_virq
= {
605 .irq_enable
= sun4v_virq_enable
,
606 .irq_disable
= sun4v_virq_disable
,
607 .irq_eoi
= sun4v_virq_eoi
,
608 .irq_set_affinity
= sun4v_virt_set_affinity
,
609 .flags
= IRQCHIP_EOI_IF_HANDLED
,
612 unsigned int build_irq(int inofixup
, unsigned long iclr
, unsigned long imap
)
614 struct irq_handler_data
*handler_data
;
615 struct ino_bucket
*bucket
;
619 BUG_ON(tlb_type
== hypervisor
);
621 ino
= (upa_readq(imap
) & (IMAP_IGN
| IMAP_INO
)) + inofixup
;
622 bucket
= &ivector_table
[ino
];
623 irq
= bucket_get_irq(__pa(bucket
));
625 irq
= irq_alloc(0, ino
);
626 bucket_set_irq(__pa(bucket
), irq
);
627 irq_set_chip_and_handler_name(irq
, &sun4u_irq
,
628 handle_fasteoi_irq
, "IVEC");
631 handler_data
= irq_get_handler_data(irq
);
632 if (unlikely(handler_data
))
635 handler_data
= kzalloc(sizeof(struct irq_handler_data
), GFP_ATOMIC
);
636 if (unlikely(!handler_data
)) {
637 prom_printf("IRQ: kzalloc(irq_handler_data) failed.\n");
640 irq_set_handler_data(irq
, handler_data
);
642 handler_data
->imap
= imap
;
643 handler_data
->iclr
= iclr
;
649 static unsigned int sun4v_build_common(u32 devhandle
, unsigned int devino
,
650 void (*handler_data_init
)(struct irq_handler_data
*data
,
651 u32 devhandle
, unsigned int devino
),
652 struct irq_chip
*chip
)
654 struct irq_handler_data
*data
;
657 irq
= irq_alloc(devhandle
, devino
);
661 data
= kzalloc(sizeof(struct irq_handler_data
), GFP_ATOMIC
);
662 if (unlikely(!data
)) {
663 pr_err("IRQ handler data allocation failed.\n");
669 irq_set_handler_data(irq
, data
);
670 handler_data_init(data
, devhandle
, devino
);
671 irq_set_chip_and_handler_name(irq
, chip
, handle_fasteoi_irq
, "IVEC");
678 static unsigned long cookie_assign(unsigned int irq
, u32 devhandle
,
681 struct irq_handler_data
*ihd
= irq_get_handler_data(irq
);
682 unsigned long hv_error
, cookie
;
684 /* handler_irq needs to find the irq. cookie is seen signed in
685 * sun4v_dev_mondo and treated as a non ivector_table delivery.
687 ihd
->bucket
.__irq
= irq
;
688 cookie
= ~__pa(&ihd
->bucket
);
690 hv_error
= sun4v_vintr_set_cookie(devhandle
, devino
, cookie
);
692 pr_err("HV vintr set cookie failed = %ld\n", hv_error
);
697 static void cookie_handler_data(struct irq_handler_data
*data
,
698 u32 devhandle
, unsigned int devino
)
700 data
->dev_handle
= devhandle
;
701 data
->dev_ino
= devino
;
704 static unsigned int cookie_build_irq(u32 devhandle
, unsigned int devino
,
705 struct irq_chip
*chip
)
707 unsigned long hv_error
;
710 irq
= sun4v_build_common(devhandle
, devino
, cookie_handler_data
, chip
);
712 hv_error
= cookie_assign(irq
, devhandle
, devino
);
721 static unsigned int sun4v_build_cookie(u32 devhandle
, unsigned int devino
)
725 irq
= cookie_exists(devhandle
, devino
);
729 irq
= cookie_build_irq(devhandle
, devino
, &sun4v_virq
);
735 static void sysino_set_bucket(unsigned int irq
)
737 struct irq_handler_data
*ihd
= irq_get_handler_data(irq
);
738 struct ino_bucket
*bucket
;
739 unsigned long sysino
;
741 sysino
= sun4v_devino_to_sysino(ihd
->dev_handle
, ihd
->dev_ino
);
742 BUG_ON(sysino
>= nr_ivec
);
743 bucket
= &ivector_table
[sysino
];
744 bucket_set_irq(__pa(bucket
), irq
);
747 static void sysino_handler_data(struct irq_handler_data
*data
,
748 u32 devhandle
, unsigned int devino
)
750 unsigned long sysino
;
752 sysino
= sun4v_devino_to_sysino(devhandle
, devino
);
753 data
->sysino
= sysino
;
756 static unsigned int sysino_build_irq(u32 devhandle
, unsigned int devino
,
757 struct irq_chip
*chip
)
761 irq
= sun4v_build_common(devhandle
, devino
, sysino_handler_data
, chip
);
765 sysino_set_bucket(irq
);
770 static int sun4v_build_sysino(u32 devhandle
, unsigned int devino
)
774 irq
= sysino_exists(devhandle
, devino
);
778 irq
= sysino_build_irq(devhandle
, devino
, &sun4v_irq
);
783 unsigned int sun4v_build_irq(u32 devhandle
, unsigned int devino
)
787 if (sun4v_cookie_only_virqs())
788 irq
= sun4v_build_cookie(devhandle
, devino
);
790 irq
= sun4v_build_sysino(devhandle
, devino
);
795 unsigned int sun4v_build_virq(u32 devhandle
, unsigned int devino
)
799 irq
= cookie_build_irq(devhandle
, devino
, &sun4v_virq
);
803 /* This is borrowed from the original function.
805 irq_set_status_flags(irq
, IRQ_NOAUTOEN
);
811 void *hardirq_stack
[NR_CPUS
];
812 void *softirq_stack
[NR_CPUS
];
814 void __irq_entry
handler_irq(int pil
, struct pt_regs
*regs
)
816 unsigned long pstate
, bucket_pa
;
817 struct pt_regs
*old_regs
;
820 clear_softint(1 << pil
);
822 old_regs
= set_irq_regs(regs
);
825 /* Grab an atomic snapshot of the pending IVECs. */
826 __asm__
__volatile__("rdpr %%pstate, %0\n\t"
827 "wrpr %0, %3, %%pstate\n\t"
830 "wrpr %0, 0x0, %%pstate\n\t"
831 : "=&r" (pstate
), "=&r" (bucket_pa
)
832 : "r" (irq_work_pa(smp_processor_id())),
836 orig_sp
= set_hardirq_stack();
839 unsigned long next_pa
;
842 next_pa
= bucket_get_chain_pa(bucket_pa
);
843 irq
= bucket_get_irq(bucket_pa
);
844 bucket_clear_chain_pa(bucket_pa
);
846 generic_handle_irq(irq
);
851 restore_hardirq_stack(orig_sp
);
854 set_irq_regs(old_regs
);
857 void do_softirq_own_stack(void)
859 void *orig_sp
, *sp
= softirq_stack
[smp_processor_id()];
861 sp
+= THREAD_SIZE
- 192 - STACK_BIAS
;
863 __asm__
__volatile__("mov %%sp, %0\n\t"
868 __asm__
__volatile__("mov %0, %%sp"
872 #ifdef CONFIG_HOTPLUG_CPU
873 void fixup_irqs(void)
877 for (irq
= 0; irq
< NR_IRQS
; irq
++) {
878 struct irq_desc
*desc
= irq_to_desc(irq
);
879 struct irq_data
*data
;
884 data
= irq_desc_get_irq_data(desc
);
885 raw_spin_lock_irqsave(&desc
->lock
, flags
);
886 if (desc
->action
&& !irqd_is_per_cpu(data
)) {
887 if (data
->chip
->irq_set_affinity
)
888 data
->chip
->irq_set_affinity(data
,
889 irq_data_get_affinity_mask(data
),
892 raw_spin_unlock_irqrestore(&desc
->lock
, flags
);
895 tick_ops
->disable_irq();
906 static struct sun5_timer
*prom_timers
;
907 static u64 prom_limit0
, prom_limit1
;
909 static void map_prom_timers(void)
911 struct device_node
*dp
;
912 const unsigned int *addr
;
914 /* PROM timer node hangs out in the top level of device siblings... */
915 dp
= of_find_node_by_path("/");
918 if (of_node_name_eq(dp
, "counter-timer"))
923 /* Assume if node is not present, PROM uses different tick mechanism
924 * which we should not care about.
927 prom_timers
= (struct sun5_timer
*) 0;
931 /* If PROM is really using this, it must be mapped by him. */
932 addr
= of_get_property(dp
, "address", NULL
);
934 prom_printf("PROM does not have timer mapped, trying to continue.\n");
935 prom_timers
= (struct sun5_timer
*) 0;
938 prom_timers
= (struct sun5_timer
*) ((unsigned long)addr
[0]);
941 static void kill_prom_timer(void)
946 /* Save them away for later. */
947 prom_limit0
= prom_timers
->limit0
;
948 prom_limit1
= prom_timers
->limit1
;
950 /* Just as in sun4c PROM uses timer which ticks at IRQ 14.
951 * We turn both off here just to be paranoid.
953 prom_timers
->limit0
= 0;
954 prom_timers
->limit1
= 0;
956 /* Wheee, eat the interrupt packet too... */
957 __asm__
__volatile__(
959 " ldxa [%%g0] %0, %%g1\n"
960 " ldxa [%%g2] %1, %%g1\n"
961 " stxa %%g0, [%%g0] %0\n"
964 : "i" (ASI_INTR_RECEIVE
), "i" (ASI_INTR_R
)
968 void notrace
init_irqwork_curcpu(void)
970 int cpu
= hard_smp_processor_id();
972 trap_block
[cpu
].irq_worklist_pa
= 0UL;
975 /* Please be very careful with register_one_mondo() and
976 * sun4v_register_mondo_queues().
978 * On SMP this gets invoked from the CPU trampoline before
979 * the cpu has fully taken over the trap table from OBP,
980 * and it's kernel stack + %g6 thread register state is
981 * not fully cooked yet.
983 * Therefore you cannot make any OBP calls, not even prom_printf,
984 * from these two routines.
986 static void notrace
register_one_mondo(unsigned long paddr
, unsigned long type
,
989 unsigned long num_entries
= (qmask
+ 1) / 64;
990 unsigned long status
;
992 status
= sun4v_cpu_qconf(type
, paddr
, num_entries
);
993 if (status
!= HV_EOK
) {
994 prom_printf("SUN4V: sun4v_cpu_qconf(%lu:%lx:%lu) failed, "
995 "err %lu\n", type
, paddr
, num_entries
, status
);
1000 void notrace
sun4v_register_mondo_queues(int this_cpu
)
1002 struct trap_per_cpu
*tb
= &trap_block
[this_cpu
];
1004 register_one_mondo(tb
->cpu_mondo_pa
, HV_CPU_QUEUE_CPU_MONDO
,
1005 tb
->cpu_mondo_qmask
);
1006 register_one_mondo(tb
->dev_mondo_pa
, HV_CPU_QUEUE_DEVICE_MONDO
,
1007 tb
->dev_mondo_qmask
);
1008 register_one_mondo(tb
->resum_mondo_pa
, HV_CPU_QUEUE_RES_ERROR
,
1010 register_one_mondo(tb
->nonresum_mondo_pa
, HV_CPU_QUEUE_NONRES_ERROR
,
1011 tb
->nonresum_qmask
);
1014 /* Each queue region must be a power of 2 multiple of 64 bytes in
1015 * size. The base real address must be aligned to the size of the
1016 * region. Thus, an 8KB queue must be 8KB aligned, for example.
1018 static void __init
alloc_one_queue(unsigned long *pa_ptr
, unsigned long qmask
)
1020 unsigned long size
= PAGE_ALIGN(qmask
+ 1);
1021 unsigned long order
= get_order(size
);
1024 p
= __get_free_pages(GFP_KERNEL
| __GFP_ZERO
, order
);
1026 prom_printf("SUN4V: Error, cannot allocate queue.\n");
1033 static void __init
init_cpu_send_mondo_info(struct trap_per_cpu
*tb
)
1039 BUILD_BUG_ON((NR_CPUS
* sizeof(u16
)) > PAGE_SIZE
);
1041 /* Make sure mondo block is 64byte aligned */
1042 p
= kzalloc(127, GFP_KERNEL
);
1044 prom_printf("SUN4V: Error, cannot allocate mondo block.\n");
1047 mondo
= (void *)(((unsigned long)p
+ 63) & ~0x3f);
1048 tb
->cpu_mondo_block_pa
= __pa(mondo
);
1050 page
= get_zeroed_page(GFP_KERNEL
);
1052 prom_printf("SUN4V: Error, cannot allocate cpu list page.\n");
1056 tb
->cpu_list_pa
= __pa(page
);
1060 /* Allocate mondo and error queues for all possible cpus. */
1061 static void __init
sun4v_init_mondo_queues(void)
1065 for_each_possible_cpu(cpu
) {
1066 struct trap_per_cpu
*tb
= &trap_block
[cpu
];
1068 alloc_one_queue(&tb
->cpu_mondo_pa
, tb
->cpu_mondo_qmask
);
1069 alloc_one_queue(&tb
->dev_mondo_pa
, tb
->dev_mondo_qmask
);
1070 alloc_one_queue(&tb
->resum_mondo_pa
, tb
->resum_qmask
);
1071 alloc_one_queue(&tb
->resum_kernel_buf_pa
, tb
->resum_qmask
);
1072 alloc_one_queue(&tb
->nonresum_mondo_pa
, tb
->nonresum_qmask
);
1073 alloc_one_queue(&tb
->nonresum_kernel_buf_pa
,
1074 tb
->nonresum_qmask
);
1078 static void __init
init_send_mondo_info(void)
1082 for_each_possible_cpu(cpu
) {
1083 struct trap_per_cpu
*tb
= &trap_block
[cpu
];
1085 init_cpu_send_mondo_info(tb
);
1089 static struct irqaction timer_irq_action
= {
1093 static void __init
irq_ivector_init(void)
1095 unsigned long size
, order
;
1098 /* If we are doing cookie only VIRQs then we do not need the ivector
1099 * table to process interrupts.
1101 if (sun4v_cookie_only_virqs())
1104 ivecs
= size_nr_ivec();
1105 size
= sizeof(struct ino_bucket
) * ivecs
;
1106 order
= get_order(size
);
1107 ivector_table
= (struct ino_bucket
*)
1108 __get_free_pages(GFP_KERNEL
| __GFP_ZERO
, order
);
1109 if (!ivector_table
) {
1110 prom_printf("Fatal error, cannot allocate ivector_table\n");
1113 __flush_dcache_range((unsigned long) ivector_table
,
1114 ((unsigned long) ivector_table
) + size
);
1116 ivector_table_pa
= __pa(ivector_table
);
1119 /* Only invoked on boot processor.*/
1120 void __init
init_IRQ(void)
1127 if (tlb_type
== hypervisor
)
1128 sun4v_init_mondo_queues();
1130 init_send_mondo_info();
1132 if (tlb_type
== hypervisor
) {
1133 /* Load up the boot cpu's entries. */
1134 sun4v_register_mondo_queues(hard_smp_processor_id());
1137 /* We need to clear any IRQ's pending in the soft interrupt
1138 * registers, a spurious one could be left around from the
1139 * PROM timer which we just disabled.
1141 clear_softint(get_softint());
1143 /* Now that ivector table is initialized, it is safe
1144 * to receive IRQ vector traps. We will normally take
1145 * one or two right now, in case some device PROM used
1146 * to boot us wants to speak to us. We just ignore them.
1148 __asm__
__volatile__("rdpr %%pstate, %%g1\n\t"
1149 "or %%g1, %0, %%g1\n\t"
1150 "wrpr %%g1, 0x0, %%pstate"
1155 irq_to_desc(0)->action
= &timer_irq_action
;