2 * Marvell Armada 370 and Armada XP SoC IRQ handling
4 * Copyright (C) 2012 Marvell
6 * Lior Amsalem <alior@marvell.com>
7 * Gregory CLEMENT <gregory.clement@free-electrons.com>
8 * Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
9 * Ben Dooks <ben.dooks@codethink.co.uk>
11 * This file is licensed under the terms of the GNU General Public
12 * License version 2. This program is licensed "as is" without any
13 * warranty of any kind, whether express or implied.
16 #include <linux/kernel.h>
17 #include <linux/module.h>
18 #include <linux/init.h>
19 #include <linux/irq.h>
20 #include <linux/interrupt.h>
21 #include <linux/irqchip.h>
22 #include <linux/irqchip/chained_irq.h>
23 #include <linux/cpu.h>
25 #include <linux/of_address.h>
26 #include <linux/of_irq.h>
27 #include <linux/of_pci.h>
28 #include <linux/irqdomain.h>
29 #include <linux/slab.h>
30 #include <linux/syscore_ops.h>
31 #include <linux/msi.h>
32 #include <asm/mach/arch.h>
33 #include <asm/exception.h>
34 #include <asm/smp_plat.h>
35 #include <asm/mach/irq.h>
37 /* Interrupt Controller Registers Map */
38 #define ARMADA_370_XP_INT_SET_MASK_OFFS (0x48)
39 #define ARMADA_370_XP_INT_CLEAR_MASK_OFFS (0x4C)
40 #define ARMADA_370_XP_INT_FABRIC_MASK_OFFS (0x54)
41 #define ARMADA_370_XP_INT_CAUSE_PERF(cpu) (1 << cpu)
43 #define ARMADA_370_XP_INT_CONTROL (0x00)
44 #define ARMADA_370_XP_INT_SET_ENABLE_OFFS (0x30)
45 #define ARMADA_370_XP_INT_CLEAR_ENABLE_OFFS (0x34)
46 #define ARMADA_370_XP_INT_SOURCE_CTL(irq) (0x100 + irq*4)
47 #define ARMADA_370_XP_INT_SOURCE_CPU_MASK 0xF
48 #define ARMADA_370_XP_INT_IRQ_FIQ_MASK(cpuid) ((BIT(0) | BIT(8)) << cpuid)
50 #define ARMADA_370_XP_CPU_INTACK_OFFS (0x44)
51 #define ARMADA_375_PPI_CAUSE (0x10)
53 #define ARMADA_370_XP_SW_TRIG_INT_OFFS (0x4)
54 #define ARMADA_370_XP_IN_DRBEL_MSK_OFFS (0xc)
55 #define ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS (0x8)
57 #define ARMADA_370_XP_MAX_PER_CPU_IRQS (28)
59 #define IPI_DOORBELL_START (0)
60 #define IPI_DOORBELL_END (8)
61 #define IPI_DOORBELL_MASK 0xFF
62 #define PCI_MSI_DOORBELL_START (16)
63 #define PCI_MSI_DOORBELL_NR (16)
64 #define PCI_MSI_DOORBELL_END (32)
65 #define PCI_MSI_DOORBELL_MASK 0xFFFF0000
67 static void __iomem
*per_cpu_int_base
;
68 static void __iomem
*main_int_base
;
69 static struct irq_domain
*armada_370_xp_mpic_domain
;
70 static u32 doorbell_mask_reg
;
71 static int parent_irq
;
73 static struct irq_domain
*armada_370_xp_msi_domain
;
74 static struct irq_domain
*armada_370_xp_msi_inner_domain
;
75 static DECLARE_BITMAP(msi_used
, PCI_MSI_DOORBELL_NR
);
76 static DEFINE_MUTEX(msi_used_lock
);
77 static phys_addr_t msi_doorbell_addr
;
80 static inline bool is_percpu_irq(irq_hw_number_t irq
)
82 if (irq
<= ARMADA_370_XP_MAX_PER_CPU_IRQS
)
90 * For shared global interrupts, mask/unmask global enable bit
91 * For CPU interrupts, mask/unmask the calling CPU's bit
93 static void armada_370_xp_irq_mask(struct irq_data
*d
)
95 irq_hw_number_t hwirq
= irqd_to_hwirq(d
);
97 if (!is_percpu_irq(hwirq
))
98 writel(hwirq
, main_int_base
+
99 ARMADA_370_XP_INT_CLEAR_ENABLE_OFFS
);
101 writel(hwirq
, per_cpu_int_base
+
102 ARMADA_370_XP_INT_SET_MASK_OFFS
);
105 static void armada_370_xp_irq_unmask(struct irq_data
*d
)
107 irq_hw_number_t hwirq
= irqd_to_hwirq(d
);
109 if (!is_percpu_irq(hwirq
))
110 writel(hwirq
, main_int_base
+
111 ARMADA_370_XP_INT_SET_ENABLE_OFFS
);
113 writel(hwirq
, per_cpu_int_base
+
114 ARMADA_370_XP_INT_CLEAR_MASK_OFFS
);
117 #ifdef CONFIG_PCI_MSI
119 static struct irq_chip armada_370_xp_msi_irq_chip
= {
121 .irq_mask
= pci_msi_mask_irq
,
122 .irq_unmask
= pci_msi_unmask_irq
,
125 static struct msi_domain_info armada_370_xp_msi_domain_info
= {
126 .flags
= (MSI_FLAG_USE_DEF_DOM_OPS
| MSI_FLAG_USE_DEF_CHIP_OPS
|
127 MSI_FLAG_MULTI_PCI_MSI
),
128 .chip
= &armada_370_xp_msi_irq_chip
,
131 static void armada_370_xp_compose_msi_msg(struct irq_data
*data
, struct msi_msg
*msg
)
133 msg
->address_lo
= lower_32_bits(msi_doorbell_addr
);
134 msg
->address_hi
= upper_32_bits(msi_doorbell_addr
);
135 msg
->data
= 0xf00 | (data
->hwirq
+ PCI_MSI_DOORBELL_START
);
138 static int armada_370_xp_msi_set_affinity(struct irq_data
*irq_data
,
139 const struct cpumask
*mask
, bool force
)
144 static struct irq_chip armada_370_xp_msi_bottom_irq_chip
= {
146 .irq_compose_msi_msg
= armada_370_xp_compose_msi_msg
,
147 .irq_set_affinity
= armada_370_xp_msi_set_affinity
,
150 static int armada_370_xp_msi_alloc(struct irq_domain
*domain
, unsigned int virq
,
151 unsigned int nr_irqs
, void *args
)
155 mutex_lock(&msi_used_lock
);
157 hwirq
= bitmap_find_next_zero_area(msi_used
, PCI_MSI_DOORBELL_NR
,
159 if (hwirq
>= PCI_MSI_DOORBELL_NR
) {
160 mutex_unlock(&msi_used_lock
);
164 bitmap_set(msi_used
, hwirq
, nr_irqs
);
165 mutex_unlock(&msi_used_lock
);
167 for (i
= 0; i
< nr_irqs
; i
++) {
168 irq_domain_set_info(domain
, virq
+ i
, hwirq
+ i
,
169 &armada_370_xp_msi_bottom_irq_chip
,
170 domain
->host_data
, handle_simple_irq
,
177 static void armada_370_xp_msi_free(struct irq_domain
*domain
,
178 unsigned int virq
, unsigned int nr_irqs
)
180 struct irq_data
*d
= irq_domain_get_irq_data(domain
, virq
);
182 mutex_lock(&msi_used_lock
);
183 bitmap_clear(msi_used
, d
->hwirq
, nr_irqs
);
184 mutex_unlock(&msi_used_lock
);
187 static const struct irq_domain_ops armada_370_xp_msi_domain_ops
= {
188 .alloc
= armada_370_xp_msi_alloc
,
189 .free
= armada_370_xp_msi_free
,
192 static int armada_370_xp_msi_init(struct device_node
*node
,
193 phys_addr_t main_int_phys_base
)
197 msi_doorbell_addr
= main_int_phys_base
+
198 ARMADA_370_XP_SW_TRIG_INT_OFFS
;
200 armada_370_xp_msi_inner_domain
=
201 irq_domain_add_linear(NULL
, PCI_MSI_DOORBELL_NR
,
202 &armada_370_xp_msi_domain_ops
, NULL
);
203 if (!armada_370_xp_msi_inner_domain
)
206 armada_370_xp_msi_domain
=
207 pci_msi_create_irq_domain(of_node_to_fwnode(node
),
208 &armada_370_xp_msi_domain_info
,
209 armada_370_xp_msi_inner_domain
);
210 if (!armada_370_xp_msi_domain
) {
211 irq_domain_remove(armada_370_xp_msi_inner_domain
);
215 reg
= readl(per_cpu_int_base
+ ARMADA_370_XP_IN_DRBEL_MSK_OFFS
)
216 | PCI_MSI_DOORBELL_MASK
;
218 writel(reg
, per_cpu_int_base
+
219 ARMADA_370_XP_IN_DRBEL_MSK_OFFS
);
221 /* Unmask IPI interrupt */
222 writel(1, per_cpu_int_base
+ ARMADA_370_XP_INT_CLEAR_MASK_OFFS
);
227 static inline int armada_370_xp_msi_init(struct device_node
*node
,
228 phys_addr_t main_int_phys_base
)
235 static DEFINE_RAW_SPINLOCK(irq_controller_lock
);
237 static int armada_xp_set_affinity(struct irq_data
*d
,
238 const struct cpumask
*mask_val
, bool force
)
240 irq_hw_number_t hwirq
= irqd_to_hwirq(d
);
241 unsigned long reg
, mask
;
244 /* Select a single core from the affinity mask which is online */
245 cpu
= cpumask_any_and(mask_val
, cpu_online_mask
);
246 mask
= 1UL << cpu_logical_map(cpu
);
248 raw_spin_lock(&irq_controller_lock
);
249 reg
= readl(main_int_base
+ ARMADA_370_XP_INT_SOURCE_CTL(hwirq
));
250 reg
= (reg
& (~ARMADA_370_XP_INT_SOURCE_CPU_MASK
)) | mask
;
251 writel(reg
, main_int_base
+ ARMADA_370_XP_INT_SOURCE_CTL(hwirq
));
252 raw_spin_unlock(&irq_controller_lock
);
254 return IRQ_SET_MASK_OK
;
258 static struct irq_chip armada_370_xp_irq_chip
= {
260 .irq_mask
= armada_370_xp_irq_mask
,
261 .irq_mask_ack
= armada_370_xp_irq_mask
,
262 .irq_unmask
= armada_370_xp_irq_unmask
,
264 .irq_set_affinity
= armada_xp_set_affinity
,
266 .flags
= IRQCHIP_SKIP_SET_WAKE
| IRQCHIP_MASK_ON_SUSPEND
,
269 static int armada_370_xp_mpic_irq_map(struct irq_domain
*h
,
270 unsigned int virq
, irq_hw_number_t hw
)
272 armada_370_xp_irq_mask(irq_get_irq_data(virq
));
273 if (!is_percpu_irq(hw
))
274 writel(hw
, per_cpu_int_base
+
275 ARMADA_370_XP_INT_CLEAR_MASK_OFFS
);
277 writel(hw
, main_int_base
+ ARMADA_370_XP_INT_SET_ENABLE_OFFS
);
278 irq_set_status_flags(virq
, IRQ_LEVEL
);
280 if (is_percpu_irq(hw
)) {
281 irq_set_percpu_devid(virq
);
282 irq_set_chip_and_handler(virq
, &armada_370_xp_irq_chip
,
283 handle_percpu_devid_irq
);
286 irq_set_chip_and_handler(virq
, &armada_370_xp_irq_chip
,
290 irq_clear_status_flags(virq
, IRQ_NOAUTOEN
);
295 static void armada_xp_mpic_smp_cpu_init(void)
300 control
= readl(main_int_base
+ ARMADA_370_XP_INT_CONTROL
);
301 nr_irqs
= (control
>> 2) & 0x3ff;
303 for (i
= 0; i
< nr_irqs
; i
++)
304 writel(i
, per_cpu_int_base
+ ARMADA_370_XP_INT_SET_MASK_OFFS
);
306 /* Clear pending IPIs */
307 writel(0, per_cpu_int_base
+ ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS
);
309 /* Enable first 8 IPIs */
310 writel(IPI_DOORBELL_MASK
, per_cpu_int_base
+
311 ARMADA_370_XP_IN_DRBEL_MSK_OFFS
);
313 /* Unmask IPI interrupt */
314 writel(0, per_cpu_int_base
+ ARMADA_370_XP_INT_CLEAR_MASK_OFFS
);
317 static void armada_xp_mpic_perf_init(void)
319 unsigned long cpuid
= cpu_logical_map(smp_processor_id());
321 /* Enable Performance Counter Overflow interrupts */
322 writel(ARMADA_370_XP_INT_CAUSE_PERF(cpuid
),
323 per_cpu_int_base
+ ARMADA_370_XP_INT_FABRIC_MASK_OFFS
);
327 static void armada_mpic_send_doorbell(const struct cpumask
*mask
,
331 unsigned long map
= 0;
333 /* Convert our logical CPU mask into a physical one. */
334 for_each_cpu(cpu
, mask
)
335 map
|= 1 << cpu_logical_map(cpu
);
338 * Ensure that stores to Normal memory are visible to the
339 * other CPUs before issuing the IPI.
344 writel((map
<< 8) | irq
, main_int_base
+
345 ARMADA_370_XP_SW_TRIG_INT_OFFS
);
348 static int armada_xp_mpic_starting_cpu(unsigned int cpu
)
350 armada_xp_mpic_perf_init();
351 armada_xp_mpic_smp_cpu_init();
355 static int mpic_cascaded_starting_cpu(unsigned int cpu
)
357 armada_xp_mpic_perf_init();
358 enable_percpu_irq(parent_irq
, IRQ_TYPE_NONE
);
363 static const struct irq_domain_ops armada_370_xp_mpic_irq_ops
= {
364 .map
= armada_370_xp_mpic_irq_map
,
365 .xlate
= irq_domain_xlate_onecell
,
368 #ifdef CONFIG_PCI_MSI
369 static void armada_370_xp_handle_msi_irq(struct pt_regs
*regs
, bool is_chained
)
373 msimask
= readl_relaxed(per_cpu_int_base
+
374 ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS
)
375 & PCI_MSI_DOORBELL_MASK
;
377 writel(~msimask
, per_cpu_int_base
+
378 ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS
);
380 for (msinr
= PCI_MSI_DOORBELL_START
;
381 msinr
< PCI_MSI_DOORBELL_END
; msinr
++) {
384 if (!(msimask
& BIT(msinr
)))
388 irq
= irq_find_mapping(armada_370_xp_msi_inner_domain
,
389 msinr
- PCI_MSI_DOORBELL_START
);
390 generic_handle_irq(irq
);
392 irq
= msinr
- PCI_MSI_DOORBELL_START
;
393 handle_domain_irq(armada_370_xp_msi_inner_domain
,
399 static void armada_370_xp_handle_msi_irq(struct pt_regs
*r
, bool b
) {}
402 static void armada_370_xp_mpic_handle_cascade_irq(struct irq_desc
*desc
)
404 struct irq_chip
*chip
= irq_desc_get_chip(desc
);
405 unsigned long irqmap
, irqn
, irqsrc
, cpuid
;
406 unsigned int cascade_irq
;
408 chained_irq_enter(chip
, desc
);
410 irqmap
= readl_relaxed(per_cpu_int_base
+ ARMADA_375_PPI_CAUSE
);
411 cpuid
= cpu_logical_map(smp_processor_id());
413 for_each_set_bit(irqn
, &irqmap
, BITS_PER_LONG
) {
414 irqsrc
= readl_relaxed(main_int_base
+
415 ARMADA_370_XP_INT_SOURCE_CTL(irqn
));
417 /* Check if the interrupt is not masked on current CPU.
418 * Test IRQ (0-1) and FIQ (8-9) mask bits.
420 if (!(irqsrc
& ARMADA_370_XP_INT_IRQ_FIQ_MASK(cpuid
)))
424 armada_370_xp_handle_msi_irq(NULL
, true);
428 cascade_irq
= irq_find_mapping(armada_370_xp_mpic_domain
, irqn
);
429 generic_handle_irq(cascade_irq
);
432 chained_irq_exit(chip
, desc
);
435 static void __exception_irq_entry
436 armada_370_xp_handle_irq(struct pt_regs
*regs
)
441 irqstat
= readl_relaxed(per_cpu_int_base
+
442 ARMADA_370_XP_CPU_INTACK_OFFS
);
443 irqnr
= irqstat
& 0x3FF;
449 handle_domain_irq(armada_370_xp_mpic_domain
,
456 armada_370_xp_handle_msi_irq(regs
, false);
463 ipimask
= readl_relaxed(per_cpu_int_base
+
464 ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS
)
467 writel(~ipimask
, per_cpu_int_base
+
468 ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS
);
470 /* Handle all pending doorbells */
471 for (ipinr
= IPI_DOORBELL_START
;
472 ipinr
< IPI_DOORBELL_END
; ipinr
++) {
473 if (ipimask
& (0x1 << ipinr
))
474 handle_IPI(ipinr
, regs
);
483 static int armada_370_xp_mpic_suspend(void)
485 doorbell_mask_reg
= readl(per_cpu_int_base
+
486 ARMADA_370_XP_IN_DRBEL_MSK_OFFS
);
490 static void armada_370_xp_mpic_resume(void)
495 /* Re-enable interrupts */
496 nirqs
= (readl(main_int_base
+ ARMADA_370_XP_INT_CONTROL
) >> 2) & 0x3ff;
497 for (irq
= 0; irq
< nirqs
; irq
++) {
498 struct irq_data
*data
;
501 virq
= irq_linear_revmap(armada_370_xp_mpic_domain
, irq
);
505 if (!is_percpu_irq(irq
))
506 writel(irq
, per_cpu_int_base
+
507 ARMADA_370_XP_INT_CLEAR_MASK_OFFS
);
509 writel(irq
, main_int_base
+
510 ARMADA_370_XP_INT_SET_ENABLE_OFFS
);
512 data
= irq_get_irq_data(virq
);
513 if (!irqd_irq_disabled(data
))
514 armada_370_xp_irq_unmask(data
);
517 /* Reconfigure doorbells for IPIs and MSIs */
518 writel(doorbell_mask_reg
,
519 per_cpu_int_base
+ ARMADA_370_XP_IN_DRBEL_MSK_OFFS
);
520 if (doorbell_mask_reg
& IPI_DOORBELL_MASK
)
521 writel(0, per_cpu_int_base
+ ARMADA_370_XP_INT_CLEAR_MASK_OFFS
);
522 if (doorbell_mask_reg
& PCI_MSI_DOORBELL_MASK
)
523 writel(1, per_cpu_int_base
+ ARMADA_370_XP_INT_CLEAR_MASK_OFFS
);
526 static struct syscore_ops armada_370_xp_mpic_syscore_ops
= {
527 .suspend
= armada_370_xp_mpic_suspend
,
528 .resume
= armada_370_xp_mpic_resume
,
531 static int __init
armada_370_xp_mpic_of_init(struct device_node
*node
,
532 struct device_node
*parent
)
534 struct resource main_int_res
, per_cpu_int_res
;
538 BUG_ON(of_address_to_resource(node
, 0, &main_int_res
));
539 BUG_ON(of_address_to_resource(node
, 1, &per_cpu_int_res
));
541 BUG_ON(!request_mem_region(main_int_res
.start
,
542 resource_size(&main_int_res
),
544 BUG_ON(!request_mem_region(per_cpu_int_res
.start
,
545 resource_size(&per_cpu_int_res
),
548 main_int_base
= ioremap(main_int_res
.start
,
549 resource_size(&main_int_res
));
550 BUG_ON(!main_int_base
);
552 per_cpu_int_base
= ioremap(per_cpu_int_res
.start
,
553 resource_size(&per_cpu_int_res
));
554 BUG_ON(!per_cpu_int_base
);
556 control
= readl(main_int_base
+ ARMADA_370_XP_INT_CONTROL
);
557 nr_irqs
= (control
>> 2) & 0x3ff;
559 for (i
= 0; i
< nr_irqs
; i
++)
560 writel(i
, main_int_base
+ ARMADA_370_XP_INT_CLEAR_ENABLE_OFFS
);
562 armada_370_xp_mpic_domain
=
563 irq_domain_add_linear(node
, nr_irqs
,
564 &armada_370_xp_mpic_irq_ops
, NULL
);
565 BUG_ON(!armada_370_xp_mpic_domain
);
566 armada_370_xp_mpic_domain
->bus_token
= DOMAIN_BUS_WIRED
;
568 /* Setup for the boot CPU */
569 armada_xp_mpic_perf_init();
570 armada_xp_mpic_smp_cpu_init();
572 armada_370_xp_msi_init(node
, main_int_res
.start
);
574 parent_irq
= irq_of_parse_and_map(node
, 0);
575 if (parent_irq
<= 0) {
576 irq_set_default_host(armada_370_xp_mpic_domain
);
577 set_handle_irq(armada_370_xp_handle_irq
);
579 set_smp_cross_call(armada_mpic_send_doorbell
);
580 cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_ARMADA_XP_STARTING
,
581 "irqchip/armada/ipi:starting",
582 armada_xp_mpic_starting_cpu
, NULL
);
586 cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_ARMADA_XP_STARTING
,
587 "irqchip/armada/cascade:starting",
588 mpic_cascaded_starting_cpu
, NULL
);
590 irq_set_chained_handler(parent_irq
,
591 armada_370_xp_mpic_handle_cascade_irq
);
594 register_syscore_ops(&armada_370_xp_mpic_syscore_ops
);
599 IRQCHIP_DECLARE(armada_370_xp_mpic
, "marvell,mpic", armada_370_xp_mpic_of_init
);