2 * Local APIC related interfaces to support IOAPIC, MSI, HT_IRQ etc.
4 * Copyright (C) 1997, 1998, 1999, 2000, 2009 Ingo Molnar, Hajnalka Szabo
5 * Moved from arch/x86/kernel/apic/io_apic.c.
6 * Jiang Liu <jiang.liu@linux.intel.com>
7 * Enable support of hierarchical irqdomains
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
13 #include <linux/interrupt.h>
14 #include <linux/init.h>
15 #include <linux/compiler.h>
16 #include <linux/slab.h>
17 #include <asm/irqdomain.h>
18 #include <asm/hw_irq.h>
20 #include <asm/i8259.h>
22 #include <asm/irq_remapping.h>
24 struct apic_chip_data
{
27 cpumask_var_t old_domain
;
28 u8 move_in_progress
: 1;
31 struct irq_domain
*x86_vector_domain
;
32 static DEFINE_RAW_SPINLOCK(vector_lock
);
33 static cpumask_var_t vector_cpumask
;
34 static struct irq_chip lapic_controller
;
35 #ifdef CONFIG_X86_IO_APIC
36 static struct apic_chip_data
*legacy_irq_data
[NR_IRQS_LEGACY
];
39 void lock_vector_lock(void)
41 /* Used to the online set of cpus does not change
42 * during assign_irq_vector.
44 raw_spin_lock(&vector_lock
);
47 void unlock_vector_lock(void)
49 raw_spin_unlock(&vector_lock
);
52 static struct apic_chip_data
*apic_chip_data(struct irq_data
*irq_data
)
57 while (irq_data
->parent_data
)
58 irq_data
= irq_data
->parent_data
;
60 return irq_data
->chip_data
;
63 struct irq_cfg
*irqd_cfg(struct irq_data
*irq_data
)
65 struct apic_chip_data
*data
= apic_chip_data(irq_data
);
67 return data
? &data
->cfg
: NULL
;
70 struct irq_cfg
*irq_cfg(unsigned int irq
)
72 return irqd_cfg(irq_get_irq_data(irq
));
75 static struct apic_chip_data
*alloc_apic_chip_data(int node
)
77 struct apic_chip_data
*data
;
79 data
= kzalloc_node(sizeof(*data
), GFP_KERNEL
, node
);
82 if (!zalloc_cpumask_var_node(&data
->domain
, GFP_KERNEL
, node
))
84 if (!zalloc_cpumask_var_node(&data
->old_domain
, GFP_KERNEL
, node
))
88 free_cpumask_var(data
->domain
);
94 static void free_apic_chip_data(struct apic_chip_data
*data
)
97 free_cpumask_var(data
->domain
);
98 free_cpumask_var(data
->old_domain
);
103 static int __assign_irq_vector(int irq
, struct apic_chip_data
*d
,
104 const struct cpumask
*mask
)
107 * NOTE! The local APIC isn't very good at handling
108 * multiple interrupts at the same interrupt level.
109 * As the interrupt level is determined by taking the
110 * vector number and shifting that right by 4, we
111 * want to spread these out a bit so that they don't
112 * all fall in the same interrupt level.
114 * Also, we've got to be careful not to trash gate
115 * 0x80, because int 0x80 is hm, kind of importantish. ;)
117 static int current_vector
= FIRST_EXTERNAL_VECTOR
+ VECTOR_OFFSET_START
;
118 static int current_offset
= VECTOR_OFFSET_START
% 16;
121 if (d
->move_in_progress
)
124 /* Only try and allocate irqs on cpus that are present */
126 cpumask_clear(d
->old_domain
);
127 cpu
= cpumask_first_and(mask
, cpu_online_mask
);
128 while (cpu
< nr_cpu_ids
) {
129 int new_cpu
, vector
, offset
;
131 apic
->vector_allocation_domain(cpu
, vector_cpumask
, mask
);
133 if (cpumask_subset(vector_cpumask
, d
->domain
)) {
135 if (cpumask_equal(vector_cpumask
, d
->domain
))
138 * New cpumask using the vector is a proper subset of
139 * the current in use mask. So cleanup the vector
140 * allocation for the members that are not used anymore.
142 cpumask_andnot(d
->old_domain
, d
->domain
,
144 d
->move_in_progress
=
145 cpumask_intersects(d
->old_domain
, cpu_online_mask
);
146 cpumask_and(d
->domain
, d
->domain
, vector_cpumask
);
150 vector
= current_vector
;
151 offset
= current_offset
;
154 if (vector
>= first_system_vector
) {
155 offset
= (offset
+ 1) % 16;
156 vector
= FIRST_EXTERNAL_VECTOR
+ offset
;
159 if (unlikely(current_vector
== vector
)) {
160 cpumask_or(d
->old_domain
, d
->old_domain
,
162 cpumask_andnot(vector_cpumask
, mask
, d
->old_domain
);
163 cpu
= cpumask_first_and(vector_cpumask
,
168 if (test_bit(vector
, used_vectors
))
171 for_each_cpu_and(new_cpu
, vector_cpumask
, cpu_online_mask
) {
172 if (per_cpu(vector_irq
, new_cpu
)[vector
] >
177 current_vector
= vector
;
178 current_offset
= offset
;
180 cpumask_copy(d
->old_domain
, d
->domain
);
181 d
->move_in_progress
=
182 cpumask_intersects(d
->old_domain
, cpu_online_mask
);
184 for_each_cpu_and(new_cpu
, vector_cpumask
, cpu_online_mask
)
185 per_cpu(vector_irq
, new_cpu
)[vector
] = irq
;
186 d
->cfg
.vector
= vector
;
187 cpumask_copy(d
->domain
, vector_cpumask
);
193 /* cache destination APIC IDs into cfg->dest_apicid */
194 err
= apic
->cpu_mask_to_apicid_and(mask
, d
->domain
,
195 &d
->cfg
.dest_apicid
);
201 static int assign_irq_vector(int irq
, struct apic_chip_data
*data
,
202 const struct cpumask
*mask
)
207 raw_spin_lock_irqsave(&vector_lock
, flags
);
208 err
= __assign_irq_vector(irq
, data
, mask
);
209 raw_spin_unlock_irqrestore(&vector_lock
, flags
);
213 static int assign_irq_vector_policy(int irq
, int node
,
214 struct apic_chip_data
*data
,
215 struct irq_alloc_info
*info
)
217 if (info
&& info
->mask
)
218 return assign_irq_vector(irq
, data
, info
->mask
);
219 if (node
!= NUMA_NO_NODE
&&
220 assign_irq_vector(irq
, data
, cpumask_of_node(node
)) == 0)
222 return assign_irq_vector(irq
, data
, apic
->target_cpus());
225 static void clear_irq_vector(int irq
, struct apic_chip_data
*data
)
230 raw_spin_lock_irqsave(&vector_lock
, flags
);
231 BUG_ON(!data
->cfg
.vector
);
233 vector
= data
->cfg
.vector
;
234 for_each_cpu_and(cpu
, data
->domain
, cpu_online_mask
)
235 per_cpu(vector_irq
, cpu
)[vector
] = VECTOR_UNDEFINED
;
237 data
->cfg
.vector
= 0;
238 cpumask_clear(data
->domain
);
240 if (likely(!data
->move_in_progress
)) {
241 raw_spin_unlock_irqrestore(&vector_lock
, flags
);
245 for_each_cpu_and(cpu
, data
->old_domain
, cpu_online_mask
) {
246 for (vector
= FIRST_EXTERNAL_VECTOR
; vector
< NR_VECTORS
;
248 if (per_cpu(vector_irq
, cpu
)[vector
] != irq
)
250 per_cpu(vector_irq
, cpu
)[vector
] = VECTOR_UNDEFINED
;
254 data
->move_in_progress
= 0;
255 raw_spin_unlock_irqrestore(&vector_lock
, flags
);
258 void init_irq_alloc_info(struct irq_alloc_info
*info
,
259 const struct cpumask
*mask
)
261 memset(info
, 0, sizeof(*info
));
265 void copy_irq_alloc_info(struct irq_alloc_info
*dst
, struct irq_alloc_info
*src
)
270 memset(dst
, 0, sizeof(*dst
));
273 static void x86_vector_free_irqs(struct irq_domain
*domain
,
274 unsigned int virq
, unsigned int nr_irqs
)
276 struct irq_data
*irq_data
;
279 for (i
= 0; i
< nr_irqs
; i
++) {
280 irq_data
= irq_domain_get_irq_data(x86_vector_domain
, virq
+ i
);
281 if (irq_data
&& irq_data
->chip_data
) {
282 clear_irq_vector(virq
+ i
, irq_data
->chip_data
);
283 free_apic_chip_data(irq_data
->chip_data
);
284 #ifdef CONFIG_X86_IO_APIC
285 if (virq
+ i
< nr_legacy_irqs())
286 legacy_irq_data
[virq
+ i
] = NULL
;
288 irq_domain_reset_irq_data(irq_data
);
293 static int x86_vector_alloc_irqs(struct irq_domain
*domain
, unsigned int virq
,
294 unsigned int nr_irqs
, void *arg
)
296 struct irq_alloc_info
*info
= arg
;
297 struct apic_chip_data
*data
;
298 struct irq_data
*irq_data
;
304 /* Currently vector allocator can't guarantee contiguous allocations */
305 if ((info
->flags
& X86_IRQ_ALLOC_CONTIGUOUS_VECTORS
) && nr_irqs
> 1)
308 for (i
= 0; i
< nr_irqs
; i
++) {
309 irq_data
= irq_domain_get_irq_data(domain
, virq
+ i
);
311 #ifdef CONFIG_X86_IO_APIC
312 if (virq
+ i
< nr_legacy_irqs() && legacy_irq_data
[virq
+ i
])
313 data
= legacy_irq_data
[virq
+ i
];
316 data
= alloc_apic_chip_data(irq_data
->node
);
322 irq_data
->chip
= &lapic_controller
;
323 irq_data
->chip_data
= data
;
324 irq_data
->hwirq
= virq
+ i
;
325 err
= assign_irq_vector_policy(virq
+ i
, irq_data
->node
, data
,
334 x86_vector_free_irqs(domain
, virq
, i
+ 1);
338 static const struct irq_domain_ops x86_vector_domain_ops
= {
339 .alloc
= x86_vector_alloc_irqs
,
340 .free
= x86_vector_free_irqs
,
343 int __init
arch_probe_nr_irqs(void)
347 if (nr_irqs
> (NR_VECTORS
* nr_cpu_ids
))
348 nr_irqs
= NR_VECTORS
* nr_cpu_ids
;
350 nr
= (gsi_top
+ nr_legacy_irqs()) + 8 * nr_cpu_ids
;
351 #if defined(CONFIG_PCI_MSI) || defined(CONFIG_HT_IRQ)
353 * for MSI and HT dyn irq
355 if (gsi_top
<= NR_IRQS_LEGACY
)
356 nr
+= 8 * nr_cpu_ids
;
363 return nr_legacy_irqs();
366 #ifdef CONFIG_X86_IO_APIC
367 static void init_legacy_irqs(void)
369 int i
, node
= cpu_to_node(0);
370 struct apic_chip_data
*data
;
373 * For legacy IRQ's, start with assigning irq0 to irq15 to
374 * ISA_IRQ_VECTOR(i) for all cpu's.
376 for (i
= 0; i
< nr_legacy_irqs(); i
++) {
377 data
= legacy_irq_data
[i
] = alloc_apic_chip_data(node
);
380 data
->cfg
.vector
= ISA_IRQ_VECTOR(i
);
381 cpumask_setall(data
->domain
);
382 irq_set_chip_data(i
, data
);
386 static void init_legacy_irqs(void) { }
389 int __init
arch_early_irq_init(void)
393 x86_vector_domain
= irq_domain_add_tree(NULL
, &x86_vector_domain_ops
,
395 BUG_ON(x86_vector_domain
== NULL
);
396 irq_set_default_host(x86_vector_domain
);
398 arch_init_msi_domain(x86_vector_domain
);
399 arch_init_htirq_domain(x86_vector_domain
);
401 BUG_ON(!alloc_cpumask_var(&vector_cpumask
, GFP_KERNEL
));
403 return arch_early_ioapic_init();
406 static void __setup_vector_irq(int cpu
)
408 /* Initialize vector_irq on a new cpu */
410 struct apic_chip_data
*data
;
412 /* Mark the inuse vectors */
413 for_each_active_irq(irq
) {
414 data
= apic_chip_data(irq_get_irq_data(irq
));
418 if (!cpumask_test_cpu(cpu
, data
->domain
))
420 vector
= data
->cfg
.vector
;
421 per_cpu(vector_irq
, cpu
)[vector
] = irq
;
423 /* Mark the free vectors */
424 for (vector
= 0; vector
< NR_VECTORS
; ++vector
) {
425 irq
= per_cpu(vector_irq
, cpu
)[vector
];
426 if (irq
<= VECTOR_UNDEFINED
)
429 data
= apic_chip_data(irq_get_irq_data(irq
));
430 if (!cpumask_test_cpu(cpu
, data
->domain
))
431 per_cpu(vector_irq
, cpu
)[vector
] = VECTOR_UNDEFINED
;
436 * Setup the vector to irq mappings. Must be called with vector_lock held.
438 void setup_vector_irq(int cpu
)
442 lockdep_assert_held(&vector_lock
);
444 * On most of the platforms, legacy PIC delivers the interrupts on the
445 * boot cpu. But there are certain platforms where PIC interrupts are
446 * delivered to multiple cpu's. If the legacy IRQ is handled by the
447 * legacy PIC, for the new cpu that is coming online, setup the static
448 * legacy vector to irq mapping:
450 for (irq
= 0; irq
< nr_legacy_irqs(); irq
++)
451 per_cpu(vector_irq
, cpu
)[ISA_IRQ_VECTOR(irq
)] = irq
;
453 __setup_vector_irq(cpu
);
456 static int apic_retrigger_irq(struct irq_data
*irq_data
)
458 struct apic_chip_data
*data
= apic_chip_data(irq_data
);
462 raw_spin_lock_irqsave(&vector_lock
, flags
);
463 cpu
= cpumask_first_and(data
->domain
, cpu_online_mask
);
464 apic
->send_IPI_mask(cpumask_of(cpu
), data
->cfg
.vector
);
465 raw_spin_unlock_irqrestore(&vector_lock
, flags
);
470 void apic_ack_edge(struct irq_data
*data
)
472 irq_complete_move(irqd_cfg(data
));
477 static int apic_set_affinity(struct irq_data
*irq_data
,
478 const struct cpumask
*dest
, bool force
)
480 struct apic_chip_data
*data
= irq_data
->chip_data
;
481 int err
, irq
= irq_data
->irq
;
483 if (!config_enabled(CONFIG_SMP
))
486 if (!cpumask_intersects(dest
, cpu_online_mask
))
489 err
= assign_irq_vector(irq
, data
, dest
);
491 struct irq_data
*top
= irq_get_irq_data(irq
);
493 if (assign_irq_vector(irq
, data
, top
->affinity
))
494 pr_err("Failed to recover vector for irq %d\n", irq
);
498 return IRQ_SET_MASK_OK
;
501 static struct irq_chip lapic_controller
= {
502 .irq_ack
= apic_ack_edge
,
503 .irq_set_affinity
= apic_set_affinity
,
504 .irq_retrigger
= apic_retrigger_irq
,
508 static void __send_cleanup_vector(struct apic_chip_data
*data
)
510 cpumask_var_t cleanup_mask
;
512 if (unlikely(!alloc_cpumask_var(&cleanup_mask
, GFP_ATOMIC
))) {
515 for_each_cpu_and(i
, data
->old_domain
, cpu_online_mask
)
516 apic
->send_IPI_mask(cpumask_of(i
),
517 IRQ_MOVE_CLEANUP_VECTOR
);
519 cpumask_and(cleanup_mask
, data
->old_domain
, cpu_online_mask
);
520 apic
->send_IPI_mask(cleanup_mask
, IRQ_MOVE_CLEANUP_VECTOR
);
521 free_cpumask_var(cleanup_mask
);
523 data
->move_in_progress
= 0;
526 void send_cleanup_vector(struct irq_cfg
*cfg
)
528 struct apic_chip_data
*data
;
530 data
= container_of(cfg
, struct apic_chip_data
, cfg
);
531 if (data
->move_in_progress
)
532 __send_cleanup_vector(data
);
535 asmlinkage __visible
void smp_irq_move_cleanup_interrupt(void)
541 me
= smp_processor_id();
542 for (vector
= FIRST_EXTERNAL_VECTOR
; vector
< NR_VECTORS
; vector
++) {
545 struct irq_desc
*desc
;
546 struct apic_chip_data
*data
;
548 irq
= __this_cpu_read(vector_irq
[vector
]);
550 if (irq
<= VECTOR_UNDEFINED
)
553 desc
= irq_to_desc(irq
);
557 data
= apic_chip_data(&desc
->irq_data
);
561 raw_spin_lock(&desc
->lock
);
564 * Check if the irq migration is in progress. If so, we
565 * haven't received the cleanup request yet for this irq.
567 if (data
->move_in_progress
)
570 if (vector
== data
->cfg
.vector
&&
571 cpumask_test_cpu(me
, data
->domain
))
574 irr
= apic_read(APIC_IRR
+ (vector
/ 32 * 0x10));
576 * Check if the vector that needs to be cleanedup is
577 * registered at the cpu's IRR. If so, then this is not
578 * the best time to clean it up. Lets clean it up in the
579 * next attempt by sending another IRQ_MOVE_CLEANUP_VECTOR
582 if (irr
& (1 << (vector
% 32))) {
583 apic
->send_IPI_self(IRQ_MOVE_CLEANUP_VECTOR
);
586 __this_cpu_write(vector_irq
[vector
], VECTOR_UNDEFINED
);
588 raw_spin_unlock(&desc
->lock
);
594 static void __irq_complete_move(struct irq_cfg
*cfg
, unsigned vector
)
597 struct apic_chip_data
*data
;
599 data
= container_of(cfg
, struct apic_chip_data
, cfg
);
600 if (likely(!data
->move_in_progress
))
603 me
= smp_processor_id();
604 if (vector
== data
->cfg
.vector
&& cpumask_test_cpu(me
, data
->domain
))
605 __send_cleanup_vector(data
);
608 void irq_complete_move(struct irq_cfg
*cfg
)
610 __irq_complete_move(cfg
, ~get_irq_regs()->orig_ax
);
613 void irq_force_complete_move(int irq
)
615 struct irq_cfg
*cfg
= irq_cfg(irq
);
618 __irq_complete_move(cfg
, cfg
->vector
);
622 static void __init
print_APIC_field(int base
)
628 for (i
= 0; i
< 8; i
++)
629 pr_cont("%08x", apic_read(base
+ i
*0x10));
634 static void __init
print_local_APIC(void *dummy
)
636 unsigned int i
, v
, ver
, maxlvt
;
639 pr_debug("printing local APIC contents on CPU#%d/%d:\n",
640 smp_processor_id(), hard_smp_processor_id());
641 v
= apic_read(APIC_ID
);
642 pr_info("... APIC ID: %08x (%01x)\n", v
, read_apic_id());
643 v
= apic_read(APIC_LVR
);
644 pr_info("... APIC VERSION: %08x\n", v
);
645 ver
= GET_APIC_VERSION(v
);
646 maxlvt
= lapic_get_maxlvt();
648 v
= apic_read(APIC_TASKPRI
);
649 pr_debug("... APIC TASKPRI: %08x (%02x)\n", v
, v
& APIC_TPRI_MASK
);
652 if (APIC_INTEGRATED(ver
)) {
653 if (!APIC_XAPIC(ver
)) {
654 v
= apic_read(APIC_ARBPRI
);
655 pr_debug("... APIC ARBPRI: %08x (%02x)\n",
656 v
, v
& APIC_ARBPRI_MASK
);
658 v
= apic_read(APIC_PROCPRI
);
659 pr_debug("... APIC PROCPRI: %08x\n", v
);
663 * Remote read supported only in the 82489DX and local APIC for
664 * Pentium processors.
666 if (!APIC_INTEGRATED(ver
) || maxlvt
== 3) {
667 v
= apic_read(APIC_RRR
);
668 pr_debug("... APIC RRR: %08x\n", v
);
671 v
= apic_read(APIC_LDR
);
672 pr_debug("... APIC LDR: %08x\n", v
);
673 if (!x2apic_enabled()) {
674 v
= apic_read(APIC_DFR
);
675 pr_debug("... APIC DFR: %08x\n", v
);
677 v
= apic_read(APIC_SPIV
);
678 pr_debug("... APIC SPIV: %08x\n", v
);
680 pr_debug("... APIC ISR field:\n");
681 print_APIC_field(APIC_ISR
);
682 pr_debug("... APIC TMR field:\n");
683 print_APIC_field(APIC_TMR
);
684 pr_debug("... APIC IRR field:\n");
685 print_APIC_field(APIC_IRR
);
688 if (APIC_INTEGRATED(ver
)) {
689 /* Due to the Pentium erratum 3AP. */
691 apic_write(APIC_ESR
, 0);
693 v
= apic_read(APIC_ESR
);
694 pr_debug("... APIC ESR: %08x\n", v
);
697 icr
= apic_icr_read();
698 pr_debug("... APIC ICR: %08x\n", (u32
)icr
);
699 pr_debug("... APIC ICR2: %08x\n", (u32
)(icr
>> 32));
701 v
= apic_read(APIC_LVTT
);
702 pr_debug("... APIC LVTT: %08x\n", v
);
706 v
= apic_read(APIC_LVTPC
);
707 pr_debug("... APIC LVTPC: %08x\n", v
);
709 v
= apic_read(APIC_LVT0
);
710 pr_debug("... APIC LVT0: %08x\n", v
);
711 v
= apic_read(APIC_LVT1
);
712 pr_debug("... APIC LVT1: %08x\n", v
);
716 v
= apic_read(APIC_LVTERR
);
717 pr_debug("... APIC LVTERR: %08x\n", v
);
720 v
= apic_read(APIC_TMICT
);
721 pr_debug("... APIC TMICT: %08x\n", v
);
722 v
= apic_read(APIC_TMCCT
);
723 pr_debug("... APIC TMCCT: %08x\n", v
);
724 v
= apic_read(APIC_TDCR
);
725 pr_debug("... APIC TDCR: %08x\n", v
);
727 if (boot_cpu_has(X86_FEATURE_EXTAPIC
)) {
728 v
= apic_read(APIC_EFEAT
);
729 maxlvt
= (v
>> 16) & 0xff;
730 pr_debug("... APIC EFEAT: %08x\n", v
);
731 v
= apic_read(APIC_ECTRL
);
732 pr_debug("... APIC ECTRL: %08x\n", v
);
733 for (i
= 0; i
< maxlvt
; i
++) {
734 v
= apic_read(APIC_EILVTn(i
));
735 pr_debug("... APIC EILVT%d: %08x\n", i
, v
);
741 static void __init
print_local_APICs(int maxcpu
)
749 for_each_online_cpu(cpu
) {
752 smp_call_function_single(cpu
, print_local_APIC
, NULL
, 1);
757 static void __init
print_PIC(void)
762 if (!nr_legacy_irqs())
765 pr_debug("\nprinting PIC contents\n");
767 raw_spin_lock_irqsave(&i8259A_lock
, flags
);
769 v
= inb(0xa1) << 8 | inb(0x21);
770 pr_debug("... PIC IMR: %04x\n", v
);
772 v
= inb(0xa0) << 8 | inb(0x20);
773 pr_debug("... PIC IRR: %04x\n", v
);
777 v
= inb(0xa0) << 8 | inb(0x20);
781 raw_spin_unlock_irqrestore(&i8259A_lock
, flags
);
783 pr_debug("... PIC ISR: %04x\n", v
);
785 v
= inb(0x4d1) << 8 | inb(0x4d0);
786 pr_debug("... PIC ELCR: %04x\n", v
);
789 static int show_lapic __initdata
= 1;
790 static __init
int setup_show_lapic(char *arg
)
794 if (strcmp(arg
, "all") == 0) {
795 show_lapic
= CONFIG_NR_CPUS
;
797 get_option(&arg
, &num
);
804 __setup("show_lapic=", setup_show_lapic
);
806 static int __init
print_ICs(void)
808 if (apic_verbosity
== APIC_QUIET
)
813 /* don't print out if apic is not there */
814 if (!cpu_has_apic
&& !apic_from_smp_config())
817 print_local_APICs(show_lapic
);
823 late_initcall(print_ICs
);