?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
x86/amd-iommu: Add per IOMMU reference counting
[linux/fpc-iii.git] / arch / x86 / kernel / apic / io_apic.c
blobdc69f28489f5bc9a2873a8d22b084bec313d9912
1 /*
2 * Intel IO-APIC support for multi-Pentium hosts.
3 *
4 * Copyright (C) 1997, 1998, 1999, 2000, 2009 Ingo Molnar, Hajnalka Szabo
5 *
6 * Many thanks to Stig Venaas for trying out countless experimental
7 * patches and reporting/debugging problems patiently!
8 *
9 * (c) 1999, Multiple IO-APIC support, developed by
10 * Ken-ichi Yaku <yaku@css1.kbnes.nec.co.jp> and
11 * Hidemi Kishimoto <kisimoto@css1.kbnes.nec.co.jp>,
12 * further tested and cleaned up by Zach Brown <zab@redhat.com>
13 * and Ingo Molnar <mingo@redhat.com>
14 *
15 * Fixes
16 * Maciej W. Rozycki : Bits for genuine 82489DX APICs;
17 * thanks to Eric Gilmore
18 * and Rolf G. Tews
19 * for testing these extensively
20 * Paul Diefenbaugh : Added full ACPI support
21 */
22
23 #include <linux/mm.h>
24 #include <linux/interrupt.h>
25 #include <linux/init.h>
26 #include <linux/delay.h>
27 #include <linux/sched.h>
28 #include <linux/pci.h>
29 #include <linux/mc146818rtc.h>
30 #include <linux/compiler.h>
31 #include <linux/acpi.h>
32 #include <linux/module.h>
33 #include <linux/sysdev.h>
34 #include <linux/msi.h>
35 #include <linux/htirq.h>
36 #include <linux/freezer.h>
37 #include <linux/kthread.h>
38 #include <linux/jiffies.h> /* time_after() */
39 #ifdef CONFIG_ACPI
40 #include <acpi/acpi_bus.h>
41 #endif
42 #include <linux/bootmem.h>
43 #include <linux/dmar.h>
44 #include <linux/hpet.h>
45
46 #include <asm/idle.h>
47 #include <asm/io.h>
48 #include <asm/smp.h>
49 #include <asm/cpu.h>
50 #include <asm/desc.h>
51 #include <asm/proto.h>
52 #include <asm/acpi.h>
53 #include <asm/dma.h>
54 #include <asm/timer.h>
55 #include <asm/i8259.h>
56 #include <asm/nmi.h>
57 #include <asm/msidef.h>
58 #include <asm/hypertransport.h>
59 #include <asm/setup.h>
60 #include <asm/irq_remapping.h>
61 #include <asm/hpet.h>
62 #include <asm/hw_irq.h>
63 #include <asm/uv/uv_hub.h>
64 #include <asm/uv/uv_irq.h>
65
66 #include <asm/apic.h>
67
68 #define __apicdebuginit(type) static type __init
69 #define for_each_irq_pin(entry, head) \
70 for (entry = head; entry; entry = entry->next)
71
72 /*
73 * Is the SiS APIC rmw bug present ?
74 * -1 = don't know, 0 = no, 1 = yes
75 */
76 int sis_apic_bug = -1;
77
78 static DEFINE_SPINLOCK(ioapic_lock);
79 static DEFINE_SPINLOCK(vector_lock);
80
81 /*
82 * # of IRQ routing registers
83 */
84 int nr_ioapic_registers[MAX_IO_APICS];
85
86 /* I/O APIC entries */
87 struct mpc_ioapic mp_ioapics[MAX_IO_APICS];
88 int nr_ioapics;
89
90 /* IO APIC gsi routing info */
91 struct mp_ioapic_gsi mp_gsi_routing[MAX_IO_APICS];
92
93 /* MP IRQ source entries */
94 struct mpc_intsrc mp_irqs[MAX_IRQ_SOURCES];
95
96 /* # of MP IRQ source entries */
97 int mp_irq_entries;
98
99 /* Number of legacy interrupts */
100 static int nr_legacy_irqs __read_mostly = NR_IRQS_LEGACY;
101 /* GSI interrupts */
102 static int nr_irqs_gsi = NR_IRQS_LEGACY;
103
104 #if defined (CONFIG_MCA) || defined (CONFIG_EISA)
105 int mp_bus_id_to_type[MAX_MP_BUSSES];
106 #endif
107
108 DECLARE_BITMAP(mp_bus_not_pci, MAX_MP_BUSSES);
109
110 int skip_ioapic_setup;
111
112 void arch_disable_smp_support(void)
113 {
114 #ifdef CONFIG_PCI
115 noioapicquirk = 1;
116 noioapicreroute = -1;
117 #endif
118 skip_ioapic_setup = 1;
119 }
120
121 static int __init parse_noapic(char *str)
122 {
123 /* disable IO-APIC */
124 arch_disable_smp_support();
125 return 0;
126 }
127 early_param("noapic", parse_noapic);
128
129 struct irq_pin_list {
130 int apic, pin;
131 struct irq_pin_list *next;
132 };
133
134 static struct irq_pin_list *get_one_free_irq_2_pin(int node)
135 {
136 struct irq_pin_list *pin;
137
138 pin = kzalloc_node(sizeof(*pin), GFP_ATOMIC, node);
139
140 return pin;
141 }
142
143 /*
144 * This is performance-critical, we want to do it O(1)
145 *
146 * Most irqs are mapped 1:1 with pins.
147 */
148 struct irq_cfg {
149 struct irq_pin_list *irq_2_pin;
150 cpumask_var_t domain;
151 cpumask_var_t old_domain;
152 unsigned move_cleanup_count;
153 u8 vector;
154 u8 move_in_progress : 1;
155 };
156
157 /* irq_cfg is indexed by the sum of all RTEs in all I/O APICs. */
158 #ifdef CONFIG_SPARSE_IRQ
159 static struct irq_cfg irq_cfgx[] = {
160 #else
161 static struct irq_cfg irq_cfgx[NR_IRQS] = {
162 #endif
163 [0] = { .vector = IRQ0_VECTOR, },
164 [1] = { .vector = IRQ1_VECTOR, },
165 [2] = { .vector = IRQ2_VECTOR, },
166 [3] = { .vector = IRQ3_VECTOR, },
167 [4] = { .vector = IRQ4_VECTOR, },
168 [5] = { .vector = IRQ5_VECTOR, },
169 [6] = { .vector = IRQ6_VECTOR, },
170 [7] = { .vector = IRQ7_VECTOR, },
171 [8] = { .vector = IRQ8_VECTOR, },
172 [9] = { .vector = IRQ9_VECTOR, },
173 [10] = { .vector = IRQ10_VECTOR, },
174 [11] = { .vector = IRQ11_VECTOR, },
175 [12] = { .vector = IRQ12_VECTOR, },
176 [13] = { .vector = IRQ13_VECTOR, },
177 [14] = { .vector = IRQ14_VECTOR, },
178 [15] = { .vector = IRQ15_VECTOR, },
179 };
180
181 void __init io_apic_disable_legacy(void)
182 {
183 nr_legacy_irqs = 0;
184 nr_irqs_gsi = 0;
185 }
186
187 int __init arch_early_irq_init(void)
188 {
189 struct irq_cfg *cfg;
190 struct irq_desc *desc;
191 int count;
192 int node;
193 int i;
194
195 cfg = irq_cfgx;
196 count = ARRAY_SIZE(irq_cfgx);
197 node= cpu_to_node(boot_cpu_id);
198
199 for (i = 0; i < count; i++) {
200 desc = irq_to_desc(i);
201 desc->chip_data = &cfg[i];
202 zalloc_cpumask_var_node(&cfg[i].domain, GFP_NOWAIT, node);
203 zalloc_cpumask_var_node(&cfg[i].old_domain, GFP_NOWAIT, node);
204 if (i < nr_legacy_irqs)
205 cpumask_setall(cfg[i].domain);
206 }
207
208 return 0;
209 }
210
211 #ifdef CONFIG_SPARSE_IRQ
212 static struct irq_cfg *irq_cfg(unsigned int irq)
213 {
214 struct irq_cfg *cfg = NULL;
215 struct irq_desc *desc;
216
217 desc = irq_to_desc(irq);
218 if (desc)
219 cfg = desc->chip_data;
220
221 return cfg;
222 }
223
224 static struct irq_cfg *get_one_free_irq_cfg(int node)
225 {
226 struct irq_cfg *cfg;
227
228 cfg = kzalloc_node(sizeof(*cfg), GFP_ATOMIC, node);
229 if (cfg) {
230 if (!zalloc_cpumask_var_node(&cfg->domain, GFP_ATOMIC, node)) {
231 kfree(cfg);
232 cfg = NULL;
233 } else if (!zalloc_cpumask_var_node(&cfg->old_domain,
234 GFP_ATOMIC, node)) {
235 free_cpumask_var(cfg->domain);
236 kfree(cfg);
237 cfg = NULL;
238 }
239 }
240
241 return cfg;
242 }
243
244 int arch_init_chip_data(struct irq_desc *desc, int node)
245 {
246 struct irq_cfg *cfg;
247
248 cfg = desc->chip_data;
249 if (!cfg) {
250 desc->chip_data = get_one_free_irq_cfg(node);
251 if (!desc->chip_data) {
252 printk(KERN_ERR "can not alloc irq_cfg\n");
253 BUG_ON(1);
254 }
255 }
256
257 return 0;
258 }
259
260 /* for move_irq_desc */
261 static void
262 init_copy_irq_2_pin(struct irq_cfg *old_cfg, struct irq_cfg *cfg, int node)
263 {
264 struct irq_pin_list *old_entry, *head, *tail, *entry;
265
266 cfg->irq_2_pin = NULL;
267 old_entry = old_cfg->irq_2_pin;
268 if (!old_entry)
269 return;
270
271 entry = get_one_free_irq_2_pin(node);
272 if (!entry)
273 return;
274
275 entry->apic = old_entry->apic;
276 entry->pin = old_entry->pin;
277 head = entry;
278 tail = entry;
279 old_entry = old_entry->next;
280 while (old_entry) {
281 entry = get_one_free_irq_2_pin(node);
282 if (!entry) {
283 entry = head;
284 while (entry) {
285 head = entry->next;
286 kfree(entry);
287 entry = head;
288 }
289 /* still use the old one */
290 return;
291 }
292 entry->apic = old_entry->apic;
293 entry->pin = old_entry->pin;
294 tail->next = entry;
295 tail = entry;
296 old_entry = old_entry->next;
297 }
298
299 tail->next = NULL;
300 cfg->irq_2_pin = head;
301 }
302
303 static void free_irq_2_pin(struct irq_cfg *old_cfg, struct irq_cfg *cfg)
304 {
305 struct irq_pin_list *entry, *next;
306
307 if (old_cfg->irq_2_pin == cfg->irq_2_pin)
308 return;
309
310 entry = old_cfg->irq_2_pin;
311
312 while (entry) {
313 next = entry->next;
314 kfree(entry);
315 entry = next;
316 }
317 old_cfg->irq_2_pin = NULL;
318 }
319
320 void arch_init_copy_chip_data(struct irq_desc *old_desc,
321 struct irq_desc *desc, int node)
322 {
323 struct irq_cfg *cfg;
324 struct irq_cfg *old_cfg;
325
326 cfg = get_one_free_irq_cfg(node);
327
328 if (!cfg)
329 return;
330
331 desc->chip_data = cfg;
332
333 old_cfg = old_desc->chip_data;
334
335 memcpy(cfg, old_cfg, sizeof(struct irq_cfg));
336
337 init_copy_irq_2_pin(old_cfg, cfg, node);
338 }
339
340 static void free_irq_cfg(struct irq_cfg *old_cfg)
341 {
342 kfree(old_cfg);
343 }
344
345 void arch_free_chip_data(struct irq_desc *old_desc, struct irq_desc *desc)
346 {
347 struct irq_cfg *old_cfg, *cfg;
348
349 old_cfg = old_desc->chip_data;
350 cfg = desc->chip_data;
351
352 if (old_cfg == cfg)
353 return;
354
355 if (old_cfg) {
356 free_irq_2_pin(old_cfg, cfg);
357 free_irq_cfg(old_cfg);
358 old_desc->chip_data = NULL;
359 }
360 }
361 /* end for move_irq_desc */
362
363 #else
364 static struct irq_cfg *irq_cfg(unsigned int irq)
365 {
366 return irq < nr_irqs ? irq_cfgx + irq : NULL;
367 }
368
369 #endif
370
371 struct io_apic {
372 unsigned int index;
373 unsigned int unused[3];
374 unsigned int data;
375 unsigned int unused2[11];
376 unsigned int eoi;
377 };
378
379 static __attribute_const__ struct io_apic __iomem *io_apic_base(int idx)
380 {
381 return (void __iomem *) __fix_to_virt(FIX_IO_APIC_BASE_0 + idx)
382 + (mp_ioapics[idx].apicaddr & ~PAGE_MASK);
383 }
384
385 static inline void io_apic_eoi(unsigned int apic, unsigned int vector)
386 {
387 struct io_apic __iomem *io_apic = io_apic_base(apic);
388 writel(vector, &io_apic->eoi);
389 }
390
391 static inline unsigned int io_apic_read(unsigned int apic, unsigned int reg)
392 {
393 struct io_apic __iomem *io_apic = io_apic_base(apic);
394 writel(reg, &io_apic->index);
395 return readl(&io_apic->data);
396 }
397
398 static inline void io_apic_write(unsigned int apic, unsigned int reg, unsigned int value)
399 {
400 struct io_apic __iomem *io_apic = io_apic_base(apic);
401 writel(reg, &io_apic->index);
402 writel(value, &io_apic->data);
403 }
404
405 /*
406 * Re-write a value: to be used for read-modify-write
407 * cycles where the read already set up the index register.
408 *
409 * Older SiS APIC requires we rewrite the index register
410 */
411 static inline void io_apic_modify(unsigned int apic, unsigned int reg, unsigned int value)
412 {
413 struct io_apic __iomem *io_apic = io_apic_base(apic);
414
415 if (sis_apic_bug)
416 writel(reg, &io_apic->index);
417 writel(value, &io_apic->data);
418 }
419
420 static bool io_apic_level_ack_pending(struct irq_cfg *cfg)
421 {
422 struct irq_pin_list *entry;
423 unsigned long flags;
424
425 spin_lock_irqsave(&ioapic_lock, flags);
426 for_each_irq_pin(entry, cfg->irq_2_pin) {
427 unsigned int reg;
428 int pin;
429
430 pin = entry->pin;
431 reg = io_apic_read(entry->apic, 0x10 + pin*2);
432 /* Is the remote IRR bit set? */
433 if (reg & IO_APIC_REDIR_REMOTE_IRR) {
434 spin_unlock_irqrestore(&ioapic_lock, flags);
435 return true;
436 }
437 }
438 spin_unlock_irqrestore(&ioapic_lock, flags);
439
440 return false;
441 }
442
443 union entry_union {
444 struct { u32 w1, w2; };
445 struct IO_APIC_route_entry entry;
446 };
447
448 static struct IO_APIC_route_entry ioapic_read_entry(int apic, int pin)
449 {
450 union entry_union eu;
451 unsigned long flags;
452 spin_lock_irqsave(&ioapic_lock, flags);
453 eu.w1 = io_apic_read(apic, 0x10 + 2 * pin);
454 eu.w2 = io_apic_read(apic, 0x11 + 2 * pin);
455 spin_unlock_irqrestore(&ioapic_lock, flags);
456 return eu.entry;
457 }
458
459 /*
460 * When we write a new IO APIC routing entry, we need to write the high
461 * word first! If the mask bit in the low word is clear, we will enable
462 * the interrupt, and we need to make sure the entry is fully populated
463 * before that happens.
464 */
465 static void
466 __ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e)
467 {
468 union entry_union eu = {{0, 0}};
469
470 eu.entry = e;
471 io_apic_write(apic, 0x11 + 2*pin, eu.w2);
472 io_apic_write(apic, 0x10 + 2*pin, eu.w1);
473 }
474
475 void ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e)
476 {
477 unsigned long flags;
478 spin_lock_irqsave(&ioapic_lock, flags);
479 __ioapic_write_entry(apic, pin, e);
480 spin_unlock_irqrestore(&ioapic_lock, flags);
481 }
482
483 /*
484 * When we mask an IO APIC routing entry, we need to write the low
485 * word first, in order to set the mask bit before we change the
486 * high bits!
487 */
488 static void ioapic_mask_entry(int apic, int pin)
489 {
490 unsigned long flags;
491 union entry_union eu = { .entry.mask = 1 };
492
493 spin_lock_irqsave(&ioapic_lock, flags);
494 io_apic_write(apic, 0x10 + 2*pin, eu.w1);
495 io_apic_write(apic, 0x11 + 2*pin, eu.w2);
496 spin_unlock_irqrestore(&ioapic_lock, flags);
497 }
498
499 /*
500 * The common case is 1:1 IRQ<->pin mappings. Sometimes there are
501 * shared ISA-space IRQs, so we have to support them. We are super
502 * fast in the common case, and fast for shared ISA-space IRQs.
503 */
504 static int
505 add_pin_to_irq_node_nopanic(struct irq_cfg *cfg, int node, int apic, int pin)
506 {
507 struct irq_pin_list **last, *entry;
508
509 /* don't allow duplicates */
510 last = &cfg->irq_2_pin;
511 for_each_irq_pin(entry, cfg->irq_2_pin) {
512 if (entry->apic == apic && entry->pin == pin)
513 return 0;
514 last = &entry->next;
515 }
516
517 entry = get_one_free_irq_2_pin(node);
518 if (!entry) {
519 printk(KERN_ERR "can not alloc irq_pin_list (%d,%d,%d)\n",
520 node, apic, pin);
521 return -ENOMEM;
522 }
523 entry->apic = apic;
524 entry->pin = pin;
525
526 *last = entry;
527 return 0;
528 }
529
530 static void add_pin_to_irq_node(struct irq_cfg *cfg, int node, int apic, int pin)
531 {
532 if (add_pin_to_irq_node_nopanic(cfg, node, apic, pin))
533 panic("IO-APIC: failed to add irq-pin. Can not proceed\n");
534 }
535
536 /*
537 * Reroute an IRQ to a different pin.
538 */
539 static void __init replace_pin_at_irq_node(struct irq_cfg *cfg, int node,
540 int oldapic, int oldpin,
541 int newapic, int newpin)
542 {
543 struct irq_pin_list *entry;
544
545 for_each_irq_pin(entry, cfg->irq_2_pin) {
546 if (entry->apic == oldapic && entry->pin == oldpin) {
547 entry->apic = newapic;
548 entry->pin = newpin;
549 /* every one is different, right? */
550 return;
551 }
552 }
553
554 /* old apic/pin didn't exist, so just add new ones */
555 add_pin_to_irq_node(cfg, node, newapic, newpin);
556 }
557
558 static void io_apic_modify_irq(struct irq_cfg *cfg,
559 int mask_and, int mask_or,
560 void (*final)(struct irq_pin_list *entry))
561 {
562 int pin;
563 struct irq_pin_list *entry;
564
565 for_each_irq_pin(entry, cfg->irq_2_pin) {
566 unsigned int reg;
567 pin = entry->pin;
568 reg = io_apic_read(entry->apic, 0x10 + pin * 2);
569 reg &= mask_and;
570 reg |= mask_or;
571 io_apic_modify(entry->apic, 0x10 + pin * 2, reg);
572 if (final)
573 final(entry);
574 }
575 }
576
577 static void __unmask_IO_APIC_irq(struct irq_cfg *cfg)
578 {
579 io_apic_modify_irq(cfg, ~IO_APIC_REDIR_MASKED, 0, NULL);
580 }
581
582 static void io_apic_sync(struct irq_pin_list *entry)
583 {
584 /*
585 * Synchronize the IO-APIC and the CPU by doing
586 * a dummy read from the IO-APIC
587 */
588 struct io_apic __iomem *io_apic;
589 io_apic = io_apic_base(entry->apic);
590 readl(&io_apic->data);
591 }
592
593 static void __mask_IO_APIC_irq(struct irq_cfg *cfg)
594 {
595 io_apic_modify_irq(cfg, ~0, IO_APIC_REDIR_MASKED, &io_apic_sync);
596 }
597
598 static void __mask_and_edge_IO_APIC_irq(struct irq_cfg *cfg)
599 {
600 io_apic_modify_irq(cfg, ~IO_APIC_REDIR_LEVEL_TRIGGER,
601 IO_APIC_REDIR_MASKED, NULL);
602 }
603
604 static void __unmask_and_level_IO_APIC_irq(struct irq_cfg *cfg)
605 {
606 io_apic_modify_irq(cfg, ~IO_APIC_REDIR_MASKED,
607 IO_APIC_REDIR_LEVEL_TRIGGER, NULL);
608 }
609
610 static void mask_IO_APIC_irq_desc(struct irq_desc *desc)
611 {
612 struct irq_cfg *cfg = desc->chip_data;
613 unsigned long flags;
614
615 BUG_ON(!cfg);
616
617 spin_lock_irqsave(&ioapic_lock, flags);
618 __mask_IO_APIC_irq(cfg);
619 spin_unlock_irqrestore(&ioapic_lock, flags);
620 }
621
622 static void unmask_IO_APIC_irq_desc(struct irq_desc *desc)
623 {
624 struct irq_cfg *cfg = desc->chip_data;
625 unsigned long flags;
626
627 spin_lock_irqsave(&ioapic_lock, flags);
628 __unmask_IO_APIC_irq(cfg);
629 spin_unlock_irqrestore(&ioapic_lock, flags);
630 }
631
632 static void mask_IO_APIC_irq(unsigned int irq)
633 {
634 struct irq_desc *desc = irq_to_desc(irq);
635
636 mask_IO_APIC_irq_desc(desc);
637 }
638 static void unmask_IO_APIC_irq(unsigned int irq)
639 {
640 struct irq_desc *desc = irq_to_desc(irq);
641
642 unmask_IO_APIC_irq_desc(desc);
643 }
644
645 static void clear_IO_APIC_pin(unsigned int apic, unsigned int pin)
646 {
647 struct IO_APIC_route_entry entry;
648
649 /* Check delivery_mode to be sure we're not clearing an SMI pin */
650 entry = ioapic_read_entry(apic, pin);
651 if (entry.delivery_mode == dest_SMI)
652 return;
653 /*
654 * Disable it in the IO-APIC irq-routing table:
655 */
656 ioapic_mask_entry(apic, pin);
657 }
658
659 static void clear_IO_APIC (void)
660 {
661 int apic, pin;
662
663 for (apic = 0; apic < nr_ioapics; apic++)
664 for (pin = 0; pin < nr_ioapic_registers[apic]; pin++)
665 clear_IO_APIC_pin(apic, pin);
666 }
667
668 #ifdef CONFIG_X86_32
669 /*
670 * support for broken MP BIOSs, enables hand-redirection of PIRQ0-7 to
671 * specific CPU-side IRQs.
672 */
673
674 #define MAX_PIRQS 8
675 static int pirq_entries[MAX_PIRQS] = {
676 [0 ... MAX_PIRQS - 1] = -1
677 };
678
679 static int __init ioapic_pirq_setup(char *str)
680 {
681 int i, max;
682 int ints[MAX_PIRQS+1];
683
684 get_options(str, ARRAY_SIZE(ints), ints);
685
686 apic_printk(APIC_VERBOSE, KERN_INFO
687 "PIRQ redirection, working around broken MP-BIOS.\n");
688 max = MAX_PIRQS;
689 if (ints[0] < MAX_PIRQS)
690 max = ints[0];
691
692 for (i = 0; i < max; i++) {
693 apic_printk(APIC_VERBOSE, KERN_DEBUG
694 "... PIRQ%d -> IRQ %d\n", i, ints[i+1]);
695 /*
696 * PIRQs are mapped upside down, usually.
697 */
698 pirq_entries[MAX_PIRQS-i-1] = ints[i+1];
699 }
700 return 1;
701 }
702
703 __setup("pirq=", ioapic_pirq_setup);
704 #endif /* CONFIG_X86_32 */
705
706 struct IO_APIC_route_entry **alloc_ioapic_entries(void)
707 {
708 int apic;
709 struct IO_APIC_route_entry **ioapic_entries;
710
711 ioapic_entries = kzalloc(sizeof(*ioapic_entries) * nr_ioapics,
712 GFP_ATOMIC);
713 if (!ioapic_entries)
714 return 0;
715
716 for (apic = 0; apic < nr_ioapics; apic++) {
717 ioapic_entries[apic] =
718 kzalloc(sizeof(struct IO_APIC_route_entry) *
719 nr_ioapic_registers[apic], GFP_ATOMIC);
720 if (!ioapic_entries[apic])
721 goto nomem;
722 }
723
724 return ioapic_entries;
725
726 nomem:
727 while (--apic >= 0)
728 kfree(ioapic_entries[apic]);
729 kfree(ioapic_entries);
730
731 return 0;
732 }
733
734 /*
735 * Saves all the IO-APIC RTE's
736 */
737 int save_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries)
738 {
739 int apic, pin;
740
741 if (!ioapic_entries)
742 return -ENOMEM;
743
744 for (apic = 0; apic < nr_ioapics; apic++) {
745 if (!ioapic_entries[apic])
746 return -ENOMEM;
747
748 for (pin = 0; pin < nr_ioapic_registers[apic]; pin++)
749 ioapic_entries[apic][pin] =
750 ioapic_read_entry(apic, pin);
751 }
752
753 return 0;
754 }
755
756 /*
757 * Mask all IO APIC entries.
758 */
759 void mask_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries)
760 {
761 int apic, pin;
762
763 if (!ioapic_entries)
764 return;
765
766 for (apic = 0; apic < nr_ioapics; apic++) {
767 if (!ioapic_entries[apic])
768 break;
769
770 for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
771 struct IO_APIC_route_entry entry;
772
773 entry = ioapic_entries[apic][pin];
774 if (!entry.mask) {
775 entry.mask = 1;
776 ioapic_write_entry(apic, pin, entry);
777 }
778 }
779 }
780 }
781
782 /*
783 * Restore IO APIC entries which was saved in ioapic_entries.
784 */
785 int restore_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries)
786 {
787 int apic, pin;
788
789 if (!ioapic_entries)
790 return -ENOMEM;
791
792 for (apic = 0; apic < nr_ioapics; apic++) {
793 if (!ioapic_entries[apic])
794 return -ENOMEM;
795
796 for (pin = 0; pin < nr_ioapic_registers[apic]; pin++)
797 ioapic_write_entry(apic, pin,
798 ioapic_entries[apic][pin]);
799 }
800 return 0;
801 }
802
803 void free_ioapic_entries(struct IO_APIC_route_entry **ioapic_entries)
804 {
805 int apic;
806
807 for (apic = 0; apic < nr_ioapics; apic++)
808 kfree(ioapic_entries[apic]);
809
810 kfree(ioapic_entries);
811 }
812
813 /*
814 * Find the IRQ entry number of a certain pin.
815 */
816 static int find_irq_entry(int apic, int pin, int type)
817 {
818 int i;
819
820 for (i = 0; i < mp_irq_entries; i++)
821 if (mp_irqs[i].irqtype == type &&
822 (mp_irqs[i].dstapic == mp_ioapics[apic].apicid ||
823 mp_irqs[i].dstapic == MP_APIC_ALL) &&
824 mp_irqs[i].dstirq == pin)
825 return i;
826
827 return -1;
828 }
829
830 /*
831 * Find the pin to which IRQ[irq] (ISA) is connected
832 */
833 static int __init find_isa_irq_pin(int irq, int type)
834 {
835 int i;
836
837 for (i = 0; i < mp_irq_entries; i++) {
838 int lbus = mp_irqs[i].srcbus;
839
840 if (test_bit(lbus, mp_bus_not_pci) &&
841 (mp_irqs[i].irqtype == type) &&
842 (mp_irqs[i].srcbusirq == irq))
843
844 return mp_irqs[i].dstirq;
845 }
846 return -1;
847 }
848
849 static int __init find_isa_irq_apic(int irq, int type)
850 {
851 int i;
852
853 for (i = 0; i < mp_irq_entries; i++) {
854 int lbus = mp_irqs[i].srcbus;
855
856 if (test_bit(lbus, mp_bus_not_pci) &&
857 (mp_irqs[i].irqtype == type) &&
858 (mp_irqs[i].srcbusirq == irq))
859 break;
860 }
861 if (i < mp_irq_entries) {
862 int apic;
863 for(apic = 0; apic < nr_ioapics; apic++) {
864 if (mp_ioapics[apic].apicid == mp_irqs[i].dstapic)
865 return apic;
866 }
867 }
868
869 return -1;
870 }
871
872 #if defined(CONFIG_EISA) || defined(CONFIG_MCA)
873 /*
874 * EISA Edge/Level control register, ELCR
875 */
876 static int EISA_ELCR(unsigned int irq)
877 {
878 if (irq < nr_legacy_irqs) {
879 unsigned int port = 0x4d0 + (irq >> 3);
880 return (inb(port) >> (irq & 7)) & 1;
881 }
882 apic_printk(APIC_VERBOSE, KERN_INFO
883 "Broken MPtable reports ISA irq %d\n", irq);
884 return 0;
885 }
886
887 #endif
888
889 /* ISA interrupts are always polarity zero edge triggered,
890 * when listed as conforming in the MP table. */
891
892 #define default_ISA_trigger(idx) (0)
893 #define default_ISA_polarity(idx) (0)
894
895 /* EISA interrupts are always polarity zero and can be edge or level
896 * trigger depending on the ELCR value. If an interrupt is listed as
897 * EISA conforming in the MP table, that means its trigger type must
898 * be read in from the ELCR */
899
900 #define default_EISA_trigger(idx) (EISA_ELCR(mp_irqs[idx].srcbusirq))
901 #define default_EISA_polarity(idx) default_ISA_polarity(idx)
902
903 /* PCI interrupts are always polarity one level triggered,
904 * when listed as conforming in the MP table. */
905
906 #define default_PCI_trigger(idx) (1)
907 #define default_PCI_polarity(idx) (1)
908
909 /* MCA interrupts are always polarity zero level triggered,
910 * when listed as conforming in the MP table. */
911
912 #define default_MCA_trigger(idx) (1)
913 #define default_MCA_polarity(idx) default_ISA_polarity(idx)
914
915 static int MPBIOS_polarity(int idx)
916 {
917 int bus = mp_irqs[idx].srcbus;
918 int polarity;
919
920 /*
921 * Determine IRQ line polarity (high active or low active):
922 */
923 switch (mp_irqs[idx].irqflag & 3)
924 {
925 case 0: /* conforms, ie. bus-type dependent polarity */
926 if (test_bit(bus, mp_bus_not_pci))
927 polarity = default_ISA_polarity(idx);
928 else
929 polarity = default_PCI_polarity(idx);
930 break;
931 case 1: /* high active */
932 {
933 polarity = 0;
934 break;
935 }
936 case 2: /* reserved */
937 {
938 printk(KERN_WARNING "broken BIOS!!\n");
939 polarity = 1;
940 break;
941 }
942 case 3: /* low active */
943 {
944 polarity = 1;
945 break;
946 }
947 default: /* invalid */
948 {
949 printk(KERN_WARNING "broken BIOS!!\n");
950 polarity = 1;
951 break;
952 }
953 }
954 return polarity;
955 }
956
957 static int MPBIOS_trigger(int idx)
958 {
959 int bus = mp_irqs[idx].srcbus;
960 int trigger;
961
962 /*
963 * Determine IRQ trigger mode (edge or level sensitive):
964 */
965 switch ((mp_irqs[idx].irqflag>>2) & 3)
966 {
967 case 0: /* conforms, ie. bus-type dependent */
968 if (test_bit(bus, mp_bus_not_pci))
969 trigger = default_ISA_trigger(idx);
970 else
971 trigger = default_PCI_trigger(idx);
972 #if defined(CONFIG_EISA) || defined(CONFIG_MCA)
973 switch (mp_bus_id_to_type[bus]) {
974 case MP_BUS_ISA: /* ISA pin */
975 {
976 /* set before the switch */
977 break;
978 }
979 case MP_BUS_EISA: /* EISA pin */
980 {
981 trigger = default_EISA_trigger(idx);
982 break;
983 }
984 case MP_BUS_PCI: /* PCI pin */
985 {
986 /* set before the switch */
987 break;
988 }
989 case MP_BUS_MCA: /* MCA pin */
990 {
991 trigger = default_MCA_trigger(idx);
992 break;
993 }
994 default:
995 {
996 printk(KERN_WARNING "broken BIOS!!\n");
997 trigger = 1;
998 break;
999 }
1000 }
1001 #endif
1002 break;
1003 case 1: /* edge */
1004 {
1005 trigger = 0;
1006 break;
1007 }
1008 case 2: /* reserved */
1009 {
1010 printk(KERN_WARNING "broken BIOS!!\n");
1011 trigger = 1;
1012 break;
1013 }
1014 case 3: /* level */
1015 {
1016 trigger = 1;
1017 break;
1018 }
1019 default: /* invalid */
1020 {
1021 printk(KERN_WARNING "broken BIOS!!\n");
1022 trigger = 0;
1023 break;
1024 }
1025 }
1026 return trigger;
1027 }
1028
1029 static inline int irq_polarity(int idx)
1030 {
1031 return MPBIOS_polarity(idx);
1032 }
1033
1034 static inline int irq_trigger(int idx)
1035 {
1036 return MPBIOS_trigger(idx);
1037 }
1038
1039 int (*ioapic_renumber_irq)(int ioapic, int irq);
1040 static int pin_2_irq(int idx, int apic, int pin)
1041 {
1042 int irq, i;
1043 int bus = mp_irqs[idx].srcbus;
1044
1045 /*
1046 * Debugging check, we are in big trouble if this message pops up!
1047 */
1048 if (mp_irqs[idx].dstirq != pin)
1049 printk(KERN_ERR "broken BIOS or MPTABLE parser, ayiee!!\n");
1050
1051 if (test_bit(bus, mp_bus_not_pci)) {
1052 irq = mp_irqs[idx].srcbusirq;
1053 } else {
1054 /*
1055 * PCI IRQs are mapped in order
1056 */
1057 i = irq = 0;
1058 while (i < apic)
1059 irq += nr_ioapic_registers[i++];
1060 irq += pin;
1061 /*
1062 * For MPS mode, so far only needed by ES7000 platform
1063 */
1064 if (ioapic_renumber_irq)
1065 irq = ioapic_renumber_irq(apic, irq);
1066 }
1067
1068 #ifdef CONFIG_X86_32
1069 /*
1070 * PCI IRQ command line redirection. Yes, limits are hardcoded.
1071 */
1072 if ((pin >= 16) && (pin <= 23)) {
1073 if (pirq_entries[pin-16] != -1) {
1074 if (!pirq_entries[pin-16]) {
1075 apic_printk(APIC_VERBOSE, KERN_DEBUG
1076 "disabling PIRQ%d\n", pin-16);
1077 } else {
1078 irq = pirq_entries[pin-16];
1079 apic_printk(APIC_VERBOSE, KERN_DEBUG
1080 "using PIRQ%d -> IRQ %d\n",
1081 pin-16, irq);
1082 }
1083 }
1084 }
1085 #endif
1086
1087 return irq;
1088 }
1089
1090 /*
1091 * Find a specific PCI IRQ entry.
1092 * Not an __init, possibly needed by modules
1093 */
1094 int IO_APIC_get_PCI_irq_vector(int bus, int slot, int pin,
1095 struct io_apic_irq_attr *irq_attr)
1096 {
1097 int apic, i, best_guess = -1;
1098
1099 apic_printk(APIC_DEBUG,
1100 "querying PCI -> IRQ mapping bus:%d, slot:%d, pin:%d.\n",
1101 bus, slot, pin);
1102 if (test_bit(bus, mp_bus_not_pci)) {
1103 apic_printk(APIC_VERBOSE,
1104 "PCI BIOS passed nonexistent PCI bus %d!\n", bus);
1105 return -1;
1106 }
1107 for (i = 0; i < mp_irq_entries; i++) {
1108 int lbus = mp_irqs[i].srcbus;
1109
1110 for (apic = 0; apic < nr_ioapics; apic++)
1111 if (mp_ioapics[apic].apicid == mp_irqs[i].dstapic ||
1112 mp_irqs[i].dstapic == MP_APIC_ALL)
1113 break;
1114
1115 if (!test_bit(lbus, mp_bus_not_pci) &&
1116 !mp_irqs[i].irqtype &&
1117 (bus == lbus) &&
1118 (slot == ((mp_irqs[i].srcbusirq >> 2) & 0x1f))) {
1119 int irq = pin_2_irq(i, apic, mp_irqs[i].dstirq);
1120
1121 if (!(apic || IO_APIC_IRQ(irq)))
1122 continue;
1123
1124 if (pin == (mp_irqs[i].srcbusirq & 3)) {
1125 set_io_apic_irq_attr(irq_attr, apic,
1126 mp_irqs[i].dstirq,
1127 irq_trigger(i),
1128 irq_polarity(i));
1129 return irq;
1130 }
1131 /*
1132 * Use the first all-but-pin matching entry as a
1133 * best-guess fuzzy result for broken mptables.
1134 */
1135 if (best_guess < 0) {
1136 set_io_apic_irq_attr(irq_attr, apic,
1137 mp_irqs[i].dstirq,
1138 irq_trigger(i),
1139 irq_polarity(i));
1140 best_guess = irq;
1141 }
1142 }
1143 }
1144 return best_guess;
1145 }
1146 EXPORT_SYMBOL(IO_APIC_get_PCI_irq_vector);
1147
1148 void lock_vector_lock(void)
1149 {
1150 /* Used to the online set of cpus does not change
1151 * during assign_irq_vector.
1152 */
1153 spin_lock(&vector_lock);
1154 }
1155
1156 void unlock_vector_lock(void)
1157 {
1158 spin_unlock(&vector_lock);
1159 }
1160
1161 static int
1162 __assign_irq_vector(int irq, struct irq_cfg *cfg, const struct cpumask *mask)
1163 {
1164 /*
1165 * NOTE! The local APIC isn't very good at handling
1166 * multiple interrupts at the same interrupt level.
1167 * As the interrupt level is determined by taking the
1168 * vector number and shifting that right by 4, we
1169 * want to spread these out a bit so that they don't
1170 * all fall in the same interrupt level.
1171 *
1172 * Also, we've got to be careful not to trash gate
1173 * 0x80, because int 0x80 is hm, kind of importantish. ;)
1174 */
1175 static int current_vector = FIRST_DEVICE_VECTOR, current_offset = 0;
1176 unsigned int old_vector;
1177 int cpu, err;
1178 cpumask_var_t tmp_mask;
1179
1180 if ((cfg->move_in_progress) || cfg->move_cleanup_count)
1181 return -EBUSY;
1182
1183 if (!alloc_cpumask_var(&tmp_mask, GFP_ATOMIC))
1184 return -ENOMEM;
1185
1186 old_vector = cfg->vector;
1187 if (old_vector) {
1188 cpumask_and(tmp_mask, mask, cpu_online_mask);
1189 cpumask_and(tmp_mask, cfg->domain, tmp_mask);
1190 if (!cpumask_empty(tmp_mask)) {
1191 free_cpumask_var(tmp_mask);
1192 return 0;
1193 }
1194 }
1195
1196 /* Only try and allocate irqs on cpus that are present */
1197 err = -ENOSPC;
1198 for_each_cpu_and(cpu, mask, cpu_online_mask) {
1199 int new_cpu;
1200 int vector, offset;
1201
1202 apic->vector_allocation_domain(cpu, tmp_mask);
1203
1204 vector = current_vector;
1205 offset = current_offset;
1206 next:
1207 vector += 8;
1208 if (vector >= first_system_vector) {
1209 /* If out of vectors on large boxen, must share them. */
1210 offset = (offset + 1) % 8;
1211 vector = FIRST_DEVICE_VECTOR + offset;
1212 }
1213 if (unlikely(current_vector == vector))
1214 continue;
1215
1216 if (test_bit(vector, used_vectors))
1217 goto next;
1218
1219 for_each_cpu_and(new_cpu, tmp_mask, cpu_online_mask)
1220 if (per_cpu(vector_irq, new_cpu)[vector] != -1)
1221 goto next;
1222 /* Found one! */
1223 current_vector = vector;
1224 current_offset = offset;
1225 if (old_vector) {
1226 cfg->move_in_progress = 1;
1227 cpumask_copy(cfg->old_domain, cfg->domain);
1228 }
1229 for_each_cpu_and(new_cpu, tmp_mask, cpu_online_mask)
1230 per_cpu(vector_irq, new_cpu)[vector] = irq;
1231 cfg->vector = vector;
1232 cpumask_copy(cfg->domain, tmp_mask);
1233 err = 0;
1234 break;
1235 }
1236 free_cpumask_var(tmp_mask);
1237 return err;
1238 }
1239
1240 static int
1241 assign_irq_vector(int irq, struct irq_cfg *cfg, const struct cpumask *mask)
1242 {
1243 int err;
1244 unsigned long flags;
1245
1246 spin_lock_irqsave(&vector_lock, flags);
1247 err = __assign_irq_vector(irq, cfg, mask);
1248 spin_unlock_irqrestore(&vector_lock, flags);
1249 return err;
1250 }
1251
1252 static void __clear_irq_vector(int irq, struct irq_cfg *cfg)
1253 {
1254 int cpu, vector;
1255
1256 BUG_ON(!cfg->vector);
1257
1258 vector = cfg->vector;
1259 for_each_cpu_and(cpu, cfg->domain, cpu_online_mask)
1260 per_cpu(vector_irq, cpu)[vector] = -1;
1261
1262 cfg->vector = 0;
1263 cpumask_clear(cfg->domain);
1264
1265 if (likely(!cfg->move_in_progress))
1266 return;
1267 for_each_cpu_and(cpu, cfg->old_domain, cpu_online_mask) {
1268 for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS;
1269 vector++) {
1270 if (per_cpu(vector_irq, cpu)[vector] != irq)
1271 continue;
1272 per_cpu(vector_irq, cpu)[vector] = -1;
1273 break;
1274 }
1275 }
1276 cfg->move_in_progress = 0;
1277 }
1278
1279 void __setup_vector_irq(int cpu)
1280 {
1281 /* Initialize vector_irq on a new cpu */
1282 /* This function must be called with vector_lock held */
1283 int irq, vector;
1284 struct irq_cfg *cfg;
1285 struct irq_desc *desc;
1286
1287 /* Mark the inuse vectors */
1288 for_each_irq_desc(irq, desc) {
1289 cfg = desc->chip_data;
1290 if (!cpumask_test_cpu(cpu, cfg->domain))
1291 continue;
1292 vector = cfg->vector;
1293 per_cpu(vector_irq, cpu)[vector] = irq;
1294 }
1295 /* Mark the free vectors */
1296 for (vector = 0; vector < NR_VECTORS; ++vector) {
1297 irq = per_cpu(vector_irq, cpu)[vector];
1298 if (irq < 0)
1299 continue;
1300
1301 cfg = irq_cfg(irq);
1302 if (!cpumask_test_cpu(cpu, cfg->domain))
1303 per_cpu(vector_irq, cpu)[vector] = -1;
1304 }
1305 }
1306
1307 static struct irq_chip ioapic_chip;
1308 static struct irq_chip ir_ioapic_chip;
1309
1310 #define IOAPIC_AUTO -1
1311 #define IOAPIC_EDGE 0
1312 #define IOAPIC_LEVEL 1
1313
1314 #ifdef CONFIG_X86_32
1315 static inline int IO_APIC_irq_trigger(int irq)
1316 {
1317 int apic, idx, pin;
1318
1319 for (apic = 0; apic < nr_ioapics; apic++) {
1320 for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
1321 idx = find_irq_entry(apic, pin, mp_INT);
1322 if ((idx != -1) && (irq == pin_2_irq(idx, apic, pin)))
1323 return irq_trigger(idx);
1324 }
1325 }
1326 /*
1327 * nonexistent IRQs are edge default
1328 */
1329 return 0;
1330 }
1331 #else
1332 static inline int IO_APIC_irq_trigger(int irq)
1333 {
1334 return 1;
1335 }
1336 #endif
1337
1338 static void ioapic_register_intr(int irq, struct irq_desc *desc, unsigned long trigger)
1339 {
1340
1341 if ((trigger == IOAPIC_AUTO && IO_APIC_irq_trigger(irq)) ||
1342 trigger == IOAPIC_LEVEL)
1343 desc->status |= IRQ_LEVEL;
1344 else
1345 desc->status &= ~IRQ_LEVEL;
1346
1347 if (irq_remapped(irq)) {
1348 desc->status |= IRQ_MOVE_PCNTXT;
1349 if (trigger)
1350 set_irq_chip_and_handler_name(irq, &ir_ioapic_chip,
1351 handle_fasteoi_irq,
1352 "fasteoi");
1353 else
1354 set_irq_chip_and_handler_name(irq, &ir_ioapic_chip,
1355 handle_edge_irq, "edge");
1356 return;
1357 }
1358
1359 if ((trigger == IOAPIC_AUTO && IO_APIC_irq_trigger(irq)) ||
1360 trigger == IOAPIC_LEVEL)
1361 set_irq_chip_and_handler_name(irq, &ioapic_chip,
1362 handle_fasteoi_irq,
1363 "fasteoi");
1364 else
1365 set_irq_chip_and_handler_name(irq, &ioapic_chip,
1366 handle_edge_irq, "edge");
1367 }
1368
1369 int setup_ioapic_entry(int apic_id, int irq,
1370 struct IO_APIC_route_entry *entry,
1371 unsigned int destination, int trigger,
1372 int polarity, int vector, int pin)
1373 {
1374 /*
1375 * add it to the IO-APIC irq-routing table:
1376 */
1377 memset(entry,0,sizeof(*entry));
1378
1379 if (intr_remapping_enabled) {
1380 struct intel_iommu *iommu = map_ioapic_to_ir(apic_id);
1381 struct irte irte;
1382 struct IR_IO_APIC_route_entry *ir_entry =
1383 (struct IR_IO_APIC_route_entry *) entry;
1384 int index;
1385
1386 if (!iommu)
1387 panic("No mapping iommu for ioapic %d\n", apic_id);
1388
1389 index = alloc_irte(iommu, irq, 1);
1390 if (index < 0)
1391 panic("Failed to allocate IRTE for ioapic %d\n", apic_id);
1392
1393 memset(&irte, 0, sizeof(irte));
1394
1395 irte.present = 1;
1396 irte.dst_mode = apic->irq_dest_mode;
1397 /*
1398 * Trigger mode in the IRTE will always be edge, and the
1399 * actual level or edge trigger will be setup in the IO-APIC
1400 * RTE. This will help simplify level triggered irq migration.
1401 * For more details, see the comments above explainig IO-APIC
1402 * irq migration in the presence of interrupt-remapping.
1403 */
1404 irte.trigger_mode = 0;
1405 irte.dlvry_mode = apic->irq_delivery_mode;
1406 irte.vector = vector;
1407 irte.dest_id = IRTE_DEST(destination);
1408
1409 /* Set source-id of interrupt request */
1410 set_ioapic_sid(&irte, apic_id);
1411
1412 modify_irte(irq, &irte);
1413
1414 ir_entry->index2 = (index >> 15) & 0x1;
1415 ir_entry->zero = 0;
1416 ir_entry->format = 1;
1417 ir_entry->index = (index & 0x7fff);
1418 /*
1419 * IO-APIC RTE will be configured with virtual vector.
1420 * irq handler will do the explicit EOI to the io-apic.
1421 */
1422 ir_entry->vector = pin;
1423 } else {
1424 entry->delivery_mode = apic->irq_delivery_mode;
1425 entry->dest_mode = apic->irq_dest_mode;
1426 entry->dest = destination;
1427 entry->vector = vector;
1428 }
1429
1430 entry->mask = 0; /* enable IRQ */
1431 entry->trigger = trigger;
1432 entry->polarity = polarity;
1433
1434 /* Mask level triggered irqs.
1435 * Use IRQ_DELAYED_DISABLE for edge triggered irqs.
1436 */
1437 if (trigger)
1438 entry->mask = 1;
1439 return 0;
1440 }
1441
1442 static void setup_IO_APIC_irq(int apic_id, int pin, unsigned int irq, struct irq_desc *desc,
1443 int trigger, int polarity)
1444 {
1445 struct irq_cfg *cfg;
1446 struct IO_APIC_route_entry entry;
1447 unsigned int dest;
1448
1449 if (!IO_APIC_IRQ(irq))
1450 return;
1451
1452 cfg = desc->chip_data;
1453
1454 if (assign_irq_vector(irq, cfg, apic->target_cpus()))
1455 return;
1456
1457 dest = apic->cpu_mask_to_apicid_and(cfg->domain, apic->target_cpus());
1458
1459 apic_printk(APIC_VERBOSE,KERN_DEBUG
1460 "IOAPIC[%d]: Set routing entry (%d-%d -> 0x%x -> "
1461 "IRQ %d Mode:%i Active:%i)\n",
1462 apic_id, mp_ioapics[apic_id].apicid, pin, cfg->vector,
1463 irq, trigger, polarity);
1464
1465
1466 if (setup_ioapic_entry(mp_ioapics[apic_id].apicid, irq, &entry,
1467 dest, trigger, polarity, cfg->vector, pin)) {
1468 printk("Failed to setup ioapic entry for ioapic %d, pin %d\n",
1469 mp_ioapics[apic_id].apicid, pin);
1470 __clear_irq_vector(irq, cfg);
1471 return;
1472 }
1473
1474 ioapic_register_intr(irq, desc, trigger);
1475 if (irq < nr_legacy_irqs)
1476 disable_8259A_irq(irq);
1477
1478 ioapic_write_entry(apic_id, pin, entry);
1479 }
1480
1481 static struct {
1482 DECLARE_BITMAP(pin_programmed, MP_MAX_IOAPIC_PIN + 1);
1483 } mp_ioapic_routing[MAX_IO_APICS];
1484
1485 static void __init setup_IO_APIC_irqs(void)
1486 {
1487 int apic_id = 0, pin, idx, irq;
1488 int notcon = 0;
1489 struct irq_desc *desc;
1490 struct irq_cfg *cfg;
1491 int node = cpu_to_node(boot_cpu_id);
1492
1493 apic_printk(APIC_VERBOSE, KERN_DEBUG "init IO_APIC IRQs\n");
1494
1495 #ifdef CONFIG_ACPI
1496 if (!acpi_disabled && acpi_ioapic) {
1497 apic_id = mp_find_ioapic(0);
1498 if (apic_id < 0)
1499 apic_id = 0;
1500 }
1501 #endif
1502
1503 for (pin = 0; pin < nr_ioapic_registers[apic_id]; pin++) {
1504 idx = find_irq_entry(apic_id, pin, mp_INT);
1505 if (idx == -1) {
1506 if (!notcon) {
1507 notcon = 1;
1508 apic_printk(APIC_VERBOSE,
1509 KERN_DEBUG " %d-%d",
1510 mp_ioapics[apic_id].apicid, pin);
1511 } else
1512 apic_printk(APIC_VERBOSE, " %d-%d",
1513 mp_ioapics[apic_id].apicid, pin);
1514 continue;
1515 }
1516 if (notcon) {
1517 apic_printk(APIC_VERBOSE,
1518 " (apicid-pin) not connected\n");
1519 notcon = 0;
1520 }
1521
1522 irq = pin_2_irq(idx, apic_id, pin);
1523
1524 /*
1525 * Skip the timer IRQ if there's a quirk handler
1526 * installed and if it returns 1:
1527 */
1528 if (apic->multi_timer_check &&
1529 apic->multi_timer_check(apic_id, irq))
1530 continue;
1531
1532 desc = irq_to_desc_alloc_node(irq, node);
1533 if (!desc) {
1534 printk(KERN_INFO "can not get irq_desc for %d\n", irq);
1535 continue;
1536 }
1537 cfg = desc->chip_data;
1538 add_pin_to_irq_node(cfg, node, apic_id, pin);
1539 /*
1540 * don't mark it in pin_programmed, so later acpi could
1541 * set it correctly when irq < 16
1542 */
1543 setup_IO_APIC_irq(apic_id, pin, irq, desc,
1544 irq_trigger(idx), irq_polarity(idx));
1545 }
1546
1547 if (notcon)
1548 apic_printk(APIC_VERBOSE,
1549 " (apicid-pin) not connected\n");
1550 }
1551
1552 /*
1553 * Set up the timer pin, possibly with the 8259A-master behind.
1554 */
1555 static void __init setup_timer_IRQ0_pin(unsigned int apic_id, unsigned int pin,
1556 int vector)
1557 {
1558 struct IO_APIC_route_entry entry;
1559
1560 if (intr_remapping_enabled)
1561 return;
1562
1563 memset(&entry, 0, sizeof(entry));
1564
1565 /*
1566 * We use logical delivery to get the timer IRQ
1567 * to the first CPU.
1568 */
1569 entry.dest_mode = apic->irq_dest_mode;
1570 entry.mask = 0; /* don't mask IRQ for edge */
1571 entry.dest = apic->cpu_mask_to_apicid(apic->target_cpus());
1572 entry.delivery_mode = apic->irq_delivery_mode;
1573 entry.polarity = 0;
1574 entry.trigger = 0;
1575 entry.vector = vector;
1576
1577 /*
1578 * The timer IRQ doesn't have to know that behind the
1579 * scene we may have a 8259A-master in AEOI mode ...
1580 */
1581 set_irq_chip_and_handler_name(0, &ioapic_chip, handle_edge_irq, "edge");
1582
1583 /*
1584 * Add it to the IO-APIC irq-routing table:
1585 */
1586 ioapic_write_entry(apic_id, pin, entry);
1587 }
1588
1589
1590 __apicdebuginit(void) print_IO_APIC(void)
1591 {
1592 int apic, i;
1593 union IO_APIC_reg_00 reg_00;
1594 union IO_APIC_reg_01 reg_01;
1595 union IO_APIC_reg_02 reg_02;
1596 union IO_APIC_reg_03 reg_03;
1597 unsigned long flags;
1598 struct irq_cfg *cfg;
1599 struct irq_desc *desc;
1600 unsigned int irq;
1601
1602 if (apic_verbosity == APIC_QUIET)
1603 return;
1604
1605 printk(KERN_DEBUG "number of MP IRQ sources: %d.\n", mp_irq_entries);
1606 for (i = 0; i < nr_ioapics; i++)
1607 printk(KERN_DEBUG "number of IO-APIC #%d registers: %d.\n",
1608 mp_ioapics[i].apicid, nr_ioapic_registers[i]);
1609
1610 /*
1611 * We are a bit conservative about what we expect. We have to
1612 * know about every hardware change ASAP.
1613 */
1614 printk(KERN_INFO "testing the IO APIC.......................\n");
1615
1616 for (apic = 0; apic < nr_ioapics; apic++) {
1617
1618 spin_lock_irqsave(&ioapic_lock, flags);
1619 reg_00.raw = io_apic_read(apic, 0);
1620 reg_01.raw = io_apic_read(apic, 1);
1621 if (reg_01.bits.version >= 0x10)
1622 reg_02.raw = io_apic_read(apic, 2);
1623 if (reg_01.bits.version >= 0x20)
1624 reg_03.raw = io_apic_read(apic, 3);
1625 spin_unlock_irqrestore(&ioapic_lock, flags);
1626
1627 printk("\n");
1628 printk(KERN_DEBUG "IO APIC #%d......\n", mp_ioapics[apic].apicid);
1629 printk(KERN_DEBUG ".... register #00: %08X\n", reg_00.raw);
1630 printk(KERN_DEBUG "....... : physical APIC id: %02X\n", reg_00.bits.ID);
1631 printk(KERN_DEBUG "....... : Delivery Type: %X\n", reg_00.bits.delivery_type);
1632 printk(KERN_DEBUG "....... : LTS : %X\n", reg_00.bits.LTS);
1633
1634 printk(KERN_DEBUG ".... register #01: %08X\n", *(int *)&reg_01);
1635 printk(KERN_DEBUG "....... : max redirection entries: %04X\n", reg_01.bits.entries);
1636
1637 printk(KERN_DEBUG "....... : PRQ implemented: %X\n", reg_01.bits.PRQ);
1638 printk(KERN_DEBUG "....... : IO APIC version: %04X\n", reg_01.bits.version);
1639
1640 /*
1641 * Some Intel chipsets with IO APIC VERSION of 0x1? don't have reg_02,
1642 * but the value of reg_02 is read as the previous read register
1643 * value, so ignore it if reg_02 == reg_01.
1644 */
1645 if (reg_01.bits.version >= 0x10 && reg_02.raw != reg_01.raw) {
1646 printk(KERN_DEBUG ".... register #02: %08X\n", reg_02.raw);
1647 printk(KERN_DEBUG "....... : arbitration: %02X\n", reg_02.bits.arbitration);
1648 }
1649
1650 /*
1651 * Some Intel chipsets with IO APIC VERSION of 0x2? don't have reg_02
1652 * or reg_03, but the value of reg_0[23] is read as the previous read
1653 * register value, so ignore it if reg_03 == reg_0[12].
1654 */
1655 if (reg_01.bits.version >= 0x20 && reg_03.raw != reg_02.raw &&
1656 reg_03.raw != reg_01.raw) {
1657 printk(KERN_DEBUG ".... register #03: %08X\n", reg_03.raw);
1658 printk(KERN_DEBUG "....... : Boot DT : %X\n", reg_03.bits.boot_DT);
1659 }
1660
1661 printk(KERN_DEBUG ".... IRQ redirection table:\n");
1662
1663 printk(KERN_DEBUG " NR Dst Mask Trig IRR Pol"
1664 " Stat Dmod Deli Vect: \n");
1665
1666 for (i = 0; i <= reg_01.bits.entries; i++) {
1667 struct IO_APIC_route_entry entry;
1668
1669 entry = ioapic_read_entry(apic, i);
1670
1671 printk(KERN_DEBUG " %02x %03X ",
1672 i,
1673 entry.dest
1674 );
1675
1676 printk("%1d %1d %1d %1d %1d %1d %1d %02X\n",
1677 entry.mask,
1678 entry.trigger,
1679 entry.irr,
1680 entry.polarity,
1681 entry.delivery_status,
1682 entry.dest_mode,
1683 entry.delivery_mode,
1684 entry.vector
1685 );
1686 }
1687 }
1688 printk(KERN_DEBUG "IRQ to pin mappings:\n");
1689 for_each_irq_desc(irq, desc) {
1690 struct irq_pin_list *entry;
1691
1692 cfg = desc->chip_data;
1693 entry = cfg->irq_2_pin;
1694 if (!entry)
1695 continue;
1696 printk(KERN_DEBUG "IRQ%d ", irq);
1697 for_each_irq_pin(entry, cfg->irq_2_pin)
1698 printk("-> %d:%d", entry->apic, entry->pin);
1699 printk("\n");
1700 }
1701
1702 printk(KERN_INFO ".................................... done.\n");
1703
1704 return;
1705 }
1706
1707 __apicdebuginit(void) print_APIC_field(int base)
1708 {
1709 int i;
1710
1711 if (apic_verbosity == APIC_QUIET)
1712 return;
1713
1714 printk(KERN_DEBUG);
1715
1716 for (i = 0; i < 8; i++)
1717 printk(KERN_CONT "%08x", apic_read(base + i*0x10));
1718
1719 printk(KERN_CONT "\n");
1720 }
1721
1722 __apicdebuginit(void) print_local_APIC(void *dummy)
1723 {
1724 unsigned int i, v, ver, maxlvt;
1725 u64 icr;
1726
1727 if (apic_verbosity == APIC_QUIET)
1728 return;
1729
1730 printk(KERN_DEBUG "printing local APIC contents on CPU#%d/%d:\n",
1731 smp_processor_id(), hard_smp_processor_id());
1732 v = apic_read(APIC_ID);
1733 printk(KERN_INFO "... APIC ID: %08x (%01x)\n", v, read_apic_id());
1734 v = apic_read(APIC_LVR);
1735 printk(KERN_INFO "... APIC VERSION: %08x\n", v);
1736 ver = GET_APIC_VERSION(v);
1737 maxlvt = lapic_get_maxlvt();
1738
1739 v = apic_read(APIC_TASKPRI);
1740 printk(KERN_DEBUG "... APIC TASKPRI: %08x (%02x)\n", v, v & APIC_TPRI_MASK);
1741
1742 if (APIC_INTEGRATED(ver)) { /* !82489DX */
1743 if (!APIC_XAPIC(ver)) {
1744 v = apic_read(APIC_ARBPRI);
1745 printk(KERN_DEBUG "... APIC ARBPRI: %08x (%02x)\n", v,
1746 v & APIC_ARBPRI_MASK);
1747 }
1748 v = apic_read(APIC_PROCPRI);
1749 printk(KERN_DEBUG "... APIC PROCPRI: %08x\n", v);
1750 }
1751
1752 /*
1753 * Remote read supported only in the 82489DX and local APIC for
1754 * Pentium processors.
1755 */
1756 if (!APIC_INTEGRATED(ver) || maxlvt == 3) {
1757 v = apic_read(APIC_RRR);
1758 printk(KERN_DEBUG "... APIC RRR: %08x\n", v);
1759 }
1760
1761 v = apic_read(APIC_LDR);
1762 printk(KERN_DEBUG "... APIC LDR: %08x\n", v);
1763 if (!x2apic_enabled()) {
1764 v = apic_read(APIC_DFR);
1765 printk(KERN_DEBUG "... APIC DFR: %08x\n", v);
1766 }
1767 v = apic_read(APIC_SPIV);
1768 printk(KERN_DEBUG "... APIC SPIV: %08x\n", v);
1769
1770 printk(KERN_DEBUG "... APIC ISR field:\n");
1771 print_APIC_field(APIC_ISR);
1772 printk(KERN_DEBUG "... APIC TMR field:\n");
1773 print_APIC_field(APIC_TMR);
1774 printk(KERN_DEBUG "... APIC IRR field:\n");
1775 print_APIC_field(APIC_IRR);
1776
1777 if (APIC_INTEGRATED(ver)) { /* !82489DX */
1778 if (maxlvt > 3) /* Due to the Pentium erratum 3AP. */
1779 apic_write(APIC_ESR, 0);
1780
1781 v = apic_read(APIC_ESR);
1782 printk(KERN_DEBUG "... APIC ESR: %08x\n", v);
1783 }
1784
1785 icr = apic_icr_read();
1786 printk(KERN_DEBUG "... APIC ICR: %08x\n", (u32)icr);
1787 printk(KERN_DEBUG "... APIC ICR2: %08x\n", (u32)(icr >> 32));
1788
1789 v = apic_read(APIC_LVTT);
1790 printk(KERN_DEBUG "... APIC LVTT: %08x\n", v);
1791
1792 if (maxlvt > 3) { /* PC is LVT#4. */
1793 v = apic_read(APIC_LVTPC);
1794 printk(KERN_DEBUG "... APIC LVTPC: %08x\n", v);
1795 }
1796 v = apic_read(APIC_LVT0);
1797 printk(KERN_DEBUG "... APIC LVT0: %08x\n", v);
1798 v = apic_read(APIC_LVT1);
1799 printk(KERN_DEBUG "... APIC LVT1: %08x\n", v);
1800
1801 if (maxlvt > 2) { /* ERR is LVT#3. */
1802 v = apic_read(APIC_LVTERR);
1803 printk(KERN_DEBUG "... APIC LVTERR: %08x\n", v);
1804 }
1805
1806 v = apic_read(APIC_TMICT);
1807 printk(KERN_DEBUG "... APIC TMICT: %08x\n", v);
1808 v = apic_read(APIC_TMCCT);
1809 printk(KERN_DEBUG "... APIC TMCCT: %08x\n", v);
1810 v = apic_read(APIC_TDCR);
1811 printk(KERN_DEBUG "... APIC TDCR: %08x\n", v);
1812
1813 if (boot_cpu_has(X86_FEATURE_EXTAPIC)) {
1814 v = apic_read(APIC_EFEAT);
1815 maxlvt = (v >> 16) & 0xff;
1816 printk(KERN_DEBUG "... APIC EFEAT: %08x\n", v);
1817 v = apic_read(APIC_ECTRL);
1818 printk(KERN_DEBUG "... APIC ECTRL: %08x\n", v);
1819 for (i = 0; i < maxlvt; i++) {
1820 v = apic_read(APIC_EILVTn(i));
1821 printk(KERN_DEBUG "... APIC EILVT%d: %08x\n", i, v);
1822 }
1823 }
1824 printk("\n");
1825 }
1826
1827 __apicdebuginit(void) print_all_local_APICs(void)
1828 {
1829 int cpu;
1830
1831 preempt_disable();
1832 for_each_online_cpu(cpu)
1833 smp_call_function_single(cpu, print_local_APIC, NULL, 1);
1834 preempt_enable();
1835 }
1836
1837 __apicdebuginit(void) print_PIC(void)
1838 {
1839 unsigned int v;
1840 unsigned long flags;
1841
1842 if (apic_verbosity == APIC_QUIET || !nr_legacy_irqs)
1843 return;
1844
1845 printk(KERN_DEBUG "\nprinting PIC contents\n");
1846
1847 spin_lock_irqsave(&i8259A_lock, flags);
1848
1849 v = inb(0xa1) << 8 | inb(0x21);
1850 printk(KERN_DEBUG "... PIC IMR: %04x\n", v);
1851
1852 v = inb(0xa0) << 8 | inb(0x20);
1853 printk(KERN_DEBUG "... PIC IRR: %04x\n", v);
1854
1855 outb(0x0b,0xa0);
1856 outb(0x0b,0x20);
1857 v = inb(0xa0) << 8 | inb(0x20);
1858 outb(0x0a,0xa0);
1859 outb(0x0a,0x20);
1860
1861 spin_unlock_irqrestore(&i8259A_lock, flags);
1862
1863 printk(KERN_DEBUG "... PIC ISR: %04x\n", v);
1864
1865 v = inb(0x4d1) << 8 | inb(0x4d0);
1866 printk(KERN_DEBUG "... PIC ELCR: %04x\n", v);
1867 }
1868
1869 __apicdebuginit(int) print_all_ICs(void)
1870 {
1871 print_PIC();
1872
1873 /* don't print out if apic is not there */
1874 if (!cpu_has_apic && !apic_from_smp_config())
1875 return 0;
1876
1877 print_all_local_APICs();
1878 print_IO_APIC();
1879
1880 return 0;
1881 }
1882
1883 fs_initcall(print_all_ICs);
1884
1885
1886 /* Where if anywhere is the i8259 connect in external int mode */
1887 static struct { int pin, apic; } ioapic_i8259 = { -1, -1 };
1888
1889 void __init enable_IO_APIC(void)
1890 {
1891 union IO_APIC_reg_01 reg_01;
1892 int i8259_apic, i8259_pin;
1893 int apic;
1894 unsigned long flags;
1895
1896 /*
1897 * The number of IO-APIC IRQ registers (== #pins):
1898 */
1899 for (apic = 0; apic < nr_ioapics; apic++) {
1900 spin_lock_irqsave(&ioapic_lock, flags);
1901 reg_01.raw = io_apic_read(apic, 1);
1902 spin_unlock_irqrestore(&ioapic_lock, flags);
1903 nr_ioapic_registers[apic] = reg_01.bits.entries+1;
1904 }
1905
1906 if (!nr_legacy_irqs)
1907 return;
1908
1909 for(apic = 0; apic < nr_ioapics; apic++) {
1910 int pin;
1911 /* See if any of the pins is in ExtINT mode */
1912 for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
1913 struct IO_APIC_route_entry entry;
1914 entry = ioapic_read_entry(apic, pin);
1915
1916 /* If the interrupt line is enabled and in ExtInt mode
1917 * I have found the pin where the i8259 is connected.
1918 */
1919 if ((entry.mask == 0) && (entry.delivery_mode == dest_ExtINT)) {
1920 ioapic_i8259.apic = apic;
1921 ioapic_i8259.pin = pin;
1922 goto found_i8259;
1923 }
1924 }
1925 }
1926 found_i8259:
1927 /* Look to see what if the MP table has reported the ExtINT */
1928 /* If we could not find the appropriate pin by looking at the ioapic
1929 * the i8259 probably is not connected the ioapic but give the
1930 * mptable a chance anyway.
1931 */
1932 i8259_pin = find_isa_irq_pin(0, mp_ExtINT);
1933 i8259_apic = find_isa_irq_apic(0, mp_ExtINT);
1934 /* Trust the MP table if nothing is setup in the hardware */
1935 if ((ioapic_i8259.pin == -1) && (i8259_pin >= 0)) {
1936 printk(KERN_WARNING "ExtINT not setup in hardware but reported by MP table\n");
1937 ioapic_i8259.pin = i8259_pin;
1938 ioapic_i8259.apic = i8259_apic;
1939 }
1940 /* Complain if the MP table and the hardware disagree */
1941 if (((ioapic_i8259.apic != i8259_apic) || (ioapic_i8259.pin != i8259_pin)) &&
1942 (i8259_pin >= 0) && (ioapic_i8259.pin >= 0))
1943 {
1944 printk(KERN_WARNING "ExtINT in hardware and MP table differ\n");
1945 }
1946
1947 /*
1948 * Do not trust the IO-APIC being empty at bootup
1949 */
1950 clear_IO_APIC();
1951 }
1952
1953 /*
1954 * Not an __init, needed by the reboot code
1955 */
1956 void disable_IO_APIC(void)
1957 {
1958 /*
1959 * Clear the IO-APIC before rebooting:
1960 */
1961 clear_IO_APIC();
1962
1963 if (!nr_legacy_irqs)
1964 return;
1965
1966 /*
1967 * If the i8259 is routed through an IOAPIC
1968 * Put that IOAPIC in virtual wire mode
1969 * so legacy interrupts can be delivered.
1970 *
1971 * With interrupt-remapping, for now we will use virtual wire A mode,
1972 * as virtual wire B is little complex (need to configure both
1973 * IOAPIC RTE aswell as interrupt-remapping table entry).
1974 * As this gets called during crash dump, keep this simple for now.
1975 */
1976 if (ioapic_i8259.pin != -1 && !intr_remapping_enabled) {
1977 struct IO_APIC_route_entry entry;
1978
1979 memset(&entry, 0, sizeof(entry));
1980 entry.mask = 0; /* Enabled */
1981 entry.trigger = 0; /* Edge */
1982 entry.irr = 0;
1983 entry.polarity = 0; /* High */
1984 entry.delivery_status = 0;
1985 entry.dest_mode = 0; /* Physical */
1986 entry.delivery_mode = dest_ExtINT; /* ExtInt */
1987 entry.vector = 0;
1988 entry.dest = read_apic_id();
1989
1990 /*
1991 * Add it to the IO-APIC irq-routing table:
1992 */
1993 ioapic_write_entry(ioapic_i8259.apic, ioapic_i8259.pin, entry);
1994 }
1995
1996 /*
1997 * Use virtual wire A mode when interrupt remapping is enabled.
1998 */
1999 if (cpu_has_apic || apic_from_smp_config())
2000 disconnect_bsp_APIC(!intr_remapping_enabled &&
2001 ioapic_i8259.pin != -1);
2002 }
2003
2004 #ifdef CONFIG_X86_32
2005 /*
2006 * function to set the IO-APIC physical IDs based on the
2007 * values stored in the MPC table.
2008 *
2009 * by Matt Domsch <Matt_Domsch@dell.com> Tue Dec 21 12:25:05 CST 1999
2010 */
2011
2012 void __init setup_ioapic_ids_from_mpc(void)
2013 {
2014 union IO_APIC_reg_00 reg_00;
2015 physid_mask_t phys_id_present_map;
2016 int apic_id;
2017 int i;
2018 unsigned char old_id;
2019 unsigned long flags;
2020
2021 if (acpi_ioapic)
2022 return;
2023 /*
2024 * Don't check I/O APIC IDs for xAPIC systems. They have
2025 * no meaning without the serial APIC bus.
2026 */
2027 if (!(boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
2028 || APIC_XAPIC(apic_version[boot_cpu_physical_apicid]))
2029 return;
2030 /*
2031 * This is broken; anything with a real cpu count has to
2032 * circumvent this idiocy regardless.
2033 */
2034 phys_id_present_map = apic->ioapic_phys_id_map(phys_cpu_present_map);
2035
2036 /*
2037 * Set the IOAPIC ID to the value stored in the MPC table.
2038 */
2039 for (apic_id = 0; apic_id < nr_ioapics; apic_id++) {
2040
2041 /* Read the register 0 value */
2042 spin_lock_irqsave(&ioapic_lock, flags);
2043 reg_00.raw = io_apic_read(apic_id, 0);
2044 spin_unlock_irqrestore(&ioapic_lock, flags);
2045
2046 old_id = mp_ioapics[apic_id].apicid;
2047
2048 if (mp_ioapics[apic_id].apicid >= get_physical_broadcast()) {
2049 printk(KERN_ERR "BIOS bug, IO-APIC#%d ID is %d in the MPC table!...\n",
2050 apic_id, mp_ioapics[apic_id].apicid);
2051 printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n",
2052 reg_00.bits.ID);
2053 mp_ioapics[apic_id].apicid = reg_00.bits.ID;
2054 }
2055
2056 /*
2057 * Sanity check, is the ID really free? Every APIC in a
2058 * system must have a unique ID or we get lots of nice
2059 * 'stuck on smp_invalidate_needed IPI wait' messages.
2060 */
2061 if (apic->check_apicid_used(phys_id_present_map,
2062 mp_ioapics[apic_id].apicid)) {
2063 printk(KERN_ERR "BIOS bug, IO-APIC#%d ID %d is already used!...\n",
2064 apic_id, mp_ioapics[apic_id].apicid);
2065 for (i = 0; i < get_physical_broadcast(); i++)
2066 if (!physid_isset(i, phys_id_present_map))
2067 break;
2068 if (i >= get_physical_broadcast())
2069 panic("Max APIC ID exceeded!\n");
2070 printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n",
2071 i);
2072 physid_set(i, phys_id_present_map);
2073 mp_ioapics[apic_id].apicid = i;
2074 } else {
2075 physid_mask_t tmp;
2076 tmp = apic->apicid_to_cpu_present(mp_ioapics[apic_id].apicid);
2077 apic_printk(APIC_VERBOSE, "Setting %d in the "
2078 "phys_id_present_map\n",
2079 mp_ioapics[apic_id].apicid);
2080 physids_or(phys_id_present_map, phys_id_present_map, tmp);
2081 }
2082
2083
2084 /*
2085 * We need to adjust the IRQ routing table
2086 * if the ID changed.
2087 */
2088 if (old_id != mp_ioapics[apic_id].apicid)
2089 for (i = 0; i < mp_irq_entries; i++)
2090 if (mp_irqs[i].dstapic == old_id)
2091 mp_irqs[i].dstapic
2092 = mp_ioapics[apic_id].apicid;
2093
2094 /*
2095 * Read the right value from the MPC table and
2096 * write it into the ID register.
2097 */
2098 apic_printk(APIC_VERBOSE, KERN_INFO
2099 "...changing IO-APIC physical APIC ID to %d ...",
2100 mp_ioapics[apic_id].apicid);
2101
2102 reg_00.bits.ID = mp_ioapics[apic_id].apicid;
2103 spin_lock_irqsave(&ioapic_lock, flags);
2104 io_apic_write(apic_id, 0, reg_00.raw);
2105 spin_unlock_irqrestore(&ioapic_lock, flags);
2106
2107 /*
2108 * Sanity check
2109 */
2110 spin_lock_irqsave(&ioapic_lock, flags);
2111 reg_00.raw = io_apic_read(apic_id, 0);
2112 spin_unlock_irqrestore(&ioapic_lock, flags);
2113 if (reg_00.bits.ID != mp_ioapics[apic_id].apicid)
2114 printk("could not set ID!\n");
2115 else
2116 apic_printk(APIC_VERBOSE, " ok.\n");
2117 }
2118 }
2119 #endif
2120
2121 int no_timer_check __initdata;
2122
2123 static int __init notimercheck(char *s)
2124 {
2125 no_timer_check = 1;
2126 return 1;
2127 }
2128 __setup("no_timer_check", notimercheck);
2129
2130 /*
2131 * There is a nasty bug in some older SMP boards, their mptable lies
2132 * about the timer IRQ. We do the following to work around the situation:
2133 *
2134 * - timer IRQ defaults to IO-APIC IRQ
2135 * - if this function detects that timer IRQs are defunct, then we fall
2136 * back to ISA timer IRQs
2137 */
2138 static int __init timer_irq_works(void)
2139 {
2140 unsigned long t1 = jiffies;
2141 unsigned long flags;
2142
2143 if (no_timer_check)
2144 return 1;
2145
2146 local_save_flags(flags);
2147 local_irq_enable();
2148 /* Let ten ticks pass... */
2149 mdelay((10 * 1000) / HZ);
2150 local_irq_restore(flags);
2151
2152 /*
2153 * Expect a few ticks at least, to be sure some possible
2154 * glue logic does not lock up after one or two first
2155 * ticks in a non-ExtINT mode. Also the local APIC
2156 * might have cached one ExtINT interrupt. Finally, at
2157 * least one tick may be lost due to delays.
2158 */
2159
2160 /* jiffies wrap? */
2161 if (time_after(jiffies, t1 + 4))
2162 return 1;
2163 return 0;
2164 }
2165
2166 /*
2167 * In the SMP+IOAPIC case it might happen that there are an unspecified
2168 * number of pending IRQ events unhandled. These cases are very rare,
2169 * so we 'resend' these IRQs via IPIs, to the same CPU. It's much
2170 * better to do it this way as thus we do not have to be aware of
2171 * 'pending' interrupts in the IRQ path, except at this point.
2172 */
2173 /*
2174 * Edge triggered needs to resend any interrupt
2175 * that was delayed but this is now handled in the device
2176 * independent code.
2177 */
2178
2179 /*
2180 * Starting up a edge-triggered IO-APIC interrupt is
2181 * nasty - we need to make sure that we get the edge.
2182 * If it is already asserted for some reason, we need
2183 * return 1 to indicate that is was pending.
2184 *
2185 * This is not complete - we should be able to fake
2186 * an edge even if it isn't on the 8259A...
2187 */
2188
2189 static unsigned int startup_ioapic_irq(unsigned int irq)
2190 {
2191 int was_pending = 0;
2192 unsigned long flags;
2193 struct irq_cfg *cfg;
2194
2195 spin_lock_irqsave(&ioapic_lock, flags);
2196 if (irq < nr_legacy_irqs) {
2197 disable_8259A_irq(irq);
2198 if (i8259A_irq_pending(irq))
2199 was_pending = 1;
2200 }
2201 cfg = irq_cfg(irq);
2202 __unmask_IO_APIC_irq(cfg);
2203 spin_unlock_irqrestore(&ioapic_lock, flags);
2204
2205 return was_pending;
2206 }
2207
2208 static int ioapic_retrigger_irq(unsigned int irq)
2209 {
2210
2211 struct irq_cfg *cfg = irq_cfg(irq);
2212 unsigned long flags;
2213
2214 spin_lock_irqsave(&vector_lock, flags);
2215 apic->send_IPI_mask(cpumask_of(cpumask_first(cfg->domain)), cfg->vector);
2216 spin_unlock_irqrestore(&vector_lock, flags);
2217
2218 return 1;
2219 }
2220
2221 /*
2222 * Level and edge triggered IO-APIC interrupts need different handling,
2223 * so we use two separate IRQ descriptors. Edge triggered IRQs can be
2224 * handled with the level-triggered descriptor, but that one has slightly
2225 * more overhead. Level-triggered interrupts cannot be handled with the
2226 * edge-triggered handler, without risking IRQ storms and other ugly
2227 * races.
2228 */
2229
2230 #ifdef CONFIG_SMP
2231 static void send_cleanup_vector(struct irq_cfg *cfg)
2232 {
2233 cpumask_var_t cleanup_mask;
2234
2235 if (unlikely(!alloc_cpumask_var(&cleanup_mask, GFP_ATOMIC))) {
2236 unsigned int i;
2237 cfg->move_cleanup_count = 0;
2238 for_each_cpu_and(i, cfg->old_domain, cpu_online_mask)
2239 cfg->move_cleanup_count++;
2240 for_each_cpu_and(i, cfg->old_domain, cpu_online_mask)
2241 apic->send_IPI_mask(cpumask_of(i), IRQ_MOVE_CLEANUP_VECTOR);
2242 } else {
2243 cpumask_and(cleanup_mask, cfg->old_domain, cpu_online_mask);
2244 cfg->move_cleanup_count = cpumask_weight(cleanup_mask);
2245 apic->send_IPI_mask(cleanup_mask, IRQ_MOVE_CLEANUP_VECTOR);
2246 free_cpumask_var(cleanup_mask);
2247 }
2248 cfg->move_in_progress = 0;
2249 }
2250
2251 static void __target_IO_APIC_irq(unsigned int irq, unsigned int dest, struct irq_cfg *cfg)
2252 {
2253 int apic, pin;
2254 struct irq_pin_list *entry;
2255 u8 vector = cfg->vector;
2256
2257 for_each_irq_pin(entry, cfg->irq_2_pin) {
2258 unsigned int reg;
2259
2260 apic = entry->apic;
2261 pin = entry->pin;
2262 /*
2263 * With interrupt-remapping, destination information comes
2264 * from interrupt-remapping table entry.
2265 */
2266 if (!irq_remapped(irq))
2267 io_apic_write(apic, 0x11 + pin*2, dest);
2268 reg = io_apic_read(apic, 0x10 + pin*2);
2269 reg &= ~IO_APIC_REDIR_VECTOR_MASK;
2270 reg |= vector;
2271 io_apic_modify(apic, 0x10 + pin*2, reg);
2272 }
2273 }
2274
2275 static int
2276 assign_irq_vector(int irq, struct irq_cfg *cfg, const struct cpumask *mask);
2277
2278 /*
2279 * Either sets desc->affinity to a valid value, and returns
2280 * ->cpu_mask_to_apicid of that, or returns BAD_APICID and
2281 * leaves desc->affinity untouched.
2282 */
2283 static unsigned int
2284 set_desc_affinity(struct irq_desc *desc, const struct cpumask *mask)
2285 {
2286 struct irq_cfg *cfg;
2287 unsigned int irq;
2288
2289 if (!cpumask_intersects(mask, cpu_online_mask))
2290 return BAD_APICID;
2291
2292 irq = desc->irq;
2293 cfg = desc->chip_data;
2294 if (assign_irq_vector(irq, cfg, mask))
2295 return BAD_APICID;
2296
2297 cpumask_copy(desc->affinity, mask);
2298
2299 return apic->cpu_mask_to_apicid_and(desc->affinity, cfg->domain);
2300 }
2301
2302 static int
2303 set_ioapic_affinity_irq_desc(struct irq_desc *desc, const struct cpumask *mask)
2304 {
2305 struct irq_cfg *cfg;
2306 unsigned long flags;
2307 unsigned int dest;
2308 unsigned int irq;
2309 int ret = -1;
2310
2311 irq = desc->irq;
2312 cfg = desc->chip_data;
2313
2314 spin_lock_irqsave(&ioapic_lock, flags);
2315 dest = set_desc_affinity(desc, mask);
2316 if (dest != BAD_APICID) {
2317 /* Only the high 8 bits are valid. */
2318 dest = SET_APIC_LOGICAL_ID(dest);
2319 __target_IO_APIC_irq(irq, dest, cfg);
2320 ret = 0;
2321 }
2322 spin_unlock_irqrestore(&ioapic_lock, flags);
2323
2324 return ret;
2325 }
2326
2327 static int
2328 set_ioapic_affinity_irq(unsigned int irq, const struct cpumask *mask)
2329 {
2330 struct irq_desc *desc;
2331
2332 desc = irq_to_desc(irq);
2333
2334 return set_ioapic_affinity_irq_desc(desc, mask);
2335 }
2336
2337 #ifdef CONFIG_INTR_REMAP
2338
2339 /*
2340 * Migrate the IO-APIC irq in the presence of intr-remapping.
2341 *
2342 * For both level and edge triggered, irq migration is a simple atomic
2343 * update(of vector and cpu destination) of IRTE and flush the hardware cache.
2344 *
2345 * For level triggered, we eliminate the io-apic RTE modification (with the
2346 * updated vector information), by using a virtual vector (io-apic pin number).
2347 * Real vector that is used for interrupting cpu will be coming from
2348 * the interrupt-remapping table entry.
2349 */
2350 static int
2351 migrate_ioapic_irq_desc(struct irq_desc *desc, const struct cpumask *mask)
2352 {
2353 struct irq_cfg *cfg;
2354 struct irte irte;
2355 unsigned int dest;
2356 unsigned int irq;
2357 int ret = -1;
2358
2359 if (!cpumask_intersects(mask, cpu_online_mask))
2360 return ret;
2361
2362 irq = desc->irq;
2363 if (get_irte(irq, &irte))
2364 return ret;
2365
2366 cfg = desc->chip_data;
2367 if (assign_irq_vector(irq, cfg, mask))
2368 return ret;
2369
2370 dest = apic->cpu_mask_to_apicid_and(cfg->domain, mask);
2371
2372 irte.vector = cfg->vector;
2373 irte.dest_id = IRTE_DEST(dest);
2374
2375 /*
2376 * Modified the IRTE and flushes the Interrupt entry cache.
2377 */
2378 modify_irte(irq, &irte);
2379
2380 if (cfg->move_in_progress)
2381 send_cleanup_vector(cfg);
2382
2383 cpumask_copy(desc->affinity, mask);
2384
2385 return 0;
2386 }
2387
2388 /*
2389 * Migrates the IRQ destination in the process context.
2390 */
2391 static int set_ir_ioapic_affinity_irq_desc(struct irq_desc *desc,
2392 const struct cpumask *mask)
2393 {
2394 return migrate_ioapic_irq_desc(desc, mask);
2395 }
2396 static int set_ir_ioapic_affinity_irq(unsigned int irq,
2397 const struct cpumask *mask)
2398 {
2399 struct irq_desc *desc = irq_to_desc(irq);
2400
2401 return set_ir_ioapic_affinity_irq_desc(desc, mask);
2402 }
2403 #else
2404 static inline int set_ir_ioapic_affinity_irq_desc(struct irq_desc *desc,
2405 const struct cpumask *mask)
2406 {
2407 return 0;
2408 }
2409 #endif
2410
2411 asmlinkage void smp_irq_move_cleanup_interrupt(void)
2412 {
2413 unsigned vector, me;
2414
2415 ack_APIC_irq();
2416 exit_idle();
2417 irq_enter();
2418
2419 me = smp_processor_id();
2420 for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) {
2421 unsigned int irq;
2422 unsigned int irr;
2423 struct irq_desc *desc;
2424 struct irq_cfg *cfg;
2425 irq = __get_cpu_var(vector_irq)[vector];
2426
2427 if (irq == -1)
2428 continue;
2429
2430 desc = irq_to_desc(irq);
2431 if (!desc)
2432 continue;
2433
2434 cfg = irq_cfg(irq);
2435 spin_lock(&desc->lock);
2436 if (!cfg->move_cleanup_count)
2437 goto unlock;
2438
2439 if (vector == cfg->vector && cpumask_test_cpu(me, cfg->domain))
2440 goto unlock;
2441
2442 irr = apic_read(APIC_IRR + (vector / 32 * 0x10));
2443 /*
2444 * Check if the vector that needs to be cleanedup is
2445 * registered at the cpu's IRR. If so, then this is not
2446 * the best time to clean it up. Lets clean it up in the
2447 * next attempt by sending another IRQ_MOVE_CLEANUP_VECTOR
2448 * to myself.
2449 */
2450 if (irr & (1 << (vector % 32))) {
2451 apic->send_IPI_self(IRQ_MOVE_CLEANUP_VECTOR);
2452 goto unlock;
2453 }
2454 __get_cpu_var(vector_irq)[vector] = -1;
2455 cfg->move_cleanup_count--;
2456 unlock:
2457 spin_unlock(&desc->lock);
2458 }
2459
2460 irq_exit();
2461 }
2462
2463 static void irq_complete_move(struct irq_desc **descp)
2464 {
2465 struct irq_desc *desc = *descp;
2466 struct irq_cfg *cfg = desc->chip_data;
2467 unsigned vector, me;
2468
2469 if (likely(!cfg->move_in_progress))
2470 return;
2471
2472 vector = ~get_irq_regs()->orig_ax;
2473 me = smp_processor_id();
2474
2475 if (vector == cfg->vector && cpumask_test_cpu(me, cfg->domain))
2476 send_cleanup_vector(cfg);
2477 }
2478 #else
2479 static inline void irq_complete_move(struct irq_desc **descp) {}
2480 #endif
2481
2482 static void ack_apic_edge(unsigned int irq)
2483 {
2484 struct irq_desc *desc = irq_to_desc(irq);
2485
2486 irq_complete_move(&desc);
2487 move_native_irq(irq);
2488 ack_APIC_irq();
2489 }
2490
2491 atomic_t irq_mis_count;
2492
2493 static void ack_apic_level(unsigned int irq)
2494 {
2495 struct irq_desc *desc = irq_to_desc(irq);
2496 unsigned long v;
2497 int i;
2498 struct irq_cfg *cfg;
2499 int do_unmask_irq = 0;
2500
2501 irq_complete_move(&desc);
2502 #ifdef CONFIG_GENERIC_PENDING_IRQ
2503 /* If we are moving the irq we need to mask it */
2504 if (unlikely(desc->status & IRQ_MOVE_PENDING)) {
2505 do_unmask_irq = 1;
2506 mask_IO_APIC_irq_desc(desc);
2507 }
2508 #endif
2509
2510 /*
2511 * It appears there is an erratum which affects at least version 0x11
2512 * of I/O APIC (that's the 82093AA and cores integrated into various
2513 * chipsets). Under certain conditions a level-triggered interrupt is
2514 * erroneously delivered as edge-triggered one but the respective IRR
2515 * bit gets set nevertheless. As a result the I/O unit expects an EOI
2516 * message but it will never arrive and further interrupts are blocked
2517 * from the source. The exact reason is so far unknown, but the
2518 * phenomenon was observed when two consecutive interrupt requests
2519 * from a given source get delivered to the same CPU and the source is
2520 * temporarily disabled in between.
2521 *
2522 * A workaround is to simulate an EOI message manually. We achieve it
2523 * by setting the trigger mode to edge and then to level when the edge
2524 * trigger mode gets detected in the TMR of a local APIC for a
2525 * level-triggered interrupt. We mask the source for the time of the
2526 * operation to prevent an edge-triggered interrupt escaping meanwhile.
2527 * The idea is from Manfred Spraul. --macro
2528 */
2529 cfg = desc->chip_data;
2530 i = cfg->vector;
2531 v = apic_read(APIC_TMR + ((i & ~0x1f) >> 1));
2532
2533 /*
2534 * We must acknowledge the irq before we move it or the acknowledge will
2535 * not propagate properly.
2536 */
2537 ack_APIC_irq();
2538
2539 /* Now we can move and renable the irq */
2540 if (unlikely(do_unmask_irq)) {
2541 /* Only migrate the irq if the ack has been received.
2542 *
2543 * On rare occasions the broadcast level triggered ack gets
2544 * delayed going to ioapics, and if we reprogram the
2545 * vector while Remote IRR is still set the irq will never
2546 * fire again.
2547 *
2548 * To prevent this scenario we read the Remote IRR bit
2549 * of the ioapic. This has two effects.
2550 * - On any sane system the read of the ioapic will
2551 * flush writes (and acks) going to the ioapic from
2552 * this cpu.
2553 * - We get to see if the ACK has actually been delivered.
2554 *
2555 * Based on failed experiments of reprogramming the
2556 * ioapic entry from outside of irq context starting
2557 * with masking the ioapic entry and then polling until
2558 * Remote IRR was clear before reprogramming the
2559 * ioapic I don't trust the Remote IRR bit to be
2560 * completey accurate.
2561 *
2562 * However there appears to be no other way to plug
2563 * this race, so if the Remote IRR bit is not
2564 * accurate and is causing problems then it is a hardware bug
2565 * and you can go talk to the chipset vendor about it.
2566 */
2567 cfg = desc->chip_data;
2568 if (!io_apic_level_ack_pending(cfg))
2569 move_masked_irq(irq);
2570 unmask_IO_APIC_irq_desc(desc);
2571 }
2572
2573 /* Tail end of version 0x11 I/O APIC bug workaround */
2574 if (!(v & (1 << (i & 0x1f)))) {
2575 atomic_inc(&irq_mis_count);
2576 spin_lock(&ioapic_lock);
2577 __mask_and_edge_IO_APIC_irq(cfg);
2578 __unmask_and_level_IO_APIC_irq(cfg);
2579 spin_unlock(&ioapic_lock);
2580 }
2581 }
2582
2583 #ifdef CONFIG_INTR_REMAP
2584 static void __eoi_ioapic_irq(unsigned int irq, struct irq_cfg *cfg)
2585 {
2586 struct irq_pin_list *entry;
2587
2588 for_each_irq_pin(entry, cfg->irq_2_pin)
2589 io_apic_eoi(entry->apic, entry->pin);
2590 }
2591
2592 static void
2593 eoi_ioapic_irq(struct irq_desc *desc)
2594 {
2595 struct irq_cfg *cfg;
2596 unsigned long flags;
2597 unsigned int irq;
2598
2599 irq = desc->irq;
2600 cfg = desc->chip_data;
2601
2602 spin_lock_irqsave(&ioapic_lock, flags);
2603 __eoi_ioapic_irq(irq, cfg);
2604 spin_unlock_irqrestore(&ioapic_lock, flags);
2605 }
2606
2607 static void ir_ack_apic_edge(unsigned int irq)
2608 {
2609 ack_APIC_irq();
2610 }
2611
2612 static void ir_ack_apic_level(unsigned int irq)
2613 {
2614 struct irq_desc *desc = irq_to_desc(irq);
2615
2616 ack_APIC_irq();
2617 eoi_ioapic_irq(desc);
2618 }
2619 #endif /* CONFIG_INTR_REMAP */
2620
2621 static struct irq_chip ioapic_chip __read_mostly = {
2622 .name = "IO-APIC",
2623 .startup = startup_ioapic_irq,
2624 .mask = mask_IO_APIC_irq,
2625 .unmask = unmask_IO_APIC_irq,
2626 .ack = ack_apic_edge,
2627 .eoi = ack_apic_level,
2628 #ifdef CONFIG_SMP
2629 .set_affinity = set_ioapic_affinity_irq,
2630 #endif
2631 .retrigger = ioapic_retrigger_irq,
2632 };
2633
2634 static struct irq_chip ir_ioapic_chip __read_mostly = {
2635 .name = "IR-IO-APIC",
2636 .startup = startup_ioapic_irq,
2637 .mask = mask_IO_APIC_irq,
2638 .unmask = unmask_IO_APIC_irq,
2639 #ifdef CONFIG_INTR_REMAP
2640 .ack = ir_ack_apic_edge,
2641 .eoi = ir_ack_apic_level,
2642 #ifdef CONFIG_SMP
2643 .set_affinity = set_ir_ioapic_affinity_irq,
2644 #endif
2645 #endif
2646 .retrigger = ioapic_retrigger_irq,
2647 };
2648
2649 static inline void init_IO_APIC_traps(void)
2650 {
2651 int irq;
2652 struct irq_desc *desc;
2653 struct irq_cfg *cfg;
2654
2655 /*
2656 * NOTE! The local APIC isn't very good at handling
2657 * multiple interrupts at the same interrupt level.
2658 * As the interrupt level is determined by taking the
2659 * vector number and shifting that right by 4, we
2660 * want to spread these out a bit so that they don't
2661 * all fall in the same interrupt level.
2662 *
2663 * Also, we've got to be careful not to trash gate
2664 * 0x80, because int 0x80 is hm, kind of importantish. ;)
2665 */
2666 for_each_irq_desc(irq, desc) {
2667 cfg = desc->chip_data;
2668 if (IO_APIC_IRQ(irq) && cfg && !cfg->vector) {
2669 /*
2670 * Hmm.. We don't have an entry for this,
2671 * so default to an old-fashioned 8259
2672 * interrupt if we can..
2673 */
2674 if (irq < nr_legacy_irqs)
2675 make_8259A_irq(irq);
2676 else
2677 /* Strange. Oh, well.. */
2678 desc->chip = &no_irq_chip;
2679 }
2680 }
2681 }
2682
2683 /*
2684 * The local APIC irq-chip implementation:
2685 */
2686
2687 static void mask_lapic_irq(unsigned int irq)
2688 {
2689 unsigned long v;
2690
2691 v = apic_read(APIC_LVT0);
2692 apic_write(APIC_LVT0, v | APIC_LVT_MASKED);
2693 }
2694
2695 static void unmask_lapic_irq(unsigned int irq)
2696 {
2697 unsigned long v;
2698
2699 v = apic_read(APIC_LVT0);
2700 apic_write(APIC_LVT0, v & ~APIC_LVT_MASKED);
2701 }
2702
2703 static void ack_lapic_irq(unsigned int irq)
2704 {
2705 ack_APIC_irq();
2706 }
2707
2708 static struct irq_chip lapic_chip __read_mostly = {
2709 .name = "local-APIC",
2710 .mask = mask_lapic_irq,
2711 .unmask = unmask_lapic_irq,
2712 .ack = ack_lapic_irq,
2713 };
2714
2715 static void lapic_register_intr(int irq, struct irq_desc *desc)
2716 {
2717 desc->status &= ~IRQ_LEVEL;
2718 set_irq_chip_and_handler_name(irq, &lapic_chip, handle_edge_irq,
2719 "edge");
2720 }
2721
2722 static void __init setup_nmi(void)
2723 {
2724 /*
2725 * Dirty trick to enable the NMI watchdog ...
2726 * We put the 8259A master into AEOI mode and
2727 * unmask on all local APICs LVT0 as NMI.
2728 *
2729 * The idea to use the 8259A in AEOI mode ('8259A Virtual Wire')
2730 * is from Maciej W. Rozycki - so we do not have to EOI from
2731 * the NMI handler or the timer interrupt.
2732 */
2733 apic_printk(APIC_VERBOSE, KERN_INFO "activating NMI Watchdog ...");
2734
2735 enable_NMI_through_LVT0();
2736
2737 apic_printk(APIC_VERBOSE, " done.\n");
2738 }
2739
2740 /*
2741 * This looks a bit hackish but it's about the only one way of sending
2742 * a few INTA cycles to 8259As and any associated glue logic. ICR does
2743 * not support the ExtINT mode, unfortunately. We need to send these
2744 * cycles as some i82489DX-based boards have glue logic that keeps the
2745 * 8259A interrupt line asserted until INTA. --macro
2746 */
2747 static inline void __init unlock_ExtINT_logic(void)
2748 {
2749 int apic, pin, i;
2750 struct IO_APIC_route_entry entry0, entry1;
2751 unsigned char save_control, save_freq_select;
2752
2753 pin = find_isa_irq_pin(8, mp_INT);
2754 if (pin == -1) {
2755 WARN_ON_ONCE(1);
2756 return;
2757 }
2758 apic = find_isa_irq_apic(8, mp_INT);
2759 if (apic == -1) {
2760 WARN_ON_ONCE(1);
2761 return;
2762 }
2763
2764 entry0 = ioapic_read_entry(apic, pin);
2765 clear_IO_APIC_pin(apic, pin);
2766
2767 memset(&entry1, 0, sizeof(entry1));
2768
2769 entry1.dest_mode = 0; /* physical delivery */
2770 entry1.mask = 0; /* unmask IRQ now */
2771 entry1.dest = hard_smp_processor_id();
2772 entry1.delivery_mode = dest_ExtINT;
2773 entry1.polarity = entry0.polarity;
2774 entry1.trigger = 0;
2775 entry1.vector = 0;
2776
2777 ioapic_write_entry(apic, pin, entry1);
2778
2779 save_control = CMOS_READ(RTC_CONTROL);
2780 save_freq_select = CMOS_READ(RTC_FREQ_SELECT);
2781 CMOS_WRITE((save_freq_select & ~RTC_RATE_SELECT) | 0x6,
2782 RTC_FREQ_SELECT);
2783 CMOS_WRITE(save_control | RTC_PIE, RTC_CONTROL);
2784
2785 i = 100;
2786 while (i-- > 0) {
2787 mdelay(10);
2788 if ((CMOS_READ(RTC_INTR_FLAGS) & RTC_PF) == RTC_PF)
2789 i -= 10;
2790 }
2791
2792 CMOS_WRITE(save_control, RTC_CONTROL);
2793 CMOS_WRITE(save_freq_select, RTC_FREQ_SELECT);
2794 clear_IO_APIC_pin(apic, pin);
2795
2796 ioapic_write_entry(apic, pin, entry0);
2797 }
2798
2799 static int disable_timer_pin_1 __initdata;
2800 /* Actually the next is obsolete, but keep it for paranoid reasons -AK */
2801 static int __init disable_timer_pin_setup(char *arg)
2802 {
2803 disable_timer_pin_1 = 1;
2804 return 0;
2805 }
2806 early_param("disable_timer_pin_1", disable_timer_pin_setup);
2807
2808 int timer_through_8259 __initdata;
2809
2810 /*
2811 * This code may look a bit paranoid, but it's supposed to cooperate with
2812 * a wide range of boards and BIOS bugs. Fortunately only the timer IRQ
2813 * is so screwy. Thanks to Brian Perkins for testing/hacking this beast
2814 * fanatically on his truly buggy board.
2815 *
2816 * FIXME: really need to revamp this for all platforms.
2817 */
2818 static inline void __init check_timer(void)
2819 {
2820 struct irq_desc *desc = irq_to_desc(0);
2821 struct irq_cfg *cfg = desc->chip_data;
2822 int node = cpu_to_node(boot_cpu_id);
2823 int apic1, pin1, apic2, pin2;
2824 unsigned long flags;
2825 int no_pin1 = 0;
2826
2827 local_irq_save(flags);
2828
2829 /*
2830 * get/set the timer IRQ vector:
2831 */
2832 disable_8259A_irq(0);
2833 assign_irq_vector(0, cfg, apic->target_cpus());
2834
2835 /*
2836 * As IRQ0 is to be enabled in the 8259A, the virtual
2837 * wire has to be disabled in the local APIC. Also
2838 * timer interrupts need to be acknowledged manually in
2839 * the 8259A for the i82489DX when using the NMI
2840 * watchdog as that APIC treats NMIs as level-triggered.
2841 * The AEOI mode will finish them in the 8259A
2842 * automatically.
2843 */
2844 apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_EXTINT);
2845 init_8259A(1);
2846 #ifdef CONFIG_X86_32
2847 {
2848 unsigned int ver;
2849
2850 ver = apic_read(APIC_LVR);
2851 ver = GET_APIC_VERSION(ver);
2852 timer_ack = (nmi_watchdog == NMI_IO_APIC && !APIC_INTEGRATED(ver));
2853 }
2854 #endif
2855
2856 pin1 = find_isa_irq_pin(0, mp_INT);
2857 apic1 = find_isa_irq_apic(0, mp_INT);
2858 pin2 = ioapic_i8259.pin;
2859 apic2 = ioapic_i8259.apic;
2860
2861 apic_printk(APIC_QUIET, KERN_INFO "..TIMER: vector=0x%02X "
2862 "apic1=%d pin1=%d apic2=%d pin2=%d\n",
2863 cfg->vector, apic1, pin1, apic2, pin2);
2864
2865 /*
2866 * Some BIOS writers are clueless and report the ExtINTA
2867 * I/O APIC input from the cascaded 8259A as the timer
2868 * interrupt input. So just in case, if only one pin
2869 * was found above, try it both directly and through the
2870 * 8259A.
2871 */
2872 if (pin1 == -1) {
2873 if (intr_remapping_enabled)
2874 panic("BIOS bug: timer not connected to IO-APIC");
2875 pin1 = pin2;
2876 apic1 = apic2;
2877 no_pin1 = 1;
2878 } else if (pin2 == -1) {
2879 pin2 = pin1;
2880 apic2 = apic1;
2881 }
2882
2883 if (pin1 != -1) {
2884 /*
2885 * Ok, does IRQ0 through the IOAPIC work?
2886 */
2887 if (no_pin1) {
2888 add_pin_to_irq_node(cfg, node, apic1, pin1);
2889 setup_timer_IRQ0_pin(apic1, pin1, cfg->vector);
2890 } else {
2891 /* for edge trigger, setup_IO_APIC_irq already
2892 * leave it unmasked.
2893 * so only need to unmask if it is level-trigger
2894 * do we really have level trigger timer?
2895 */
2896 int idx;
2897 idx = find_irq_entry(apic1, pin1, mp_INT);
2898 if (idx != -1 && irq_trigger(idx))
2899 unmask_IO_APIC_irq_desc(desc);
2900 }
2901 if (timer_irq_works()) {
2902 if (nmi_watchdog == NMI_IO_APIC) {
2903 setup_nmi();
2904 enable_8259A_irq(0);
2905 }
2906 if (disable_timer_pin_1 > 0)
2907 clear_IO_APIC_pin(0, pin1);
2908 goto out;
2909 }
2910 if (intr_remapping_enabled)
2911 panic("timer doesn't work through Interrupt-remapped IO-APIC");
2912 local_irq_disable();
2913 clear_IO_APIC_pin(apic1, pin1);
2914 if (!no_pin1)
2915 apic_printk(APIC_QUIET, KERN_ERR "..MP-BIOS bug: "
2916 "8254 timer not connected to IO-APIC\n");
2917
2918 apic_printk(APIC_QUIET, KERN_INFO "...trying to set up timer "
2919 "(IRQ0) through the 8259A ...\n");
2920 apic_printk(APIC_QUIET, KERN_INFO
2921 "..... (found apic %d pin %d) ...\n", apic2, pin2);
2922 /*
2923 * legacy devices should be connected to IO APIC #0
2924 */
2925 replace_pin_at_irq_node(cfg, node, apic1, pin1, apic2, pin2);
2926 setup_timer_IRQ0_pin(apic2, pin2, cfg->vector);
2927 enable_8259A_irq(0);
2928 if (timer_irq_works()) {
2929 apic_printk(APIC_QUIET, KERN_INFO "....... works.\n");
2930 timer_through_8259 = 1;
2931 if (nmi_watchdog == NMI_IO_APIC) {
2932 disable_8259A_irq(0);
2933 setup_nmi();
2934 enable_8259A_irq(0);
2935 }
2936 goto out;
2937 }
2938 /*
2939 * Cleanup, just in case ...
2940 */
2941 local_irq_disable();
2942 disable_8259A_irq(0);
2943 clear_IO_APIC_pin(apic2, pin2);
2944 apic_printk(APIC_QUIET, KERN_INFO "....... failed.\n");
2945 }
2946
2947 if (nmi_watchdog == NMI_IO_APIC) {
2948 apic_printk(APIC_QUIET, KERN_WARNING "timer doesn't work "
2949 "through the IO-APIC - disabling NMI Watchdog!\n");
2950 nmi_watchdog = NMI_NONE;
2951 }
2952 #ifdef CONFIG_X86_32
2953 timer_ack = 0;
2954 #endif
2955
2956 apic_printk(APIC_QUIET, KERN_INFO
2957 "...trying to set up timer as Virtual Wire IRQ...\n");
2958
2959 lapic_register_intr(0, desc);
2960 apic_write(APIC_LVT0, APIC_DM_FIXED | cfg->vector); /* Fixed mode */
2961 enable_8259A_irq(0);
2962
2963 if (timer_irq_works()) {
2964 apic_printk(APIC_QUIET, KERN_INFO "..... works.\n");
2965 goto out;
2966 }
2967 local_irq_disable();
2968 disable_8259A_irq(0);
2969 apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_FIXED | cfg->vector);
2970 apic_printk(APIC_QUIET, KERN_INFO "..... failed.\n");
2971
2972 apic_printk(APIC_QUIET, KERN_INFO
2973 "...trying to set up timer as ExtINT IRQ...\n");
2974
2975 init_8259A(0);
2976 make_8259A_irq(0);
2977 apic_write(APIC_LVT0, APIC_DM_EXTINT);
2978
2979 unlock_ExtINT_logic();
2980
2981 if (timer_irq_works()) {
2982 apic_printk(APIC_QUIET, KERN_INFO "..... works.\n");
2983 goto out;
2984 }
2985 local_irq_disable();
2986 apic_printk(APIC_QUIET, KERN_INFO "..... failed :(.\n");
2987 panic("IO-APIC + timer doesn't work! Boot with apic=debug and send a "
2988 "report. Then try booting with the 'noapic' option.\n");
2989 out:
2990 local_irq_restore(flags);
2991 }
2992
2993 /*
2994 * Traditionally ISA IRQ2 is the cascade IRQ, and is not available
2995 * to devices. However there may be an I/O APIC pin available for
2996 * this interrupt regardless. The pin may be left unconnected, but
2997 * typically it will be reused as an ExtINT cascade interrupt for
2998 * the master 8259A. In the MPS case such a pin will normally be
2999 * reported as an ExtINT interrupt in the MP table. With ACPI
3000 * there is no provision for ExtINT interrupts, and in the absence
3001 * of an override it would be treated as an ordinary ISA I/O APIC
3002 * interrupt, that is edge-triggered and unmasked by default. We
3003 * used to do this, but it caused problems on some systems because
3004 * of the NMI watchdog and sometimes IRQ0 of the 8254 timer using
3005 * the same ExtINT cascade interrupt to drive the local APIC of the
3006 * bootstrap processor. Therefore we refrain from routing IRQ2 to
3007 * the I/O APIC in all cases now. No actual device should request
3008 * it anyway. --macro
3009 */
3010 #define PIC_IRQS (1UL << PIC_CASCADE_IR)
3011
3012 void __init setup_IO_APIC(void)
3013 {
3014
3015 /*
3016 * calling enable_IO_APIC() is moved to setup_local_APIC for BP
3017 */
3018 io_apic_irqs = nr_legacy_irqs ? ~PIC_IRQS : ~0UL;
3019
3020 apic_printk(APIC_VERBOSE, "ENABLING IO-APIC IRQs\n");
3021 /*
3022 * Set up IO-APIC IRQ routing.
3023 */
3024 x86_init.mpparse.setup_ioapic_ids();
3025
3026 sync_Arb_IDs();
3027 setup_IO_APIC_irqs();
3028 init_IO_APIC_traps();
3029 if (nr_legacy_irqs)
3030 check_timer();
3031 }
3032
3033 /*
3034 * Called after all the initialization is done. If we didnt find any
3035 * APIC bugs then we can allow the modify fast path
3036 */
3037
3038 static int __init io_apic_bug_finalize(void)
3039 {
3040 if (sis_apic_bug == -1)
3041 sis_apic_bug = 0;
3042 return 0;
3043 }
3044
3045 late_initcall(io_apic_bug_finalize);
3046
3047 struct sysfs_ioapic_data {
3048 struct sys_device dev;
3049 struct IO_APIC_route_entry entry[0];
3050 };
3051 static struct sysfs_ioapic_data * mp_ioapic_data[MAX_IO_APICS];
3052
3053 static int ioapic_suspend(struct sys_device *dev, pm_message_t state)
3054 {
3055 struct IO_APIC_route_entry *entry;
3056 struct sysfs_ioapic_data *data;
3057 int i;
3058
3059 data = container_of(dev, struct sysfs_ioapic_data, dev);
3060 entry = data->entry;
3061 for (i = 0; i < nr_ioapic_registers[dev->id]; i ++, entry ++ )
3062 *entry = ioapic_read_entry(dev->id, i);
3063
3064 return 0;
3065 }
3066
3067 static int ioapic_resume(struct sys_device *dev)
3068 {
3069 struct IO_APIC_route_entry *entry;
3070 struct sysfs_ioapic_data *data;
3071 unsigned long flags;
3072 union IO_APIC_reg_00 reg_00;
3073 int i;
3074
3075 data = container_of(dev, struct sysfs_ioapic_data, dev);
3076 entry = data->entry;
3077
3078 spin_lock_irqsave(&ioapic_lock, flags);
3079 reg_00.raw = io_apic_read(dev->id, 0);
3080 if (reg_00.bits.ID != mp_ioapics[dev->id].apicid) {
3081 reg_00.bits.ID = mp_ioapics[dev->id].apicid;
3082 io_apic_write(dev->id, 0, reg_00.raw);
3083 }
3084 spin_unlock_irqrestore(&ioapic_lock, flags);
3085 for (i = 0; i < nr_ioapic_registers[dev->id]; i++)
3086 ioapic_write_entry(dev->id, i, entry[i]);
3087
3088 return 0;
3089 }
3090
3091 static struct sysdev_class ioapic_sysdev_class = {
3092 .name = "ioapic",
3093 .suspend = ioapic_suspend,
3094 .resume = ioapic_resume,
3095 };
3096
3097 static int __init ioapic_init_sysfs(void)
3098 {
3099 struct sys_device * dev;
3100 int i, size, error;
3101
3102 error = sysdev_class_register(&ioapic_sysdev_class);
3103 if (error)
3104 return error;
3105
3106 for (i = 0; i < nr_ioapics; i++ ) {
3107 size = sizeof(struct sys_device) + nr_ioapic_registers[i]
3108 * sizeof(struct IO_APIC_route_entry);
3109 mp_ioapic_data[i] = kzalloc(size, GFP_KERNEL);
3110 if (!mp_ioapic_data[i]) {
3111 printk(KERN_ERR "Can't suspend/resume IOAPIC %d\n", i);
3112 continue;
3113 }
3114 dev = &mp_ioapic_data[i]->dev;
3115 dev->id = i;
3116 dev->cls = &ioapic_sysdev_class;
3117 error = sysdev_register(dev);
3118 if (error) {
3119 kfree(mp_ioapic_data[i]);
3120 mp_ioapic_data[i] = NULL;
3121 printk(KERN_ERR "Can't suspend/resume IOAPIC %d\n", i);
3122 continue;
3123 }
3124 }
3125
3126 return 0;
3127 }
3128
3129 device_initcall(ioapic_init_sysfs);
3130
3131 /*
3132 * Dynamic irq allocate and deallocation
3133 */
3134 unsigned int create_irq_nr(unsigned int irq_want, int node)
3135 {
3136 /* Allocate an unused irq */
3137 unsigned int irq;
3138 unsigned int new;
3139 unsigned long flags;
3140 struct irq_cfg *cfg_new = NULL;
3141 struct irq_desc *desc_new = NULL;
3142
3143 irq = 0;
3144 if (irq_want < nr_irqs_gsi)
3145 irq_want = nr_irqs_gsi;
3146
3147 spin_lock_irqsave(&vector_lock, flags);
3148 for (new = irq_want; new < nr_irqs; new++) {
3149 desc_new = irq_to_desc_alloc_node(new, node);
3150 if (!desc_new) {
3151 printk(KERN_INFO "can not get irq_desc for %d\n", new);
3152 continue;
3153 }
3154 cfg_new = desc_new->chip_data;
3155
3156 if (cfg_new->vector != 0)
3157 continue;
3158
3159 desc_new = move_irq_desc(desc_new, node);
3160
3161 if (__assign_irq_vector(new, cfg_new, apic->target_cpus()) == 0)
3162 irq = new;
3163 break;
3164 }
3165 spin_unlock_irqrestore(&vector_lock, flags);
3166
3167 if (irq > 0) {
3168 dynamic_irq_init(irq);
3169 /* restore it, in case dynamic_irq_init clear it */
3170 if (desc_new)
3171 desc_new->chip_data = cfg_new;
3172 }
3173 return irq;
3174 }
3175
3176 int create_irq(void)
3177 {
3178 int node = cpu_to_node(boot_cpu_id);
3179 unsigned int irq_want;
3180 int irq;
3181
3182 irq_want = nr_irqs_gsi;
3183 irq = create_irq_nr(irq_want, node);
3184
3185 if (irq == 0)
3186 irq = -1;
3187
3188 return irq;
3189 }
3190
3191 void destroy_irq(unsigned int irq)
3192 {
3193 unsigned long flags;
3194 struct irq_cfg *cfg;
3195 struct irq_desc *desc;
3196
3197 /* store it, in case dynamic_irq_cleanup clear it */
3198 desc = irq_to_desc(irq);
3199 cfg = desc->chip_data;
3200 dynamic_irq_cleanup(irq);
3201 /* connect back irq_cfg */
3202 desc->chip_data = cfg;
3203
3204 free_irte(irq);
3205 spin_lock_irqsave(&vector_lock, flags);
3206 __clear_irq_vector(irq, cfg);
3207 spin_unlock_irqrestore(&vector_lock, flags);
3208 }
3209
3210 /*
3211 * MSI message composition
3212 */
3213 #ifdef CONFIG_PCI_MSI
3214 static int msi_compose_msg(struct pci_dev *pdev, unsigned int irq, struct msi_msg *msg)
3215 {
3216 struct irq_cfg *cfg;
3217 int err;
3218 unsigned dest;
3219
3220 if (disable_apic)
3221 return -ENXIO;
3222
3223 cfg = irq_cfg(irq);
3224 err = assign_irq_vector(irq, cfg, apic->target_cpus());
3225 if (err)
3226 return err;
3227
3228 dest = apic->cpu_mask_to_apicid_and(cfg->domain, apic->target_cpus());
3229
3230 if (irq_remapped(irq)) {
3231 struct irte irte;
3232 int ir_index;
3233 u16 sub_handle;
3234
3235 ir_index = map_irq_to_irte_handle(irq, &sub_handle);
3236 BUG_ON(ir_index == -1);
3237
3238 memset (&irte, 0, sizeof(irte));
3239
3240 irte.present = 1;
3241 irte.dst_mode = apic->irq_dest_mode;
3242 irte.trigger_mode = 0; /* edge */
3243 irte.dlvry_mode = apic->irq_delivery_mode;
3244 irte.vector = cfg->vector;
3245 irte.dest_id = IRTE_DEST(dest);
3246
3247 /* Set source-id of interrupt request */
3248 set_msi_sid(&irte, pdev);
3249
3250 modify_irte(irq, &irte);
3251
3252 msg->address_hi = MSI_ADDR_BASE_HI;
3253 msg->data = sub_handle;
3254 msg->address_lo = MSI_ADDR_BASE_LO | MSI_ADDR_IR_EXT_INT |
3255 MSI_ADDR_IR_SHV |
3256 MSI_ADDR_IR_INDEX1(ir_index) |
3257 MSI_ADDR_IR_INDEX2(ir_index);
3258 } else {
3259 if (x2apic_enabled())
3260 msg->address_hi = MSI_ADDR_BASE_HI |
3261 MSI_ADDR_EXT_DEST_ID(dest);
3262 else
3263 msg->address_hi = MSI_ADDR_BASE_HI;
3264
3265 msg->address_lo =
3266 MSI_ADDR_BASE_LO |
3267 ((apic->irq_dest_mode == 0) ?
3268 MSI_ADDR_DEST_MODE_PHYSICAL:
3269 MSI_ADDR_DEST_MODE_LOGICAL) |
3270 ((apic->irq_delivery_mode != dest_LowestPrio) ?
3271 MSI_ADDR_REDIRECTION_CPU:
3272 MSI_ADDR_REDIRECTION_LOWPRI) |
3273 MSI_ADDR_DEST_ID(dest);
3274
3275 msg->data =
3276 MSI_DATA_TRIGGER_EDGE |
3277 MSI_DATA_LEVEL_ASSERT |
3278 ((apic->irq_delivery_mode != dest_LowestPrio) ?
3279 MSI_DATA_DELIVERY_FIXED:
3280 MSI_DATA_DELIVERY_LOWPRI) |
3281 MSI_DATA_VECTOR(cfg->vector);
3282 }
3283 return err;
3284 }
3285
3286 #ifdef CONFIG_SMP
3287 static int set_msi_irq_affinity(unsigned int irq, const struct cpumask *mask)
3288 {
3289 struct irq_desc *desc = irq_to_desc(irq);
3290 struct irq_cfg *cfg;
3291 struct msi_msg msg;
3292 unsigned int dest;
3293
3294 dest = set_desc_affinity(desc, mask);
3295 if (dest == BAD_APICID)
3296 return -1;
3297
3298 cfg = desc->chip_data;
3299
3300 read_msi_msg_desc(desc, &msg);
3301
3302 msg.data &= ~MSI_DATA_VECTOR_MASK;
3303 msg.data |= MSI_DATA_VECTOR(cfg->vector);
3304 msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
3305 msg.address_lo |= MSI_ADDR_DEST_ID(dest);
3306
3307 write_msi_msg_desc(desc, &msg);
3308
3309 return 0;
3310 }
3311 #ifdef CONFIG_INTR_REMAP
3312 /*
3313 * Migrate the MSI irq to another cpumask. This migration is
3314 * done in the process context using interrupt-remapping hardware.
3315 */
3316 static int
3317 ir_set_msi_irq_affinity(unsigned int irq, const struct cpumask *mask)
3318 {
3319 struct irq_desc *desc = irq_to_desc(irq);
3320 struct irq_cfg *cfg = desc->chip_data;
3321 unsigned int dest;
3322 struct irte irte;
3323
3324 if (get_irte(irq, &irte))
3325 return -1;
3326
3327 dest = set_desc_affinity(desc, mask);
3328 if (dest == BAD_APICID)
3329 return -1;
3330
3331 irte.vector = cfg->vector;
3332 irte.dest_id = IRTE_DEST(dest);
3333
3334 /*
3335 * atomically update the IRTE with the new destination and vector.
3336 */
3337 modify_irte(irq, &irte);
3338
3339 /*
3340 * After this point, all the interrupts will start arriving
3341 * at the new destination. So, time to cleanup the previous
3342 * vector allocation.
3343 */
3344 if (cfg->move_in_progress)
3345 send_cleanup_vector(cfg);
3346
3347 return 0;
3348 }
3349
3350 #endif
3351 #endif /* CONFIG_SMP */
3352
3353 /*
3354 * IRQ Chip for MSI PCI/PCI-X/PCI-Express Devices,
3355 * which implement the MSI or MSI-X Capability Structure.
3356 */
3357 static struct irq_chip msi_chip = {
3358 .name = "PCI-MSI",
3359 .unmask = unmask_msi_irq,
3360 .mask = mask_msi_irq,
3361 .ack = ack_apic_edge,
3362 #ifdef CONFIG_SMP
3363 .set_affinity = set_msi_irq_affinity,
3364 #endif
3365 .retrigger = ioapic_retrigger_irq,
3366 };
3367
3368 static struct irq_chip msi_ir_chip = {
3369 .name = "IR-PCI-MSI",
3370 .unmask = unmask_msi_irq,
3371 .mask = mask_msi_irq,
3372 #ifdef CONFIG_INTR_REMAP
3373 .ack = ir_ack_apic_edge,
3374 #ifdef CONFIG_SMP
3375 .set_affinity = ir_set_msi_irq_affinity,
3376 #endif
3377 #endif
3378 .retrigger = ioapic_retrigger_irq,
3379 };
3380
3381 /*
3382 * Map the PCI dev to the corresponding remapping hardware unit
3383 * and allocate 'nvec' consecutive interrupt-remapping table entries
3384 * in it.
3385 */
3386 static int msi_alloc_irte(struct pci_dev *dev, int irq, int nvec)
3387 {
3388 struct intel_iommu *iommu;
3389 int index;
3390
3391 iommu = map_dev_to_ir(dev);
3392 if (!iommu) {
3393 printk(KERN_ERR
3394 "Unable to map PCI %s to iommu\n", pci_name(dev));
3395 return -ENOENT;
3396 }
3397
3398 index = alloc_irte(iommu, irq, nvec);
3399 if (index < 0) {
3400 printk(KERN_ERR
3401 "Unable to allocate %d IRTE for PCI %s\n", nvec,
3402 pci_name(dev));
3403 return -ENOSPC;
3404 }
3405 return index;
3406 }
3407
3408 static int setup_msi_irq(struct pci_dev *dev, struct msi_desc *msidesc, int irq)
3409 {
3410 int ret;
3411 struct msi_msg msg;
3412
3413 ret = msi_compose_msg(dev, irq, &msg);
3414 if (ret < 0)
3415 return ret;
3416
3417 set_irq_msi(irq, msidesc);
3418 write_msi_msg(irq, &msg);
3419
3420 if (irq_remapped(irq)) {
3421 struct irq_desc *desc = irq_to_desc(irq);
3422 /*
3423 * irq migration in process context
3424 */
3425 desc->status |= IRQ_MOVE_PCNTXT;
3426 set_irq_chip_and_handler_name(irq, &msi_ir_chip, handle_edge_irq, "edge");
3427 } else
3428 set_irq_chip_and_handler_name(irq, &msi_chip, handle_edge_irq, "edge");
3429
3430 dev_printk(KERN_DEBUG, &dev->dev, "irq %d for MSI/MSI-X\n", irq);
3431
3432 return 0;
3433 }
3434
3435 int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
3436 {
3437 unsigned int irq;
3438 int ret, sub_handle;
3439 struct msi_desc *msidesc;
3440 unsigned int irq_want;
3441 struct intel_iommu *iommu = NULL;
3442 int index = 0;
3443 int node;
3444
3445 /* x86 doesn't support multiple MSI yet */
3446 if (type == PCI_CAP_ID_MSI && nvec > 1)
3447 return 1;
3448
3449 node = dev_to_node(&dev->dev);
3450 irq_want = nr_irqs_gsi;
3451 sub_handle = 0;
3452 list_for_each_entry(msidesc, &dev->msi_list, list) {
3453 irq = create_irq_nr(irq_want, node);
3454 if (irq == 0)
3455 return -1;
3456 irq_want = irq + 1;
3457 if (!intr_remapping_enabled)
3458 goto no_ir;
3459
3460 if (!sub_handle) {
3461 /*
3462 * allocate the consecutive block of IRTE's
3463 * for 'nvec'
3464 */
3465 index = msi_alloc_irte(dev, irq, nvec);
3466 if (index < 0) {
3467 ret = index;
3468 goto error;
3469 }
3470 } else {
3471 iommu = map_dev_to_ir(dev);
3472 if (!iommu) {
3473 ret = -ENOENT;
3474 goto error;
3475 }
3476 /*
3477 * setup the mapping between the irq and the IRTE
3478 * base index, the sub_handle pointing to the
3479 * appropriate interrupt remap table entry.
3480 */
3481 set_irte_irq(irq, iommu, index, sub_handle);
3482 }
3483 no_ir:
3484 ret = setup_msi_irq(dev, msidesc, irq);
3485 if (ret < 0)
3486 goto error;
3487 sub_handle++;
3488 }
3489 return 0;
3490
3491 error:
3492 destroy_irq(irq);
3493 return ret;
3494 }
3495
3496 void arch_teardown_msi_irq(unsigned int irq)
3497 {
3498 destroy_irq(irq);
3499 }
3500
3501 #if defined (CONFIG_DMAR) || defined (CONFIG_INTR_REMAP)
3502 #ifdef CONFIG_SMP
3503 static int dmar_msi_set_affinity(unsigned int irq, const struct cpumask *mask)
3504 {
3505 struct irq_desc *desc = irq_to_desc(irq);
3506 struct irq_cfg *cfg;
3507 struct msi_msg msg;
3508 unsigned int dest;
3509
3510 dest = set_desc_affinity(desc, mask);
3511 if (dest == BAD_APICID)
3512 return -1;
3513
3514 cfg = desc->chip_data;
3515
3516 dmar_msi_read(irq, &msg);
3517
3518 msg.data &= ~MSI_DATA_VECTOR_MASK;
3519 msg.data |= MSI_DATA_VECTOR(cfg->vector);
3520 msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
3521 msg.address_lo |= MSI_ADDR_DEST_ID(dest);
3522
3523 dmar_msi_write(irq, &msg);
3524
3525 return 0;
3526 }
3527
3528 #endif /* CONFIG_SMP */
3529
3530 static struct irq_chip dmar_msi_type = {
3531 .name = "DMAR_MSI",
3532 .unmask = dmar_msi_unmask,
3533 .mask = dmar_msi_mask,
3534 .ack = ack_apic_edge,
3535 #ifdef CONFIG_SMP
3536 .set_affinity = dmar_msi_set_affinity,
3537 #endif
3538 .retrigger = ioapic_retrigger_irq,
3539 };
3540
3541 int arch_setup_dmar_msi(unsigned int irq)
3542 {
3543 int ret;
3544 struct msi_msg msg;
3545
3546 ret = msi_compose_msg(NULL, irq, &msg);
3547 if (ret < 0)
3548 return ret;
3549 dmar_msi_write(irq, &msg);
3550 set_irq_chip_and_handler_name(irq, &dmar_msi_type, handle_edge_irq,
3551 "edge");
3552 return 0;
3553 }
3554 #endif
3555
3556 #ifdef CONFIG_HPET_TIMER
3557
3558 #ifdef CONFIG_SMP
3559 static int hpet_msi_set_affinity(unsigned int irq, const struct cpumask *mask)
3560 {
3561 struct irq_desc *desc = irq_to_desc(irq);
3562 struct irq_cfg *cfg;
3563 struct msi_msg msg;
3564 unsigned int dest;
3565
3566 dest = set_desc_affinity(desc, mask);
3567 if (dest == BAD_APICID)
3568 return -1;
3569
3570 cfg = desc->chip_data;
3571
3572 hpet_msi_read(irq, &msg);
3573
3574 msg.data &= ~MSI_DATA_VECTOR_MASK;
3575 msg.data |= MSI_DATA_VECTOR(cfg->vector);
3576 msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
3577 msg.address_lo |= MSI_ADDR_DEST_ID(dest);
3578
3579 hpet_msi_write(irq, &msg);
3580
3581 return 0;
3582 }
3583
3584 #endif /* CONFIG_SMP */
3585
3586 static struct irq_chip hpet_msi_type = {
3587 .name = "HPET_MSI",
3588 .unmask = hpet_msi_unmask,
3589 .mask = hpet_msi_mask,
3590 .ack = ack_apic_edge,
3591 #ifdef CONFIG_SMP
3592 .set_affinity = hpet_msi_set_affinity,
3593 #endif
3594 .retrigger = ioapic_retrigger_irq,
3595 };
3596
3597 int arch_setup_hpet_msi(unsigned int irq)
3598 {
3599 int ret;
3600 struct msi_msg msg;
3601 struct irq_desc *desc = irq_to_desc(irq);
3602
3603 ret = msi_compose_msg(NULL, irq, &msg);
3604 if (ret < 0)
3605 return ret;
3606
3607 hpet_msi_write(irq, &msg);
3608 desc->status |= IRQ_MOVE_PCNTXT;
3609 set_irq_chip_and_handler_name(irq, &hpet_msi_type, handle_edge_irq,
3610 "edge");
3611
3612 return 0;
3613 }
3614 #endif
3615
3616 #endif /* CONFIG_PCI_MSI */
3617 /*
3618 * Hypertransport interrupt support
3619 */
3620 #ifdef CONFIG_HT_IRQ
3621
3622 #ifdef CONFIG_SMP
3623
3624 static void target_ht_irq(unsigned int irq, unsigned int dest, u8 vector)
3625 {
3626 struct ht_irq_msg msg;
3627 fetch_ht_irq_msg(irq, &msg);
3628
3629 msg.address_lo &= ~(HT_IRQ_LOW_VECTOR_MASK | HT_IRQ_LOW_DEST_ID_MASK);
3630 msg.address_hi &= ~(HT_IRQ_HIGH_DEST_ID_MASK);
3631
3632 msg.address_lo |= HT_IRQ_LOW_VECTOR(vector) | HT_IRQ_LOW_DEST_ID(dest);
3633 msg.address_hi |= HT_IRQ_HIGH_DEST_ID(dest);
3634
3635 write_ht_irq_msg(irq, &msg);
3636 }
3637
3638 static int set_ht_irq_affinity(unsigned int irq, const struct cpumask *mask)
3639 {
3640 struct irq_desc *desc = irq_to_desc(irq);
3641 struct irq_cfg *cfg;
3642 unsigned int dest;
3643
3644 dest = set_desc_affinity(desc, mask);
3645 if (dest == BAD_APICID)
3646 return -1;
3647
3648 cfg = desc->chip_data;
3649
3650 target_ht_irq(irq, dest, cfg->vector);
3651
3652 return 0;
3653 }
3654
3655 #endif
3656
3657 static struct irq_chip ht_irq_chip = {
3658 .name = "PCI-HT",
3659 .mask = mask_ht_irq,
3660 .unmask = unmask_ht_irq,
3661 .ack = ack_apic_edge,
3662 #ifdef CONFIG_SMP
3663 .set_affinity = set_ht_irq_affinity,
3664 #endif
3665 .retrigger = ioapic_retrigger_irq,
3666 };
3667
3668 int arch_setup_ht_irq(unsigned int irq, struct pci_dev *dev)
3669 {
3670 struct irq_cfg *cfg;
3671 int err;
3672
3673 if (disable_apic)
3674 return -ENXIO;
3675
3676 cfg = irq_cfg(irq);
3677 err = assign_irq_vector(irq, cfg, apic->target_cpus());
3678 if (!err) {
3679 struct ht_irq_msg msg;
3680 unsigned dest;
3681
3682 dest = apic->cpu_mask_to_apicid_and(cfg->domain,
3683 apic->target_cpus());
3684
3685 msg.address_hi = HT_IRQ_HIGH_DEST_ID(dest);
3686
3687 msg.address_lo =
3688 HT_IRQ_LOW_BASE |
3689 HT_IRQ_LOW_DEST_ID(dest) |
3690 HT_IRQ_LOW_VECTOR(cfg->vector) |
3691 ((apic->irq_dest_mode == 0) ?
3692 HT_IRQ_LOW_DM_PHYSICAL :
3693 HT_IRQ_LOW_DM_LOGICAL) |
3694 HT_IRQ_LOW_RQEOI_EDGE |
3695 ((apic->irq_delivery_mode != dest_LowestPrio) ?
3696 HT_IRQ_LOW_MT_FIXED :
3697 HT_IRQ_LOW_MT_ARBITRATED) |
3698 HT_IRQ_LOW_IRQ_MASKED;
3699
3700 write_ht_irq_msg(irq, &msg);
3701
3702 set_irq_chip_and_handler_name(irq, &ht_irq_chip,
3703 handle_edge_irq, "edge");
3704
3705 dev_printk(KERN_DEBUG, &dev->dev, "irq %d for HT\n", irq);
3706 }
3707 return err;
3708 }
3709 #endif /* CONFIG_HT_IRQ */
3710
3711 #ifdef CONFIG_X86_UV
3712 /*
3713 * Re-target the irq to the specified CPU and enable the specified MMR located
3714 * on the specified blade to allow the sending of MSIs to the specified CPU.
3715 */
3716 int arch_enable_uv_irq(char *irq_name, unsigned int irq, int cpu, int mmr_blade,
3717 unsigned long mmr_offset)
3718 {
3719 const struct cpumask *eligible_cpu = cpumask_of(cpu);
3720 struct irq_cfg *cfg;
3721 int mmr_pnode;
3722 unsigned long mmr_value;
3723 struct uv_IO_APIC_route_entry *entry;
3724 unsigned long flags;
3725 int err;
3726
3727 BUILD_BUG_ON(sizeof(struct uv_IO_APIC_route_entry) != sizeof(unsigned long));
3728
3729 cfg = irq_cfg(irq);
3730
3731 err = assign_irq_vector(irq, cfg, eligible_cpu);
3732 if (err != 0)
3733 return err;
3734
3735 spin_lock_irqsave(&vector_lock, flags);
3736 set_irq_chip_and_handler_name(irq, &uv_irq_chip, handle_percpu_irq,
3737 irq_name);
3738 spin_unlock_irqrestore(&vector_lock, flags);
3739
3740 mmr_value = 0;
3741 entry = (struct uv_IO_APIC_route_entry *)&mmr_value;
3742 entry->vector = cfg->vector;
3743 entry->delivery_mode = apic->irq_delivery_mode;
3744 entry->dest_mode = apic->irq_dest_mode;
3745 entry->polarity = 0;
3746 entry->trigger = 0;
3747 entry->mask = 0;
3748 entry->dest = apic->cpu_mask_to_apicid(eligible_cpu);
3749
3750 mmr_pnode = uv_blade_to_pnode(mmr_blade);
3751 uv_write_global_mmr64(mmr_pnode, mmr_offset, mmr_value);
3752
3753 if (cfg->move_in_progress)
3754 send_cleanup_vector(cfg);
3755
3756 return irq;
3757 }
3758
3759 /*
3760 * Disable the specified MMR located on the specified blade so that MSIs are
3761 * longer allowed to be sent.
3762 */
3763 void arch_disable_uv_irq(int mmr_blade, unsigned long mmr_offset)
3764 {
3765 unsigned long mmr_value;
3766 struct uv_IO_APIC_route_entry *entry;
3767 int mmr_pnode;
3768
3769 BUILD_BUG_ON(sizeof(struct uv_IO_APIC_route_entry) != sizeof(unsigned long));
3770
3771 mmr_value = 0;
3772 entry = (struct uv_IO_APIC_route_entry *)&mmr_value;
3773 entry->mask = 1;
3774
3775 mmr_pnode = uv_blade_to_pnode(mmr_blade);
3776 uv_write_global_mmr64(mmr_pnode, mmr_offset, mmr_value);
3777 }
3778 #endif /* CONFIG_X86_64 */
3779
3780 int __init io_apic_get_redir_entries (int ioapic)
3781 {
3782 union IO_APIC_reg_01 reg_01;
3783 unsigned long flags;
3784
3785 spin_lock_irqsave(&ioapic_lock, flags);
3786 reg_01.raw = io_apic_read(ioapic, 1);
3787 spin_unlock_irqrestore(&ioapic_lock, flags);
3788
3789 return reg_01.bits.entries;
3790 }
3791
3792 void __init probe_nr_irqs_gsi(void)
3793 {
3794 int nr = 0;
3795
3796 nr = acpi_probe_gsi();
3797 if (nr > nr_irqs_gsi) {
3798 nr_irqs_gsi = nr;
3799 } else {
3800 /* for acpi=off or acpi is not compiled in */
3801 int idx;
3802
3803 nr = 0;
3804 for (idx = 0; idx < nr_ioapics; idx++)
3805 nr += io_apic_get_redir_entries(idx) + 1;
3806
3807 if (nr > nr_irqs_gsi)
3808 nr_irqs_gsi = nr;
3809 }
3810
3811 printk(KERN_DEBUG "nr_irqs_gsi: %d\n", nr_irqs_gsi);
3812 }
3813
3814 #ifdef CONFIG_SPARSE_IRQ
3815 int __init arch_probe_nr_irqs(void)
3816 {
3817 int nr;
3818
3819 if (nr_irqs > (NR_VECTORS * nr_cpu_ids))
3820 nr_irqs = NR_VECTORS * nr_cpu_ids;
3821
3822 nr = nr_irqs_gsi + 8 * nr_cpu_ids;
3823 #if defined(CONFIG_PCI_MSI) || defined(CONFIG_HT_IRQ)
3824 /*
3825 * for MSI and HT dyn irq
3826 */
3827 nr += nr_irqs_gsi * 16;
3828 #endif
3829 if (nr < nr_irqs)
3830 nr_irqs = nr;
3831
3832 return 0;
3833 }
3834 #endif
3835
3836 static int __io_apic_set_pci_routing(struct device *dev, int irq,
3837 struct io_apic_irq_attr *irq_attr)
3838 {
3839 struct irq_desc *desc;
3840 struct irq_cfg *cfg;
3841 int node;
3842 int ioapic, pin;
3843 int trigger, polarity;
3844
3845 ioapic = irq_attr->ioapic;
3846 if (!IO_APIC_IRQ(irq)) {
3847 apic_printk(APIC_QUIET,KERN_ERR "IOAPIC[%d]: Invalid reference to IRQ 0\n",
3848 ioapic);
3849 return -EINVAL;
3850 }
3851
3852 if (dev)
3853 node = dev_to_node(dev);
3854 else
3855 node = cpu_to_node(boot_cpu_id);
3856
3857 desc = irq_to_desc_alloc_node(irq, node);
3858 if (!desc) {
3859 printk(KERN_INFO "can not get irq_desc %d\n", irq);
3860 return 0;
3861 }
3862
3863 pin = irq_attr->ioapic_pin;
3864 trigger = irq_attr->trigger;
3865 polarity = irq_attr->polarity;
3866
3867 /*
3868 * IRQs < 16 are already in the irq_2_pin[] map
3869 */
3870 if (irq >= nr_legacy_irqs) {
3871 cfg = desc->chip_data;
3872 if (add_pin_to_irq_node_nopanic(cfg, node, ioapic, pin)) {
3873 printk(KERN_INFO "can not add pin %d for irq %d\n",
3874 pin, irq);
3875 return 0;
3876 }
3877 }
3878
3879 setup_IO_APIC_irq(ioapic, pin, irq, desc, trigger, polarity);
3880
3881 return 0;
3882 }
3883
3884 int io_apic_set_pci_routing(struct device *dev, int irq,
3885 struct io_apic_irq_attr *irq_attr)
3886 {
3887 int ioapic, pin;
3888 /*
3889 * Avoid pin reprogramming. PRTs typically include entries
3890 * with redundant pin->gsi mappings (but unique PCI devices);
3891 * we only program the IOAPIC on the first.
3892 */
3893 ioapic = irq_attr->ioapic;
3894 pin = irq_attr->ioapic_pin;
3895 if (test_bit(pin, mp_ioapic_routing[ioapic].pin_programmed)) {
3896 pr_debug("Pin %d-%d already programmed\n",
3897 mp_ioapics[ioapic].apicid, pin);
3898 return 0;
3899 }
3900 set_bit(pin, mp_ioapic_routing[ioapic].pin_programmed);
3901
3902 return __io_apic_set_pci_routing(dev, irq, irq_attr);
3903 }
3904
3905 u8 __init io_apic_unique_id(u8 id)
3906 {
3907 #ifdef CONFIG_X86_32
3908 if ((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) &&
3909 !APIC_XAPIC(apic_version[boot_cpu_physical_apicid]))
3910 return io_apic_get_unique_id(nr_ioapics, id);
3911 else
3912 return id;
3913 #else
3914 int i;
3915 DECLARE_BITMAP(used, 256);
3916
3917 bitmap_zero(used, 256);
3918 for (i = 0; i < nr_ioapics; i++) {
3919 struct mpc_ioapic *ia = &mp_ioapics[i];
3920 __set_bit(ia->apicid, used);
3921 }
3922 if (!test_bit(id, used))
3923 return id;
3924 return find_first_zero_bit(used, 256);
3925 #endif
3926 }
3927
3928 #ifdef CONFIG_X86_32
3929 int __init io_apic_get_unique_id(int ioapic, int apic_id)
3930 {
3931 union IO_APIC_reg_00 reg_00;
3932 static physid_mask_t apic_id_map = PHYSID_MASK_NONE;
3933 physid_mask_t tmp;
3934 unsigned long flags;
3935 int i = 0;
3936
3937 /*
3938 * The P4 platform supports up to 256 APIC IDs on two separate APIC
3939 * buses (one for LAPICs, one for IOAPICs), where predecessors only
3940 * supports up to 16 on one shared APIC bus.
3941 *
3942 * TBD: Expand LAPIC/IOAPIC support on P4-class systems to take full
3943 * advantage of new APIC bus architecture.
3944 */
3945
3946 if (physids_empty(apic_id_map))
3947 apic_id_map = apic->ioapic_phys_id_map(phys_cpu_present_map);
3948
3949 spin_lock_irqsave(&ioapic_lock, flags);
3950 reg_00.raw = io_apic_read(ioapic, 0);
3951 spin_unlock_irqrestore(&ioapic_lock, flags);
3952
3953 if (apic_id >= get_physical_broadcast()) {
3954 printk(KERN_WARNING "IOAPIC[%d]: Invalid apic_id %d, trying "
3955 "%d\n", ioapic, apic_id, reg_00.bits.ID);
3956 apic_id = reg_00.bits.ID;
3957 }
3958
3959 /*
3960 * Every APIC in a system must have a unique ID or we get lots of nice
3961 * 'stuck on smp_invalidate_needed IPI wait' messages.
3962 */
3963 if (apic->check_apicid_used(apic_id_map, apic_id)) {
3964
3965 for (i = 0; i < get_physical_broadcast(); i++) {
3966 if (!apic->check_apicid_used(apic_id_map, i))
3967 break;
3968 }
3969
3970 if (i == get_physical_broadcast())
3971 panic("Max apic_id exceeded!\n");
3972
3973 printk(KERN_WARNING "IOAPIC[%d]: apic_id %d already used, "
3974 "trying %d\n", ioapic, apic_id, i);
3975
3976 apic_id = i;
3977 }
3978
3979 tmp = apic->apicid_to_cpu_present(apic_id);
3980 physids_or(apic_id_map, apic_id_map, tmp);
3981
3982 if (reg_00.bits.ID != apic_id) {
3983 reg_00.bits.ID = apic_id;
3984
3985 spin_lock_irqsave(&ioapic_lock, flags);
3986 io_apic_write(ioapic, 0, reg_00.raw);
3987 reg_00.raw = io_apic_read(ioapic, 0);
3988 spin_unlock_irqrestore(&ioapic_lock, flags);
3989
3990 /* Sanity check */
3991 if (reg_00.bits.ID != apic_id) {
3992 printk("IOAPIC[%d]: Unable to change apic_id!\n", ioapic);
3993 return -1;
3994 }
3995 }
3996
3997 apic_printk(APIC_VERBOSE, KERN_INFO
3998 "IOAPIC[%d]: Assigned apic_id %d\n", ioapic, apic_id);
3999
4000 return apic_id;
4001 }
4002 #endif
4003
4004 int __init io_apic_get_version(int ioapic)
4005 {
4006 union IO_APIC_reg_01 reg_01;
4007 unsigned long flags;
4008
4009 spin_lock_irqsave(&ioapic_lock, flags);
4010 reg_01.raw = io_apic_read(ioapic, 1);
4011 spin_unlock_irqrestore(&ioapic_lock, flags);
4012
4013 return reg_01.bits.version;
4014 }
4015
4016 int acpi_get_override_irq(int bus_irq, int *trigger, int *polarity)
4017 {
4018 int i;
4019
4020 if (skip_ioapic_setup)
4021 return -1;
4022
4023 for (i = 0; i < mp_irq_entries; i++)
4024 if (mp_irqs[i].irqtype == mp_INT &&
4025 mp_irqs[i].srcbusirq == bus_irq)
4026 break;
4027 if (i >= mp_irq_entries)
4028 return -1;
4029
4030 *trigger = irq_trigger(i);
4031 *polarity = irq_polarity(i);
4032 return 0;
4033 }
4034
4035 /*
4036 * This function currently is only a helper for the i386 smp boot process where
4037 * we need to reprogram the ioredtbls to cater for the cpus which have come online
4038 * so mask in all cases should simply be apic->target_cpus()
4039 */
4040 #ifdef CONFIG_SMP
4041 void __init setup_ioapic_dest(void)
4042 {
4043 int pin, ioapic = 0, irq, irq_entry;
4044 struct irq_desc *desc;
4045 const struct cpumask *mask;
4046
4047 if (skip_ioapic_setup == 1)
4048 return;
4049
4050 #ifdef CONFIG_ACPI
4051 if (!acpi_disabled && acpi_ioapic) {
4052 ioapic = mp_find_ioapic(0);
4053 if (ioapic < 0)
4054 ioapic = 0;
4055 }
4056 #endif
4057
4058 for (pin = 0; pin < nr_ioapic_registers[ioapic]; pin++) {
4059 irq_entry = find_irq_entry(ioapic, pin, mp_INT);
4060 if (irq_entry == -1)
4061 continue;
4062 irq = pin_2_irq(irq_entry, ioapic, pin);
4063
4064 desc = irq_to_desc(irq);
4065
4066 /*
4067 * Honour affinities which have been set in early boot
4068 */
4069 if (desc->status &
4070 (IRQ_NO_BALANCING | IRQ_AFFINITY_SET))
4071 mask = desc->affinity;
4072 else
4073 mask = apic->target_cpus();
4074
4075 if (intr_remapping_enabled)
4076 set_ir_ioapic_affinity_irq_desc(desc, mask);
4077 else
4078 set_ioapic_affinity_irq_desc(desc, mask);
4079 }
4080
4081 }
4082 #endif
4083
4084 #define IOAPIC_RESOURCE_NAME_SIZE 11
4085
4086 static struct resource *ioapic_resources;
4087
4088 static struct resource * __init ioapic_setup_resources(int nr_ioapics)
4089 {
4090 unsigned long n;
4091 struct resource *res;
4092 char *mem;
4093 int i;
4094
4095 if (nr_ioapics <= 0)
4096 return NULL;
4097
4098 n = IOAPIC_RESOURCE_NAME_SIZE + sizeof(struct resource);
4099 n *= nr_ioapics;
4100
4101 mem = alloc_bootmem(n);
4102 res = (void *)mem;
4103
4104 mem += sizeof(struct resource) * nr_ioapics;
4105
4106 for (i = 0; i < nr_ioapics; i++) {
4107 res[i].name = mem;
4108 res[i].flags = IORESOURCE_MEM | IORESOURCE_BUSY;
4109 sprintf(mem, "IOAPIC %u", i);
4110 mem += IOAPIC_RESOURCE_NAME_SIZE;
4111 }
4112
4113 ioapic_resources = res;
4114
4115 return res;
4116 }
4117
4118 void __init ioapic_init_mappings(void)
4119 {
4120 unsigned long ioapic_phys, idx = FIX_IO_APIC_BASE_0;
4121 struct resource *ioapic_res;
4122 int i;
4123
4124 ioapic_res = ioapic_setup_resources(nr_ioapics);
4125 for (i = 0; i < nr_ioapics; i++) {
4126 if (smp_found_config) {
4127 ioapic_phys = mp_ioapics[i].apicaddr;
4128 #ifdef CONFIG_X86_32
4129 if (!ioapic_phys) {
4130 printk(KERN_ERR
4131 "WARNING: bogus zero IO-APIC "
4132 "address found in MPTABLE, "
4133 "disabling IO/APIC support!\n");
4134 smp_found_config = 0;
4135 skip_ioapic_setup = 1;
4136 goto fake_ioapic_page;
4137 }
4138 #endif
4139 } else {
4140 #ifdef CONFIG_X86_32
4141 fake_ioapic_page:
4142 #endif
4143 ioapic_phys = (unsigned long)
4144 alloc_bootmem_pages(PAGE_SIZE);
4145 ioapic_phys = __pa(ioapic_phys);
4146 }
4147 set_fixmap_nocache(idx, ioapic_phys);
4148 apic_printk(APIC_VERBOSE,
4149 "mapped IOAPIC to %08lx (%08lx)\n",
4150 __fix_to_virt(idx), ioapic_phys);
4151 idx++;
4152
4153 ioapic_res->start = ioapic_phys;
4154 ioapic_res->end = ioapic_phys + (4 * 1024) - 1;
4155 ioapic_res++;
4156 }
4157 }
4158
4159 void __init ioapic_insert_resources(void)
4160 {
4161 int i;
4162 struct resource *r = ioapic_resources;
4163
4164 if (!r) {
4165 if (nr_ioapics > 0)
4166 printk(KERN_ERR
4167 "IO APIC resources couldn't be allocated.\n");
4168 return;
4169 }
4170
4171 for (i = 0; i < nr_ioapics; i++) {
4172 insert_resource(&iomem_resource, r);
4173 r++;
4174 }
4175 }
4176
4177 int mp_find_ioapic(int gsi)
4178 {
4179 int i = 0;
4180
4181 /* Find the IOAPIC that manages this GSI. */
4182 for (i = 0; i < nr_ioapics; i++) {
4183 if ((gsi >= mp_gsi_routing[i].gsi_base)
4184 && (gsi <= mp_gsi_routing[i].gsi_end))
4185 return i;
4186 }
4187
4188 printk(KERN_ERR "ERROR: Unable to locate IOAPIC for GSI %d\n", gsi);
4189 return -1;
4190 }
4191
4192 int mp_find_ioapic_pin(int ioapic, int gsi)
4193 {
4194 if (WARN_ON(ioapic == -1))
4195 return -1;
4196 if (WARN_ON(gsi > mp_gsi_routing[ioapic].gsi_end))
4197 return -1;
4198
4199 return gsi - mp_gsi_routing[ioapic].gsi_base;
4200 }
4201
4202 static int bad_ioapic(unsigned long address)
4203 {
4204 if (nr_ioapics >= MAX_IO_APICS) {
4205 printk(KERN_WARNING "WARING: Max # of I/O APICs (%d) exceeded "
4206 "(found %d), skipping\n", MAX_IO_APICS, nr_ioapics);
4207 return 1;
4208 }
4209 if (!address) {
4210 printk(KERN_WARNING "WARNING: Bogus (zero) I/O APIC address"
4211 " found in table, skipping!\n");
4212 return 1;
4213 }
4214 return 0;
4215 }
4216
4217 void __init mp_register_ioapic(int id, u32 address, u32 gsi_base)
4218 {
4219 int idx = 0;
4220
4221 if (bad_ioapic(address))
4222 return;
4223
4224 idx = nr_ioapics;
4225
4226 mp_ioapics[idx].type = MP_IOAPIC;
4227 mp_ioapics[idx].flags = MPC_APIC_USABLE;
4228 mp_ioapics[idx].apicaddr = address;
4229
4230 set_fixmap_nocache(FIX_IO_APIC_BASE_0 + idx, address);
4231 mp_ioapics[idx].apicid = io_apic_unique_id(id);
4232 mp_ioapics[idx].apicver = io_apic_get_version(idx);
4233
4234 /*
4235 * Build basic GSI lookup table to facilitate gsi->io_apic lookups
4236 * and to prevent reprogramming of IOAPIC pins (PCI GSIs).
4237 */
4238 mp_gsi_routing[idx].gsi_base = gsi_base;
4239 mp_gsi_routing[idx].gsi_end = gsi_base +
4240 io_apic_get_redir_entries(idx);
4241
4242 printk(KERN_INFO "IOAPIC[%d]: apic_id %d, version %d, address 0x%x, "
4243 "GSI %d-%d\n", idx, mp_ioapics[idx].apicid,
4244 mp_ioapics[idx].apicver, mp_ioapics[idx].apicaddr,
4245 mp_gsi_routing[idx].gsi_base, mp_gsi_routing[idx].gsi_end);
4246
4247 nr_ioapics++;
4248 }
Linux with added FPC-III support