3 * Purpose: PCI Message Signaled Interrupt (MSI)
5 * Copyright (C) 2003-2004 Intel
6 * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com)
7 * Copyright (C) 2016 Christoph Hellwig.
10 #include <linux/err.h>
12 #include <linux/irq.h>
13 #include <linux/interrupt.h>
14 #include <linux/export.h>
15 #include <linux/ioport.h>
16 #include <linux/pci.h>
17 #include <linux/proc_fs.h>
18 #include <linux/msi.h>
19 #include <linux/smp.h>
20 #include <linux/errno.h>
22 #include <linux/acpi_iort.h>
23 #include <linux/slab.h>
24 #include <linux/irqdomain.h>
25 #include <linux/of_irq.h>
29 static int pci_msi_enable
= 1;
30 int pci_msi_ignore_mask
;
32 #define msix_table_size(flags) ((flags & PCI_MSIX_FLAGS_QSIZE) + 1)
34 #ifdef CONFIG_PCI_MSI_IRQ_DOMAIN
35 static int pci_msi_setup_msi_irqs(struct pci_dev
*dev
, int nvec
, int type
)
37 struct irq_domain
*domain
;
39 domain
= dev_get_msi_domain(&dev
->dev
);
40 if (domain
&& irq_domain_is_hierarchy(domain
))
41 return msi_domain_alloc_irqs(domain
, &dev
->dev
, nvec
);
43 return arch_setup_msi_irqs(dev
, nvec
, type
);
46 static void pci_msi_teardown_msi_irqs(struct pci_dev
*dev
)
48 struct irq_domain
*domain
;
50 domain
= dev_get_msi_domain(&dev
->dev
);
51 if (domain
&& irq_domain_is_hierarchy(domain
))
52 msi_domain_free_irqs(domain
, &dev
->dev
);
54 arch_teardown_msi_irqs(dev
);
57 #define pci_msi_setup_msi_irqs arch_setup_msi_irqs
58 #define pci_msi_teardown_msi_irqs arch_teardown_msi_irqs
63 int __weak
arch_setup_msi_irq(struct pci_dev
*dev
, struct msi_desc
*desc
)
65 struct msi_controller
*chip
= dev
->bus
->msi
;
68 if (!chip
|| !chip
->setup_irq
)
71 err
= chip
->setup_irq(chip
, dev
, desc
);
75 irq_set_chip_data(desc
->irq
, chip
);
80 void __weak
arch_teardown_msi_irq(unsigned int irq
)
82 struct msi_controller
*chip
= irq_get_chip_data(irq
);
84 if (!chip
|| !chip
->teardown_irq
)
87 chip
->teardown_irq(chip
, irq
);
90 int __weak
arch_setup_msi_irqs(struct pci_dev
*dev
, int nvec
, int type
)
92 struct msi_controller
*chip
= dev
->bus
->msi
;
93 struct msi_desc
*entry
;
96 if (chip
&& chip
->setup_irqs
)
97 return chip
->setup_irqs(chip
, dev
, nvec
, type
);
99 * If an architecture wants to support multiple MSI, it needs to
100 * override arch_setup_msi_irqs()
102 if (type
== PCI_CAP_ID_MSI
&& nvec
> 1)
105 for_each_pci_msi_entry(entry
, dev
) {
106 ret
= arch_setup_msi_irq(dev
, entry
);
117 * We have a default implementation available as a separate non-weak
118 * function, as it is used by the Xen x86 PCI code
120 void default_teardown_msi_irqs(struct pci_dev
*dev
)
123 struct msi_desc
*entry
;
125 for_each_pci_msi_entry(entry
, dev
)
127 for (i
= 0; i
< entry
->nvec_used
; i
++)
128 arch_teardown_msi_irq(entry
->irq
+ i
);
131 void __weak
arch_teardown_msi_irqs(struct pci_dev
*dev
)
133 return default_teardown_msi_irqs(dev
);
136 static void default_restore_msi_irq(struct pci_dev
*dev
, int irq
)
138 struct msi_desc
*entry
;
141 if (dev
->msix_enabled
) {
142 for_each_pci_msi_entry(entry
, dev
) {
143 if (irq
== entry
->irq
)
146 } else if (dev
->msi_enabled
) {
147 entry
= irq_get_msi_desc(irq
);
151 __pci_write_msi_msg(entry
, &entry
->msg
);
154 void __weak
arch_restore_msi_irqs(struct pci_dev
*dev
)
156 return default_restore_msi_irqs(dev
);
159 static inline __attribute_const__ u32
msi_mask(unsigned x
)
161 /* Don't shift by >= width of type */
164 return (1 << (1 << x
)) - 1;
168 * PCI 2.3 does not specify mask bits for each MSI interrupt. Attempting to
169 * mask all MSI interrupts by clearing the MSI enable bit does not work
170 * reliably as devices without an INTx disable bit will then generate a
171 * level IRQ which will never be cleared.
173 u32
__pci_msi_desc_mask_irq(struct msi_desc
*desc
, u32 mask
, u32 flag
)
175 u32 mask_bits
= desc
->masked
;
177 if (pci_msi_ignore_mask
|| !desc
->msi_attrib
.maskbit
)
182 pci_write_config_dword(msi_desc_to_pci_dev(desc
), desc
->mask_pos
,
188 static void msi_mask_irq(struct msi_desc
*desc
, u32 mask
, u32 flag
)
190 desc
->masked
= __pci_msi_desc_mask_irq(desc
, mask
, flag
);
193 static void __iomem
*pci_msix_desc_addr(struct msi_desc
*desc
)
195 return desc
->mask_base
+
196 desc
->msi_attrib
.entry_nr
* PCI_MSIX_ENTRY_SIZE
;
200 * This internal function does not flush PCI writes to the device.
201 * All users must ensure that they read from the device before either
202 * assuming that the device state is up to date, or returning out of this
203 * file. This saves a few milliseconds when initialising devices with lots
204 * of MSI-X interrupts.
206 u32
__pci_msix_desc_mask_irq(struct msi_desc
*desc
, u32 flag
)
208 u32 mask_bits
= desc
->masked
;
210 if (pci_msi_ignore_mask
)
213 mask_bits
&= ~PCI_MSIX_ENTRY_CTRL_MASKBIT
;
215 mask_bits
|= PCI_MSIX_ENTRY_CTRL_MASKBIT
;
216 writel(mask_bits
, pci_msix_desc_addr(desc
) + PCI_MSIX_ENTRY_VECTOR_CTRL
);
221 static void msix_mask_irq(struct msi_desc
*desc
, u32 flag
)
223 desc
->masked
= __pci_msix_desc_mask_irq(desc
, flag
);
226 static void msi_set_mask_bit(struct irq_data
*data
, u32 flag
)
228 struct msi_desc
*desc
= irq_data_get_msi_desc(data
);
230 if (desc
->msi_attrib
.is_msix
) {
231 msix_mask_irq(desc
, flag
);
232 readl(desc
->mask_base
); /* Flush write to device */
234 unsigned offset
= data
->irq
- desc
->irq
;
235 msi_mask_irq(desc
, 1 << offset
, flag
<< offset
);
240 * pci_msi_mask_irq - Generic irq chip callback to mask PCI/MSI interrupts
241 * @data: pointer to irqdata associated to that interrupt
243 void pci_msi_mask_irq(struct irq_data
*data
)
245 msi_set_mask_bit(data
, 1);
247 EXPORT_SYMBOL_GPL(pci_msi_mask_irq
);
250 * pci_msi_unmask_irq - Generic irq chip callback to unmask PCI/MSI interrupts
251 * @data: pointer to irqdata associated to that interrupt
253 void pci_msi_unmask_irq(struct irq_data
*data
)
255 msi_set_mask_bit(data
, 0);
257 EXPORT_SYMBOL_GPL(pci_msi_unmask_irq
);
259 void default_restore_msi_irqs(struct pci_dev
*dev
)
261 struct msi_desc
*entry
;
263 for_each_pci_msi_entry(entry
, dev
)
264 default_restore_msi_irq(dev
, entry
->irq
);
267 void __pci_read_msi_msg(struct msi_desc
*entry
, struct msi_msg
*msg
)
269 struct pci_dev
*dev
= msi_desc_to_pci_dev(entry
);
271 BUG_ON(dev
->current_state
!= PCI_D0
);
273 if (entry
->msi_attrib
.is_msix
) {
274 void __iomem
*base
= pci_msix_desc_addr(entry
);
276 msg
->address_lo
= readl(base
+ PCI_MSIX_ENTRY_LOWER_ADDR
);
277 msg
->address_hi
= readl(base
+ PCI_MSIX_ENTRY_UPPER_ADDR
);
278 msg
->data
= readl(base
+ PCI_MSIX_ENTRY_DATA
);
280 int pos
= dev
->msi_cap
;
283 pci_read_config_dword(dev
, pos
+ PCI_MSI_ADDRESS_LO
,
285 if (entry
->msi_attrib
.is_64
) {
286 pci_read_config_dword(dev
, pos
+ PCI_MSI_ADDRESS_HI
,
288 pci_read_config_word(dev
, pos
+ PCI_MSI_DATA_64
, &data
);
291 pci_read_config_word(dev
, pos
+ PCI_MSI_DATA_32
, &data
);
297 void __pci_write_msi_msg(struct msi_desc
*entry
, struct msi_msg
*msg
)
299 struct pci_dev
*dev
= msi_desc_to_pci_dev(entry
);
301 if (dev
->current_state
!= PCI_D0
|| pci_dev_is_disconnected(dev
)) {
302 /* Don't touch the hardware now */
303 } else if (entry
->msi_attrib
.is_msix
) {
304 void __iomem
*base
= pci_msix_desc_addr(entry
);
306 writel(msg
->address_lo
, base
+ PCI_MSIX_ENTRY_LOWER_ADDR
);
307 writel(msg
->address_hi
, base
+ PCI_MSIX_ENTRY_UPPER_ADDR
);
308 writel(msg
->data
, base
+ PCI_MSIX_ENTRY_DATA
);
310 int pos
= dev
->msi_cap
;
313 pci_read_config_word(dev
, pos
+ PCI_MSI_FLAGS
, &msgctl
);
314 msgctl
&= ~PCI_MSI_FLAGS_QSIZE
;
315 msgctl
|= entry
->msi_attrib
.multiple
<< 4;
316 pci_write_config_word(dev
, pos
+ PCI_MSI_FLAGS
, msgctl
);
318 pci_write_config_dword(dev
, pos
+ PCI_MSI_ADDRESS_LO
,
320 if (entry
->msi_attrib
.is_64
) {
321 pci_write_config_dword(dev
, pos
+ PCI_MSI_ADDRESS_HI
,
323 pci_write_config_word(dev
, pos
+ PCI_MSI_DATA_64
,
326 pci_write_config_word(dev
, pos
+ PCI_MSI_DATA_32
,
333 void pci_write_msi_msg(unsigned int irq
, struct msi_msg
*msg
)
335 struct msi_desc
*entry
= irq_get_msi_desc(irq
);
337 __pci_write_msi_msg(entry
, msg
);
339 EXPORT_SYMBOL_GPL(pci_write_msi_msg
);
341 static void free_msi_irqs(struct pci_dev
*dev
)
343 struct list_head
*msi_list
= dev_to_msi_list(&dev
->dev
);
344 struct msi_desc
*entry
, *tmp
;
345 struct attribute
**msi_attrs
;
346 struct device_attribute
*dev_attr
;
349 for_each_pci_msi_entry(entry
, dev
)
351 for (i
= 0; i
< entry
->nvec_used
; i
++)
352 BUG_ON(irq_has_action(entry
->irq
+ i
));
354 pci_msi_teardown_msi_irqs(dev
);
356 list_for_each_entry_safe(entry
, tmp
, msi_list
, list
) {
357 if (entry
->msi_attrib
.is_msix
) {
358 if (list_is_last(&entry
->list
, msi_list
))
359 iounmap(entry
->mask_base
);
362 list_del(&entry
->list
);
363 free_msi_entry(entry
);
366 if (dev
->msi_irq_groups
) {
367 sysfs_remove_groups(&dev
->dev
.kobj
, dev
->msi_irq_groups
);
368 msi_attrs
= dev
->msi_irq_groups
[0]->attrs
;
369 while (msi_attrs
[count
]) {
370 dev_attr
= container_of(msi_attrs
[count
],
371 struct device_attribute
, attr
);
372 kfree(dev_attr
->attr
.name
);
377 kfree(dev
->msi_irq_groups
[0]);
378 kfree(dev
->msi_irq_groups
);
379 dev
->msi_irq_groups
= NULL
;
383 static void pci_intx_for_msi(struct pci_dev
*dev
, int enable
)
385 if (!(dev
->dev_flags
& PCI_DEV_FLAGS_MSI_INTX_DISABLE_BUG
))
386 pci_intx(dev
, enable
);
389 static void __pci_restore_msi_state(struct pci_dev
*dev
)
392 struct msi_desc
*entry
;
394 if (!dev
->msi_enabled
)
397 entry
= irq_get_msi_desc(dev
->irq
);
399 pci_intx_for_msi(dev
, 0);
400 pci_msi_set_enable(dev
, 0);
401 arch_restore_msi_irqs(dev
);
403 pci_read_config_word(dev
, dev
->msi_cap
+ PCI_MSI_FLAGS
, &control
);
404 msi_mask_irq(entry
, msi_mask(entry
->msi_attrib
.multi_cap
),
406 control
&= ~PCI_MSI_FLAGS_QSIZE
;
407 control
|= (entry
->msi_attrib
.multiple
<< 4) | PCI_MSI_FLAGS_ENABLE
;
408 pci_write_config_word(dev
, dev
->msi_cap
+ PCI_MSI_FLAGS
, control
);
411 static void __pci_restore_msix_state(struct pci_dev
*dev
)
413 struct msi_desc
*entry
;
415 if (!dev
->msix_enabled
)
417 BUG_ON(list_empty(dev_to_msi_list(&dev
->dev
)));
419 /* route the table */
420 pci_intx_for_msi(dev
, 0);
421 pci_msix_clear_and_set_ctrl(dev
, 0,
422 PCI_MSIX_FLAGS_ENABLE
| PCI_MSIX_FLAGS_MASKALL
);
424 arch_restore_msi_irqs(dev
);
425 for_each_pci_msi_entry(entry
, dev
)
426 msix_mask_irq(entry
, entry
->masked
);
428 pci_msix_clear_and_set_ctrl(dev
, PCI_MSIX_FLAGS_MASKALL
, 0);
431 void pci_restore_msi_state(struct pci_dev
*dev
)
433 __pci_restore_msi_state(dev
);
434 __pci_restore_msix_state(dev
);
436 EXPORT_SYMBOL_GPL(pci_restore_msi_state
);
438 static ssize_t
msi_mode_show(struct device
*dev
, struct device_attribute
*attr
,
441 struct msi_desc
*entry
;
445 retval
= kstrtoul(attr
->attr
.name
, 10, &irq
);
449 entry
= irq_get_msi_desc(irq
);
451 return sprintf(buf
, "%s\n",
452 entry
->msi_attrib
.is_msix
? "msix" : "msi");
457 static int populate_msi_sysfs(struct pci_dev
*pdev
)
459 struct attribute
**msi_attrs
;
460 struct attribute
*msi_attr
;
461 struct device_attribute
*msi_dev_attr
;
462 struct attribute_group
*msi_irq_group
;
463 const struct attribute_group
**msi_irq_groups
;
464 struct msi_desc
*entry
;
470 /* Determine how many msi entries we have */
471 for_each_pci_msi_entry(entry
, pdev
)
472 num_msi
+= entry
->nvec_used
;
476 /* Dynamically create the MSI attributes for the PCI device */
477 msi_attrs
= kzalloc(sizeof(void *) * (num_msi
+ 1), GFP_KERNEL
);
480 for_each_pci_msi_entry(entry
, pdev
) {
481 for (i
= 0; i
< entry
->nvec_used
; i
++) {
482 msi_dev_attr
= kzalloc(sizeof(*msi_dev_attr
), GFP_KERNEL
);
485 msi_attrs
[count
] = &msi_dev_attr
->attr
;
487 sysfs_attr_init(&msi_dev_attr
->attr
);
488 msi_dev_attr
->attr
.name
= kasprintf(GFP_KERNEL
, "%d",
490 if (!msi_dev_attr
->attr
.name
)
492 msi_dev_attr
->attr
.mode
= S_IRUGO
;
493 msi_dev_attr
->show
= msi_mode_show
;
498 msi_irq_group
= kzalloc(sizeof(*msi_irq_group
), GFP_KERNEL
);
501 msi_irq_group
->name
= "msi_irqs";
502 msi_irq_group
->attrs
= msi_attrs
;
504 msi_irq_groups
= kzalloc(sizeof(void *) * 2, GFP_KERNEL
);
506 goto error_irq_group
;
507 msi_irq_groups
[0] = msi_irq_group
;
509 ret
= sysfs_create_groups(&pdev
->dev
.kobj
, msi_irq_groups
);
511 goto error_irq_groups
;
512 pdev
->msi_irq_groups
= msi_irq_groups
;
517 kfree(msi_irq_groups
);
519 kfree(msi_irq_group
);
522 msi_attr
= msi_attrs
[count
];
524 msi_dev_attr
= container_of(msi_attr
, struct device_attribute
, attr
);
525 kfree(msi_attr
->name
);
528 msi_attr
= msi_attrs
[count
];
534 static struct msi_desc
*
535 msi_setup_entry(struct pci_dev
*dev
, int nvec
, const struct irq_affinity
*affd
)
537 struct cpumask
*masks
= NULL
;
538 struct msi_desc
*entry
;
542 masks
= irq_create_affinity_masks(nvec
, affd
);
545 /* MSI Entry Initialization */
546 entry
= alloc_msi_entry(&dev
->dev
, nvec
, masks
);
550 pci_read_config_word(dev
, dev
->msi_cap
+ PCI_MSI_FLAGS
, &control
);
552 entry
->msi_attrib
.is_msix
= 0;
553 entry
->msi_attrib
.is_64
= !!(control
& PCI_MSI_FLAGS_64BIT
);
554 entry
->msi_attrib
.entry_nr
= 0;
555 entry
->msi_attrib
.maskbit
= !!(control
& PCI_MSI_FLAGS_MASKBIT
);
556 entry
->msi_attrib
.default_irq
= dev
->irq
; /* Save IOAPIC IRQ */
557 entry
->msi_attrib
.multi_cap
= (control
& PCI_MSI_FLAGS_QMASK
) >> 1;
558 entry
->msi_attrib
.multiple
= ilog2(__roundup_pow_of_two(nvec
));
560 if (control
& PCI_MSI_FLAGS_64BIT
)
561 entry
->mask_pos
= dev
->msi_cap
+ PCI_MSI_MASK_64
;
563 entry
->mask_pos
= dev
->msi_cap
+ PCI_MSI_MASK_32
;
565 /* Save the initial mask status */
566 if (entry
->msi_attrib
.maskbit
)
567 pci_read_config_dword(dev
, entry
->mask_pos
, &entry
->masked
);
574 static int msi_verify_entries(struct pci_dev
*dev
)
576 struct msi_desc
*entry
;
578 for_each_pci_msi_entry(entry
, dev
) {
579 if (!dev
->no_64bit_msi
|| !entry
->msg
.address_hi
)
581 dev_err(&dev
->dev
, "Device has broken 64-bit MSI but arch"
582 " tried to assign one above 4G\n");
589 * msi_capability_init - configure device's MSI capability structure
590 * @dev: pointer to the pci_dev data structure of MSI device function
591 * @nvec: number of interrupts to allocate
592 * @affd: description of automatic irq affinity assignments (may be %NULL)
594 * Setup the MSI capability structure of the device with the requested
595 * number of interrupts. A return value of zero indicates the successful
596 * setup of an entry with the new MSI irq. A negative return value indicates
597 * an error, and a positive return value indicates the number of interrupts
598 * which could have been allocated.
600 static int msi_capability_init(struct pci_dev
*dev
, int nvec
,
601 const struct irq_affinity
*affd
)
603 struct msi_desc
*entry
;
607 pci_msi_set_enable(dev
, 0); /* Disable MSI during set up */
609 entry
= msi_setup_entry(dev
, nvec
, affd
);
613 /* All MSIs are unmasked by default, Mask them all */
614 mask
= msi_mask(entry
->msi_attrib
.multi_cap
);
615 msi_mask_irq(entry
, mask
, mask
);
617 list_add_tail(&entry
->list
, dev_to_msi_list(&dev
->dev
));
619 /* Configure MSI capability structure */
620 ret
= pci_msi_setup_msi_irqs(dev
, nvec
, PCI_CAP_ID_MSI
);
622 msi_mask_irq(entry
, mask
, ~mask
);
627 ret
= msi_verify_entries(dev
);
629 msi_mask_irq(entry
, mask
, ~mask
);
634 ret
= populate_msi_sysfs(dev
);
636 msi_mask_irq(entry
, mask
, ~mask
);
641 /* Set MSI enabled bits */
642 pci_intx_for_msi(dev
, 0);
643 pci_msi_set_enable(dev
, 1);
644 dev
->msi_enabled
= 1;
646 pcibios_free_irq(dev
);
647 dev
->irq
= entry
->irq
;
651 static void __iomem
*msix_map_region(struct pci_dev
*dev
, unsigned nr_entries
)
653 resource_size_t phys_addr
;
658 pci_read_config_dword(dev
, dev
->msix_cap
+ PCI_MSIX_TABLE
,
660 bir
= (u8
)(table_offset
& PCI_MSIX_TABLE_BIR
);
661 flags
= pci_resource_flags(dev
, bir
);
662 if (!flags
|| (flags
& IORESOURCE_UNSET
))
665 table_offset
&= PCI_MSIX_TABLE_OFFSET
;
666 phys_addr
= pci_resource_start(dev
, bir
) + table_offset
;
668 return ioremap_nocache(phys_addr
, nr_entries
* PCI_MSIX_ENTRY_SIZE
);
671 static int msix_setup_entries(struct pci_dev
*dev
, void __iomem
*base
,
672 struct msix_entry
*entries
, int nvec
,
673 const struct irq_affinity
*affd
)
675 struct cpumask
*curmsk
, *masks
= NULL
;
676 struct msi_desc
*entry
;
680 masks
= irq_create_affinity_masks(nvec
, affd
);
682 for (i
= 0, curmsk
= masks
; i
< nvec
; i
++) {
683 entry
= alloc_msi_entry(&dev
->dev
, 1, curmsk
);
689 /* No enough memory. Don't try again */
694 entry
->msi_attrib
.is_msix
= 1;
695 entry
->msi_attrib
.is_64
= 1;
697 entry
->msi_attrib
.entry_nr
= entries
[i
].entry
;
699 entry
->msi_attrib
.entry_nr
= i
;
700 entry
->msi_attrib
.default_irq
= dev
->irq
;
701 entry
->mask_base
= base
;
703 list_add_tail(&entry
->list
, dev_to_msi_list(&dev
->dev
));
713 static void msix_program_entries(struct pci_dev
*dev
,
714 struct msix_entry
*entries
)
716 struct msi_desc
*entry
;
719 for_each_pci_msi_entry(entry
, dev
) {
721 entries
[i
++].vector
= entry
->irq
;
722 entry
->masked
= readl(pci_msix_desc_addr(entry
) +
723 PCI_MSIX_ENTRY_VECTOR_CTRL
);
724 msix_mask_irq(entry
, 1);
729 * msix_capability_init - configure device's MSI-X capability
730 * @dev: pointer to the pci_dev data structure of MSI-X device function
731 * @entries: pointer to an array of struct msix_entry entries
732 * @nvec: number of @entries
733 * @affd: Optional pointer to enable automatic affinity assignement
735 * Setup the MSI-X capability structure of device function with a
736 * single MSI-X irq. A return of zero indicates the successful setup of
737 * requested MSI-X entries with allocated irqs or non-zero for otherwise.
739 static int msix_capability_init(struct pci_dev
*dev
, struct msix_entry
*entries
,
740 int nvec
, const struct irq_affinity
*affd
)
746 /* Ensure MSI-X is disabled while it is set up */
747 pci_msix_clear_and_set_ctrl(dev
, PCI_MSIX_FLAGS_ENABLE
, 0);
749 pci_read_config_word(dev
, dev
->msix_cap
+ PCI_MSIX_FLAGS
, &control
);
750 /* Request & Map MSI-X table region */
751 base
= msix_map_region(dev
, msix_table_size(control
));
755 ret
= msix_setup_entries(dev
, base
, entries
, nvec
, affd
);
759 ret
= pci_msi_setup_msi_irqs(dev
, nvec
, PCI_CAP_ID_MSIX
);
763 /* Check if all MSI entries honor device restrictions */
764 ret
= msi_verify_entries(dev
);
769 * Some devices require MSI-X to be enabled before we can touch the
770 * MSI-X registers. We need to mask all the vectors to prevent
771 * interrupts coming in before they're fully set up.
773 pci_msix_clear_and_set_ctrl(dev
, 0,
774 PCI_MSIX_FLAGS_MASKALL
| PCI_MSIX_FLAGS_ENABLE
);
776 msix_program_entries(dev
, entries
);
778 ret
= populate_msi_sysfs(dev
);
782 /* Set MSI-X enabled bits and unmask the function */
783 pci_intx_for_msi(dev
, 0);
784 dev
->msix_enabled
= 1;
785 pci_msix_clear_and_set_ctrl(dev
, PCI_MSIX_FLAGS_MASKALL
, 0);
787 pcibios_free_irq(dev
);
793 * If we had some success, report the number of irqs
794 * we succeeded in setting up.
796 struct msi_desc
*entry
;
799 for_each_pci_msi_entry(entry
, dev
) {
814 * pci_msi_supported - check whether MSI may be enabled on a device
815 * @dev: pointer to the pci_dev data structure of MSI device function
816 * @nvec: how many MSIs have been requested ?
818 * Look at global flags, the device itself, and its parent buses
819 * to determine if MSI/-X are supported for the device. If MSI/-X is
820 * supported return 1, else return 0.
822 static int pci_msi_supported(struct pci_dev
*dev
, int nvec
)
826 /* MSI must be globally enabled and supported by the device */
830 if (!dev
|| dev
->no_msi
|| dev
->current_state
!= PCI_D0
)
834 * You can't ask to have 0 or less MSIs configured.
836 * b) the list manipulation code assumes nvec >= 1.
842 * Any bridge which does NOT route MSI transactions from its
843 * secondary bus to its primary bus must set NO_MSI flag on
844 * the secondary pci_bus.
845 * We expect only arch-specific PCI host bus controller driver
846 * or quirks for specific PCI bridges to be setting NO_MSI.
848 for (bus
= dev
->bus
; bus
; bus
= bus
->parent
)
849 if (bus
->bus_flags
& PCI_BUS_FLAGS_NO_MSI
)
856 * pci_msi_vec_count - Return the number of MSI vectors a device can send
857 * @dev: device to report about
859 * This function returns the number of MSI vectors a device requested via
860 * Multiple Message Capable register. It returns a negative errno if the
861 * device is not capable sending MSI interrupts. Otherwise, the call succeeds
862 * and returns a power of two, up to a maximum of 2^5 (32), according to the
865 int pci_msi_vec_count(struct pci_dev
*dev
)
873 pci_read_config_word(dev
, dev
->msi_cap
+ PCI_MSI_FLAGS
, &msgctl
);
874 ret
= 1 << ((msgctl
& PCI_MSI_FLAGS_QMASK
) >> 1);
878 EXPORT_SYMBOL(pci_msi_vec_count
);
880 static void pci_msi_shutdown(struct pci_dev
*dev
)
882 struct msi_desc
*desc
;
885 if (!pci_msi_enable
|| !dev
|| !dev
->msi_enabled
)
888 BUG_ON(list_empty(dev_to_msi_list(&dev
->dev
)));
889 desc
= first_pci_msi_entry(dev
);
891 pci_msi_set_enable(dev
, 0);
892 pci_intx_for_msi(dev
, 1);
893 dev
->msi_enabled
= 0;
895 /* Return the device with MSI unmasked as initial states */
896 mask
= msi_mask(desc
->msi_attrib
.multi_cap
);
897 /* Keep cached state to be restored */
898 __pci_msi_desc_mask_irq(desc
, mask
, ~mask
);
900 /* Restore dev->irq to its default pin-assertion irq */
901 dev
->irq
= desc
->msi_attrib
.default_irq
;
902 pcibios_alloc_irq(dev
);
905 void pci_disable_msi(struct pci_dev
*dev
)
907 if (!pci_msi_enable
|| !dev
|| !dev
->msi_enabled
)
910 pci_msi_shutdown(dev
);
913 EXPORT_SYMBOL(pci_disable_msi
);
916 * pci_msix_vec_count - return the number of device's MSI-X table entries
917 * @dev: pointer to the pci_dev data structure of MSI-X device function
918 * This function returns the number of device's MSI-X table entries and
919 * therefore the number of MSI-X vectors device is capable of sending.
920 * It returns a negative errno if the device is not capable of sending MSI-X
923 int pci_msix_vec_count(struct pci_dev
*dev
)
930 pci_read_config_word(dev
, dev
->msix_cap
+ PCI_MSIX_FLAGS
, &control
);
931 return msix_table_size(control
);
933 EXPORT_SYMBOL(pci_msix_vec_count
);
935 static int __pci_enable_msix(struct pci_dev
*dev
, struct msix_entry
*entries
,
936 int nvec
, const struct irq_affinity
*affd
)
941 if (!pci_msi_supported(dev
, nvec
))
944 nr_entries
= pci_msix_vec_count(dev
);
947 if (nvec
> nr_entries
)
951 /* Check for any invalid entries */
952 for (i
= 0; i
< nvec
; i
++) {
953 if (entries
[i
].entry
>= nr_entries
)
954 return -EINVAL
; /* invalid entry */
955 for (j
= i
+ 1; j
< nvec
; j
++) {
956 if (entries
[i
].entry
== entries
[j
].entry
)
957 return -EINVAL
; /* duplicate entry */
962 /* Check whether driver already requested for MSI irq */
963 if (dev
->msi_enabled
) {
964 dev_info(&dev
->dev
, "can't enable MSI-X (MSI IRQ already assigned)\n");
967 return msix_capability_init(dev
, entries
, nvec
, affd
);
970 static void pci_msix_shutdown(struct pci_dev
*dev
)
972 struct msi_desc
*entry
;
974 if (!pci_msi_enable
|| !dev
|| !dev
->msix_enabled
)
977 if (pci_dev_is_disconnected(dev
)) {
978 dev
->msix_enabled
= 0;
982 /* Return the device with MSI-X masked as initial states */
983 for_each_pci_msi_entry(entry
, dev
) {
984 /* Keep cached states to be restored */
985 __pci_msix_desc_mask_irq(entry
, 1);
988 pci_msix_clear_and_set_ctrl(dev
, PCI_MSIX_FLAGS_ENABLE
, 0);
989 pci_intx_for_msi(dev
, 1);
990 dev
->msix_enabled
= 0;
991 pcibios_alloc_irq(dev
);
994 void pci_disable_msix(struct pci_dev
*dev
)
996 if (!pci_msi_enable
|| !dev
|| !dev
->msix_enabled
)
999 pci_msix_shutdown(dev
);
1002 EXPORT_SYMBOL(pci_disable_msix
);
1004 void pci_no_msi(void)
1010 * pci_msi_enabled - is MSI enabled?
1012 * Returns true if MSI has not been disabled by the command-line option
1015 int pci_msi_enabled(void)
1017 return pci_msi_enable
;
1019 EXPORT_SYMBOL(pci_msi_enabled
);
1021 static int __pci_enable_msi_range(struct pci_dev
*dev
, int minvec
, int maxvec
,
1022 const struct irq_affinity
*affd
)
1027 if (!pci_msi_supported(dev
, minvec
))
1030 /* Check whether driver already requested MSI-X irqs */
1031 if (dev
->msix_enabled
) {
1033 "can't enable MSI (MSI-X already enabled)\n");
1037 if (maxvec
< minvec
)
1040 if (WARN_ON_ONCE(dev
->msi_enabled
))
1043 nvec
= pci_msi_vec_count(dev
);
1054 nvec
= irq_calc_affinity_vectors(minvec
, nvec
, affd
);
1059 rc
= msi_capability_init(dev
, nvec
, affd
);
1072 /* deprecated, don't use */
1073 int pci_enable_msi(struct pci_dev
*dev
)
1075 int rc
= __pci_enable_msi_range(dev
, 1, 1, NULL
);
1080 EXPORT_SYMBOL(pci_enable_msi
);
1082 static int __pci_enable_msix_range(struct pci_dev
*dev
,
1083 struct msix_entry
*entries
, int minvec
,
1084 int maxvec
, const struct irq_affinity
*affd
)
1086 int rc
, nvec
= maxvec
;
1088 if (maxvec
< minvec
)
1091 if (WARN_ON_ONCE(dev
->msix_enabled
))
1096 nvec
= irq_calc_affinity_vectors(minvec
, nvec
, affd
);
1101 rc
= __pci_enable_msix(dev
, entries
, nvec
, affd
);
1115 * pci_enable_msix_range - configure device's MSI-X capability structure
1116 * @dev: pointer to the pci_dev data structure of MSI-X device function
1117 * @entries: pointer to an array of MSI-X entries
1118 * @minvec: minimum number of MSI-X irqs requested
1119 * @maxvec: maximum number of MSI-X irqs requested
1121 * Setup the MSI-X capability structure of device function with a maximum
1122 * possible number of interrupts in the range between @minvec and @maxvec
1123 * upon its software driver call to request for MSI-X mode enabled on its
1124 * hardware device function. It returns a negative errno if an error occurs.
1125 * If it succeeds, it returns the actual number of interrupts allocated and
1126 * indicates the successful configuration of MSI-X capability structure
1127 * with new allocated MSI-X interrupts.
1129 int pci_enable_msix_range(struct pci_dev
*dev
, struct msix_entry
*entries
,
1130 int minvec
, int maxvec
)
1132 return __pci_enable_msix_range(dev
, entries
, minvec
, maxvec
, NULL
);
1134 EXPORT_SYMBOL(pci_enable_msix_range
);
1137 * pci_alloc_irq_vectors_affinity - allocate multiple IRQs for a device
1138 * @dev: PCI device to operate on
1139 * @min_vecs: minimum number of vectors required (must be >= 1)
1140 * @max_vecs: maximum (desired) number of vectors
1141 * @flags: flags or quirks for the allocation
1142 * @affd: optional description of the affinity requirements
1144 * Allocate up to @max_vecs interrupt vectors for @dev, using MSI-X or MSI
1145 * vectors if available, and fall back to a single legacy vector
1146 * if neither is available. Return the number of vectors allocated,
1147 * (which might be smaller than @max_vecs) if successful, or a negative
1148 * error code on error. If less than @min_vecs interrupt vectors are
1149 * available for @dev the function will fail with -ENOSPC.
1151 * To get the Linux IRQ number used for a vector that can be passed to
1152 * request_irq() use the pci_irq_vector() helper.
1154 int pci_alloc_irq_vectors_affinity(struct pci_dev
*dev
, unsigned int min_vecs
,
1155 unsigned int max_vecs
, unsigned int flags
,
1156 const struct irq_affinity
*affd
)
1158 static const struct irq_affinity msi_default_affd
;
1161 if (flags
& PCI_IRQ_AFFINITY
) {
1163 affd
= &msi_default_affd
;
1169 if (flags
& PCI_IRQ_MSIX
) {
1170 vecs
= __pci_enable_msix_range(dev
, NULL
, min_vecs
, max_vecs
,
1176 if (flags
& PCI_IRQ_MSI
) {
1177 vecs
= __pci_enable_msi_range(dev
, min_vecs
, max_vecs
, affd
);
1182 /* use legacy irq if allowed */
1183 if (flags
& PCI_IRQ_LEGACY
) {
1184 if (min_vecs
== 1 && dev
->irq
) {
1192 EXPORT_SYMBOL(pci_alloc_irq_vectors_affinity
);
1195 * pci_free_irq_vectors - free previously allocated IRQs for a device
1196 * @dev: PCI device to operate on
1198 * Undoes the allocations and enabling in pci_alloc_irq_vectors().
1200 void pci_free_irq_vectors(struct pci_dev
*dev
)
1202 pci_disable_msix(dev
);
1203 pci_disable_msi(dev
);
1205 EXPORT_SYMBOL(pci_free_irq_vectors
);
1208 * pci_irq_vector - return Linux IRQ number of a device vector
1209 * @dev: PCI device to operate on
1210 * @nr: device-relative interrupt vector index (0-based).
1212 int pci_irq_vector(struct pci_dev
*dev
, unsigned int nr
)
1214 if (dev
->msix_enabled
) {
1215 struct msi_desc
*entry
;
1218 for_each_pci_msi_entry(entry
, dev
) {
1227 if (dev
->msi_enabled
) {
1228 struct msi_desc
*entry
= first_pci_msi_entry(dev
);
1230 if (WARN_ON_ONCE(nr
>= entry
->nvec_used
))
1233 if (WARN_ON_ONCE(nr
> 0))
1237 return dev
->irq
+ nr
;
1239 EXPORT_SYMBOL(pci_irq_vector
);
1242 * pci_irq_get_affinity - return the affinity of a particular msi vector
1243 * @dev: PCI device to operate on
1244 * @nr: device-relative interrupt vector index (0-based).
1246 const struct cpumask
*pci_irq_get_affinity(struct pci_dev
*dev
, int nr
)
1248 if (dev
->msix_enabled
) {
1249 struct msi_desc
*entry
;
1252 for_each_pci_msi_entry(entry
, dev
) {
1254 return entry
->affinity
;
1259 } else if (dev
->msi_enabled
) {
1260 struct msi_desc
*entry
= first_pci_msi_entry(dev
);
1262 if (WARN_ON_ONCE(!entry
|| !entry
->affinity
||
1263 nr
>= entry
->nvec_used
))
1266 return &entry
->affinity
[nr
];
1268 return cpu_possible_mask
;
1271 EXPORT_SYMBOL(pci_irq_get_affinity
);
1274 * pci_irq_get_node - return the numa node of a particular msi vector
1275 * @pdev: PCI device to operate on
1276 * @vec: device-relative interrupt vector index (0-based).
1278 int pci_irq_get_node(struct pci_dev
*pdev
, int vec
)
1280 const struct cpumask
*mask
;
1282 mask
= pci_irq_get_affinity(pdev
, vec
);
1284 return local_memory_node(cpu_to_node(cpumask_first(mask
)));
1285 return dev_to_node(&pdev
->dev
);
1287 EXPORT_SYMBOL(pci_irq_get_node
);
1289 struct pci_dev
*msi_desc_to_pci_dev(struct msi_desc
*desc
)
1291 return to_pci_dev(desc
->dev
);
1293 EXPORT_SYMBOL(msi_desc_to_pci_dev
);
1295 void *msi_desc_to_pci_sysdata(struct msi_desc
*desc
)
1297 struct pci_dev
*dev
= msi_desc_to_pci_dev(desc
);
1299 return dev
->bus
->sysdata
;
1301 EXPORT_SYMBOL_GPL(msi_desc_to_pci_sysdata
);
1303 #ifdef CONFIG_PCI_MSI_IRQ_DOMAIN
1305 * pci_msi_domain_write_msg - Helper to write MSI message to PCI config space
1306 * @irq_data: Pointer to interrupt data of the MSI interrupt
1307 * @msg: Pointer to the message
1309 void pci_msi_domain_write_msg(struct irq_data
*irq_data
, struct msi_msg
*msg
)
1311 struct msi_desc
*desc
= irq_data_get_msi_desc(irq_data
);
1314 * For MSI-X desc->irq is always equal to irq_data->irq. For
1315 * MSI only the first interrupt of MULTI MSI passes the test.
1317 if (desc
->irq
== irq_data
->irq
)
1318 __pci_write_msi_msg(desc
, msg
);
1322 * pci_msi_domain_calc_hwirq - Generate a unique ID for an MSI source
1323 * @dev: Pointer to the PCI device
1324 * @desc: Pointer to the msi descriptor
1326 * The ID number is only used within the irqdomain.
1328 irq_hw_number_t
pci_msi_domain_calc_hwirq(struct pci_dev
*dev
,
1329 struct msi_desc
*desc
)
1331 return (irq_hw_number_t
)desc
->msi_attrib
.entry_nr
|
1332 PCI_DEVID(dev
->bus
->number
, dev
->devfn
) << 11 |
1333 (pci_domain_nr(dev
->bus
) & 0xFFFFFFFF) << 27;
1336 static inline bool pci_msi_desc_is_multi_msi(struct msi_desc
*desc
)
1338 return !desc
->msi_attrib
.is_msix
&& desc
->nvec_used
> 1;
1342 * pci_msi_domain_check_cap - Verify that @domain supports the capabilities for @dev
1343 * @domain: The interrupt domain to check
1344 * @info: The domain info for verification
1345 * @dev: The device to check
1348 * 0 if the functionality is supported
1349 * 1 if Multi MSI is requested, but the domain does not support it
1350 * -ENOTSUPP otherwise
1352 int pci_msi_domain_check_cap(struct irq_domain
*domain
,
1353 struct msi_domain_info
*info
, struct device
*dev
)
1355 struct msi_desc
*desc
= first_pci_msi_entry(to_pci_dev(dev
));
1357 /* Special handling to support __pci_enable_msi_range() */
1358 if (pci_msi_desc_is_multi_msi(desc
) &&
1359 !(info
->flags
& MSI_FLAG_MULTI_PCI_MSI
))
1361 else if (desc
->msi_attrib
.is_msix
&& !(info
->flags
& MSI_FLAG_PCI_MSIX
))
1367 static int pci_msi_domain_handle_error(struct irq_domain
*domain
,
1368 struct msi_desc
*desc
, int error
)
1370 /* Special handling to support __pci_enable_msi_range() */
1371 if (pci_msi_desc_is_multi_msi(desc
) && error
== -ENOSPC
)
1377 #ifdef GENERIC_MSI_DOMAIN_OPS
1378 static void pci_msi_domain_set_desc(msi_alloc_info_t
*arg
,
1379 struct msi_desc
*desc
)
1382 arg
->hwirq
= pci_msi_domain_calc_hwirq(msi_desc_to_pci_dev(desc
),
1386 #define pci_msi_domain_set_desc NULL
1389 static struct msi_domain_ops pci_msi_domain_ops_default
= {
1390 .set_desc
= pci_msi_domain_set_desc
,
1391 .msi_check
= pci_msi_domain_check_cap
,
1392 .handle_error
= pci_msi_domain_handle_error
,
1395 static void pci_msi_domain_update_dom_ops(struct msi_domain_info
*info
)
1397 struct msi_domain_ops
*ops
= info
->ops
;
1400 info
->ops
= &pci_msi_domain_ops_default
;
1402 if (ops
->set_desc
== NULL
)
1403 ops
->set_desc
= pci_msi_domain_set_desc
;
1404 if (ops
->msi_check
== NULL
)
1405 ops
->msi_check
= pci_msi_domain_check_cap
;
1406 if (ops
->handle_error
== NULL
)
1407 ops
->handle_error
= pci_msi_domain_handle_error
;
1411 static void pci_msi_domain_update_chip_ops(struct msi_domain_info
*info
)
1413 struct irq_chip
*chip
= info
->chip
;
1416 if (!chip
->irq_write_msi_msg
)
1417 chip
->irq_write_msi_msg
= pci_msi_domain_write_msg
;
1418 if (!chip
->irq_mask
)
1419 chip
->irq_mask
= pci_msi_mask_irq
;
1420 if (!chip
->irq_unmask
)
1421 chip
->irq_unmask
= pci_msi_unmask_irq
;
1425 * pci_msi_create_irq_domain - Create a MSI interrupt domain
1426 * @fwnode: Optional fwnode of the interrupt controller
1427 * @info: MSI domain info
1428 * @parent: Parent irq domain
1430 * Updates the domain and chip ops and creates a MSI interrupt domain.
1433 * A domain pointer or NULL in case of failure.
1435 struct irq_domain
*pci_msi_create_irq_domain(struct fwnode_handle
*fwnode
,
1436 struct msi_domain_info
*info
,
1437 struct irq_domain
*parent
)
1439 struct irq_domain
*domain
;
1441 if (info
->flags
& MSI_FLAG_USE_DEF_DOM_OPS
)
1442 pci_msi_domain_update_dom_ops(info
);
1443 if (info
->flags
& MSI_FLAG_USE_DEF_CHIP_OPS
)
1444 pci_msi_domain_update_chip_ops(info
);
1446 info
->flags
|= MSI_FLAG_ACTIVATE_EARLY
;
1448 domain
= msi_create_irq_domain(fwnode
, info
, parent
);
1452 irq_domain_update_bus_token(domain
, DOMAIN_BUS_PCI_MSI
);
1455 EXPORT_SYMBOL_GPL(pci_msi_create_irq_domain
);
1458 * Users of the generic MSI infrastructure expect a device to have a single ID,
1459 * so with DMA aliases we have to pick the least-worst compromise. Devices with
1460 * DMA phantom functions tend to still emit MSIs from the real function number,
1461 * so we ignore those and only consider topological aliases where either the
1462 * alias device or RID appears on a different bus number. We also make the
1463 * reasonable assumption that bridges are walked in an upstream direction (so
1464 * the last one seen wins), and the much braver assumption that the most likely
1465 * case is that of PCI->PCIe so we should always use the alias RID. This echoes
1466 * the logic from intel_irq_remapping's set_msi_sid(), which presumably works
1467 * well enough in practice; in the face of the horrible PCIe<->PCI-X conditions
1468 * for taking ownership all we can really do is close our eyes and hope...
1470 static int get_msi_id_cb(struct pci_dev
*pdev
, u16 alias
, void *data
)
1473 u8 bus
= PCI_BUS_NUM(*pa
);
1475 if (pdev
->bus
->number
!= bus
|| PCI_BUS_NUM(alias
) != bus
)
1482 * pci_msi_domain_get_msi_rid - Get the MSI requester id (RID)
1483 * @domain: The interrupt domain
1484 * @pdev: The PCI device.
1486 * The RID for a device is formed from the alias, with a firmware
1487 * supplied mapping applied
1491 u32
pci_msi_domain_get_msi_rid(struct irq_domain
*domain
, struct pci_dev
*pdev
)
1493 struct device_node
*of_node
;
1494 u32 rid
= PCI_DEVID(pdev
->bus
->number
, pdev
->devfn
);
1496 pci_for_each_dma_alias(pdev
, get_msi_id_cb
, &rid
);
1498 of_node
= irq_domain_get_of_node(domain
);
1499 rid
= of_node
? of_msi_map_rid(&pdev
->dev
, of_node
, rid
) :
1500 iort_msi_map_rid(&pdev
->dev
, rid
);
1506 * pci_msi_get_device_domain - Get the MSI domain for a given PCI device
1507 * @pdev: The PCI device
1509 * Use the firmware data to find a device-specific MSI domain
1510 * (i.e. not one that is set as a default).
1512 * Returns: The corresponding MSI domain or NULL if none has been found.
1514 struct irq_domain
*pci_msi_get_device_domain(struct pci_dev
*pdev
)
1516 struct irq_domain
*dom
;
1517 u32 rid
= PCI_DEVID(pdev
->bus
->number
, pdev
->devfn
);
1519 pci_for_each_dma_alias(pdev
, get_msi_id_cb
, &rid
);
1520 dom
= of_msi_map_get_device_domain(&pdev
->dev
, rid
);
1522 dom
= iort_get_device_domain(&pdev
->dev
, rid
);
1525 #endif /* CONFIG_PCI_MSI_IRQ_DOMAIN */