1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) 2007-2011 Freescale Semiconductor, Inc.
5 * Author: Tony Li <tony.li@freescale.com>
6 * Jason Jin <Jason.jin@freescale.com>
8 * The hwirq alloc and free code reuse from sysdev/mpic_msi.c
10 #include <linux/irq.h>
11 #include <linux/msi.h>
12 #include <linux/pci.h>
13 #include <linux/slab.h>
14 #include <linux/of_platform.h>
15 #include <linux/interrupt.h>
16 #include <linux/seq_file.h>
17 #include <sysdev/fsl_soc.h>
19 #include <asm/hw_irq.h>
20 #include <asm/ppc-pci.h>
22 #include <asm/fsl_hcalls.h>
27 #define MSIIR_OFFSET_MASK 0xfffff
28 #define MSIIR_IBS_SHIFT 0
29 #define MSIIR_SRS_SHIFT 5
30 #define MSIIR1_IBS_SHIFT 4
31 #define MSIIR1_SRS_SHIFT 0
32 #define MSI_SRS_MASK 0xf
33 #define MSI_IBS_MASK 0x1f
35 #define msi_hwirq(msi, msir_index, intr_index) \
36 ((msir_index) << (msi)->srs_shift | \
37 ((intr_index) << (msi)->ibs_shift))
39 static LIST_HEAD(msi_head
);
41 struct fsl_msi_feature
{
43 u32 msiir_offset
; /* Offset of MSIIR, relative to start of MSIR bank */
46 struct fsl_msi_cascade_data
{
47 struct fsl_msi
*msi_data
;
52 static inline u32
fsl_msi_read(u32 __iomem
*base
, unsigned int reg
)
54 return in_be32(base
+ (reg
>> 2));
58 * We do not need this actually. The MSIR register has been read once
59 * in the cascade interrupt. So, this MSI interrupt has been acked
61 static void fsl_msi_end_irq(struct irq_data
*d
)
65 static void fsl_msi_print_chip(struct irq_data
*irqd
, struct seq_file
*p
)
67 struct fsl_msi
*msi_data
= irqd
->domain
->host_data
;
68 irq_hw_number_t hwirq
= irqd_to_hwirq(irqd
);
69 int cascade_virq
, srs
;
71 srs
= (hwirq
>> msi_data
->srs_shift
) & MSI_SRS_MASK
;
72 cascade_virq
= msi_data
->cascade_array
[srs
]->virq
;
74 seq_printf(p
, " fsl-msi-%d", cascade_virq
);
78 static struct irq_chip fsl_msi_chip
= {
79 .irq_mask
= pci_msi_mask_irq
,
80 .irq_unmask
= pci_msi_unmask_irq
,
81 .irq_ack
= fsl_msi_end_irq
,
82 .irq_print_chip
= fsl_msi_print_chip
,
85 static int fsl_msi_host_map(struct irq_domain
*h
, unsigned int virq
,
88 struct fsl_msi
*msi_data
= h
->host_data
;
89 struct irq_chip
*chip
= &fsl_msi_chip
;
91 irq_set_status_flags(virq
, IRQ_TYPE_EDGE_FALLING
);
93 irq_set_chip_data(virq
, msi_data
);
94 irq_set_chip_and_handler(virq
, chip
, handle_edge_irq
);
99 static const struct irq_domain_ops fsl_msi_host_ops
= {
100 .map
= fsl_msi_host_map
,
103 static int fsl_msi_init_allocator(struct fsl_msi
*msi_data
)
107 rc
= msi_bitmap_alloc(&msi_data
->bitmap
, NR_MSI_IRQS_MAX
,
108 irq_domain_get_of_node(msi_data
->irqhost
));
113 * Reserve all the hwirqs
114 * The available hwirqs will be released in fsl_msi_setup_hwirq()
116 for (hwirq
= 0; hwirq
< NR_MSI_IRQS_MAX
; hwirq
++)
117 msi_bitmap_reserve_hwirq(&msi_data
->bitmap
, hwirq
);
122 static void fsl_teardown_msi_irqs(struct pci_dev
*pdev
)
124 struct msi_desc
*entry
;
125 struct fsl_msi
*msi_data
;
126 irq_hw_number_t hwirq
;
128 for_each_pci_msi_entry(entry
, pdev
) {
131 hwirq
= virq_to_hw(entry
->irq
);
132 msi_data
= irq_get_chip_data(entry
->irq
);
133 irq_set_msi_desc(entry
->irq
, NULL
);
134 irq_dispose_mapping(entry
->irq
);
135 msi_bitmap_free_hwirqs(&msi_data
->bitmap
, hwirq
, 1);
141 static void fsl_compose_msi_msg(struct pci_dev
*pdev
, int hwirq
,
143 struct fsl_msi
*fsl_msi_data
)
145 struct fsl_msi
*msi_data
= fsl_msi_data
;
146 struct pci_controller
*hose
= pci_bus_to_host(pdev
->bus
);
147 u64 address
; /* Physical address of the MSIIR */
151 /* If the msi-address-64 property exists, then use it */
152 reg
= of_get_property(hose
->dn
, "msi-address-64", &len
);
153 if (reg
&& (len
== sizeof(u64
)))
154 address
= be64_to_cpup(reg
);
156 address
= fsl_pci_immrbar_base(hose
) + msi_data
->msiir_offset
;
158 msg
->address_lo
= lower_32_bits(address
);
159 msg
->address_hi
= upper_32_bits(address
);
162 * MPIC version 2.0 has erratum PIC1. It causes
163 * that neither MSI nor MSI-X can work fine.
164 * This is a workaround to allow MSI-X to function
165 * properly. It only works for MSI-X, we prevent
166 * MSI on buggy chips in fsl_setup_msi_irqs().
168 if (msi_data
->feature
& MSI_HW_ERRATA_ENDIAN
)
169 msg
->data
= __swab32(hwirq
);
173 pr_debug("%s: allocated srs: %d, ibs: %d\n", __func__
,
174 (hwirq
>> msi_data
->srs_shift
) & MSI_SRS_MASK
,
175 (hwirq
>> msi_data
->ibs_shift
) & MSI_IBS_MASK
);
178 static int fsl_setup_msi_irqs(struct pci_dev
*pdev
, int nvec
, int type
)
180 struct pci_controller
*hose
= pci_bus_to_host(pdev
->bus
);
181 struct device_node
*np
;
183 int rc
, hwirq
= -ENOMEM
;
185 struct msi_desc
*entry
;
187 struct fsl_msi
*msi_data
;
189 if (type
== PCI_CAP_ID_MSI
) {
191 * MPIC version 2.0 has erratum PIC1. For now MSI
192 * could not work. So check to prevent MSI from
193 * being used on the board with this erratum.
195 list_for_each_entry(msi_data
, &msi_head
, list
)
196 if (msi_data
->feature
& MSI_HW_ERRATA_ENDIAN
)
201 * If the PCI node has an fsl,msi property, then we need to use it
202 * to find the specific MSI.
204 np
= of_parse_phandle(hose
->dn
, "fsl,msi", 0);
206 if (of_device_is_compatible(np
, "fsl,mpic-msi") ||
207 of_device_is_compatible(np
, "fsl,vmpic-msi") ||
208 of_device_is_compatible(np
, "fsl,vmpic-msi-v4.3"))
209 phandle
= np
->phandle
;
212 "node %pOF has an invalid fsl,msi phandle %u\n",
213 hose
->dn
, np
->phandle
);
218 for_each_pci_msi_entry(entry
, pdev
) {
220 * Loop over all the MSI devices until we find one that has an
221 * available interrupt.
223 list_for_each_entry(msi_data
, &msi_head
, list
) {
225 * If the PCI node has an fsl,msi property, then we
226 * restrict our search to the corresponding MSI node.
227 * The simplest way is to skip over MSI nodes with the
228 * wrong phandle. Under the Freescale hypervisor, this
229 * has the additional benefit of skipping over MSI
230 * nodes that are not mapped in the PAMU.
232 if (phandle
&& (phandle
!= msi_data
->phandle
))
235 hwirq
= msi_bitmap_alloc_hwirqs(&msi_data
->bitmap
, 1);
242 dev_err(&pdev
->dev
, "could not allocate MSI interrupt\n");
246 virq
= irq_create_mapping(msi_data
->irqhost
, hwirq
);
249 dev_err(&pdev
->dev
, "fail mapping hwirq %i\n", hwirq
);
250 msi_bitmap_free_hwirqs(&msi_data
->bitmap
, hwirq
, 1);
254 /* chip_data is msi_data via host->hostdata in host->map() */
255 irq_set_msi_desc(virq
, entry
);
257 fsl_compose_msi_msg(pdev
, hwirq
, &msg
, msi_data
);
258 pci_write_msi_msg(virq
, &msg
);
263 /* free by the caller of this function */
267 static irqreturn_t
fsl_msi_cascade(int irq
, void *data
)
269 unsigned int cascade_irq
;
270 struct fsl_msi
*msi_data
;
275 struct fsl_msi_cascade_data
*cascade_data
= data
;
276 irqreturn_t ret
= IRQ_NONE
;
278 msi_data
= cascade_data
->msi_data
;
280 msir_index
= cascade_data
->index
;
282 if (msir_index
>= NR_MSI_REG_MAX
)
285 switch (msi_data
->feature
& FSL_PIC_IP_MASK
) {
286 case FSL_PIC_IP_MPIC
:
287 msir_value
= fsl_msi_read(msi_data
->msi_regs
,
290 case FSL_PIC_IP_IPIC
:
291 msir_value
= fsl_msi_read(msi_data
->msi_regs
, msir_index
* 0x4);
293 #ifdef CONFIG_EPAPR_PARAVIRT
294 case FSL_PIC_IP_VMPIC
: {
296 ret
= fh_vmpic_get_msir(virq_to_hw(irq
), &msir_value
);
298 pr_err("fsl-msi: fh_vmpic_get_msir() failed for "
299 "irq %u (ret=%u)\n", irq
, ret
);
308 intr_index
= ffs(msir_value
) - 1;
310 cascade_irq
= irq_linear_revmap(msi_data
->irqhost
,
311 msi_hwirq(msi_data
, msir_index
,
312 intr_index
+ have_shift
));
314 generic_handle_irq(cascade_irq
);
317 have_shift
+= intr_index
+ 1;
318 msir_value
= msir_value
>> (intr_index
+ 1);
324 static int fsl_of_msi_remove(struct platform_device
*ofdev
)
326 struct fsl_msi
*msi
= platform_get_drvdata(ofdev
);
329 if (msi
->list
.prev
!= NULL
)
330 list_del(&msi
->list
);
331 for (i
= 0; i
< NR_MSI_REG_MAX
; i
++) {
332 if (msi
->cascade_array
[i
]) {
333 virq
= msi
->cascade_array
[i
]->virq
;
337 free_irq(virq
, msi
->cascade_array
[i
]);
338 kfree(msi
->cascade_array
[i
]);
339 irq_dispose_mapping(virq
);
342 if (msi
->bitmap
.bitmap
)
343 msi_bitmap_free(&msi
->bitmap
);
344 if ((msi
->feature
& FSL_PIC_IP_MASK
) != FSL_PIC_IP_VMPIC
)
345 iounmap(msi
->msi_regs
);
351 static struct lock_class_key fsl_msi_irq_class
;
352 static struct lock_class_key fsl_msi_irq_request_class
;
354 static int fsl_msi_setup_hwirq(struct fsl_msi
*msi
, struct platform_device
*dev
,
355 int offset
, int irq_index
)
357 struct fsl_msi_cascade_data
*cascade_data
= NULL
;
358 int virt_msir
, i
, ret
;
360 virt_msir
= irq_of_parse_and_map(dev
->dev
.of_node
, irq_index
);
362 dev_err(&dev
->dev
, "%s: Cannot translate IRQ index %d\n",
363 __func__
, irq_index
);
367 cascade_data
= kzalloc(sizeof(struct fsl_msi_cascade_data
), GFP_KERNEL
);
369 dev_err(&dev
->dev
, "No memory for MSI cascade data\n");
372 irq_set_lockdep_class(virt_msir
, &fsl_msi_irq_class
,
373 &fsl_msi_irq_request_class
);
374 cascade_data
->index
= offset
;
375 cascade_data
->msi_data
= msi
;
376 cascade_data
->virq
= virt_msir
;
377 msi
->cascade_array
[irq_index
] = cascade_data
;
379 ret
= request_irq(virt_msir
, fsl_msi_cascade
, IRQF_NO_THREAD
,
380 "fsl-msi-cascade", cascade_data
);
382 dev_err(&dev
->dev
, "failed to request_irq(%d), ret = %d\n",
387 /* Release the hwirqs corresponding to this MSI register */
388 for (i
= 0; i
< IRQS_PER_MSI_REG
; i
++)
389 msi_bitmap_free_hwirqs(&msi
->bitmap
,
390 msi_hwirq(msi
, offset
, i
), 1);
395 static const struct of_device_id fsl_of_msi_ids
[];
396 static int fsl_of_msi_probe(struct platform_device
*dev
)
398 const struct of_device_id
*match
;
400 struct resource res
, msiir
;
401 int err
, i
, j
, irq_index
, count
;
403 const struct fsl_msi_feature
*features
;
406 struct pci_controller
*phb
;
408 match
= of_match_device(fsl_of_msi_ids
, &dev
->dev
);
411 features
= match
->data
;
413 printk(KERN_DEBUG
"Setting up Freescale MSI support\n");
415 msi
= kzalloc(sizeof(struct fsl_msi
), GFP_KERNEL
);
417 dev_err(&dev
->dev
, "No memory for MSI structure\n");
420 platform_set_drvdata(dev
, msi
);
422 msi
->irqhost
= irq_domain_add_linear(dev
->dev
.of_node
,
423 NR_MSI_IRQS_MAX
, &fsl_msi_host_ops
, msi
);
425 if (msi
->irqhost
== NULL
) {
426 dev_err(&dev
->dev
, "No memory for MSI irqhost\n");
432 * Under the Freescale hypervisor, the msi nodes don't have a 'reg'
433 * property. Instead, we use hypercalls to access the MSI.
435 if ((features
->fsl_pic_ip
& FSL_PIC_IP_MASK
) != FSL_PIC_IP_VMPIC
) {
436 err
= of_address_to_resource(dev
->dev
.of_node
, 0, &res
);
438 dev_err(&dev
->dev
, "invalid resource for node %pOF\n",
443 msi
->msi_regs
= ioremap(res
.start
, resource_size(&res
));
444 if (!msi
->msi_regs
) {
446 dev_err(&dev
->dev
, "could not map node %pOF\n",
451 features
->msiir_offset
+ (res
.start
& 0xfffff);
454 * First read the MSIIR/MSIIR1 offset from dts
455 * On failure use the hardcode MSIIR offset
457 if (of_address_to_resource(dev
->dev
.of_node
, 1, &msiir
))
458 msi
->msiir_offset
= features
->msiir_offset
+
459 (res
.start
& MSIIR_OFFSET_MASK
);
461 msi
->msiir_offset
= msiir
.start
& MSIIR_OFFSET_MASK
;
464 msi
->feature
= features
->fsl_pic_ip
;
466 /* For erratum PIC1 on MPIC version 2.0*/
467 if ((features
->fsl_pic_ip
& FSL_PIC_IP_MASK
) == FSL_PIC_IP_MPIC
468 && (fsl_mpic_primary_get_version() == 0x0200))
469 msi
->feature
|= MSI_HW_ERRATA_ENDIAN
;
472 * Remember the phandle, so that we can match with any PCI nodes
473 * that have an "fsl,msi" property.
475 msi
->phandle
= dev
->dev
.of_node
->phandle
;
477 err
= fsl_msi_init_allocator(msi
);
479 dev_err(&dev
->dev
, "Error allocating MSI bitmap\n");
483 p
= of_get_property(dev
->dev
.of_node
, "msi-available-ranges", &len
);
485 if (of_device_is_compatible(dev
->dev
.of_node
, "fsl,mpic-msi-v4.3") ||
486 of_device_is_compatible(dev
->dev
.of_node
, "fsl,vmpic-msi-v4.3")) {
487 msi
->srs_shift
= MSIIR1_SRS_SHIFT
;
488 msi
->ibs_shift
= MSIIR1_IBS_SHIFT
;
490 dev_warn(&dev
->dev
, "%s: dose not support msi-available-ranges property\n",
493 for (irq_index
= 0; irq_index
< NR_MSI_REG_MSIIR1
;
495 err
= fsl_msi_setup_hwirq(msi
, dev
,
496 irq_index
, irq_index
);
501 static const u32 all_avail
[] =
502 { 0, NR_MSI_REG_MSIIR
* IRQS_PER_MSI_REG
};
504 msi
->srs_shift
= MSIIR_SRS_SHIFT
;
505 msi
->ibs_shift
= MSIIR_IBS_SHIFT
;
507 if (p
&& len
% (2 * sizeof(u32
)) != 0) {
508 dev_err(&dev
->dev
, "%s: Malformed msi-available-ranges property\n",
516 len
= sizeof(all_avail
);
519 for (irq_index
= 0, i
= 0; i
< len
/ (2 * sizeof(u32
)); i
++) {
520 if (p
[i
* 2] % IRQS_PER_MSI_REG
||
521 p
[i
* 2 + 1] % IRQS_PER_MSI_REG
) {
522 pr_warn("%s: %pOF: msi available range of %u at %u is not IRQ-aligned\n",
523 __func__
, dev
->dev
.of_node
,
524 p
[i
* 2 + 1], p
[i
* 2]);
529 offset
= p
[i
* 2] / IRQS_PER_MSI_REG
;
530 count
= p
[i
* 2 + 1] / IRQS_PER_MSI_REG
;
532 for (j
= 0; j
< count
; j
++, irq_index
++) {
533 err
= fsl_msi_setup_hwirq(msi
, dev
, offset
+ j
,
541 list_add_tail(&msi
->list
, &msi_head
);
544 * Apply the MSI ops to all the controllers.
545 * It doesn't hurt to reassign the same ops,
546 * but bail out if we find another MSI driver.
548 list_for_each_entry(phb
, &hose_list
, list_node
) {
549 if (!phb
->controller_ops
.setup_msi_irqs
) {
550 phb
->controller_ops
.setup_msi_irqs
= fsl_setup_msi_irqs
;
551 phb
->controller_ops
.teardown_msi_irqs
= fsl_teardown_msi_irqs
;
552 } else if (phb
->controller_ops
.setup_msi_irqs
!= fsl_setup_msi_irqs
) {
553 dev_err(&dev
->dev
, "Different MSI driver already installed!\n");
560 fsl_of_msi_remove(dev
);
564 static const struct fsl_msi_feature mpic_msi_feature
= {
565 .fsl_pic_ip
= FSL_PIC_IP_MPIC
,
566 .msiir_offset
= 0x140,
569 static const struct fsl_msi_feature ipic_msi_feature
= {
570 .fsl_pic_ip
= FSL_PIC_IP_IPIC
,
571 .msiir_offset
= 0x38,
574 static const struct fsl_msi_feature vmpic_msi_feature
= {
575 .fsl_pic_ip
= FSL_PIC_IP_VMPIC
,
579 static const struct of_device_id fsl_of_msi_ids
[] = {
581 .compatible
= "fsl,mpic-msi",
582 .data
= &mpic_msi_feature
,
585 .compatible
= "fsl,mpic-msi-v4.3",
586 .data
= &mpic_msi_feature
,
589 .compatible
= "fsl,ipic-msi",
590 .data
= &ipic_msi_feature
,
592 #ifdef CONFIG_EPAPR_PARAVIRT
594 .compatible
= "fsl,vmpic-msi",
595 .data
= &vmpic_msi_feature
,
598 .compatible
= "fsl,vmpic-msi-v4.3",
599 .data
= &vmpic_msi_feature
,
605 static struct platform_driver fsl_of_msi_driver
= {
608 .of_match_table
= fsl_of_msi_ids
,
610 .probe
= fsl_of_msi_probe
,
611 .remove
= fsl_of_msi_remove
,
614 static __init
int fsl_of_msi_init(void)
616 return platform_driver_register(&fsl_of_msi_driver
);
619 subsys_initcall(fsl_of_msi_init
);