1 // SPDX-License-Identifier: GPL-2.0+
3 * PCIe host controller driver for Xilinx Versal CPM DMA Bridge
5 * (C) Copyright 2019 - 2020, Xilinx, Inc.
8 #include <linux/bitfield.h>
9 #include <linux/interrupt.h>
10 #include <linux/irq.h>
11 #include <linux/irqchip.h>
12 #include <linux/irqchip/chained_irq.h>
13 #include <linux/irqdomain.h>
14 #include <linux/kernel.h>
15 #include <linux/module.h>
16 #include <linux/of_address.h>
17 #include <linux/of_pci.h>
18 #include <linux/of_platform.h>
19 #include <linux/of_irq.h>
20 #include <linux/pci.h>
21 #include <linux/platform_device.h>
22 #include <linux/pci-ecam.h>
26 /* Register definitions */
27 #define XILINX_CPM_PCIE_REG_IDR 0x00000E10
28 #define XILINX_CPM_PCIE_REG_IMR 0x00000E14
29 #define XILINX_CPM_PCIE_REG_PSCR 0x00000E1C
30 #define XILINX_CPM_PCIE_REG_RPSC 0x00000E20
31 #define XILINX_CPM_PCIE_REG_RPEFR 0x00000E2C
32 #define XILINX_CPM_PCIE_REG_IDRN 0x00000E38
33 #define XILINX_CPM_PCIE_REG_IDRN_MASK 0x00000E3C
34 #define XILINX_CPM_PCIE_MISC_IR_STATUS 0x00000340
35 #define XILINX_CPM_PCIE_MISC_IR_ENABLE 0x00000348
36 #define XILINX_CPM_PCIE_MISC_IR_LOCAL BIT(1)
38 /* Interrupt registers definitions */
39 #define XILINX_CPM_PCIE_INTR_LINK_DOWN 0
40 #define XILINX_CPM_PCIE_INTR_HOT_RESET 3
41 #define XILINX_CPM_PCIE_INTR_CFG_PCIE_TIMEOUT 4
42 #define XILINX_CPM_PCIE_INTR_CFG_TIMEOUT 8
43 #define XILINX_CPM_PCIE_INTR_CORRECTABLE 9
44 #define XILINX_CPM_PCIE_INTR_NONFATAL 10
45 #define XILINX_CPM_PCIE_INTR_FATAL 11
46 #define XILINX_CPM_PCIE_INTR_CFG_ERR_POISON 12
47 #define XILINX_CPM_PCIE_INTR_PME_TO_ACK_RCVD 15
48 #define XILINX_CPM_PCIE_INTR_INTX 16
49 #define XILINX_CPM_PCIE_INTR_PM_PME_RCVD 17
50 #define XILINX_CPM_PCIE_INTR_SLV_UNSUPP 20
51 #define XILINX_CPM_PCIE_INTR_SLV_UNEXP 21
52 #define XILINX_CPM_PCIE_INTR_SLV_COMPL 22
53 #define XILINX_CPM_PCIE_INTR_SLV_ERRP 23
54 #define XILINX_CPM_PCIE_INTR_SLV_CMPABT 24
55 #define XILINX_CPM_PCIE_INTR_SLV_ILLBUR 25
56 #define XILINX_CPM_PCIE_INTR_MST_DECERR 26
57 #define XILINX_CPM_PCIE_INTR_MST_SLVERR 27
58 #define XILINX_CPM_PCIE_INTR_SLV_PCIE_TIMEOUT 28
60 #define IMR(x) BIT(XILINX_CPM_PCIE_INTR_ ##x)
62 #define XILINX_CPM_PCIE_IMR_ALL_MASK \
66 IMR(CFG_PCIE_TIMEOUT) | \
71 IMR(CFG_ERR_POISON) | \
72 IMR(PME_TO_ACK_RCVD) | \
83 IMR(SLV_PCIE_TIMEOUT) \
86 #define XILINX_CPM_PCIE_IDR_ALL_MASK 0xFFFFFFFF
87 #define XILINX_CPM_PCIE_IDRN_MASK GENMASK(19, 16)
88 #define XILINX_CPM_PCIE_IDRN_SHIFT 16
90 /* Root Port Error FIFO Read Register definitions */
91 #define XILINX_CPM_PCIE_RPEFR_ERR_VALID BIT(18)
92 #define XILINX_CPM_PCIE_RPEFR_REQ_ID GENMASK(15, 0)
93 #define XILINX_CPM_PCIE_RPEFR_ALL_MASK 0xFFFFFFFF
95 /* Root Port Status/control Register definitions */
96 #define XILINX_CPM_PCIE_REG_RPSC_BEN BIT(0)
98 /* Phy Status/Control Register definitions */
99 #define XILINX_CPM_PCIE_REG_PSCR_LNKUP BIT(11)
102 * struct xilinx_cpm_pcie_port - PCIe port information
103 * @reg_base: Bridge Register Base
104 * @cpm_base: CPM System Level Control and Status Register(SLCR) Base
105 * @dev: Device pointer
106 * @intx_domain: Legacy IRQ domain pointer
107 * @cpm_domain: CPM IRQ domain pointer
108 * @cfg: Holds mappings of config space window
109 * @intx_irq: legacy interrupt number
110 * @irq: Error interrupt number
111 * @lock: lock protecting shared register access
113 struct xilinx_cpm_pcie_port
{
114 void __iomem
*reg_base
;
115 void __iomem
*cpm_base
;
117 struct irq_domain
*intx_domain
;
118 struct irq_domain
*cpm_domain
;
119 struct pci_config_window
*cfg
;
125 static u32
pcie_read(struct xilinx_cpm_pcie_port
*port
, u32 reg
)
127 return readl_relaxed(port
->reg_base
+ reg
);
130 static void pcie_write(struct xilinx_cpm_pcie_port
*port
,
133 writel_relaxed(val
, port
->reg_base
+ reg
);
136 static bool cpm_pcie_link_up(struct xilinx_cpm_pcie_port
*port
)
138 return (pcie_read(port
, XILINX_CPM_PCIE_REG_PSCR
) &
139 XILINX_CPM_PCIE_REG_PSCR_LNKUP
);
142 static void cpm_pcie_clear_err_interrupts(struct xilinx_cpm_pcie_port
*port
)
144 unsigned long val
= pcie_read(port
, XILINX_CPM_PCIE_REG_RPEFR
);
146 if (val
& XILINX_CPM_PCIE_RPEFR_ERR_VALID
) {
147 dev_dbg(port
->dev
, "Requester ID %lu\n",
148 val
& XILINX_CPM_PCIE_RPEFR_REQ_ID
);
149 pcie_write(port
, XILINX_CPM_PCIE_RPEFR_ALL_MASK
,
150 XILINX_CPM_PCIE_REG_RPEFR
);
154 static void xilinx_cpm_mask_leg_irq(struct irq_data
*data
)
156 struct xilinx_cpm_pcie_port
*port
= irq_data_get_irq_chip_data(data
);
161 mask
= BIT(data
->hwirq
+ XILINX_CPM_PCIE_IDRN_SHIFT
);
162 raw_spin_lock_irqsave(&port
->lock
, flags
);
163 val
= pcie_read(port
, XILINX_CPM_PCIE_REG_IDRN_MASK
);
164 pcie_write(port
, (val
& (~mask
)), XILINX_CPM_PCIE_REG_IDRN_MASK
);
165 raw_spin_unlock_irqrestore(&port
->lock
, flags
);
168 static void xilinx_cpm_unmask_leg_irq(struct irq_data
*data
)
170 struct xilinx_cpm_pcie_port
*port
= irq_data_get_irq_chip_data(data
);
175 mask
= BIT(data
->hwirq
+ XILINX_CPM_PCIE_IDRN_SHIFT
);
176 raw_spin_lock_irqsave(&port
->lock
, flags
);
177 val
= pcie_read(port
, XILINX_CPM_PCIE_REG_IDRN_MASK
);
178 pcie_write(port
, (val
| mask
), XILINX_CPM_PCIE_REG_IDRN_MASK
);
179 raw_spin_unlock_irqrestore(&port
->lock
, flags
);
182 static struct irq_chip xilinx_cpm_leg_irq_chip
= {
184 .irq_mask
= xilinx_cpm_mask_leg_irq
,
185 .irq_unmask
= xilinx_cpm_unmask_leg_irq
,
189 * xilinx_cpm_pcie_intx_map - Set the handler for the INTx and mark IRQ as valid
190 * @domain: IRQ domain
191 * @irq: Virtual IRQ number
192 * @hwirq: HW interrupt number
194 * Return: Always returns 0.
196 static int xilinx_cpm_pcie_intx_map(struct irq_domain
*domain
,
197 unsigned int irq
, irq_hw_number_t hwirq
)
199 irq_set_chip_and_handler(irq
, &xilinx_cpm_leg_irq_chip
,
201 irq_set_chip_data(irq
, domain
->host_data
);
202 irq_set_status_flags(irq
, IRQ_LEVEL
);
207 /* INTx IRQ Domain operations */
208 static const struct irq_domain_ops intx_domain_ops
= {
209 .map
= xilinx_cpm_pcie_intx_map
,
212 static void xilinx_cpm_pcie_intx_flow(struct irq_desc
*desc
)
214 struct xilinx_cpm_pcie_port
*port
= irq_desc_get_handler_data(desc
);
215 struct irq_chip
*chip
= irq_desc_get_chip(desc
);
219 chained_irq_enter(chip
, desc
);
221 val
= FIELD_GET(XILINX_CPM_PCIE_IDRN_MASK
,
222 pcie_read(port
, XILINX_CPM_PCIE_REG_IDRN
));
224 for_each_set_bit(i
, &val
, PCI_NUM_INTX
)
225 generic_handle_irq(irq_find_mapping(port
->intx_domain
, i
));
227 chained_irq_exit(chip
, desc
);
230 static void xilinx_cpm_mask_event_irq(struct irq_data
*d
)
232 struct xilinx_cpm_pcie_port
*port
= irq_data_get_irq_chip_data(d
);
235 raw_spin_lock(&port
->lock
);
236 val
= pcie_read(port
, XILINX_CPM_PCIE_REG_IMR
);
237 val
&= ~BIT(d
->hwirq
);
238 pcie_write(port
, val
, XILINX_CPM_PCIE_REG_IMR
);
239 raw_spin_unlock(&port
->lock
);
242 static void xilinx_cpm_unmask_event_irq(struct irq_data
*d
)
244 struct xilinx_cpm_pcie_port
*port
= irq_data_get_irq_chip_data(d
);
247 raw_spin_lock(&port
->lock
);
248 val
= pcie_read(port
, XILINX_CPM_PCIE_REG_IMR
);
249 val
|= BIT(d
->hwirq
);
250 pcie_write(port
, val
, XILINX_CPM_PCIE_REG_IMR
);
251 raw_spin_unlock(&port
->lock
);
254 static struct irq_chip xilinx_cpm_event_irq_chip
= {
256 .irq_mask
= xilinx_cpm_mask_event_irq
,
257 .irq_unmask
= xilinx_cpm_unmask_event_irq
,
260 static int xilinx_cpm_pcie_event_map(struct irq_domain
*domain
,
261 unsigned int irq
, irq_hw_number_t hwirq
)
263 irq_set_chip_and_handler(irq
, &xilinx_cpm_event_irq_chip
,
265 irq_set_chip_data(irq
, domain
->host_data
);
266 irq_set_status_flags(irq
, IRQ_LEVEL
);
270 static const struct irq_domain_ops event_domain_ops
= {
271 .map
= xilinx_cpm_pcie_event_map
,
274 static void xilinx_cpm_pcie_event_flow(struct irq_desc
*desc
)
276 struct xilinx_cpm_pcie_port
*port
= irq_desc_get_handler_data(desc
);
277 struct irq_chip
*chip
= irq_desc_get_chip(desc
);
281 chained_irq_enter(chip
, desc
);
282 val
= pcie_read(port
, XILINX_CPM_PCIE_REG_IDR
);
283 val
&= pcie_read(port
, XILINX_CPM_PCIE_REG_IMR
);
284 for_each_set_bit(i
, &val
, 32)
285 generic_handle_irq(irq_find_mapping(port
->cpm_domain
, i
));
286 pcie_write(port
, val
, XILINX_CPM_PCIE_REG_IDR
);
289 * XILINX_CPM_PCIE_MISC_IR_STATUS register is mapped to
292 val
= readl_relaxed(port
->cpm_base
+ XILINX_CPM_PCIE_MISC_IR_STATUS
);
295 port
->cpm_base
+ XILINX_CPM_PCIE_MISC_IR_STATUS
);
297 chained_irq_exit(chip
, desc
);
301 [XILINX_CPM_PCIE_INTR_ ## x] = { __stringify(x), s }
303 static const struct {
307 _IC(LINK_DOWN
, "Link Down"),
308 _IC(HOT_RESET
, "Hot reset"),
309 _IC(CFG_TIMEOUT
, "ECAM access timeout"),
310 _IC(CORRECTABLE
, "Correctable error message"),
311 _IC(NONFATAL
, "Non fatal error message"),
312 _IC(FATAL
, "Fatal error message"),
313 _IC(SLV_UNSUPP
, "Slave unsupported request"),
314 _IC(SLV_UNEXP
, "Slave unexpected completion"),
315 _IC(SLV_COMPL
, "Slave completion timeout"),
316 _IC(SLV_ERRP
, "Slave Error Poison"),
317 _IC(SLV_CMPABT
, "Slave Completer Abort"),
318 _IC(SLV_ILLBUR
, "Slave Illegal Burst"),
319 _IC(MST_DECERR
, "Master decode error"),
320 _IC(MST_SLVERR
, "Master slave error"),
321 _IC(CFG_PCIE_TIMEOUT
, "PCIe ECAM access timeout"),
322 _IC(CFG_ERR_POISON
, "ECAM poisoned completion received"),
323 _IC(PME_TO_ACK_RCVD
, "PME_TO_ACK message received"),
324 _IC(PM_PME_RCVD
, "PM_PME message received"),
325 _IC(SLV_PCIE_TIMEOUT
, "PCIe completion timeout received"),
328 static irqreturn_t
xilinx_cpm_pcie_intr_handler(int irq
, void *dev_id
)
330 struct xilinx_cpm_pcie_port
*port
= dev_id
;
331 struct device
*dev
= port
->dev
;
334 d
= irq_domain_get_irq_data(port
->cpm_domain
, irq
);
337 case XILINX_CPM_PCIE_INTR_CORRECTABLE
:
338 case XILINX_CPM_PCIE_INTR_NONFATAL
:
339 case XILINX_CPM_PCIE_INTR_FATAL
:
340 cpm_pcie_clear_err_interrupts(port
);
344 if (intr_cause
[d
->hwirq
].str
)
345 dev_warn(dev
, "%s\n", intr_cause
[d
->hwirq
].str
);
347 dev_warn(dev
, "Unknown IRQ %ld\n", d
->hwirq
);
353 static void xilinx_cpm_free_irq_domains(struct xilinx_cpm_pcie_port
*port
)
355 if (port
->intx_domain
) {
356 irq_domain_remove(port
->intx_domain
);
357 port
->intx_domain
= NULL
;
360 if (port
->cpm_domain
) {
361 irq_domain_remove(port
->cpm_domain
);
362 port
->cpm_domain
= NULL
;
367 * xilinx_cpm_pcie_init_irq_domain - Initialize IRQ domain
368 * @port: PCIe port information
370 * Return: '0' on success and error value on failure
372 static int xilinx_cpm_pcie_init_irq_domain(struct xilinx_cpm_pcie_port
*port
)
374 struct device
*dev
= port
->dev
;
375 struct device_node
*node
= dev
->of_node
;
376 struct device_node
*pcie_intc_node
;
379 pcie_intc_node
= of_get_next_child(node
, NULL
);
380 if (!pcie_intc_node
) {
381 dev_err(dev
, "No PCIe Intc node found\n");
385 port
->cpm_domain
= irq_domain_add_linear(pcie_intc_node
, 32,
388 if (!port
->cpm_domain
)
391 irq_domain_update_bus_token(port
->cpm_domain
, DOMAIN_BUS_NEXUS
);
393 port
->intx_domain
= irq_domain_add_linear(pcie_intc_node
, PCI_NUM_INTX
,
396 if (!port
->intx_domain
)
399 irq_domain_update_bus_token(port
->intx_domain
, DOMAIN_BUS_WIRED
);
401 of_node_put(pcie_intc_node
);
402 raw_spin_lock_init(&port
->lock
);
406 xilinx_cpm_free_irq_domains(port
);
407 dev_err(dev
, "Failed to allocate IRQ domains\n");
412 static int xilinx_cpm_setup_irq(struct xilinx_cpm_pcie_port
*port
)
414 struct device
*dev
= port
->dev
;
415 struct platform_device
*pdev
= to_platform_device(dev
);
418 port
->irq
= platform_get_irq(pdev
, 0);
422 for (i
= 0; i
< ARRAY_SIZE(intr_cause
); i
++) {
425 if (!intr_cause
[i
].str
)
428 irq
= irq_create_mapping(port
->cpm_domain
, i
);
430 dev_err(dev
, "Failed to map interrupt\n");
434 err
= devm_request_irq(dev
, irq
, xilinx_cpm_pcie_intr_handler
,
435 0, intr_cause
[i
].sym
, port
);
437 dev_err(dev
, "Failed to request IRQ %d\n", irq
);
442 port
->intx_irq
= irq_create_mapping(port
->cpm_domain
,
443 XILINX_CPM_PCIE_INTR_INTX
);
444 if (!port
->intx_irq
) {
445 dev_err(dev
, "Failed to map INTx interrupt\n");
449 /* Plug the INTx chained handler */
450 irq_set_chained_handler_and_data(port
->intx_irq
,
451 xilinx_cpm_pcie_intx_flow
, port
);
453 /* Plug the main event chained handler */
454 irq_set_chained_handler_and_data(port
->irq
,
455 xilinx_cpm_pcie_event_flow
, port
);
461 * xilinx_cpm_pcie_init_port - Initialize hardware
462 * @port: PCIe port information
464 static void xilinx_cpm_pcie_init_port(struct xilinx_cpm_pcie_port
*port
)
466 if (cpm_pcie_link_up(port
))
467 dev_info(port
->dev
, "PCIe Link is UP\n");
469 dev_info(port
->dev
, "PCIe Link is DOWN\n");
471 /* Disable all interrupts */
472 pcie_write(port
, ~XILINX_CPM_PCIE_IDR_ALL_MASK
,
473 XILINX_CPM_PCIE_REG_IMR
);
475 /* Clear pending interrupts */
476 pcie_write(port
, pcie_read(port
, XILINX_CPM_PCIE_REG_IDR
) &
477 XILINX_CPM_PCIE_IMR_ALL_MASK
,
478 XILINX_CPM_PCIE_REG_IDR
);
481 * XILINX_CPM_PCIE_MISC_IR_ENABLE register is mapped to
484 writel(XILINX_CPM_PCIE_MISC_IR_LOCAL
,
485 port
->cpm_base
+ XILINX_CPM_PCIE_MISC_IR_ENABLE
);
486 /* Enable the Bridge enable bit */
487 pcie_write(port
, pcie_read(port
, XILINX_CPM_PCIE_REG_RPSC
) |
488 XILINX_CPM_PCIE_REG_RPSC_BEN
,
489 XILINX_CPM_PCIE_REG_RPSC
);
493 * xilinx_cpm_pcie_parse_dt - Parse Device tree
494 * @port: PCIe port information
495 * @bus_range: Bus resource
497 * Return: '0' on success and error value on failure
499 static int xilinx_cpm_pcie_parse_dt(struct xilinx_cpm_pcie_port
*port
,
500 struct resource
*bus_range
)
502 struct device
*dev
= port
->dev
;
503 struct platform_device
*pdev
= to_platform_device(dev
);
504 struct resource
*res
;
506 port
->cpm_base
= devm_platform_ioremap_resource_byname(pdev
,
508 if (IS_ERR(port
->cpm_base
))
509 return PTR_ERR(port
->cpm_base
);
511 res
= platform_get_resource_byname(pdev
, IORESOURCE_MEM
, "cfg");
515 port
->cfg
= pci_ecam_create(dev
, res
, bus_range
,
516 &pci_generic_ecam_ops
);
517 if (IS_ERR(port
->cfg
))
518 return PTR_ERR(port
->cfg
);
520 port
->reg_base
= port
->cfg
->win
;
525 static void xilinx_cpm_free_interrupts(struct xilinx_cpm_pcie_port
*port
)
527 irq_set_chained_handler_and_data(port
->intx_irq
, NULL
, NULL
);
528 irq_set_chained_handler_and_data(port
->irq
, NULL
, NULL
);
532 * xilinx_cpm_pcie_probe - Probe function
533 * @pdev: Platform device pointer
535 * Return: '0' on success and error value on failure
537 static int xilinx_cpm_pcie_probe(struct platform_device
*pdev
)
539 struct xilinx_cpm_pcie_port
*port
;
540 struct device
*dev
= &pdev
->dev
;
541 struct pci_host_bridge
*bridge
;
542 struct resource_entry
*bus
;
545 bridge
= devm_pci_alloc_host_bridge(dev
, sizeof(*port
));
549 port
= pci_host_bridge_priv(bridge
);
553 err
= xilinx_cpm_pcie_init_irq_domain(port
);
557 bus
= resource_list_first_type(&bridge
->windows
, IORESOURCE_BUS
);
561 err
= xilinx_cpm_pcie_parse_dt(port
, bus
->res
);
563 dev_err(dev
, "Parsing DT failed\n");
567 xilinx_cpm_pcie_init_port(port
);
569 err
= xilinx_cpm_setup_irq(port
);
571 dev_err(dev
, "Failed to set up interrupts\n");
575 bridge
->sysdata
= port
->cfg
;
576 bridge
->ops
= (struct pci_ops
*)&pci_generic_ecam_ops
.pci_ops
;
578 err
= pci_host_probe(bridge
);
580 goto err_host_bridge
;
585 xilinx_cpm_free_interrupts(port
);
587 pci_ecam_free(port
->cfg
);
589 xilinx_cpm_free_irq_domains(port
);
593 static const struct of_device_id xilinx_cpm_pcie_of_match
[] = {
594 { .compatible
= "xlnx,versal-cpm-host-1.00", },
598 static struct platform_driver xilinx_cpm_pcie_driver
= {
600 .name
= "xilinx-cpm-pcie",
601 .of_match_table
= xilinx_cpm_pcie_of_match
,
602 .suppress_bind_attrs
= true,
604 .probe
= xilinx_cpm_pcie_probe
,
607 builtin_platform_driver(xilinx_cpm_pcie_driver
);