1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) 2017 Cadence
3 // Cadence PCIe endpoint controller driver.
4 // Author: Cyrille Pitchen <cyrille.pitchen@free-electrons.com>
6 #include <linux/delay.h>
7 #include <linux/kernel.h>
9 #include <linux/pci-epc.h>
10 #include <linux/platform_device.h>
11 #include <linux/pm_runtime.h>
12 #include <linux/sizes.h>
14 #include "pcie-cadence.h"
16 #define CDNS_PCIE_EP_MIN_APERTURE 128 /* 128 bytes */
17 #define CDNS_PCIE_EP_IRQ_PCI_ADDR_NONE 0x1
18 #define CDNS_PCIE_EP_IRQ_PCI_ADDR_LEGACY 0x3
21 * struct cdns_pcie_ep - private data for this PCIe endpoint controller driver
22 * @pcie: Cadence PCIe controller
23 * @max_regions: maximum number of regions supported by hardware
24 * @ob_region_map: bitmask of mapped outbound regions
25 * @ob_addr: base addresses in the AXI bus where the outbound regions start
26 * @irq_phys_addr: base address on the AXI bus where the MSI/legacy IRQ
27 * dedicated outbound regions is mapped.
28 * @irq_cpu_addr: base address in the CPU space where a write access triggers
29 * the sending of a memory write (MSI) / normal message (legacy
30 * IRQ) TLP through the PCIe bus.
31 * @irq_pci_addr: used to save the current mapping of the MSI/legacy IRQ
32 * dedicated outbound region.
33 * @irq_pci_fn: the latest PCI function that has updated the mapping of
34 * the MSI/legacy IRQ dedicated outbound region.
35 * @irq_pending: bitmask of asserted legacy IRQs.
38 struct cdns_pcie pcie
;
40 unsigned long ob_region_map
;
42 phys_addr_t irq_phys_addr
;
43 void __iomem
*irq_cpu_addr
;
49 static int cdns_pcie_ep_write_header(struct pci_epc
*epc
, u8 fn
,
50 struct pci_epf_header
*hdr
)
52 struct cdns_pcie_ep
*ep
= epc_get_drvdata(epc
);
53 struct cdns_pcie
*pcie
= &ep
->pcie
;
55 cdns_pcie_ep_fn_writew(pcie
, fn
, PCI_DEVICE_ID
, hdr
->deviceid
);
56 cdns_pcie_ep_fn_writeb(pcie
, fn
, PCI_REVISION_ID
, hdr
->revid
);
57 cdns_pcie_ep_fn_writeb(pcie
, fn
, PCI_CLASS_PROG
, hdr
->progif_code
);
58 cdns_pcie_ep_fn_writew(pcie
, fn
, PCI_CLASS_DEVICE
,
59 hdr
->subclass_code
| hdr
->baseclass_code
<< 8);
60 cdns_pcie_ep_fn_writeb(pcie
, fn
, PCI_CACHE_LINE_SIZE
,
61 hdr
->cache_line_size
);
62 cdns_pcie_ep_fn_writew(pcie
, fn
, PCI_SUBSYSTEM_ID
, hdr
->subsys_id
);
63 cdns_pcie_ep_fn_writeb(pcie
, fn
, PCI_INTERRUPT_PIN
, hdr
->interrupt_pin
);
66 * Vendor ID can only be modified from function 0, all other functions
67 * use the same vendor ID as function 0.
70 /* Update the vendor IDs. */
71 u32 id
= CDNS_PCIE_LM_ID_VENDOR(hdr
->vendorid
) |
72 CDNS_PCIE_LM_ID_SUBSYS(hdr
->subsys_vendor_id
);
74 cdns_pcie_writel(pcie
, CDNS_PCIE_LM_ID
, id
);
80 static int cdns_pcie_ep_set_bar(struct pci_epc
*epc
, u8 fn
,
81 struct pci_epf_bar
*epf_bar
)
83 struct cdns_pcie_ep
*ep
= epc_get_drvdata(epc
);
84 struct cdns_pcie
*pcie
= &ep
->pcie
;
85 dma_addr_t bar_phys
= epf_bar
->phys_addr
;
86 enum pci_barno bar
= epf_bar
->barno
;
87 int flags
= epf_bar
->flags
;
88 u32 addr0
, addr1
, reg
, cfg
, b
, aperture
, ctrl
;
91 /* BAR size is 2^(aperture + 7) */
92 sz
= max_t(size_t, epf_bar
->size
, CDNS_PCIE_EP_MIN_APERTURE
);
94 * roundup_pow_of_two() returns an unsigned long, which is not suited
97 sz
= 1ULL << fls64(sz
- 1);
98 aperture
= ilog2(sz
) - 7; /* 128B -> 0, 256B -> 1, 512B -> 2, ... */
100 if ((flags
& PCI_BASE_ADDRESS_SPACE
) == PCI_BASE_ADDRESS_SPACE_IO
) {
101 ctrl
= CDNS_PCIE_LM_BAR_CFG_CTRL_IO_32BITS
;
103 bool is_prefetch
= !!(flags
& PCI_BASE_ADDRESS_MEM_PREFETCH
);
104 bool is_64bits
= sz
> SZ_2G
;
106 if (is_64bits
&& (bar
& 1))
109 if (is_64bits
&& !(flags
& PCI_BASE_ADDRESS_MEM_TYPE_64
))
110 epf_bar
->flags
|= PCI_BASE_ADDRESS_MEM_TYPE_64
;
112 if (is_64bits
&& is_prefetch
)
113 ctrl
= CDNS_PCIE_LM_BAR_CFG_CTRL_PREFETCH_MEM_64BITS
;
114 else if (is_prefetch
)
115 ctrl
= CDNS_PCIE_LM_BAR_CFG_CTRL_PREFETCH_MEM_32BITS
;
117 ctrl
= CDNS_PCIE_LM_BAR_CFG_CTRL_MEM_64BITS
;
119 ctrl
= CDNS_PCIE_LM_BAR_CFG_CTRL_MEM_32BITS
;
122 addr0
= lower_32_bits(bar_phys
);
123 addr1
= upper_32_bits(bar_phys
);
124 cdns_pcie_writel(pcie
, CDNS_PCIE_AT_IB_EP_FUNC_BAR_ADDR0(fn
, bar
),
126 cdns_pcie_writel(pcie
, CDNS_PCIE_AT_IB_EP_FUNC_BAR_ADDR1(fn
, bar
),
130 reg
= CDNS_PCIE_LM_EP_FUNC_BAR_CFG0(fn
);
133 reg
= CDNS_PCIE_LM_EP_FUNC_BAR_CFG1(fn
);
137 cfg
= cdns_pcie_readl(pcie
, reg
);
138 cfg
&= ~(CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b
) |
139 CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL_MASK(b
));
140 cfg
|= (CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_APERTURE(b
, aperture
) |
141 CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL(b
, ctrl
));
142 cdns_pcie_writel(pcie
, reg
, cfg
);
147 static void cdns_pcie_ep_clear_bar(struct pci_epc
*epc
, u8 fn
,
148 struct pci_epf_bar
*epf_bar
)
150 struct cdns_pcie_ep
*ep
= epc_get_drvdata(epc
);
151 struct cdns_pcie
*pcie
= &ep
->pcie
;
152 enum pci_barno bar
= epf_bar
->barno
;
153 u32 reg
, cfg
, b
, ctrl
;
156 reg
= CDNS_PCIE_LM_EP_FUNC_BAR_CFG0(fn
);
159 reg
= CDNS_PCIE_LM_EP_FUNC_BAR_CFG1(fn
);
163 ctrl
= CDNS_PCIE_LM_BAR_CFG_CTRL_DISABLED
;
164 cfg
= cdns_pcie_readl(pcie
, reg
);
165 cfg
&= ~(CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b
) |
166 CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL_MASK(b
));
167 cfg
|= CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL(b
, ctrl
);
168 cdns_pcie_writel(pcie
, reg
, cfg
);
170 cdns_pcie_writel(pcie
, CDNS_PCIE_AT_IB_EP_FUNC_BAR_ADDR0(fn
, bar
), 0);
171 cdns_pcie_writel(pcie
, CDNS_PCIE_AT_IB_EP_FUNC_BAR_ADDR1(fn
, bar
), 0);
174 static int cdns_pcie_ep_map_addr(struct pci_epc
*epc
, u8 fn
, phys_addr_t addr
,
175 u64 pci_addr
, size_t size
)
177 struct cdns_pcie_ep
*ep
= epc_get_drvdata(epc
);
178 struct cdns_pcie
*pcie
= &ep
->pcie
;
181 r
= find_first_zero_bit(&ep
->ob_region_map
,
182 sizeof(ep
->ob_region_map
) * BITS_PER_LONG
);
183 if (r
>= ep
->max_regions
- 1) {
184 dev_err(&epc
->dev
, "no free outbound region\n");
188 cdns_pcie_set_outbound_region(pcie
, fn
, r
, false, addr
, pci_addr
, size
);
190 set_bit(r
, &ep
->ob_region_map
);
191 ep
->ob_addr
[r
] = addr
;
196 static void cdns_pcie_ep_unmap_addr(struct pci_epc
*epc
, u8 fn
,
199 struct cdns_pcie_ep
*ep
= epc_get_drvdata(epc
);
200 struct cdns_pcie
*pcie
= &ep
->pcie
;
203 for (r
= 0; r
< ep
->max_regions
- 1; r
++)
204 if (ep
->ob_addr
[r
] == addr
)
207 if (r
== ep
->max_regions
- 1)
210 cdns_pcie_reset_outbound_region(pcie
, r
);
213 clear_bit(r
, &ep
->ob_region_map
);
216 static int cdns_pcie_ep_set_msi(struct pci_epc
*epc
, u8 fn
, u8 mmc
)
218 struct cdns_pcie_ep
*ep
= epc_get_drvdata(epc
);
219 struct cdns_pcie
*pcie
= &ep
->pcie
;
220 u32 cap
= CDNS_PCIE_EP_FUNC_MSI_CAP_OFFSET
;
224 * Set the Multiple Message Capable bitfield into the Message Control
227 flags
= cdns_pcie_ep_fn_readw(pcie
, fn
, cap
+ PCI_MSI_FLAGS
);
228 flags
= (flags
& ~PCI_MSI_FLAGS_QMASK
) | (mmc
<< 1);
229 flags
|= PCI_MSI_FLAGS_64BIT
;
230 flags
&= ~PCI_MSI_FLAGS_MASKBIT
;
231 cdns_pcie_ep_fn_writew(pcie
, fn
, cap
+ PCI_MSI_FLAGS
, flags
);
236 static int cdns_pcie_ep_get_msi(struct pci_epc
*epc
, u8 fn
)
238 struct cdns_pcie_ep
*ep
= epc_get_drvdata(epc
);
239 struct cdns_pcie
*pcie
= &ep
->pcie
;
240 u32 cap
= CDNS_PCIE_EP_FUNC_MSI_CAP_OFFSET
;
243 /* Validate that the MSI feature is actually enabled. */
244 flags
= cdns_pcie_ep_fn_readw(pcie
, fn
, cap
+ PCI_MSI_FLAGS
);
245 if (!(flags
& PCI_MSI_FLAGS_ENABLE
))
249 * Get the Multiple Message Enable bitfield from the Message Control
252 mme
= (flags
& PCI_MSI_FLAGS_QSIZE
) >> 4;
257 static void cdns_pcie_ep_assert_intx(struct cdns_pcie_ep
*ep
, u8 fn
,
258 u8 intx
, bool is_asserted
)
260 struct cdns_pcie
*pcie
= &ep
->pcie
;
267 /* Set the outbound region if needed. */
268 if (unlikely(ep
->irq_pci_addr
!= CDNS_PCIE_EP_IRQ_PCI_ADDR_LEGACY
||
269 ep
->irq_pci_fn
!= fn
)) {
270 /* First region was reserved for IRQ writes. */
271 cdns_pcie_set_outbound_region_for_normal_msg(pcie
, fn
, 0,
273 ep
->irq_pci_addr
= CDNS_PCIE_EP_IRQ_PCI_ADDR_LEGACY
;
278 ep
->irq_pending
|= BIT(intx
);
279 msg_code
= MSG_CODE_ASSERT_INTA
+ intx
;
281 ep
->irq_pending
&= ~BIT(intx
);
282 msg_code
= MSG_CODE_DEASSERT_INTA
+ intx
;
285 status
= cdns_pcie_ep_fn_readw(pcie
, fn
, PCI_STATUS
);
286 if (((status
& PCI_STATUS_INTERRUPT
) != 0) ^ (ep
->irq_pending
!= 0)) {
287 status
^= PCI_STATUS_INTERRUPT
;
288 cdns_pcie_ep_fn_writew(pcie
, fn
, PCI_STATUS
, status
);
291 offset
= CDNS_PCIE_NORMAL_MSG_ROUTING(MSG_ROUTING_LOCAL
) |
292 CDNS_PCIE_NORMAL_MSG_CODE(msg_code
) |
293 CDNS_PCIE_MSG_NO_DATA
;
294 writel(0, ep
->irq_cpu_addr
+ offset
);
297 static int cdns_pcie_ep_send_legacy_irq(struct cdns_pcie_ep
*ep
, u8 fn
, u8 intx
)
301 cmd
= cdns_pcie_ep_fn_readw(&ep
->pcie
, fn
, PCI_COMMAND
);
302 if (cmd
& PCI_COMMAND_INTX_DISABLE
)
305 cdns_pcie_ep_assert_intx(ep
, fn
, intx
, true);
307 * The mdelay() value was taken from dra7xx_pcie_raise_legacy_irq()
308 * from drivers/pci/dwc/pci-dra7xx.c
311 cdns_pcie_ep_assert_intx(ep
, fn
, intx
, false);
315 static int cdns_pcie_ep_send_msi_irq(struct cdns_pcie_ep
*ep
, u8 fn
,
318 struct cdns_pcie
*pcie
= &ep
->pcie
;
319 u32 cap
= CDNS_PCIE_EP_FUNC_MSI_CAP_OFFSET
;
320 u16 flags
, mme
, data
, data_mask
;
322 u64 pci_addr
, pci_addr_mask
= 0xff;
324 /* Check whether the MSI feature has been enabled by the PCI host. */
325 flags
= cdns_pcie_ep_fn_readw(pcie
, fn
, cap
+ PCI_MSI_FLAGS
);
326 if (!(flags
& PCI_MSI_FLAGS_ENABLE
))
329 /* Get the number of enabled MSIs */
330 mme
= (flags
& PCI_MSI_FLAGS_QSIZE
) >> 4;
331 msi_count
= 1 << mme
;
332 if (!interrupt_num
|| interrupt_num
> msi_count
)
335 /* Compute the data value to be written. */
336 data_mask
= msi_count
- 1;
337 data
= cdns_pcie_ep_fn_readw(pcie
, fn
, cap
+ PCI_MSI_DATA_64
);
338 data
= (data
& ~data_mask
) | ((interrupt_num
- 1) & data_mask
);
340 /* Get the PCI address where to write the data into. */
341 pci_addr
= cdns_pcie_ep_fn_readl(pcie
, fn
, cap
+ PCI_MSI_ADDRESS_HI
);
343 pci_addr
|= cdns_pcie_ep_fn_readl(pcie
, fn
, cap
+ PCI_MSI_ADDRESS_LO
);
344 pci_addr
&= GENMASK_ULL(63, 2);
346 /* Set the outbound region if needed. */
347 if (unlikely(ep
->irq_pci_addr
!= (pci_addr
& ~pci_addr_mask
) ||
348 ep
->irq_pci_fn
!= fn
)) {
349 /* First region was reserved for IRQ writes. */
350 cdns_pcie_set_outbound_region(pcie
, fn
, 0,
353 pci_addr
& ~pci_addr_mask
,
355 ep
->irq_pci_addr
= (pci_addr
& ~pci_addr_mask
);
358 writel(data
, ep
->irq_cpu_addr
+ (pci_addr
& pci_addr_mask
));
363 static int cdns_pcie_ep_raise_irq(struct pci_epc
*epc
, u8 fn
,
364 enum pci_epc_irq_type type
,
367 struct cdns_pcie_ep
*ep
= epc_get_drvdata(epc
);
370 case PCI_EPC_IRQ_LEGACY
:
371 return cdns_pcie_ep_send_legacy_irq(ep
, fn
, 0);
373 case PCI_EPC_IRQ_MSI
:
374 return cdns_pcie_ep_send_msi_irq(ep
, fn
, interrupt_num
);
383 static int cdns_pcie_ep_start(struct pci_epc
*epc
)
385 struct cdns_pcie_ep
*ep
= epc_get_drvdata(epc
);
386 struct cdns_pcie
*pcie
= &ep
->pcie
;
391 * BIT(0) is hardwired to 1, hence function 0 is always enabled
392 * and can't be disabled anyway.
395 list_for_each_entry(epf
, &epc
->pci_epf
, list
)
396 cfg
|= BIT(epf
->func_no
);
397 cdns_pcie_writel(pcie
, CDNS_PCIE_LM_EP_FUNC_CFG
, cfg
);
400 * The PCIe links are automatically established by the controller
401 * once for all at powerup: the software can neither start nor stop
402 * those links later at runtime.
404 * Then we only have to notify the EP core that our links are already
405 * established. However we don't call directly pci_epc_linkup() because
406 * we've already locked the epc->lock.
408 list_for_each_entry(epf
, &epc
->pci_epf
, list
)
414 static const struct pci_epc_ops cdns_pcie_epc_ops
= {
415 .write_header
= cdns_pcie_ep_write_header
,
416 .set_bar
= cdns_pcie_ep_set_bar
,
417 .clear_bar
= cdns_pcie_ep_clear_bar
,
418 .map_addr
= cdns_pcie_ep_map_addr
,
419 .unmap_addr
= cdns_pcie_ep_unmap_addr
,
420 .set_msi
= cdns_pcie_ep_set_msi
,
421 .get_msi
= cdns_pcie_ep_get_msi
,
422 .raise_irq
= cdns_pcie_ep_raise_irq
,
423 .start
= cdns_pcie_ep_start
,
426 static const struct of_device_id cdns_pcie_ep_of_match
[] = {
427 { .compatible
= "cdns,cdns-pcie-ep" },
432 static int cdns_pcie_ep_probe(struct platform_device
*pdev
)
434 struct device
*dev
= &pdev
->dev
;
435 struct device_node
*np
= dev
->of_node
;
436 struct cdns_pcie_ep
*ep
;
437 struct cdns_pcie
*pcie
;
439 struct resource
*res
;
443 ep
= devm_kzalloc(dev
, sizeof(*ep
), GFP_KERNEL
);
450 res
= platform_get_resource_byname(pdev
, IORESOURCE_MEM
, "reg");
451 pcie
->reg_base
= devm_ioremap_resource(dev
, res
);
452 if (IS_ERR(pcie
->reg_base
)) {
453 dev_err(dev
, "missing \"reg\"\n");
454 return PTR_ERR(pcie
->reg_base
);
457 res
= platform_get_resource_byname(pdev
, IORESOURCE_MEM
, "mem");
459 dev_err(dev
, "missing \"mem\"\n");
464 ret
= of_property_read_u32(np
, "cdns,max-outbound-regions",
467 dev_err(dev
, "missing \"cdns,max-outbound-regions\"\n");
470 ep
->ob_addr
= devm_kcalloc(dev
,
471 ep
->max_regions
, sizeof(*ep
->ob_addr
),
476 ret
= cdns_pcie_init_phy(dev
, pcie
);
478 dev_err(dev
, "failed to init phy\n");
481 platform_set_drvdata(pdev
, pcie
);
482 pm_runtime_enable(dev
);
483 ret
= pm_runtime_get_sync(dev
);
485 dev_err(dev
, "pm_runtime_get_sync() failed\n");
489 /* Disable all but function 0 (anyway BIT(0) is hardwired to 1). */
490 cdns_pcie_writel(pcie
, CDNS_PCIE_LM_EP_FUNC_CFG
, BIT(0));
492 epc
= devm_pci_epc_create(dev
, &cdns_pcie_epc_ops
);
494 dev_err(dev
, "failed to create epc device\n");
499 epc_set_drvdata(epc
, ep
);
501 if (of_property_read_u8(np
, "max-functions", &epc
->max_functions
) < 0)
502 epc
->max_functions
= 1;
504 ret
= pci_epc_mem_init(epc
, pcie
->mem_res
->start
,
505 resource_size(pcie
->mem_res
));
507 dev_err(dev
, "failed to initialize the memory space\n");
511 ep
->irq_cpu_addr
= pci_epc_mem_alloc_addr(epc
, &ep
->irq_phys_addr
,
513 if (!ep
->irq_cpu_addr
) {
514 dev_err(dev
, "failed to reserve memory space for MSI\n");
518 ep
->irq_pci_addr
= CDNS_PCIE_EP_IRQ_PCI_ADDR_NONE
;
519 /* Reserve region 0 for IRQs */
520 set_bit(0, &ep
->ob_region_map
);
525 pci_epc_mem_exit(epc
);
528 pm_runtime_put_sync(dev
);
531 pm_runtime_disable(dev
);
532 cdns_pcie_disable_phy(pcie
);
533 phy_count
= pcie
->phy_count
;
535 device_link_del(pcie
->link
[phy_count
]);
540 static void cdns_pcie_ep_shutdown(struct platform_device
*pdev
)
542 struct device
*dev
= &pdev
->dev
;
543 struct cdns_pcie
*pcie
= dev_get_drvdata(dev
);
546 ret
= pm_runtime_put_sync(dev
);
548 dev_dbg(dev
, "pm_runtime_put_sync failed\n");
550 pm_runtime_disable(dev
);
552 cdns_pcie_disable_phy(pcie
);
555 static struct platform_driver cdns_pcie_ep_driver
= {
557 .name
= "cdns-pcie-ep",
558 .of_match_table
= cdns_pcie_ep_of_match
,
559 .pm
= &cdns_pcie_pm_ops
,
561 .probe
= cdns_pcie_ep_probe
,
562 .shutdown
= cdns_pcie_ep_shutdown
,
564 builtin_platform_driver(cdns_pcie_ep_driver
);