1 // SPDX-License-Identifier: GPL-2.0+
3 * Rockchip AXI PCIe host controller driver
5 * Copyright (c) 2016 Rockchip, Inc.
7 * Author: Shawn Lin <shawn.lin@rock-chips.com>
8 * Wenrui Li <wenrui.li@rock-chips.com>
10 * Bits taken from Synopsys DesignWare Host controller driver and
11 * ARM PCI Host generic driver.
14 #include <linux/bitrev.h>
15 #include <linux/clk.h>
16 #include <linux/delay.h>
17 #include <linux/gpio/consumer.h>
18 #include <linux/init.h>
19 #include <linux/interrupt.h>
20 #include <linux/iopoll.h>
21 #include <linux/irq.h>
22 #include <linux/irqchip/chained_irq.h>
23 #include <linux/irqdomain.h>
24 #include <linux/kernel.h>
25 #include <linux/mfd/syscon.h>
26 #include <linux/module.h>
27 #include <linux/of_address.h>
28 #include <linux/of_device.h>
29 #include <linux/of_pci.h>
30 #include <linux/of_platform.h>
31 #include <linux/of_irq.h>
32 #include <linux/pci.h>
33 #include <linux/pci_ids.h>
34 #include <linux/phy/phy.h>
35 #include <linux/platform_device.h>
36 #include <linux/reset.h>
37 #include <linux/regmap.h>
40 #include "pcie-rockchip.h"
42 static void rockchip_pcie_enable_bw_int(struct rockchip_pcie
*rockchip
)
46 status
= rockchip_pcie_read(rockchip
, PCIE_RC_CONFIG_LCS
);
47 status
|= (PCI_EXP_LNKCTL_LBMIE
| PCI_EXP_LNKCTL_LABIE
);
48 rockchip_pcie_write(rockchip
, status
, PCIE_RC_CONFIG_LCS
);
51 static void rockchip_pcie_clr_bw_int(struct rockchip_pcie
*rockchip
)
55 status
= rockchip_pcie_read(rockchip
, PCIE_RC_CONFIG_LCS
);
56 status
|= (PCI_EXP_LNKSTA_LBMS
| PCI_EXP_LNKSTA_LABS
) << 16;
57 rockchip_pcie_write(rockchip
, status
, PCIE_RC_CONFIG_LCS
);
60 static void rockchip_pcie_update_txcredit_mui(struct rockchip_pcie
*rockchip
)
64 /* Update Tx credit maximum update interval */
65 val
= rockchip_pcie_read(rockchip
, PCIE_CORE_TXCREDIT_CFG1
);
66 val
&= ~PCIE_CORE_TXCREDIT_CFG1_MUI_MASK
;
67 val
|= PCIE_CORE_TXCREDIT_CFG1_MUI_ENCODE(24000); /* ns */
68 rockchip_pcie_write(rockchip
, val
, PCIE_CORE_TXCREDIT_CFG1
);
71 static int rockchip_pcie_valid_device(struct rockchip_pcie
*rockchip
,
72 struct pci_bus
*bus
, int dev
)
75 * Access only one slot on each root port.
76 * Do not read more than one device on the bus directly attached
77 * to RC's downstream side.
79 if (pci_is_root_bus(bus
) || pci_is_root_bus(bus
->parent
))
85 static u8
rockchip_pcie_lane_map(struct rockchip_pcie
*rockchip
)
90 if (rockchip
->legacy_phy
)
91 return GENMASK(MAX_LANE_NUM
- 1, 0);
93 val
= rockchip_pcie_read(rockchip
, PCIE_CORE_LANE_MAP
);
94 map
= val
& PCIE_CORE_LANE_MAP_MASK
;
96 /* The link may be using a reverse-indexed mapping. */
97 if (val
& PCIE_CORE_LANE_MAP_REVERSE
)
98 map
= bitrev8(map
) >> 4;
103 static int rockchip_pcie_rd_own_conf(struct rockchip_pcie
*rockchip
,
104 int where
, int size
, u32
*val
)
108 addr
= rockchip
->apb_base
+ PCIE_RC_CONFIG_NORMAL_BASE
+ where
;
110 if (!IS_ALIGNED((uintptr_t)addr
, size
)) {
112 return PCIBIOS_BAD_REGISTER_NUMBER
;
117 } else if (size
== 2) {
119 } else if (size
== 1) {
123 return PCIBIOS_BAD_REGISTER_NUMBER
;
125 return PCIBIOS_SUCCESSFUL
;
128 static int rockchip_pcie_wr_own_conf(struct rockchip_pcie
*rockchip
,
129 int where
, int size
, u32 val
)
131 u32 mask
, tmp
, offset
;
134 offset
= where
& ~0x3;
135 addr
= rockchip
->apb_base
+ PCIE_RC_CONFIG_NORMAL_BASE
+ offset
;
139 return PCIBIOS_SUCCESSFUL
;
142 mask
= ~(((1 << (size
* 8)) - 1) << ((where
& 0x3) * 8));
145 * N.B. This read/modify/write isn't safe in general because it can
146 * corrupt RW1C bits in adjacent registers. But the hardware
147 * doesn't support smaller writes.
149 tmp
= readl(addr
) & mask
;
150 tmp
|= val
<< ((where
& 0x3) * 8);
153 return PCIBIOS_SUCCESSFUL
;
156 static int rockchip_pcie_rd_other_conf(struct rockchip_pcie
*rockchip
,
157 struct pci_bus
*bus
, u32 devfn
,
158 int where
, int size
, u32
*val
)
162 addr
= rockchip
->reg_base
+ PCIE_ECAM_OFFSET(bus
->number
, devfn
, where
);
164 if (!IS_ALIGNED((uintptr_t)addr
, size
)) {
166 return PCIBIOS_BAD_REGISTER_NUMBER
;
169 if (pci_is_root_bus(bus
->parent
))
170 rockchip_pcie_cfg_configuration_accesses(rockchip
,
171 AXI_WRAPPER_TYPE0_CFG
);
173 rockchip_pcie_cfg_configuration_accesses(rockchip
,
174 AXI_WRAPPER_TYPE1_CFG
);
178 } else if (size
== 2) {
180 } else if (size
== 1) {
184 return PCIBIOS_BAD_REGISTER_NUMBER
;
186 return PCIBIOS_SUCCESSFUL
;
189 static int rockchip_pcie_wr_other_conf(struct rockchip_pcie
*rockchip
,
190 struct pci_bus
*bus
, u32 devfn
,
191 int where
, int size
, u32 val
)
195 addr
= rockchip
->reg_base
+ PCIE_ECAM_OFFSET(bus
->number
, devfn
, where
);
197 if (!IS_ALIGNED((uintptr_t)addr
, size
))
198 return PCIBIOS_BAD_REGISTER_NUMBER
;
200 if (pci_is_root_bus(bus
->parent
))
201 rockchip_pcie_cfg_configuration_accesses(rockchip
,
202 AXI_WRAPPER_TYPE0_CFG
);
204 rockchip_pcie_cfg_configuration_accesses(rockchip
,
205 AXI_WRAPPER_TYPE1_CFG
);
214 return PCIBIOS_BAD_REGISTER_NUMBER
;
216 return PCIBIOS_SUCCESSFUL
;
219 static int rockchip_pcie_rd_conf(struct pci_bus
*bus
, u32 devfn
, int where
,
222 struct rockchip_pcie
*rockchip
= bus
->sysdata
;
224 if (!rockchip_pcie_valid_device(rockchip
, bus
, PCI_SLOT(devfn
))) {
226 return PCIBIOS_DEVICE_NOT_FOUND
;
229 if (pci_is_root_bus(bus
))
230 return rockchip_pcie_rd_own_conf(rockchip
, where
, size
, val
);
232 return rockchip_pcie_rd_other_conf(rockchip
, bus
, devfn
, where
, size
,
236 static int rockchip_pcie_wr_conf(struct pci_bus
*bus
, u32 devfn
,
237 int where
, int size
, u32 val
)
239 struct rockchip_pcie
*rockchip
= bus
->sysdata
;
241 if (!rockchip_pcie_valid_device(rockchip
, bus
, PCI_SLOT(devfn
)))
242 return PCIBIOS_DEVICE_NOT_FOUND
;
244 if (pci_is_root_bus(bus
))
245 return rockchip_pcie_wr_own_conf(rockchip
, where
, size
, val
);
247 return rockchip_pcie_wr_other_conf(rockchip
, bus
, devfn
, where
, size
,
251 static struct pci_ops rockchip_pcie_ops
= {
252 .read
= rockchip_pcie_rd_conf
,
253 .write
= rockchip_pcie_wr_conf
,
256 static void rockchip_pcie_set_power_limit(struct rockchip_pcie
*rockchip
)
259 u32 status
, scale
, power
;
261 if (IS_ERR(rockchip
->vpcie3v3
))
265 * Set RC's captured slot power limit and scale if
266 * vpcie3v3 available. The default values are both zero
267 * which means the software should set these two according
268 * to the actual power supply.
270 curr
= regulator_get_current_limit(rockchip
->vpcie3v3
);
274 scale
= 3; /* 0.001x */
275 curr
= curr
/ 1000; /* convert to mA */
276 power
= (curr
* 3300) / 1000; /* milliwatt */
277 while (power
> PCIE_RC_CONFIG_DCR_CSPL_LIMIT
) {
279 dev_warn(rockchip
->dev
, "invalid power supply\n");
286 status
= rockchip_pcie_read(rockchip
, PCIE_RC_CONFIG_DCR
);
287 status
|= (power
<< PCIE_RC_CONFIG_DCR_CSPL_SHIFT
) |
288 (scale
<< PCIE_RC_CONFIG_DCR_CPLS_SHIFT
);
289 rockchip_pcie_write(rockchip
, status
, PCIE_RC_CONFIG_DCR
);
293 * rockchip_pcie_host_init_port - Initialize hardware
294 * @rockchip: PCIe port information
296 static int rockchip_pcie_host_init_port(struct rockchip_pcie
*rockchip
)
298 struct device
*dev
= rockchip
->dev
;
299 int err
, i
= MAX_LANE_NUM
;
302 gpiod_set_value_cansleep(rockchip
->ep_gpio
, 0);
304 err
= rockchip_pcie_init_port(rockchip
);
308 /* Fix the transmitted FTS count desired to exit from L0s. */
309 status
= rockchip_pcie_read(rockchip
, PCIE_CORE_CTRL_PLC1
);
310 status
= (status
& ~PCIE_CORE_CTRL_PLC1_FTS_MASK
) |
311 (PCIE_CORE_CTRL_PLC1_FTS_CNT
<< PCIE_CORE_CTRL_PLC1_FTS_SHIFT
);
312 rockchip_pcie_write(rockchip
, status
, PCIE_CORE_CTRL_PLC1
);
314 rockchip_pcie_set_power_limit(rockchip
);
316 /* Set RC's clock architecture as common clock */
317 status
= rockchip_pcie_read(rockchip
, PCIE_RC_CONFIG_LCS
);
318 status
|= PCI_EXP_LNKSTA_SLC
<< 16;
319 rockchip_pcie_write(rockchip
, status
, PCIE_RC_CONFIG_LCS
);
321 /* Set RC's RCB to 128 */
322 status
= rockchip_pcie_read(rockchip
, PCIE_RC_CONFIG_LCS
);
323 status
|= PCI_EXP_LNKCTL_RCB
;
324 rockchip_pcie_write(rockchip
, status
, PCIE_RC_CONFIG_LCS
);
326 /* Enable Gen1 training */
327 rockchip_pcie_write(rockchip
, PCIE_CLIENT_LINK_TRAIN_ENABLE
,
330 gpiod_set_value_cansleep(rockchip
->ep_gpio
, 1);
332 /* 500ms timeout value should be enough for Gen1/2 training */
333 err
= readl_poll_timeout(rockchip
->apb_base
+ PCIE_CLIENT_BASIC_STATUS1
,
334 status
, PCIE_LINK_UP(status
), 20,
335 500 * USEC_PER_MSEC
);
337 dev_err(dev
, "PCIe link training gen1 timeout!\n");
338 goto err_power_off_phy
;
341 if (rockchip
->link_gen
== 2) {
343 * Enable retrain for gen2. This should be configured only after
346 status
= rockchip_pcie_read(rockchip
, PCIE_RC_CONFIG_LCS
);
347 status
|= PCI_EXP_LNKCTL_RL
;
348 rockchip_pcie_write(rockchip
, status
, PCIE_RC_CONFIG_LCS
);
350 err
= readl_poll_timeout(rockchip
->apb_base
+ PCIE_CORE_CTRL
,
351 status
, PCIE_LINK_IS_GEN2(status
), 20,
352 500 * USEC_PER_MSEC
);
354 dev_dbg(dev
, "PCIe link training gen2 timeout, fall back to gen1!\n");
357 /* Check the final link width from negotiated lane counter from MGMT */
358 status
= rockchip_pcie_read(rockchip
, PCIE_CORE_CTRL
);
359 status
= 0x1 << ((status
& PCIE_CORE_PL_CONF_LANE_MASK
) >>
360 PCIE_CORE_PL_CONF_LANE_SHIFT
);
361 dev_dbg(dev
, "current link width is x%d\n", status
);
363 /* Power off unused lane(s) */
364 rockchip
->lanes_map
= rockchip_pcie_lane_map(rockchip
);
365 for (i
= 0; i
< MAX_LANE_NUM
; i
++) {
366 if (!(rockchip
->lanes_map
& BIT(i
))) {
367 dev_dbg(dev
, "idling lane %d\n", i
);
368 phy_power_off(rockchip
->phys
[i
]);
372 rockchip_pcie_write(rockchip
, ROCKCHIP_VENDOR_ID
,
373 PCIE_CORE_CONFIG_VENDOR
);
374 rockchip_pcie_write(rockchip
,
375 PCI_CLASS_BRIDGE_PCI
<< PCIE_RC_CONFIG_SCC_SHIFT
,
376 PCIE_RC_CONFIG_RID_CCR
);
378 /* Clear THP cap's next cap pointer to remove L1 substate cap */
379 status
= rockchip_pcie_read(rockchip
, PCIE_RC_CONFIG_THP_CAP
);
380 status
&= ~PCIE_RC_CONFIG_THP_CAP_NEXT_MASK
;
381 rockchip_pcie_write(rockchip
, status
, PCIE_RC_CONFIG_THP_CAP
);
383 /* Clear L0s from RC's link cap */
384 if (of_property_read_bool(dev
->of_node
, "aspm-no-l0s")) {
385 status
= rockchip_pcie_read(rockchip
, PCIE_RC_CONFIG_LINK_CAP
);
386 status
&= ~PCIE_RC_CONFIG_LINK_CAP_L0S
;
387 rockchip_pcie_write(rockchip
, status
, PCIE_RC_CONFIG_LINK_CAP
);
390 status
= rockchip_pcie_read(rockchip
, PCIE_RC_CONFIG_DCSR
);
391 status
&= ~PCIE_RC_CONFIG_DCSR_MPS_MASK
;
392 status
|= PCIE_RC_CONFIG_DCSR_MPS_256
;
393 rockchip_pcie_write(rockchip
, status
, PCIE_RC_CONFIG_DCSR
);
398 phy_power_off(rockchip
->phys
[i
]);
401 phy_exit(rockchip
->phys
[i
]);
405 static irqreturn_t
rockchip_pcie_subsys_irq_handler(int irq
, void *arg
)
407 struct rockchip_pcie
*rockchip
= arg
;
408 struct device
*dev
= rockchip
->dev
;
412 reg
= rockchip_pcie_read(rockchip
, PCIE_CLIENT_INT_STATUS
);
413 if (reg
& PCIE_CLIENT_INT_LOCAL
) {
414 dev_dbg(dev
, "local interrupt received\n");
415 sub_reg
= rockchip_pcie_read(rockchip
, PCIE_CORE_INT_STATUS
);
416 if (sub_reg
& PCIE_CORE_INT_PRFPE
)
417 dev_dbg(dev
, "parity error detected while reading from the PNP receive FIFO RAM\n");
419 if (sub_reg
& PCIE_CORE_INT_CRFPE
)
420 dev_dbg(dev
, "parity error detected while reading from the Completion Receive FIFO RAM\n");
422 if (sub_reg
& PCIE_CORE_INT_RRPE
)
423 dev_dbg(dev
, "parity error detected while reading from replay buffer RAM\n");
425 if (sub_reg
& PCIE_CORE_INT_PRFO
)
426 dev_dbg(dev
, "overflow occurred in the PNP receive FIFO\n");
428 if (sub_reg
& PCIE_CORE_INT_CRFO
)
429 dev_dbg(dev
, "overflow occurred in the completion receive FIFO\n");
431 if (sub_reg
& PCIE_CORE_INT_RT
)
432 dev_dbg(dev
, "replay timer timed out\n");
434 if (sub_reg
& PCIE_CORE_INT_RTR
)
435 dev_dbg(dev
, "replay timer rolled over after 4 transmissions of the same TLP\n");
437 if (sub_reg
& PCIE_CORE_INT_PE
)
438 dev_dbg(dev
, "phy error detected on receive side\n");
440 if (sub_reg
& PCIE_CORE_INT_MTR
)
441 dev_dbg(dev
, "malformed TLP received from the link\n");
443 if (sub_reg
& PCIE_CORE_INT_UCR
)
444 dev_dbg(dev
, "malformed TLP received from the link\n");
446 if (sub_reg
& PCIE_CORE_INT_FCE
)
447 dev_dbg(dev
, "an error was observed in the flow control advertisements from the other side\n");
449 if (sub_reg
& PCIE_CORE_INT_CT
)
450 dev_dbg(dev
, "a request timed out waiting for completion\n");
452 if (sub_reg
& PCIE_CORE_INT_UTC
)
453 dev_dbg(dev
, "unmapped TC error\n");
455 if (sub_reg
& PCIE_CORE_INT_MMVC
)
456 dev_dbg(dev
, "MSI mask register changes\n");
458 rockchip_pcie_write(rockchip
, sub_reg
, PCIE_CORE_INT_STATUS
);
459 } else if (reg
& PCIE_CLIENT_INT_PHY
) {
460 dev_dbg(dev
, "phy link changes\n");
461 rockchip_pcie_update_txcredit_mui(rockchip
);
462 rockchip_pcie_clr_bw_int(rockchip
);
465 rockchip_pcie_write(rockchip
, reg
& PCIE_CLIENT_INT_LOCAL
,
466 PCIE_CLIENT_INT_STATUS
);
471 static irqreturn_t
rockchip_pcie_client_irq_handler(int irq
, void *arg
)
473 struct rockchip_pcie
*rockchip
= arg
;
474 struct device
*dev
= rockchip
->dev
;
477 reg
= rockchip_pcie_read(rockchip
, PCIE_CLIENT_INT_STATUS
);
478 if (reg
& PCIE_CLIENT_INT_LEGACY_DONE
)
479 dev_dbg(dev
, "legacy done interrupt received\n");
481 if (reg
& PCIE_CLIENT_INT_MSG
)
482 dev_dbg(dev
, "message done interrupt received\n");
484 if (reg
& PCIE_CLIENT_INT_HOT_RST
)
485 dev_dbg(dev
, "hot reset interrupt received\n");
487 if (reg
& PCIE_CLIENT_INT_DPA
)
488 dev_dbg(dev
, "dpa interrupt received\n");
490 if (reg
& PCIE_CLIENT_INT_FATAL_ERR
)
491 dev_dbg(dev
, "fatal error interrupt received\n");
493 if (reg
& PCIE_CLIENT_INT_NFATAL_ERR
)
494 dev_dbg(dev
, "no fatal error interrupt received\n");
496 if (reg
& PCIE_CLIENT_INT_CORR_ERR
)
497 dev_dbg(dev
, "correctable error interrupt received\n");
499 if (reg
& PCIE_CLIENT_INT_PHY
)
500 dev_dbg(dev
, "phy interrupt received\n");
502 rockchip_pcie_write(rockchip
, reg
& (PCIE_CLIENT_INT_LEGACY_DONE
|
503 PCIE_CLIENT_INT_MSG
| PCIE_CLIENT_INT_HOT_RST
|
504 PCIE_CLIENT_INT_DPA
| PCIE_CLIENT_INT_FATAL_ERR
|
505 PCIE_CLIENT_INT_NFATAL_ERR
|
506 PCIE_CLIENT_INT_CORR_ERR
|
507 PCIE_CLIENT_INT_PHY
),
508 PCIE_CLIENT_INT_STATUS
);
513 static void rockchip_pcie_legacy_int_handler(struct irq_desc
*desc
)
515 struct irq_chip
*chip
= irq_desc_get_chip(desc
);
516 struct rockchip_pcie
*rockchip
= irq_desc_get_handler_data(desc
);
517 struct device
*dev
= rockchip
->dev
;
522 chained_irq_enter(chip
, desc
);
524 reg
= rockchip_pcie_read(rockchip
, PCIE_CLIENT_INT_STATUS
);
525 reg
= (reg
& PCIE_CLIENT_INTR_MASK
) >> PCIE_CLIENT_INTR_SHIFT
;
528 hwirq
= ffs(reg
) - 1;
531 virq
= irq_find_mapping(rockchip
->irq_domain
, hwirq
);
533 generic_handle_irq(virq
);
535 dev_err(dev
, "unexpected IRQ, INT%d\n", hwirq
);
538 chained_irq_exit(chip
, desc
);
541 static int rockchip_pcie_setup_irq(struct rockchip_pcie
*rockchip
)
544 struct device
*dev
= rockchip
->dev
;
545 struct platform_device
*pdev
= to_platform_device(dev
);
547 irq
= platform_get_irq_byname(pdev
, "sys");
551 err
= devm_request_irq(dev
, irq
, rockchip_pcie_subsys_irq_handler
,
552 IRQF_SHARED
, "pcie-sys", rockchip
);
554 dev_err(dev
, "failed to request PCIe subsystem IRQ\n");
558 irq
= platform_get_irq_byname(pdev
, "legacy");
562 irq_set_chained_handler_and_data(irq
,
563 rockchip_pcie_legacy_int_handler
,
566 irq
= platform_get_irq_byname(pdev
, "client");
570 err
= devm_request_irq(dev
, irq
, rockchip_pcie_client_irq_handler
,
571 IRQF_SHARED
, "pcie-client", rockchip
);
573 dev_err(dev
, "failed to request PCIe client IRQ\n");
581 * rockchip_pcie_parse_host_dt - Parse Device Tree
582 * @rockchip: PCIe port information
584 * Return: '0' on success and error value on failure
586 static int rockchip_pcie_parse_host_dt(struct rockchip_pcie
*rockchip
)
588 struct device
*dev
= rockchip
->dev
;
591 err
= rockchip_pcie_parse_dt(rockchip
);
595 err
= rockchip_pcie_setup_irq(rockchip
);
599 rockchip
->vpcie12v
= devm_regulator_get_optional(dev
, "vpcie12v");
600 if (IS_ERR(rockchip
->vpcie12v
)) {
601 if (PTR_ERR(rockchip
->vpcie12v
) != -ENODEV
)
602 return PTR_ERR(rockchip
->vpcie12v
);
603 dev_info(dev
, "no vpcie12v regulator found\n");
606 rockchip
->vpcie3v3
= devm_regulator_get_optional(dev
, "vpcie3v3");
607 if (IS_ERR(rockchip
->vpcie3v3
)) {
608 if (PTR_ERR(rockchip
->vpcie3v3
) != -ENODEV
)
609 return PTR_ERR(rockchip
->vpcie3v3
);
610 dev_info(dev
, "no vpcie3v3 regulator found\n");
613 rockchip
->vpcie1v8
= devm_regulator_get(dev
, "vpcie1v8");
614 if (IS_ERR(rockchip
->vpcie1v8
))
615 return PTR_ERR(rockchip
->vpcie1v8
);
617 rockchip
->vpcie0v9
= devm_regulator_get(dev
, "vpcie0v9");
618 if (IS_ERR(rockchip
->vpcie0v9
))
619 return PTR_ERR(rockchip
->vpcie0v9
);
624 static int rockchip_pcie_set_vpcie(struct rockchip_pcie
*rockchip
)
626 struct device
*dev
= rockchip
->dev
;
629 if (!IS_ERR(rockchip
->vpcie12v
)) {
630 err
= regulator_enable(rockchip
->vpcie12v
);
632 dev_err(dev
, "fail to enable vpcie12v regulator\n");
637 if (!IS_ERR(rockchip
->vpcie3v3
)) {
638 err
= regulator_enable(rockchip
->vpcie3v3
);
640 dev_err(dev
, "fail to enable vpcie3v3 regulator\n");
641 goto err_disable_12v
;
645 err
= regulator_enable(rockchip
->vpcie1v8
);
647 dev_err(dev
, "fail to enable vpcie1v8 regulator\n");
648 goto err_disable_3v3
;
651 err
= regulator_enable(rockchip
->vpcie0v9
);
653 dev_err(dev
, "fail to enable vpcie0v9 regulator\n");
654 goto err_disable_1v8
;
660 regulator_disable(rockchip
->vpcie1v8
);
662 if (!IS_ERR(rockchip
->vpcie3v3
))
663 regulator_disable(rockchip
->vpcie3v3
);
665 if (!IS_ERR(rockchip
->vpcie12v
))
666 regulator_disable(rockchip
->vpcie12v
);
671 static void rockchip_pcie_enable_interrupts(struct rockchip_pcie
*rockchip
)
673 rockchip_pcie_write(rockchip
, (PCIE_CLIENT_INT_CLI
<< 16) &
674 (~PCIE_CLIENT_INT_CLI
), PCIE_CLIENT_INT_MASK
);
675 rockchip_pcie_write(rockchip
, (u32
)(~PCIE_CORE_INT
),
678 rockchip_pcie_enable_bw_int(rockchip
);
681 static int rockchip_pcie_intx_map(struct irq_domain
*domain
, unsigned int irq
,
682 irq_hw_number_t hwirq
)
684 irq_set_chip_and_handler(irq
, &dummy_irq_chip
, handle_simple_irq
);
685 irq_set_chip_data(irq
, domain
->host_data
);
690 static const struct irq_domain_ops intx_domain_ops
= {
691 .map
= rockchip_pcie_intx_map
,
694 static int rockchip_pcie_init_irq_domain(struct rockchip_pcie
*rockchip
)
696 struct device
*dev
= rockchip
->dev
;
697 struct device_node
*intc
= of_get_next_child(dev
->of_node
, NULL
);
700 dev_err(dev
, "missing child interrupt-controller node\n");
704 rockchip
->irq_domain
= irq_domain_add_linear(intc
, PCI_NUM_INTX
,
705 &intx_domain_ops
, rockchip
);
707 if (!rockchip
->irq_domain
) {
708 dev_err(dev
, "failed to get a INTx IRQ domain\n");
715 static int rockchip_pcie_prog_ob_atu(struct rockchip_pcie
*rockchip
,
716 int region_no
, int type
, u8 num_pass_bits
,
717 u32 lower_addr
, u32 upper_addr
)
724 if (region_no
>= MAX_AXI_WRAPPER_REGION_NUM
)
726 if (num_pass_bits
+ 1 < 8)
728 if (num_pass_bits
> 63)
730 if (region_no
== 0) {
731 if (AXI_REGION_0_SIZE
< (2ULL << num_pass_bits
))
734 if (region_no
!= 0) {
735 if (AXI_REGION_SIZE
< (2ULL << num_pass_bits
))
739 aw_offset
= (region_no
<< OB_REG_SIZE_SHIFT
);
741 ob_addr_0
= num_pass_bits
& PCIE_CORE_OB_REGION_ADDR0_NUM_BITS
;
742 ob_addr_0
|= lower_addr
& PCIE_CORE_OB_REGION_ADDR0_LO_ADDR
;
743 ob_addr_1
= upper_addr
;
744 ob_desc_0
= (1 << 23 | type
);
746 rockchip_pcie_write(rockchip
, ob_addr_0
,
747 PCIE_CORE_OB_REGION_ADDR0
+ aw_offset
);
748 rockchip_pcie_write(rockchip
, ob_addr_1
,
749 PCIE_CORE_OB_REGION_ADDR1
+ aw_offset
);
750 rockchip_pcie_write(rockchip
, ob_desc_0
,
751 PCIE_CORE_OB_REGION_DESC0
+ aw_offset
);
752 rockchip_pcie_write(rockchip
, 0,
753 PCIE_CORE_OB_REGION_DESC1
+ aw_offset
);
758 static int rockchip_pcie_prog_ib_atu(struct rockchip_pcie
*rockchip
,
759 int region_no
, u8 num_pass_bits
,
760 u32 lower_addr
, u32 upper_addr
)
766 if (region_no
> MAX_AXI_IB_ROOTPORT_REGION_NUM
)
768 if (num_pass_bits
+ 1 < MIN_AXI_ADDR_BITS_PASSED
)
770 if (num_pass_bits
> 63)
773 aw_offset
= (region_no
<< IB_ROOT_PORT_REG_SIZE_SHIFT
);
775 ib_addr_0
= num_pass_bits
& PCIE_CORE_IB_REGION_ADDR0_NUM_BITS
;
776 ib_addr_0
|= (lower_addr
<< 8) & PCIE_CORE_IB_REGION_ADDR0_LO_ADDR
;
777 ib_addr_1
= upper_addr
;
779 rockchip_pcie_write(rockchip
, ib_addr_0
, PCIE_RP_IB_ADDR0
+ aw_offset
);
780 rockchip_pcie_write(rockchip
, ib_addr_1
, PCIE_RP_IB_ADDR1
+ aw_offset
);
785 static int rockchip_pcie_cfg_atu(struct rockchip_pcie
*rockchip
)
787 struct device
*dev
= rockchip
->dev
;
788 struct pci_host_bridge
*bridge
= pci_host_bridge_from_priv(rockchip
);
789 struct resource_entry
*entry
;
795 rockchip_pcie_cfg_configuration_accesses(rockchip
,
796 AXI_WRAPPER_TYPE0_CFG
);
797 entry
= resource_list_first_type(&bridge
->windows
, IORESOURCE_MEM
);
801 size
= resource_size(entry
->res
);
802 pci_addr
= entry
->res
->start
- entry
->offset
;
803 rockchip
->msg_bus_addr
= pci_addr
;
805 for (reg_no
= 0; reg_no
< (size
>> 20); reg_no
++) {
806 err
= rockchip_pcie_prog_ob_atu(rockchip
, reg_no
+ 1,
807 AXI_WRAPPER_MEM_WRITE
,
809 pci_addr
+ (reg_no
<< 20),
812 dev_err(dev
, "program RC mem outbound ATU failed\n");
817 err
= rockchip_pcie_prog_ib_atu(rockchip
, 2, 32 - 1, 0x0, 0);
819 dev_err(dev
, "program RC mem inbound ATU failed\n");
823 entry
= resource_list_first_type(&bridge
->windows
, IORESOURCE_IO
);
827 /* store the register number offset to program RC io outbound ATU */
830 size
= resource_size(entry
->res
);
831 pci_addr
= entry
->res
->start
- entry
->offset
;
833 for (reg_no
= 0; reg_no
< (size
>> 20); reg_no
++) {
834 err
= rockchip_pcie_prog_ob_atu(rockchip
,
836 AXI_WRAPPER_IO_WRITE
,
838 pci_addr
+ (reg_no
<< 20),
841 dev_err(dev
, "program RC io outbound ATU failed\n");
846 /* assign message regions */
847 rockchip_pcie_prog_ob_atu(rockchip
, reg_no
+ 1 + offset
,
851 rockchip
->msg_bus_addr
+= ((reg_no
+ offset
) << 20);
855 static int rockchip_pcie_wait_l2(struct rockchip_pcie
*rockchip
)
860 /* send PME_TURN_OFF message */
861 writel(0x0, rockchip
->msg_region
+ PCIE_RC_SEND_PME_OFF
);
863 /* read LTSSM and wait for falling into L2 link state */
864 err
= readl_poll_timeout(rockchip
->apb_base
+ PCIE_CLIENT_DEBUG_OUT_0
,
865 value
, PCIE_LINK_IS_L2(value
), 20,
866 jiffies_to_usecs(5 * HZ
));
868 dev_err(rockchip
->dev
, "PCIe link enter L2 timeout!\n");
875 static int __maybe_unused
rockchip_pcie_suspend_noirq(struct device
*dev
)
877 struct rockchip_pcie
*rockchip
= dev_get_drvdata(dev
);
880 /* disable core and cli int since we don't need to ack PME_ACK */
881 rockchip_pcie_write(rockchip
, (PCIE_CLIENT_INT_CLI
<< 16) |
882 PCIE_CLIENT_INT_CLI
, PCIE_CLIENT_INT_MASK
);
883 rockchip_pcie_write(rockchip
, (u32
)PCIE_CORE_INT
, PCIE_CORE_INT_MASK
);
885 ret
= rockchip_pcie_wait_l2(rockchip
);
887 rockchip_pcie_enable_interrupts(rockchip
);
891 rockchip_pcie_deinit_phys(rockchip
);
893 rockchip_pcie_disable_clocks(rockchip
);
895 regulator_disable(rockchip
->vpcie0v9
);
900 static int __maybe_unused
rockchip_pcie_resume_noirq(struct device
*dev
)
902 struct rockchip_pcie
*rockchip
= dev_get_drvdata(dev
);
905 err
= regulator_enable(rockchip
->vpcie0v9
);
907 dev_err(dev
, "fail to enable vpcie0v9 regulator\n");
911 err
= rockchip_pcie_enable_clocks(rockchip
);
913 goto err_disable_0v9
;
915 err
= rockchip_pcie_host_init_port(rockchip
);
917 goto err_pcie_resume
;
919 err
= rockchip_pcie_cfg_atu(rockchip
);
921 goto err_err_deinit_port
;
923 /* Need this to enter L1 again */
924 rockchip_pcie_update_txcredit_mui(rockchip
);
925 rockchip_pcie_enable_interrupts(rockchip
);
930 rockchip_pcie_deinit_phys(rockchip
);
932 rockchip_pcie_disable_clocks(rockchip
);
934 regulator_disable(rockchip
->vpcie0v9
);
938 static int rockchip_pcie_probe(struct platform_device
*pdev
)
940 struct rockchip_pcie
*rockchip
;
941 struct device
*dev
= &pdev
->dev
;
942 struct pci_host_bridge
*bridge
;
948 bridge
= devm_pci_alloc_host_bridge(dev
, sizeof(*rockchip
));
952 rockchip
= pci_host_bridge_priv(bridge
);
954 platform_set_drvdata(pdev
, rockchip
);
956 rockchip
->is_rc
= true;
958 err
= rockchip_pcie_parse_host_dt(rockchip
);
962 err
= rockchip_pcie_enable_clocks(rockchip
);
966 err
= rockchip_pcie_set_vpcie(rockchip
);
968 dev_err(dev
, "failed to set vpcie regulator\n");
972 err
= rockchip_pcie_host_init_port(rockchip
);
976 rockchip_pcie_enable_interrupts(rockchip
);
978 err
= rockchip_pcie_init_irq_domain(rockchip
);
980 goto err_deinit_port
;
982 err
= rockchip_pcie_cfg_atu(rockchip
);
984 goto err_remove_irq_domain
;
986 rockchip
->msg_region
= devm_ioremap(dev
, rockchip
->msg_bus_addr
, SZ_1M
);
987 if (!rockchip
->msg_region
) {
989 goto err_remove_irq_domain
;
992 bridge
->sysdata
= rockchip
;
993 bridge
->ops
= &rockchip_pcie_ops
;
995 err
= pci_host_probe(bridge
);
997 goto err_remove_irq_domain
;
1001 err_remove_irq_domain
:
1002 irq_domain_remove(rockchip
->irq_domain
);
1004 rockchip_pcie_deinit_phys(rockchip
);
1006 if (!IS_ERR(rockchip
->vpcie12v
))
1007 regulator_disable(rockchip
->vpcie12v
);
1008 if (!IS_ERR(rockchip
->vpcie3v3
))
1009 regulator_disable(rockchip
->vpcie3v3
);
1010 regulator_disable(rockchip
->vpcie1v8
);
1011 regulator_disable(rockchip
->vpcie0v9
);
1013 rockchip_pcie_disable_clocks(rockchip
);
1017 static int rockchip_pcie_remove(struct platform_device
*pdev
)
1019 struct device
*dev
= &pdev
->dev
;
1020 struct rockchip_pcie
*rockchip
= dev_get_drvdata(dev
);
1021 struct pci_host_bridge
*bridge
= pci_host_bridge_from_priv(rockchip
);
1023 pci_stop_root_bus(bridge
->bus
);
1024 pci_remove_root_bus(bridge
->bus
);
1025 irq_domain_remove(rockchip
->irq_domain
);
1027 rockchip_pcie_deinit_phys(rockchip
);
1029 rockchip_pcie_disable_clocks(rockchip
);
1031 if (!IS_ERR(rockchip
->vpcie12v
))
1032 regulator_disable(rockchip
->vpcie12v
);
1033 if (!IS_ERR(rockchip
->vpcie3v3
))
1034 regulator_disable(rockchip
->vpcie3v3
);
1035 regulator_disable(rockchip
->vpcie1v8
);
1036 regulator_disable(rockchip
->vpcie0v9
);
1041 static const struct dev_pm_ops rockchip_pcie_pm_ops
= {
1042 SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(rockchip_pcie_suspend_noirq
,
1043 rockchip_pcie_resume_noirq
)
1046 static const struct of_device_id rockchip_pcie_of_match
[] = {
1047 { .compatible
= "rockchip,rk3399-pcie", },
1050 MODULE_DEVICE_TABLE(of
, rockchip_pcie_of_match
);
1052 static struct platform_driver rockchip_pcie_driver
= {
1054 .name
= "rockchip-pcie",
1055 .of_match_table
= rockchip_pcie_of_match
,
1056 .pm
= &rockchip_pcie_pm_ops
,
1058 .probe
= rockchip_pcie_probe
,
1059 .remove
= rockchip_pcie_remove
,
1061 module_platform_driver(rockchip_pcie_driver
);
1063 MODULE_AUTHOR("Rockchip Inc");
1064 MODULE_DESCRIPTION("Rockchip AXI PCIe driver");
1065 MODULE_LICENSE("GPL v2");