?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Linux 4.16.11
[linux/fpc-iii.git] / drivers / pci / host / pcie-rockchip.c
blobf1e8f97ea1fb185852d88a12a36d2fd0ece3a937
1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3 * Rockchip AXI PCIe host controller driver
4 *
5 * Copyright (c) 2016 Rockchip, Inc.
6 *
7 * Author: Shawn Lin <shawn.lin@rock-chips.com>
8 * Wenrui Li <wenrui.li@rock-chips.com>
9 *
10 * Bits taken from Synopsys DesignWare Host controller driver and
11 * ARM PCI Host generic driver.
12 */
13
14 #include <linux/bitrev.h>
15 #include <linux/clk.h>
16 #include <linux/delay.h>
17 #include <linux/gpio/consumer.h>
18 #include <linux/init.h>
19 #include <linux/interrupt.h>
20 #include <linux/iopoll.h>
21 #include <linux/irq.h>
22 #include <linux/irqchip/chained_irq.h>
23 #include <linux/irqdomain.h>
24 #include <linux/kernel.h>
25 #include <linux/mfd/syscon.h>
26 #include <linux/module.h>
27 #include <linux/of_address.h>
28 #include <linux/of_device.h>
29 #include <linux/of_pci.h>
30 #include <linux/of_platform.h>
31 #include <linux/of_irq.h>
32 #include <linux/pci.h>
33 #include <linux/pci_ids.h>
34 #include <linux/phy/phy.h>
35 #include <linux/platform_device.h>
36 #include <linux/reset.h>
37 #include <linux/regmap.h>
38
39 /*
40 * The upper 16 bits of PCIE_CLIENT_CONFIG are a write mask for the lower 16
41 * bits. This allows atomic updates of the register without locking.
42 */
43 #define HIWORD_UPDATE(mask, val) (((mask) << 16) | (val))
44 #define HIWORD_UPDATE_BIT(val) HIWORD_UPDATE(val, val)
45
46 #define ENCODE_LANES(x) ((((x) >> 1) & 3) << 4)
47 #define MAX_LANE_NUM 4
48
49 #define PCIE_CLIENT_BASE 0x0
50 #define PCIE_CLIENT_CONFIG (PCIE_CLIENT_BASE + 0x00)
51 #define PCIE_CLIENT_CONF_ENABLE HIWORD_UPDATE_BIT(0x0001)
52 #define PCIE_CLIENT_LINK_TRAIN_ENABLE HIWORD_UPDATE_BIT(0x0002)
53 #define PCIE_CLIENT_ARI_ENABLE HIWORD_UPDATE_BIT(0x0008)
54 #define PCIE_CLIENT_CONF_LANE_NUM(x) HIWORD_UPDATE(0x0030, ENCODE_LANES(x))
55 #define PCIE_CLIENT_MODE_RC HIWORD_UPDATE_BIT(0x0040)
56 #define PCIE_CLIENT_GEN_SEL_1 HIWORD_UPDATE(0x0080, 0)
57 #define PCIE_CLIENT_GEN_SEL_2 HIWORD_UPDATE_BIT(0x0080)
58 #define PCIE_CLIENT_DEBUG_OUT_0 (PCIE_CLIENT_BASE + 0x3c)
59 #define PCIE_CLIENT_DEBUG_LTSSM_MASK GENMASK(5, 0)
60 #define PCIE_CLIENT_DEBUG_LTSSM_L1 0x18
61 #define PCIE_CLIENT_DEBUG_LTSSM_L2 0x19
62 #define PCIE_CLIENT_BASIC_STATUS1 (PCIE_CLIENT_BASE + 0x48)
63 #define PCIE_CLIENT_LINK_STATUS_UP 0x00300000
64 #define PCIE_CLIENT_LINK_STATUS_MASK 0x00300000
65 #define PCIE_CLIENT_INT_MASK (PCIE_CLIENT_BASE + 0x4c)
66 #define PCIE_CLIENT_INT_STATUS (PCIE_CLIENT_BASE + 0x50)
67 #define PCIE_CLIENT_INTR_MASK GENMASK(8, 5)
68 #define PCIE_CLIENT_INTR_SHIFT 5
69 #define PCIE_CLIENT_INT_LEGACY_DONE BIT(15)
70 #define PCIE_CLIENT_INT_MSG BIT(14)
71 #define PCIE_CLIENT_INT_HOT_RST BIT(13)
72 #define PCIE_CLIENT_INT_DPA BIT(12)
73 #define PCIE_CLIENT_INT_FATAL_ERR BIT(11)
74 #define PCIE_CLIENT_INT_NFATAL_ERR BIT(10)
75 #define PCIE_CLIENT_INT_CORR_ERR BIT(9)
76 #define PCIE_CLIENT_INT_INTD BIT(8)
77 #define PCIE_CLIENT_INT_INTC BIT(7)
78 #define PCIE_CLIENT_INT_INTB BIT(6)
79 #define PCIE_CLIENT_INT_INTA BIT(5)
80 #define PCIE_CLIENT_INT_LOCAL BIT(4)
81 #define PCIE_CLIENT_INT_UDMA BIT(3)
82 #define PCIE_CLIENT_INT_PHY BIT(2)
83 #define PCIE_CLIENT_INT_HOT_PLUG BIT(1)
84 #define PCIE_CLIENT_INT_PWR_STCG BIT(0)
85
86 #define PCIE_CLIENT_INT_LEGACY \
87 (PCIE_CLIENT_INT_INTA | PCIE_CLIENT_INT_INTB | \
88 PCIE_CLIENT_INT_INTC | PCIE_CLIENT_INT_INTD)
89
90 #define PCIE_CLIENT_INT_CLI \
91 (PCIE_CLIENT_INT_CORR_ERR | PCIE_CLIENT_INT_NFATAL_ERR | \
92 PCIE_CLIENT_INT_FATAL_ERR | PCIE_CLIENT_INT_DPA | \
93 PCIE_CLIENT_INT_HOT_RST | PCIE_CLIENT_INT_MSG | \
94 PCIE_CLIENT_INT_LEGACY_DONE | PCIE_CLIENT_INT_LEGACY | \
95 PCIE_CLIENT_INT_PHY)
96
97 #define PCIE_CORE_CTRL_MGMT_BASE 0x900000
98 #define PCIE_CORE_CTRL (PCIE_CORE_CTRL_MGMT_BASE + 0x000)
99 #define PCIE_CORE_PL_CONF_SPEED_5G 0x00000008
100 #define PCIE_CORE_PL_CONF_SPEED_MASK 0x00000018
101 #define PCIE_CORE_PL_CONF_LANE_MASK 0x00000006
102 #define PCIE_CORE_PL_CONF_LANE_SHIFT 1
103 #define PCIE_CORE_CTRL_PLC1 (PCIE_CORE_CTRL_MGMT_BASE + 0x004)
104 #define PCIE_CORE_CTRL_PLC1_FTS_MASK GENMASK(23, 8)
105 #define PCIE_CORE_CTRL_PLC1_FTS_SHIFT 8
106 #define PCIE_CORE_CTRL_PLC1_FTS_CNT 0xffff
107 #define PCIE_CORE_TXCREDIT_CFG1 (PCIE_CORE_CTRL_MGMT_BASE + 0x020)
108 #define PCIE_CORE_TXCREDIT_CFG1_MUI_MASK 0xFFFF0000
109 #define PCIE_CORE_TXCREDIT_CFG1_MUI_SHIFT 16
110 #define PCIE_CORE_TXCREDIT_CFG1_MUI_ENCODE(x) \
111 (((x) >> 3) << PCIE_CORE_TXCREDIT_CFG1_MUI_SHIFT)
112 #define PCIE_CORE_LANE_MAP (PCIE_CORE_CTRL_MGMT_BASE + 0x200)
113 #define PCIE_CORE_LANE_MAP_MASK 0x0000000f
114 #define PCIE_CORE_LANE_MAP_REVERSE BIT(16)
115 #define PCIE_CORE_INT_STATUS (PCIE_CORE_CTRL_MGMT_BASE + 0x20c)
116 #define PCIE_CORE_INT_PRFPE BIT(0)
117 #define PCIE_CORE_INT_CRFPE BIT(1)
118 #define PCIE_CORE_INT_RRPE BIT(2)
119 #define PCIE_CORE_INT_PRFO BIT(3)
120 #define PCIE_CORE_INT_CRFO BIT(4)
121 #define PCIE_CORE_INT_RT BIT(5)
122 #define PCIE_CORE_INT_RTR BIT(6)
123 #define PCIE_CORE_INT_PE BIT(7)
124 #define PCIE_CORE_INT_MTR BIT(8)
125 #define PCIE_CORE_INT_UCR BIT(9)
126 #define PCIE_CORE_INT_FCE BIT(10)
127 #define PCIE_CORE_INT_CT BIT(11)
128 #define PCIE_CORE_INT_UTC BIT(18)
129 #define PCIE_CORE_INT_MMVC BIT(19)
130 #define PCIE_CORE_CONFIG_VENDOR (PCIE_CORE_CTRL_MGMT_BASE + 0x44)
131 #define PCIE_CORE_INT_MASK (PCIE_CORE_CTRL_MGMT_BASE + 0x210)
132 #define PCIE_RC_BAR_CONF (PCIE_CORE_CTRL_MGMT_BASE + 0x300)
133
134 #define PCIE_CORE_INT \
135 (PCIE_CORE_INT_PRFPE | PCIE_CORE_INT_CRFPE | \
136 PCIE_CORE_INT_RRPE | PCIE_CORE_INT_CRFO | \
137 PCIE_CORE_INT_RT | PCIE_CORE_INT_RTR | \
138 PCIE_CORE_INT_PE | PCIE_CORE_INT_MTR | \
139 PCIE_CORE_INT_UCR | PCIE_CORE_INT_FCE | \
140 PCIE_CORE_INT_CT | PCIE_CORE_INT_UTC | \
141 PCIE_CORE_INT_MMVC)
142
143 #define PCIE_RC_CONFIG_NORMAL_BASE 0x800000
144 #define PCIE_RC_CONFIG_BASE 0xa00000
145 #define PCIE_RC_CONFIG_RID_CCR (PCIE_RC_CONFIG_BASE + 0x08)
146 #define PCIE_RC_CONFIG_SCC_SHIFT 16
147 #define PCIE_RC_CONFIG_DCR (PCIE_RC_CONFIG_BASE + 0xc4)
148 #define PCIE_RC_CONFIG_DCR_CSPL_SHIFT 18
149 #define PCIE_RC_CONFIG_DCR_CSPL_LIMIT 0xff
150 #define PCIE_RC_CONFIG_DCR_CPLS_SHIFT 26
151 #define PCIE_RC_CONFIG_DCSR (PCIE_RC_CONFIG_BASE + 0xc8)
152 #define PCIE_RC_CONFIG_DCSR_MPS_MASK GENMASK(7, 5)
153 #define PCIE_RC_CONFIG_DCSR_MPS_256 (0x1 << 5)
154 #define PCIE_RC_CONFIG_LINK_CAP (PCIE_RC_CONFIG_BASE + 0xcc)
155 #define PCIE_RC_CONFIG_LINK_CAP_L0S BIT(10)
156 #define PCIE_RC_CONFIG_LCS (PCIE_RC_CONFIG_BASE + 0xd0)
157 #define PCIE_RC_CONFIG_L1_SUBSTATE_CTRL2 (PCIE_RC_CONFIG_BASE + 0x90c)
158 #define PCIE_RC_CONFIG_THP_CAP (PCIE_RC_CONFIG_BASE + 0x274)
159 #define PCIE_RC_CONFIG_THP_CAP_NEXT_MASK GENMASK(31, 20)
160
161 #define PCIE_CORE_AXI_CONF_BASE 0xc00000
162 #define PCIE_CORE_OB_REGION_ADDR0 (PCIE_CORE_AXI_CONF_BASE + 0x0)
163 #define PCIE_CORE_OB_REGION_ADDR0_NUM_BITS 0x3f
164 #define PCIE_CORE_OB_REGION_ADDR0_LO_ADDR 0xffffff00
165 #define PCIE_CORE_OB_REGION_ADDR1 (PCIE_CORE_AXI_CONF_BASE + 0x4)
166 #define PCIE_CORE_OB_REGION_DESC0 (PCIE_CORE_AXI_CONF_BASE + 0x8)
167 #define PCIE_CORE_OB_REGION_DESC1 (PCIE_CORE_AXI_CONF_BASE + 0xc)
168
169 #define PCIE_CORE_AXI_INBOUND_BASE 0xc00800
170 #define PCIE_RP_IB_ADDR0 (PCIE_CORE_AXI_INBOUND_BASE + 0x0)
171 #define PCIE_CORE_IB_REGION_ADDR0_NUM_BITS 0x3f
172 #define PCIE_CORE_IB_REGION_ADDR0_LO_ADDR 0xffffff00
173 #define PCIE_RP_IB_ADDR1 (PCIE_CORE_AXI_INBOUND_BASE + 0x4)
174
175 /* Size of one AXI Region (not Region 0) */
176 #define AXI_REGION_SIZE BIT(20)
177 /* Size of Region 0, equal to sum of sizes of other regions */
178 #define AXI_REGION_0_SIZE (32 * (0x1 << 20))
179 #define OB_REG_SIZE_SHIFT 5
180 #define IB_ROOT_PORT_REG_SIZE_SHIFT 3
181 #define AXI_WRAPPER_IO_WRITE 0x6
182 #define AXI_WRAPPER_MEM_WRITE 0x2
183 #define AXI_WRAPPER_TYPE0_CFG 0xa
184 #define AXI_WRAPPER_TYPE1_CFG 0xb
185 #define AXI_WRAPPER_NOR_MSG 0xc
186
187 #define MAX_AXI_IB_ROOTPORT_REGION_NUM 3
188 #define MIN_AXI_ADDR_BITS_PASSED 8
189 #define PCIE_RC_SEND_PME_OFF 0x11960
190 #define ROCKCHIP_VENDOR_ID 0x1d87
191 #define PCIE_ECAM_BUS(x) (((x) & 0xff) << 20)
192 #define PCIE_ECAM_DEV(x) (((x) & 0x1f) << 15)
193 #define PCIE_ECAM_FUNC(x) (((x) & 0x7) << 12)
194 #define PCIE_ECAM_REG(x) (((x) & 0xfff) << 0)
195 #define PCIE_ECAM_ADDR(bus, dev, func, reg) \
196 (PCIE_ECAM_BUS(bus) | PCIE_ECAM_DEV(dev) | \
197 PCIE_ECAM_FUNC(func) | PCIE_ECAM_REG(reg))
198 #define PCIE_LINK_IS_L2(x) \
199 (((x) & PCIE_CLIENT_DEBUG_LTSSM_MASK) == PCIE_CLIENT_DEBUG_LTSSM_L2)
200 #define PCIE_LINK_UP(x) \
201 (((x) & PCIE_CLIENT_LINK_STATUS_MASK) == PCIE_CLIENT_LINK_STATUS_UP)
202 #define PCIE_LINK_IS_GEN2(x) \
203 (((x) & PCIE_CORE_PL_CONF_SPEED_MASK) == PCIE_CORE_PL_CONF_SPEED_5G)
204
205 #define RC_REGION_0_ADDR_TRANS_H 0x00000000
206 #define RC_REGION_0_ADDR_TRANS_L 0x00000000
207 #define RC_REGION_0_PASS_BITS (25 - 1)
208 #define RC_REGION_0_TYPE_MASK GENMASK(3, 0)
209 #define MAX_AXI_WRAPPER_REGION_NUM 33
210
211 struct rockchip_pcie {
212 void __iomem *reg_base; /* DT axi-base */
213 void __iomem *apb_base; /* DT apb-base */
214 bool legacy_phy;
215 struct phy *phys[MAX_LANE_NUM];
216 struct reset_control *core_rst;
217 struct reset_control *mgmt_rst;
218 struct reset_control *mgmt_sticky_rst;
219 struct reset_control *pipe_rst;
220 struct reset_control *pm_rst;
221 struct reset_control *aclk_rst;
222 struct reset_control *pclk_rst;
223 struct clk *aclk_pcie;
224 struct clk *aclk_perf_pcie;
225 struct clk *hclk_pcie;
226 struct clk *clk_pcie_pm;
227 struct regulator *vpcie12v; /* 12V power supply */
228 struct regulator *vpcie3v3; /* 3.3V power supply */
229 struct regulator *vpcie1v8; /* 1.8V power supply */
230 struct regulator *vpcie0v9; /* 0.9V power supply */
231 struct gpio_desc *ep_gpio;
232 u32 lanes;
233 u8 lanes_map;
234 u8 root_bus_nr;
235 int link_gen;
236 struct device *dev;
237 struct irq_domain *irq_domain;
238 int offset;
239 struct pci_bus *root_bus;
240 struct resource *io;
241 phys_addr_t io_bus_addr;
242 u32 io_size;
243 void __iomem *msg_region;
244 u32 mem_size;
245 phys_addr_t msg_bus_addr;
246 phys_addr_t mem_bus_addr;
247 };
248
249 static u32 rockchip_pcie_read(struct rockchip_pcie *rockchip, u32 reg)
250 {
251 return readl(rockchip->apb_base + reg);
252 }
253
254 static void rockchip_pcie_write(struct rockchip_pcie *rockchip, u32 val,
255 u32 reg)
256 {
257 writel(val, rockchip->apb_base + reg);
258 }
259
260 static void rockchip_pcie_enable_bw_int(struct rockchip_pcie *rockchip)
261 {
262 u32 status;
263
264 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
265 status |= (PCI_EXP_LNKCTL_LBMIE | PCI_EXP_LNKCTL_LABIE);
266 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
267 }
268
269 static void rockchip_pcie_clr_bw_int(struct rockchip_pcie *rockchip)
270 {
271 u32 status;
272
273 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
274 status |= (PCI_EXP_LNKSTA_LBMS | PCI_EXP_LNKSTA_LABS) << 16;
275 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
276 }
277
278 static void rockchip_pcie_update_txcredit_mui(struct rockchip_pcie *rockchip)
279 {
280 u32 val;
281
282 /* Update Tx credit maximum update interval */
283 val = rockchip_pcie_read(rockchip, PCIE_CORE_TXCREDIT_CFG1);
284 val &= ~PCIE_CORE_TXCREDIT_CFG1_MUI_MASK;
285 val |= PCIE_CORE_TXCREDIT_CFG1_MUI_ENCODE(24000); /* ns */
286 rockchip_pcie_write(rockchip, val, PCIE_CORE_TXCREDIT_CFG1);
287 }
288
289 static int rockchip_pcie_valid_device(struct rockchip_pcie *rockchip,
290 struct pci_bus *bus, int dev)
291 {
292 /* access only one slot on each root port */
293 if (bus->number == rockchip->root_bus_nr && dev > 0)
294 return 0;
295
296 /*
297 * do not read more than one device on the bus directly attached
298 * to RC's downstream side.
299 */
300 if (bus->primary == rockchip->root_bus_nr && dev > 0)
301 return 0;
302
303 return 1;
304 }
305
306 static u8 rockchip_pcie_lane_map(struct rockchip_pcie *rockchip)
307 {
308 u32 val;
309 u8 map;
310
311 if (rockchip->legacy_phy)
312 return GENMASK(MAX_LANE_NUM - 1, 0);
313
314 val = rockchip_pcie_read(rockchip, PCIE_CORE_LANE_MAP);
315 map = val & PCIE_CORE_LANE_MAP_MASK;
316
317 /* The link may be using a reverse-indexed mapping. */
318 if (val & PCIE_CORE_LANE_MAP_REVERSE)
319 map = bitrev8(map) >> 4;
320
321 return map;
322 }
323
324 static int rockchip_pcie_rd_own_conf(struct rockchip_pcie *rockchip,
325 int where, int size, u32 *val)
326 {
327 void __iomem *addr;
328
329 addr = rockchip->apb_base + PCIE_RC_CONFIG_NORMAL_BASE + where;
330
331 if (!IS_ALIGNED((uintptr_t)addr, size)) {
332 *val = 0;
333 return PCIBIOS_BAD_REGISTER_NUMBER;
334 }
335
336 if (size == 4) {
337 *val = readl(addr);
338 } else if (size == 2) {
339 *val = readw(addr);
340 } else if (size == 1) {
341 *val = readb(addr);
342 } else {
343 *val = 0;
344 return PCIBIOS_BAD_REGISTER_NUMBER;
345 }
346 return PCIBIOS_SUCCESSFUL;
347 }
348
349 static int rockchip_pcie_wr_own_conf(struct rockchip_pcie *rockchip,
350 int where, int size, u32 val)
351 {
352 u32 mask, tmp, offset;
353 void __iomem *addr;
354
355 offset = where & ~0x3;
356 addr = rockchip->apb_base + PCIE_RC_CONFIG_NORMAL_BASE + offset;
357
358 if (size == 4) {
359 writel(val, addr);
360 return PCIBIOS_SUCCESSFUL;
361 }
362
363 mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8));
364
365 /*
366 * N.B. This read/modify/write isn't safe in general because it can
367 * corrupt RW1C bits in adjacent registers. But the hardware
368 * doesn't support smaller writes.
369 */
370 tmp = readl(addr) & mask;
371 tmp |= val << ((where & 0x3) * 8);
372 writel(tmp, addr);
373
374 return PCIBIOS_SUCCESSFUL;
375 }
376
377 static void rockchip_pcie_cfg_configuration_accesses(
378 struct rockchip_pcie *rockchip, u32 type)
379 {
380 u32 ob_desc_0;
381
382 /* Configuration Accesses for region 0 */
383 rockchip_pcie_write(rockchip, 0x0, PCIE_RC_BAR_CONF);
384
385 rockchip_pcie_write(rockchip,
386 (RC_REGION_0_ADDR_TRANS_L + RC_REGION_0_PASS_BITS),
387 PCIE_CORE_OB_REGION_ADDR0);
388 rockchip_pcie_write(rockchip, RC_REGION_0_ADDR_TRANS_H,
389 PCIE_CORE_OB_REGION_ADDR1);
390 ob_desc_0 = rockchip_pcie_read(rockchip, PCIE_CORE_OB_REGION_DESC0);
391 ob_desc_0 &= ~(RC_REGION_0_TYPE_MASK);
392 ob_desc_0 |= (type | (0x1 << 23));
393 rockchip_pcie_write(rockchip, ob_desc_0, PCIE_CORE_OB_REGION_DESC0);
394 rockchip_pcie_write(rockchip, 0x0, PCIE_CORE_OB_REGION_DESC1);
395 }
396
397 static int rockchip_pcie_rd_other_conf(struct rockchip_pcie *rockchip,
398 struct pci_bus *bus, u32 devfn,
399 int where, int size, u32 *val)
400 {
401 u32 busdev;
402
403 busdev = PCIE_ECAM_ADDR(bus->number, PCI_SLOT(devfn),
404 PCI_FUNC(devfn), where);
405
406 if (!IS_ALIGNED(busdev, size)) {
407 *val = 0;
408 return PCIBIOS_BAD_REGISTER_NUMBER;
409 }
410
411 if (bus->parent->number == rockchip->root_bus_nr)
412 rockchip_pcie_cfg_configuration_accesses(rockchip,
413 AXI_WRAPPER_TYPE0_CFG);
414 else
415 rockchip_pcie_cfg_configuration_accesses(rockchip,
416 AXI_WRAPPER_TYPE1_CFG);
417
418 if (size == 4) {
419 *val = readl(rockchip->reg_base + busdev);
420 } else if (size == 2) {
421 *val = readw(rockchip->reg_base + busdev);
422 } else if (size == 1) {
423 *val = readb(rockchip->reg_base + busdev);
424 } else {
425 *val = 0;
426 return PCIBIOS_BAD_REGISTER_NUMBER;
427 }
428 return PCIBIOS_SUCCESSFUL;
429 }
430
431 static int rockchip_pcie_wr_other_conf(struct rockchip_pcie *rockchip,
432 struct pci_bus *bus, u32 devfn,
433 int where, int size, u32 val)
434 {
435 u32 busdev;
436
437 busdev = PCIE_ECAM_ADDR(bus->number, PCI_SLOT(devfn),
438 PCI_FUNC(devfn), where);
439 if (!IS_ALIGNED(busdev, size))
440 return PCIBIOS_BAD_REGISTER_NUMBER;
441
442 if (bus->parent->number == rockchip->root_bus_nr)
443 rockchip_pcie_cfg_configuration_accesses(rockchip,
444 AXI_WRAPPER_TYPE0_CFG);
445 else
446 rockchip_pcie_cfg_configuration_accesses(rockchip,
447 AXI_WRAPPER_TYPE1_CFG);
448
449 if (size == 4)
450 writel(val, rockchip->reg_base + busdev);
451 else if (size == 2)
452 writew(val, rockchip->reg_base + busdev);
453 else if (size == 1)
454 writeb(val, rockchip->reg_base + busdev);
455 else
456 return PCIBIOS_BAD_REGISTER_NUMBER;
457
458 return PCIBIOS_SUCCESSFUL;
459 }
460
461 static int rockchip_pcie_rd_conf(struct pci_bus *bus, u32 devfn, int where,
462 int size, u32 *val)
463 {
464 struct rockchip_pcie *rockchip = bus->sysdata;
465
466 if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn))) {
467 *val = 0xffffffff;
468 return PCIBIOS_DEVICE_NOT_FOUND;
469 }
470
471 if (bus->number == rockchip->root_bus_nr)
472 return rockchip_pcie_rd_own_conf(rockchip, where, size, val);
473
474 return rockchip_pcie_rd_other_conf(rockchip, bus, devfn, where, size, val);
475 }
476
477 static int rockchip_pcie_wr_conf(struct pci_bus *bus, u32 devfn,
478 int where, int size, u32 val)
479 {
480 struct rockchip_pcie *rockchip = bus->sysdata;
481
482 if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn)))
483 return PCIBIOS_DEVICE_NOT_FOUND;
484
485 if (bus->number == rockchip->root_bus_nr)
486 return rockchip_pcie_wr_own_conf(rockchip, where, size, val);
487
488 return rockchip_pcie_wr_other_conf(rockchip, bus, devfn, where, size, val);
489 }
490
491 static struct pci_ops rockchip_pcie_ops = {
492 .read = rockchip_pcie_rd_conf,
493 .write = rockchip_pcie_wr_conf,
494 };
495
496 static void rockchip_pcie_set_power_limit(struct rockchip_pcie *rockchip)
497 {
498 int curr;
499 u32 status, scale, power;
500
501 if (IS_ERR(rockchip->vpcie3v3))
502 return;
503
504 /*
505 * Set RC's captured slot power limit and scale if
506 * vpcie3v3 available. The default values are both zero
507 * which means the software should set these two according
508 * to the actual power supply.
509 */
510 curr = regulator_get_current_limit(rockchip->vpcie3v3);
511 if (curr <= 0)
512 return;
513
514 scale = 3; /* 0.001x */
515 curr = curr / 1000; /* convert to mA */
516 power = (curr * 3300) / 1000; /* milliwatt */
517 while (power > PCIE_RC_CONFIG_DCR_CSPL_LIMIT) {
518 if (!scale) {
519 dev_warn(rockchip->dev, "invalid power supply\n");
520 return;
521 }
522 scale--;
523 power = power / 10;
524 }
525
526 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_DCR);
527 status |= (power << PCIE_RC_CONFIG_DCR_CSPL_SHIFT) |
528 (scale << PCIE_RC_CONFIG_DCR_CPLS_SHIFT);
529 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_DCR);
530 }
531
532 /**
533 * rockchip_pcie_init_port - Initialize hardware
534 * @rockchip: PCIe port information
535 */
536 static int rockchip_pcie_init_port(struct rockchip_pcie *rockchip)
537 {
538 struct device *dev = rockchip->dev;
539 int err, i;
540 u32 status;
541
542 gpiod_set_value_cansleep(rockchip->ep_gpio, 0);
543
544 err = reset_control_assert(rockchip->aclk_rst);
545 if (err) {
546 dev_err(dev, "assert aclk_rst err %d\n", err);
547 return err;
548 }
549
550 err = reset_control_assert(rockchip->pclk_rst);
551 if (err) {
552 dev_err(dev, "assert pclk_rst err %d\n", err);
553 return err;
554 }
555
556 err = reset_control_assert(rockchip->pm_rst);
557 if (err) {
558 dev_err(dev, "assert pm_rst err %d\n", err);
559 return err;
560 }
561
562 for (i = 0; i < MAX_LANE_NUM; i++) {
563 err = phy_init(rockchip->phys[i]);
564 if (err) {
565 dev_err(dev, "init phy%d err %d\n", i, err);
566 goto err_exit_phy;
567 }
568 }
569
570 err = reset_control_assert(rockchip->core_rst);
571 if (err) {
572 dev_err(dev, "assert core_rst err %d\n", err);
573 goto err_exit_phy;
574 }
575
576 err = reset_control_assert(rockchip->mgmt_rst);
577 if (err) {
578 dev_err(dev, "assert mgmt_rst err %d\n", err);
579 goto err_exit_phy;
580 }
581
582 err = reset_control_assert(rockchip->mgmt_sticky_rst);
583 if (err) {
584 dev_err(dev, "assert mgmt_sticky_rst err %d\n", err);
585 goto err_exit_phy;
586 }
587
588 err = reset_control_assert(rockchip->pipe_rst);
589 if (err) {
590 dev_err(dev, "assert pipe_rst err %d\n", err);
591 goto err_exit_phy;
592 }
593
594 udelay(10);
595
596 err = reset_control_deassert(rockchip->pm_rst);
597 if (err) {
598 dev_err(dev, "deassert pm_rst err %d\n", err);
599 goto err_exit_phy;
600 }
601
602 err = reset_control_deassert(rockchip->aclk_rst);
603 if (err) {
604 dev_err(dev, "deassert aclk_rst err %d\n", err);
605 goto err_exit_phy;
606 }
607
608 err = reset_control_deassert(rockchip->pclk_rst);
609 if (err) {
610 dev_err(dev, "deassert pclk_rst err %d\n", err);
611 goto err_exit_phy;
612 }
613
614 if (rockchip->link_gen == 2)
615 rockchip_pcie_write(rockchip, PCIE_CLIENT_GEN_SEL_2,
616 PCIE_CLIENT_CONFIG);
617 else
618 rockchip_pcie_write(rockchip, PCIE_CLIENT_GEN_SEL_1,
619 PCIE_CLIENT_CONFIG);
620
621 rockchip_pcie_write(rockchip,
622 PCIE_CLIENT_CONF_ENABLE |
623 PCIE_CLIENT_LINK_TRAIN_ENABLE |
624 PCIE_CLIENT_ARI_ENABLE |
625 PCIE_CLIENT_CONF_LANE_NUM(rockchip->lanes) |
626 PCIE_CLIENT_MODE_RC,
627 PCIE_CLIENT_CONFIG);
628
629 for (i = 0; i < MAX_LANE_NUM; i++) {
630 err = phy_power_on(rockchip->phys[i]);
631 if (err) {
632 dev_err(dev, "power on phy%d err %d\n", i, err);
633 goto err_power_off_phy;
634 }
635 }
636
637 /*
638 * Please don't reorder the deassert sequence of the following
639 * four reset pins.
640 */
641 err = reset_control_deassert(rockchip->mgmt_sticky_rst);
642 if (err) {
643 dev_err(dev, "deassert mgmt_sticky_rst err %d\n", err);
644 goto err_power_off_phy;
645 }
646
647 err = reset_control_deassert(rockchip->core_rst);
648 if (err) {
649 dev_err(dev, "deassert core_rst err %d\n", err);
650 goto err_power_off_phy;
651 }
652
653 err = reset_control_deassert(rockchip->mgmt_rst);
654 if (err) {
655 dev_err(dev, "deassert mgmt_rst err %d\n", err);
656 goto err_power_off_phy;
657 }
658
659 err = reset_control_deassert(rockchip->pipe_rst);
660 if (err) {
661 dev_err(dev, "deassert pipe_rst err %d\n", err);
662 goto err_power_off_phy;
663 }
664
665 /* Fix the transmitted FTS count desired to exit from L0s. */
666 status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL_PLC1);
667 status = (status & ~PCIE_CORE_CTRL_PLC1_FTS_MASK) |
668 (PCIE_CORE_CTRL_PLC1_FTS_CNT << PCIE_CORE_CTRL_PLC1_FTS_SHIFT);
669 rockchip_pcie_write(rockchip, status, PCIE_CORE_CTRL_PLC1);
670
671 rockchip_pcie_set_power_limit(rockchip);
672
673 /* Set RC's clock architecture as common clock */
674 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
675 status |= PCI_EXP_LNKSTA_SLC << 16;
676 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
677
678 /* Set RC's RCB to 128 */
679 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
680 status |= PCI_EXP_LNKCTL_RCB;
681 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
682
683 /* Enable Gen1 training */
684 rockchip_pcie_write(rockchip, PCIE_CLIENT_LINK_TRAIN_ENABLE,
685 PCIE_CLIENT_CONFIG);
686
687 gpiod_set_value_cansleep(rockchip->ep_gpio, 1);
688
689 /* 500ms timeout value should be enough for Gen1/2 training */
690 err = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_BASIC_STATUS1,
691 status, PCIE_LINK_UP(status), 20,
692 500 * USEC_PER_MSEC);
693 if (err) {
694 dev_err(dev, "PCIe link training gen1 timeout!\n");
695 goto err_power_off_phy;
696 }
697
698 if (rockchip->link_gen == 2) {
699 /*
700 * Enable retrain for gen2. This should be configured only after
701 * gen1 finished.
702 */
703 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
704 status |= PCI_EXP_LNKCTL_RL;
705 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
706
707 err = readl_poll_timeout(rockchip->apb_base + PCIE_CORE_CTRL,
708 status, PCIE_LINK_IS_GEN2(status), 20,
709 500 * USEC_PER_MSEC);
710 if (err)
711 dev_dbg(dev, "PCIe link training gen2 timeout, fall back to gen1!\n");
712 }
713
714 /* Check the final link width from negotiated lane counter from MGMT */
715 status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL);
716 status = 0x1 << ((status & PCIE_CORE_PL_CONF_LANE_MASK) >>
717 PCIE_CORE_PL_CONF_LANE_SHIFT);
718 dev_dbg(dev, "current link width is x%d\n", status);
719
720 /* Power off unused lane(s) */
721 rockchip->lanes_map = rockchip_pcie_lane_map(rockchip);
722 for (i = 0; i < MAX_LANE_NUM; i++) {
723 if (!(rockchip->lanes_map & BIT(i))) {
724 dev_dbg(dev, "idling lane %d\n", i);
725 phy_power_off(rockchip->phys[i]);
726 }
727 }
728
729 rockchip_pcie_write(rockchip, ROCKCHIP_VENDOR_ID,
730 PCIE_CORE_CONFIG_VENDOR);
731 rockchip_pcie_write(rockchip,
732 PCI_CLASS_BRIDGE_PCI << PCIE_RC_CONFIG_SCC_SHIFT,
733 PCIE_RC_CONFIG_RID_CCR);
734
735 /* Clear THP cap's next cap pointer to remove L1 substate cap */
736 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_THP_CAP);
737 status &= ~PCIE_RC_CONFIG_THP_CAP_NEXT_MASK;
738 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_THP_CAP);
739
740 /* Clear L0s from RC's link cap */
741 if (of_property_read_bool(dev->of_node, "aspm-no-l0s")) {
742 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LINK_CAP);
743 status &= ~PCIE_RC_CONFIG_LINK_CAP_L0S;
744 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LINK_CAP);
745 }
746
747 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_DCSR);
748 status &= ~PCIE_RC_CONFIG_DCSR_MPS_MASK;
749 status |= PCIE_RC_CONFIG_DCSR_MPS_256;
750 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_DCSR);
751
752 return 0;
753 err_power_off_phy:
754 while (i--)
755 phy_power_off(rockchip->phys[i]);
756 i = MAX_LANE_NUM;
757 err_exit_phy:
758 while (i--)
759 phy_exit(rockchip->phys[i]);
760 return err;
761 }
762
763 static void rockchip_pcie_deinit_phys(struct rockchip_pcie *rockchip)
764 {
765 int i;
766
767 for (i = 0; i < MAX_LANE_NUM; i++) {
768 /* inactive lanes are already powered off */
769 if (rockchip->lanes_map & BIT(i))
770 phy_power_off(rockchip->phys[i]);
771 phy_exit(rockchip->phys[i]);
772 }
773 }
774
775 static irqreturn_t rockchip_pcie_subsys_irq_handler(int irq, void *arg)
776 {
777 struct rockchip_pcie *rockchip = arg;
778 struct device *dev = rockchip->dev;
779 u32 reg;
780 u32 sub_reg;
781
782 reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS);
783 if (reg & PCIE_CLIENT_INT_LOCAL) {
784 dev_dbg(dev, "local interrupt received\n");
785 sub_reg = rockchip_pcie_read(rockchip, PCIE_CORE_INT_STATUS);
786 if (sub_reg & PCIE_CORE_INT_PRFPE)
787 dev_dbg(dev, "parity error detected while reading from the PNP receive FIFO RAM\n");
788
789 if (sub_reg & PCIE_CORE_INT_CRFPE)
790 dev_dbg(dev, "parity error detected while reading from the Completion Receive FIFO RAM\n");
791
792 if (sub_reg & PCIE_CORE_INT_RRPE)
793 dev_dbg(dev, "parity error detected while reading from replay buffer RAM\n");
794
795 if (sub_reg & PCIE_CORE_INT_PRFO)
796 dev_dbg(dev, "overflow occurred in the PNP receive FIFO\n");
797
798 if (sub_reg & PCIE_CORE_INT_CRFO)
799 dev_dbg(dev, "overflow occurred in the completion receive FIFO\n");
800
801 if (sub_reg & PCIE_CORE_INT_RT)
802 dev_dbg(dev, "replay timer timed out\n");
803
804 if (sub_reg & PCIE_CORE_INT_RTR)
805 dev_dbg(dev, "replay timer rolled over after 4 transmissions of the same TLP\n");
806
807 if (sub_reg & PCIE_CORE_INT_PE)
808 dev_dbg(dev, "phy error detected on receive side\n");
809
810 if (sub_reg & PCIE_CORE_INT_MTR)
811 dev_dbg(dev, "malformed TLP received from the link\n");
812
813 if (sub_reg & PCIE_CORE_INT_UCR)
814 dev_dbg(dev, "malformed TLP received from the link\n");
815
816 if (sub_reg & PCIE_CORE_INT_FCE)
817 dev_dbg(dev, "an error was observed in the flow control advertisements from the other side\n");
818
819 if (sub_reg & PCIE_CORE_INT_CT)
820 dev_dbg(dev, "a request timed out waiting for completion\n");
821
822 if (sub_reg & PCIE_CORE_INT_UTC)
823 dev_dbg(dev, "unmapped TC error\n");
824
825 if (sub_reg & PCIE_CORE_INT_MMVC)
826 dev_dbg(dev, "MSI mask register changes\n");
827
828 rockchip_pcie_write(rockchip, sub_reg, PCIE_CORE_INT_STATUS);
829 } else if (reg & PCIE_CLIENT_INT_PHY) {
830 dev_dbg(dev, "phy link changes\n");
831 rockchip_pcie_update_txcredit_mui(rockchip);
832 rockchip_pcie_clr_bw_int(rockchip);
833 }
834
835 rockchip_pcie_write(rockchip, reg & PCIE_CLIENT_INT_LOCAL,
836 PCIE_CLIENT_INT_STATUS);
837
838 return IRQ_HANDLED;
839 }
840
841 static irqreturn_t rockchip_pcie_client_irq_handler(int irq, void *arg)
842 {
843 struct rockchip_pcie *rockchip = arg;
844 struct device *dev = rockchip->dev;
845 u32 reg;
846
847 reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS);
848 if (reg & PCIE_CLIENT_INT_LEGACY_DONE)
849 dev_dbg(dev, "legacy done interrupt received\n");
850
851 if (reg & PCIE_CLIENT_INT_MSG)
852 dev_dbg(dev, "message done interrupt received\n");
853
854 if (reg & PCIE_CLIENT_INT_HOT_RST)
855 dev_dbg(dev, "hot reset interrupt received\n");
856
857 if (reg & PCIE_CLIENT_INT_DPA)
858 dev_dbg(dev, "dpa interrupt received\n");
859
860 if (reg & PCIE_CLIENT_INT_FATAL_ERR)
861 dev_dbg(dev, "fatal error interrupt received\n");
862
863 if (reg & PCIE_CLIENT_INT_NFATAL_ERR)
864 dev_dbg(dev, "no fatal error interrupt received\n");
865
866 if (reg & PCIE_CLIENT_INT_CORR_ERR)
867 dev_dbg(dev, "correctable error interrupt received\n");
868
869 if (reg & PCIE_CLIENT_INT_PHY)
870 dev_dbg(dev, "phy interrupt received\n");
871
872 rockchip_pcie_write(rockchip, reg & (PCIE_CLIENT_INT_LEGACY_DONE |
873 PCIE_CLIENT_INT_MSG | PCIE_CLIENT_INT_HOT_RST |
874 PCIE_CLIENT_INT_DPA | PCIE_CLIENT_INT_FATAL_ERR |
875 PCIE_CLIENT_INT_NFATAL_ERR |
876 PCIE_CLIENT_INT_CORR_ERR |
877 PCIE_CLIENT_INT_PHY),
878 PCIE_CLIENT_INT_STATUS);
879
880 return IRQ_HANDLED;
881 }
882
883 static void rockchip_pcie_legacy_int_handler(struct irq_desc *desc)
884 {
885 struct irq_chip *chip = irq_desc_get_chip(desc);
886 struct rockchip_pcie *rockchip = irq_desc_get_handler_data(desc);
887 struct device *dev = rockchip->dev;
888 u32 reg;
889 u32 hwirq;
890 u32 virq;
891
892 chained_irq_enter(chip, desc);
893
894 reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS);
895 reg = (reg & PCIE_CLIENT_INTR_MASK) >> PCIE_CLIENT_INTR_SHIFT;
896
897 while (reg) {
898 hwirq = ffs(reg) - 1;
899 reg &= ~BIT(hwirq);
900
901 virq = irq_find_mapping(rockchip->irq_domain, hwirq);
902 if (virq)
903 generic_handle_irq(virq);
904 else
905 dev_err(dev, "unexpected IRQ, INT%d\n", hwirq);
906 }
907
908 chained_irq_exit(chip, desc);
909 }
910
911 static int rockchip_pcie_get_phys(struct rockchip_pcie *rockchip)
912 {
913 struct device *dev = rockchip->dev;
914 struct phy *phy;
915 char *name;
916 u32 i;
917
918 phy = devm_phy_get(dev, "pcie-phy");
919 if (!IS_ERR(phy)) {
920 rockchip->legacy_phy = true;
921 rockchip->phys[0] = phy;
922 dev_warn(dev, "legacy phy model is deprecated!\n");
923 return 0;
924 }
925
926 if (PTR_ERR(phy) == -EPROBE_DEFER)
927 return PTR_ERR(phy);
928
929 dev_dbg(dev, "missing legacy phy; search for per-lane PHY\n");
930
931 for (i = 0; i < MAX_LANE_NUM; i++) {
932 name = kasprintf(GFP_KERNEL, "pcie-phy-%u", i);
933 if (!name)
934 return -ENOMEM;
935
936 phy = devm_of_phy_get(dev, dev->of_node, name);
937 kfree(name);
938
939 if (IS_ERR(phy)) {
940 if (PTR_ERR(phy) != -EPROBE_DEFER)
941 dev_err(dev, "missing phy for lane %d: %ld\n",
942 i, PTR_ERR(phy));
943 return PTR_ERR(phy);
944 }
945
946 rockchip->phys[i] = phy;
947 }
948
949 return 0;
950 }
951
952 static int rockchip_pcie_setup_irq(struct rockchip_pcie *rockchip)
953 {
954 int irq, err;
955 struct device *dev = rockchip->dev;
956 struct platform_device *pdev = to_platform_device(dev);
957
958 irq = platform_get_irq_byname(pdev, "sys");
959 if (irq < 0) {
960 dev_err(dev, "missing sys IRQ resource\n");
961 return irq;
962 }
963
964 err = devm_request_irq(dev, irq, rockchip_pcie_subsys_irq_handler,
965 IRQF_SHARED, "pcie-sys", rockchip);
966 if (err) {
967 dev_err(dev, "failed to request PCIe subsystem IRQ\n");
968 return err;
969 }
970
971 irq = platform_get_irq_byname(pdev, "legacy");
972 if (irq < 0) {
973 dev_err(dev, "missing legacy IRQ resource\n");
974 return irq;
975 }
976
977 irq_set_chained_handler_and_data(irq,
978 rockchip_pcie_legacy_int_handler,
979 rockchip);
980
981 irq = platform_get_irq_byname(pdev, "client");
982 if (irq < 0) {
983 dev_err(dev, "missing client IRQ resource\n");
984 return irq;
985 }
986
987 err = devm_request_irq(dev, irq, rockchip_pcie_client_irq_handler,
988 IRQF_SHARED, "pcie-client", rockchip);
989 if (err) {
990 dev_err(dev, "failed to request PCIe client IRQ\n");
991 return err;
992 }
993
994 return 0;
995 }
996
997 /**
998 * rockchip_pcie_parse_dt - Parse Device Tree
999 * @rockchip: PCIe port information
1000 *
1001 * Return: '0' on success and error value on failure
1002 */
1003 static int rockchip_pcie_parse_dt(struct rockchip_pcie *rockchip)
1004 {
1005 struct device *dev = rockchip->dev;
1006 struct platform_device *pdev = to_platform_device(dev);
1007 struct device_node *node = dev->of_node;
1008 struct resource *regs;
1009 int err;
1010
1011 regs = platform_get_resource_byname(pdev,
1012 IORESOURCE_MEM,
1013 "axi-base");
1014 rockchip->reg_base = devm_pci_remap_cfg_resource(dev, regs);
1015 if (IS_ERR(rockchip->reg_base))
1016 return PTR_ERR(rockchip->reg_base);
1017
1018 regs = platform_get_resource_byname(pdev,
1019 IORESOURCE_MEM,
1020 "apb-base");
1021 rockchip->apb_base = devm_ioremap_resource(dev, regs);
1022 if (IS_ERR(rockchip->apb_base))
1023 return PTR_ERR(rockchip->apb_base);
1024
1025 err = rockchip_pcie_get_phys(rockchip);
1026 if (err)
1027 return err;
1028
1029 rockchip->lanes = 1;
1030 err = of_property_read_u32(node, "num-lanes", &rockchip->lanes);
1031 if (!err && (rockchip->lanes == 0 ||
1032 rockchip->lanes == 3 ||
1033 rockchip->lanes > 4)) {
1034 dev_warn(dev, "invalid num-lanes, default to use one lane\n");
1035 rockchip->lanes = 1;
1036 }
1037
1038 rockchip->link_gen = of_pci_get_max_link_speed(node);
1039 if (rockchip->link_gen < 0 || rockchip->link_gen > 2)
1040 rockchip->link_gen = 2;
1041
1042 rockchip->core_rst = devm_reset_control_get_exclusive(dev, "core");
1043 if (IS_ERR(rockchip->core_rst)) {
1044 if (PTR_ERR(rockchip->core_rst) != -EPROBE_DEFER)
1045 dev_err(dev, "missing core reset property in node\n");
1046 return PTR_ERR(rockchip->core_rst);
1047 }
1048
1049 rockchip->mgmt_rst = devm_reset_control_get_exclusive(dev, "mgmt");
1050 if (IS_ERR(rockchip->mgmt_rst)) {
1051 if (PTR_ERR(rockchip->mgmt_rst) != -EPROBE_DEFER)
1052 dev_err(dev, "missing mgmt reset property in node\n");
1053 return PTR_ERR(rockchip->mgmt_rst);
1054 }
1055
1056 rockchip->mgmt_sticky_rst = devm_reset_control_get_exclusive(dev,
1057 "mgmt-sticky");
1058 if (IS_ERR(rockchip->mgmt_sticky_rst)) {
1059 if (PTR_ERR(rockchip->mgmt_sticky_rst) != -EPROBE_DEFER)
1060 dev_err(dev, "missing mgmt-sticky reset property in node\n");
1061 return PTR_ERR(rockchip->mgmt_sticky_rst);
1062 }
1063
1064 rockchip->pipe_rst = devm_reset_control_get_exclusive(dev, "pipe");
1065 if (IS_ERR(rockchip->pipe_rst)) {
1066 if (PTR_ERR(rockchip->pipe_rst) != -EPROBE_DEFER)
1067 dev_err(dev, "missing pipe reset property in node\n");
1068 return PTR_ERR(rockchip->pipe_rst);
1069 }
1070
1071 rockchip->pm_rst = devm_reset_control_get_exclusive(dev, "pm");
1072 if (IS_ERR(rockchip->pm_rst)) {
1073 if (PTR_ERR(rockchip->pm_rst) != -EPROBE_DEFER)
1074 dev_err(dev, "missing pm reset property in node\n");
1075 return PTR_ERR(rockchip->pm_rst);
1076 }
1077
1078 rockchip->pclk_rst = devm_reset_control_get_exclusive(dev, "pclk");
1079 if (IS_ERR(rockchip->pclk_rst)) {
1080 if (PTR_ERR(rockchip->pclk_rst) != -EPROBE_DEFER)
1081 dev_err(dev, "missing pclk reset property in node\n");
1082 return PTR_ERR(rockchip->pclk_rst);
1083 }
1084
1085 rockchip->aclk_rst = devm_reset_control_get_exclusive(dev, "aclk");
1086 if (IS_ERR(rockchip->aclk_rst)) {
1087 if (PTR_ERR(rockchip->aclk_rst) != -EPROBE_DEFER)
1088 dev_err(dev, "missing aclk reset property in node\n");
1089 return PTR_ERR(rockchip->aclk_rst);
1090 }
1091
1092 rockchip->ep_gpio = devm_gpiod_get(dev, "ep", GPIOD_OUT_HIGH);
1093 if (IS_ERR(rockchip->ep_gpio)) {
1094 dev_err(dev, "missing ep-gpios property in node\n");
1095 return PTR_ERR(rockchip->ep_gpio);
1096 }
1097
1098 rockchip->aclk_pcie = devm_clk_get(dev, "aclk");
1099 if (IS_ERR(rockchip->aclk_pcie)) {
1100 dev_err(dev, "aclk clock not found\n");
1101 return PTR_ERR(rockchip->aclk_pcie);
1102 }
1103
1104 rockchip->aclk_perf_pcie = devm_clk_get(dev, "aclk-perf");
1105 if (IS_ERR(rockchip->aclk_perf_pcie)) {
1106 dev_err(dev, "aclk_perf clock not found\n");
1107 return PTR_ERR(rockchip->aclk_perf_pcie);
1108 }
1109
1110 rockchip->hclk_pcie = devm_clk_get(dev, "hclk");
1111 if (IS_ERR(rockchip->hclk_pcie)) {
1112 dev_err(dev, "hclk clock not found\n");
1113 return PTR_ERR(rockchip->hclk_pcie);
1114 }
1115
1116 rockchip->clk_pcie_pm = devm_clk_get(dev, "pm");
1117 if (IS_ERR(rockchip->clk_pcie_pm)) {
1118 dev_err(dev, "pm clock not found\n");
1119 return PTR_ERR(rockchip->clk_pcie_pm);
1120 }
1121
1122 err = rockchip_pcie_setup_irq(rockchip);
1123 if (err)
1124 return err;
1125
1126 rockchip->vpcie12v = devm_regulator_get_optional(dev, "vpcie12v");
1127 if (IS_ERR(rockchip->vpcie12v)) {
1128 if (PTR_ERR(rockchip->vpcie12v) == -EPROBE_DEFER)
1129 return -EPROBE_DEFER;
1130 dev_info(dev, "no vpcie12v regulator found\n");
1131 }
1132
1133 rockchip->vpcie3v3 = devm_regulator_get_optional(dev, "vpcie3v3");
1134 if (IS_ERR(rockchip->vpcie3v3)) {
1135 if (PTR_ERR(rockchip->vpcie3v3) == -EPROBE_DEFER)
1136 return -EPROBE_DEFER;
1137 dev_info(dev, "no vpcie3v3 regulator found\n");
1138 }
1139
1140 rockchip->vpcie1v8 = devm_regulator_get_optional(dev, "vpcie1v8");
1141 if (IS_ERR(rockchip->vpcie1v8)) {
1142 if (PTR_ERR(rockchip->vpcie1v8) == -EPROBE_DEFER)
1143 return -EPROBE_DEFER;
1144 dev_info(dev, "no vpcie1v8 regulator found\n");
1145 }
1146
1147 rockchip->vpcie0v9 = devm_regulator_get_optional(dev, "vpcie0v9");
1148 if (IS_ERR(rockchip->vpcie0v9)) {
1149 if (PTR_ERR(rockchip->vpcie0v9) == -EPROBE_DEFER)
1150 return -EPROBE_DEFER;
1151 dev_info(dev, "no vpcie0v9 regulator found\n");
1152 }
1153
1154 return 0;
1155 }
1156
1157 static int rockchip_pcie_set_vpcie(struct rockchip_pcie *rockchip)
1158 {
1159 struct device *dev = rockchip->dev;
1160 int err;
1161
1162 if (!IS_ERR(rockchip->vpcie12v)) {
1163 err = regulator_enable(rockchip->vpcie12v);
1164 if (err) {
1165 dev_err(dev, "fail to enable vpcie12v regulator\n");
1166 goto err_out;
1167 }
1168 }
1169
1170 if (!IS_ERR(rockchip->vpcie3v3)) {
1171 err = regulator_enable(rockchip->vpcie3v3);
1172 if (err) {
1173 dev_err(dev, "fail to enable vpcie3v3 regulator\n");
1174 goto err_disable_12v;
1175 }
1176 }
1177
1178 if (!IS_ERR(rockchip->vpcie1v8)) {
1179 err = regulator_enable(rockchip->vpcie1v8);
1180 if (err) {
1181 dev_err(dev, "fail to enable vpcie1v8 regulator\n");
1182 goto err_disable_3v3;
1183 }
1184 }
1185
1186 if (!IS_ERR(rockchip->vpcie0v9)) {
1187 err = regulator_enable(rockchip->vpcie0v9);
1188 if (err) {
1189 dev_err(dev, "fail to enable vpcie0v9 regulator\n");
1190 goto err_disable_1v8;
1191 }
1192 }
1193
1194 return 0;
1195
1196 err_disable_1v8:
1197 if (!IS_ERR(rockchip->vpcie1v8))
1198 regulator_disable(rockchip->vpcie1v8);
1199 err_disable_3v3:
1200 if (!IS_ERR(rockchip->vpcie3v3))
1201 regulator_disable(rockchip->vpcie3v3);
1202 err_disable_12v:
1203 if (!IS_ERR(rockchip->vpcie12v))
1204 regulator_disable(rockchip->vpcie12v);
1205 err_out:
1206 return err;
1207 }
1208
1209 static void rockchip_pcie_enable_interrupts(struct rockchip_pcie *rockchip)
1210 {
1211 rockchip_pcie_write(rockchip, (PCIE_CLIENT_INT_CLI << 16) &
1212 (~PCIE_CLIENT_INT_CLI), PCIE_CLIENT_INT_MASK);
1213 rockchip_pcie_write(rockchip, (u32)(~PCIE_CORE_INT),
1214 PCIE_CORE_INT_MASK);
1215
1216 rockchip_pcie_enable_bw_int(rockchip);
1217 }
1218
1219 static int rockchip_pcie_intx_map(struct irq_domain *domain, unsigned int irq,
1220 irq_hw_number_t hwirq)
1221 {
1222 irq_set_chip_and_handler(irq, &dummy_irq_chip, handle_simple_irq);
1223 irq_set_chip_data(irq, domain->host_data);
1224
1225 return 0;
1226 }
1227
1228 static const struct irq_domain_ops intx_domain_ops = {
1229 .map = rockchip_pcie_intx_map,
1230 };
1231
1232 static int rockchip_pcie_init_irq_domain(struct rockchip_pcie *rockchip)
1233 {
1234 struct device *dev = rockchip->dev;
1235 struct device_node *intc = of_get_next_child(dev->of_node, NULL);
1236
1237 if (!intc) {
1238 dev_err(dev, "missing child interrupt-controller node\n");
1239 return -EINVAL;
1240 }
1241
1242 rockchip->irq_domain = irq_domain_add_linear(intc, PCI_NUM_INTX,
1243 &intx_domain_ops, rockchip);
1244 if (!rockchip->irq_domain) {
1245 dev_err(dev, "failed to get a INTx IRQ domain\n");
1246 return -EINVAL;
1247 }
1248
1249 return 0;
1250 }
1251
1252 static int rockchip_pcie_prog_ob_atu(struct rockchip_pcie *rockchip,
1253 int region_no, int type, u8 num_pass_bits,
1254 u32 lower_addr, u32 upper_addr)
1255 {
1256 u32 ob_addr_0;
1257 u32 ob_addr_1;
1258 u32 ob_desc_0;
1259 u32 aw_offset;
1260
1261 if (region_no >= MAX_AXI_WRAPPER_REGION_NUM)
1262 return -EINVAL;
1263 if (num_pass_bits + 1 < 8)
1264 return -EINVAL;
1265 if (num_pass_bits > 63)
1266 return -EINVAL;
1267 if (region_no == 0) {
1268 if (AXI_REGION_0_SIZE < (2ULL << num_pass_bits))
1269 return -EINVAL;
1270 }
1271 if (region_no != 0) {
1272 if (AXI_REGION_SIZE < (2ULL << num_pass_bits))
1273 return -EINVAL;
1274 }
1275
1276 aw_offset = (region_no << OB_REG_SIZE_SHIFT);
1277
1278 ob_addr_0 = num_pass_bits & PCIE_CORE_OB_REGION_ADDR0_NUM_BITS;
1279 ob_addr_0 |= lower_addr & PCIE_CORE_OB_REGION_ADDR0_LO_ADDR;
1280 ob_addr_1 = upper_addr;
1281 ob_desc_0 = (1 << 23 | type);
1282
1283 rockchip_pcie_write(rockchip, ob_addr_0,
1284 PCIE_CORE_OB_REGION_ADDR0 + aw_offset);
1285 rockchip_pcie_write(rockchip, ob_addr_1,
1286 PCIE_CORE_OB_REGION_ADDR1 + aw_offset);
1287 rockchip_pcie_write(rockchip, ob_desc_0,
1288 PCIE_CORE_OB_REGION_DESC0 + aw_offset);
1289 rockchip_pcie_write(rockchip, 0,
1290 PCIE_CORE_OB_REGION_DESC1 + aw_offset);
1291
1292 return 0;
1293 }
1294
1295 static int rockchip_pcie_prog_ib_atu(struct rockchip_pcie *rockchip,
1296 int region_no, u8 num_pass_bits,
1297 u32 lower_addr, u32 upper_addr)
1298 {
1299 u32 ib_addr_0;
1300 u32 ib_addr_1;
1301 u32 aw_offset;
1302
1303 if (region_no > MAX_AXI_IB_ROOTPORT_REGION_NUM)
1304 return -EINVAL;
1305 if (num_pass_bits + 1 < MIN_AXI_ADDR_BITS_PASSED)
1306 return -EINVAL;
1307 if (num_pass_bits > 63)
1308 return -EINVAL;
1309
1310 aw_offset = (region_no << IB_ROOT_PORT_REG_SIZE_SHIFT);
1311
1312 ib_addr_0 = num_pass_bits & PCIE_CORE_IB_REGION_ADDR0_NUM_BITS;
1313 ib_addr_0 |= (lower_addr << 8) & PCIE_CORE_IB_REGION_ADDR0_LO_ADDR;
1314 ib_addr_1 = upper_addr;
1315
1316 rockchip_pcie_write(rockchip, ib_addr_0, PCIE_RP_IB_ADDR0 + aw_offset);
1317 rockchip_pcie_write(rockchip, ib_addr_1, PCIE_RP_IB_ADDR1 + aw_offset);
1318
1319 return 0;
1320 }
1321
1322 static int rockchip_pcie_cfg_atu(struct rockchip_pcie *rockchip)
1323 {
1324 struct device *dev = rockchip->dev;
1325 int offset;
1326 int err;
1327 int reg_no;
1328
1329 rockchip_pcie_cfg_configuration_accesses(rockchip,
1330 AXI_WRAPPER_TYPE0_CFG);
1331
1332 for (reg_no = 0; reg_no < (rockchip->mem_size >> 20); reg_no++) {
1333 err = rockchip_pcie_prog_ob_atu(rockchip, reg_no + 1,
1334 AXI_WRAPPER_MEM_WRITE,
1335 20 - 1,
1336 rockchip->mem_bus_addr +
1337 (reg_no << 20),
1338 0);
1339 if (err) {
1340 dev_err(dev, "program RC mem outbound ATU failed\n");
1341 return err;
1342 }
1343 }
1344
1345 err = rockchip_pcie_prog_ib_atu(rockchip, 2, 32 - 1, 0x0, 0);
1346 if (err) {
1347 dev_err(dev, "program RC mem inbound ATU failed\n");
1348 return err;
1349 }
1350
1351 offset = rockchip->mem_size >> 20;
1352 for (reg_no = 0; reg_no < (rockchip->io_size >> 20); reg_no++) {
1353 err = rockchip_pcie_prog_ob_atu(rockchip,
1354 reg_no + 1 + offset,
1355 AXI_WRAPPER_IO_WRITE,
1356 20 - 1,
1357 rockchip->io_bus_addr +
1358 (reg_no << 20),
1359 0);
1360 if (err) {
1361 dev_err(dev, "program RC io outbound ATU failed\n");
1362 return err;
1363 }
1364 }
1365
1366 /* assign message regions */
1367 rockchip_pcie_prog_ob_atu(rockchip, reg_no + 1 + offset,
1368 AXI_WRAPPER_NOR_MSG,
1369 20 - 1, 0, 0);
1370
1371 rockchip->msg_bus_addr = rockchip->mem_bus_addr +
1372 ((reg_no + offset) << 20);
1373 return err;
1374 }
1375
1376 static int rockchip_pcie_wait_l2(struct rockchip_pcie *rockchip)
1377 {
1378 u32 value;
1379 int err;
1380
1381 /* send PME_TURN_OFF message */
1382 writel(0x0, rockchip->msg_region + PCIE_RC_SEND_PME_OFF);
1383
1384 /* read LTSSM and wait for falling into L2 link state */
1385 err = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_DEBUG_OUT_0,
1386 value, PCIE_LINK_IS_L2(value), 20,
1387 jiffies_to_usecs(5 * HZ));
1388 if (err) {
1389 dev_err(rockchip->dev, "PCIe link enter L2 timeout!\n");
1390 return err;
1391 }
1392
1393 return 0;
1394 }
1395
1396 static int rockchip_pcie_enable_clocks(struct rockchip_pcie *rockchip)
1397 {
1398 struct device *dev = rockchip->dev;
1399 int err;
1400
1401 err = clk_prepare_enable(rockchip->aclk_pcie);
1402 if (err) {
1403 dev_err(dev, "unable to enable aclk_pcie clock\n");
1404 return err;
1405 }
1406
1407 err = clk_prepare_enable(rockchip->aclk_perf_pcie);
1408 if (err) {
1409 dev_err(dev, "unable to enable aclk_perf_pcie clock\n");
1410 goto err_aclk_perf_pcie;
1411 }
1412
1413 err = clk_prepare_enable(rockchip->hclk_pcie);
1414 if (err) {
1415 dev_err(dev, "unable to enable hclk_pcie clock\n");
1416 goto err_hclk_pcie;
1417 }
1418
1419 err = clk_prepare_enable(rockchip->clk_pcie_pm);
1420 if (err) {
1421 dev_err(dev, "unable to enable clk_pcie_pm clock\n");
1422 goto err_clk_pcie_pm;
1423 }
1424
1425 return 0;
1426
1427 err_clk_pcie_pm:
1428 clk_disable_unprepare(rockchip->hclk_pcie);
1429 err_hclk_pcie:
1430 clk_disable_unprepare(rockchip->aclk_perf_pcie);
1431 err_aclk_perf_pcie:
1432 clk_disable_unprepare(rockchip->aclk_pcie);
1433 return err;
1434 }
1435
1436 static void rockchip_pcie_disable_clocks(void *data)
1437 {
1438 struct rockchip_pcie *rockchip = data;
1439
1440 clk_disable_unprepare(rockchip->clk_pcie_pm);
1441 clk_disable_unprepare(rockchip->hclk_pcie);
1442 clk_disable_unprepare(rockchip->aclk_perf_pcie);
1443 clk_disable_unprepare(rockchip->aclk_pcie);
1444 }
1445
1446 static int __maybe_unused rockchip_pcie_suspend_noirq(struct device *dev)
1447 {
1448 struct rockchip_pcie *rockchip = dev_get_drvdata(dev);
1449 int ret;
1450
1451 /* disable core and cli int since we don't need to ack PME_ACK */
1452 rockchip_pcie_write(rockchip, (PCIE_CLIENT_INT_CLI << 16) |
1453 PCIE_CLIENT_INT_CLI, PCIE_CLIENT_INT_MASK);
1454 rockchip_pcie_write(rockchip, (u32)PCIE_CORE_INT, PCIE_CORE_INT_MASK);
1455
1456 ret = rockchip_pcie_wait_l2(rockchip);
1457 if (ret) {
1458 rockchip_pcie_enable_interrupts(rockchip);
1459 return ret;
1460 }
1461
1462 rockchip_pcie_deinit_phys(rockchip);
1463
1464 rockchip_pcie_disable_clocks(rockchip);
1465
1466 if (!IS_ERR(rockchip->vpcie0v9))
1467 regulator_disable(rockchip->vpcie0v9);
1468
1469 return ret;
1470 }
1471
1472 static int __maybe_unused rockchip_pcie_resume_noirq(struct device *dev)
1473 {
1474 struct rockchip_pcie *rockchip = dev_get_drvdata(dev);
1475 int err;
1476
1477 if (!IS_ERR(rockchip->vpcie0v9)) {
1478 err = regulator_enable(rockchip->vpcie0v9);
1479 if (err) {
1480 dev_err(dev, "fail to enable vpcie0v9 regulator\n");
1481 return err;
1482 }
1483 }
1484
1485 err = rockchip_pcie_enable_clocks(rockchip);
1486 if (err)
1487 goto err_disable_0v9;
1488
1489 err = rockchip_pcie_init_port(rockchip);
1490 if (err)
1491 goto err_pcie_resume;
1492
1493 err = rockchip_pcie_cfg_atu(rockchip);
1494 if (err)
1495 goto err_err_deinit_port;
1496
1497 /* Need this to enter L1 again */
1498 rockchip_pcie_update_txcredit_mui(rockchip);
1499 rockchip_pcie_enable_interrupts(rockchip);
1500
1501 return 0;
1502
1503 err_err_deinit_port:
1504 rockchip_pcie_deinit_phys(rockchip);
1505 err_pcie_resume:
1506 rockchip_pcie_disable_clocks(rockchip);
1507 err_disable_0v9:
1508 if (!IS_ERR(rockchip->vpcie0v9))
1509 regulator_disable(rockchip->vpcie0v9);
1510 return err;
1511 }
1512
1513 static int rockchip_pcie_probe(struct platform_device *pdev)
1514 {
1515 struct rockchip_pcie *rockchip;
1516 struct device *dev = &pdev->dev;
1517 struct pci_bus *bus, *child;
1518 struct pci_host_bridge *bridge;
1519 struct resource_entry *win;
1520 resource_size_t io_base;
1521 struct resource *mem;
1522 struct resource *io;
1523 int err;
1524
1525 LIST_HEAD(res);
1526
1527 if (!dev->of_node)
1528 return -ENODEV;
1529
1530 bridge = devm_pci_alloc_host_bridge(dev, sizeof(*rockchip));
1531 if (!bridge)
1532 return -ENOMEM;
1533
1534 rockchip = pci_host_bridge_priv(bridge);
1535
1536 platform_set_drvdata(pdev, rockchip);
1537 rockchip->dev = dev;
1538
1539 err = rockchip_pcie_parse_dt(rockchip);
1540 if (err)
1541 return err;
1542
1543 err = rockchip_pcie_enable_clocks(rockchip);
1544 if (err)
1545 return err;
1546
1547 err = rockchip_pcie_set_vpcie(rockchip);
1548 if (err) {
1549 dev_err(dev, "failed to set vpcie regulator\n");
1550 goto err_set_vpcie;
1551 }
1552
1553 err = rockchip_pcie_init_port(rockchip);
1554 if (err)
1555 goto err_vpcie;
1556
1557 rockchip_pcie_enable_interrupts(rockchip);
1558
1559 err = rockchip_pcie_init_irq_domain(rockchip);
1560 if (err < 0)
1561 goto err_deinit_port;
1562
1563 err = of_pci_get_host_bridge_resources(dev->of_node, 0, 0xff,
1564 &res, &io_base);
1565 if (err)
1566 goto err_remove_irq_domain;
1567
1568 err = devm_request_pci_bus_resources(dev, &res);
1569 if (err)
1570 goto err_free_res;
1571
1572 /* Get the I/O and memory ranges from DT */
1573 resource_list_for_each_entry(win, &res) {
1574 switch (resource_type(win->res)) {
1575 case IORESOURCE_IO:
1576 io = win->res;
1577 io->name = "I/O";
1578 rockchip->io_size = resource_size(io);
1579 rockchip->io_bus_addr = io->start - win->offset;
1580 err = pci_remap_iospace(io, io_base);
1581 if (err) {
1582 dev_warn(dev, "error %d: failed to map resource %pR\n",
1583 err, io);
1584 continue;
1585 }
1586 rockchip->io = io;
1587 break;
1588 case IORESOURCE_MEM:
1589 mem = win->res;
1590 mem->name = "MEM";
1591 rockchip->mem_size = resource_size(mem);
1592 rockchip->mem_bus_addr = mem->start - win->offset;
1593 break;
1594 case IORESOURCE_BUS:
1595 rockchip->root_bus_nr = win->res->start;
1596 break;
1597 default:
1598 continue;
1599 }
1600 }
1601
1602 err = rockchip_pcie_cfg_atu(rockchip);
1603 if (err)
1604 goto err_unmap_iospace;
1605
1606 rockchip->msg_region = devm_ioremap(dev, rockchip->msg_bus_addr, SZ_1M);
1607 if (!rockchip->msg_region) {
1608 err = -ENOMEM;
1609 goto err_unmap_iospace;
1610 }
1611
1612 list_splice_init(&res, &bridge->windows);
1613 bridge->dev.parent = dev;
1614 bridge->sysdata = rockchip;
1615 bridge->busnr = 0;
1616 bridge->ops = &rockchip_pcie_ops;
1617 bridge->map_irq = of_irq_parse_and_map_pci;
1618 bridge->swizzle_irq = pci_common_swizzle;
1619
1620 err = pci_scan_root_bus_bridge(bridge);
1621 if (err < 0)
1622 goto err_unmap_iospace;
1623
1624 bus = bridge->bus;
1625
1626 rockchip->root_bus = bus;
1627
1628 pci_bus_size_bridges(bus);
1629 pci_bus_assign_resources(bus);
1630 list_for_each_entry(child, &bus->children, node)
1631 pcie_bus_configure_settings(child);
1632
1633 pci_bus_add_devices(bus);
1634 return 0;
1635
1636 err_unmap_iospace:
1637 pci_unmap_iospace(rockchip->io);
1638 err_free_res:
1639 pci_free_resource_list(&res);
1640 err_remove_irq_domain:
1641 irq_domain_remove(rockchip->irq_domain);
1642 err_deinit_port:
1643 rockchip_pcie_deinit_phys(rockchip);
1644 err_vpcie:
1645 if (!IS_ERR(rockchip->vpcie12v))
1646 regulator_disable(rockchip->vpcie12v);
1647 if (!IS_ERR(rockchip->vpcie3v3))
1648 regulator_disable(rockchip->vpcie3v3);
1649 if (!IS_ERR(rockchip->vpcie1v8))
1650 regulator_disable(rockchip->vpcie1v8);
1651 if (!IS_ERR(rockchip->vpcie0v9))
1652 regulator_disable(rockchip->vpcie0v9);
1653 err_set_vpcie:
1654 rockchip_pcie_disable_clocks(rockchip);
1655 return err;
1656 }
1657
1658 static int rockchip_pcie_remove(struct platform_device *pdev)
1659 {
1660 struct device *dev = &pdev->dev;
1661 struct rockchip_pcie *rockchip = dev_get_drvdata(dev);
1662
1663 pci_stop_root_bus(rockchip->root_bus);
1664 pci_remove_root_bus(rockchip->root_bus);
1665 pci_unmap_iospace(rockchip->io);
1666 irq_domain_remove(rockchip->irq_domain);
1667
1668 rockchip_pcie_deinit_phys(rockchip);
1669
1670 rockchip_pcie_disable_clocks(rockchip);
1671
1672 if (!IS_ERR(rockchip->vpcie12v))
1673 regulator_disable(rockchip->vpcie12v);
1674 if (!IS_ERR(rockchip->vpcie3v3))
1675 regulator_disable(rockchip->vpcie3v3);
1676 if (!IS_ERR(rockchip->vpcie1v8))
1677 regulator_disable(rockchip->vpcie1v8);
1678 if (!IS_ERR(rockchip->vpcie0v9))
1679 regulator_disable(rockchip->vpcie0v9);
1680
1681 return 0;
1682 }
1683
1684 static const struct dev_pm_ops rockchip_pcie_pm_ops = {
1685 SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(rockchip_pcie_suspend_noirq,
1686 rockchip_pcie_resume_noirq)
1687 };
1688
1689 static const struct of_device_id rockchip_pcie_of_match[] = {
1690 { .compatible = "rockchip,rk3399-pcie", },
1691 {}
1692 };
1693 MODULE_DEVICE_TABLE(of, rockchip_pcie_of_match);
1694
1695 static struct platform_driver rockchip_pcie_driver = {
1696 .driver = {
1697 .name = "rockchip-pcie",
1698 .of_match_table = rockchip_pcie_of_match,
1699 .pm = &rockchip_pcie_pm_ops,
1700 },
1701 .probe = rockchip_pcie_probe,
1702 .remove = rockchip_pcie_remove,
1703 };
1704 module_platform_driver(rockchip_pcie_driver);
1705
1706 MODULE_AUTHOR("Rockchip Inc");
1707 MODULE_DESCRIPTION("Rockchip AXI PCIe driver");
1708 MODULE_LICENSE("GPL v2");
Linux with added FPC-III support