?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
mm: memcontrol: use per-cpu stocks for socket memory uncharging
[linux/fpc-iii.git] / drivers / pci / host / pcie-rockchip.c
blob7bb9870f6d8ce0bd001529ef4c8dba1659e624a1
1 /*
2 * Rockchip AXI PCIe host controller driver
3 *
4 * Copyright (c) 2016 Rockchip, Inc.
5 *
6 * Author: Shawn Lin <shawn.lin@rock-chips.com>
7 * Wenrui Li <wenrui.li@rock-chips.com>
8 *
9 * Bits taken from Synopsys Designware Host controller driver and
10 * ARM PCI Host generic driver.
11 *
12 * This program is free software: you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License as published by
14 * the Free Software Foundation, either version 2 of the License, or
15 * (at your option) any later version.
16 */
17
18 #include <linux/clk.h>
19 #include <linux/delay.h>
20 #include <linux/gpio/consumer.h>
21 #include <linux/init.h>
22 #include <linux/interrupt.h>
23 #include <linux/iopoll.h>
24 #include <linux/irq.h>
25 #include <linux/irqchip/chained_irq.h>
26 #include <linux/irqdomain.h>
27 #include <linux/kernel.h>
28 #include <linux/mfd/syscon.h>
29 #include <linux/module.h>
30 #include <linux/of_address.h>
31 #include <linux/of_device.h>
32 #include <linux/of_pci.h>
33 #include <linux/of_platform.h>
34 #include <linux/of_irq.h>
35 #include <linux/pci.h>
36 #include <linux/pci_ids.h>
37 #include <linux/phy/phy.h>
38 #include <linux/platform_device.h>
39 #include <linux/reset.h>
40 #include <linux/regmap.h>
41
42 /*
43 * The upper 16 bits of PCIE_CLIENT_CONFIG are a write mask for the lower 16
44 * bits. This allows atomic updates of the register without locking.
45 */
46 #define HIWORD_UPDATE(mask, val) (((mask) << 16) | (val))
47 #define HIWORD_UPDATE_BIT(val) HIWORD_UPDATE(val, val)
48
49 #define ENCODE_LANES(x) ((((x) >> 1) & 3) << 4)
50
51 #define PCIE_CLIENT_BASE 0x0
52 #define PCIE_CLIENT_CONFIG (PCIE_CLIENT_BASE + 0x00)
53 #define PCIE_CLIENT_CONF_ENABLE HIWORD_UPDATE_BIT(0x0001)
54 #define PCIE_CLIENT_LINK_TRAIN_ENABLE HIWORD_UPDATE_BIT(0x0002)
55 #define PCIE_CLIENT_ARI_ENABLE HIWORD_UPDATE_BIT(0x0008)
56 #define PCIE_CLIENT_CONF_LANE_NUM(x) HIWORD_UPDATE(0x0030, ENCODE_LANES(x))
57 #define PCIE_CLIENT_MODE_RC HIWORD_UPDATE_BIT(0x0040)
58 #define PCIE_CLIENT_GEN_SEL_1 HIWORD_UPDATE(0x0080, 0)
59 #define PCIE_CLIENT_GEN_SEL_2 HIWORD_UPDATE_BIT(0x0080)
60 #define PCIE_CLIENT_DEBUG_OUT_0 (PCIE_CLIENT_BASE + 0x3c)
61 #define PCIE_CLIENT_DEBUG_LTSSM_MASK GENMASK(5, 0)
62 #define PCIE_CLIENT_DEBUG_LTSSM_L1 0x18
63 #define PCIE_CLIENT_DEBUG_LTSSM_L2 0x19
64 #define PCIE_CLIENT_BASIC_STATUS1 (PCIE_CLIENT_BASE + 0x48)
65 #define PCIE_CLIENT_LINK_STATUS_UP 0x00300000
66 #define PCIE_CLIENT_LINK_STATUS_MASK 0x00300000
67 #define PCIE_CLIENT_INT_MASK (PCIE_CLIENT_BASE + 0x4c)
68 #define PCIE_CLIENT_INT_STATUS (PCIE_CLIENT_BASE + 0x50)
69 #define PCIE_CLIENT_INTR_MASK GENMASK(8, 5)
70 #define PCIE_CLIENT_INTR_SHIFT 5
71 #define PCIE_CLIENT_INT_LEGACY_DONE BIT(15)
72 #define PCIE_CLIENT_INT_MSG BIT(14)
73 #define PCIE_CLIENT_INT_HOT_RST BIT(13)
74 #define PCIE_CLIENT_INT_DPA BIT(12)
75 #define PCIE_CLIENT_INT_FATAL_ERR BIT(11)
76 #define PCIE_CLIENT_INT_NFATAL_ERR BIT(10)
77 #define PCIE_CLIENT_INT_CORR_ERR BIT(9)
78 #define PCIE_CLIENT_INT_INTD BIT(8)
79 #define PCIE_CLIENT_INT_INTC BIT(7)
80 #define PCIE_CLIENT_INT_INTB BIT(6)
81 #define PCIE_CLIENT_INT_INTA BIT(5)
82 #define PCIE_CLIENT_INT_LOCAL BIT(4)
83 #define PCIE_CLIENT_INT_UDMA BIT(3)
84 #define PCIE_CLIENT_INT_PHY BIT(2)
85 #define PCIE_CLIENT_INT_HOT_PLUG BIT(1)
86 #define PCIE_CLIENT_INT_PWR_STCG BIT(0)
87
88 #define PCIE_CLIENT_INT_LEGACY \
89 (PCIE_CLIENT_INT_INTA | PCIE_CLIENT_INT_INTB | \
90 PCIE_CLIENT_INT_INTC | PCIE_CLIENT_INT_INTD)
91
92 #define PCIE_CLIENT_INT_CLI \
93 (PCIE_CLIENT_INT_CORR_ERR | PCIE_CLIENT_INT_NFATAL_ERR | \
94 PCIE_CLIENT_INT_FATAL_ERR | PCIE_CLIENT_INT_DPA | \
95 PCIE_CLIENT_INT_HOT_RST | PCIE_CLIENT_INT_MSG | \
96 PCIE_CLIENT_INT_LEGACY_DONE | PCIE_CLIENT_INT_LEGACY | \
97 PCIE_CLIENT_INT_PHY)
98
99 #define PCIE_CORE_CTRL_MGMT_BASE 0x900000
100 #define PCIE_CORE_CTRL (PCIE_CORE_CTRL_MGMT_BASE + 0x000)
101 #define PCIE_CORE_PL_CONF_SPEED_5G 0x00000008
102 #define PCIE_CORE_PL_CONF_SPEED_MASK 0x00000018
103 #define PCIE_CORE_PL_CONF_LANE_MASK 0x00000006
104 #define PCIE_CORE_PL_CONF_LANE_SHIFT 1
105 #define PCIE_CORE_CTRL_PLC1 (PCIE_CORE_CTRL_MGMT_BASE + 0x004)
106 #define PCIE_CORE_CTRL_PLC1_FTS_MASK GENMASK(23, 8)
107 #define PCIE_CORE_CTRL_PLC1_FTS_SHIFT 8
108 #define PCIE_CORE_CTRL_PLC1_FTS_CNT 0xffff
109 #define PCIE_CORE_TXCREDIT_CFG1 (PCIE_CORE_CTRL_MGMT_BASE + 0x020)
110 #define PCIE_CORE_TXCREDIT_CFG1_MUI_MASK 0xFFFF0000
111 #define PCIE_CORE_TXCREDIT_CFG1_MUI_SHIFT 16
112 #define PCIE_CORE_TXCREDIT_CFG1_MUI_ENCODE(x) \
113 (((x) >> 3) << PCIE_CORE_TXCREDIT_CFG1_MUI_SHIFT)
114 #define PCIE_CORE_INT_STATUS (PCIE_CORE_CTRL_MGMT_BASE + 0x20c)
115 #define PCIE_CORE_INT_PRFPE BIT(0)
116 #define PCIE_CORE_INT_CRFPE BIT(1)
117 #define PCIE_CORE_INT_RRPE BIT(2)
118 #define PCIE_CORE_INT_PRFO BIT(3)
119 #define PCIE_CORE_INT_CRFO BIT(4)
120 #define PCIE_CORE_INT_RT BIT(5)
121 #define PCIE_CORE_INT_RTR BIT(6)
122 #define PCIE_CORE_INT_PE BIT(7)
123 #define PCIE_CORE_INT_MTR BIT(8)
124 #define PCIE_CORE_INT_UCR BIT(9)
125 #define PCIE_CORE_INT_FCE BIT(10)
126 #define PCIE_CORE_INT_CT BIT(11)
127 #define PCIE_CORE_INT_UTC BIT(18)
128 #define PCIE_CORE_INT_MMVC BIT(19)
129 #define PCIE_CORE_CONFIG_VENDOR (PCIE_CORE_CTRL_MGMT_BASE + 0x44)
130 #define PCIE_CORE_INT_MASK (PCIE_CORE_CTRL_MGMT_BASE + 0x210)
131 #define PCIE_RC_BAR_CONF (PCIE_CORE_CTRL_MGMT_BASE + 0x300)
132
133 #define PCIE_CORE_INT \
134 (PCIE_CORE_INT_PRFPE | PCIE_CORE_INT_CRFPE | \
135 PCIE_CORE_INT_RRPE | PCIE_CORE_INT_CRFO | \
136 PCIE_CORE_INT_RT | PCIE_CORE_INT_RTR | \
137 PCIE_CORE_INT_PE | PCIE_CORE_INT_MTR | \
138 PCIE_CORE_INT_UCR | PCIE_CORE_INT_FCE | \
139 PCIE_CORE_INT_CT | PCIE_CORE_INT_UTC | \
140 PCIE_CORE_INT_MMVC)
141
142 #define PCIE_RC_CONFIG_NORMAL_BASE 0x800000
143 #define PCIE_RC_CONFIG_BASE 0xa00000
144 #define PCIE_RC_CONFIG_RID_CCR (PCIE_RC_CONFIG_BASE + 0x08)
145 #define PCIE_RC_CONFIG_SCC_SHIFT 16
146 #define PCIE_RC_CONFIG_DCR (PCIE_RC_CONFIG_BASE + 0xc4)
147 #define PCIE_RC_CONFIG_DCR_CSPL_SHIFT 18
148 #define PCIE_RC_CONFIG_DCR_CSPL_LIMIT 0xff
149 #define PCIE_RC_CONFIG_DCR_CPLS_SHIFT 26
150 #define PCIE_RC_CONFIG_DCSR (PCIE_RC_CONFIG_BASE + 0xc8)
151 #define PCIE_RC_CONFIG_DCSR_MPS_MASK GENMASK(7, 5)
152 #define PCIE_RC_CONFIG_DCSR_MPS_256 (0x1 << 5)
153 #define PCIE_RC_CONFIG_LINK_CAP (PCIE_RC_CONFIG_BASE + 0xcc)
154 #define PCIE_RC_CONFIG_LINK_CAP_L0S BIT(10)
155 #define PCIE_RC_CONFIG_LCS (PCIE_RC_CONFIG_BASE + 0xd0)
156 #define PCIE_RC_CONFIG_L1_SUBSTATE_CTRL2 (PCIE_RC_CONFIG_BASE + 0x90c)
157 #define PCIE_RC_CONFIG_THP_CAP (PCIE_RC_CONFIG_BASE + 0x274)
158 #define PCIE_RC_CONFIG_THP_CAP_NEXT_MASK GENMASK(31, 20)
159
160 #define PCIE_CORE_AXI_CONF_BASE 0xc00000
161 #define PCIE_CORE_OB_REGION_ADDR0 (PCIE_CORE_AXI_CONF_BASE + 0x0)
162 #define PCIE_CORE_OB_REGION_ADDR0_NUM_BITS 0x3f
163 #define PCIE_CORE_OB_REGION_ADDR0_LO_ADDR 0xffffff00
164 #define PCIE_CORE_OB_REGION_ADDR1 (PCIE_CORE_AXI_CONF_BASE + 0x4)
165 #define PCIE_CORE_OB_REGION_DESC0 (PCIE_CORE_AXI_CONF_BASE + 0x8)
166 #define PCIE_CORE_OB_REGION_DESC1 (PCIE_CORE_AXI_CONF_BASE + 0xc)
167
168 #define PCIE_CORE_AXI_INBOUND_BASE 0xc00800
169 #define PCIE_RP_IB_ADDR0 (PCIE_CORE_AXI_INBOUND_BASE + 0x0)
170 #define PCIE_CORE_IB_REGION_ADDR0_NUM_BITS 0x3f
171 #define PCIE_CORE_IB_REGION_ADDR0_LO_ADDR 0xffffff00
172 #define PCIE_RP_IB_ADDR1 (PCIE_CORE_AXI_INBOUND_BASE + 0x4)
173
174 /* Size of one AXI Region (not Region 0) */
175 #define AXI_REGION_SIZE BIT(20)
176 /* Size of Region 0, equal to sum of sizes of other regions */
177 #define AXI_REGION_0_SIZE (32 * (0x1 << 20))
178 #define OB_REG_SIZE_SHIFT 5
179 #define IB_ROOT_PORT_REG_SIZE_SHIFT 3
180 #define AXI_WRAPPER_IO_WRITE 0x6
181 #define AXI_WRAPPER_MEM_WRITE 0x2
182 #define AXI_WRAPPER_TYPE0_CFG 0xa
183 #define AXI_WRAPPER_TYPE1_CFG 0xb
184 #define AXI_WRAPPER_NOR_MSG 0xc
185
186 #define MAX_AXI_IB_ROOTPORT_REGION_NUM 3
187 #define MIN_AXI_ADDR_BITS_PASSED 8
188 #define PCIE_RC_SEND_PME_OFF 0x11960
189 #define ROCKCHIP_VENDOR_ID 0x1d87
190 #define PCIE_ECAM_BUS(x) (((x) & 0xff) << 20)
191 #define PCIE_ECAM_DEV(x) (((x) & 0x1f) << 15)
192 #define PCIE_ECAM_FUNC(x) (((x) & 0x7) << 12)
193 #define PCIE_ECAM_REG(x) (((x) & 0xfff) << 0)
194 #define PCIE_ECAM_ADDR(bus, dev, func, reg) \
195 (PCIE_ECAM_BUS(bus) | PCIE_ECAM_DEV(dev) | \
196 PCIE_ECAM_FUNC(func) | PCIE_ECAM_REG(reg))
197 #define PCIE_LINK_IS_L2(x) \
198 (((x) & PCIE_CLIENT_DEBUG_LTSSM_MASK) == PCIE_CLIENT_DEBUG_LTSSM_L2)
199 #define PCIE_LINK_UP(x) \
200 (((x) & PCIE_CLIENT_LINK_STATUS_MASK) == PCIE_CLIENT_LINK_STATUS_UP)
201 #define PCIE_LINK_IS_GEN2(x) \
202 (((x) & PCIE_CORE_PL_CONF_SPEED_MASK) == PCIE_CORE_PL_CONF_SPEED_5G)
203
204 #define RC_REGION_0_ADDR_TRANS_H 0x00000000
205 #define RC_REGION_0_ADDR_TRANS_L 0x00000000
206 #define RC_REGION_0_PASS_BITS (25 - 1)
207 #define RC_REGION_0_TYPE_MASK GENMASK(3, 0)
208 #define MAX_AXI_WRAPPER_REGION_NUM 33
209
210 struct rockchip_pcie {
211 void __iomem *reg_base; /* DT axi-base */
212 void __iomem *apb_base; /* DT apb-base */
213 struct phy *phy;
214 struct reset_control *core_rst;
215 struct reset_control *mgmt_rst;
216 struct reset_control *mgmt_sticky_rst;
217 struct reset_control *pipe_rst;
218 struct reset_control *pm_rst;
219 struct reset_control *aclk_rst;
220 struct reset_control *pclk_rst;
221 struct clk *aclk_pcie;
222 struct clk *aclk_perf_pcie;
223 struct clk *hclk_pcie;
224 struct clk *clk_pcie_pm;
225 struct regulator *vpcie3v3; /* 3.3V power supply */
226 struct regulator *vpcie1v8; /* 1.8V power supply */
227 struct regulator *vpcie0v9; /* 0.9V power supply */
228 struct gpio_desc *ep_gpio;
229 u32 lanes;
230 u8 root_bus_nr;
231 int link_gen;
232 struct device *dev;
233 struct irq_domain *irq_domain;
234 int offset;
235 struct pci_bus *root_bus;
236 struct resource *io;
237 phys_addr_t io_bus_addr;
238 u32 io_size;
239 void __iomem *msg_region;
240 u32 mem_size;
241 phys_addr_t msg_bus_addr;
242 phys_addr_t mem_bus_addr;
243 };
244
245 static u32 rockchip_pcie_read(struct rockchip_pcie *rockchip, u32 reg)
246 {
247 return readl(rockchip->apb_base + reg);
248 }
249
250 static void rockchip_pcie_write(struct rockchip_pcie *rockchip, u32 val,
251 u32 reg)
252 {
253 writel(val, rockchip->apb_base + reg);
254 }
255
256 static void rockchip_pcie_enable_bw_int(struct rockchip_pcie *rockchip)
257 {
258 u32 status;
259
260 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
261 status |= (PCI_EXP_LNKCTL_LBMIE | PCI_EXP_LNKCTL_LABIE);
262 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
263 }
264
265 static void rockchip_pcie_clr_bw_int(struct rockchip_pcie *rockchip)
266 {
267 u32 status;
268
269 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
270 status |= (PCI_EXP_LNKSTA_LBMS | PCI_EXP_LNKSTA_LABS) << 16;
271 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
272 }
273
274 static void rockchip_pcie_update_txcredit_mui(struct rockchip_pcie *rockchip)
275 {
276 u32 val;
277
278 /* Update Tx credit maximum update interval */
279 val = rockchip_pcie_read(rockchip, PCIE_CORE_TXCREDIT_CFG1);
280 val &= ~PCIE_CORE_TXCREDIT_CFG1_MUI_MASK;
281 val |= PCIE_CORE_TXCREDIT_CFG1_MUI_ENCODE(24000); /* ns */
282 rockchip_pcie_write(rockchip, val, PCIE_CORE_TXCREDIT_CFG1);
283 }
284
285 static int rockchip_pcie_valid_device(struct rockchip_pcie *rockchip,
286 struct pci_bus *bus, int dev)
287 {
288 /* access only one slot on each root port */
289 if (bus->number == rockchip->root_bus_nr && dev > 0)
290 return 0;
291
292 /*
293 * do not read more than one device on the bus directly attached
294 * to RC's downstream side.
295 */
296 if (bus->primary == rockchip->root_bus_nr && dev > 0)
297 return 0;
298
299 return 1;
300 }
301
302 static int rockchip_pcie_rd_own_conf(struct rockchip_pcie *rockchip,
303 int where, int size, u32 *val)
304 {
305 void __iomem *addr;
306
307 addr = rockchip->apb_base + PCIE_RC_CONFIG_NORMAL_BASE + where;
308
309 if (!IS_ALIGNED((uintptr_t)addr, size)) {
310 *val = 0;
311 return PCIBIOS_BAD_REGISTER_NUMBER;
312 }
313
314 if (size == 4) {
315 *val = readl(addr);
316 } else if (size == 2) {
317 *val = readw(addr);
318 } else if (size == 1) {
319 *val = readb(addr);
320 } else {
321 *val = 0;
322 return PCIBIOS_BAD_REGISTER_NUMBER;
323 }
324 return PCIBIOS_SUCCESSFUL;
325 }
326
327 static int rockchip_pcie_wr_own_conf(struct rockchip_pcie *rockchip,
328 int where, int size, u32 val)
329 {
330 u32 mask, tmp, offset;
331 void __iomem *addr;
332
333 offset = where & ~0x3;
334 addr = rockchip->apb_base + PCIE_RC_CONFIG_NORMAL_BASE + offset;
335
336 if (size == 4) {
337 writel(val, addr);
338 return PCIBIOS_SUCCESSFUL;
339 }
340
341 mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8));
342
343 /*
344 * N.B. This read/modify/write isn't safe in general because it can
345 * corrupt RW1C bits in adjacent registers. But the hardware
346 * doesn't support smaller writes.
347 */
348 tmp = readl(addr) & mask;
349 tmp |= val << ((where & 0x3) * 8);
350 writel(tmp, addr);
351
352 return PCIBIOS_SUCCESSFUL;
353 }
354
355 static void rockchip_pcie_cfg_configuration_accesses(
356 struct rockchip_pcie *rockchip, u32 type)
357 {
358 u32 ob_desc_0;
359
360 /* Configuration Accesses for region 0 */
361 rockchip_pcie_write(rockchip, 0x0, PCIE_RC_BAR_CONF);
362
363 rockchip_pcie_write(rockchip,
364 (RC_REGION_0_ADDR_TRANS_L + RC_REGION_0_PASS_BITS),
365 PCIE_CORE_OB_REGION_ADDR0);
366 rockchip_pcie_write(rockchip, RC_REGION_0_ADDR_TRANS_H,
367 PCIE_CORE_OB_REGION_ADDR1);
368 ob_desc_0 = rockchip_pcie_read(rockchip, PCIE_CORE_OB_REGION_DESC0);
369 ob_desc_0 &= ~(RC_REGION_0_TYPE_MASK);
370 ob_desc_0 |= (type | (0x1 << 23));
371 rockchip_pcie_write(rockchip, ob_desc_0, PCIE_CORE_OB_REGION_DESC0);
372 rockchip_pcie_write(rockchip, 0x0, PCIE_CORE_OB_REGION_DESC1);
373 }
374
375 static int rockchip_pcie_rd_other_conf(struct rockchip_pcie *rockchip,
376 struct pci_bus *bus, u32 devfn,
377 int where, int size, u32 *val)
378 {
379 u32 busdev;
380
381 busdev = PCIE_ECAM_ADDR(bus->number, PCI_SLOT(devfn),
382 PCI_FUNC(devfn), where);
383
384 if (!IS_ALIGNED(busdev, size)) {
385 *val = 0;
386 return PCIBIOS_BAD_REGISTER_NUMBER;
387 }
388
389 if (bus->parent->number == rockchip->root_bus_nr)
390 rockchip_pcie_cfg_configuration_accesses(rockchip,
391 AXI_WRAPPER_TYPE0_CFG);
392 else
393 rockchip_pcie_cfg_configuration_accesses(rockchip,
394 AXI_WRAPPER_TYPE1_CFG);
395
396 if (size == 4) {
397 *val = readl(rockchip->reg_base + busdev);
398 } else if (size == 2) {
399 *val = readw(rockchip->reg_base + busdev);
400 } else if (size == 1) {
401 *val = readb(rockchip->reg_base + busdev);
402 } else {
403 *val = 0;
404 return PCIBIOS_BAD_REGISTER_NUMBER;
405 }
406 return PCIBIOS_SUCCESSFUL;
407 }
408
409 static int rockchip_pcie_wr_other_conf(struct rockchip_pcie *rockchip,
410 struct pci_bus *bus, u32 devfn,
411 int where, int size, u32 val)
412 {
413 u32 busdev;
414
415 busdev = PCIE_ECAM_ADDR(bus->number, PCI_SLOT(devfn),
416 PCI_FUNC(devfn), where);
417 if (!IS_ALIGNED(busdev, size))
418 return PCIBIOS_BAD_REGISTER_NUMBER;
419
420 if (bus->parent->number == rockchip->root_bus_nr)
421 rockchip_pcie_cfg_configuration_accesses(rockchip,
422 AXI_WRAPPER_TYPE0_CFG);
423 else
424 rockchip_pcie_cfg_configuration_accesses(rockchip,
425 AXI_WRAPPER_TYPE1_CFG);
426
427 if (size == 4)
428 writel(val, rockchip->reg_base + busdev);
429 else if (size == 2)
430 writew(val, rockchip->reg_base + busdev);
431 else if (size == 1)
432 writeb(val, rockchip->reg_base + busdev);
433 else
434 return PCIBIOS_BAD_REGISTER_NUMBER;
435
436 return PCIBIOS_SUCCESSFUL;
437 }
438
439 static int rockchip_pcie_rd_conf(struct pci_bus *bus, u32 devfn, int where,
440 int size, u32 *val)
441 {
442 struct rockchip_pcie *rockchip = bus->sysdata;
443
444 if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn))) {
445 *val = 0xffffffff;
446 return PCIBIOS_DEVICE_NOT_FOUND;
447 }
448
449 if (bus->number == rockchip->root_bus_nr)
450 return rockchip_pcie_rd_own_conf(rockchip, where, size, val);
451
452 return rockchip_pcie_rd_other_conf(rockchip, bus, devfn, where, size, val);
453 }
454
455 static int rockchip_pcie_wr_conf(struct pci_bus *bus, u32 devfn,
456 int where, int size, u32 val)
457 {
458 struct rockchip_pcie *rockchip = bus->sysdata;
459
460 if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn)))
461 return PCIBIOS_DEVICE_NOT_FOUND;
462
463 if (bus->number == rockchip->root_bus_nr)
464 return rockchip_pcie_wr_own_conf(rockchip, where, size, val);
465
466 return rockchip_pcie_wr_other_conf(rockchip, bus, devfn, where, size, val);
467 }
468
469 static struct pci_ops rockchip_pcie_ops = {
470 .read = rockchip_pcie_rd_conf,
471 .write = rockchip_pcie_wr_conf,
472 };
473
474 static void rockchip_pcie_set_power_limit(struct rockchip_pcie *rockchip)
475 {
476 int curr;
477 u32 status, scale, power;
478
479 if (IS_ERR(rockchip->vpcie3v3))
480 return;
481
482 /*
483 * Set RC's captured slot power limit and scale if
484 * vpcie3v3 available. The default values are both zero
485 * which means the software should set these two according
486 * to the actual power supply.
487 */
488 curr = regulator_get_current_limit(rockchip->vpcie3v3);
489 if (curr <= 0)
490 return;
491
492 scale = 3; /* 0.001x */
493 curr = curr / 1000; /* convert to mA */
494 power = (curr * 3300) / 1000; /* milliwatt */
495 while (power > PCIE_RC_CONFIG_DCR_CSPL_LIMIT) {
496 if (!scale) {
497 dev_warn(rockchip->dev, "invalid power supply\n");
498 return;
499 }
500 scale--;
501 power = power / 10;
502 }
503
504 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_DCR);
505 status |= (power << PCIE_RC_CONFIG_DCR_CSPL_SHIFT) |
506 (scale << PCIE_RC_CONFIG_DCR_CPLS_SHIFT);
507 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_DCR);
508 }
509
510 /**
511 * rockchip_pcie_init_port - Initialize hardware
512 * @rockchip: PCIe port information
513 */
514 static int rockchip_pcie_init_port(struct rockchip_pcie *rockchip)
515 {
516 struct device *dev = rockchip->dev;
517 int err;
518 u32 status;
519
520 gpiod_set_value(rockchip->ep_gpio, 0);
521
522 err = reset_control_assert(rockchip->aclk_rst);
523 if (err) {
524 dev_err(dev, "assert aclk_rst err %d\n", err);
525 return err;
526 }
527
528 err = reset_control_assert(rockchip->pclk_rst);
529 if (err) {
530 dev_err(dev, "assert pclk_rst err %d\n", err);
531 return err;
532 }
533
534 err = reset_control_assert(rockchip->pm_rst);
535 if (err) {
536 dev_err(dev, "assert pm_rst err %d\n", err);
537 return err;
538 }
539
540 err = phy_init(rockchip->phy);
541 if (err < 0) {
542 dev_err(dev, "fail to init phy, err %d\n", err);
543 return err;
544 }
545
546 err = reset_control_assert(rockchip->core_rst);
547 if (err) {
548 dev_err(dev, "assert core_rst err %d\n", err);
549 return err;
550 }
551
552 err = reset_control_assert(rockchip->mgmt_rst);
553 if (err) {
554 dev_err(dev, "assert mgmt_rst err %d\n", err);
555 return err;
556 }
557
558 err = reset_control_assert(rockchip->mgmt_sticky_rst);
559 if (err) {
560 dev_err(dev, "assert mgmt_sticky_rst err %d\n", err);
561 return err;
562 }
563
564 err = reset_control_assert(rockchip->pipe_rst);
565 if (err) {
566 dev_err(dev, "assert pipe_rst err %d\n", err);
567 return err;
568 }
569
570 udelay(10);
571
572 err = reset_control_deassert(rockchip->pm_rst);
573 if (err) {
574 dev_err(dev, "deassert pm_rst err %d\n", err);
575 return err;
576 }
577
578 err = reset_control_deassert(rockchip->aclk_rst);
579 if (err) {
580 dev_err(dev, "deassert aclk_rst err %d\n", err);
581 return err;
582 }
583
584 err = reset_control_deassert(rockchip->pclk_rst);
585 if (err) {
586 dev_err(dev, "deassert pclk_rst err %d\n", err);
587 return err;
588 }
589
590 if (rockchip->link_gen == 2)
591 rockchip_pcie_write(rockchip, PCIE_CLIENT_GEN_SEL_2,
592 PCIE_CLIENT_CONFIG);
593 else
594 rockchip_pcie_write(rockchip, PCIE_CLIENT_GEN_SEL_1,
595 PCIE_CLIENT_CONFIG);
596
597 rockchip_pcie_write(rockchip,
598 PCIE_CLIENT_CONF_ENABLE |
599 PCIE_CLIENT_LINK_TRAIN_ENABLE |
600 PCIE_CLIENT_ARI_ENABLE |
601 PCIE_CLIENT_CONF_LANE_NUM(rockchip->lanes) |
602 PCIE_CLIENT_MODE_RC,
603 PCIE_CLIENT_CONFIG);
604
605 err = phy_power_on(rockchip->phy);
606 if (err) {
607 dev_err(dev, "fail to power on phy, err %d\n", err);
608 return err;
609 }
610
611 /*
612 * Please don't reorder the deassert sequence of the following
613 * four reset pins.
614 */
615 err = reset_control_deassert(rockchip->mgmt_sticky_rst);
616 if (err) {
617 dev_err(dev, "deassert mgmt_sticky_rst err %d\n", err);
618 return err;
619 }
620
621 err = reset_control_deassert(rockchip->core_rst);
622 if (err) {
623 dev_err(dev, "deassert core_rst err %d\n", err);
624 return err;
625 }
626
627 err = reset_control_deassert(rockchip->mgmt_rst);
628 if (err) {
629 dev_err(dev, "deassert mgmt_rst err %d\n", err);
630 return err;
631 }
632
633 err = reset_control_deassert(rockchip->pipe_rst);
634 if (err) {
635 dev_err(dev, "deassert pipe_rst err %d\n", err);
636 return err;
637 }
638
639 /* Fix the transmitted FTS count desired to exit from L0s. */
640 status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL_PLC1);
641 status = (status & ~PCIE_CORE_CTRL_PLC1_FTS_MASK) |
642 (PCIE_CORE_CTRL_PLC1_FTS_CNT << PCIE_CORE_CTRL_PLC1_FTS_SHIFT);
643 rockchip_pcie_write(rockchip, status, PCIE_CORE_CTRL_PLC1);
644
645 rockchip_pcie_set_power_limit(rockchip);
646
647 /* Set RC's clock architecture as common clock */
648 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
649 status |= PCI_EXP_LNKSTA_SLC << 16;
650 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
651
652 /* Set RC's RCB to 128 */
653 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
654 status |= PCI_EXP_LNKCTL_RCB;
655 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
656
657 /* Enable Gen1 training */
658 rockchip_pcie_write(rockchip, PCIE_CLIENT_LINK_TRAIN_ENABLE,
659 PCIE_CLIENT_CONFIG);
660
661 gpiod_set_value(rockchip->ep_gpio, 1);
662
663 /* 500ms timeout value should be enough for Gen1/2 training */
664 err = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_BASIC_STATUS1,
665 status, PCIE_LINK_UP(status), 20,
666 500 * USEC_PER_MSEC);
667 if (err) {
668 dev_err(dev, "PCIe link training gen1 timeout!\n");
669 return -ETIMEDOUT;
670 }
671
672 if (rockchip->link_gen == 2) {
673 /*
674 * Enable retrain for gen2. This should be configured only after
675 * gen1 finished.
676 */
677 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
678 status |= PCI_EXP_LNKCTL_RL;
679 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
680
681 err = readl_poll_timeout(rockchip->apb_base + PCIE_CORE_CTRL,
682 status, PCIE_LINK_IS_GEN2(status), 20,
683 500 * USEC_PER_MSEC);
684 if (err)
685 dev_dbg(dev, "PCIe link training gen2 timeout, fall back to gen1!\n");
686 }
687
688 /* Check the final link width from negotiated lane counter from MGMT */
689 status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL);
690 status = 0x1 << ((status & PCIE_CORE_PL_CONF_LANE_MASK) >>
691 PCIE_CORE_PL_CONF_LANE_SHIFT);
692 dev_dbg(dev, "current link width is x%d\n", status);
693
694 rockchip_pcie_write(rockchip, ROCKCHIP_VENDOR_ID,
695 PCIE_CORE_CONFIG_VENDOR);
696 rockchip_pcie_write(rockchip,
697 PCI_CLASS_BRIDGE_PCI << PCIE_RC_CONFIG_SCC_SHIFT,
698 PCIE_RC_CONFIG_RID_CCR);
699
700 /* Clear THP cap's next cap pointer to remove L1 substate cap */
701 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_THP_CAP);
702 status &= ~PCIE_RC_CONFIG_THP_CAP_NEXT_MASK;
703 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_THP_CAP);
704
705 /* Clear L0s from RC's link cap */
706 if (of_property_read_bool(dev->of_node, "aspm-no-l0s")) {
707 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LINK_CAP);
708 status &= ~PCIE_RC_CONFIG_LINK_CAP_L0S;
709 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LINK_CAP);
710 }
711
712 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_DCSR);
713 status &= ~PCIE_RC_CONFIG_DCSR_MPS_MASK;
714 status |= PCIE_RC_CONFIG_DCSR_MPS_256;
715 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_DCSR);
716
717 return 0;
718 }
719
720 static irqreturn_t rockchip_pcie_subsys_irq_handler(int irq, void *arg)
721 {
722 struct rockchip_pcie *rockchip = arg;
723 struct device *dev = rockchip->dev;
724 u32 reg;
725 u32 sub_reg;
726
727 reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS);
728 if (reg & PCIE_CLIENT_INT_LOCAL) {
729 dev_dbg(dev, "local interrupt received\n");
730 sub_reg = rockchip_pcie_read(rockchip, PCIE_CORE_INT_STATUS);
731 if (sub_reg & PCIE_CORE_INT_PRFPE)
732 dev_dbg(dev, "parity error detected while reading from the PNP receive FIFO RAM\n");
733
734 if (sub_reg & PCIE_CORE_INT_CRFPE)
735 dev_dbg(dev, "parity error detected while reading from the Completion Receive FIFO RAM\n");
736
737 if (sub_reg & PCIE_CORE_INT_RRPE)
738 dev_dbg(dev, "parity error detected while reading from replay buffer RAM\n");
739
740 if (sub_reg & PCIE_CORE_INT_PRFO)
741 dev_dbg(dev, "overflow occurred in the PNP receive FIFO\n");
742
743 if (sub_reg & PCIE_CORE_INT_CRFO)
744 dev_dbg(dev, "overflow occurred in the completion receive FIFO\n");
745
746 if (sub_reg & PCIE_CORE_INT_RT)
747 dev_dbg(dev, "replay timer timed out\n");
748
749 if (sub_reg & PCIE_CORE_INT_RTR)
750 dev_dbg(dev, "replay timer rolled over after 4 transmissions of the same TLP\n");
751
752 if (sub_reg & PCIE_CORE_INT_PE)
753 dev_dbg(dev, "phy error detected on receive side\n");
754
755 if (sub_reg & PCIE_CORE_INT_MTR)
756 dev_dbg(dev, "malformed TLP received from the link\n");
757
758 if (sub_reg & PCIE_CORE_INT_UCR)
759 dev_dbg(dev, "malformed TLP received from the link\n");
760
761 if (sub_reg & PCIE_CORE_INT_FCE)
762 dev_dbg(dev, "an error was observed in the flow control advertisements from the other side\n");
763
764 if (sub_reg & PCIE_CORE_INT_CT)
765 dev_dbg(dev, "a request timed out waiting for completion\n");
766
767 if (sub_reg & PCIE_CORE_INT_UTC)
768 dev_dbg(dev, "unmapped TC error\n");
769
770 if (sub_reg & PCIE_CORE_INT_MMVC)
771 dev_dbg(dev, "MSI mask register changes\n");
772
773 rockchip_pcie_write(rockchip, sub_reg, PCIE_CORE_INT_STATUS);
774 } else if (reg & PCIE_CLIENT_INT_PHY) {
775 dev_dbg(dev, "phy link changes\n");
776 rockchip_pcie_update_txcredit_mui(rockchip);
777 rockchip_pcie_clr_bw_int(rockchip);
778 }
779
780 rockchip_pcie_write(rockchip, reg & PCIE_CLIENT_INT_LOCAL,
781 PCIE_CLIENT_INT_STATUS);
782
783 return IRQ_HANDLED;
784 }
785
786 static irqreturn_t rockchip_pcie_client_irq_handler(int irq, void *arg)
787 {
788 struct rockchip_pcie *rockchip = arg;
789 struct device *dev = rockchip->dev;
790 u32 reg;
791
792 reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS);
793 if (reg & PCIE_CLIENT_INT_LEGACY_DONE)
794 dev_dbg(dev, "legacy done interrupt received\n");
795
796 if (reg & PCIE_CLIENT_INT_MSG)
797 dev_dbg(dev, "message done interrupt received\n");
798
799 if (reg & PCIE_CLIENT_INT_HOT_RST)
800 dev_dbg(dev, "hot reset interrupt received\n");
801
802 if (reg & PCIE_CLIENT_INT_DPA)
803 dev_dbg(dev, "dpa interrupt received\n");
804
805 if (reg & PCIE_CLIENT_INT_FATAL_ERR)
806 dev_dbg(dev, "fatal error interrupt received\n");
807
808 if (reg & PCIE_CLIENT_INT_NFATAL_ERR)
809 dev_dbg(dev, "no fatal error interrupt received\n");
810
811 if (reg & PCIE_CLIENT_INT_CORR_ERR)
812 dev_dbg(dev, "correctable error interrupt received\n");
813
814 if (reg & PCIE_CLIENT_INT_PHY)
815 dev_dbg(dev, "phy interrupt received\n");
816
817 rockchip_pcie_write(rockchip, reg & (PCIE_CLIENT_INT_LEGACY_DONE |
818 PCIE_CLIENT_INT_MSG | PCIE_CLIENT_INT_HOT_RST |
819 PCIE_CLIENT_INT_DPA | PCIE_CLIENT_INT_FATAL_ERR |
820 PCIE_CLIENT_INT_NFATAL_ERR |
821 PCIE_CLIENT_INT_CORR_ERR |
822 PCIE_CLIENT_INT_PHY),
823 PCIE_CLIENT_INT_STATUS);
824
825 return IRQ_HANDLED;
826 }
827
828 static void rockchip_pcie_legacy_int_handler(struct irq_desc *desc)
829 {
830 struct irq_chip *chip = irq_desc_get_chip(desc);
831 struct rockchip_pcie *rockchip = irq_desc_get_handler_data(desc);
832 struct device *dev = rockchip->dev;
833 u32 reg;
834 u32 hwirq;
835 u32 virq;
836
837 chained_irq_enter(chip, desc);
838
839 reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS);
840 reg = (reg & PCIE_CLIENT_INTR_MASK) >> PCIE_CLIENT_INTR_SHIFT;
841
842 while (reg) {
843 hwirq = ffs(reg) - 1;
844 reg &= ~BIT(hwirq);
845
846 virq = irq_find_mapping(rockchip->irq_domain, hwirq);
847 if (virq)
848 generic_handle_irq(virq);
849 else
850 dev_err(dev, "unexpected IRQ, INT%d\n", hwirq);
851 }
852
853 chained_irq_exit(chip, desc);
854 }
855
856
857 /**
858 * rockchip_pcie_parse_dt - Parse Device Tree
859 * @rockchip: PCIe port information
860 *
861 * Return: '0' on success and error value on failure
862 */
863 static int rockchip_pcie_parse_dt(struct rockchip_pcie *rockchip)
864 {
865 struct device *dev = rockchip->dev;
866 struct platform_device *pdev = to_platform_device(dev);
867 struct device_node *node = dev->of_node;
868 struct resource *regs;
869 int irq;
870 int err;
871
872 regs = platform_get_resource_byname(pdev,
873 IORESOURCE_MEM,
874 "axi-base");
875 rockchip->reg_base = devm_pci_remap_cfg_resource(dev, regs);
876 if (IS_ERR(rockchip->reg_base))
877 return PTR_ERR(rockchip->reg_base);
878
879 regs = platform_get_resource_byname(pdev,
880 IORESOURCE_MEM,
881 "apb-base");
882 rockchip->apb_base = devm_ioremap_resource(dev, regs);
883 if (IS_ERR(rockchip->apb_base))
884 return PTR_ERR(rockchip->apb_base);
885
886 rockchip->phy = devm_phy_get(dev, "pcie-phy");
887 if (IS_ERR(rockchip->phy)) {
888 if (PTR_ERR(rockchip->phy) != -EPROBE_DEFER)
889 dev_err(dev, "missing phy\n");
890 return PTR_ERR(rockchip->phy);
891 }
892
893 rockchip->lanes = 1;
894 err = of_property_read_u32(node, "num-lanes", &rockchip->lanes);
895 if (!err && (rockchip->lanes == 0 ||
896 rockchip->lanes == 3 ||
897 rockchip->lanes > 4)) {
898 dev_warn(dev, "invalid num-lanes, default to use one lane\n");
899 rockchip->lanes = 1;
900 }
901
902 rockchip->link_gen = of_pci_get_max_link_speed(node);
903 if (rockchip->link_gen < 0 || rockchip->link_gen > 2)
904 rockchip->link_gen = 2;
905
906 rockchip->core_rst = devm_reset_control_get(dev, "core");
907 if (IS_ERR(rockchip->core_rst)) {
908 if (PTR_ERR(rockchip->core_rst) != -EPROBE_DEFER)
909 dev_err(dev, "missing core reset property in node\n");
910 return PTR_ERR(rockchip->core_rst);
911 }
912
913 rockchip->mgmt_rst = devm_reset_control_get(dev, "mgmt");
914 if (IS_ERR(rockchip->mgmt_rst)) {
915 if (PTR_ERR(rockchip->mgmt_rst) != -EPROBE_DEFER)
916 dev_err(dev, "missing mgmt reset property in node\n");
917 return PTR_ERR(rockchip->mgmt_rst);
918 }
919
920 rockchip->mgmt_sticky_rst = devm_reset_control_get(dev, "mgmt-sticky");
921 if (IS_ERR(rockchip->mgmt_sticky_rst)) {
922 if (PTR_ERR(rockchip->mgmt_sticky_rst) != -EPROBE_DEFER)
923 dev_err(dev, "missing mgmt-sticky reset property in node\n");
924 return PTR_ERR(rockchip->mgmt_sticky_rst);
925 }
926
927 rockchip->pipe_rst = devm_reset_control_get(dev, "pipe");
928 if (IS_ERR(rockchip->pipe_rst)) {
929 if (PTR_ERR(rockchip->pipe_rst) != -EPROBE_DEFER)
930 dev_err(dev, "missing pipe reset property in node\n");
931 return PTR_ERR(rockchip->pipe_rst);
932 }
933
934 rockchip->pm_rst = devm_reset_control_get(dev, "pm");
935 if (IS_ERR(rockchip->pm_rst)) {
936 if (PTR_ERR(rockchip->pm_rst) != -EPROBE_DEFER)
937 dev_err(dev, "missing pm reset property in node\n");
938 return PTR_ERR(rockchip->pm_rst);
939 }
940
941 rockchip->pclk_rst = devm_reset_control_get(dev, "pclk");
942 if (IS_ERR(rockchip->pclk_rst)) {
943 if (PTR_ERR(rockchip->pclk_rst) != -EPROBE_DEFER)
944 dev_err(dev, "missing pclk reset property in node\n");
945 return PTR_ERR(rockchip->pclk_rst);
946 }
947
948 rockchip->aclk_rst = devm_reset_control_get(dev, "aclk");
949 if (IS_ERR(rockchip->aclk_rst)) {
950 if (PTR_ERR(rockchip->aclk_rst) != -EPROBE_DEFER)
951 dev_err(dev, "missing aclk reset property in node\n");
952 return PTR_ERR(rockchip->aclk_rst);
953 }
954
955 rockchip->ep_gpio = devm_gpiod_get(dev, "ep", GPIOD_OUT_HIGH);
956 if (IS_ERR(rockchip->ep_gpio)) {
957 dev_err(dev, "missing ep-gpios property in node\n");
958 return PTR_ERR(rockchip->ep_gpio);
959 }
960
961 rockchip->aclk_pcie = devm_clk_get(dev, "aclk");
962 if (IS_ERR(rockchip->aclk_pcie)) {
963 dev_err(dev, "aclk clock not found\n");
964 return PTR_ERR(rockchip->aclk_pcie);
965 }
966
967 rockchip->aclk_perf_pcie = devm_clk_get(dev, "aclk-perf");
968 if (IS_ERR(rockchip->aclk_perf_pcie)) {
969 dev_err(dev, "aclk_perf clock not found\n");
970 return PTR_ERR(rockchip->aclk_perf_pcie);
971 }
972
973 rockchip->hclk_pcie = devm_clk_get(dev, "hclk");
974 if (IS_ERR(rockchip->hclk_pcie)) {
975 dev_err(dev, "hclk clock not found\n");
976 return PTR_ERR(rockchip->hclk_pcie);
977 }
978
979 rockchip->clk_pcie_pm = devm_clk_get(dev, "pm");
980 if (IS_ERR(rockchip->clk_pcie_pm)) {
981 dev_err(dev, "pm clock not found\n");
982 return PTR_ERR(rockchip->clk_pcie_pm);
983 }
984
985 irq = platform_get_irq_byname(pdev, "sys");
986 if (irq < 0) {
987 dev_err(dev, "missing sys IRQ resource\n");
988 return -EINVAL;
989 }
990
991 err = devm_request_irq(dev, irq, rockchip_pcie_subsys_irq_handler,
992 IRQF_SHARED, "pcie-sys", rockchip);
993 if (err) {
994 dev_err(dev, "failed to request PCIe subsystem IRQ\n");
995 return err;
996 }
997
998 irq = platform_get_irq_byname(pdev, "legacy");
999 if (irq < 0) {
1000 dev_err(dev, "missing legacy IRQ resource\n");
1001 return -EINVAL;
1002 }
1003
1004 irq_set_chained_handler_and_data(irq,
1005 rockchip_pcie_legacy_int_handler,
1006 rockchip);
1007
1008 irq = platform_get_irq_byname(pdev, "client");
1009 if (irq < 0) {
1010 dev_err(dev, "missing client IRQ resource\n");
1011 return -EINVAL;
1012 }
1013
1014 err = devm_request_irq(dev, irq, rockchip_pcie_client_irq_handler,
1015 IRQF_SHARED, "pcie-client", rockchip);
1016 if (err) {
1017 dev_err(dev, "failed to request PCIe client IRQ\n");
1018 return err;
1019 }
1020
1021 rockchip->vpcie3v3 = devm_regulator_get_optional(dev, "vpcie3v3");
1022 if (IS_ERR(rockchip->vpcie3v3)) {
1023 if (PTR_ERR(rockchip->vpcie3v3) == -EPROBE_DEFER)
1024 return -EPROBE_DEFER;
1025 dev_info(dev, "no vpcie3v3 regulator found\n");
1026 }
1027
1028 rockchip->vpcie1v8 = devm_regulator_get_optional(dev, "vpcie1v8");
1029 if (IS_ERR(rockchip->vpcie1v8)) {
1030 if (PTR_ERR(rockchip->vpcie1v8) == -EPROBE_DEFER)
1031 return -EPROBE_DEFER;
1032 dev_info(dev, "no vpcie1v8 regulator found\n");
1033 }
1034
1035 rockchip->vpcie0v9 = devm_regulator_get_optional(dev, "vpcie0v9");
1036 if (IS_ERR(rockchip->vpcie0v9)) {
1037 if (PTR_ERR(rockchip->vpcie0v9) == -EPROBE_DEFER)
1038 return -EPROBE_DEFER;
1039 dev_info(dev, "no vpcie0v9 regulator found\n");
1040 }
1041
1042 return 0;
1043 }
1044
1045 static int rockchip_pcie_set_vpcie(struct rockchip_pcie *rockchip)
1046 {
1047 struct device *dev = rockchip->dev;
1048 int err;
1049
1050 if (!IS_ERR(rockchip->vpcie3v3)) {
1051 err = regulator_enable(rockchip->vpcie3v3);
1052 if (err) {
1053 dev_err(dev, "fail to enable vpcie3v3 regulator\n");
1054 goto err_out;
1055 }
1056 }
1057
1058 if (!IS_ERR(rockchip->vpcie1v8)) {
1059 err = regulator_enable(rockchip->vpcie1v8);
1060 if (err) {
1061 dev_err(dev, "fail to enable vpcie1v8 regulator\n");
1062 goto err_disable_3v3;
1063 }
1064 }
1065
1066 if (!IS_ERR(rockchip->vpcie0v9)) {
1067 err = regulator_enable(rockchip->vpcie0v9);
1068 if (err) {
1069 dev_err(dev, "fail to enable vpcie0v9 regulator\n");
1070 goto err_disable_1v8;
1071 }
1072 }
1073
1074 return 0;
1075
1076 err_disable_1v8:
1077 if (!IS_ERR(rockchip->vpcie1v8))
1078 regulator_disable(rockchip->vpcie1v8);
1079 err_disable_3v3:
1080 if (!IS_ERR(rockchip->vpcie3v3))
1081 regulator_disable(rockchip->vpcie3v3);
1082 err_out:
1083 return err;
1084 }
1085
1086 static void rockchip_pcie_enable_interrupts(struct rockchip_pcie *rockchip)
1087 {
1088 rockchip_pcie_write(rockchip, (PCIE_CLIENT_INT_CLI << 16) &
1089 (~PCIE_CLIENT_INT_CLI), PCIE_CLIENT_INT_MASK);
1090 rockchip_pcie_write(rockchip, (u32)(~PCIE_CORE_INT),
1091 PCIE_CORE_INT_MASK);
1092
1093 rockchip_pcie_enable_bw_int(rockchip);
1094 }
1095
1096 static int rockchip_pcie_intx_map(struct irq_domain *domain, unsigned int irq,
1097 irq_hw_number_t hwirq)
1098 {
1099 irq_set_chip_and_handler(irq, &dummy_irq_chip, handle_simple_irq);
1100 irq_set_chip_data(irq, domain->host_data);
1101
1102 return 0;
1103 }
1104
1105 static const struct irq_domain_ops intx_domain_ops = {
1106 .map = rockchip_pcie_intx_map,
1107 };
1108
1109 static int rockchip_pcie_init_irq_domain(struct rockchip_pcie *rockchip)
1110 {
1111 struct device *dev = rockchip->dev;
1112 struct device_node *intc = of_get_next_child(dev->of_node, NULL);
1113
1114 if (!intc) {
1115 dev_err(dev, "missing child interrupt-controller node\n");
1116 return -EINVAL;
1117 }
1118
1119 rockchip->irq_domain = irq_domain_add_linear(intc, 4,
1120 &intx_domain_ops, rockchip);
1121 if (!rockchip->irq_domain) {
1122 dev_err(dev, "failed to get a INTx IRQ domain\n");
1123 return -EINVAL;
1124 }
1125
1126 return 0;
1127 }
1128
1129 static int rockchip_pcie_prog_ob_atu(struct rockchip_pcie *rockchip,
1130 int region_no, int type, u8 num_pass_bits,
1131 u32 lower_addr, u32 upper_addr)
1132 {
1133 u32 ob_addr_0;
1134 u32 ob_addr_1;
1135 u32 ob_desc_0;
1136 u32 aw_offset;
1137
1138 if (region_no >= MAX_AXI_WRAPPER_REGION_NUM)
1139 return -EINVAL;
1140 if (num_pass_bits + 1 < 8)
1141 return -EINVAL;
1142 if (num_pass_bits > 63)
1143 return -EINVAL;
1144 if (region_no == 0) {
1145 if (AXI_REGION_0_SIZE < (2ULL << num_pass_bits))
1146 return -EINVAL;
1147 }
1148 if (region_no != 0) {
1149 if (AXI_REGION_SIZE < (2ULL << num_pass_bits))
1150 return -EINVAL;
1151 }
1152
1153 aw_offset = (region_no << OB_REG_SIZE_SHIFT);
1154
1155 ob_addr_0 = num_pass_bits & PCIE_CORE_OB_REGION_ADDR0_NUM_BITS;
1156 ob_addr_0 |= lower_addr & PCIE_CORE_OB_REGION_ADDR0_LO_ADDR;
1157 ob_addr_1 = upper_addr;
1158 ob_desc_0 = (1 << 23 | type);
1159
1160 rockchip_pcie_write(rockchip, ob_addr_0,
1161 PCIE_CORE_OB_REGION_ADDR0 + aw_offset);
1162 rockchip_pcie_write(rockchip, ob_addr_1,
1163 PCIE_CORE_OB_REGION_ADDR1 + aw_offset);
1164 rockchip_pcie_write(rockchip, ob_desc_0,
1165 PCIE_CORE_OB_REGION_DESC0 + aw_offset);
1166 rockchip_pcie_write(rockchip, 0,
1167 PCIE_CORE_OB_REGION_DESC1 + aw_offset);
1168
1169 return 0;
1170 }
1171
1172 static int rockchip_pcie_prog_ib_atu(struct rockchip_pcie *rockchip,
1173 int region_no, u8 num_pass_bits,
1174 u32 lower_addr, u32 upper_addr)
1175 {
1176 u32 ib_addr_0;
1177 u32 ib_addr_1;
1178 u32 aw_offset;
1179
1180 if (region_no > MAX_AXI_IB_ROOTPORT_REGION_NUM)
1181 return -EINVAL;
1182 if (num_pass_bits + 1 < MIN_AXI_ADDR_BITS_PASSED)
1183 return -EINVAL;
1184 if (num_pass_bits > 63)
1185 return -EINVAL;
1186
1187 aw_offset = (region_no << IB_ROOT_PORT_REG_SIZE_SHIFT);
1188
1189 ib_addr_0 = num_pass_bits & PCIE_CORE_IB_REGION_ADDR0_NUM_BITS;
1190 ib_addr_0 |= (lower_addr << 8) & PCIE_CORE_IB_REGION_ADDR0_LO_ADDR;
1191 ib_addr_1 = upper_addr;
1192
1193 rockchip_pcie_write(rockchip, ib_addr_0, PCIE_RP_IB_ADDR0 + aw_offset);
1194 rockchip_pcie_write(rockchip, ib_addr_1, PCIE_RP_IB_ADDR1 + aw_offset);
1195
1196 return 0;
1197 }
1198
1199 static int rockchip_pcie_cfg_atu(struct rockchip_pcie *rockchip)
1200 {
1201 struct device *dev = rockchip->dev;
1202 int offset;
1203 int err;
1204 int reg_no;
1205
1206 rockchip_pcie_cfg_configuration_accesses(rockchip,
1207 AXI_WRAPPER_TYPE0_CFG);
1208
1209 for (reg_no = 0; reg_no < (rockchip->mem_size >> 20); reg_no++) {
1210 err = rockchip_pcie_prog_ob_atu(rockchip, reg_no + 1,
1211 AXI_WRAPPER_MEM_WRITE,
1212 20 - 1,
1213 rockchip->mem_bus_addr +
1214 (reg_no << 20),
1215 0);
1216 if (err) {
1217 dev_err(dev, "program RC mem outbound ATU failed\n");
1218 return err;
1219 }
1220 }
1221
1222 err = rockchip_pcie_prog_ib_atu(rockchip, 2, 32 - 1, 0x0, 0);
1223 if (err) {
1224 dev_err(dev, "program RC mem inbound ATU failed\n");
1225 return err;
1226 }
1227
1228 offset = rockchip->mem_size >> 20;
1229 for (reg_no = 0; reg_no < (rockchip->io_size >> 20); reg_no++) {
1230 err = rockchip_pcie_prog_ob_atu(rockchip,
1231 reg_no + 1 + offset,
1232 AXI_WRAPPER_IO_WRITE,
1233 20 - 1,
1234 rockchip->io_bus_addr +
1235 (reg_no << 20),
1236 0);
1237 if (err) {
1238 dev_err(dev, "program RC io outbound ATU failed\n");
1239 return err;
1240 }
1241 }
1242
1243 /* assign message regions */
1244 rockchip_pcie_prog_ob_atu(rockchip, reg_no + 1 + offset,
1245 AXI_WRAPPER_NOR_MSG,
1246 20 - 1, 0, 0);
1247
1248 rockchip->msg_bus_addr = rockchip->mem_bus_addr +
1249 ((reg_no + offset) << 20);
1250 return err;
1251 }
1252
1253 static int rockchip_pcie_wait_l2(struct rockchip_pcie *rockchip)
1254 {
1255 u32 value;
1256 int err;
1257
1258 /* send PME_TURN_OFF message */
1259 writel(0x0, rockchip->msg_region + PCIE_RC_SEND_PME_OFF);
1260
1261 /* read LTSSM and wait for falling into L2 link state */
1262 err = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_DEBUG_OUT_0,
1263 value, PCIE_LINK_IS_L2(value), 20,
1264 jiffies_to_usecs(5 * HZ));
1265 if (err) {
1266 dev_err(rockchip->dev, "PCIe link enter L2 timeout!\n");
1267 return err;
1268 }
1269
1270 return 0;
1271 }
1272
1273 static int __maybe_unused rockchip_pcie_suspend_noirq(struct device *dev)
1274 {
1275 struct rockchip_pcie *rockchip = dev_get_drvdata(dev);
1276 int ret;
1277
1278 /* disable core and cli int since we don't need to ack PME_ACK */
1279 rockchip_pcie_write(rockchip, (PCIE_CLIENT_INT_CLI << 16) |
1280 PCIE_CLIENT_INT_CLI, PCIE_CLIENT_INT_MASK);
1281 rockchip_pcie_write(rockchip, (u32)PCIE_CORE_INT, PCIE_CORE_INT_MASK);
1282
1283 ret = rockchip_pcie_wait_l2(rockchip);
1284 if (ret) {
1285 rockchip_pcie_enable_interrupts(rockchip);
1286 return ret;
1287 }
1288
1289 phy_power_off(rockchip->phy);
1290 phy_exit(rockchip->phy);
1291
1292 clk_disable_unprepare(rockchip->clk_pcie_pm);
1293 clk_disable_unprepare(rockchip->hclk_pcie);
1294 clk_disable_unprepare(rockchip->aclk_perf_pcie);
1295 clk_disable_unprepare(rockchip->aclk_pcie);
1296
1297 if (!IS_ERR(rockchip->vpcie0v9))
1298 regulator_disable(rockchip->vpcie0v9);
1299
1300 return ret;
1301 }
1302
1303 static int __maybe_unused rockchip_pcie_resume_noirq(struct device *dev)
1304 {
1305 struct rockchip_pcie *rockchip = dev_get_drvdata(dev);
1306 int err;
1307
1308 if (!IS_ERR(rockchip->vpcie0v9)) {
1309 err = regulator_enable(rockchip->vpcie0v9);
1310 if (err) {
1311 dev_err(dev, "fail to enable vpcie0v9 regulator\n");
1312 return err;
1313 }
1314 }
1315
1316 err = clk_prepare_enable(rockchip->clk_pcie_pm);
1317 if (err)
1318 goto err_pcie_pm;
1319
1320 err = clk_prepare_enable(rockchip->hclk_pcie);
1321 if (err)
1322 goto err_hclk_pcie;
1323
1324 err = clk_prepare_enable(rockchip->aclk_perf_pcie);
1325 if (err)
1326 goto err_aclk_perf_pcie;
1327
1328 err = clk_prepare_enable(rockchip->aclk_pcie);
1329 if (err)
1330 goto err_aclk_pcie;
1331
1332 err = rockchip_pcie_init_port(rockchip);
1333 if (err)
1334 goto err_pcie_resume;
1335
1336 err = rockchip_pcie_cfg_atu(rockchip);
1337 if (err)
1338 goto err_pcie_resume;
1339
1340 /* Need this to enter L1 again */
1341 rockchip_pcie_update_txcredit_mui(rockchip);
1342 rockchip_pcie_enable_interrupts(rockchip);
1343
1344 return 0;
1345
1346 err_pcie_resume:
1347 clk_disable_unprepare(rockchip->aclk_pcie);
1348 err_aclk_pcie:
1349 clk_disable_unprepare(rockchip->aclk_perf_pcie);
1350 err_aclk_perf_pcie:
1351 clk_disable_unprepare(rockchip->hclk_pcie);
1352 err_hclk_pcie:
1353 clk_disable_unprepare(rockchip->clk_pcie_pm);
1354 err_pcie_pm:
1355 return err;
1356 }
1357
1358 static int rockchip_pcie_probe(struct platform_device *pdev)
1359 {
1360 struct rockchip_pcie *rockchip;
1361 struct device *dev = &pdev->dev;
1362 struct pci_bus *bus, *child;
1363 struct pci_host_bridge *bridge;
1364 struct resource_entry *win;
1365 resource_size_t io_base;
1366 struct resource *mem;
1367 struct resource *io;
1368 int err;
1369
1370 LIST_HEAD(res);
1371
1372 if (!dev->of_node)
1373 return -ENODEV;
1374
1375 bridge = devm_pci_alloc_host_bridge(dev, sizeof(*rockchip));
1376 if (!bridge)
1377 return -ENOMEM;
1378
1379 rockchip = pci_host_bridge_priv(bridge);
1380
1381 platform_set_drvdata(pdev, rockchip);
1382 rockchip->dev = dev;
1383
1384 err = rockchip_pcie_parse_dt(rockchip);
1385 if (err)
1386 return err;
1387
1388 err = clk_prepare_enable(rockchip->aclk_pcie);
1389 if (err) {
1390 dev_err(dev, "unable to enable aclk_pcie clock\n");
1391 goto err_aclk_pcie;
1392 }
1393
1394 err = clk_prepare_enable(rockchip->aclk_perf_pcie);
1395 if (err) {
1396 dev_err(dev, "unable to enable aclk_perf_pcie clock\n");
1397 goto err_aclk_perf_pcie;
1398 }
1399
1400 err = clk_prepare_enable(rockchip->hclk_pcie);
1401 if (err) {
1402 dev_err(dev, "unable to enable hclk_pcie clock\n");
1403 goto err_hclk_pcie;
1404 }
1405
1406 err = clk_prepare_enable(rockchip->clk_pcie_pm);
1407 if (err) {
1408 dev_err(dev, "unable to enable hclk_pcie clock\n");
1409 goto err_pcie_pm;
1410 }
1411
1412 err = rockchip_pcie_set_vpcie(rockchip);
1413 if (err) {
1414 dev_err(dev, "failed to set vpcie regulator\n");
1415 goto err_set_vpcie;
1416 }
1417
1418 err = rockchip_pcie_init_port(rockchip);
1419 if (err)
1420 goto err_vpcie;
1421
1422 rockchip_pcie_enable_interrupts(rockchip);
1423
1424 err = rockchip_pcie_init_irq_domain(rockchip);
1425 if (err < 0)
1426 goto err_vpcie;
1427
1428 err = of_pci_get_host_bridge_resources(dev->of_node, 0, 0xff,
1429 &res, &io_base);
1430 if (err)
1431 goto err_vpcie;
1432
1433 err = devm_request_pci_bus_resources(dev, &res);
1434 if (err)
1435 goto err_free_res;
1436
1437 /* Get the I/O and memory ranges from DT */
1438 resource_list_for_each_entry(win, &res) {
1439 switch (resource_type(win->res)) {
1440 case IORESOURCE_IO:
1441 io = win->res;
1442 io->name = "I/O";
1443 rockchip->io_size = resource_size(io);
1444 rockchip->io_bus_addr = io->start - win->offset;
1445 err = pci_remap_iospace(io, io_base);
1446 if (err) {
1447 dev_warn(dev, "error %d: failed to map resource %pR\n",
1448 err, io);
1449 continue;
1450 }
1451 rockchip->io = io;
1452 break;
1453 case IORESOURCE_MEM:
1454 mem = win->res;
1455 mem->name = "MEM";
1456 rockchip->mem_size = resource_size(mem);
1457 rockchip->mem_bus_addr = mem->start - win->offset;
1458 break;
1459 case IORESOURCE_BUS:
1460 rockchip->root_bus_nr = win->res->start;
1461 break;
1462 default:
1463 continue;
1464 }
1465 }
1466
1467 err = rockchip_pcie_cfg_atu(rockchip);
1468 if (err)
1469 goto err_free_res;
1470
1471 rockchip->msg_region = devm_ioremap(dev, rockchip->msg_bus_addr, SZ_1M);
1472 if (!rockchip->msg_region) {
1473 err = -ENOMEM;
1474 goto err_free_res;
1475 }
1476
1477 list_splice_init(&res, &bridge->windows);
1478 bridge->dev.parent = dev;
1479 bridge->sysdata = rockchip;
1480 bridge->busnr = 0;
1481 bridge->ops = &rockchip_pcie_ops;
1482 bridge->map_irq = of_irq_parse_and_map_pci;
1483 bridge->swizzle_irq = pci_common_swizzle;
1484
1485 err = pci_scan_root_bus_bridge(bridge);
1486 if (err < 0)
1487 goto err_free_res;
1488
1489 bus = bridge->bus;
1490
1491 rockchip->root_bus = bus;
1492
1493 pci_bus_size_bridges(bus);
1494 pci_bus_assign_resources(bus);
1495 list_for_each_entry(child, &bus->children, node)
1496 pcie_bus_configure_settings(child);
1497
1498 pci_bus_add_devices(bus);
1499 return 0;
1500
1501 err_free_res:
1502 pci_free_resource_list(&res);
1503 err_vpcie:
1504 if (!IS_ERR(rockchip->vpcie3v3))
1505 regulator_disable(rockchip->vpcie3v3);
1506 if (!IS_ERR(rockchip->vpcie1v8))
1507 regulator_disable(rockchip->vpcie1v8);
1508 if (!IS_ERR(rockchip->vpcie0v9))
1509 regulator_disable(rockchip->vpcie0v9);
1510 err_set_vpcie:
1511 clk_disable_unprepare(rockchip->clk_pcie_pm);
1512 err_pcie_pm:
1513 clk_disable_unprepare(rockchip->hclk_pcie);
1514 err_hclk_pcie:
1515 clk_disable_unprepare(rockchip->aclk_perf_pcie);
1516 err_aclk_perf_pcie:
1517 clk_disable_unprepare(rockchip->aclk_pcie);
1518 err_aclk_pcie:
1519 return err;
1520 }
1521
1522 static int rockchip_pcie_remove(struct platform_device *pdev)
1523 {
1524 struct device *dev = &pdev->dev;
1525 struct rockchip_pcie *rockchip = dev_get_drvdata(dev);
1526
1527 pci_stop_root_bus(rockchip->root_bus);
1528 pci_remove_root_bus(rockchip->root_bus);
1529 pci_unmap_iospace(rockchip->io);
1530 irq_domain_remove(rockchip->irq_domain);
1531
1532 phy_power_off(rockchip->phy);
1533 phy_exit(rockchip->phy);
1534
1535 clk_disable_unprepare(rockchip->clk_pcie_pm);
1536 clk_disable_unprepare(rockchip->hclk_pcie);
1537 clk_disable_unprepare(rockchip->aclk_perf_pcie);
1538 clk_disable_unprepare(rockchip->aclk_pcie);
1539
1540 if (!IS_ERR(rockchip->vpcie3v3))
1541 regulator_disable(rockchip->vpcie3v3);
1542 if (!IS_ERR(rockchip->vpcie1v8))
1543 regulator_disable(rockchip->vpcie1v8);
1544 if (!IS_ERR(rockchip->vpcie0v9))
1545 regulator_disable(rockchip->vpcie0v9);
1546
1547 return 0;
1548 }
1549
1550 static const struct dev_pm_ops rockchip_pcie_pm_ops = {
1551 SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(rockchip_pcie_suspend_noirq,
1552 rockchip_pcie_resume_noirq)
1553 };
1554
1555 static const struct of_device_id rockchip_pcie_of_match[] = {
1556 { .compatible = "rockchip,rk3399-pcie", },
1557 {}
1558 };
1559 MODULE_DEVICE_TABLE(of, rockchip_pcie_of_match);
1560
1561 static struct platform_driver rockchip_pcie_driver = {
1562 .driver = {
1563 .name = "rockchip-pcie",
1564 .of_match_table = rockchip_pcie_of_match,
1565 .pm = &rockchip_pcie_pm_ops,
1566 },
1567 .probe = rockchip_pcie_probe,
1568 .remove = rockchip_pcie_remove,
1569 };
1570 module_platform_driver(rockchip_pcie_driver);
1571
1572 MODULE_AUTHOR("Rockchip Inc");
1573 MODULE_DESCRIPTION("Rockchip AXI PCIe driver");
1574 MODULE_LICENSE("GPL v2");
Linux with added FPC-III support