?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'block-5.11-2021-01-10' of git://git.kernel.dk/linux-block
[linux/fpc-iii.git] / drivers / memory / tegra / tegra210-emc-core.c
blob5f224796e32e213d4688cecde07c957b33ddf00c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (c) 2015-2020, NVIDIA CORPORATION. All rights reserved.
4 */
5
6 #include <linux/bitfield.h>
7 #include <linux/clk.h>
8 #include <linux/clk/tegra.h>
9 #include <linux/debugfs.h>
10 #include <linux/delay.h>
11 #include <linux/kernel.h>
12 #include <linux/module.h>
13 #include <linux/of_address.h>
14 #include <linux/of_platform.h>
15 #include <linux/of_reserved_mem.h>
16 #include <linux/slab.h>
17 #include <linux/thermal.h>
18 #include <soc/tegra/fuse.h>
19 #include <soc/tegra/mc.h>
20
21 #include "tegra210-emc.h"
22 #include "tegra210-mc.h"
23
24 /* CLK_RST_CONTROLLER_CLK_SOURCE_EMC */
25 #define EMC_CLK_EMC_2X_CLK_SRC_SHIFT 29
26 #define EMC_CLK_EMC_2X_CLK_SRC_MASK \
27 (0x7 << EMC_CLK_EMC_2X_CLK_SRC_SHIFT)
28 #define EMC_CLK_SOURCE_PLLM_LJ 0x4
29 #define EMC_CLK_SOURCE_PLLMB_LJ 0x5
30 #define EMC_CLK_FORCE_CC_TRIGGER BIT(27)
31 #define EMC_CLK_MC_EMC_SAME_FREQ BIT(16)
32 #define EMC_CLK_EMC_2X_CLK_DIVISOR_SHIFT 0
33 #define EMC_CLK_EMC_2X_CLK_DIVISOR_MASK \
34 (0xff << EMC_CLK_EMC_2X_CLK_DIVISOR_SHIFT)
35
36 /* CLK_RST_CONTROLLER_CLK_SOURCE_EMC_DLL */
37 #define DLL_CLK_EMC_DLL_CLK_SRC_SHIFT 29
38 #define DLL_CLK_EMC_DLL_CLK_SRC_MASK \
39 (0x7 << DLL_CLK_EMC_DLL_CLK_SRC_SHIFT)
40 #define DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT 10
41 #define DLL_CLK_EMC_DLL_DDLL_CLK_SEL_MASK \
42 (0x3 << DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT)
43 #define PLLM_VCOA 0
44 #define PLLM_VCOB 1
45 #define EMC_DLL_SWITCH_OUT 2
46 #define DLL_CLK_EMC_DLL_CLK_DIVISOR_SHIFT 0
47 #define DLL_CLK_EMC_DLL_CLK_DIVISOR_MASK \
48 (0xff << DLL_CLK_EMC_DLL_CLK_DIVISOR_SHIFT)
49
50 /* MC_EMEM_ARB_MISC0 */
51 #define MC_EMEM_ARB_MISC0_EMC_SAME_FREQ BIT(27)
52
53 /* EMC_DATA_BRLSHFT_X */
54 #define EMC0_EMC_DATA_BRLSHFT_0_INDEX 2
55 #define EMC1_EMC_DATA_BRLSHFT_0_INDEX 3
56 #define EMC0_EMC_DATA_BRLSHFT_1_INDEX 4
57 #define EMC1_EMC_DATA_BRLSHFT_1_INDEX 5
58
59 #define TRIM_REG(chan, rank, reg, byte) \
60 (((EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ## \
61 _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte ## _MASK & \
62 next->trim_regs[EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## \
63 rank ## _ ## reg ## _INDEX]) >> \
64 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ## \
65 _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte ## _SHIFT) \
66 + \
67 (((EMC_DATA_BRLSHFT_ ## rank ## _RANK ## rank ## _BYTE ## \
68 byte ## _DATA_BRLSHFT_MASK & \
69 next->trim_perch_regs[EMC ## chan ## \
70 _EMC_DATA_BRLSHFT_ ## rank ## _INDEX]) >> \
71 EMC_DATA_BRLSHFT_ ## rank ## _RANK ## rank ## _BYTE ## \
72 byte ## _DATA_BRLSHFT_SHIFT) * 64))
73
74 #define CALC_TEMP(rank, reg, byte1, byte2, n) \
75 (((new[n] << EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## \
76 reg ## _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte1 ## _SHIFT) & \
77 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ## \
78 _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte1 ## _MASK) \
79 | \
80 ((new[n + 1] << EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ##\
81 reg ## _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte2 ## _SHIFT) & \
82 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ## \
83 _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte2 ## _MASK))
84
85 #define REFRESH_SPEEDUP(value, speedup) \
86 (((value) & 0xffff0000) | ((value) & 0xffff) * (speedup))
87
88 #define LPDDR2_MR4_SRR GENMASK(2, 0)
89
90 static const struct tegra210_emc_sequence *tegra210_emc_sequences[] = {
91 &tegra210_emc_r21021,
92 };
93
94 static const struct tegra210_emc_table_register_offsets
95 tegra210_emc_table_register_offsets = {
96 .burst = {
97 EMC_RC,
98 EMC_RFC,
99 EMC_RFCPB,
100 EMC_REFCTRL2,
101 EMC_RFC_SLR,
102 EMC_RAS,
103 EMC_RP,
104 EMC_R2W,
105 EMC_W2R,
106 EMC_R2P,
107 EMC_W2P,
108 EMC_R2R,
109 EMC_TPPD,
110 EMC_CCDMW,
111 EMC_RD_RCD,
112 EMC_WR_RCD,
113 EMC_RRD,
114 EMC_REXT,
115 EMC_WEXT,
116 EMC_WDV_CHK,
117 EMC_WDV,
118 EMC_WSV,
119 EMC_WEV,
120 EMC_WDV_MASK,
121 EMC_WS_DURATION,
122 EMC_WE_DURATION,
123 EMC_QUSE,
124 EMC_QUSE_WIDTH,
125 EMC_IBDLY,
126 EMC_OBDLY,
127 EMC_EINPUT,
128 EMC_MRW6,
129 EMC_EINPUT_DURATION,
130 EMC_PUTERM_EXTRA,
131 EMC_PUTERM_WIDTH,
132 EMC_QRST,
133 EMC_QSAFE,
134 EMC_RDV,
135 EMC_RDV_MASK,
136 EMC_RDV_EARLY,
137 EMC_RDV_EARLY_MASK,
138 EMC_REFRESH,
139 EMC_BURST_REFRESH_NUM,
140 EMC_PRE_REFRESH_REQ_CNT,
141 EMC_PDEX2WR,
142 EMC_PDEX2RD,
143 EMC_PCHG2PDEN,
144 EMC_ACT2PDEN,
145 EMC_AR2PDEN,
146 EMC_RW2PDEN,
147 EMC_CKE2PDEN,
148 EMC_PDEX2CKE,
149 EMC_PDEX2MRR,
150 EMC_TXSR,
151 EMC_TXSRDLL,
152 EMC_TCKE,
153 EMC_TCKESR,
154 EMC_TPD,
155 EMC_TFAW,
156 EMC_TRPAB,
157 EMC_TCLKSTABLE,
158 EMC_TCLKSTOP,
159 EMC_MRW7,
160 EMC_TREFBW,
161 EMC_ODT_WRITE,
162 EMC_FBIO_CFG5,
163 EMC_FBIO_CFG7,
164 EMC_CFG_DIG_DLL,
165 EMC_CFG_DIG_DLL_PERIOD,
166 EMC_PMACRO_IB_RXRT,
167 EMC_CFG_PIPE_1,
168 EMC_CFG_PIPE_2,
169 EMC_PMACRO_QUSE_DDLL_RANK0_4,
170 EMC_PMACRO_QUSE_DDLL_RANK0_5,
171 EMC_PMACRO_QUSE_DDLL_RANK1_4,
172 EMC_PMACRO_QUSE_DDLL_RANK1_5,
173 EMC_MRW8,
174 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_4,
175 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_5,
176 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_0,
177 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_1,
178 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_2,
179 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_3,
180 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_4,
181 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_5,
182 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_0,
183 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_1,
184 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_2,
185 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_3,
186 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_4,
187 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_5,
188 EMC_PMACRO_DDLL_LONG_CMD_0,
189 EMC_PMACRO_DDLL_LONG_CMD_1,
190 EMC_PMACRO_DDLL_LONG_CMD_2,
191 EMC_PMACRO_DDLL_LONG_CMD_3,
192 EMC_PMACRO_DDLL_LONG_CMD_4,
193 EMC_PMACRO_DDLL_SHORT_CMD_0,
194 EMC_PMACRO_DDLL_SHORT_CMD_1,
195 EMC_PMACRO_DDLL_SHORT_CMD_2,
196 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_3,
197 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_3,
198 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_3,
199 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_3,
200 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_3,
201 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_3,
202 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_3,
203 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_3,
204 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_3,
205 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_3,
206 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_3,
207 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_3,
208 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_3,
209 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_3,
210 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_3,
211 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_3,
212 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_3,
213 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_3,
214 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_3,
215 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_3,
216 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_0,
217 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_1,
218 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_2,
219 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_3,
220 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_0,
221 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_1,
222 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_2,
223 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_3,
224 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_0,
225 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_1,
226 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_2,
227 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_3,
228 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_0,
229 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_1,
230 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_2,
231 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_3,
232 EMC_TXDSRVTTGEN,
233 EMC_FDPD_CTRL_DQ,
234 EMC_FDPD_CTRL_CMD,
235 EMC_FBIO_SPARE,
236 EMC_ZCAL_INTERVAL,
237 EMC_ZCAL_WAIT_CNT,
238 EMC_MRS_WAIT_CNT,
239 EMC_MRS_WAIT_CNT2,
240 EMC_AUTO_CAL_CHANNEL,
241 EMC_DLL_CFG_0,
242 EMC_DLL_CFG_1,
243 EMC_PMACRO_AUTOCAL_CFG_COMMON,
244 EMC_PMACRO_ZCTRL,
245 EMC_CFG,
246 EMC_CFG_PIPE,
247 EMC_DYN_SELF_REF_CONTROL,
248 EMC_QPOP,
249 EMC_DQS_BRLSHFT_0,
250 EMC_DQS_BRLSHFT_1,
251 EMC_CMD_BRLSHFT_2,
252 EMC_CMD_BRLSHFT_3,
253 EMC_PMACRO_PAD_CFG_CTRL,
254 EMC_PMACRO_DATA_PAD_RX_CTRL,
255 EMC_PMACRO_CMD_PAD_RX_CTRL,
256 EMC_PMACRO_DATA_RX_TERM_MODE,
257 EMC_PMACRO_CMD_RX_TERM_MODE,
258 EMC_PMACRO_CMD_PAD_TX_CTRL,
259 EMC_PMACRO_DATA_PAD_TX_CTRL,
260 EMC_PMACRO_COMMON_PAD_TX_CTRL,
261 EMC_PMACRO_VTTGEN_CTRL_0,
262 EMC_PMACRO_VTTGEN_CTRL_1,
263 EMC_PMACRO_VTTGEN_CTRL_2,
264 EMC_PMACRO_BRICK_CTRL_RFU1,
265 EMC_PMACRO_CMD_BRICK_CTRL_FDPD,
266 EMC_PMACRO_BRICK_CTRL_RFU2,
267 EMC_PMACRO_DATA_BRICK_CTRL_FDPD,
268 EMC_PMACRO_BG_BIAS_CTRL_0,
269 EMC_CFG_3,
270 EMC_PMACRO_TX_PWRD_0,
271 EMC_PMACRO_TX_PWRD_1,
272 EMC_PMACRO_TX_PWRD_2,
273 EMC_PMACRO_TX_PWRD_3,
274 EMC_PMACRO_TX_PWRD_4,
275 EMC_PMACRO_TX_PWRD_5,
276 EMC_CONFIG_SAMPLE_DELAY,
277 EMC_PMACRO_TX_SEL_CLK_SRC_0,
278 EMC_PMACRO_TX_SEL_CLK_SRC_1,
279 EMC_PMACRO_TX_SEL_CLK_SRC_2,
280 EMC_PMACRO_TX_SEL_CLK_SRC_3,
281 EMC_PMACRO_TX_SEL_CLK_SRC_4,
282 EMC_PMACRO_TX_SEL_CLK_SRC_5,
283 EMC_PMACRO_DDLL_BYPASS,
284 EMC_PMACRO_DDLL_PWRD_0,
285 EMC_PMACRO_DDLL_PWRD_1,
286 EMC_PMACRO_DDLL_PWRD_2,
287 EMC_PMACRO_CMD_CTRL_0,
288 EMC_PMACRO_CMD_CTRL_1,
289 EMC_PMACRO_CMD_CTRL_2,
290 EMC_TR_TIMING_0,
291 EMC_TR_DVFS,
292 EMC_TR_CTRL_1,
293 EMC_TR_RDV,
294 EMC_TR_QPOP,
295 EMC_TR_RDV_MASK,
296 EMC_MRW14,
297 EMC_TR_QSAFE,
298 EMC_TR_QRST,
299 EMC_TRAINING_CTRL,
300 EMC_TRAINING_SETTLE,
301 EMC_TRAINING_VREF_SETTLE,
302 EMC_TRAINING_CA_FINE_CTRL,
303 EMC_TRAINING_CA_CTRL_MISC,
304 EMC_TRAINING_CA_CTRL_MISC1,
305 EMC_TRAINING_CA_VREF_CTRL,
306 EMC_TRAINING_QUSE_CORS_CTRL,
307 EMC_TRAINING_QUSE_FINE_CTRL,
308 EMC_TRAINING_QUSE_CTRL_MISC,
309 EMC_TRAINING_QUSE_VREF_CTRL,
310 EMC_TRAINING_READ_FINE_CTRL,
311 EMC_TRAINING_READ_CTRL_MISC,
312 EMC_TRAINING_READ_VREF_CTRL,
313 EMC_TRAINING_WRITE_FINE_CTRL,
314 EMC_TRAINING_WRITE_CTRL_MISC,
315 EMC_TRAINING_WRITE_VREF_CTRL,
316 EMC_TRAINING_MPC,
317 EMC_MRW15,
318 },
319 .trim = {
320 EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_0,
321 EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_1,
322 EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_2,
323 EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_3,
324 EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_0,
325 EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_1,
326 EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_2,
327 EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_3,
328 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_0,
329 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_1,
330 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_2,
331 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_0,
332 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_1,
333 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_2,
334 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_0,
335 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_1,
336 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_2,
337 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_0,
338 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_1,
339 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_2,
340 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_0,
341 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_1,
342 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_2,
343 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_0,
344 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_1,
345 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_2,
346 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_0,
347 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_1,
348 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_2,
349 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_0,
350 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_1,
351 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_2,
352 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_0,
353 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_1,
354 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_2,
355 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_0,
356 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_1,
357 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_2,
358 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_0,
359 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_1,
360 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_2,
361 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_0,
362 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_1,
363 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_2,
364 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_0,
365 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_1,
366 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_2,
367 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_0,
368 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_1,
369 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_2,
370 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_0,
371 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_1,
372 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_2,
373 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_0,
374 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_1,
375 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_2,
376 EMC_PMACRO_IB_VREF_DQS_0,
377 EMC_PMACRO_IB_VREF_DQS_1,
378 EMC_PMACRO_IB_VREF_DQ_0,
379 EMC_PMACRO_IB_VREF_DQ_1,
380 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0,
381 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1,
382 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2,
383 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3,
384 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_4,
385 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_5,
386 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0,
387 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1,
388 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2,
389 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3,
390 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_0,
391 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_1,
392 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_2,
393 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_0,
394 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_1,
395 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_2,
396 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_0,
397 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_1,
398 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_2,
399 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_0,
400 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_1,
401 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_2,
402 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_0,
403 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_1,
404 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_2,
405 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_0,
406 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_1,
407 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_2,
408 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_0,
409 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_1,
410 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_2,
411 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_0,
412 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_1,
413 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_2,
414 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_0,
415 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_1,
416 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_2,
417 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_0,
418 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_1,
419 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_2,
420 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_0,
421 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_1,
422 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_2,
423 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_0,
424 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_1,
425 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_2,
426 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_0,
427 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_1,
428 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_2,
429 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_0,
430 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_1,
431 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_2,
432 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_0,
433 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_1,
434 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_2,
435 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_0,
436 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_1,
437 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_2,
438 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_0,
439 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_1,
440 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_2,
441 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_0,
442 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_1,
443 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_2,
444 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_0,
445 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_1,
446 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_2,
447 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_0,
448 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_1,
449 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_2,
450 EMC_PMACRO_QUSE_DDLL_RANK0_0,
451 EMC_PMACRO_QUSE_DDLL_RANK0_1,
452 EMC_PMACRO_QUSE_DDLL_RANK0_2,
453 EMC_PMACRO_QUSE_DDLL_RANK0_3,
454 EMC_PMACRO_QUSE_DDLL_RANK1_0,
455 EMC_PMACRO_QUSE_DDLL_RANK1_1,
456 EMC_PMACRO_QUSE_DDLL_RANK1_2,
457 EMC_PMACRO_QUSE_DDLL_RANK1_3
458 },
459 .burst_mc = {
460 MC_EMEM_ARB_CFG,
461 MC_EMEM_ARB_OUTSTANDING_REQ,
462 MC_EMEM_ARB_REFPB_HP_CTRL,
463 MC_EMEM_ARB_REFPB_BANK_CTRL,
464 MC_EMEM_ARB_TIMING_RCD,
465 MC_EMEM_ARB_TIMING_RP,
466 MC_EMEM_ARB_TIMING_RC,
467 MC_EMEM_ARB_TIMING_RAS,
468 MC_EMEM_ARB_TIMING_FAW,
469 MC_EMEM_ARB_TIMING_RRD,
470 MC_EMEM_ARB_TIMING_RAP2PRE,
471 MC_EMEM_ARB_TIMING_WAP2PRE,
472 MC_EMEM_ARB_TIMING_R2R,
473 MC_EMEM_ARB_TIMING_W2W,
474 MC_EMEM_ARB_TIMING_R2W,
475 MC_EMEM_ARB_TIMING_CCDMW,
476 MC_EMEM_ARB_TIMING_W2R,
477 MC_EMEM_ARB_TIMING_RFCPB,
478 MC_EMEM_ARB_DA_TURNS,
479 MC_EMEM_ARB_DA_COVERS,
480 MC_EMEM_ARB_MISC0,
481 MC_EMEM_ARB_MISC1,
482 MC_EMEM_ARB_MISC2,
483 MC_EMEM_ARB_RING1_THROTTLE,
484 MC_EMEM_ARB_DHYST_CTRL,
485 MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_0,
486 MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_1,
487 MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_2,
488 MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_3,
489 MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_4,
490 MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_5,
491 MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_6,
492 MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_7,
493 },
494 .la_scale = {
495 MC_MLL_MPCORER_PTSA_RATE,
496 MC_FTOP_PTSA_RATE,
497 MC_PTSA_GRANT_DECREMENT,
498 MC_LATENCY_ALLOWANCE_XUSB_0,
499 MC_LATENCY_ALLOWANCE_XUSB_1,
500 MC_LATENCY_ALLOWANCE_TSEC_0,
501 MC_LATENCY_ALLOWANCE_SDMMCA_0,
502 MC_LATENCY_ALLOWANCE_SDMMCAA_0,
503 MC_LATENCY_ALLOWANCE_SDMMC_0,
504 MC_LATENCY_ALLOWANCE_SDMMCAB_0,
505 MC_LATENCY_ALLOWANCE_PPCS_0,
506 MC_LATENCY_ALLOWANCE_PPCS_1,
507 MC_LATENCY_ALLOWANCE_MPCORE_0,
508 MC_LATENCY_ALLOWANCE_HC_0,
509 MC_LATENCY_ALLOWANCE_HC_1,
510 MC_LATENCY_ALLOWANCE_AVPC_0,
511 MC_LATENCY_ALLOWANCE_GPU_0,
512 MC_LATENCY_ALLOWANCE_GPU2_0,
513 MC_LATENCY_ALLOWANCE_NVENC_0,
514 MC_LATENCY_ALLOWANCE_NVDEC_0,
515 MC_LATENCY_ALLOWANCE_VIC_0,
516 MC_LATENCY_ALLOWANCE_VI2_0,
517 MC_LATENCY_ALLOWANCE_ISP2_0,
518 MC_LATENCY_ALLOWANCE_ISP2_1,
519 },
520 .burst_per_channel = {
521 { .bank = 0, .offset = EMC_MRW10, },
522 { .bank = 1, .offset = EMC_MRW10, },
523 { .bank = 0, .offset = EMC_MRW11, },
524 { .bank = 1, .offset = EMC_MRW11, },
525 { .bank = 0, .offset = EMC_MRW12, },
526 { .bank = 1, .offset = EMC_MRW12, },
527 { .bank = 0, .offset = EMC_MRW13, },
528 { .bank = 1, .offset = EMC_MRW13, },
529 },
530 .trim_per_channel = {
531 { .bank = 0, .offset = EMC_CMD_BRLSHFT_0, },
532 { .bank = 1, .offset = EMC_CMD_BRLSHFT_1, },
533 { .bank = 0, .offset = EMC_DATA_BRLSHFT_0, },
534 { .bank = 1, .offset = EMC_DATA_BRLSHFT_0, },
535 { .bank = 0, .offset = EMC_DATA_BRLSHFT_1, },
536 { .bank = 1, .offset = EMC_DATA_BRLSHFT_1, },
537 { .bank = 0, .offset = EMC_QUSE_BRLSHFT_0, },
538 { .bank = 1, .offset = EMC_QUSE_BRLSHFT_1, },
539 { .bank = 0, .offset = EMC_QUSE_BRLSHFT_2, },
540 { .bank = 1, .offset = EMC_QUSE_BRLSHFT_3, },
541 },
542 .vref_per_channel = {
543 {
544 .bank = 0,
545 .offset = EMC_TRAINING_OPT_DQS_IB_VREF_RANK0,
546 }, {
547 .bank = 1,
548 .offset = EMC_TRAINING_OPT_DQS_IB_VREF_RANK0,
549 }, {
550 .bank = 0,
551 .offset = EMC_TRAINING_OPT_DQS_IB_VREF_RANK1,
552 }, {
553 .bank = 1,
554 .offset = EMC_TRAINING_OPT_DQS_IB_VREF_RANK1,
555 },
556 },
557 };
558
559 static void tegra210_emc_train(struct timer_list *timer)
560 {
561 struct tegra210_emc *emc = from_timer(emc, timer, training);
562 unsigned long flags;
563
564 if (!emc->last)
565 return;
566
567 spin_lock_irqsave(&emc->lock, flags);
568
569 if (emc->sequence->periodic_compensation)
570 emc->sequence->periodic_compensation(emc);
571
572 spin_unlock_irqrestore(&emc->lock, flags);
573
574 mod_timer(&emc->training,
575 jiffies + msecs_to_jiffies(emc->training_interval));
576 }
577
578 static void tegra210_emc_training_start(struct tegra210_emc *emc)
579 {
580 mod_timer(&emc->training,
581 jiffies + msecs_to_jiffies(emc->training_interval));
582 }
583
584 static void tegra210_emc_training_stop(struct tegra210_emc *emc)
585 {
586 del_timer(&emc->training);
587 }
588
589 static unsigned int tegra210_emc_get_temperature(struct tegra210_emc *emc)
590 {
591 unsigned long flags;
592 u32 value, max = 0;
593 unsigned int i;
594
595 spin_lock_irqsave(&emc->lock, flags);
596
597 for (i = 0; i < emc->num_devices; i++) {
598 value = tegra210_emc_mrr_read(emc, i, 4);
599
600 if (value & BIT(7))
601 dev_dbg(emc->dev,
602 "sensor reading changed for device %u: %08x\n",
603 i, value);
604
605 value = FIELD_GET(LPDDR2_MR4_SRR, value);
606 if (value > max)
607 max = value;
608 }
609
610 spin_unlock_irqrestore(&emc->lock, flags);
611
612 return max;
613 }
614
615 static void tegra210_emc_poll_refresh(struct timer_list *timer)
616 {
617 struct tegra210_emc *emc = from_timer(emc, timer, refresh_timer);
618 unsigned int temperature;
619
620 if (!emc->debugfs.temperature)
621 temperature = tegra210_emc_get_temperature(emc);
622 else
623 temperature = emc->debugfs.temperature;
624
625 if (temperature == emc->temperature)
626 goto reset;
627
628 switch (temperature) {
629 case 0 ... 3:
630 /* temperature is fine, using regular refresh */
631 dev_dbg(emc->dev, "switching to nominal refresh...\n");
632 tegra210_emc_set_refresh(emc, TEGRA210_EMC_REFRESH_NOMINAL);
633 break;
634
635 case 4:
636 dev_dbg(emc->dev, "switching to 2x refresh...\n");
637 tegra210_emc_set_refresh(emc, TEGRA210_EMC_REFRESH_2X);
638 break;
639
640 case 5:
641 dev_dbg(emc->dev, "switching to 4x refresh...\n");
642 tegra210_emc_set_refresh(emc, TEGRA210_EMC_REFRESH_4X);
643 break;
644
645 case 6 ... 7:
646 dev_dbg(emc->dev, "switching to throttle refresh...\n");
647 tegra210_emc_set_refresh(emc, TEGRA210_EMC_REFRESH_THROTTLE);
648 break;
649
650 default:
651 WARN(1, "invalid DRAM temperature state %u\n", temperature);
652 return;
653 }
654
655 emc->temperature = temperature;
656
657 reset:
658 if (atomic_read(&emc->refresh_poll) > 0) {
659 unsigned int interval = emc->refresh_poll_interval;
660 unsigned int timeout = msecs_to_jiffies(interval);
661
662 mod_timer(&emc->refresh_timer, jiffies + timeout);
663 }
664 }
665
666 static void tegra210_emc_poll_refresh_stop(struct tegra210_emc *emc)
667 {
668 atomic_set(&emc->refresh_poll, 0);
669 del_timer_sync(&emc->refresh_timer);
670 }
671
672 static void tegra210_emc_poll_refresh_start(struct tegra210_emc *emc)
673 {
674 atomic_set(&emc->refresh_poll, 1);
675
676 mod_timer(&emc->refresh_timer,
677 jiffies + msecs_to_jiffies(emc->refresh_poll_interval));
678 }
679
680 static int tegra210_emc_cd_max_state(struct thermal_cooling_device *cd,
681 unsigned long *state)
682 {
683 *state = 1;
684
685 return 0;
686 }
687
688 static int tegra210_emc_cd_get_state(struct thermal_cooling_device *cd,
689 unsigned long *state)
690 {
691 struct tegra210_emc *emc = cd->devdata;
692
693 *state = atomic_read(&emc->refresh_poll);
694
695 return 0;
696 }
697
698 static int tegra210_emc_cd_set_state(struct thermal_cooling_device *cd,
699 unsigned long state)
700 {
701 struct tegra210_emc *emc = cd->devdata;
702
703 if (state == atomic_read(&emc->refresh_poll))
704 return 0;
705
706 if (state)
707 tegra210_emc_poll_refresh_start(emc);
708 else
709 tegra210_emc_poll_refresh_stop(emc);
710
711 return 0;
712 }
713
714 static struct thermal_cooling_device_ops tegra210_emc_cd_ops = {
715 .get_max_state = tegra210_emc_cd_max_state,
716 .get_cur_state = tegra210_emc_cd_get_state,
717 .set_cur_state = tegra210_emc_cd_set_state,
718 };
719
720 static void tegra210_emc_set_clock(struct tegra210_emc *emc, u32 clksrc)
721 {
722 emc->sequence->set_clock(emc, clksrc);
723
724 if (emc->next->periodic_training)
725 tegra210_emc_training_start(emc);
726 else
727 tegra210_emc_training_stop(emc);
728 }
729
730 static void tegra210_change_dll_src(struct tegra210_emc *emc,
731 u32 clksrc)
732 {
733 u32 dll_setting = emc->next->dll_clk_src;
734 u32 emc_clk_src;
735 u32 emc_clk_div;
736
737 emc_clk_src = (clksrc & EMC_CLK_EMC_2X_CLK_SRC_MASK) >>
738 EMC_CLK_EMC_2X_CLK_SRC_SHIFT;
739 emc_clk_div = (clksrc & EMC_CLK_EMC_2X_CLK_DIVISOR_MASK) >>
740 EMC_CLK_EMC_2X_CLK_DIVISOR_SHIFT;
741
742 dll_setting &= ~(DLL_CLK_EMC_DLL_CLK_SRC_MASK |
743 DLL_CLK_EMC_DLL_CLK_DIVISOR_MASK);
744 dll_setting |= emc_clk_src << DLL_CLK_EMC_DLL_CLK_SRC_SHIFT;
745 dll_setting |= emc_clk_div << DLL_CLK_EMC_DLL_CLK_DIVISOR_SHIFT;
746
747 dll_setting &= ~DLL_CLK_EMC_DLL_DDLL_CLK_SEL_MASK;
748 if (emc_clk_src == EMC_CLK_SOURCE_PLLMB_LJ)
749 dll_setting |= (PLLM_VCOB <<
750 DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT);
751 else if (emc_clk_src == EMC_CLK_SOURCE_PLLM_LJ)
752 dll_setting |= (PLLM_VCOA <<
753 DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT);
754 else
755 dll_setting |= (EMC_DLL_SWITCH_OUT <<
756 DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT);
757
758 tegra210_clk_emc_dll_update_setting(dll_setting);
759
760 if (emc->next->clk_out_enb_x_0_clk_enb_emc_dll)
761 tegra210_clk_emc_dll_enable(true);
762 else
763 tegra210_clk_emc_dll_enable(false);
764 }
765
766 int tegra210_emc_set_refresh(struct tegra210_emc *emc,
767 enum tegra210_emc_refresh refresh)
768 {
769 struct tegra210_emc_timing *timings;
770 unsigned long flags;
771
772 if ((emc->dram_type != DRAM_TYPE_LPDDR2 &&
773 emc->dram_type != DRAM_TYPE_LPDDR4) ||
774 !emc->last)
775 return -ENODEV;
776
777 if (refresh > TEGRA210_EMC_REFRESH_THROTTLE)
778 return -EINVAL;
779
780 if (refresh == emc->refresh)
781 return 0;
782
783 spin_lock_irqsave(&emc->lock, flags);
784
785 if (refresh == TEGRA210_EMC_REFRESH_THROTTLE && emc->derated)
786 timings = emc->derated;
787 else
788 timings = emc->nominal;
789
790 if (timings != emc->timings) {
791 unsigned int index = emc->last - emc->timings;
792 u32 clksrc;
793
794 clksrc = emc->provider.configs[index].value |
795 EMC_CLK_FORCE_CC_TRIGGER;
796
797 emc->next = &timings[index];
798 emc->timings = timings;
799
800 tegra210_emc_set_clock(emc, clksrc);
801 } else {
802 tegra210_emc_adjust_timing(emc, emc->last);
803 tegra210_emc_timing_update(emc);
804
805 if (refresh != TEGRA210_EMC_REFRESH_NOMINAL)
806 emc_writel(emc, EMC_REF_REF_CMD, EMC_REF);
807 }
808
809 spin_unlock_irqrestore(&emc->lock, flags);
810
811 return 0;
812 }
813
814 u32 tegra210_emc_mrr_read(struct tegra210_emc *emc, unsigned int chip,
815 unsigned int address)
816 {
817 u32 value, ret = 0;
818 unsigned int i;
819
820 value = (chip & EMC_MRR_DEV_SEL_MASK) << EMC_MRR_DEV_SEL_SHIFT |
821 (address & EMC_MRR_MA_MASK) << EMC_MRR_MA_SHIFT;
822 emc_writel(emc, value, EMC_MRR);
823
824 for (i = 0; i < emc->num_channels; i++)
825 WARN(tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS,
826 EMC_EMC_STATUS_MRR_DIVLD, 1),
827 "Timed out waiting for MRR %u (ch=%u)\n", address, i);
828
829 for (i = 0; i < emc->num_channels; i++) {
830 value = emc_channel_readl(emc, i, EMC_MRR);
831 value &= EMC_MRR_DATA_MASK;
832
833 ret = (ret << 16) | value;
834 }
835
836 return ret;
837 }
838
839 void tegra210_emc_do_clock_change(struct tegra210_emc *emc, u32 clksrc)
840 {
841 int err;
842
843 mc_readl(emc->mc, MC_EMEM_ADR_CFG);
844 emc_readl(emc, EMC_INTSTATUS);
845
846 tegra210_clk_emc_update_setting(clksrc);
847
848 err = tegra210_emc_wait_for_update(emc, 0, EMC_INTSTATUS,
849 EMC_INTSTATUS_CLKCHANGE_COMPLETE,
850 true);
851 if (err)
852 dev_warn(emc->dev, "clock change completion error: %d\n", err);
853 }
854
855 struct tegra210_emc_timing *tegra210_emc_find_timing(struct tegra210_emc *emc,
856 unsigned long rate)
857 {
858 unsigned int i;
859
860 for (i = 0; i < emc->num_timings; i++)
861 if (emc->timings[i].rate * 1000UL == rate)
862 return &emc->timings[i];
863
864 return NULL;
865 }
866
867 int tegra210_emc_wait_for_update(struct tegra210_emc *emc, unsigned int channel,
868 unsigned int offset, u32 bit_mask, bool state)
869 {
870 unsigned int i;
871 u32 value;
872
873 for (i = 0; i < EMC_STATUS_UPDATE_TIMEOUT; i++) {
874 value = emc_channel_readl(emc, channel, offset);
875 if (!!(value & bit_mask) == state)
876 return 0;
877
878 udelay(1);
879 }
880
881 return -ETIMEDOUT;
882 }
883
884 void tegra210_emc_set_shadow_bypass(struct tegra210_emc *emc, int set)
885 {
886 u32 emc_dbg = emc_readl(emc, EMC_DBG);
887
888 if (set)
889 emc_writel(emc, emc_dbg | EMC_DBG_WRITE_MUX_ACTIVE, EMC_DBG);
890 else
891 emc_writel(emc, emc_dbg & ~EMC_DBG_WRITE_MUX_ACTIVE, EMC_DBG);
892 }
893
894 u32 tegra210_emc_get_dll_state(struct tegra210_emc_timing *next)
895 {
896 if (next->emc_emrs & 0x1)
897 return 0;
898
899 return 1;
900 }
901
902 void tegra210_emc_timing_update(struct tegra210_emc *emc)
903 {
904 unsigned int i;
905 int err = 0;
906
907 emc_writel(emc, 0x1, EMC_TIMING_CONTROL);
908
909 for (i = 0; i < emc->num_channels; i++) {
910 err |= tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS,
911 EMC_EMC_STATUS_TIMING_UPDATE_STALLED,
912 false);
913 }
914
915 if (err)
916 dev_warn(emc->dev, "timing update error: %d\n", err);
917 }
918
919 unsigned long tegra210_emc_actual_osc_clocks(u32 in)
920 {
921 if (in < 0x40)
922 return in * 16;
923 else if (in < 0x80)
924 return 2048;
925 else if (in < 0xc0)
926 return 4096;
927 else
928 return 8192;
929 }
930
931 void tegra210_emc_start_periodic_compensation(struct tegra210_emc *emc)
932 {
933 u32 mpc_req = 0x4b;
934
935 emc_writel(emc, mpc_req, EMC_MPC);
936 mpc_req = emc_readl(emc, EMC_MPC);
937 }
938
939 u32 tegra210_emc_compensate(struct tegra210_emc_timing *next, u32 offset)
940 {
941 u32 temp = 0, rate = next->rate / 1000;
942 s32 delta[4], delta_taps[4];
943 s32 new[] = {
944 TRIM_REG(0, 0, 0, 0),
945 TRIM_REG(0, 0, 0, 1),
946 TRIM_REG(0, 0, 1, 2),
947 TRIM_REG(0, 0, 1, 3),
948
949 TRIM_REG(1, 0, 2, 4),
950 TRIM_REG(1, 0, 2, 5),
951 TRIM_REG(1, 0, 3, 6),
952 TRIM_REG(1, 0, 3, 7),
953
954 TRIM_REG(0, 1, 0, 0),
955 TRIM_REG(0, 1, 0, 1),
956 TRIM_REG(0, 1, 1, 2),
957 TRIM_REG(0, 1, 1, 3),
958
959 TRIM_REG(1, 1, 2, 4),
960 TRIM_REG(1, 1, 2, 5),
961 TRIM_REG(1, 1, 3, 6),
962 TRIM_REG(1, 1, 3, 7)
963 };
964 unsigned i;
965
966 switch (offset) {
967 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0:
968 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1:
969 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2:
970 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3:
971 case EMC_DATA_BRLSHFT_0:
972 delta[0] = 128 * (next->current_dram_clktree[C0D0U0] -
973 next->trained_dram_clktree[C0D0U0]);
974 delta[1] = 128 * (next->current_dram_clktree[C0D0U1] -
975 next->trained_dram_clktree[C0D0U1]);
976 delta[2] = 128 * (next->current_dram_clktree[C1D0U0] -
977 next->trained_dram_clktree[C1D0U0]);
978 delta[3] = 128 * (next->current_dram_clktree[C1D0U1] -
979 next->trained_dram_clktree[C1D0U1]);
980
981 delta_taps[0] = (delta[0] * (s32)rate) / 1000000;
982 delta_taps[1] = (delta[1] * (s32)rate) / 1000000;
983 delta_taps[2] = (delta[2] * (s32)rate) / 1000000;
984 delta_taps[3] = (delta[3] * (s32)rate) / 1000000;
985
986 for (i = 0; i < 4; i++) {
987 if ((delta_taps[i] > next->tree_margin) ||
988 (delta_taps[i] < (-1 * next->tree_margin))) {
989 new[i * 2] = new[i * 2] + delta_taps[i];
990 new[i * 2 + 1] = new[i * 2 + 1] +
991 delta_taps[i];
992 }
993 }
994
995 if (offset == EMC_DATA_BRLSHFT_0) {
996 for (i = 0; i < 8; i++)
997 new[i] = new[i] / 64;
998 } else {
999 for (i = 0; i < 8; i++)
1000 new[i] = new[i] % 64;
1001 }
1002
1003 break;
1004
1005 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0:
1006 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1:
1007 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2:
1008 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3:
1009 case EMC_DATA_BRLSHFT_1:
1010 delta[0] = 128 * (next->current_dram_clktree[C0D1U0] -
1011 next->trained_dram_clktree[C0D1U0]);
1012 delta[1] = 128 * (next->current_dram_clktree[C0D1U1] -
1013 next->trained_dram_clktree[C0D1U1]);
1014 delta[2] = 128 * (next->current_dram_clktree[C1D1U0] -
1015 next->trained_dram_clktree[C1D1U0]);
1016 delta[3] = 128 * (next->current_dram_clktree[C1D1U1] -
1017 next->trained_dram_clktree[C1D1U1]);
1018
1019 delta_taps[0] = (delta[0] * (s32)rate) / 1000000;
1020 delta_taps[1] = (delta[1] * (s32)rate) / 1000000;
1021 delta_taps[2] = (delta[2] * (s32)rate) / 1000000;
1022 delta_taps[3] = (delta[3] * (s32)rate) / 1000000;
1023
1024 for (i = 0; i < 4; i++) {
1025 if ((delta_taps[i] > next->tree_margin) ||
1026 (delta_taps[i] < (-1 * next->tree_margin))) {
1027 new[8 + i * 2] = new[8 + i * 2] +
1028 delta_taps[i];
1029 new[8 + i * 2 + 1] = new[8 + i * 2 + 1] +
1030 delta_taps[i];
1031 }
1032 }
1033
1034 if (offset == EMC_DATA_BRLSHFT_1) {
1035 for (i = 0; i < 8; i++)
1036 new[i + 8] = new[i + 8] / 64;
1037 } else {
1038 for (i = 0; i < 8; i++)
1039 new[i + 8] = new[i + 8] % 64;
1040 }
1041
1042 break;
1043 }
1044
1045 switch (offset) {
1046 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0:
1047 temp = CALC_TEMP(0, 0, 0, 1, 0);
1048 break;
1049
1050 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1:
1051 temp = CALC_TEMP(0, 1, 2, 3, 2);
1052 break;
1053
1054 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2:
1055 temp = CALC_TEMP(0, 2, 4, 5, 4);
1056 break;
1057
1058 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3:
1059 temp = CALC_TEMP(0, 3, 6, 7, 6);
1060 break;
1061
1062 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0:
1063 temp = CALC_TEMP(1, 0, 0, 1, 8);
1064 break;
1065
1066 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1:
1067 temp = CALC_TEMP(1, 1, 2, 3, 10);
1068 break;
1069
1070 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2:
1071 temp = CALC_TEMP(1, 2, 4, 5, 12);
1072 break;
1073
1074 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3:
1075 temp = CALC_TEMP(1, 3, 6, 7, 14);
1076 break;
1077
1078 case EMC_DATA_BRLSHFT_0:
1079 temp = ((new[0] <<
1080 EMC_DATA_BRLSHFT_0_RANK0_BYTE0_DATA_BRLSHFT_SHIFT) &
1081 EMC_DATA_BRLSHFT_0_RANK0_BYTE0_DATA_BRLSHFT_MASK) |
1082 ((new[1] <<
1083 EMC_DATA_BRLSHFT_0_RANK0_BYTE1_DATA_BRLSHFT_SHIFT) &
1084 EMC_DATA_BRLSHFT_0_RANK0_BYTE1_DATA_BRLSHFT_MASK) |
1085 ((new[2] <<
1086 EMC_DATA_BRLSHFT_0_RANK0_BYTE2_DATA_BRLSHFT_SHIFT) &
1087 EMC_DATA_BRLSHFT_0_RANK0_BYTE2_DATA_BRLSHFT_MASK) |
1088 ((new[3] <<
1089 EMC_DATA_BRLSHFT_0_RANK0_BYTE3_DATA_BRLSHFT_SHIFT) &
1090 EMC_DATA_BRLSHFT_0_RANK0_BYTE3_DATA_BRLSHFT_MASK) |
1091 ((new[4] <<
1092 EMC_DATA_BRLSHFT_0_RANK0_BYTE4_DATA_BRLSHFT_SHIFT) &
1093 EMC_DATA_BRLSHFT_0_RANK0_BYTE4_DATA_BRLSHFT_MASK) |
1094 ((new[5] <<
1095 EMC_DATA_BRLSHFT_0_RANK0_BYTE5_DATA_BRLSHFT_SHIFT) &
1096 EMC_DATA_BRLSHFT_0_RANK0_BYTE5_DATA_BRLSHFT_MASK) |
1097 ((new[6] <<
1098 EMC_DATA_BRLSHFT_0_RANK0_BYTE6_DATA_BRLSHFT_SHIFT) &
1099 EMC_DATA_BRLSHFT_0_RANK0_BYTE6_DATA_BRLSHFT_MASK) |
1100 ((new[7] <<
1101 EMC_DATA_BRLSHFT_0_RANK0_BYTE7_DATA_BRLSHFT_SHIFT) &
1102 EMC_DATA_BRLSHFT_0_RANK0_BYTE7_DATA_BRLSHFT_MASK);
1103 break;
1104
1105 case EMC_DATA_BRLSHFT_1:
1106 temp = ((new[8] <<
1107 EMC_DATA_BRLSHFT_1_RANK1_BYTE0_DATA_BRLSHFT_SHIFT) &
1108 EMC_DATA_BRLSHFT_1_RANK1_BYTE0_DATA_BRLSHFT_MASK) |
1109 ((new[9] <<
1110 EMC_DATA_BRLSHFT_1_RANK1_BYTE1_DATA_BRLSHFT_SHIFT) &
1111 EMC_DATA_BRLSHFT_1_RANK1_BYTE1_DATA_BRLSHFT_MASK) |
1112 ((new[10] <<
1113 EMC_DATA_BRLSHFT_1_RANK1_BYTE2_DATA_BRLSHFT_SHIFT) &
1114 EMC_DATA_BRLSHFT_1_RANK1_BYTE2_DATA_BRLSHFT_MASK) |
1115 ((new[11] <<
1116 EMC_DATA_BRLSHFT_1_RANK1_BYTE3_DATA_BRLSHFT_SHIFT) &
1117 EMC_DATA_BRLSHFT_1_RANK1_BYTE3_DATA_BRLSHFT_MASK) |
1118 ((new[12] <<
1119 EMC_DATA_BRLSHFT_1_RANK1_BYTE4_DATA_BRLSHFT_SHIFT) &
1120 EMC_DATA_BRLSHFT_1_RANK1_BYTE4_DATA_BRLSHFT_MASK) |
1121 ((new[13] <<
1122 EMC_DATA_BRLSHFT_1_RANK1_BYTE5_DATA_BRLSHFT_SHIFT) &
1123 EMC_DATA_BRLSHFT_1_RANK1_BYTE5_DATA_BRLSHFT_MASK) |
1124 ((new[14] <<
1125 EMC_DATA_BRLSHFT_1_RANK1_BYTE6_DATA_BRLSHFT_SHIFT) &
1126 EMC_DATA_BRLSHFT_1_RANK1_BYTE6_DATA_BRLSHFT_MASK) |
1127 ((new[15] <<
1128 EMC_DATA_BRLSHFT_1_RANK1_BYTE7_DATA_BRLSHFT_SHIFT) &
1129 EMC_DATA_BRLSHFT_1_RANK1_BYTE7_DATA_BRLSHFT_MASK);
1130 break;
1131
1132 default:
1133 break;
1134 }
1135
1136 return temp;
1137 }
1138
1139 u32 tegra210_emc_dll_prelock(struct tegra210_emc *emc, u32 clksrc)
1140 {
1141 unsigned int i;
1142 u32 value;
1143
1144 value = emc_readl(emc, EMC_CFG_DIG_DLL);
1145 value &= ~EMC_CFG_DIG_DLL_CFG_DLL_LOCK_LIMIT_MASK;
1146 value |= (3 << EMC_CFG_DIG_DLL_CFG_DLL_LOCK_LIMIT_SHIFT);
1147 value &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN;
1148 value &= ~EMC_CFG_DIG_DLL_CFG_DLL_MODE_MASK;
1149 value |= (3 << EMC_CFG_DIG_DLL_CFG_DLL_MODE_SHIFT);
1150 value |= EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_TRAFFIC;
1151 value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_RW_UNTIL_LOCK;
1152 value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_UNTIL_LOCK;
1153 emc_writel(emc, value, EMC_CFG_DIG_DLL);
1154 emc_writel(emc, 1, EMC_TIMING_CONTROL);
1155
1156 for (i = 0; i < emc->num_channels; i++)
1157 tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS,
1158 EMC_EMC_STATUS_TIMING_UPDATE_STALLED,
1159 0);
1160
1161 for (i = 0; i < emc->num_channels; i++) {
1162 while (true) {
1163 value = emc_channel_readl(emc, i, EMC_CFG_DIG_DLL);
1164 if ((value & EMC_CFG_DIG_DLL_CFG_DLL_EN) == 0)
1165 break;
1166 }
1167 }
1168
1169 value = emc->next->burst_regs[EMC_DLL_CFG_0_INDEX];
1170 emc_writel(emc, value, EMC_DLL_CFG_0);
1171
1172 value = emc_readl(emc, EMC_DLL_CFG_1);
1173 value &= EMC_DLL_CFG_1_DDLLCAL_CTRL_START_TRIM_MASK;
1174
1175 if (emc->next->rate >= 400000 && emc->next->rate < 600000)
1176 value |= 150;
1177 else if (emc->next->rate >= 600000 && emc->next->rate < 800000)
1178 value |= 100;
1179 else if (emc->next->rate >= 800000 && emc->next->rate < 1000000)
1180 value |= 70;
1181 else if (emc->next->rate >= 1000000 && emc->next->rate < 1200000)
1182 value |= 30;
1183 else
1184 value |= 20;
1185
1186 emc_writel(emc, value, EMC_DLL_CFG_1);
1187
1188 tegra210_change_dll_src(emc, clksrc);
1189
1190 value = emc_readl(emc, EMC_CFG_DIG_DLL);
1191 value |= EMC_CFG_DIG_DLL_CFG_DLL_EN;
1192 emc_writel(emc, value, EMC_CFG_DIG_DLL);
1193
1194 tegra210_emc_timing_update(emc);
1195
1196 for (i = 0; i < emc->num_channels; i++) {
1197 while (true) {
1198 value = emc_channel_readl(emc, 0, EMC_CFG_DIG_DLL);
1199 if (value & EMC_CFG_DIG_DLL_CFG_DLL_EN)
1200 break;
1201 }
1202 }
1203
1204 while (true) {
1205 value = emc_readl(emc, EMC_DIG_DLL_STATUS);
1206
1207 if ((value & EMC_DIG_DLL_STATUS_DLL_PRIV_UPDATED) == 0)
1208 continue;
1209
1210 if ((value & EMC_DIG_DLL_STATUS_DLL_LOCK) == 0)
1211 continue;
1212
1213 break;
1214 }
1215
1216 value = emc_readl(emc, EMC_DIG_DLL_STATUS);
1217
1218 return value & EMC_DIG_DLL_STATUS_DLL_OUT_MASK;
1219 }
1220
1221 u32 tegra210_emc_dvfs_power_ramp_up(struct tegra210_emc *emc, u32 clk,
1222 bool flip_backward)
1223 {
1224 u32 cmd_pad, dq_pad, rfu1, cfg5, common_tx, ramp_up_wait = 0;
1225 const struct tegra210_emc_timing *timing;
1226
1227 if (flip_backward)
1228 timing = emc->last;
1229 else
1230 timing = emc->next;
1231
1232 cmd_pad = timing->burst_regs[EMC_PMACRO_CMD_PAD_TX_CTRL_INDEX];
1233 dq_pad = timing->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX];
1234 rfu1 = timing->burst_regs[EMC_PMACRO_BRICK_CTRL_RFU1_INDEX];
1235 cfg5 = timing->burst_regs[EMC_FBIO_CFG5_INDEX];
1236 common_tx = timing->burst_regs[EMC_PMACRO_COMMON_PAD_TX_CTRL_INDEX];
1237
1238 cmd_pad |= EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON;
1239
1240 if (clk < 1000000 / DVFS_FGCG_MID_SPEED_THRESHOLD) {
1241 ccfifo_writel(emc, common_tx & 0xa,
1242 EMC_PMACRO_COMMON_PAD_TX_CTRL, 0);
1243 ccfifo_writel(emc, common_tx & 0xf,
1244 EMC_PMACRO_COMMON_PAD_TX_CTRL,
1245 (100000 / clk) + 1);
1246 ramp_up_wait += 100000;
1247 } else {
1248 ccfifo_writel(emc, common_tx | 0x8,
1249 EMC_PMACRO_COMMON_PAD_TX_CTRL, 0);
1250 }
1251
1252 if (clk < 1000000 / DVFS_FGCG_HIGH_SPEED_THRESHOLD) {
1253 if (clk < 1000000 / IOBRICK_DCC_THRESHOLD) {
1254 cmd_pad |=
1255 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC |
1256 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC;
1257 cmd_pad &=
1258 ~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC |
1259 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC);
1260 ccfifo_writel(emc, cmd_pad,
1261 EMC_PMACRO_CMD_PAD_TX_CTRL,
1262 (100000 / clk) + 1);
1263 ramp_up_wait += 100000;
1264
1265 dq_pad |=
1266 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC |
1267 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC;
1268 dq_pad &=
1269 ~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC |
1270 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC);
1271 ccfifo_writel(emc, dq_pad,
1272 EMC_PMACRO_DATA_PAD_TX_CTRL, 0);
1273 ccfifo_writel(emc, rfu1 & 0xfe40fe40,
1274 EMC_PMACRO_BRICK_CTRL_RFU1, 0);
1275 } else {
1276 ccfifo_writel(emc, rfu1 & 0xfe40fe40,
1277 EMC_PMACRO_BRICK_CTRL_RFU1,
1278 (100000 / clk) + 1);
1279 ramp_up_wait += 100000;
1280 }
1281
1282 ccfifo_writel(emc, rfu1 & 0xfeedfeed,
1283 EMC_PMACRO_BRICK_CTRL_RFU1, (100000 / clk) + 1);
1284 ramp_up_wait += 100000;
1285
1286 if (clk < 1000000 / IOBRICK_DCC_THRESHOLD) {
1287 cmd_pad |=
1288 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC |
1289 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC |
1290 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC |
1291 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC;
1292 ccfifo_writel(emc, cmd_pad,
1293 EMC_PMACRO_CMD_PAD_TX_CTRL,
1294 (100000 / clk) + 1);
1295 ramp_up_wait += 100000;
1296
1297 dq_pad |=
1298 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC |
1299 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC |
1300 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC |
1301 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC;
1302 ccfifo_writel(emc, dq_pad,
1303 EMC_PMACRO_DATA_PAD_TX_CTRL, 0);
1304 ccfifo_writel(emc, rfu1,
1305 EMC_PMACRO_BRICK_CTRL_RFU1, 0);
1306 } else {
1307 ccfifo_writel(emc, rfu1,
1308 EMC_PMACRO_BRICK_CTRL_RFU1,
1309 (100000 / clk) + 1);
1310 ramp_up_wait += 100000;
1311 }
1312
1313 ccfifo_writel(emc, cfg5 & ~EMC_FBIO_CFG5_CMD_TX_DIS,
1314 EMC_FBIO_CFG5, (100000 / clk) + 10);
1315 ramp_up_wait += 100000 + (10 * clk);
1316 } else if (clk < 1000000 / DVFS_FGCG_MID_SPEED_THRESHOLD) {
1317 ccfifo_writel(emc, rfu1 | 0x06000600,
1318 EMC_PMACRO_BRICK_CTRL_RFU1, (100000 / clk) + 1);
1319 ccfifo_writel(emc, cfg5 & ~EMC_FBIO_CFG5_CMD_TX_DIS,
1320 EMC_FBIO_CFG5, (100000 / clk) + 10);
1321 ramp_up_wait += 100000 + 10 * clk;
1322 } else {
1323 ccfifo_writel(emc, rfu1 | 0x00000600,
1324 EMC_PMACRO_BRICK_CTRL_RFU1, 0);
1325 ccfifo_writel(emc, cfg5 & ~EMC_FBIO_CFG5_CMD_TX_DIS,
1326 EMC_FBIO_CFG5, 12);
1327 ramp_up_wait += 12 * clk;
1328 }
1329
1330 cmd_pad &= ~EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON;
1331 ccfifo_writel(emc, cmd_pad, EMC_PMACRO_CMD_PAD_TX_CTRL, 5);
1332
1333 return ramp_up_wait;
1334 }
1335
1336 u32 tegra210_emc_dvfs_power_ramp_down(struct tegra210_emc *emc, u32 clk,
1337 bool flip_backward)
1338 {
1339 u32 ramp_down_wait = 0, cmd_pad, dq_pad, rfu1, cfg5, common_tx;
1340 const struct tegra210_emc_timing *entry;
1341 u32 seq_wait;
1342
1343 if (flip_backward)
1344 entry = emc->next;
1345 else
1346 entry = emc->last;
1347
1348 cmd_pad = entry->burst_regs[EMC_PMACRO_CMD_PAD_TX_CTRL_INDEX];
1349 dq_pad = entry->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX];
1350 rfu1 = entry->burst_regs[EMC_PMACRO_BRICK_CTRL_RFU1_INDEX];
1351 cfg5 = entry->burst_regs[EMC_FBIO_CFG5_INDEX];
1352 common_tx = entry->burst_regs[EMC_PMACRO_COMMON_PAD_TX_CTRL_INDEX];
1353
1354 cmd_pad |= EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON;
1355
1356 ccfifo_writel(emc, cmd_pad, EMC_PMACRO_CMD_PAD_TX_CTRL, 0);
1357 ccfifo_writel(emc, cfg5 | EMC_FBIO_CFG5_CMD_TX_DIS,
1358 EMC_FBIO_CFG5, 12);
1359 ramp_down_wait = 12 * clk;
1360
1361 seq_wait = (100000 / clk) + 1;
1362
1363 if (clk < (1000000 / DVFS_FGCG_HIGH_SPEED_THRESHOLD)) {
1364 if (clk < (1000000 / IOBRICK_DCC_THRESHOLD)) {
1365 cmd_pad &=
1366 ~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC |
1367 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC);
1368 cmd_pad |=
1369 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC |
1370 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC;
1371 ccfifo_writel(emc, cmd_pad,
1372 EMC_PMACRO_CMD_PAD_TX_CTRL, seq_wait);
1373 ramp_down_wait += 100000;
1374
1375 dq_pad &=
1376 ~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC |
1377 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC);
1378 dq_pad |=
1379 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC |
1380 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC;
1381 ccfifo_writel(emc, dq_pad,
1382 EMC_PMACRO_DATA_PAD_TX_CTRL, 0);
1383 ccfifo_writel(emc, rfu1 & ~0x01120112,
1384 EMC_PMACRO_BRICK_CTRL_RFU1, 0);
1385 } else {
1386 ccfifo_writel(emc, rfu1 & ~0x01120112,
1387 EMC_PMACRO_BRICK_CTRL_RFU1, seq_wait);
1388 ramp_down_wait += 100000;
1389 }
1390
1391 ccfifo_writel(emc, rfu1 & ~0x01bf01bf,
1392 EMC_PMACRO_BRICK_CTRL_RFU1, seq_wait);
1393 ramp_down_wait += 100000;
1394
1395 if (clk < (1000000 / IOBRICK_DCC_THRESHOLD)) {
1396 cmd_pad &=
1397 ~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC |
1398 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC |
1399 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC |
1400 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC);
1401 ccfifo_writel(emc, cmd_pad,
1402 EMC_PMACRO_CMD_PAD_TX_CTRL, seq_wait);
1403 ramp_down_wait += 100000;
1404
1405 dq_pad &=
1406 ~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC |
1407 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC |
1408 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC |
1409 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC);
1410 ccfifo_writel(emc, dq_pad,
1411 EMC_PMACRO_DATA_PAD_TX_CTRL, 0);
1412 ccfifo_writel(emc, rfu1 & ~0x07ff07ff,
1413 EMC_PMACRO_BRICK_CTRL_RFU1, 0);
1414 } else {
1415 ccfifo_writel(emc, rfu1 & ~0x07ff07ff,
1416 EMC_PMACRO_BRICK_CTRL_RFU1, seq_wait);
1417 ramp_down_wait += 100000;
1418 }
1419 } else {
1420 ccfifo_writel(emc, rfu1 & ~0xffff07ff,
1421 EMC_PMACRO_BRICK_CTRL_RFU1, seq_wait + 19);
1422 ramp_down_wait += 100000 + (20 * clk);
1423 }
1424
1425 if (clk < (1000000 / DVFS_FGCG_MID_SPEED_THRESHOLD)) {
1426 ramp_down_wait += 100000;
1427 ccfifo_writel(emc, common_tx & ~0x5,
1428 EMC_PMACRO_COMMON_PAD_TX_CTRL, seq_wait);
1429 ramp_down_wait += 100000;
1430 ccfifo_writel(emc, common_tx & ~0xf,
1431 EMC_PMACRO_COMMON_PAD_TX_CTRL, seq_wait);
1432 ramp_down_wait += 100000;
1433 ccfifo_writel(emc, 0, 0, seq_wait);
1434 ramp_down_wait += 100000;
1435 } else {
1436 ccfifo_writel(emc, common_tx & ~0xf,
1437 EMC_PMACRO_COMMON_PAD_TX_CTRL, seq_wait);
1438 }
1439
1440 return ramp_down_wait;
1441 }
1442
1443 void tegra210_emc_reset_dram_clktree_values(struct tegra210_emc_timing *timing)
1444 {
1445 timing->current_dram_clktree[C0D0U0] =
1446 timing->trained_dram_clktree[C0D0U0];
1447 timing->current_dram_clktree[C0D0U1] =
1448 timing->trained_dram_clktree[C0D0U1];
1449 timing->current_dram_clktree[C1D0U0] =
1450 timing->trained_dram_clktree[C1D0U0];
1451 timing->current_dram_clktree[C1D0U1] =
1452 timing->trained_dram_clktree[C1D0U1];
1453 timing->current_dram_clktree[C1D1U0] =
1454 timing->trained_dram_clktree[C1D1U0];
1455 timing->current_dram_clktree[C1D1U1] =
1456 timing->trained_dram_clktree[C1D1U1];
1457 }
1458
1459 static void update_dll_control(struct tegra210_emc *emc, u32 value, bool state)
1460 {
1461 unsigned int i;
1462
1463 emc_writel(emc, value, EMC_CFG_DIG_DLL);
1464 tegra210_emc_timing_update(emc);
1465
1466 for (i = 0; i < emc->num_channels; i++)
1467 tegra210_emc_wait_for_update(emc, i, EMC_CFG_DIG_DLL,
1468 EMC_CFG_DIG_DLL_CFG_DLL_EN,
1469 state);
1470 }
1471
1472 void tegra210_emc_dll_disable(struct tegra210_emc *emc)
1473 {
1474 u32 value;
1475
1476 value = emc_readl(emc, EMC_CFG_DIG_DLL);
1477 value &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN;
1478
1479 update_dll_control(emc, value, false);
1480 }
1481
1482 void tegra210_emc_dll_enable(struct tegra210_emc *emc)
1483 {
1484 u32 value;
1485
1486 value = emc_readl(emc, EMC_CFG_DIG_DLL);
1487 value |= EMC_CFG_DIG_DLL_CFG_DLL_EN;
1488
1489 update_dll_control(emc, value, true);
1490 }
1491
1492 void tegra210_emc_adjust_timing(struct tegra210_emc *emc,
1493 struct tegra210_emc_timing *timing)
1494 {
1495 u32 dsr_cntrl = timing->burst_regs[EMC_DYN_SELF_REF_CONTROL_INDEX];
1496 u32 pre_ref = timing->burst_regs[EMC_PRE_REFRESH_REQ_CNT_INDEX];
1497 u32 ref = timing->burst_regs[EMC_REFRESH_INDEX];
1498
1499 switch (emc->refresh) {
1500 case TEGRA210_EMC_REFRESH_NOMINAL:
1501 case TEGRA210_EMC_REFRESH_THROTTLE:
1502 break;
1503
1504 case TEGRA210_EMC_REFRESH_2X:
1505 ref = REFRESH_SPEEDUP(ref, 2);
1506 pre_ref = REFRESH_SPEEDUP(pre_ref, 2);
1507 dsr_cntrl = REFRESH_SPEEDUP(dsr_cntrl, 2);
1508 break;
1509
1510 case TEGRA210_EMC_REFRESH_4X:
1511 ref = REFRESH_SPEEDUP(ref, 4);
1512 pre_ref = REFRESH_SPEEDUP(pre_ref, 4);
1513 dsr_cntrl = REFRESH_SPEEDUP(dsr_cntrl, 4);
1514 break;
1515
1516 default:
1517 dev_warn(emc->dev, "failed to set refresh: %d\n", emc->refresh);
1518 return;
1519 }
1520
1521 emc_writel(emc, ref, emc->offsets->burst[EMC_REFRESH_INDEX]);
1522 emc_writel(emc, pre_ref,
1523 emc->offsets->burst[EMC_PRE_REFRESH_REQ_CNT_INDEX]);
1524 emc_writel(emc, dsr_cntrl,
1525 emc->offsets->burst[EMC_DYN_SELF_REF_CONTROL_INDEX]);
1526 }
1527
1528 static int tegra210_emc_set_rate(struct device *dev,
1529 const struct tegra210_clk_emc_config *config)
1530 {
1531 struct tegra210_emc *emc = dev_get_drvdata(dev);
1532 struct tegra210_emc_timing *timing = NULL;
1533 unsigned long rate = config->rate;
1534 s64 last_change_delay;
1535 unsigned long flags;
1536 unsigned int i;
1537
1538 if (rate == emc->last->rate * 1000UL)
1539 return 0;
1540
1541 for (i = 0; i < emc->num_timings; i++) {
1542 if (emc->timings[i].rate * 1000UL == rate) {
1543 timing = &emc->timings[i];
1544 break;
1545 }
1546 }
1547
1548 if (!timing)
1549 return -EINVAL;
1550
1551 if (rate > 204000000 && !timing->trained)
1552 return -EINVAL;
1553
1554 emc->next = timing;
1555 last_change_delay = ktime_us_delta(ktime_get(), emc->clkchange_time);
1556
1557 /* XXX use non-busy-looping sleep? */
1558 if ((last_change_delay >= 0) &&
1559 (last_change_delay < emc->clkchange_delay))
1560 udelay(emc->clkchange_delay - (int)last_change_delay);
1561
1562 spin_lock_irqsave(&emc->lock, flags);
1563 tegra210_emc_set_clock(emc, config->value);
1564 emc->clkchange_time = ktime_get();
1565 emc->last = timing;
1566 spin_unlock_irqrestore(&emc->lock, flags);
1567
1568 return 0;
1569 }
1570
1571 /*
1572 * debugfs interface
1573 *
1574 * The memory controller driver exposes some files in debugfs that can be used
1575 * to control the EMC frequency. The top-level directory can be found here:
1576 *
1577 * /sys/kernel/debug/emc
1578 *
1579 * It contains the following files:
1580 *
1581 * - available_rates: This file contains a list of valid, space-separated
1582 * EMC frequencies.
1583 *
1584 * - min_rate: Writing a value to this file sets the given frequency as the
1585 * floor of the permitted range. If this is higher than the currently
1586 * configured EMC frequency, this will cause the frequency to be
1587 * increased so that it stays within the valid range.
1588 *
1589 * - max_rate: Similarily to the min_rate file, writing a value to this file
1590 * sets the given frequency as the ceiling of the permitted range. If
1591 * the value is lower than the currently configured EMC frequency, this
1592 * will cause the frequency to be decreased so that it stays within the
1593 * valid range.
1594 */
1595
1596 static bool tegra210_emc_validate_rate(struct tegra210_emc *emc,
1597 unsigned long rate)
1598 {
1599 unsigned int i;
1600
1601 for (i = 0; i < emc->num_timings; i++)
1602 if (rate == emc->timings[i].rate * 1000UL)
1603 return true;
1604
1605 return false;
1606 }
1607
1608 static int tegra210_emc_debug_available_rates_show(struct seq_file *s,
1609 void *data)
1610 {
1611 struct tegra210_emc *emc = s->private;
1612 const char *prefix = "";
1613 unsigned int i;
1614
1615 for (i = 0; i < emc->num_timings; i++) {
1616 seq_printf(s, "%s%u", prefix, emc->timings[i].rate * 1000);
1617 prefix = " ";
1618 }
1619
1620 seq_puts(s, "\n");
1621
1622 return 0;
1623 }
1624
1625 static int tegra210_emc_debug_available_rates_open(struct inode *inode,
1626 struct file *file)
1627 {
1628 return single_open(file, tegra210_emc_debug_available_rates_show,
1629 inode->i_private);
1630 }
1631
1632 static const struct file_operations tegra210_emc_debug_available_rates_fops = {
1633 .open = tegra210_emc_debug_available_rates_open,
1634 .read = seq_read,
1635 .llseek = seq_lseek,
1636 .release = single_release,
1637 };
1638
1639 static int tegra210_emc_debug_min_rate_get(void *data, u64 *rate)
1640 {
1641 struct tegra210_emc *emc = data;
1642
1643 *rate = emc->debugfs.min_rate;
1644
1645 return 0;
1646 }
1647
1648 static int tegra210_emc_debug_min_rate_set(void *data, u64 rate)
1649 {
1650 struct tegra210_emc *emc = data;
1651 int err;
1652
1653 if (!tegra210_emc_validate_rate(emc, rate))
1654 return -EINVAL;
1655
1656 err = clk_set_min_rate(emc->clk, rate);
1657 if (err < 0)
1658 return err;
1659
1660 emc->debugfs.min_rate = rate;
1661
1662 return 0;
1663 }
1664
1665 DEFINE_SIMPLE_ATTRIBUTE(tegra210_emc_debug_min_rate_fops,
1666 tegra210_emc_debug_min_rate_get,
1667 tegra210_emc_debug_min_rate_set, "%llu\n");
1668
1669 static int tegra210_emc_debug_max_rate_get(void *data, u64 *rate)
1670 {
1671 struct tegra210_emc *emc = data;
1672
1673 *rate = emc->debugfs.max_rate;
1674
1675 return 0;
1676 }
1677
1678 static int tegra210_emc_debug_max_rate_set(void *data, u64 rate)
1679 {
1680 struct tegra210_emc *emc = data;
1681 int err;
1682
1683 if (!tegra210_emc_validate_rate(emc, rate))
1684 return -EINVAL;
1685
1686 err = clk_set_max_rate(emc->clk, rate);
1687 if (err < 0)
1688 return err;
1689
1690 emc->debugfs.max_rate = rate;
1691
1692 return 0;
1693 }
1694
1695 DEFINE_SIMPLE_ATTRIBUTE(tegra210_emc_debug_max_rate_fops,
1696 tegra210_emc_debug_max_rate_get,
1697 tegra210_emc_debug_max_rate_set, "%llu\n");
1698
1699 static int tegra210_emc_debug_temperature_get(void *data, u64 *temperature)
1700 {
1701 struct tegra210_emc *emc = data;
1702 unsigned int value;
1703
1704 if (!emc->debugfs.temperature)
1705 value = tegra210_emc_get_temperature(emc);
1706 else
1707 value = emc->debugfs.temperature;
1708
1709 *temperature = value;
1710
1711 return 0;
1712 }
1713
1714 static int tegra210_emc_debug_temperature_set(void *data, u64 temperature)
1715 {
1716 struct tegra210_emc *emc = data;
1717
1718 if (temperature > 7)
1719 return -EINVAL;
1720
1721 emc->debugfs.temperature = temperature;
1722
1723 return 0;
1724 }
1725
1726 DEFINE_SIMPLE_ATTRIBUTE(tegra210_emc_debug_temperature_fops,
1727 tegra210_emc_debug_temperature_get,
1728 tegra210_emc_debug_temperature_set, "%llu\n");
1729
1730 static void tegra210_emc_debugfs_init(struct tegra210_emc *emc)
1731 {
1732 struct device *dev = emc->dev;
1733 unsigned int i;
1734 int err;
1735
1736 emc->debugfs.min_rate = ULONG_MAX;
1737 emc->debugfs.max_rate = 0;
1738
1739 for (i = 0; i < emc->num_timings; i++) {
1740 if (emc->timings[i].rate * 1000UL < emc->debugfs.min_rate)
1741 emc->debugfs.min_rate = emc->timings[i].rate * 1000UL;
1742
1743 if (emc->timings[i].rate * 1000UL > emc->debugfs.max_rate)
1744 emc->debugfs.max_rate = emc->timings[i].rate * 1000UL;
1745 }
1746
1747 if (!emc->num_timings) {
1748 emc->debugfs.min_rate = clk_get_rate(emc->clk);
1749 emc->debugfs.max_rate = emc->debugfs.min_rate;
1750 }
1751
1752 err = clk_set_rate_range(emc->clk, emc->debugfs.min_rate,
1753 emc->debugfs.max_rate);
1754 if (err < 0) {
1755 dev_err(dev, "failed to set rate range [%lu-%lu] for %pC\n",
1756 emc->debugfs.min_rate, emc->debugfs.max_rate,
1757 emc->clk);
1758 return;
1759 }
1760
1761 emc->debugfs.root = debugfs_create_dir("emc", NULL);
1762 if (!emc->debugfs.root) {
1763 dev_err(dev, "failed to create debugfs directory\n");
1764 return;
1765 }
1766
1767 debugfs_create_file("available_rates", 0444, emc->debugfs.root, emc,
1768 &tegra210_emc_debug_available_rates_fops);
1769 debugfs_create_file("min_rate", 0644, emc->debugfs.root, emc,
1770 &tegra210_emc_debug_min_rate_fops);
1771 debugfs_create_file("max_rate", 0644, emc->debugfs.root, emc,
1772 &tegra210_emc_debug_max_rate_fops);
1773 debugfs_create_file("temperature", 0644, emc->debugfs.root, emc,
1774 &tegra210_emc_debug_temperature_fops);
1775 }
1776
1777 static void tegra210_emc_detect(struct tegra210_emc *emc)
1778 {
1779 u32 value;
1780
1781 /* probe the number of connected DRAM devices */
1782 value = mc_readl(emc->mc, MC_EMEM_ADR_CFG);
1783
1784 if (value & MC_EMEM_ADR_CFG_EMEM_NUMDEV)
1785 emc->num_devices = 2;
1786 else
1787 emc->num_devices = 1;
1788
1789 /* probe the type of DRAM */
1790 value = emc_readl(emc, EMC_FBIO_CFG5);
1791 emc->dram_type = value & 0x3;
1792
1793 /* probe the number of channels */
1794 value = emc_readl(emc, EMC_FBIO_CFG7);
1795
1796 if ((value & EMC_FBIO_CFG7_CH1_ENABLE) &&
1797 (value & EMC_FBIO_CFG7_CH0_ENABLE))
1798 emc->num_channels = 2;
1799 else
1800 emc->num_channels = 1;
1801 }
1802
1803 static int tegra210_emc_validate_timings(struct tegra210_emc *emc,
1804 struct tegra210_emc_timing *timings,
1805 unsigned int num_timings)
1806 {
1807 unsigned int i;
1808
1809 for (i = 0; i < num_timings; i++) {
1810 u32 min_volt = timings[i].min_volt;
1811 u32 rate = timings[i].rate;
1812
1813 if (!rate)
1814 return -EINVAL;
1815
1816 if ((i > 0) && ((rate <= timings[i - 1].rate) ||
1817 (min_volt < timings[i - 1].min_volt)))
1818 return -EINVAL;
1819
1820 if (timings[i].revision != timings[0].revision)
1821 continue;
1822 }
1823
1824 return 0;
1825 }
1826
1827 static int tegra210_emc_probe(struct platform_device *pdev)
1828 {
1829 struct thermal_cooling_device *cd;
1830 unsigned long current_rate;
1831 struct tegra210_emc *emc;
1832 struct device_node *np;
1833 unsigned int i;
1834 int err;
1835
1836 emc = devm_kzalloc(&pdev->dev, sizeof(*emc), GFP_KERNEL);
1837 if (!emc)
1838 return -ENOMEM;
1839
1840 emc->clk = devm_clk_get(&pdev->dev, "emc");
1841 if (IS_ERR(emc->clk))
1842 return PTR_ERR(emc->clk);
1843
1844 platform_set_drvdata(pdev, emc);
1845 spin_lock_init(&emc->lock);
1846 emc->dev = &pdev->dev;
1847
1848 emc->mc = devm_tegra_memory_controller_get(&pdev->dev);
1849 if (IS_ERR(emc->mc))
1850 return PTR_ERR(emc->mc);
1851
1852 emc->regs = devm_platform_ioremap_resource(pdev, 0);
1853 if (IS_ERR(emc->regs))
1854 return PTR_ERR(emc->regs);
1855
1856 for (i = 0; i < 2; i++) {
1857 emc->channel[i] = devm_platform_ioremap_resource(pdev, 1 + i);
1858 if (IS_ERR(emc->channel[i]))
1859 return PTR_ERR(emc->channel[i]);
1860
1861 }
1862
1863 tegra210_emc_detect(emc);
1864 np = pdev->dev.of_node;
1865
1866 /* attach to the nominal and (optional) derated tables */
1867 err = of_reserved_mem_device_init_by_name(emc->dev, np, "nominal");
1868 if (err < 0) {
1869 dev_err(emc->dev, "failed to get nominal EMC table: %d\n", err);
1870 return err;
1871 }
1872
1873 err = of_reserved_mem_device_init_by_name(emc->dev, np, "derated");
1874 if (err < 0 && err != -ENODEV) {
1875 dev_err(emc->dev, "failed to get derated EMC table: %d\n", err);
1876 goto release;
1877 }
1878
1879 /* validate the tables */
1880 if (emc->nominal) {
1881 err = tegra210_emc_validate_timings(emc, emc->nominal,
1882 emc->num_timings);
1883 if (err < 0)
1884 goto release;
1885 }
1886
1887 if (emc->derated) {
1888 err = tegra210_emc_validate_timings(emc, emc->derated,
1889 emc->num_timings);
1890 if (err < 0)
1891 goto release;
1892 }
1893
1894 /* default to the nominal table */
1895 emc->timings = emc->nominal;
1896
1897 /* pick the current timing based on the current EMC clock rate */
1898 current_rate = clk_get_rate(emc->clk) / 1000;
1899
1900 for (i = 0; i < emc->num_timings; i++) {
1901 if (emc->timings[i].rate == current_rate) {
1902 emc->last = &emc->timings[i];
1903 break;
1904 }
1905 }
1906
1907 if (i == emc->num_timings) {
1908 dev_err(emc->dev, "no EMC table entry found for %lu kHz\n",
1909 current_rate);
1910 err = -ENOENT;
1911 goto release;
1912 }
1913
1914 /* pick a compatible clock change sequence for the EMC table */
1915 for (i = 0; i < ARRAY_SIZE(tegra210_emc_sequences); i++) {
1916 const struct tegra210_emc_sequence *sequence =
1917 tegra210_emc_sequences[i];
1918
1919 if (emc->timings[0].revision == sequence->revision) {
1920 emc->sequence = sequence;
1921 break;
1922 }
1923 }
1924
1925 if (!emc->sequence) {
1926 dev_err(&pdev->dev, "sequence %u not supported\n",
1927 emc->timings[0].revision);
1928 err = -ENOTSUPP;
1929 goto release;
1930 }
1931
1932 emc->offsets = &tegra210_emc_table_register_offsets;
1933 emc->refresh = TEGRA210_EMC_REFRESH_NOMINAL;
1934
1935 emc->provider.owner = THIS_MODULE;
1936 emc->provider.dev = &pdev->dev;
1937 emc->provider.set_rate = tegra210_emc_set_rate;
1938
1939 emc->provider.configs = devm_kcalloc(&pdev->dev, emc->num_timings,
1940 sizeof(*emc->provider.configs),
1941 GFP_KERNEL);
1942 if (!emc->provider.configs) {
1943 err = -ENOMEM;
1944 goto release;
1945 }
1946
1947 emc->provider.num_configs = emc->num_timings;
1948
1949 for (i = 0; i < emc->provider.num_configs; i++) {
1950 struct tegra210_emc_timing *timing = &emc->timings[i];
1951 struct tegra210_clk_emc_config *config =
1952 &emc->provider.configs[i];
1953 u32 value;
1954
1955 config->rate = timing->rate * 1000UL;
1956 config->value = timing->clk_src_emc;
1957
1958 value = timing->burst_mc_regs[MC_EMEM_ARB_MISC0_INDEX];
1959
1960 if ((value & MC_EMEM_ARB_MISC0_EMC_SAME_FREQ) == 0)
1961 config->same_freq = false;
1962 else
1963 config->same_freq = true;
1964 }
1965
1966 err = tegra210_clk_emc_attach(emc->clk, &emc->provider);
1967 if (err < 0) {
1968 dev_err(&pdev->dev, "failed to attach to EMC clock: %d\n", err);
1969 goto release;
1970 }
1971
1972 emc->clkchange_delay = 100;
1973 emc->training_interval = 100;
1974 dev_set_drvdata(emc->dev, emc);
1975
1976 timer_setup(&emc->refresh_timer, tegra210_emc_poll_refresh,
1977 TIMER_DEFERRABLE);
1978 atomic_set(&emc->refresh_poll, 0);
1979 emc->refresh_poll_interval = 1000;
1980
1981 timer_setup(&emc->training, tegra210_emc_train, 0);
1982
1983 tegra210_emc_debugfs_init(emc);
1984
1985 cd = devm_thermal_of_cooling_device_register(emc->dev, np, "emc", emc,
1986 &tegra210_emc_cd_ops);
1987 if (IS_ERR(cd)) {
1988 err = PTR_ERR(cd);
1989 dev_err(emc->dev, "failed to register cooling device: %d\n",
1990 err);
1991 goto detach;
1992 }
1993
1994 return 0;
1995
1996 detach:
1997 debugfs_remove_recursive(emc->debugfs.root);
1998 tegra210_clk_emc_detach(emc->clk);
1999 release:
2000 of_reserved_mem_device_release(emc->dev);
2001
2002 return err;
2003 }
2004
2005 static int tegra210_emc_remove(struct platform_device *pdev)
2006 {
2007 struct tegra210_emc *emc = platform_get_drvdata(pdev);
2008
2009 debugfs_remove_recursive(emc->debugfs.root);
2010 tegra210_clk_emc_detach(emc->clk);
2011 of_reserved_mem_device_release(emc->dev);
2012
2013 return 0;
2014 }
2015
2016 static int __maybe_unused tegra210_emc_suspend(struct device *dev)
2017 {
2018 struct tegra210_emc *emc = dev_get_drvdata(dev);
2019 int err;
2020
2021 err = clk_rate_exclusive_get(emc->clk);
2022 if (err < 0) {
2023 dev_err(emc->dev, "failed to acquire clock: %d\n", err);
2024 return err;
2025 }
2026
2027 emc->resume_rate = clk_get_rate(emc->clk);
2028
2029 clk_set_rate(emc->clk, 204000000);
2030 tegra210_clk_emc_detach(emc->clk);
2031
2032 dev_dbg(dev, "suspending at %lu Hz\n", clk_get_rate(emc->clk));
2033
2034 return 0;
2035 }
2036
2037 static int __maybe_unused tegra210_emc_resume(struct device *dev)
2038 {
2039 struct tegra210_emc *emc = dev_get_drvdata(dev);
2040 int err;
2041
2042 err = tegra210_clk_emc_attach(emc->clk, &emc->provider);
2043 if (err < 0) {
2044 dev_err(dev, "failed to attach to EMC clock: %d\n", err);
2045 return err;
2046 }
2047
2048 clk_set_rate(emc->clk, emc->resume_rate);
2049 clk_rate_exclusive_put(emc->clk);
2050
2051 dev_dbg(dev, "resuming at %lu Hz\n", clk_get_rate(emc->clk));
2052
2053 return 0;
2054 }
2055
2056 static const struct dev_pm_ops tegra210_emc_pm_ops = {
2057 SET_SYSTEM_SLEEP_PM_OPS(tegra210_emc_suspend, tegra210_emc_resume)
2058 };
2059
2060 static const struct of_device_id tegra210_emc_of_match[] = {
2061 { .compatible = "nvidia,tegra210-emc", },
2062 { },
2063 };
2064 MODULE_DEVICE_TABLE(of, tegra210_emc_of_match);
2065
2066 static struct platform_driver tegra210_emc_driver = {
2067 .driver = {
2068 .name = "tegra210-emc",
2069 .of_match_table = tegra210_emc_of_match,
2070 .pm = &tegra210_emc_pm_ops,
2071 },
2072 .probe = tegra210_emc_probe,
2073 .remove = tegra210_emc_remove,
2074 };
2075
2076 module_platform_driver(tegra210_emc_driver);
2077
2078 MODULE_AUTHOR("Thierry Reding <treding@nvidia.com>");
2079 MODULE_AUTHOR("Joseph Lo <josephl@nvidia.com>");
2080 MODULE_DESCRIPTION("NVIDIA Tegra210 EMC driver");
2081 MODULE_LICENSE("GPL v2");
Linux with added FPC-III support