?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
mmc: host: sh_mobile_sdhi: don't populate unneeded functions
[linux/fpc-iii.git] / drivers / gpu / drm / radeon / cik.c
blob0c1b9ff433af706c15347467a756a7f71e36746a
1 /*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
27 #include "drmP.h"
28 #include "radeon.h"
29 #include "radeon_asic.h"
30 #include "radeon_audio.h"
31 #include "cikd.h"
32 #include "atom.h"
33 #include "cik_blit_shaders.h"
34 #include "radeon_ucode.h"
35 #include "clearstate_ci.h"
36 #include "radeon_kfd.h"
37
38 MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
39 MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
40 MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
41 MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
42 MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
43 MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin");
44 MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
45 MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
46 MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
47
48 MODULE_FIRMWARE("radeon/bonaire_pfp.bin");
49 MODULE_FIRMWARE("radeon/bonaire_me.bin");
50 MODULE_FIRMWARE("radeon/bonaire_ce.bin");
51 MODULE_FIRMWARE("radeon/bonaire_mec.bin");
52 MODULE_FIRMWARE("radeon/bonaire_mc.bin");
53 MODULE_FIRMWARE("radeon/bonaire_rlc.bin");
54 MODULE_FIRMWARE("radeon/bonaire_sdma.bin");
55 MODULE_FIRMWARE("radeon/bonaire_smc.bin");
56 MODULE_FIRMWARE("radeon/bonaire_k_smc.bin");
57
58 MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
59 MODULE_FIRMWARE("radeon/HAWAII_me.bin");
60 MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
61 MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
62 MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
63 MODULE_FIRMWARE("radeon/HAWAII_mc2.bin");
64 MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
65 MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
66 MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
67
68 MODULE_FIRMWARE("radeon/hawaii_pfp.bin");
69 MODULE_FIRMWARE("radeon/hawaii_me.bin");
70 MODULE_FIRMWARE("radeon/hawaii_ce.bin");
71 MODULE_FIRMWARE("radeon/hawaii_mec.bin");
72 MODULE_FIRMWARE("radeon/hawaii_mc.bin");
73 MODULE_FIRMWARE("radeon/hawaii_rlc.bin");
74 MODULE_FIRMWARE("radeon/hawaii_sdma.bin");
75 MODULE_FIRMWARE("radeon/hawaii_smc.bin");
76 MODULE_FIRMWARE("radeon/hawaii_k_smc.bin");
77
78 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
79 MODULE_FIRMWARE("radeon/KAVERI_me.bin");
80 MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
81 MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
82 MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
83 MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
84
85 MODULE_FIRMWARE("radeon/kaveri_pfp.bin");
86 MODULE_FIRMWARE("radeon/kaveri_me.bin");
87 MODULE_FIRMWARE("radeon/kaveri_ce.bin");
88 MODULE_FIRMWARE("radeon/kaveri_mec.bin");
89 MODULE_FIRMWARE("radeon/kaveri_mec2.bin");
90 MODULE_FIRMWARE("radeon/kaveri_rlc.bin");
91 MODULE_FIRMWARE("radeon/kaveri_sdma.bin");
92
93 MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
94 MODULE_FIRMWARE("radeon/KABINI_me.bin");
95 MODULE_FIRMWARE("radeon/KABINI_ce.bin");
96 MODULE_FIRMWARE("radeon/KABINI_mec.bin");
97 MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
98 MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
99
100 MODULE_FIRMWARE("radeon/kabini_pfp.bin");
101 MODULE_FIRMWARE("radeon/kabini_me.bin");
102 MODULE_FIRMWARE("radeon/kabini_ce.bin");
103 MODULE_FIRMWARE("radeon/kabini_mec.bin");
104 MODULE_FIRMWARE("radeon/kabini_rlc.bin");
105 MODULE_FIRMWARE("radeon/kabini_sdma.bin");
106
107 MODULE_FIRMWARE("radeon/MULLINS_pfp.bin");
108 MODULE_FIRMWARE("radeon/MULLINS_me.bin");
109 MODULE_FIRMWARE("radeon/MULLINS_ce.bin");
110 MODULE_FIRMWARE("radeon/MULLINS_mec.bin");
111 MODULE_FIRMWARE("radeon/MULLINS_rlc.bin");
112 MODULE_FIRMWARE("radeon/MULLINS_sdma.bin");
113
114 MODULE_FIRMWARE("radeon/mullins_pfp.bin");
115 MODULE_FIRMWARE("radeon/mullins_me.bin");
116 MODULE_FIRMWARE("radeon/mullins_ce.bin");
117 MODULE_FIRMWARE("radeon/mullins_mec.bin");
118 MODULE_FIRMWARE("radeon/mullins_rlc.bin");
119 MODULE_FIRMWARE("radeon/mullins_sdma.bin");
120
121 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
122 extern void r600_ih_ring_fini(struct radeon_device *rdev);
123 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
124 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
125 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
126 extern void sumo_rlc_fini(struct radeon_device *rdev);
127 extern int sumo_rlc_init(struct radeon_device *rdev);
128 extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
129 extern void si_rlc_reset(struct radeon_device *rdev);
130 extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
131 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
132 extern int cik_sdma_resume(struct radeon_device *rdev);
133 extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
134 extern void cik_sdma_fini(struct radeon_device *rdev);
135 extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable);
136 static void cik_rlc_stop(struct radeon_device *rdev);
137 static void cik_pcie_gen3_enable(struct radeon_device *rdev);
138 static void cik_program_aspm(struct radeon_device *rdev);
139 static void cik_init_pg(struct radeon_device *rdev);
140 static void cik_init_cg(struct radeon_device *rdev);
141 static void cik_fini_pg(struct radeon_device *rdev);
142 static void cik_fini_cg(struct radeon_device *rdev);
143 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
144 bool enable);
145
146 /**
147 * cik_get_allowed_info_register - fetch the register for the info ioctl
148 *
149 * @rdev: radeon_device pointer
150 * @reg: register offset in bytes
151 * @val: register value
152 *
153 * Returns 0 for success or -EINVAL for an invalid register
154 *
155 */
156 int cik_get_allowed_info_register(struct radeon_device *rdev,
157 u32 reg, u32 *val)
158 {
159 switch (reg) {
160 case GRBM_STATUS:
161 case GRBM_STATUS2:
162 case GRBM_STATUS_SE0:
163 case GRBM_STATUS_SE1:
164 case GRBM_STATUS_SE2:
165 case GRBM_STATUS_SE3:
166 case SRBM_STATUS:
167 case SRBM_STATUS2:
168 case (SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET):
169 case (SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET):
170 case UVD_STATUS:
171 /* TODO VCE */
172 *val = RREG32(reg);
173 return 0;
174 default:
175 return -EINVAL;
176 }
177 }
178
179 /*
180 * Indirect registers accessor
181 */
182 u32 cik_didt_rreg(struct radeon_device *rdev, u32 reg)
183 {
184 unsigned long flags;
185 u32 r;
186
187 spin_lock_irqsave(&rdev->didt_idx_lock, flags);
188 WREG32(CIK_DIDT_IND_INDEX, (reg));
189 r = RREG32(CIK_DIDT_IND_DATA);
190 spin_unlock_irqrestore(&rdev->didt_idx_lock, flags);
191 return r;
192 }
193
194 void cik_didt_wreg(struct radeon_device *rdev, u32 reg, u32 v)
195 {
196 unsigned long flags;
197
198 spin_lock_irqsave(&rdev->didt_idx_lock, flags);
199 WREG32(CIK_DIDT_IND_INDEX, (reg));
200 WREG32(CIK_DIDT_IND_DATA, (v));
201 spin_unlock_irqrestore(&rdev->didt_idx_lock, flags);
202 }
203
204 /* get temperature in millidegrees */
205 int ci_get_temp(struct radeon_device *rdev)
206 {
207 u32 temp;
208 int actual_temp = 0;
209
210 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
211 CTF_TEMP_SHIFT;
212
213 if (temp & 0x200)
214 actual_temp = 255;
215 else
216 actual_temp = temp & 0x1ff;
217
218 actual_temp = actual_temp * 1000;
219
220 return actual_temp;
221 }
222
223 /* get temperature in millidegrees */
224 int kv_get_temp(struct radeon_device *rdev)
225 {
226 u32 temp;
227 int actual_temp = 0;
228
229 temp = RREG32_SMC(0xC0300E0C);
230
231 if (temp)
232 actual_temp = (temp / 8) - 49;
233 else
234 actual_temp = 0;
235
236 actual_temp = actual_temp * 1000;
237
238 return actual_temp;
239 }
240
241 /*
242 * Indirect registers accessor
243 */
244 u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
245 {
246 unsigned long flags;
247 u32 r;
248
249 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
250 WREG32(PCIE_INDEX, reg);
251 (void)RREG32(PCIE_INDEX);
252 r = RREG32(PCIE_DATA);
253 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
254 return r;
255 }
256
257 void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
258 {
259 unsigned long flags;
260
261 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
262 WREG32(PCIE_INDEX, reg);
263 (void)RREG32(PCIE_INDEX);
264 WREG32(PCIE_DATA, v);
265 (void)RREG32(PCIE_DATA);
266 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
267 }
268
269 static const u32 spectre_rlc_save_restore_register_list[] =
270 {
271 (0x0e00 << 16) | (0xc12c >> 2),
272 0x00000000,
273 (0x0e00 << 16) | (0xc140 >> 2),
274 0x00000000,
275 (0x0e00 << 16) | (0xc150 >> 2),
276 0x00000000,
277 (0x0e00 << 16) | (0xc15c >> 2),
278 0x00000000,
279 (0x0e00 << 16) | (0xc168 >> 2),
280 0x00000000,
281 (0x0e00 << 16) | (0xc170 >> 2),
282 0x00000000,
283 (0x0e00 << 16) | (0xc178 >> 2),
284 0x00000000,
285 (0x0e00 << 16) | (0xc204 >> 2),
286 0x00000000,
287 (0x0e00 << 16) | (0xc2b4 >> 2),
288 0x00000000,
289 (0x0e00 << 16) | (0xc2b8 >> 2),
290 0x00000000,
291 (0x0e00 << 16) | (0xc2bc >> 2),
292 0x00000000,
293 (0x0e00 << 16) | (0xc2c0 >> 2),
294 0x00000000,
295 (0x0e00 << 16) | (0x8228 >> 2),
296 0x00000000,
297 (0x0e00 << 16) | (0x829c >> 2),
298 0x00000000,
299 (0x0e00 << 16) | (0x869c >> 2),
300 0x00000000,
301 (0x0600 << 16) | (0x98f4 >> 2),
302 0x00000000,
303 (0x0e00 << 16) | (0x98f8 >> 2),
304 0x00000000,
305 (0x0e00 << 16) | (0x9900 >> 2),
306 0x00000000,
307 (0x0e00 << 16) | (0xc260 >> 2),
308 0x00000000,
309 (0x0e00 << 16) | (0x90e8 >> 2),
310 0x00000000,
311 (0x0e00 << 16) | (0x3c000 >> 2),
312 0x00000000,
313 (0x0e00 << 16) | (0x3c00c >> 2),
314 0x00000000,
315 (0x0e00 << 16) | (0x8c1c >> 2),
316 0x00000000,
317 (0x0e00 << 16) | (0x9700 >> 2),
318 0x00000000,
319 (0x0e00 << 16) | (0xcd20 >> 2),
320 0x00000000,
321 (0x4e00 << 16) | (0xcd20 >> 2),
322 0x00000000,
323 (0x5e00 << 16) | (0xcd20 >> 2),
324 0x00000000,
325 (0x6e00 << 16) | (0xcd20 >> 2),
326 0x00000000,
327 (0x7e00 << 16) | (0xcd20 >> 2),
328 0x00000000,
329 (0x8e00 << 16) | (0xcd20 >> 2),
330 0x00000000,
331 (0x9e00 << 16) | (0xcd20 >> 2),
332 0x00000000,
333 (0xae00 << 16) | (0xcd20 >> 2),
334 0x00000000,
335 (0xbe00 << 16) | (0xcd20 >> 2),
336 0x00000000,
337 (0x0e00 << 16) | (0x89bc >> 2),
338 0x00000000,
339 (0x0e00 << 16) | (0x8900 >> 2),
340 0x00000000,
341 0x3,
342 (0x0e00 << 16) | (0xc130 >> 2),
343 0x00000000,
344 (0x0e00 << 16) | (0xc134 >> 2),
345 0x00000000,
346 (0x0e00 << 16) | (0xc1fc >> 2),
347 0x00000000,
348 (0x0e00 << 16) | (0xc208 >> 2),
349 0x00000000,
350 (0x0e00 << 16) | (0xc264 >> 2),
351 0x00000000,
352 (0x0e00 << 16) | (0xc268 >> 2),
353 0x00000000,
354 (0x0e00 << 16) | (0xc26c >> 2),
355 0x00000000,
356 (0x0e00 << 16) | (0xc270 >> 2),
357 0x00000000,
358 (0x0e00 << 16) | (0xc274 >> 2),
359 0x00000000,
360 (0x0e00 << 16) | (0xc278 >> 2),
361 0x00000000,
362 (0x0e00 << 16) | (0xc27c >> 2),
363 0x00000000,
364 (0x0e00 << 16) | (0xc280 >> 2),
365 0x00000000,
366 (0x0e00 << 16) | (0xc284 >> 2),
367 0x00000000,
368 (0x0e00 << 16) | (0xc288 >> 2),
369 0x00000000,
370 (0x0e00 << 16) | (0xc28c >> 2),
371 0x00000000,
372 (0x0e00 << 16) | (0xc290 >> 2),
373 0x00000000,
374 (0x0e00 << 16) | (0xc294 >> 2),
375 0x00000000,
376 (0x0e00 << 16) | (0xc298 >> 2),
377 0x00000000,
378 (0x0e00 << 16) | (0xc29c >> 2),
379 0x00000000,
380 (0x0e00 << 16) | (0xc2a0 >> 2),
381 0x00000000,
382 (0x0e00 << 16) | (0xc2a4 >> 2),
383 0x00000000,
384 (0x0e00 << 16) | (0xc2a8 >> 2),
385 0x00000000,
386 (0x0e00 << 16) | (0xc2ac >> 2),
387 0x00000000,
388 (0x0e00 << 16) | (0xc2b0 >> 2),
389 0x00000000,
390 (0x0e00 << 16) | (0x301d0 >> 2),
391 0x00000000,
392 (0x0e00 << 16) | (0x30238 >> 2),
393 0x00000000,
394 (0x0e00 << 16) | (0x30250 >> 2),
395 0x00000000,
396 (0x0e00 << 16) | (0x30254 >> 2),
397 0x00000000,
398 (0x0e00 << 16) | (0x30258 >> 2),
399 0x00000000,
400 (0x0e00 << 16) | (0x3025c >> 2),
401 0x00000000,
402 (0x4e00 << 16) | (0xc900 >> 2),
403 0x00000000,
404 (0x5e00 << 16) | (0xc900 >> 2),
405 0x00000000,
406 (0x6e00 << 16) | (0xc900 >> 2),
407 0x00000000,
408 (0x7e00 << 16) | (0xc900 >> 2),
409 0x00000000,
410 (0x8e00 << 16) | (0xc900 >> 2),
411 0x00000000,
412 (0x9e00 << 16) | (0xc900 >> 2),
413 0x00000000,
414 (0xae00 << 16) | (0xc900 >> 2),
415 0x00000000,
416 (0xbe00 << 16) | (0xc900 >> 2),
417 0x00000000,
418 (0x4e00 << 16) | (0xc904 >> 2),
419 0x00000000,
420 (0x5e00 << 16) | (0xc904 >> 2),
421 0x00000000,
422 (0x6e00 << 16) | (0xc904 >> 2),
423 0x00000000,
424 (0x7e00 << 16) | (0xc904 >> 2),
425 0x00000000,
426 (0x8e00 << 16) | (0xc904 >> 2),
427 0x00000000,
428 (0x9e00 << 16) | (0xc904 >> 2),
429 0x00000000,
430 (0xae00 << 16) | (0xc904 >> 2),
431 0x00000000,
432 (0xbe00 << 16) | (0xc904 >> 2),
433 0x00000000,
434 (0x4e00 << 16) | (0xc908 >> 2),
435 0x00000000,
436 (0x5e00 << 16) | (0xc908 >> 2),
437 0x00000000,
438 (0x6e00 << 16) | (0xc908 >> 2),
439 0x00000000,
440 (0x7e00 << 16) | (0xc908 >> 2),
441 0x00000000,
442 (0x8e00 << 16) | (0xc908 >> 2),
443 0x00000000,
444 (0x9e00 << 16) | (0xc908 >> 2),
445 0x00000000,
446 (0xae00 << 16) | (0xc908 >> 2),
447 0x00000000,
448 (0xbe00 << 16) | (0xc908 >> 2),
449 0x00000000,
450 (0x4e00 << 16) | (0xc90c >> 2),
451 0x00000000,
452 (0x5e00 << 16) | (0xc90c >> 2),
453 0x00000000,
454 (0x6e00 << 16) | (0xc90c >> 2),
455 0x00000000,
456 (0x7e00 << 16) | (0xc90c >> 2),
457 0x00000000,
458 (0x8e00 << 16) | (0xc90c >> 2),
459 0x00000000,
460 (0x9e00 << 16) | (0xc90c >> 2),
461 0x00000000,
462 (0xae00 << 16) | (0xc90c >> 2),
463 0x00000000,
464 (0xbe00 << 16) | (0xc90c >> 2),
465 0x00000000,
466 (0x4e00 << 16) | (0xc910 >> 2),
467 0x00000000,
468 (0x5e00 << 16) | (0xc910 >> 2),
469 0x00000000,
470 (0x6e00 << 16) | (0xc910 >> 2),
471 0x00000000,
472 (0x7e00 << 16) | (0xc910 >> 2),
473 0x00000000,
474 (0x8e00 << 16) | (0xc910 >> 2),
475 0x00000000,
476 (0x9e00 << 16) | (0xc910 >> 2),
477 0x00000000,
478 (0xae00 << 16) | (0xc910 >> 2),
479 0x00000000,
480 (0xbe00 << 16) | (0xc910 >> 2),
481 0x00000000,
482 (0x0e00 << 16) | (0xc99c >> 2),
483 0x00000000,
484 (0x0e00 << 16) | (0x9834 >> 2),
485 0x00000000,
486 (0x0000 << 16) | (0x30f00 >> 2),
487 0x00000000,
488 (0x0001 << 16) | (0x30f00 >> 2),
489 0x00000000,
490 (0x0000 << 16) | (0x30f04 >> 2),
491 0x00000000,
492 (0x0001 << 16) | (0x30f04 >> 2),
493 0x00000000,
494 (0x0000 << 16) | (0x30f08 >> 2),
495 0x00000000,
496 (0x0001 << 16) | (0x30f08 >> 2),
497 0x00000000,
498 (0x0000 << 16) | (0x30f0c >> 2),
499 0x00000000,
500 (0x0001 << 16) | (0x30f0c >> 2),
501 0x00000000,
502 (0x0600 << 16) | (0x9b7c >> 2),
503 0x00000000,
504 (0x0e00 << 16) | (0x8a14 >> 2),
505 0x00000000,
506 (0x0e00 << 16) | (0x8a18 >> 2),
507 0x00000000,
508 (0x0600 << 16) | (0x30a00 >> 2),
509 0x00000000,
510 (0x0e00 << 16) | (0x8bf0 >> 2),
511 0x00000000,
512 (0x0e00 << 16) | (0x8bcc >> 2),
513 0x00000000,
514 (0x0e00 << 16) | (0x8b24 >> 2),
515 0x00000000,
516 (0x0e00 << 16) | (0x30a04 >> 2),
517 0x00000000,
518 (0x0600 << 16) | (0x30a10 >> 2),
519 0x00000000,
520 (0x0600 << 16) | (0x30a14 >> 2),
521 0x00000000,
522 (0x0600 << 16) | (0x30a18 >> 2),
523 0x00000000,
524 (0x0600 << 16) | (0x30a2c >> 2),
525 0x00000000,
526 (0x0e00 << 16) | (0xc700 >> 2),
527 0x00000000,
528 (0x0e00 << 16) | (0xc704 >> 2),
529 0x00000000,
530 (0x0e00 << 16) | (0xc708 >> 2),
531 0x00000000,
532 (0x0e00 << 16) | (0xc768 >> 2),
533 0x00000000,
534 (0x0400 << 16) | (0xc770 >> 2),
535 0x00000000,
536 (0x0400 << 16) | (0xc774 >> 2),
537 0x00000000,
538 (0x0400 << 16) | (0xc778 >> 2),
539 0x00000000,
540 (0x0400 << 16) | (0xc77c >> 2),
541 0x00000000,
542 (0x0400 << 16) | (0xc780 >> 2),
543 0x00000000,
544 (0x0400 << 16) | (0xc784 >> 2),
545 0x00000000,
546 (0x0400 << 16) | (0xc788 >> 2),
547 0x00000000,
548 (0x0400 << 16) | (0xc78c >> 2),
549 0x00000000,
550 (0x0400 << 16) | (0xc798 >> 2),
551 0x00000000,
552 (0x0400 << 16) | (0xc79c >> 2),
553 0x00000000,
554 (0x0400 << 16) | (0xc7a0 >> 2),
555 0x00000000,
556 (0x0400 << 16) | (0xc7a4 >> 2),
557 0x00000000,
558 (0x0400 << 16) | (0xc7a8 >> 2),
559 0x00000000,
560 (0x0400 << 16) | (0xc7ac >> 2),
561 0x00000000,
562 (0x0400 << 16) | (0xc7b0 >> 2),
563 0x00000000,
564 (0x0400 << 16) | (0xc7b4 >> 2),
565 0x00000000,
566 (0x0e00 << 16) | (0x9100 >> 2),
567 0x00000000,
568 (0x0e00 << 16) | (0x3c010 >> 2),
569 0x00000000,
570 (0x0e00 << 16) | (0x92a8 >> 2),
571 0x00000000,
572 (0x0e00 << 16) | (0x92ac >> 2),
573 0x00000000,
574 (0x0e00 << 16) | (0x92b4 >> 2),
575 0x00000000,
576 (0x0e00 << 16) | (0x92b8 >> 2),
577 0x00000000,
578 (0x0e00 << 16) | (0x92bc >> 2),
579 0x00000000,
580 (0x0e00 << 16) | (0x92c0 >> 2),
581 0x00000000,
582 (0x0e00 << 16) | (0x92c4 >> 2),
583 0x00000000,
584 (0x0e00 << 16) | (0x92c8 >> 2),
585 0x00000000,
586 (0x0e00 << 16) | (0x92cc >> 2),
587 0x00000000,
588 (0x0e00 << 16) | (0x92d0 >> 2),
589 0x00000000,
590 (0x0e00 << 16) | (0x8c00 >> 2),
591 0x00000000,
592 (0x0e00 << 16) | (0x8c04 >> 2),
593 0x00000000,
594 (0x0e00 << 16) | (0x8c20 >> 2),
595 0x00000000,
596 (0x0e00 << 16) | (0x8c38 >> 2),
597 0x00000000,
598 (0x0e00 << 16) | (0x8c3c >> 2),
599 0x00000000,
600 (0x0e00 << 16) | (0xae00 >> 2),
601 0x00000000,
602 (0x0e00 << 16) | (0x9604 >> 2),
603 0x00000000,
604 (0x0e00 << 16) | (0xac08 >> 2),
605 0x00000000,
606 (0x0e00 << 16) | (0xac0c >> 2),
607 0x00000000,
608 (0x0e00 << 16) | (0xac10 >> 2),
609 0x00000000,
610 (0x0e00 << 16) | (0xac14 >> 2),
611 0x00000000,
612 (0x0e00 << 16) | (0xac58 >> 2),
613 0x00000000,
614 (0x0e00 << 16) | (0xac68 >> 2),
615 0x00000000,
616 (0x0e00 << 16) | (0xac6c >> 2),
617 0x00000000,
618 (0x0e00 << 16) | (0xac70 >> 2),
619 0x00000000,
620 (0x0e00 << 16) | (0xac74 >> 2),
621 0x00000000,
622 (0x0e00 << 16) | (0xac78 >> 2),
623 0x00000000,
624 (0x0e00 << 16) | (0xac7c >> 2),
625 0x00000000,
626 (0x0e00 << 16) | (0xac80 >> 2),
627 0x00000000,
628 (0x0e00 << 16) | (0xac84 >> 2),
629 0x00000000,
630 (0x0e00 << 16) | (0xac88 >> 2),
631 0x00000000,
632 (0x0e00 << 16) | (0xac8c >> 2),
633 0x00000000,
634 (0x0e00 << 16) | (0x970c >> 2),
635 0x00000000,
636 (0x0e00 << 16) | (0x9714 >> 2),
637 0x00000000,
638 (0x0e00 << 16) | (0x9718 >> 2),
639 0x00000000,
640 (0x0e00 << 16) | (0x971c >> 2),
641 0x00000000,
642 (0x0e00 << 16) | (0x31068 >> 2),
643 0x00000000,
644 (0x4e00 << 16) | (0x31068 >> 2),
645 0x00000000,
646 (0x5e00 << 16) | (0x31068 >> 2),
647 0x00000000,
648 (0x6e00 << 16) | (0x31068 >> 2),
649 0x00000000,
650 (0x7e00 << 16) | (0x31068 >> 2),
651 0x00000000,
652 (0x8e00 << 16) | (0x31068 >> 2),
653 0x00000000,
654 (0x9e00 << 16) | (0x31068 >> 2),
655 0x00000000,
656 (0xae00 << 16) | (0x31068 >> 2),
657 0x00000000,
658 (0xbe00 << 16) | (0x31068 >> 2),
659 0x00000000,
660 (0x0e00 << 16) | (0xcd10 >> 2),
661 0x00000000,
662 (0x0e00 << 16) | (0xcd14 >> 2),
663 0x00000000,
664 (0x0e00 << 16) | (0x88b0 >> 2),
665 0x00000000,
666 (0x0e00 << 16) | (0x88b4 >> 2),
667 0x00000000,
668 (0x0e00 << 16) | (0x88b8 >> 2),
669 0x00000000,
670 (0x0e00 << 16) | (0x88bc >> 2),
671 0x00000000,
672 (0x0400 << 16) | (0x89c0 >> 2),
673 0x00000000,
674 (0x0e00 << 16) | (0x88c4 >> 2),
675 0x00000000,
676 (0x0e00 << 16) | (0x88c8 >> 2),
677 0x00000000,
678 (0x0e00 << 16) | (0x88d0 >> 2),
679 0x00000000,
680 (0x0e00 << 16) | (0x88d4 >> 2),
681 0x00000000,
682 (0x0e00 << 16) | (0x88d8 >> 2),
683 0x00000000,
684 (0x0e00 << 16) | (0x8980 >> 2),
685 0x00000000,
686 (0x0e00 << 16) | (0x30938 >> 2),
687 0x00000000,
688 (0x0e00 << 16) | (0x3093c >> 2),
689 0x00000000,
690 (0x0e00 << 16) | (0x30940 >> 2),
691 0x00000000,
692 (0x0e00 << 16) | (0x89a0 >> 2),
693 0x00000000,
694 (0x0e00 << 16) | (0x30900 >> 2),
695 0x00000000,
696 (0x0e00 << 16) | (0x30904 >> 2),
697 0x00000000,
698 (0x0e00 << 16) | (0x89b4 >> 2),
699 0x00000000,
700 (0x0e00 << 16) | (0x3c210 >> 2),
701 0x00000000,
702 (0x0e00 << 16) | (0x3c214 >> 2),
703 0x00000000,
704 (0x0e00 << 16) | (0x3c218 >> 2),
705 0x00000000,
706 (0x0e00 << 16) | (0x8904 >> 2),
707 0x00000000,
708 0x5,
709 (0x0e00 << 16) | (0x8c28 >> 2),
710 (0x0e00 << 16) | (0x8c2c >> 2),
711 (0x0e00 << 16) | (0x8c30 >> 2),
712 (0x0e00 << 16) | (0x8c34 >> 2),
713 (0x0e00 << 16) | (0x9600 >> 2),
714 };
715
716 static const u32 kalindi_rlc_save_restore_register_list[] =
717 {
718 (0x0e00 << 16) | (0xc12c >> 2),
719 0x00000000,
720 (0x0e00 << 16) | (0xc140 >> 2),
721 0x00000000,
722 (0x0e00 << 16) | (0xc150 >> 2),
723 0x00000000,
724 (0x0e00 << 16) | (0xc15c >> 2),
725 0x00000000,
726 (0x0e00 << 16) | (0xc168 >> 2),
727 0x00000000,
728 (0x0e00 << 16) | (0xc170 >> 2),
729 0x00000000,
730 (0x0e00 << 16) | (0xc204 >> 2),
731 0x00000000,
732 (0x0e00 << 16) | (0xc2b4 >> 2),
733 0x00000000,
734 (0x0e00 << 16) | (0xc2b8 >> 2),
735 0x00000000,
736 (0x0e00 << 16) | (0xc2bc >> 2),
737 0x00000000,
738 (0x0e00 << 16) | (0xc2c0 >> 2),
739 0x00000000,
740 (0x0e00 << 16) | (0x8228 >> 2),
741 0x00000000,
742 (0x0e00 << 16) | (0x829c >> 2),
743 0x00000000,
744 (0x0e00 << 16) | (0x869c >> 2),
745 0x00000000,
746 (0x0600 << 16) | (0x98f4 >> 2),
747 0x00000000,
748 (0x0e00 << 16) | (0x98f8 >> 2),
749 0x00000000,
750 (0x0e00 << 16) | (0x9900 >> 2),
751 0x00000000,
752 (0x0e00 << 16) | (0xc260 >> 2),
753 0x00000000,
754 (0x0e00 << 16) | (0x90e8 >> 2),
755 0x00000000,
756 (0x0e00 << 16) | (0x3c000 >> 2),
757 0x00000000,
758 (0x0e00 << 16) | (0x3c00c >> 2),
759 0x00000000,
760 (0x0e00 << 16) | (0x8c1c >> 2),
761 0x00000000,
762 (0x0e00 << 16) | (0x9700 >> 2),
763 0x00000000,
764 (0x0e00 << 16) | (0xcd20 >> 2),
765 0x00000000,
766 (0x4e00 << 16) | (0xcd20 >> 2),
767 0x00000000,
768 (0x5e00 << 16) | (0xcd20 >> 2),
769 0x00000000,
770 (0x6e00 << 16) | (0xcd20 >> 2),
771 0x00000000,
772 (0x7e00 << 16) | (0xcd20 >> 2),
773 0x00000000,
774 (0x0e00 << 16) | (0x89bc >> 2),
775 0x00000000,
776 (0x0e00 << 16) | (0x8900 >> 2),
777 0x00000000,
778 0x3,
779 (0x0e00 << 16) | (0xc130 >> 2),
780 0x00000000,
781 (0x0e00 << 16) | (0xc134 >> 2),
782 0x00000000,
783 (0x0e00 << 16) | (0xc1fc >> 2),
784 0x00000000,
785 (0x0e00 << 16) | (0xc208 >> 2),
786 0x00000000,
787 (0x0e00 << 16) | (0xc264 >> 2),
788 0x00000000,
789 (0x0e00 << 16) | (0xc268 >> 2),
790 0x00000000,
791 (0x0e00 << 16) | (0xc26c >> 2),
792 0x00000000,
793 (0x0e00 << 16) | (0xc270 >> 2),
794 0x00000000,
795 (0x0e00 << 16) | (0xc274 >> 2),
796 0x00000000,
797 (0x0e00 << 16) | (0xc28c >> 2),
798 0x00000000,
799 (0x0e00 << 16) | (0xc290 >> 2),
800 0x00000000,
801 (0x0e00 << 16) | (0xc294 >> 2),
802 0x00000000,
803 (0x0e00 << 16) | (0xc298 >> 2),
804 0x00000000,
805 (0x0e00 << 16) | (0xc2a0 >> 2),
806 0x00000000,
807 (0x0e00 << 16) | (0xc2a4 >> 2),
808 0x00000000,
809 (0x0e00 << 16) | (0xc2a8 >> 2),
810 0x00000000,
811 (0x0e00 << 16) | (0xc2ac >> 2),
812 0x00000000,
813 (0x0e00 << 16) | (0x301d0 >> 2),
814 0x00000000,
815 (0x0e00 << 16) | (0x30238 >> 2),
816 0x00000000,
817 (0x0e00 << 16) | (0x30250 >> 2),
818 0x00000000,
819 (0x0e00 << 16) | (0x30254 >> 2),
820 0x00000000,
821 (0x0e00 << 16) | (0x30258 >> 2),
822 0x00000000,
823 (0x0e00 << 16) | (0x3025c >> 2),
824 0x00000000,
825 (0x4e00 << 16) | (0xc900 >> 2),
826 0x00000000,
827 (0x5e00 << 16) | (0xc900 >> 2),
828 0x00000000,
829 (0x6e00 << 16) | (0xc900 >> 2),
830 0x00000000,
831 (0x7e00 << 16) | (0xc900 >> 2),
832 0x00000000,
833 (0x4e00 << 16) | (0xc904 >> 2),
834 0x00000000,
835 (0x5e00 << 16) | (0xc904 >> 2),
836 0x00000000,
837 (0x6e00 << 16) | (0xc904 >> 2),
838 0x00000000,
839 (0x7e00 << 16) | (0xc904 >> 2),
840 0x00000000,
841 (0x4e00 << 16) | (0xc908 >> 2),
842 0x00000000,
843 (0x5e00 << 16) | (0xc908 >> 2),
844 0x00000000,
845 (0x6e00 << 16) | (0xc908 >> 2),
846 0x00000000,
847 (0x7e00 << 16) | (0xc908 >> 2),
848 0x00000000,
849 (0x4e00 << 16) | (0xc90c >> 2),
850 0x00000000,
851 (0x5e00 << 16) | (0xc90c >> 2),
852 0x00000000,
853 (0x6e00 << 16) | (0xc90c >> 2),
854 0x00000000,
855 (0x7e00 << 16) | (0xc90c >> 2),
856 0x00000000,
857 (0x4e00 << 16) | (0xc910 >> 2),
858 0x00000000,
859 (0x5e00 << 16) | (0xc910 >> 2),
860 0x00000000,
861 (0x6e00 << 16) | (0xc910 >> 2),
862 0x00000000,
863 (0x7e00 << 16) | (0xc910 >> 2),
864 0x00000000,
865 (0x0e00 << 16) | (0xc99c >> 2),
866 0x00000000,
867 (0x0e00 << 16) | (0x9834 >> 2),
868 0x00000000,
869 (0x0000 << 16) | (0x30f00 >> 2),
870 0x00000000,
871 (0x0000 << 16) | (0x30f04 >> 2),
872 0x00000000,
873 (0x0000 << 16) | (0x30f08 >> 2),
874 0x00000000,
875 (0x0000 << 16) | (0x30f0c >> 2),
876 0x00000000,
877 (0x0600 << 16) | (0x9b7c >> 2),
878 0x00000000,
879 (0x0e00 << 16) | (0x8a14 >> 2),
880 0x00000000,
881 (0x0e00 << 16) | (0x8a18 >> 2),
882 0x00000000,
883 (0x0600 << 16) | (0x30a00 >> 2),
884 0x00000000,
885 (0x0e00 << 16) | (0x8bf0 >> 2),
886 0x00000000,
887 (0x0e00 << 16) | (0x8bcc >> 2),
888 0x00000000,
889 (0x0e00 << 16) | (0x8b24 >> 2),
890 0x00000000,
891 (0x0e00 << 16) | (0x30a04 >> 2),
892 0x00000000,
893 (0x0600 << 16) | (0x30a10 >> 2),
894 0x00000000,
895 (0x0600 << 16) | (0x30a14 >> 2),
896 0x00000000,
897 (0x0600 << 16) | (0x30a18 >> 2),
898 0x00000000,
899 (0x0600 << 16) | (0x30a2c >> 2),
900 0x00000000,
901 (0x0e00 << 16) | (0xc700 >> 2),
902 0x00000000,
903 (0x0e00 << 16) | (0xc704 >> 2),
904 0x00000000,
905 (0x0e00 << 16) | (0xc708 >> 2),
906 0x00000000,
907 (0x0e00 << 16) | (0xc768 >> 2),
908 0x00000000,
909 (0x0400 << 16) | (0xc770 >> 2),
910 0x00000000,
911 (0x0400 << 16) | (0xc774 >> 2),
912 0x00000000,
913 (0x0400 << 16) | (0xc798 >> 2),
914 0x00000000,
915 (0x0400 << 16) | (0xc79c >> 2),
916 0x00000000,
917 (0x0e00 << 16) | (0x9100 >> 2),
918 0x00000000,
919 (0x0e00 << 16) | (0x3c010 >> 2),
920 0x00000000,
921 (0x0e00 << 16) | (0x8c00 >> 2),
922 0x00000000,
923 (0x0e00 << 16) | (0x8c04 >> 2),
924 0x00000000,
925 (0x0e00 << 16) | (0x8c20 >> 2),
926 0x00000000,
927 (0x0e00 << 16) | (0x8c38 >> 2),
928 0x00000000,
929 (0x0e00 << 16) | (0x8c3c >> 2),
930 0x00000000,
931 (0x0e00 << 16) | (0xae00 >> 2),
932 0x00000000,
933 (0x0e00 << 16) | (0x9604 >> 2),
934 0x00000000,
935 (0x0e00 << 16) | (0xac08 >> 2),
936 0x00000000,
937 (0x0e00 << 16) | (0xac0c >> 2),
938 0x00000000,
939 (0x0e00 << 16) | (0xac10 >> 2),
940 0x00000000,
941 (0x0e00 << 16) | (0xac14 >> 2),
942 0x00000000,
943 (0x0e00 << 16) | (0xac58 >> 2),
944 0x00000000,
945 (0x0e00 << 16) | (0xac68 >> 2),
946 0x00000000,
947 (0x0e00 << 16) | (0xac6c >> 2),
948 0x00000000,
949 (0x0e00 << 16) | (0xac70 >> 2),
950 0x00000000,
951 (0x0e00 << 16) | (0xac74 >> 2),
952 0x00000000,
953 (0x0e00 << 16) | (0xac78 >> 2),
954 0x00000000,
955 (0x0e00 << 16) | (0xac7c >> 2),
956 0x00000000,
957 (0x0e00 << 16) | (0xac80 >> 2),
958 0x00000000,
959 (0x0e00 << 16) | (0xac84 >> 2),
960 0x00000000,
961 (0x0e00 << 16) | (0xac88 >> 2),
962 0x00000000,
963 (0x0e00 << 16) | (0xac8c >> 2),
964 0x00000000,
965 (0x0e00 << 16) | (0x970c >> 2),
966 0x00000000,
967 (0x0e00 << 16) | (0x9714 >> 2),
968 0x00000000,
969 (0x0e00 << 16) | (0x9718 >> 2),
970 0x00000000,
971 (0x0e00 << 16) | (0x971c >> 2),
972 0x00000000,
973 (0x0e00 << 16) | (0x31068 >> 2),
974 0x00000000,
975 (0x4e00 << 16) | (0x31068 >> 2),
976 0x00000000,
977 (0x5e00 << 16) | (0x31068 >> 2),
978 0x00000000,
979 (0x6e00 << 16) | (0x31068 >> 2),
980 0x00000000,
981 (0x7e00 << 16) | (0x31068 >> 2),
982 0x00000000,
983 (0x0e00 << 16) | (0xcd10 >> 2),
984 0x00000000,
985 (0x0e00 << 16) | (0xcd14 >> 2),
986 0x00000000,
987 (0x0e00 << 16) | (0x88b0 >> 2),
988 0x00000000,
989 (0x0e00 << 16) | (0x88b4 >> 2),
990 0x00000000,
991 (0x0e00 << 16) | (0x88b8 >> 2),
992 0x00000000,
993 (0x0e00 << 16) | (0x88bc >> 2),
994 0x00000000,
995 (0x0400 << 16) | (0x89c0 >> 2),
996 0x00000000,
997 (0x0e00 << 16) | (0x88c4 >> 2),
998 0x00000000,
999 (0x0e00 << 16) | (0x88c8 >> 2),
1000 0x00000000,
1001 (0x0e00 << 16) | (0x88d0 >> 2),
1002 0x00000000,
1003 (0x0e00 << 16) | (0x88d4 >> 2),
1004 0x00000000,
1005 (0x0e00 << 16) | (0x88d8 >> 2),
1006 0x00000000,
1007 (0x0e00 << 16) | (0x8980 >> 2),
1008 0x00000000,
1009 (0x0e00 << 16) | (0x30938 >> 2),
1010 0x00000000,
1011 (0x0e00 << 16) | (0x3093c >> 2),
1012 0x00000000,
1013 (0x0e00 << 16) | (0x30940 >> 2),
1014 0x00000000,
1015 (0x0e00 << 16) | (0x89a0 >> 2),
1016 0x00000000,
1017 (0x0e00 << 16) | (0x30900 >> 2),
1018 0x00000000,
1019 (0x0e00 << 16) | (0x30904 >> 2),
1020 0x00000000,
1021 (0x0e00 << 16) | (0x89b4 >> 2),
1022 0x00000000,
1023 (0x0e00 << 16) | (0x3e1fc >> 2),
1024 0x00000000,
1025 (0x0e00 << 16) | (0x3c210 >> 2),
1026 0x00000000,
1027 (0x0e00 << 16) | (0x3c214 >> 2),
1028 0x00000000,
1029 (0x0e00 << 16) | (0x3c218 >> 2),
1030 0x00000000,
1031 (0x0e00 << 16) | (0x8904 >> 2),
1032 0x00000000,
1033 0x5,
1034 (0x0e00 << 16) | (0x8c28 >> 2),
1035 (0x0e00 << 16) | (0x8c2c >> 2),
1036 (0x0e00 << 16) | (0x8c30 >> 2),
1037 (0x0e00 << 16) | (0x8c34 >> 2),
1038 (0x0e00 << 16) | (0x9600 >> 2),
1039 };
1040
1041 static const u32 bonaire_golden_spm_registers[] =
1042 {
1043 0x30800, 0xe0ffffff, 0xe0000000
1044 };
1045
1046 static const u32 bonaire_golden_common_registers[] =
1047 {
1048 0xc770, 0xffffffff, 0x00000800,
1049 0xc774, 0xffffffff, 0x00000800,
1050 0xc798, 0xffffffff, 0x00007fbf,
1051 0xc79c, 0xffffffff, 0x00007faf
1052 };
1053
1054 static const u32 bonaire_golden_registers[] =
1055 {
1056 0x3354, 0x00000333, 0x00000333,
1057 0x3350, 0x000c0fc0, 0x00040200,
1058 0x9a10, 0x00010000, 0x00058208,
1059 0x3c000, 0xffff1fff, 0x00140000,
1060 0x3c200, 0xfdfc0fff, 0x00000100,
1061 0x3c234, 0x40000000, 0x40000200,
1062 0x9830, 0xffffffff, 0x00000000,
1063 0x9834, 0xf00fffff, 0x00000400,
1064 0x9838, 0x0002021c, 0x00020200,
1065 0xc78, 0x00000080, 0x00000000,
1066 0x5bb0, 0x000000f0, 0x00000070,
1067 0x5bc0, 0xf0311fff, 0x80300000,
1068 0x98f8, 0x73773777, 0x12010001,
1069 0x350c, 0x00810000, 0x408af000,
1070 0x7030, 0x31000111, 0x00000011,
1071 0x2f48, 0x73773777, 0x12010001,
1072 0x220c, 0x00007fb6, 0x0021a1b1,
1073 0x2210, 0x00007fb6, 0x002021b1,
1074 0x2180, 0x00007fb6, 0x00002191,
1075 0x2218, 0x00007fb6, 0x002121b1,
1076 0x221c, 0x00007fb6, 0x002021b1,
1077 0x21dc, 0x00007fb6, 0x00002191,
1078 0x21e0, 0x00007fb6, 0x00002191,
1079 0x3628, 0x0000003f, 0x0000000a,
1080 0x362c, 0x0000003f, 0x0000000a,
1081 0x2ae4, 0x00073ffe, 0x000022a2,
1082 0x240c, 0x000007ff, 0x00000000,
1083 0x8a14, 0xf000003f, 0x00000007,
1084 0x8bf0, 0x00002001, 0x00000001,
1085 0x8b24, 0xffffffff, 0x00ffffff,
1086 0x30a04, 0x0000ff0f, 0x00000000,
1087 0x28a4c, 0x07ffffff, 0x06000000,
1088 0x4d8, 0x00000fff, 0x00000100,
1089 0x3e78, 0x00000001, 0x00000002,
1090 0x9100, 0x03000000, 0x0362c688,
1091 0x8c00, 0x000000ff, 0x00000001,
1092 0xe40, 0x00001fff, 0x00001fff,
1093 0x9060, 0x0000007f, 0x00000020,
1094 0x9508, 0x00010000, 0x00010000,
1095 0xac14, 0x000003ff, 0x000000f3,
1096 0xac0c, 0xffffffff, 0x00001032
1097 };
1098
1099 static const u32 bonaire_mgcg_cgcg_init[] =
1100 {
1101 0xc420, 0xffffffff, 0xfffffffc,
1102 0x30800, 0xffffffff, 0xe0000000,
1103 0x3c2a0, 0xffffffff, 0x00000100,
1104 0x3c208, 0xffffffff, 0x00000100,
1105 0x3c2c0, 0xffffffff, 0xc0000100,
1106 0x3c2c8, 0xffffffff, 0xc0000100,
1107 0x3c2c4, 0xffffffff, 0xc0000100,
1108 0x55e4, 0xffffffff, 0x00600100,
1109 0x3c280, 0xffffffff, 0x00000100,
1110 0x3c214, 0xffffffff, 0x06000100,
1111 0x3c220, 0xffffffff, 0x00000100,
1112 0x3c218, 0xffffffff, 0x06000100,
1113 0x3c204, 0xffffffff, 0x00000100,
1114 0x3c2e0, 0xffffffff, 0x00000100,
1115 0x3c224, 0xffffffff, 0x00000100,
1116 0x3c200, 0xffffffff, 0x00000100,
1117 0x3c230, 0xffffffff, 0x00000100,
1118 0x3c234, 0xffffffff, 0x00000100,
1119 0x3c250, 0xffffffff, 0x00000100,
1120 0x3c254, 0xffffffff, 0x00000100,
1121 0x3c258, 0xffffffff, 0x00000100,
1122 0x3c25c, 0xffffffff, 0x00000100,
1123 0x3c260, 0xffffffff, 0x00000100,
1124 0x3c27c, 0xffffffff, 0x00000100,
1125 0x3c278, 0xffffffff, 0x00000100,
1126 0x3c210, 0xffffffff, 0x06000100,
1127 0x3c290, 0xffffffff, 0x00000100,
1128 0x3c274, 0xffffffff, 0x00000100,
1129 0x3c2b4, 0xffffffff, 0x00000100,
1130 0x3c2b0, 0xffffffff, 0x00000100,
1131 0x3c270, 0xffffffff, 0x00000100,
1132 0x30800, 0xffffffff, 0xe0000000,
1133 0x3c020, 0xffffffff, 0x00010000,
1134 0x3c024, 0xffffffff, 0x00030002,
1135 0x3c028, 0xffffffff, 0x00040007,
1136 0x3c02c, 0xffffffff, 0x00060005,
1137 0x3c030, 0xffffffff, 0x00090008,
1138 0x3c034, 0xffffffff, 0x00010000,
1139 0x3c038, 0xffffffff, 0x00030002,
1140 0x3c03c, 0xffffffff, 0x00040007,
1141 0x3c040, 0xffffffff, 0x00060005,
1142 0x3c044, 0xffffffff, 0x00090008,
1143 0x3c048, 0xffffffff, 0x00010000,
1144 0x3c04c, 0xffffffff, 0x00030002,
1145 0x3c050, 0xffffffff, 0x00040007,
1146 0x3c054, 0xffffffff, 0x00060005,
1147 0x3c058, 0xffffffff, 0x00090008,
1148 0x3c05c, 0xffffffff, 0x00010000,
1149 0x3c060, 0xffffffff, 0x00030002,
1150 0x3c064, 0xffffffff, 0x00040007,
1151 0x3c068, 0xffffffff, 0x00060005,
1152 0x3c06c, 0xffffffff, 0x00090008,
1153 0x3c070, 0xffffffff, 0x00010000,
1154 0x3c074, 0xffffffff, 0x00030002,
1155 0x3c078, 0xffffffff, 0x00040007,
1156 0x3c07c, 0xffffffff, 0x00060005,
1157 0x3c080, 0xffffffff, 0x00090008,
1158 0x3c084, 0xffffffff, 0x00010000,
1159 0x3c088, 0xffffffff, 0x00030002,
1160 0x3c08c, 0xffffffff, 0x00040007,
1161 0x3c090, 0xffffffff, 0x00060005,
1162 0x3c094, 0xffffffff, 0x00090008,
1163 0x3c098, 0xffffffff, 0x00010000,
1164 0x3c09c, 0xffffffff, 0x00030002,
1165 0x3c0a0, 0xffffffff, 0x00040007,
1166 0x3c0a4, 0xffffffff, 0x00060005,
1167 0x3c0a8, 0xffffffff, 0x00090008,
1168 0x3c000, 0xffffffff, 0x96e00200,
1169 0x8708, 0xffffffff, 0x00900100,
1170 0xc424, 0xffffffff, 0x0020003f,
1171 0x38, 0xffffffff, 0x0140001c,
1172 0x3c, 0x000f0000, 0x000f0000,
1173 0x220, 0xffffffff, 0xC060000C,
1174 0x224, 0xc0000fff, 0x00000100,
1175 0xf90, 0xffffffff, 0x00000100,
1176 0xf98, 0x00000101, 0x00000000,
1177 0x20a8, 0xffffffff, 0x00000104,
1178 0x55e4, 0xff000fff, 0x00000100,
1179 0x30cc, 0xc0000fff, 0x00000104,
1180 0xc1e4, 0x00000001, 0x00000001,
1181 0xd00c, 0xff000ff0, 0x00000100,
1182 0xd80c, 0xff000ff0, 0x00000100
1183 };
1184
1185 static const u32 spectre_golden_spm_registers[] =
1186 {
1187 0x30800, 0xe0ffffff, 0xe0000000
1188 };
1189
1190 static const u32 spectre_golden_common_registers[] =
1191 {
1192 0xc770, 0xffffffff, 0x00000800,
1193 0xc774, 0xffffffff, 0x00000800,
1194 0xc798, 0xffffffff, 0x00007fbf,
1195 0xc79c, 0xffffffff, 0x00007faf
1196 };
1197
1198 static const u32 spectre_golden_registers[] =
1199 {
1200 0x3c000, 0xffff1fff, 0x96940200,
1201 0x3c00c, 0xffff0001, 0xff000000,
1202 0x3c200, 0xfffc0fff, 0x00000100,
1203 0x6ed8, 0x00010101, 0x00010000,
1204 0x9834, 0xf00fffff, 0x00000400,
1205 0x9838, 0xfffffffc, 0x00020200,
1206 0x5bb0, 0x000000f0, 0x00000070,
1207 0x5bc0, 0xf0311fff, 0x80300000,
1208 0x98f8, 0x73773777, 0x12010001,
1209 0x9b7c, 0x00ff0000, 0x00fc0000,
1210 0x2f48, 0x73773777, 0x12010001,
1211 0x8a14, 0xf000003f, 0x00000007,
1212 0x8b24, 0xffffffff, 0x00ffffff,
1213 0x28350, 0x3f3f3fff, 0x00000082,
1214 0x28354, 0x0000003f, 0x00000000,
1215 0x3e78, 0x00000001, 0x00000002,
1216 0x913c, 0xffff03df, 0x00000004,
1217 0xc768, 0x00000008, 0x00000008,
1218 0x8c00, 0x000008ff, 0x00000800,
1219 0x9508, 0x00010000, 0x00010000,
1220 0xac0c, 0xffffffff, 0x54763210,
1221 0x214f8, 0x01ff01ff, 0x00000002,
1222 0x21498, 0x007ff800, 0x00200000,
1223 0x2015c, 0xffffffff, 0x00000f40,
1224 0x30934, 0xffffffff, 0x00000001
1225 };
1226
1227 static const u32 spectre_mgcg_cgcg_init[] =
1228 {
1229 0xc420, 0xffffffff, 0xfffffffc,
1230 0x30800, 0xffffffff, 0xe0000000,
1231 0x3c2a0, 0xffffffff, 0x00000100,
1232 0x3c208, 0xffffffff, 0x00000100,
1233 0x3c2c0, 0xffffffff, 0x00000100,
1234 0x3c2c8, 0xffffffff, 0x00000100,
1235 0x3c2c4, 0xffffffff, 0x00000100,
1236 0x55e4, 0xffffffff, 0x00600100,
1237 0x3c280, 0xffffffff, 0x00000100,
1238 0x3c214, 0xffffffff, 0x06000100,
1239 0x3c220, 0xffffffff, 0x00000100,
1240 0x3c218, 0xffffffff, 0x06000100,
1241 0x3c204, 0xffffffff, 0x00000100,
1242 0x3c2e0, 0xffffffff, 0x00000100,
1243 0x3c224, 0xffffffff, 0x00000100,
1244 0x3c200, 0xffffffff, 0x00000100,
1245 0x3c230, 0xffffffff, 0x00000100,
1246 0x3c234, 0xffffffff, 0x00000100,
1247 0x3c250, 0xffffffff, 0x00000100,
1248 0x3c254, 0xffffffff, 0x00000100,
1249 0x3c258, 0xffffffff, 0x00000100,
1250 0x3c25c, 0xffffffff, 0x00000100,
1251 0x3c260, 0xffffffff, 0x00000100,
1252 0x3c27c, 0xffffffff, 0x00000100,
1253 0x3c278, 0xffffffff, 0x00000100,
1254 0x3c210, 0xffffffff, 0x06000100,
1255 0x3c290, 0xffffffff, 0x00000100,
1256 0x3c274, 0xffffffff, 0x00000100,
1257 0x3c2b4, 0xffffffff, 0x00000100,
1258 0x3c2b0, 0xffffffff, 0x00000100,
1259 0x3c270, 0xffffffff, 0x00000100,
1260 0x30800, 0xffffffff, 0xe0000000,
1261 0x3c020, 0xffffffff, 0x00010000,
1262 0x3c024, 0xffffffff, 0x00030002,
1263 0x3c028, 0xffffffff, 0x00040007,
1264 0x3c02c, 0xffffffff, 0x00060005,
1265 0x3c030, 0xffffffff, 0x00090008,
1266 0x3c034, 0xffffffff, 0x00010000,
1267 0x3c038, 0xffffffff, 0x00030002,
1268 0x3c03c, 0xffffffff, 0x00040007,
1269 0x3c040, 0xffffffff, 0x00060005,
1270 0x3c044, 0xffffffff, 0x00090008,
1271 0x3c048, 0xffffffff, 0x00010000,
1272 0x3c04c, 0xffffffff, 0x00030002,
1273 0x3c050, 0xffffffff, 0x00040007,
1274 0x3c054, 0xffffffff, 0x00060005,
1275 0x3c058, 0xffffffff, 0x00090008,
1276 0x3c05c, 0xffffffff, 0x00010000,
1277 0x3c060, 0xffffffff, 0x00030002,
1278 0x3c064, 0xffffffff, 0x00040007,
1279 0x3c068, 0xffffffff, 0x00060005,
1280 0x3c06c, 0xffffffff, 0x00090008,
1281 0x3c070, 0xffffffff, 0x00010000,
1282 0x3c074, 0xffffffff, 0x00030002,
1283 0x3c078, 0xffffffff, 0x00040007,
1284 0x3c07c, 0xffffffff, 0x00060005,
1285 0x3c080, 0xffffffff, 0x00090008,
1286 0x3c084, 0xffffffff, 0x00010000,
1287 0x3c088, 0xffffffff, 0x00030002,
1288 0x3c08c, 0xffffffff, 0x00040007,
1289 0x3c090, 0xffffffff, 0x00060005,
1290 0x3c094, 0xffffffff, 0x00090008,
1291 0x3c098, 0xffffffff, 0x00010000,
1292 0x3c09c, 0xffffffff, 0x00030002,
1293 0x3c0a0, 0xffffffff, 0x00040007,
1294 0x3c0a4, 0xffffffff, 0x00060005,
1295 0x3c0a8, 0xffffffff, 0x00090008,
1296 0x3c0ac, 0xffffffff, 0x00010000,
1297 0x3c0b0, 0xffffffff, 0x00030002,
1298 0x3c0b4, 0xffffffff, 0x00040007,
1299 0x3c0b8, 0xffffffff, 0x00060005,
1300 0x3c0bc, 0xffffffff, 0x00090008,
1301 0x3c000, 0xffffffff, 0x96e00200,
1302 0x8708, 0xffffffff, 0x00900100,
1303 0xc424, 0xffffffff, 0x0020003f,
1304 0x38, 0xffffffff, 0x0140001c,
1305 0x3c, 0x000f0000, 0x000f0000,
1306 0x220, 0xffffffff, 0xC060000C,
1307 0x224, 0xc0000fff, 0x00000100,
1308 0xf90, 0xffffffff, 0x00000100,
1309 0xf98, 0x00000101, 0x00000000,
1310 0x20a8, 0xffffffff, 0x00000104,
1311 0x55e4, 0xff000fff, 0x00000100,
1312 0x30cc, 0xc0000fff, 0x00000104,
1313 0xc1e4, 0x00000001, 0x00000001,
1314 0xd00c, 0xff000ff0, 0x00000100,
1315 0xd80c, 0xff000ff0, 0x00000100
1316 };
1317
1318 static const u32 kalindi_golden_spm_registers[] =
1319 {
1320 0x30800, 0xe0ffffff, 0xe0000000
1321 };
1322
1323 static const u32 kalindi_golden_common_registers[] =
1324 {
1325 0xc770, 0xffffffff, 0x00000800,
1326 0xc774, 0xffffffff, 0x00000800,
1327 0xc798, 0xffffffff, 0x00007fbf,
1328 0xc79c, 0xffffffff, 0x00007faf
1329 };
1330
1331 static const u32 kalindi_golden_registers[] =
1332 {
1333 0x3c000, 0xffffdfff, 0x6e944040,
1334 0x55e4, 0xff607fff, 0xfc000100,
1335 0x3c220, 0xff000fff, 0x00000100,
1336 0x3c224, 0xff000fff, 0x00000100,
1337 0x3c200, 0xfffc0fff, 0x00000100,
1338 0x6ed8, 0x00010101, 0x00010000,
1339 0x9830, 0xffffffff, 0x00000000,
1340 0x9834, 0xf00fffff, 0x00000400,
1341 0x5bb0, 0x000000f0, 0x00000070,
1342 0x5bc0, 0xf0311fff, 0x80300000,
1343 0x98f8, 0x73773777, 0x12010001,
1344 0x98fc, 0xffffffff, 0x00000010,
1345 0x9b7c, 0x00ff0000, 0x00fc0000,
1346 0x8030, 0x00001f0f, 0x0000100a,
1347 0x2f48, 0x73773777, 0x12010001,
1348 0x2408, 0x000fffff, 0x000c007f,
1349 0x8a14, 0xf000003f, 0x00000007,
1350 0x8b24, 0x3fff3fff, 0x00ffcfff,
1351 0x30a04, 0x0000ff0f, 0x00000000,
1352 0x28a4c, 0x07ffffff, 0x06000000,
1353 0x4d8, 0x00000fff, 0x00000100,
1354 0x3e78, 0x00000001, 0x00000002,
1355 0xc768, 0x00000008, 0x00000008,
1356 0x8c00, 0x000000ff, 0x00000003,
1357 0x214f8, 0x01ff01ff, 0x00000002,
1358 0x21498, 0x007ff800, 0x00200000,
1359 0x2015c, 0xffffffff, 0x00000f40,
1360 0x88c4, 0x001f3ae3, 0x00000082,
1361 0x88d4, 0x0000001f, 0x00000010,
1362 0x30934, 0xffffffff, 0x00000000
1363 };
1364
1365 static const u32 kalindi_mgcg_cgcg_init[] =
1366 {
1367 0xc420, 0xffffffff, 0xfffffffc,
1368 0x30800, 0xffffffff, 0xe0000000,
1369 0x3c2a0, 0xffffffff, 0x00000100,
1370 0x3c208, 0xffffffff, 0x00000100,
1371 0x3c2c0, 0xffffffff, 0x00000100,
1372 0x3c2c8, 0xffffffff, 0x00000100,
1373 0x3c2c4, 0xffffffff, 0x00000100,
1374 0x55e4, 0xffffffff, 0x00600100,
1375 0x3c280, 0xffffffff, 0x00000100,
1376 0x3c214, 0xffffffff, 0x06000100,
1377 0x3c220, 0xffffffff, 0x00000100,
1378 0x3c218, 0xffffffff, 0x06000100,
1379 0x3c204, 0xffffffff, 0x00000100,
1380 0x3c2e0, 0xffffffff, 0x00000100,
1381 0x3c224, 0xffffffff, 0x00000100,
1382 0x3c200, 0xffffffff, 0x00000100,
1383 0x3c230, 0xffffffff, 0x00000100,
1384 0x3c234, 0xffffffff, 0x00000100,
1385 0x3c250, 0xffffffff, 0x00000100,
1386 0x3c254, 0xffffffff, 0x00000100,
1387 0x3c258, 0xffffffff, 0x00000100,
1388 0x3c25c, 0xffffffff, 0x00000100,
1389 0x3c260, 0xffffffff, 0x00000100,
1390 0x3c27c, 0xffffffff, 0x00000100,
1391 0x3c278, 0xffffffff, 0x00000100,
1392 0x3c210, 0xffffffff, 0x06000100,
1393 0x3c290, 0xffffffff, 0x00000100,
1394 0x3c274, 0xffffffff, 0x00000100,
1395 0x3c2b4, 0xffffffff, 0x00000100,
1396 0x3c2b0, 0xffffffff, 0x00000100,
1397 0x3c270, 0xffffffff, 0x00000100,
1398 0x30800, 0xffffffff, 0xe0000000,
1399 0x3c020, 0xffffffff, 0x00010000,
1400 0x3c024, 0xffffffff, 0x00030002,
1401 0x3c028, 0xffffffff, 0x00040007,
1402 0x3c02c, 0xffffffff, 0x00060005,
1403 0x3c030, 0xffffffff, 0x00090008,
1404 0x3c034, 0xffffffff, 0x00010000,
1405 0x3c038, 0xffffffff, 0x00030002,
1406 0x3c03c, 0xffffffff, 0x00040007,
1407 0x3c040, 0xffffffff, 0x00060005,
1408 0x3c044, 0xffffffff, 0x00090008,
1409 0x3c000, 0xffffffff, 0x96e00200,
1410 0x8708, 0xffffffff, 0x00900100,
1411 0xc424, 0xffffffff, 0x0020003f,
1412 0x38, 0xffffffff, 0x0140001c,
1413 0x3c, 0x000f0000, 0x000f0000,
1414 0x220, 0xffffffff, 0xC060000C,
1415 0x224, 0xc0000fff, 0x00000100,
1416 0x20a8, 0xffffffff, 0x00000104,
1417 0x55e4, 0xff000fff, 0x00000100,
1418 0x30cc, 0xc0000fff, 0x00000104,
1419 0xc1e4, 0x00000001, 0x00000001,
1420 0xd00c, 0xff000ff0, 0x00000100,
1421 0xd80c, 0xff000ff0, 0x00000100
1422 };
1423
1424 static const u32 hawaii_golden_spm_registers[] =
1425 {
1426 0x30800, 0xe0ffffff, 0xe0000000
1427 };
1428
1429 static const u32 hawaii_golden_common_registers[] =
1430 {
1431 0x30800, 0xffffffff, 0xe0000000,
1432 0x28350, 0xffffffff, 0x3a00161a,
1433 0x28354, 0xffffffff, 0x0000002e,
1434 0x9a10, 0xffffffff, 0x00018208,
1435 0x98f8, 0xffffffff, 0x12011003
1436 };
1437
1438 static const u32 hawaii_golden_registers[] =
1439 {
1440 0x3354, 0x00000333, 0x00000333,
1441 0x9a10, 0x00010000, 0x00058208,
1442 0x9830, 0xffffffff, 0x00000000,
1443 0x9834, 0xf00fffff, 0x00000400,
1444 0x9838, 0x0002021c, 0x00020200,
1445 0xc78, 0x00000080, 0x00000000,
1446 0x5bb0, 0x000000f0, 0x00000070,
1447 0x5bc0, 0xf0311fff, 0x80300000,
1448 0x350c, 0x00810000, 0x408af000,
1449 0x7030, 0x31000111, 0x00000011,
1450 0x2f48, 0x73773777, 0x12010001,
1451 0x2120, 0x0000007f, 0x0000001b,
1452 0x21dc, 0x00007fb6, 0x00002191,
1453 0x3628, 0x0000003f, 0x0000000a,
1454 0x362c, 0x0000003f, 0x0000000a,
1455 0x2ae4, 0x00073ffe, 0x000022a2,
1456 0x240c, 0x000007ff, 0x00000000,
1457 0x8bf0, 0x00002001, 0x00000001,
1458 0x8b24, 0xffffffff, 0x00ffffff,
1459 0x30a04, 0x0000ff0f, 0x00000000,
1460 0x28a4c, 0x07ffffff, 0x06000000,
1461 0x3e78, 0x00000001, 0x00000002,
1462 0xc768, 0x00000008, 0x00000008,
1463 0xc770, 0x00000f00, 0x00000800,
1464 0xc774, 0x00000f00, 0x00000800,
1465 0xc798, 0x00ffffff, 0x00ff7fbf,
1466 0xc79c, 0x00ffffff, 0x00ff7faf,
1467 0x8c00, 0x000000ff, 0x00000800,
1468 0xe40, 0x00001fff, 0x00001fff,
1469 0x9060, 0x0000007f, 0x00000020,
1470 0x9508, 0x00010000, 0x00010000,
1471 0xae00, 0x00100000, 0x000ff07c,
1472 0xac14, 0x000003ff, 0x0000000f,
1473 0xac10, 0xffffffff, 0x7564fdec,
1474 0xac0c, 0xffffffff, 0x3120b9a8,
1475 0xac08, 0x20000000, 0x0f9c0000
1476 };
1477
1478 static const u32 hawaii_mgcg_cgcg_init[] =
1479 {
1480 0xc420, 0xffffffff, 0xfffffffd,
1481 0x30800, 0xffffffff, 0xe0000000,
1482 0x3c2a0, 0xffffffff, 0x00000100,
1483 0x3c208, 0xffffffff, 0x00000100,
1484 0x3c2c0, 0xffffffff, 0x00000100,
1485 0x3c2c8, 0xffffffff, 0x00000100,
1486 0x3c2c4, 0xffffffff, 0x00000100,
1487 0x55e4, 0xffffffff, 0x00200100,
1488 0x3c280, 0xffffffff, 0x00000100,
1489 0x3c214, 0xffffffff, 0x06000100,
1490 0x3c220, 0xffffffff, 0x00000100,
1491 0x3c218, 0xffffffff, 0x06000100,
1492 0x3c204, 0xffffffff, 0x00000100,
1493 0x3c2e0, 0xffffffff, 0x00000100,
1494 0x3c224, 0xffffffff, 0x00000100,
1495 0x3c200, 0xffffffff, 0x00000100,
1496 0x3c230, 0xffffffff, 0x00000100,
1497 0x3c234, 0xffffffff, 0x00000100,
1498 0x3c250, 0xffffffff, 0x00000100,
1499 0x3c254, 0xffffffff, 0x00000100,
1500 0x3c258, 0xffffffff, 0x00000100,
1501 0x3c25c, 0xffffffff, 0x00000100,
1502 0x3c260, 0xffffffff, 0x00000100,
1503 0x3c27c, 0xffffffff, 0x00000100,
1504 0x3c278, 0xffffffff, 0x00000100,
1505 0x3c210, 0xffffffff, 0x06000100,
1506 0x3c290, 0xffffffff, 0x00000100,
1507 0x3c274, 0xffffffff, 0x00000100,
1508 0x3c2b4, 0xffffffff, 0x00000100,
1509 0x3c2b0, 0xffffffff, 0x00000100,
1510 0x3c270, 0xffffffff, 0x00000100,
1511 0x30800, 0xffffffff, 0xe0000000,
1512 0x3c020, 0xffffffff, 0x00010000,
1513 0x3c024, 0xffffffff, 0x00030002,
1514 0x3c028, 0xffffffff, 0x00040007,
1515 0x3c02c, 0xffffffff, 0x00060005,
1516 0x3c030, 0xffffffff, 0x00090008,
1517 0x3c034, 0xffffffff, 0x00010000,
1518 0x3c038, 0xffffffff, 0x00030002,
1519 0x3c03c, 0xffffffff, 0x00040007,
1520 0x3c040, 0xffffffff, 0x00060005,
1521 0x3c044, 0xffffffff, 0x00090008,
1522 0x3c048, 0xffffffff, 0x00010000,
1523 0x3c04c, 0xffffffff, 0x00030002,
1524 0x3c050, 0xffffffff, 0x00040007,
1525 0x3c054, 0xffffffff, 0x00060005,
1526 0x3c058, 0xffffffff, 0x00090008,
1527 0x3c05c, 0xffffffff, 0x00010000,
1528 0x3c060, 0xffffffff, 0x00030002,
1529 0x3c064, 0xffffffff, 0x00040007,
1530 0x3c068, 0xffffffff, 0x00060005,
1531 0x3c06c, 0xffffffff, 0x00090008,
1532 0x3c070, 0xffffffff, 0x00010000,
1533 0x3c074, 0xffffffff, 0x00030002,
1534 0x3c078, 0xffffffff, 0x00040007,
1535 0x3c07c, 0xffffffff, 0x00060005,
1536 0x3c080, 0xffffffff, 0x00090008,
1537 0x3c084, 0xffffffff, 0x00010000,
1538 0x3c088, 0xffffffff, 0x00030002,
1539 0x3c08c, 0xffffffff, 0x00040007,
1540 0x3c090, 0xffffffff, 0x00060005,
1541 0x3c094, 0xffffffff, 0x00090008,
1542 0x3c098, 0xffffffff, 0x00010000,
1543 0x3c09c, 0xffffffff, 0x00030002,
1544 0x3c0a0, 0xffffffff, 0x00040007,
1545 0x3c0a4, 0xffffffff, 0x00060005,
1546 0x3c0a8, 0xffffffff, 0x00090008,
1547 0x3c0ac, 0xffffffff, 0x00010000,
1548 0x3c0b0, 0xffffffff, 0x00030002,
1549 0x3c0b4, 0xffffffff, 0x00040007,
1550 0x3c0b8, 0xffffffff, 0x00060005,
1551 0x3c0bc, 0xffffffff, 0x00090008,
1552 0x3c0c0, 0xffffffff, 0x00010000,
1553 0x3c0c4, 0xffffffff, 0x00030002,
1554 0x3c0c8, 0xffffffff, 0x00040007,
1555 0x3c0cc, 0xffffffff, 0x00060005,
1556 0x3c0d0, 0xffffffff, 0x00090008,
1557 0x3c0d4, 0xffffffff, 0x00010000,
1558 0x3c0d8, 0xffffffff, 0x00030002,
1559 0x3c0dc, 0xffffffff, 0x00040007,
1560 0x3c0e0, 0xffffffff, 0x00060005,
1561 0x3c0e4, 0xffffffff, 0x00090008,
1562 0x3c0e8, 0xffffffff, 0x00010000,
1563 0x3c0ec, 0xffffffff, 0x00030002,
1564 0x3c0f0, 0xffffffff, 0x00040007,
1565 0x3c0f4, 0xffffffff, 0x00060005,
1566 0x3c0f8, 0xffffffff, 0x00090008,
1567 0xc318, 0xffffffff, 0x00020200,
1568 0x3350, 0xffffffff, 0x00000200,
1569 0x15c0, 0xffffffff, 0x00000400,
1570 0x55e8, 0xffffffff, 0x00000000,
1571 0x2f50, 0xffffffff, 0x00000902,
1572 0x3c000, 0xffffffff, 0x96940200,
1573 0x8708, 0xffffffff, 0x00900100,
1574 0xc424, 0xffffffff, 0x0020003f,
1575 0x38, 0xffffffff, 0x0140001c,
1576 0x3c, 0x000f0000, 0x000f0000,
1577 0x220, 0xffffffff, 0xc060000c,
1578 0x224, 0xc0000fff, 0x00000100,
1579 0xf90, 0xffffffff, 0x00000100,
1580 0xf98, 0x00000101, 0x00000000,
1581 0x20a8, 0xffffffff, 0x00000104,
1582 0x55e4, 0xff000fff, 0x00000100,
1583 0x30cc, 0xc0000fff, 0x00000104,
1584 0xc1e4, 0x00000001, 0x00000001,
1585 0xd00c, 0xff000ff0, 0x00000100,
1586 0xd80c, 0xff000ff0, 0x00000100
1587 };
1588
1589 static const u32 godavari_golden_registers[] =
1590 {
1591 0x55e4, 0xff607fff, 0xfc000100,
1592 0x6ed8, 0x00010101, 0x00010000,
1593 0x9830, 0xffffffff, 0x00000000,
1594 0x98302, 0xf00fffff, 0x00000400,
1595 0x6130, 0xffffffff, 0x00010000,
1596 0x5bb0, 0x000000f0, 0x00000070,
1597 0x5bc0, 0xf0311fff, 0x80300000,
1598 0x98f8, 0x73773777, 0x12010001,
1599 0x98fc, 0xffffffff, 0x00000010,
1600 0x8030, 0x00001f0f, 0x0000100a,
1601 0x2f48, 0x73773777, 0x12010001,
1602 0x2408, 0x000fffff, 0x000c007f,
1603 0x8a14, 0xf000003f, 0x00000007,
1604 0x8b24, 0xffffffff, 0x00ff0fff,
1605 0x30a04, 0x0000ff0f, 0x00000000,
1606 0x28a4c, 0x07ffffff, 0x06000000,
1607 0x4d8, 0x00000fff, 0x00000100,
1608 0xd014, 0x00010000, 0x00810001,
1609 0xd814, 0x00010000, 0x00810001,
1610 0x3e78, 0x00000001, 0x00000002,
1611 0xc768, 0x00000008, 0x00000008,
1612 0xc770, 0x00000f00, 0x00000800,
1613 0xc774, 0x00000f00, 0x00000800,
1614 0xc798, 0x00ffffff, 0x00ff7fbf,
1615 0xc79c, 0x00ffffff, 0x00ff7faf,
1616 0x8c00, 0x000000ff, 0x00000001,
1617 0x214f8, 0x01ff01ff, 0x00000002,
1618 0x21498, 0x007ff800, 0x00200000,
1619 0x2015c, 0xffffffff, 0x00000f40,
1620 0x88c4, 0x001f3ae3, 0x00000082,
1621 0x88d4, 0x0000001f, 0x00000010,
1622 0x30934, 0xffffffff, 0x00000000
1623 };
1624
1625
1626 static void cik_init_golden_registers(struct radeon_device *rdev)
1627 {
1628 /* Some of the registers might be dependent on GRBM_GFX_INDEX */
1629 mutex_lock(&rdev->grbm_idx_mutex);
1630 switch (rdev->family) {
1631 case CHIP_BONAIRE:
1632 radeon_program_register_sequence(rdev,
1633 bonaire_mgcg_cgcg_init,
1634 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1635 radeon_program_register_sequence(rdev,
1636 bonaire_golden_registers,
1637 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1638 radeon_program_register_sequence(rdev,
1639 bonaire_golden_common_registers,
1640 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1641 radeon_program_register_sequence(rdev,
1642 bonaire_golden_spm_registers,
1643 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1644 break;
1645 case CHIP_KABINI:
1646 radeon_program_register_sequence(rdev,
1647 kalindi_mgcg_cgcg_init,
1648 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1649 radeon_program_register_sequence(rdev,
1650 kalindi_golden_registers,
1651 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1652 radeon_program_register_sequence(rdev,
1653 kalindi_golden_common_registers,
1654 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1655 radeon_program_register_sequence(rdev,
1656 kalindi_golden_spm_registers,
1657 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1658 break;
1659 case CHIP_MULLINS:
1660 radeon_program_register_sequence(rdev,
1661 kalindi_mgcg_cgcg_init,
1662 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1663 radeon_program_register_sequence(rdev,
1664 godavari_golden_registers,
1665 (const u32)ARRAY_SIZE(godavari_golden_registers));
1666 radeon_program_register_sequence(rdev,
1667 kalindi_golden_common_registers,
1668 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1669 radeon_program_register_sequence(rdev,
1670 kalindi_golden_spm_registers,
1671 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1672 break;
1673 case CHIP_KAVERI:
1674 radeon_program_register_sequence(rdev,
1675 spectre_mgcg_cgcg_init,
1676 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1677 radeon_program_register_sequence(rdev,
1678 spectre_golden_registers,
1679 (const u32)ARRAY_SIZE(spectre_golden_registers));
1680 radeon_program_register_sequence(rdev,
1681 spectre_golden_common_registers,
1682 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1683 radeon_program_register_sequence(rdev,
1684 spectre_golden_spm_registers,
1685 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1686 break;
1687 case CHIP_HAWAII:
1688 radeon_program_register_sequence(rdev,
1689 hawaii_mgcg_cgcg_init,
1690 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1691 radeon_program_register_sequence(rdev,
1692 hawaii_golden_registers,
1693 (const u32)ARRAY_SIZE(hawaii_golden_registers));
1694 radeon_program_register_sequence(rdev,
1695 hawaii_golden_common_registers,
1696 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1697 radeon_program_register_sequence(rdev,
1698 hawaii_golden_spm_registers,
1699 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1700 break;
1701 default:
1702 break;
1703 }
1704 mutex_unlock(&rdev->grbm_idx_mutex);
1705 }
1706
1707 /**
1708 * cik_get_xclk - get the xclk
1709 *
1710 * @rdev: radeon_device pointer
1711 *
1712 * Returns the reference clock used by the gfx engine
1713 * (CIK).
1714 */
1715 u32 cik_get_xclk(struct radeon_device *rdev)
1716 {
1717 u32 reference_clock = rdev->clock.spll.reference_freq;
1718
1719 if (rdev->flags & RADEON_IS_IGP) {
1720 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1721 return reference_clock / 2;
1722 } else {
1723 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1724 return reference_clock / 4;
1725 }
1726 return reference_clock;
1727 }
1728
1729 /**
1730 * cik_mm_rdoorbell - read a doorbell dword
1731 *
1732 * @rdev: radeon_device pointer
1733 * @index: doorbell index
1734 *
1735 * Returns the value in the doorbell aperture at the
1736 * requested doorbell index (CIK).
1737 */
1738 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
1739 {
1740 if (index < rdev->doorbell.num_doorbells) {
1741 return readl(rdev->doorbell.ptr + index);
1742 } else {
1743 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
1744 return 0;
1745 }
1746 }
1747
1748 /**
1749 * cik_mm_wdoorbell - write a doorbell dword
1750 *
1751 * @rdev: radeon_device pointer
1752 * @index: doorbell index
1753 * @v: value to write
1754 *
1755 * Writes @v to the doorbell aperture at the
1756 * requested doorbell index (CIK).
1757 */
1758 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
1759 {
1760 if (index < rdev->doorbell.num_doorbells) {
1761 writel(v, rdev->doorbell.ptr + index);
1762 } else {
1763 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
1764 }
1765 }
1766
1767 #define BONAIRE_IO_MC_REGS_SIZE 36
1768
1769 static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1770 {
1771 {0x00000070, 0x04400000},
1772 {0x00000071, 0x80c01803},
1773 {0x00000072, 0x00004004},
1774 {0x00000073, 0x00000100},
1775 {0x00000074, 0x00ff0000},
1776 {0x00000075, 0x34000000},
1777 {0x00000076, 0x08000014},
1778 {0x00000077, 0x00cc08ec},
1779 {0x00000078, 0x00000400},
1780 {0x00000079, 0x00000000},
1781 {0x0000007a, 0x04090000},
1782 {0x0000007c, 0x00000000},
1783 {0x0000007e, 0x4408a8e8},
1784 {0x0000007f, 0x00000304},
1785 {0x00000080, 0x00000000},
1786 {0x00000082, 0x00000001},
1787 {0x00000083, 0x00000002},
1788 {0x00000084, 0xf3e4f400},
1789 {0x00000085, 0x052024e3},
1790 {0x00000087, 0x00000000},
1791 {0x00000088, 0x01000000},
1792 {0x0000008a, 0x1c0a0000},
1793 {0x0000008b, 0xff010000},
1794 {0x0000008d, 0xffffefff},
1795 {0x0000008e, 0xfff3efff},
1796 {0x0000008f, 0xfff3efbf},
1797 {0x00000092, 0xf7ffffff},
1798 {0x00000093, 0xffffff7f},
1799 {0x00000095, 0x00101101},
1800 {0x00000096, 0x00000fff},
1801 {0x00000097, 0x00116fff},
1802 {0x00000098, 0x60010000},
1803 {0x00000099, 0x10010000},
1804 {0x0000009a, 0x00006000},
1805 {0x0000009b, 0x00001000},
1806 {0x0000009f, 0x00b48000}
1807 };
1808
1809 #define HAWAII_IO_MC_REGS_SIZE 22
1810
1811 static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1812 {
1813 {0x0000007d, 0x40000000},
1814 {0x0000007e, 0x40180304},
1815 {0x0000007f, 0x0000ff00},
1816 {0x00000081, 0x00000000},
1817 {0x00000083, 0x00000800},
1818 {0x00000086, 0x00000000},
1819 {0x00000087, 0x00000100},
1820 {0x00000088, 0x00020100},
1821 {0x00000089, 0x00000000},
1822 {0x0000008b, 0x00040000},
1823 {0x0000008c, 0x00000100},
1824 {0x0000008e, 0xff010000},
1825 {0x00000090, 0xffffefff},
1826 {0x00000091, 0xfff3efff},
1827 {0x00000092, 0xfff3efbf},
1828 {0x00000093, 0xf7ffffff},
1829 {0x00000094, 0xffffff7f},
1830 {0x00000095, 0x00000fff},
1831 {0x00000096, 0x00116fff},
1832 {0x00000097, 0x60010000},
1833 {0x00000098, 0x10010000},
1834 {0x0000009f, 0x00c79000}
1835 };
1836
1837
1838 /**
1839 * cik_srbm_select - select specific register instances
1840 *
1841 * @rdev: radeon_device pointer
1842 * @me: selected ME (micro engine)
1843 * @pipe: pipe
1844 * @queue: queue
1845 * @vmid: VMID
1846 *
1847 * Switches the currently active registers instances. Some
1848 * registers are instanced per VMID, others are instanced per
1849 * me/pipe/queue combination.
1850 */
1851 static void cik_srbm_select(struct radeon_device *rdev,
1852 u32 me, u32 pipe, u32 queue, u32 vmid)
1853 {
1854 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1855 MEID(me & 0x3) |
1856 VMID(vmid & 0xf) |
1857 QUEUEID(queue & 0x7));
1858 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1859 }
1860
1861 /* ucode loading */
1862 /**
1863 * ci_mc_load_microcode - load MC ucode into the hw
1864 *
1865 * @rdev: radeon_device pointer
1866 *
1867 * Load the GDDR MC ucode into the hw (CIK).
1868 * Returns 0 on success, error on failure.
1869 */
1870 int ci_mc_load_microcode(struct radeon_device *rdev)
1871 {
1872 const __be32 *fw_data = NULL;
1873 const __le32 *new_fw_data = NULL;
1874 u32 running, blackout = 0, tmp;
1875 u32 *io_mc_regs = NULL;
1876 const __le32 *new_io_mc_regs = NULL;
1877 int i, regs_size, ucode_size;
1878
1879 if (!rdev->mc_fw)
1880 return -EINVAL;
1881
1882 if (rdev->new_fw) {
1883 const struct mc_firmware_header_v1_0 *hdr =
1884 (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
1885
1886 radeon_ucode_print_mc_hdr(&hdr->header);
1887
1888 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1889 new_io_mc_regs = (const __le32 *)
1890 (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1891 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1892 new_fw_data = (const __le32 *)
1893 (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1894 } else {
1895 ucode_size = rdev->mc_fw->size / 4;
1896
1897 switch (rdev->family) {
1898 case CHIP_BONAIRE:
1899 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1900 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1901 break;
1902 case CHIP_HAWAII:
1903 io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1904 regs_size = HAWAII_IO_MC_REGS_SIZE;
1905 break;
1906 default:
1907 return -EINVAL;
1908 }
1909 fw_data = (const __be32 *)rdev->mc_fw->data;
1910 }
1911
1912 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1913
1914 if (running == 0) {
1915 if (running) {
1916 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1917 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1918 }
1919
1920 /* reset the engine and set to writable */
1921 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1922 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1923
1924 /* load mc io regs */
1925 for (i = 0; i < regs_size; i++) {
1926 if (rdev->new_fw) {
1927 WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1928 WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1929 } else {
1930 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1931 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1932 }
1933 }
1934
1935 tmp = RREG32(MC_SEQ_MISC0);
1936 if ((rdev->pdev->device == 0x6649) && ((tmp & 0xff00) == 0x5600)) {
1937 WREG32(MC_SEQ_IO_DEBUG_INDEX, 5);
1938 WREG32(MC_SEQ_IO_DEBUG_DATA, 0x00000023);
1939 WREG32(MC_SEQ_IO_DEBUG_INDEX, 9);
1940 WREG32(MC_SEQ_IO_DEBUG_DATA, 0x000001f0);
1941 }
1942
1943 /* load the MC ucode */
1944 for (i = 0; i < ucode_size; i++) {
1945 if (rdev->new_fw)
1946 WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1947 else
1948 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1949 }
1950
1951 /* put the engine back into the active state */
1952 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1953 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1954 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1955
1956 /* wait for training to complete */
1957 for (i = 0; i < rdev->usec_timeout; i++) {
1958 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1959 break;
1960 udelay(1);
1961 }
1962 for (i = 0; i < rdev->usec_timeout; i++) {
1963 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1964 break;
1965 udelay(1);
1966 }
1967
1968 if (running)
1969 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1970 }
1971
1972 return 0;
1973 }
1974
1975 /**
1976 * cik_init_microcode - load ucode images from disk
1977 *
1978 * @rdev: radeon_device pointer
1979 *
1980 * Use the firmware interface to load the ucode images into
1981 * the driver (not loaded into hw).
1982 * Returns 0 on success, error on failure.
1983 */
1984 static int cik_init_microcode(struct radeon_device *rdev)
1985 {
1986 const char *chip_name;
1987 const char *new_chip_name;
1988 size_t pfp_req_size, me_req_size, ce_req_size,
1989 mec_req_size, rlc_req_size, mc_req_size = 0,
1990 sdma_req_size, smc_req_size = 0, mc2_req_size = 0;
1991 char fw_name[30];
1992 int new_fw = 0;
1993 int err;
1994 int num_fw;
1995 bool new_smc = false;
1996
1997 DRM_DEBUG("\n");
1998
1999 switch (rdev->family) {
2000 case CHIP_BONAIRE:
2001 chip_name = "BONAIRE";
2002 if ((rdev->pdev->revision == 0x80) ||
2003 (rdev->pdev->revision == 0x81) ||
2004 (rdev->pdev->device == 0x665f))
2005 new_smc = true;
2006 new_chip_name = "bonaire";
2007 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2008 me_req_size = CIK_ME_UCODE_SIZE * 4;
2009 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2010 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2011 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
2012 mc_req_size = BONAIRE_MC_UCODE_SIZE * 4;
2013 mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4;
2014 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2015 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
2016 num_fw = 8;
2017 break;
2018 case CHIP_HAWAII:
2019 chip_name = "HAWAII";
2020 if (rdev->pdev->revision == 0x80)
2021 new_smc = true;
2022 new_chip_name = "hawaii";
2023 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2024 me_req_size = CIK_ME_UCODE_SIZE * 4;
2025 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2026 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2027 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
2028 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
2029 mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4;
2030 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2031 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
2032 num_fw = 8;
2033 break;
2034 case CHIP_KAVERI:
2035 chip_name = "KAVERI";
2036 new_chip_name = "kaveri";
2037 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2038 me_req_size = CIK_ME_UCODE_SIZE * 4;
2039 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2040 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2041 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
2042 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2043 num_fw = 7;
2044 break;
2045 case CHIP_KABINI:
2046 chip_name = "KABINI";
2047 new_chip_name = "kabini";
2048 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2049 me_req_size = CIK_ME_UCODE_SIZE * 4;
2050 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2051 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2052 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
2053 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2054 num_fw = 6;
2055 break;
2056 case CHIP_MULLINS:
2057 chip_name = "MULLINS";
2058 new_chip_name = "mullins";
2059 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2060 me_req_size = CIK_ME_UCODE_SIZE * 4;
2061 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2062 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2063 rlc_req_size = ML_RLC_UCODE_SIZE * 4;
2064 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2065 num_fw = 6;
2066 break;
2067 default: BUG();
2068 }
2069
2070 DRM_INFO("Loading %s Microcode\n", new_chip_name);
2071
2072 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name);
2073 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
2074 if (err) {
2075 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
2076 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
2077 if (err)
2078 goto out;
2079 if (rdev->pfp_fw->size != pfp_req_size) {
2080 printk(KERN_ERR
2081 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2082 rdev->pfp_fw->size, fw_name);
2083 err = -EINVAL;
2084 goto out;
2085 }
2086 } else {
2087 err = radeon_ucode_validate(rdev->pfp_fw);
2088 if (err) {
2089 printk(KERN_ERR
2090 "cik_fw: validation failed for firmware \"%s\"\n",
2091 fw_name);
2092 goto out;
2093 } else {
2094 new_fw++;
2095 }
2096 }
2097
2098 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name);
2099 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2100 if (err) {
2101 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
2102 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2103 if (err)
2104 goto out;
2105 if (rdev->me_fw->size != me_req_size) {
2106 printk(KERN_ERR
2107 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2108 rdev->me_fw->size, fw_name);
2109 err = -EINVAL;
2110 }
2111 } else {
2112 err = radeon_ucode_validate(rdev->me_fw);
2113 if (err) {
2114 printk(KERN_ERR
2115 "cik_fw: validation failed for firmware \"%s\"\n",
2116 fw_name);
2117 goto out;
2118 } else {
2119 new_fw++;
2120 }
2121 }
2122
2123 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name);
2124 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2125 if (err) {
2126 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
2127 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2128 if (err)
2129 goto out;
2130 if (rdev->ce_fw->size != ce_req_size) {
2131 printk(KERN_ERR
2132 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2133 rdev->ce_fw->size, fw_name);
2134 err = -EINVAL;
2135 }
2136 } else {
2137 err = radeon_ucode_validate(rdev->ce_fw);
2138 if (err) {
2139 printk(KERN_ERR
2140 "cik_fw: validation failed for firmware \"%s\"\n",
2141 fw_name);
2142 goto out;
2143 } else {
2144 new_fw++;
2145 }
2146 }
2147
2148 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", new_chip_name);
2149 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2150 if (err) {
2151 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
2152 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2153 if (err)
2154 goto out;
2155 if (rdev->mec_fw->size != mec_req_size) {
2156 printk(KERN_ERR
2157 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2158 rdev->mec_fw->size, fw_name);
2159 err = -EINVAL;
2160 }
2161 } else {
2162 err = radeon_ucode_validate(rdev->mec_fw);
2163 if (err) {
2164 printk(KERN_ERR
2165 "cik_fw: validation failed for firmware \"%s\"\n",
2166 fw_name);
2167 goto out;
2168 } else {
2169 new_fw++;
2170 }
2171 }
2172
2173 if (rdev->family == CHIP_KAVERI) {
2174 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec2.bin", new_chip_name);
2175 err = request_firmware(&rdev->mec2_fw, fw_name, rdev->dev);
2176 if (err) {
2177 goto out;
2178 } else {
2179 err = radeon_ucode_validate(rdev->mec2_fw);
2180 if (err) {
2181 goto out;
2182 } else {
2183 new_fw++;
2184 }
2185 }
2186 }
2187
2188 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name);
2189 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2190 if (err) {
2191 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
2192 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2193 if (err)
2194 goto out;
2195 if (rdev->rlc_fw->size != rlc_req_size) {
2196 printk(KERN_ERR
2197 "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
2198 rdev->rlc_fw->size, fw_name);
2199 err = -EINVAL;
2200 }
2201 } else {
2202 err = radeon_ucode_validate(rdev->rlc_fw);
2203 if (err) {
2204 printk(KERN_ERR
2205 "cik_fw: validation failed for firmware \"%s\"\n",
2206 fw_name);
2207 goto out;
2208 } else {
2209 new_fw++;
2210 }
2211 }
2212
2213 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", new_chip_name);
2214 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2215 if (err) {
2216 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
2217 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2218 if (err)
2219 goto out;
2220 if (rdev->sdma_fw->size != sdma_req_size) {
2221 printk(KERN_ERR
2222 "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
2223 rdev->sdma_fw->size, fw_name);
2224 err = -EINVAL;
2225 }
2226 } else {
2227 err = radeon_ucode_validate(rdev->sdma_fw);
2228 if (err) {
2229 printk(KERN_ERR
2230 "cik_fw: validation failed for firmware \"%s\"\n",
2231 fw_name);
2232 goto out;
2233 } else {
2234 new_fw++;
2235 }
2236 }
2237
2238 /* No SMC, MC ucode on APUs */
2239 if (!(rdev->flags & RADEON_IS_IGP)) {
2240 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name);
2241 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2242 if (err) {
2243 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
2244 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2245 if (err) {
2246 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
2247 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2248 if (err)
2249 goto out;
2250 }
2251 if ((rdev->mc_fw->size != mc_req_size) &&
2252 (rdev->mc_fw->size != mc2_req_size)){
2253 printk(KERN_ERR
2254 "cik_mc: Bogus length %zu in firmware \"%s\"\n",
2255 rdev->mc_fw->size, fw_name);
2256 err = -EINVAL;
2257 }
2258 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
2259 } else {
2260 err = radeon_ucode_validate(rdev->mc_fw);
2261 if (err) {
2262 printk(KERN_ERR
2263 "cik_fw: validation failed for firmware \"%s\"\n",
2264 fw_name);
2265 goto out;
2266 } else {
2267 new_fw++;
2268 }
2269 }
2270
2271 if (new_smc)
2272 snprintf(fw_name, sizeof(fw_name), "radeon/%s_k_smc.bin", new_chip_name);
2273 else
2274 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
2275 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2276 if (err) {
2277 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
2278 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2279 if (err) {
2280 printk(KERN_ERR
2281 "smc: error loading firmware \"%s\"\n",
2282 fw_name);
2283 release_firmware(rdev->smc_fw);
2284 rdev->smc_fw = NULL;
2285 err = 0;
2286 } else if (rdev->smc_fw->size != smc_req_size) {
2287 printk(KERN_ERR
2288 "cik_smc: Bogus length %zu in firmware \"%s\"\n",
2289 rdev->smc_fw->size, fw_name);
2290 err = -EINVAL;
2291 }
2292 } else {
2293 err = radeon_ucode_validate(rdev->smc_fw);
2294 if (err) {
2295 printk(KERN_ERR
2296 "cik_fw: validation failed for firmware \"%s\"\n",
2297 fw_name);
2298 goto out;
2299 } else {
2300 new_fw++;
2301 }
2302 }
2303 }
2304
2305 if (new_fw == 0) {
2306 rdev->new_fw = false;
2307 } else if (new_fw < num_fw) {
2308 printk(KERN_ERR "ci_fw: mixing new and old firmware!\n");
2309 err = -EINVAL;
2310 } else {
2311 rdev->new_fw = true;
2312 }
2313
2314 out:
2315 if (err) {
2316 if (err != -EINVAL)
2317 printk(KERN_ERR
2318 "cik_cp: Failed to load firmware \"%s\"\n",
2319 fw_name);
2320 release_firmware(rdev->pfp_fw);
2321 rdev->pfp_fw = NULL;
2322 release_firmware(rdev->me_fw);
2323 rdev->me_fw = NULL;
2324 release_firmware(rdev->ce_fw);
2325 rdev->ce_fw = NULL;
2326 release_firmware(rdev->mec_fw);
2327 rdev->mec_fw = NULL;
2328 release_firmware(rdev->mec2_fw);
2329 rdev->mec2_fw = NULL;
2330 release_firmware(rdev->rlc_fw);
2331 rdev->rlc_fw = NULL;
2332 release_firmware(rdev->sdma_fw);
2333 rdev->sdma_fw = NULL;
2334 release_firmware(rdev->mc_fw);
2335 rdev->mc_fw = NULL;
2336 release_firmware(rdev->smc_fw);
2337 rdev->smc_fw = NULL;
2338 }
2339 return err;
2340 }
2341
2342 /*
2343 * Core functions
2344 */
2345 /**
2346 * cik_tiling_mode_table_init - init the hw tiling table
2347 *
2348 * @rdev: radeon_device pointer
2349 *
2350 * Starting with SI, the tiling setup is done globally in a
2351 * set of 32 tiling modes. Rather than selecting each set of
2352 * parameters per surface as on older asics, we just select
2353 * which index in the tiling table we want to use, and the
2354 * surface uses those parameters (CIK).
2355 */
2356 static void cik_tiling_mode_table_init(struct radeon_device *rdev)
2357 {
2358 u32 *tile = rdev->config.cik.tile_mode_array;
2359 u32 *macrotile = rdev->config.cik.macrotile_mode_array;
2360 const u32 num_tile_mode_states =
2361 ARRAY_SIZE(rdev->config.cik.tile_mode_array);
2362 const u32 num_secondary_tile_mode_states =
2363 ARRAY_SIZE(rdev->config.cik.macrotile_mode_array);
2364 u32 reg_offset, split_equal_to_row_size;
2365 u32 num_pipe_configs;
2366 u32 num_rbs = rdev->config.cik.max_backends_per_se *
2367 rdev->config.cik.max_shader_engines;
2368
2369 switch (rdev->config.cik.mem_row_size_in_kb) {
2370 case 1:
2371 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2372 break;
2373 case 2:
2374 default:
2375 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2376 break;
2377 case 4:
2378 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2379 break;
2380 }
2381
2382 num_pipe_configs = rdev->config.cik.max_tile_pipes;
2383 if (num_pipe_configs > 8)
2384 num_pipe_configs = 16;
2385
2386 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2387 tile[reg_offset] = 0;
2388 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2389 macrotile[reg_offset] = 0;
2390
2391 switch(num_pipe_configs) {
2392 case 16:
2393 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2394 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2395 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2396 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2397 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2398 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2399 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2400 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2401 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2402 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2403 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2404 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2405 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2406 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2407 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2408 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2409 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2410 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2411 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2412 TILE_SPLIT(split_equal_to_row_size));
2413 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2414 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2415 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2416 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2417 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2418 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2419 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2420 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2421 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2422 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2423 TILE_SPLIT(split_equal_to_row_size));
2424 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2425 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2426 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2427 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2428 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2429 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2430 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2431 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2432 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2433 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2434 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2435 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2436 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2437 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2438 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2439 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2440 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2441 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2442 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2443 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2444 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2445 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2446 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2447 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2448 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2449 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2450 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2451 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2452 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2453 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2454 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2455 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2456 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2457 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2458 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2459 tile[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2460 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2461 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2462 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2463 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2464 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2465 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2466 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2467 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2468 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2469 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2470 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2471
2472 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2473 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2474 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2475 NUM_BANKS(ADDR_SURF_16_BANK));
2476 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2477 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2478 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2479 NUM_BANKS(ADDR_SURF_16_BANK));
2480 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2481 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2482 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2483 NUM_BANKS(ADDR_SURF_16_BANK));
2484 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2485 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2486 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2487 NUM_BANKS(ADDR_SURF_16_BANK));
2488 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2489 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2490 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2491 NUM_BANKS(ADDR_SURF_8_BANK));
2492 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2493 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2494 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2495 NUM_BANKS(ADDR_SURF_4_BANK));
2496 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2497 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2498 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2499 NUM_BANKS(ADDR_SURF_2_BANK));
2500 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2501 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2502 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2503 NUM_BANKS(ADDR_SURF_16_BANK));
2504 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2505 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2506 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2507 NUM_BANKS(ADDR_SURF_16_BANK));
2508 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2509 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2510 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2511 NUM_BANKS(ADDR_SURF_16_BANK));
2512 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2513 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2514 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2515 NUM_BANKS(ADDR_SURF_8_BANK));
2516 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2517 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2518 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2519 NUM_BANKS(ADDR_SURF_4_BANK));
2520 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2521 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2522 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2523 NUM_BANKS(ADDR_SURF_2_BANK));
2524 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2525 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2526 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2527 NUM_BANKS(ADDR_SURF_2_BANK));
2528
2529 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2530 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2531 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2532 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
2533 break;
2534
2535 case 8:
2536 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2537 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2538 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2539 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2540 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2541 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2542 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2543 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2544 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2545 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2546 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2547 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2548 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2549 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2550 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2551 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2552 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2553 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2554 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2555 TILE_SPLIT(split_equal_to_row_size));
2556 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2557 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2558 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2559 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2560 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2561 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2562 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2563 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2564 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2565 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2566 TILE_SPLIT(split_equal_to_row_size));
2567 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2568 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2569 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2570 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2571 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2572 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2573 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2574 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2575 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2576 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2577 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2578 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2579 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2580 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2581 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2582 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2583 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2584 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2585 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2586 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2587 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2588 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2589 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2590 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2591 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2592 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2593 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2594 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2595 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2596 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2597 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2598 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2599 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2600 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2601 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2602 tile[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2603 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2604 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2605 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2606 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2607 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2608 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2609 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2610 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2611 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2612 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2613 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2614
2615 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2616 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2617 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2618 NUM_BANKS(ADDR_SURF_16_BANK));
2619 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2620 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2621 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2622 NUM_BANKS(ADDR_SURF_16_BANK));
2623 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2624 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2625 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2626 NUM_BANKS(ADDR_SURF_16_BANK));
2627 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2628 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2629 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2630 NUM_BANKS(ADDR_SURF_16_BANK));
2631 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2632 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2633 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2634 NUM_BANKS(ADDR_SURF_8_BANK));
2635 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2636 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2637 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2638 NUM_BANKS(ADDR_SURF_4_BANK));
2639 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2640 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2641 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2642 NUM_BANKS(ADDR_SURF_2_BANK));
2643 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2644 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2645 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2646 NUM_BANKS(ADDR_SURF_16_BANK));
2647 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2648 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2649 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2650 NUM_BANKS(ADDR_SURF_16_BANK));
2651 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2652 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2653 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2654 NUM_BANKS(ADDR_SURF_16_BANK));
2655 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2656 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2657 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2658 NUM_BANKS(ADDR_SURF_16_BANK));
2659 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2660 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2661 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2662 NUM_BANKS(ADDR_SURF_8_BANK));
2663 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2664 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2665 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2666 NUM_BANKS(ADDR_SURF_4_BANK));
2667 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2668 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2669 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2670 NUM_BANKS(ADDR_SURF_2_BANK));
2671
2672 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2673 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2674 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2675 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
2676 break;
2677
2678 case 4:
2679 if (num_rbs == 4) {
2680 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2681 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2682 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2683 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2684 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2685 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2686 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2687 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2688 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2689 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2690 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2691 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2692 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2693 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2694 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2695 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2696 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2697 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2698 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2699 TILE_SPLIT(split_equal_to_row_size));
2700 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2701 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2702 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2703 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2704 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2705 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2706 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2707 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2708 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2709 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2710 TILE_SPLIT(split_equal_to_row_size));
2711 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2712 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2713 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2714 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2715 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2716 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2717 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2718 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2719 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2720 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2721 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2722 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2723 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2724 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2725 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2726 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2727 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2728 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2729 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2730 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2731 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2732 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2733 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2734 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2735 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2736 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2737 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2738 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2739 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2740 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2741 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2742 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2743 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2744 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2745 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2746 tile[28] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2747 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2748 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2749 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2750 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2751 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2752 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2753 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2754 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2755 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2756 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2757 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2758
2759 } else if (num_rbs < 4) {
2760 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2761 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2762 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2763 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2764 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2765 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2766 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2767 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2768 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2769 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2770 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2771 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2772 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2773 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2774 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2775 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2776 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2777 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2778 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2779 TILE_SPLIT(split_equal_to_row_size));
2780 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2781 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2782 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2783 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2784 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2785 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2786 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2787 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2788 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2789 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2790 TILE_SPLIT(split_equal_to_row_size));
2791 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2792 PIPE_CONFIG(ADDR_SURF_P4_8x16));
2793 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2794 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2795 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2796 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2797 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2798 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2799 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2800 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2801 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2802 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2803 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2804 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2805 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2806 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2807 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2808 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2809 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2810 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2811 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2812 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2813 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2814 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2815 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2816 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2817 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2818 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2819 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2820 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2821 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2822 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2823 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2824 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2825 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2826 tile[28] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2827 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2828 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2829 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2830 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2831 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2832 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2833 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2834 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2835 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2836 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2837 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2838 }
2839
2840 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2841 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2842 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2843 NUM_BANKS(ADDR_SURF_16_BANK));
2844 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2845 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2846 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2847 NUM_BANKS(ADDR_SURF_16_BANK));
2848 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2849 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2850 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2851 NUM_BANKS(ADDR_SURF_16_BANK));
2852 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2853 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2854 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2855 NUM_BANKS(ADDR_SURF_16_BANK));
2856 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2857 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2858 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2859 NUM_BANKS(ADDR_SURF_16_BANK));
2860 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2861 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2862 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2863 NUM_BANKS(ADDR_SURF_8_BANK));
2864 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2865 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2866 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2867 NUM_BANKS(ADDR_SURF_4_BANK));
2868 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2869 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2870 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2871 NUM_BANKS(ADDR_SURF_16_BANK));
2872 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2873 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2874 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2875 NUM_BANKS(ADDR_SURF_16_BANK));
2876 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2877 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2878 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2879 NUM_BANKS(ADDR_SURF_16_BANK));
2880 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2881 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2882 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2883 NUM_BANKS(ADDR_SURF_16_BANK));
2884 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2885 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2886 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2887 NUM_BANKS(ADDR_SURF_16_BANK));
2888 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2889 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2890 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2891 NUM_BANKS(ADDR_SURF_8_BANK));
2892 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2893 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2894 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2895 NUM_BANKS(ADDR_SURF_4_BANK));
2896
2897 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2898 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2899 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2900 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
2901 break;
2902
2903 case 2:
2904 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2905 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2906 PIPE_CONFIG(ADDR_SURF_P2) |
2907 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2908 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2909 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2910 PIPE_CONFIG(ADDR_SURF_P2) |
2911 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2912 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2913 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2914 PIPE_CONFIG(ADDR_SURF_P2) |
2915 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2916 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2917 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2918 PIPE_CONFIG(ADDR_SURF_P2) |
2919 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2920 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2921 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2922 PIPE_CONFIG(ADDR_SURF_P2) |
2923 TILE_SPLIT(split_equal_to_row_size));
2924 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2925 PIPE_CONFIG(ADDR_SURF_P2) |
2926 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2927 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2928 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2929 PIPE_CONFIG(ADDR_SURF_P2) |
2930 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2931 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2932 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2933 PIPE_CONFIG(ADDR_SURF_P2) |
2934 TILE_SPLIT(split_equal_to_row_size));
2935 tile[8] = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2936 PIPE_CONFIG(ADDR_SURF_P2);
2937 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2938 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2939 PIPE_CONFIG(ADDR_SURF_P2));
2940 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2941 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2942 PIPE_CONFIG(ADDR_SURF_P2) |
2943 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2944 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2945 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2946 PIPE_CONFIG(ADDR_SURF_P2) |
2947 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2948 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2949 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2950 PIPE_CONFIG(ADDR_SURF_P2) |
2951 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2952 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2953 PIPE_CONFIG(ADDR_SURF_P2) |
2954 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2955 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2956 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2957 PIPE_CONFIG(ADDR_SURF_P2) |
2958 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2959 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2960 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2961 PIPE_CONFIG(ADDR_SURF_P2) |
2962 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2963 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2964 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2965 PIPE_CONFIG(ADDR_SURF_P2) |
2966 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2967 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2968 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2969 PIPE_CONFIG(ADDR_SURF_P2));
2970 tile[28] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2971 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2972 PIPE_CONFIG(ADDR_SURF_P2) |
2973 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2974 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2975 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2976 PIPE_CONFIG(ADDR_SURF_P2) |
2977 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2978 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2979 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2980 PIPE_CONFIG(ADDR_SURF_P2) |
2981 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2982
2983 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2984 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2985 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2986 NUM_BANKS(ADDR_SURF_16_BANK));
2987 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2988 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2989 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2990 NUM_BANKS(ADDR_SURF_16_BANK));
2991 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2992 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2993 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2994 NUM_BANKS(ADDR_SURF_16_BANK));
2995 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2996 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2997 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2998 NUM_BANKS(ADDR_SURF_16_BANK));
2999 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3000 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3001 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3002 NUM_BANKS(ADDR_SURF_16_BANK));
3003 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3004 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3005 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3006 NUM_BANKS(ADDR_SURF_16_BANK));
3007 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3008 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3009 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3010 NUM_BANKS(ADDR_SURF_8_BANK));
3011 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3012 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3013 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3014 NUM_BANKS(ADDR_SURF_16_BANK));
3015 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3016 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3017 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3018 NUM_BANKS(ADDR_SURF_16_BANK));
3019 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3020 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3021 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3022 NUM_BANKS(ADDR_SURF_16_BANK));
3023 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3024 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3025 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3026 NUM_BANKS(ADDR_SURF_16_BANK));
3027 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3028 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3029 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3030 NUM_BANKS(ADDR_SURF_16_BANK));
3031 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3032 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3033 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3034 NUM_BANKS(ADDR_SURF_16_BANK));
3035 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3036 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3037 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3038 NUM_BANKS(ADDR_SURF_8_BANK));
3039
3040 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
3041 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
3042 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
3043 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
3044 break;
3045
3046 default:
3047 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
3048 }
3049 }
3050
3051 /**
3052 * cik_select_se_sh - select which SE, SH to address
3053 *
3054 * @rdev: radeon_device pointer
3055 * @se_num: shader engine to address
3056 * @sh_num: sh block to address
3057 *
3058 * Select which SE, SH combinations to address. Certain
3059 * registers are instanced per SE or SH. 0xffffffff means
3060 * broadcast to all SEs or SHs (CIK).
3061 */
3062 static void cik_select_se_sh(struct radeon_device *rdev,
3063 u32 se_num, u32 sh_num)
3064 {
3065 u32 data = INSTANCE_BROADCAST_WRITES;
3066
3067 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
3068 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
3069 else if (se_num == 0xffffffff)
3070 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3071 else if (sh_num == 0xffffffff)
3072 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3073 else
3074 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3075 WREG32(GRBM_GFX_INDEX, data);
3076 }
3077
3078 /**
3079 * cik_create_bitmask - create a bitmask
3080 *
3081 * @bit_width: length of the mask
3082 *
3083 * create a variable length bit mask (CIK).
3084 * Returns the bitmask.
3085 */
3086 static u32 cik_create_bitmask(u32 bit_width)
3087 {
3088 u32 i, mask = 0;
3089
3090 for (i = 0; i < bit_width; i++) {
3091 mask <<= 1;
3092 mask |= 1;
3093 }
3094 return mask;
3095 }
3096
3097 /**
3098 * cik_get_rb_disabled - computes the mask of disabled RBs
3099 *
3100 * @rdev: radeon_device pointer
3101 * @max_rb_num: max RBs (render backends) for the asic
3102 * @se_num: number of SEs (shader engines) for the asic
3103 * @sh_per_se: number of SH blocks per SE for the asic
3104 *
3105 * Calculates the bitmask of disabled RBs (CIK).
3106 * Returns the disabled RB bitmask.
3107 */
3108 static u32 cik_get_rb_disabled(struct radeon_device *rdev,
3109 u32 max_rb_num_per_se,
3110 u32 sh_per_se)
3111 {
3112 u32 data, mask;
3113
3114 data = RREG32(CC_RB_BACKEND_DISABLE);
3115 if (data & 1)
3116 data &= BACKEND_DISABLE_MASK;
3117 else
3118 data = 0;
3119 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3120
3121 data >>= BACKEND_DISABLE_SHIFT;
3122
3123 mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
3124
3125 return data & mask;
3126 }
3127
3128 /**
3129 * cik_setup_rb - setup the RBs on the asic
3130 *
3131 * @rdev: radeon_device pointer
3132 * @se_num: number of SEs (shader engines) for the asic
3133 * @sh_per_se: number of SH blocks per SE for the asic
3134 * @max_rb_num: max RBs (render backends) for the asic
3135 *
3136 * Configures per-SE/SH RB registers (CIK).
3137 */
3138 static void cik_setup_rb(struct radeon_device *rdev,
3139 u32 se_num, u32 sh_per_se,
3140 u32 max_rb_num_per_se)
3141 {
3142 int i, j;
3143 u32 data, mask;
3144 u32 disabled_rbs = 0;
3145 u32 enabled_rbs = 0;
3146
3147 mutex_lock(&rdev->grbm_idx_mutex);
3148 for (i = 0; i < se_num; i++) {
3149 for (j = 0; j < sh_per_se; j++) {
3150 cik_select_se_sh(rdev, i, j);
3151 data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
3152 if (rdev->family == CHIP_HAWAII)
3153 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3154 else
3155 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
3156 }
3157 }
3158 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3159 mutex_unlock(&rdev->grbm_idx_mutex);
3160
3161 mask = 1;
3162 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
3163 if (!(disabled_rbs & mask))
3164 enabled_rbs |= mask;
3165 mask <<= 1;
3166 }
3167
3168 rdev->config.cik.backend_enable_mask = enabled_rbs;
3169
3170 mutex_lock(&rdev->grbm_idx_mutex);
3171 for (i = 0; i < se_num; i++) {
3172 cik_select_se_sh(rdev, i, 0xffffffff);
3173 data = 0;
3174 for (j = 0; j < sh_per_se; j++) {
3175 switch (enabled_rbs & 3) {
3176 case 0:
3177 if (j == 0)
3178 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3179 else
3180 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3181 break;
3182 case 1:
3183 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3184 break;
3185 case 2:
3186 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3187 break;
3188 case 3:
3189 default:
3190 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3191 break;
3192 }
3193 enabled_rbs >>= 2;
3194 }
3195 WREG32(PA_SC_RASTER_CONFIG, data);
3196 }
3197 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3198 mutex_unlock(&rdev->grbm_idx_mutex);
3199 }
3200
3201 /**
3202 * cik_gpu_init - setup the 3D engine
3203 *
3204 * @rdev: radeon_device pointer
3205 *
3206 * Configures the 3D engine and tiling configuration
3207 * registers so that the 3D engine is usable.
3208 */
3209 static void cik_gpu_init(struct radeon_device *rdev)
3210 {
3211 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3212 u32 mc_shared_chmap, mc_arb_ramcfg;
3213 u32 hdp_host_path_cntl;
3214 u32 tmp;
3215 int i, j;
3216
3217 switch (rdev->family) {
3218 case CHIP_BONAIRE:
3219 rdev->config.cik.max_shader_engines = 2;
3220 rdev->config.cik.max_tile_pipes = 4;
3221 rdev->config.cik.max_cu_per_sh = 7;
3222 rdev->config.cik.max_sh_per_se = 1;
3223 rdev->config.cik.max_backends_per_se = 2;
3224 rdev->config.cik.max_texture_channel_caches = 4;
3225 rdev->config.cik.max_gprs = 256;
3226 rdev->config.cik.max_gs_threads = 32;
3227 rdev->config.cik.max_hw_contexts = 8;
3228
3229 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3230 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3231 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3232 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3233 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3234 break;
3235 case CHIP_HAWAII:
3236 rdev->config.cik.max_shader_engines = 4;
3237 rdev->config.cik.max_tile_pipes = 16;
3238 rdev->config.cik.max_cu_per_sh = 11;
3239 rdev->config.cik.max_sh_per_se = 1;
3240 rdev->config.cik.max_backends_per_se = 4;
3241 rdev->config.cik.max_texture_channel_caches = 16;
3242 rdev->config.cik.max_gprs = 256;
3243 rdev->config.cik.max_gs_threads = 32;
3244 rdev->config.cik.max_hw_contexts = 8;
3245
3246 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3247 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3248 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3249 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3250 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3251 break;
3252 case CHIP_KAVERI:
3253 rdev->config.cik.max_shader_engines = 1;
3254 rdev->config.cik.max_tile_pipes = 4;
3255 if ((rdev->pdev->device == 0x1304) ||
3256 (rdev->pdev->device == 0x1305) ||
3257 (rdev->pdev->device == 0x130C) ||
3258 (rdev->pdev->device == 0x130F) ||
3259 (rdev->pdev->device == 0x1310) ||
3260 (rdev->pdev->device == 0x1311) ||
3261 (rdev->pdev->device == 0x131C)) {
3262 rdev->config.cik.max_cu_per_sh = 8;
3263 rdev->config.cik.max_backends_per_se = 2;
3264 } else if ((rdev->pdev->device == 0x1309) ||
3265 (rdev->pdev->device == 0x130A) ||
3266 (rdev->pdev->device == 0x130D) ||
3267 (rdev->pdev->device == 0x1313) ||
3268 (rdev->pdev->device == 0x131D)) {
3269 rdev->config.cik.max_cu_per_sh = 6;
3270 rdev->config.cik.max_backends_per_se = 2;
3271 } else if ((rdev->pdev->device == 0x1306) ||
3272 (rdev->pdev->device == 0x1307) ||
3273 (rdev->pdev->device == 0x130B) ||
3274 (rdev->pdev->device == 0x130E) ||
3275 (rdev->pdev->device == 0x1315) ||
3276 (rdev->pdev->device == 0x1318) ||
3277 (rdev->pdev->device == 0x131B)) {
3278 rdev->config.cik.max_cu_per_sh = 4;
3279 rdev->config.cik.max_backends_per_se = 1;
3280 } else {
3281 rdev->config.cik.max_cu_per_sh = 3;
3282 rdev->config.cik.max_backends_per_se = 1;
3283 }
3284 rdev->config.cik.max_sh_per_se = 1;
3285 rdev->config.cik.max_texture_channel_caches = 4;
3286 rdev->config.cik.max_gprs = 256;
3287 rdev->config.cik.max_gs_threads = 16;
3288 rdev->config.cik.max_hw_contexts = 8;
3289
3290 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3291 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3292 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3293 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3294 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3295 break;
3296 case CHIP_KABINI:
3297 case CHIP_MULLINS:
3298 default:
3299 rdev->config.cik.max_shader_engines = 1;
3300 rdev->config.cik.max_tile_pipes = 2;
3301 rdev->config.cik.max_cu_per_sh = 2;
3302 rdev->config.cik.max_sh_per_se = 1;
3303 rdev->config.cik.max_backends_per_se = 1;
3304 rdev->config.cik.max_texture_channel_caches = 2;
3305 rdev->config.cik.max_gprs = 256;
3306 rdev->config.cik.max_gs_threads = 16;
3307 rdev->config.cik.max_hw_contexts = 8;
3308
3309 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3310 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3311 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3312 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3313 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3314 break;
3315 }
3316
3317 /* Initialize HDP */
3318 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3319 WREG32((0x2c14 + j), 0x00000000);
3320 WREG32((0x2c18 + j), 0x00000000);
3321 WREG32((0x2c1c + j), 0x00000000);
3322 WREG32((0x2c20 + j), 0x00000000);
3323 WREG32((0x2c24 + j), 0x00000000);
3324 }
3325
3326 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3327 WREG32(SRBM_INT_CNTL, 0x1);
3328 WREG32(SRBM_INT_ACK, 0x1);
3329
3330 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3331
3332 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3333 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3334
3335 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3336 rdev->config.cik.mem_max_burst_length_bytes = 256;
3337 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3338 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3339 if (rdev->config.cik.mem_row_size_in_kb > 4)
3340 rdev->config.cik.mem_row_size_in_kb = 4;
3341 /* XXX use MC settings? */
3342 rdev->config.cik.shader_engine_tile_size = 32;
3343 rdev->config.cik.num_gpus = 1;
3344 rdev->config.cik.multi_gpu_tile_size = 64;
3345
3346 /* fix up row size */
3347 gb_addr_config &= ~ROW_SIZE_MASK;
3348 switch (rdev->config.cik.mem_row_size_in_kb) {
3349 case 1:
3350 default:
3351 gb_addr_config |= ROW_SIZE(0);
3352 break;
3353 case 2:
3354 gb_addr_config |= ROW_SIZE(1);
3355 break;
3356 case 4:
3357 gb_addr_config |= ROW_SIZE(2);
3358 break;
3359 }
3360
3361 /* setup tiling info dword. gb_addr_config is not adequate since it does
3362 * not have bank info, so create a custom tiling dword.
3363 * bits 3:0 num_pipes
3364 * bits 7:4 num_banks
3365 * bits 11:8 group_size
3366 * bits 15:12 row_size
3367 */
3368 rdev->config.cik.tile_config = 0;
3369 switch (rdev->config.cik.num_tile_pipes) {
3370 case 1:
3371 rdev->config.cik.tile_config |= (0 << 0);
3372 break;
3373 case 2:
3374 rdev->config.cik.tile_config |= (1 << 0);
3375 break;
3376 case 4:
3377 rdev->config.cik.tile_config |= (2 << 0);
3378 break;
3379 case 8:
3380 default:
3381 /* XXX what about 12? */
3382 rdev->config.cik.tile_config |= (3 << 0);
3383 break;
3384 }
3385 rdev->config.cik.tile_config |=
3386 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
3387 rdev->config.cik.tile_config |=
3388 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3389 rdev->config.cik.tile_config |=
3390 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3391
3392 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3393 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3394 WREG32(DMIF_ADDR_CALC, gb_addr_config);
3395 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3396 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
3397 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3398 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3399 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3400
3401 cik_tiling_mode_table_init(rdev);
3402
3403 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3404 rdev->config.cik.max_sh_per_se,
3405 rdev->config.cik.max_backends_per_se);
3406
3407 rdev->config.cik.active_cus = 0;
3408 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
3409 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
3410 rdev->config.cik.active_cus +=
3411 hweight32(cik_get_cu_active_bitmap(rdev, i, j));
3412 }
3413 }
3414
3415 /* set HW defaults for 3D engine */
3416 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3417
3418 mutex_lock(&rdev->grbm_idx_mutex);
3419 /*
3420 * making sure that the following register writes will be broadcasted
3421 * to all the shaders
3422 */
3423 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3424 WREG32(SX_DEBUG_1, 0x20);
3425
3426 WREG32(TA_CNTL_AUX, 0x00010000);
3427
3428 tmp = RREG32(SPI_CONFIG_CNTL);
3429 tmp |= 0x03000000;
3430 WREG32(SPI_CONFIG_CNTL, tmp);
3431
3432 WREG32(SQ_CONFIG, 1);
3433
3434 WREG32(DB_DEBUG, 0);
3435
3436 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3437 tmp |= 0x00000400;
3438 WREG32(DB_DEBUG2, tmp);
3439
3440 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3441 tmp |= 0x00020200;
3442 WREG32(DB_DEBUG3, tmp);
3443
3444 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3445 tmp |= 0x00018208;
3446 WREG32(CB_HW_CONTROL, tmp);
3447
3448 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3449
3450 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3451 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3452 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3453 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3454
3455 WREG32(VGT_NUM_INSTANCES, 1);
3456
3457 WREG32(CP_PERFMON_CNTL, 0);
3458
3459 WREG32(SQ_CONFIG, 0);
3460
3461 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3462 FORCE_EOV_MAX_REZ_CNT(255)));
3463
3464 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3465 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3466
3467 WREG32(VGT_GS_VERTEX_REUSE, 16);
3468 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3469
3470 tmp = RREG32(HDP_MISC_CNTL);
3471 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3472 WREG32(HDP_MISC_CNTL, tmp);
3473
3474 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3475 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3476
3477 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3478 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3479 mutex_unlock(&rdev->grbm_idx_mutex);
3480
3481 udelay(50);
3482 }
3483
3484 /*
3485 * GPU scratch registers helpers function.
3486 */
3487 /**
3488 * cik_scratch_init - setup driver info for CP scratch regs
3489 *
3490 * @rdev: radeon_device pointer
3491 *
3492 * Set up the number and offset of the CP scratch registers.
3493 * NOTE: use of CP scratch registers is a legacy inferface and
3494 * is not used by default on newer asics (r6xx+). On newer asics,
3495 * memory buffers are used for fences rather than scratch regs.
3496 */
3497 static void cik_scratch_init(struct radeon_device *rdev)
3498 {
3499 int i;
3500
3501 rdev->scratch.num_reg = 7;
3502 rdev->scratch.reg_base = SCRATCH_REG0;
3503 for (i = 0; i < rdev->scratch.num_reg; i++) {
3504 rdev->scratch.free[i] = true;
3505 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3506 }
3507 }
3508
3509 /**
3510 * cik_ring_test - basic gfx ring test
3511 *
3512 * @rdev: radeon_device pointer
3513 * @ring: radeon_ring structure holding ring information
3514 *
3515 * Allocate a scratch register and write to it using the gfx ring (CIK).
3516 * Provides a basic gfx ring test to verify that the ring is working.
3517 * Used by cik_cp_gfx_resume();
3518 * Returns 0 on success, error on failure.
3519 */
3520 int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3521 {
3522 uint32_t scratch;
3523 uint32_t tmp = 0;
3524 unsigned i;
3525 int r;
3526
3527 r = radeon_scratch_get(rdev, &scratch);
3528 if (r) {
3529 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3530 return r;
3531 }
3532 WREG32(scratch, 0xCAFEDEAD);
3533 r = radeon_ring_lock(rdev, ring, 3);
3534 if (r) {
3535 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3536 radeon_scratch_free(rdev, scratch);
3537 return r;
3538 }
3539 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3540 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3541 radeon_ring_write(ring, 0xDEADBEEF);
3542 radeon_ring_unlock_commit(rdev, ring, false);
3543
3544 for (i = 0; i < rdev->usec_timeout; i++) {
3545 tmp = RREG32(scratch);
3546 if (tmp == 0xDEADBEEF)
3547 break;
3548 DRM_UDELAY(1);
3549 }
3550 if (i < rdev->usec_timeout) {
3551 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3552 } else {
3553 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3554 ring->idx, scratch, tmp);
3555 r = -EINVAL;
3556 }
3557 radeon_scratch_free(rdev, scratch);
3558 return r;
3559 }
3560
3561 /**
3562 * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3563 *
3564 * @rdev: radeon_device pointer
3565 * @ridx: radeon ring index
3566 *
3567 * Emits an hdp flush on the cp.
3568 */
3569 static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,
3570 int ridx)
3571 {
3572 struct radeon_ring *ring = &rdev->ring[ridx];
3573 u32 ref_and_mask;
3574
3575 switch (ring->idx) {
3576 case CAYMAN_RING_TYPE_CP1_INDEX:
3577 case CAYMAN_RING_TYPE_CP2_INDEX:
3578 default:
3579 switch (ring->me) {
3580 case 0:
3581 ref_and_mask = CP2 << ring->pipe;
3582 break;
3583 case 1:
3584 ref_and_mask = CP6 << ring->pipe;
3585 break;
3586 default:
3587 return;
3588 }
3589 break;
3590 case RADEON_RING_TYPE_GFX_INDEX:
3591 ref_and_mask = CP0;
3592 break;
3593 }
3594
3595 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3596 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
3597 WAIT_REG_MEM_FUNCTION(3) | /* == */
3598 WAIT_REG_MEM_ENGINE(1))); /* pfp */
3599 radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2);
3600 radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2);
3601 radeon_ring_write(ring, ref_and_mask);
3602 radeon_ring_write(ring, ref_and_mask);
3603 radeon_ring_write(ring, 0x20); /* poll interval */
3604 }
3605
3606 /**
3607 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
3608 *
3609 * @rdev: radeon_device pointer
3610 * @fence: radeon fence object
3611 *
3612 * Emits a fence sequnce number on the gfx ring and flushes
3613 * GPU caches.
3614 */
3615 void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3616 struct radeon_fence *fence)
3617 {
3618 struct radeon_ring *ring = &rdev->ring[fence->ring];
3619 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3620
3621 /* Workaround for cache flush problems. First send a dummy EOP
3622 * event down the pipe with seq one below.
3623 */
3624 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3625 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3626 EOP_TC_ACTION_EN |
3627 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3628 EVENT_INDEX(5)));
3629 radeon_ring_write(ring, addr & 0xfffffffc);
3630 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) |
3631 DATA_SEL(1) | INT_SEL(0));
3632 radeon_ring_write(ring, fence->seq - 1);
3633 radeon_ring_write(ring, 0);
3634
3635 /* Then send the real EOP event down the pipe. */
3636 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3637 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3638 EOP_TC_ACTION_EN |
3639 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3640 EVENT_INDEX(5)));
3641 radeon_ring_write(ring, addr & 0xfffffffc);
3642 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3643 radeon_ring_write(ring, fence->seq);
3644 radeon_ring_write(ring, 0);
3645 }
3646
3647 /**
3648 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3649 *
3650 * @rdev: radeon_device pointer
3651 * @fence: radeon fence object
3652 *
3653 * Emits a fence sequnce number on the compute ring and flushes
3654 * GPU caches.
3655 */
3656 void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3657 struct radeon_fence *fence)
3658 {
3659 struct radeon_ring *ring = &rdev->ring[fence->ring];
3660 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3661
3662 /* RELEASE_MEM - flush caches, send int */
3663 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3664 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3665 EOP_TC_ACTION_EN |
3666 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3667 EVENT_INDEX(5)));
3668 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3669 radeon_ring_write(ring, addr & 0xfffffffc);
3670 radeon_ring_write(ring, upper_32_bits(addr));
3671 radeon_ring_write(ring, fence->seq);
3672 radeon_ring_write(ring, 0);
3673 }
3674
3675 /**
3676 * cik_semaphore_ring_emit - emit a semaphore on the CP ring
3677 *
3678 * @rdev: radeon_device pointer
3679 * @ring: radeon ring buffer object
3680 * @semaphore: radeon semaphore object
3681 * @emit_wait: Is this a sempahore wait?
3682 *
3683 * Emits a semaphore signal/wait packet to the CP ring and prevents the PFP
3684 * from running ahead of semaphore waits.
3685 */
3686 bool cik_semaphore_ring_emit(struct radeon_device *rdev,
3687 struct radeon_ring *ring,
3688 struct radeon_semaphore *semaphore,
3689 bool emit_wait)
3690 {
3691 uint64_t addr = semaphore->gpu_addr;
3692 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3693
3694 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3695 radeon_ring_write(ring, lower_32_bits(addr));
3696 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
3697
3698 if (emit_wait && ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
3699 /* Prevent the PFP from running ahead of the semaphore wait */
3700 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
3701 radeon_ring_write(ring, 0x0);
3702 }
3703
3704 return true;
3705 }
3706
3707 /**
3708 * cik_copy_cpdma - copy pages using the CP DMA engine
3709 *
3710 * @rdev: radeon_device pointer
3711 * @src_offset: src GPU address
3712 * @dst_offset: dst GPU address
3713 * @num_gpu_pages: number of GPU pages to xfer
3714 * @resv: reservation object to sync to
3715 *
3716 * Copy GPU paging using the CP DMA engine (CIK+).
3717 * Used by the radeon ttm implementation to move pages if
3718 * registered as the asic copy callback.
3719 */
3720 struct radeon_fence *cik_copy_cpdma(struct radeon_device *rdev,
3721 uint64_t src_offset, uint64_t dst_offset,
3722 unsigned num_gpu_pages,
3723 struct reservation_object *resv)
3724 {
3725 struct radeon_fence *fence;
3726 struct radeon_sync sync;
3727 int ring_index = rdev->asic->copy.blit_ring_index;
3728 struct radeon_ring *ring = &rdev->ring[ring_index];
3729 u32 size_in_bytes, cur_size_in_bytes, control;
3730 int i, num_loops;
3731 int r = 0;
3732
3733 radeon_sync_create(&sync);
3734
3735 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3736 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3737 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3738 if (r) {
3739 DRM_ERROR("radeon: moving bo (%d).\n", r);
3740 radeon_sync_free(rdev, &sync, NULL);
3741 return ERR_PTR(r);
3742 }
3743
3744 radeon_sync_resv(rdev, &sync, resv, false);
3745 radeon_sync_rings(rdev, &sync, ring->idx);
3746
3747 for (i = 0; i < num_loops; i++) {
3748 cur_size_in_bytes = size_in_bytes;
3749 if (cur_size_in_bytes > 0x1fffff)
3750 cur_size_in_bytes = 0x1fffff;
3751 size_in_bytes -= cur_size_in_bytes;
3752 control = 0;
3753 if (size_in_bytes == 0)
3754 control |= PACKET3_DMA_DATA_CP_SYNC;
3755 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
3756 radeon_ring_write(ring, control);
3757 radeon_ring_write(ring, lower_32_bits(src_offset));
3758 radeon_ring_write(ring, upper_32_bits(src_offset));
3759 radeon_ring_write(ring, lower_32_bits(dst_offset));
3760 radeon_ring_write(ring, upper_32_bits(dst_offset));
3761 radeon_ring_write(ring, cur_size_in_bytes);
3762 src_offset += cur_size_in_bytes;
3763 dst_offset += cur_size_in_bytes;
3764 }
3765
3766 r = radeon_fence_emit(rdev, &fence, ring->idx);
3767 if (r) {
3768 radeon_ring_unlock_undo(rdev, ring);
3769 radeon_sync_free(rdev, &sync, NULL);
3770 return ERR_PTR(r);
3771 }
3772
3773 radeon_ring_unlock_commit(rdev, ring, false);
3774 radeon_sync_free(rdev, &sync, fence);
3775
3776 return fence;
3777 }
3778
3779 /*
3780 * IB stuff
3781 */
3782 /**
3783 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3784 *
3785 * @rdev: radeon_device pointer
3786 * @ib: radeon indirect buffer object
3787 *
3788 * Emits a DE (drawing engine) or CE (constant engine) IB
3789 * on the gfx ring. IBs are usually generated by userspace
3790 * acceleration drivers and submitted to the kernel for
3791 * scheduling on the ring. This function schedules the IB
3792 * on the gfx ring for execution by the GPU.
3793 */
3794 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3795 {
3796 struct radeon_ring *ring = &rdev->ring[ib->ring];
3797 unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
3798 u32 header, control = INDIRECT_BUFFER_VALID;
3799
3800 if (ib->is_const_ib) {
3801 /* set switch buffer packet before const IB */
3802 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3803 radeon_ring_write(ring, 0);
3804
3805 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3806 } else {
3807 u32 next_rptr;
3808 if (ring->rptr_save_reg) {
3809 next_rptr = ring->wptr + 3 + 4;
3810 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3811 radeon_ring_write(ring, ((ring->rptr_save_reg -
3812 PACKET3_SET_UCONFIG_REG_START) >> 2));
3813 radeon_ring_write(ring, next_rptr);
3814 } else if (rdev->wb.enabled) {
3815 next_rptr = ring->wptr + 5 + 4;
3816 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3817 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
3818 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3819 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
3820 radeon_ring_write(ring, next_rptr);
3821 }
3822
3823 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3824 }
3825
3826 control |= ib->length_dw | (vm_id << 24);
3827
3828 radeon_ring_write(ring, header);
3829 radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFFC));
3830 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3831 radeon_ring_write(ring, control);
3832 }
3833
3834 /**
3835 * cik_ib_test - basic gfx ring IB test
3836 *
3837 * @rdev: radeon_device pointer
3838 * @ring: radeon_ring structure holding ring information
3839 *
3840 * Allocate an IB and execute it on the gfx ring (CIK).
3841 * Provides a basic gfx ring test to verify that IBs are working.
3842 * Returns 0 on success, error on failure.
3843 */
3844 int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3845 {
3846 struct radeon_ib ib;
3847 uint32_t scratch;
3848 uint32_t tmp = 0;
3849 unsigned i;
3850 int r;
3851
3852 r = radeon_scratch_get(rdev, &scratch);
3853 if (r) {
3854 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3855 return r;
3856 }
3857 WREG32(scratch, 0xCAFEDEAD);
3858 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3859 if (r) {
3860 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
3861 radeon_scratch_free(rdev, scratch);
3862 return r;
3863 }
3864 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
3865 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
3866 ib.ptr[2] = 0xDEADBEEF;
3867 ib.length_dw = 3;
3868 r = radeon_ib_schedule(rdev, &ib, NULL, false);
3869 if (r) {
3870 radeon_scratch_free(rdev, scratch);
3871 radeon_ib_free(rdev, &ib);
3872 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3873 return r;
3874 }
3875 r = radeon_fence_wait_timeout(ib.fence, false, usecs_to_jiffies(
3876 RADEON_USEC_IB_TEST_TIMEOUT));
3877 if (r < 0) {
3878 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3879 radeon_scratch_free(rdev, scratch);
3880 radeon_ib_free(rdev, &ib);
3881 return r;
3882 } else if (r == 0) {
3883 DRM_ERROR("radeon: fence wait timed out.\n");
3884 radeon_scratch_free(rdev, scratch);
3885 radeon_ib_free(rdev, &ib);
3886 return -ETIMEDOUT;
3887 }
3888 r = 0;
3889 for (i = 0; i < rdev->usec_timeout; i++) {
3890 tmp = RREG32(scratch);
3891 if (tmp == 0xDEADBEEF)
3892 break;
3893 DRM_UDELAY(1);
3894 }
3895 if (i < rdev->usec_timeout) {
3896 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3897 } else {
3898 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3899 scratch, tmp);
3900 r = -EINVAL;
3901 }
3902 radeon_scratch_free(rdev, scratch);
3903 radeon_ib_free(rdev, &ib);
3904 return r;
3905 }
3906
3907 /*
3908 * CP.
3909 * On CIK, gfx and compute now have independant command processors.
3910 *
3911 * GFX
3912 * Gfx consists of a single ring and can process both gfx jobs and
3913 * compute jobs. The gfx CP consists of three microengines (ME):
3914 * PFP - Pre-Fetch Parser
3915 * ME - Micro Engine
3916 * CE - Constant Engine
3917 * The PFP and ME make up what is considered the Drawing Engine (DE).
3918 * The CE is an asynchronous engine used for updating buffer desciptors
3919 * used by the DE so that they can be loaded into cache in parallel
3920 * while the DE is processing state update packets.
3921 *
3922 * Compute
3923 * The compute CP consists of two microengines (ME):
3924 * MEC1 - Compute MicroEngine 1
3925 * MEC2 - Compute MicroEngine 2
3926 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3927 * The queues are exposed to userspace and are programmed directly
3928 * by the compute runtime.
3929 */
3930 /**
3931 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3932 *
3933 * @rdev: radeon_device pointer
3934 * @enable: enable or disable the MEs
3935 *
3936 * Halts or unhalts the gfx MEs.
3937 */
3938 static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
3939 {
3940 if (enable)
3941 WREG32(CP_ME_CNTL, 0);
3942 else {
3943 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3944 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3945 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3946 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3947 }
3948 udelay(50);
3949 }
3950
3951 /**
3952 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3953 *
3954 * @rdev: radeon_device pointer
3955 *
3956 * Loads the gfx PFP, ME, and CE ucode.
3957 * Returns 0 for success, -EINVAL if the ucode is not available.
3958 */
3959 static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
3960 {
3961 int i;
3962
3963 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3964 return -EINVAL;
3965
3966 cik_cp_gfx_enable(rdev, false);
3967
3968 if (rdev->new_fw) {
3969 const struct gfx_firmware_header_v1_0 *pfp_hdr =
3970 (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
3971 const struct gfx_firmware_header_v1_0 *ce_hdr =
3972 (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
3973 const struct gfx_firmware_header_v1_0 *me_hdr =
3974 (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
3975 const __le32 *fw_data;
3976 u32 fw_size;
3977
3978 radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
3979 radeon_ucode_print_gfx_hdr(&ce_hdr->header);
3980 radeon_ucode_print_gfx_hdr(&me_hdr->header);
3981
3982 /* PFP */
3983 fw_data = (const __le32 *)
3984 (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3985 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3986 WREG32(CP_PFP_UCODE_ADDR, 0);
3987 for (i = 0; i < fw_size; i++)
3988 WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
3989 WREG32(CP_PFP_UCODE_ADDR, le32_to_cpu(pfp_hdr->header.ucode_version));
3990
3991 /* CE */
3992 fw_data = (const __le32 *)
3993 (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3994 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3995 WREG32(CP_CE_UCODE_ADDR, 0);
3996 for (i = 0; i < fw_size; i++)
3997 WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
3998 WREG32(CP_CE_UCODE_ADDR, le32_to_cpu(ce_hdr->header.ucode_version));
3999
4000 /* ME */
4001 fw_data = (const __be32 *)
4002 (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
4003 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
4004 WREG32(CP_ME_RAM_WADDR, 0);
4005 for (i = 0; i < fw_size; i++)
4006 WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
4007 WREG32(CP_ME_RAM_WADDR, le32_to_cpu(me_hdr->header.ucode_version));
4008 WREG32(CP_ME_RAM_RADDR, le32_to_cpu(me_hdr->header.ucode_version));
4009 } else {
4010 const __be32 *fw_data;
4011
4012 /* PFP */
4013 fw_data = (const __be32 *)rdev->pfp_fw->data;
4014 WREG32(CP_PFP_UCODE_ADDR, 0);
4015 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
4016 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
4017 WREG32(CP_PFP_UCODE_ADDR, 0);
4018
4019 /* CE */
4020 fw_data = (const __be32 *)rdev->ce_fw->data;
4021 WREG32(CP_CE_UCODE_ADDR, 0);
4022 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
4023 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
4024 WREG32(CP_CE_UCODE_ADDR, 0);
4025
4026 /* ME */
4027 fw_data = (const __be32 *)rdev->me_fw->data;
4028 WREG32(CP_ME_RAM_WADDR, 0);
4029 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
4030 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
4031 WREG32(CP_ME_RAM_WADDR, 0);
4032 }
4033
4034 return 0;
4035 }
4036
4037 /**
4038 * cik_cp_gfx_start - start the gfx ring
4039 *
4040 * @rdev: radeon_device pointer
4041 *
4042 * Enables the ring and loads the clear state context and other
4043 * packets required to init the ring.
4044 * Returns 0 for success, error for failure.
4045 */
4046 static int cik_cp_gfx_start(struct radeon_device *rdev)
4047 {
4048 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4049 int r, i;
4050
4051 /* init the CP */
4052 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
4053 WREG32(CP_ENDIAN_SWAP, 0);
4054 WREG32(CP_DEVICE_ID, 1);
4055
4056 cik_cp_gfx_enable(rdev, true);
4057
4058 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
4059 if (r) {
4060 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
4061 return r;
4062 }
4063
4064 /* init the CE partitions. CE only used for gfx on CIK */
4065 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
4066 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
4067 radeon_ring_write(ring, 0x8000);
4068 radeon_ring_write(ring, 0x8000);
4069
4070 /* setup clear context state */
4071 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4072 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
4073
4074 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
4075 radeon_ring_write(ring, 0x80000000);
4076 radeon_ring_write(ring, 0x80000000);
4077
4078 for (i = 0; i < cik_default_size; i++)
4079 radeon_ring_write(ring, cik_default_state[i]);
4080
4081 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4082 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
4083
4084 /* set clear context state */
4085 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
4086 radeon_ring_write(ring, 0);
4087
4088 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
4089 radeon_ring_write(ring, 0x00000316);
4090 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
4091 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
4092
4093 radeon_ring_unlock_commit(rdev, ring, false);
4094
4095 return 0;
4096 }
4097
4098 /**
4099 * cik_cp_gfx_fini - stop the gfx ring
4100 *
4101 * @rdev: radeon_device pointer
4102 *
4103 * Stop the gfx ring and tear down the driver ring
4104 * info.
4105 */
4106 static void cik_cp_gfx_fini(struct radeon_device *rdev)
4107 {
4108 cik_cp_gfx_enable(rdev, false);
4109 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4110 }
4111
4112 /**
4113 * cik_cp_gfx_resume - setup the gfx ring buffer registers
4114 *
4115 * @rdev: radeon_device pointer
4116 *
4117 * Program the location and size of the gfx ring buffer
4118 * and test it to make sure it's working.
4119 * Returns 0 for success, error for failure.
4120 */
4121 static int cik_cp_gfx_resume(struct radeon_device *rdev)
4122 {
4123 struct radeon_ring *ring;
4124 u32 tmp;
4125 u32 rb_bufsz;
4126 u64 rb_addr;
4127 int r;
4128
4129 WREG32(CP_SEM_WAIT_TIMER, 0x0);
4130 if (rdev->family != CHIP_HAWAII)
4131 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
4132
4133 /* Set the write pointer delay */
4134 WREG32(CP_RB_WPTR_DELAY, 0);
4135
4136 /* set the RB to use vmid 0 */
4137 WREG32(CP_RB_VMID, 0);
4138
4139 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
4140
4141 /* ring 0 - compute and gfx */
4142 /* Set ring buffer size */
4143 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4144 rb_bufsz = order_base_2(ring->ring_size / 8);
4145 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
4146 #ifdef __BIG_ENDIAN
4147 tmp |= BUF_SWAP_32BIT;
4148 #endif
4149 WREG32(CP_RB0_CNTL, tmp);
4150
4151 /* Initialize the ring buffer's read and write pointers */
4152 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
4153 ring->wptr = 0;
4154 WREG32(CP_RB0_WPTR, ring->wptr);
4155
4156 /* set the wb address wether it's enabled or not */
4157 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
4158 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
4159
4160 /* scratch register shadowing is no longer supported */
4161 WREG32(SCRATCH_UMSK, 0);
4162
4163 if (!rdev->wb.enabled)
4164 tmp |= RB_NO_UPDATE;
4165
4166 mdelay(1);
4167 WREG32(CP_RB0_CNTL, tmp);
4168
4169 rb_addr = ring->gpu_addr >> 8;
4170 WREG32(CP_RB0_BASE, rb_addr);
4171 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4172
4173 /* start the ring */
4174 cik_cp_gfx_start(rdev);
4175 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4176 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4177 if (r) {
4178 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4179 return r;
4180 }
4181
4182 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4183 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
4184
4185 return 0;
4186 }
4187
4188 u32 cik_gfx_get_rptr(struct radeon_device *rdev,
4189 struct radeon_ring *ring)
4190 {
4191 u32 rptr;
4192
4193 if (rdev->wb.enabled)
4194 rptr = rdev->wb.wb[ring->rptr_offs/4];
4195 else
4196 rptr = RREG32(CP_RB0_RPTR);
4197
4198 return rptr;
4199 }
4200
4201 u32 cik_gfx_get_wptr(struct radeon_device *rdev,
4202 struct radeon_ring *ring)
4203 {
4204 u32 wptr;
4205
4206 wptr = RREG32(CP_RB0_WPTR);
4207
4208 return wptr;
4209 }
4210
4211 void cik_gfx_set_wptr(struct radeon_device *rdev,
4212 struct radeon_ring *ring)
4213 {
4214 WREG32(CP_RB0_WPTR, ring->wptr);
4215 (void)RREG32(CP_RB0_WPTR);
4216 }
4217
4218 u32 cik_compute_get_rptr(struct radeon_device *rdev,
4219 struct radeon_ring *ring)
4220 {
4221 u32 rptr;
4222
4223 if (rdev->wb.enabled) {
4224 rptr = rdev->wb.wb[ring->rptr_offs/4];
4225 } else {
4226 mutex_lock(&rdev->srbm_mutex);
4227 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4228 rptr = RREG32(CP_HQD_PQ_RPTR);
4229 cik_srbm_select(rdev, 0, 0, 0, 0);
4230 mutex_unlock(&rdev->srbm_mutex);
4231 }
4232
4233 return rptr;
4234 }
4235
4236 u32 cik_compute_get_wptr(struct radeon_device *rdev,
4237 struct radeon_ring *ring)
4238 {
4239 u32 wptr;
4240
4241 if (rdev->wb.enabled) {
4242 /* XXX check if swapping is necessary on BE */
4243 wptr = rdev->wb.wb[ring->wptr_offs/4];
4244 } else {
4245 mutex_lock(&rdev->srbm_mutex);
4246 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4247 wptr = RREG32(CP_HQD_PQ_WPTR);
4248 cik_srbm_select(rdev, 0, 0, 0, 0);
4249 mutex_unlock(&rdev->srbm_mutex);
4250 }
4251
4252 return wptr;
4253 }
4254
4255 void cik_compute_set_wptr(struct radeon_device *rdev,
4256 struct radeon_ring *ring)
4257 {
4258 /* XXX check if swapping is necessary on BE */
4259 rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;
4260 WDOORBELL32(ring->doorbell_index, ring->wptr);
4261 }
4262
4263 static void cik_compute_stop(struct radeon_device *rdev,
4264 struct radeon_ring *ring)
4265 {
4266 u32 j, tmp;
4267
4268 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4269 /* Disable wptr polling. */
4270 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4271 tmp &= ~WPTR_POLL_EN;
4272 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4273 /* Disable HQD. */
4274 if (RREG32(CP_HQD_ACTIVE) & 1) {
4275 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4276 for (j = 0; j < rdev->usec_timeout; j++) {
4277 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4278 break;
4279 udelay(1);
4280 }
4281 WREG32(CP_HQD_DEQUEUE_REQUEST, 0);
4282 WREG32(CP_HQD_PQ_RPTR, 0);
4283 WREG32(CP_HQD_PQ_WPTR, 0);
4284 }
4285 cik_srbm_select(rdev, 0, 0, 0, 0);
4286 }
4287
4288 /**
4289 * cik_cp_compute_enable - enable/disable the compute CP MEs
4290 *
4291 * @rdev: radeon_device pointer
4292 * @enable: enable or disable the MEs
4293 *
4294 * Halts or unhalts the compute MEs.
4295 */
4296 static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4297 {
4298 if (enable)
4299 WREG32(CP_MEC_CNTL, 0);
4300 else {
4301 /*
4302 * To make hibernation reliable we need to clear compute ring
4303 * configuration before halting the compute ring.
4304 */
4305 mutex_lock(&rdev->srbm_mutex);
4306 cik_compute_stop(rdev,&rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
4307 cik_compute_stop(rdev,&rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
4308 mutex_unlock(&rdev->srbm_mutex);
4309
4310 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
4311 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
4312 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
4313 }
4314 udelay(50);
4315 }
4316
4317 /**
4318 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4319 *
4320 * @rdev: radeon_device pointer
4321 *
4322 * Loads the compute MEC1&2 ucode.
4323 * Returns 0 for success, -EINVAL if the ucode is not available.
4324 */
4325 static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4326 {
4327 int i;
4328
4329 if (!rdev->mec_fw)
4330 return -EINVAL;
4331
4332 cik_cp_compute_enable(rdev, false);
4333
4334 if (rdev->new_fw) {
4335 const struct gfx_firmware_header_v1_0 *mec_hdr =
4336 (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
4337 const __le32 *fw_data;
4338 u32 fw_size;
4339
4340 radeon_ucode_print_gfx_hdr(&mec_hdr->header);
4341
4342 /* MEC1 */
4343 fw_data = (const __le32 *)
4344 (rdev->mec_fw->data + le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
4345 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4;
4346 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4347 for (i = 0; i < fw_size; i++)
4348 WREG32(CP_MEC_ME1_UCODE_DATA, le32_to_cpup(fw_data++));
4349 WREG32(CP_MEC_ME1_UCODE_ADDR, le32_to_cpu(mec_hdr->header.ucode_version));
4350
4351 /* MEC2 */
4352 if (rdev->family == CHIP_KAVERI) {
4353 const struct gfx_firmware_header_v1_0 *mec2_hdr =
4354 (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
4355
4356 fw_data = (const __le32 *)
4357 (rdev->mec2_fw->data +
4358 le32_to_cpu(mec2_hdr->header.ucode_array_offset_bytes));
4359 fw_size = le32_to_cpu(mec2_hdr->header.ucode_size_bytes) / 4;
4360 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4361 for (i = 0; i < fw_size; i++)
4362 WREG32(CP_MEC_ME2_UCODE_DATA, le32_to_cpup(fw_data++));
4363 WREG32(CP_MEC_ME2_UCODE_ADDR, le32_to_cpu(mec2_hdr->header.ucode_version));
4364 }
4365 } else {
4366 const __be32 *fw_data;
4367
4368 /* MEC1 */
4369 fw_data = (const __be32 *)rdev->mec_fw->data;
4370 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4371 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4372 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4373 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4374
4375 if (rdev->family == CHIP_KAVERI) {
4376 /* MEC2 */
4377 fw_data = (const __be32 *)rdev->mec_fw->data;
4378 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4379 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4380 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4381 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4382 }
4383 }
4384
4385 return 0;
4386 }
4387
4388 /**
4389 * cik_cp_compute_start - start the compute queues
4390 *
4391 * @rdev: radeon_device pointer
4392 *
4393 * Enable the compute queues.
4394 * Returns 0 for success, error for failure.
4395 */
4396 static int cik_cp_compute_start(struct radeon_device *rdev)
4397 {
4398 cik_cp_compute_enable(rdev, true);
4399
4400 return 0;
4401 }
4402
4403 /**
4404 * cik_cp_compute_fini - stop the compute queues
4405 *
4406 * @rdev: radeon_device pointer
4407 *
4408 * Stop the compute queues and tear down the driver queue
4409 * info.
4410 */
4411 static void cik_cp_compute_fini(struct radeon_device *rdev)
4412 {
4413 int i, idx, r;
4414
4415 cik_cp_compute_enable(rdev, false);
4416
4417 for (i = 0; i < 2; i++) {
4418 if (i == 0)
4419 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4420 else
4421 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4422
4423 if (rdev->ring[idx].mqd_obj) {
4424 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4425 if (unlikely(r != 0))
4426 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4427
4428 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4429 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4430
4431 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4432 rdev->ring[idx].mqd_obj = NULL;
4433 }
4434 }
4435 }
4436
4437 static void cik_mec_fini(struct radeon_device *rdev)
4438 {
4439 int r;
4440
4441 if (rdev->mec.hpd_eop_obj) {
4442 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4443 if (unlikely(r != 0))
4444 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4445 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4446 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4447
4448 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4449 rdev->mec.hpd_eop_obj = NULL;
4450 }
4451 }
4452
4453 #define MEC_HPD_SIZE 2048
4454
4455 static int cik_mec_init(struct radeon_device *rdev)
4456 {
4457 int r;
4458 u32 *hpd;
4459
4460 /*
4461 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4462 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4463 * Nonetheless, we assign only 1 pipe because all other pipes will
4464 * be handled by KFD
4465 */
4466 rdev->mec.num_mec = 1;
4467 rdev->mec.num_pipe = 1;
4468 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4469
4470 if (rdev->mec.hpd_eop_obj == NULL) {
4471 r = radeon_bo_create(rdev,
4472 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4473 PAGE_SIZE, true,
4474 RADEON_GEM_DOMAIN_GTT, 0, NULL, NULL,
4475 &rdev->mec.hpd_eop_obj);
4476 if (r) {
4477 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4478 return r;
4479 }
4480 }
4481
4482 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4483 if (unlikely(r != 0)) {
4484 cik_mec_fini(rdev);
4485 return r;
4486 }
4487 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4488 &rdev->mec.hpd_eop_gpu_addr);
4489 if (r) {
4490 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4491 cik_mec_fini(rdev);
4492 return r;
4493 }
4494 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4495 if (r) {
4496 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4497 cik_mec_fini(rdev);
4498 return r;
4499 }
4500
4501 /* clear memory. Not sure if this is required or not */
4502 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4503
4504 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4505 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4506
4507 return 0;
4508 }
4509
4510 struct hqd_registers
4511 {
4512 u32 cp_mqd_base_addr;
4513 u32 cp_mqd_base_addr_hi;
4514 u32 cp_hqd_active;
4515 u32 cp_hqd_vmid;
4516 u32 cp_hqd_persistent_state;
4517 u32 cp_hqd_pipe_priority;
4518 u32 cp_hqd_queue_priority;
4519 u32 cp_hqd_quantum;
4520 u32 cp_hqd_pq_base;
4521 u32 cp_hqd_pq_base_hi;
4522 u32 cp_hqd_pq_rptr;
4523 u32 cp_hqd_pq_rptr_report_addr;
4524 u32 cp_hqd_pq_rptr_report_addr_hi;
4525 u32 cp_hqd_pq_wptr_poll_addr;
4526 u32 cp_hqd_pq_wptr_poll_addr_hi;
4527 u32 cp_hqd_pq_doorbell_control;
4528 u32 cp_hqd_pq_wptr;
4529 u32 cp_hqd_pq_control;
4530 u32 cp_hqd_ib_base_addr;
4531 u32 cp_hqd_ib_base_addr_hi;
4532 u32 cp_hqd_ib_rptr;
4533 u32 cp_hqd_ib_control;
4534 u32 cp_hqd_iq_timer;
4535 u32 cp_hqd_iq_rptr;
4536 u32 cp_hqd_dequeue_request;
4537 u32 cp_hqd_dma_offload;
4538 u32 cp_hqd_sema_cmd;
4539 u32 cp_hqd_msg_type;
4540 u32 cp_hqd_atomic0_preop_lo;
4541 u32 cp_hqd_atomic0_preop_hi;
4542 u32 cp_hqd_atomic1_preop_lo;
4543 u32 cp_hqd_atomic1_preop_hi;
4544 u32 cp_hqd_hq_scheduler0;
4545 u32 cp_hqd_hq_scheduler1;
4546 u32 cp_mqd_control;
4547 };
4548
4549 struct bonaire_mqd
4550 {
4551 u32 header;
4552 u32 dispatch_initiator;
4553 u32 dimensions[3];
4554 u32 start_idx[3];
4555 u32 num_threads[3];
4556 u32 pipeline_stat_enable;
4557 u32 perf_counter_enable;
4558 u32 pgm[2];
4559 u32 tba[2];
4560 u32 tma[2];
4561 u32 pgm_rsrc[2];
4562 u32 vmid;
4563 u32 resource_limits;
4564 u32 static_thread_mgmt01[2];
4565 u32 tmp_ring_size;
4566 u32 static_thread_mgmt23[2];
4567 u32 restart[3];
4568 u32 thread_trace_enable;
4569 u32 reserved1;
4570 u32 user_data[16];
4571 u32 vgtcs_invoke_count[2];
4572 struct hqd_registers queue_state;
4573 u32 dequeue_cntr;
4574 u32 interrupt_queue[64];
4575 };
4576
4577 /**
4578 * cik_cp_compute_resume - setup the compute queue registers
4579 *
4580 * @rdev: radeon_device pointer
4581 *
4582 * Program the compute queues and test them to make sure they
4583 * are working.
4584 * Returns 0 for success, error for failure.
4585 */
4586 static int cik_cp_compute_resume(struct radeon_device *rdev)
4587 {
4588 int r, i, j, idx;
4589 u32 tmp;
4590 bool use_doorbell = true;
4591 u64 hqd_gpu_addr;
4592 u64 mqd_gpu_addr;
4593 u64 eop_gpu_addr;
4594 u64 wb_gpu_addr;
4595 u32 *buf;
4596 struct bonaire_mqd *mqd;
4597
4598 r = cik_cp_compute_start(rdev);
4599 if (r)
4600 return r;
4601
4602 /* fix up chicken bits */
4603 tmp = RREG32(CP_CPF_DEBUG);
4604 tmp |= (1 << 23);
4605 WREG32(CP_CPF_DEBUG, tmp);
4606
4607 /* init the pipes */
4608 mutex_lock(&rdev->srbm_mutex);
4609
4610 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr;
4611
4612 cik_srbm_select(rdev, 0, 0, 0, 0);
4613
4614 /* write the EOP addr */
4615 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4616 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4617
4618 /* set the VMID assigned */
4619 WREG32(CP_HPD_EOP_VMID, 0);
4620
4621 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4622 tmp = RREG32(CP_HPD_EOP_CONTROL);
4623 tmp &= ~EOP_SIZE_MASK;
4624 tmp |= order_base_2(MEC_HPD_SIZE / 8);
4625 WREG32(CP_HPD_EOP_CONTROL, tmp);
4626
4627 mutex_unlock(&rdev->srbm_mutex);
4628
4629 /* init the queues. Just two for now. */
4630 for (i = 0; i < 2; i++) {
4631 if (i == 0)
4632 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4633 else
4634 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4635
4636 if (rdev->ring[idx].mqd_obj == NULL) {
4637 r = radeon_bo_create(rdev,
4638 sizeof(struct bonaire_mqd),
4639 PAGE_SIZE, true,
4640 RADEON_GEM_DOMAIN_GTT, 0, NULL,
4641 NULL, &rdev->ring[idx].mqd_obj);
4642 if (r) {
4643 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4644 return r;
4645 }
4646 }
4647
4648 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4649 if (unlikely(r != 0)) {
4650 cik_cp_compute_fini(rdev);
4651 return r;
4652 }
4653 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4654 &mqd_gpu_addr);
4655 if (r) {
4656 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4657 cik_cp_compute_fini(rdev);
4658 return r;
4659 }
4660 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4661 if (r) {
4662 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4663 cik_cp_compute_fini(rdev);
4664 return r;
4665 }
4666
4667 /* init the mqd struct */
4668 memset(buf, 0, sizeof(struct bonaire_mqd));
4669
4670 mqd = (struct bonaire_mqd *)buf;
4671 mqd->header = 0xC0310800;
4672 mqd->static_thread_mgmt01[0] = 0xffffffff;
4673 mqd->static_thread_mgmt01[1] = 0xffffffff;
4674 mqd->static_thread_mgmt23[0] = 0xffffffff;
4675 mqd->static_thread_mgmt23[1] = 0xffffffff;
4676
4677 mutex_lock(&rdev->srbm_mutex);
4678 cik_srbm_select(rdev, rdev->ring[idx].me,
4679 rdev->ring[idx].pipe,
4680 rdev->ring[idx].queue, 0);
4681
4682 /* disable wptr polling */
4683 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4684 tmp &= ~WPTR_POLL_EN;
4685 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4686
4687 /* enable doorbell? */
4688 mqd->queue_state.cp_hqd_pq_doorbell_control =
4689 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4690 if (use_doorbell)
4691 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4692 else
4693 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4694 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4695 mqd->queue_state.cp_hqd_pq_doorbell_control);
4696
4697 /* disable the queue if it's active */
4698 mqd->queue_state.cp_hqd_dequeue_request = 0;
4699 mqd->queue_state.cp_hqd_pq_rptr = 0;
4700 mqd->queue_state.cp_hqd_pq_wptr= 0;
4701 if (RREG32(CP_HQD_ACTIVE) & 1) {
4702 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4703 for (j = 0; j < rdev->usec_timeout; j++) {
4704 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4705 break;
4706 udelay(1);
4707 }
4708 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4709 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4710 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4711 }
4712
4713 /* set the pointer to the MQD */
4714 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4715 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4716 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4717 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4718 /* set MQD vmid to 0 */
4719 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4720 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4721 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4722
4723 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4724 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4725 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4726 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4727 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4728 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4729
4730 /* set up the HQD, this is similar to CP_RB0_CNTL */
4731 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4732 mqd->queue_state.cp_hqd_pq_control &=
4733 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4734
4735 mqd->queue_state.cp_hqd_pq_control |=
4736 order_base_2(rdev->ring[idx].ring_size / 8);
4737 mqd->queue_state.cp_hqd_pq_control |=
4738 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
4739 #ifdef __BIG_ENDIAN
4740 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4741 #endif
4742 mqd->queue_state.cp_hqd_pq_control &=
4743 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4744 mqd->queue_state.cp_hqd_pq_control |=
4745 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4746 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4747
4748 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4749 if (i == 0)
4750 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4751 else
4752 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4753 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4754 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4755 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4756 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4757 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4758
4759 /* set the wb address wether it's enabled or not */
4760 if (i == 0)
4761 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4762 else
4763 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4764 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4765 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4766 upper_32_bits(wb_gpu_addr) & 0xffff;
4767 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4768 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4769 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4770 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4771
4772 /* enable the doorbell if requested */
4773 if (use_doorbell) {
4774 mqd->queue_state.cp_hqd_pq_doorbell_control =
4775 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4776 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
4777 mqd->queue_state.cp_hqd_pq_doorbell_control |=
4778 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
4779 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4780 mqd->queue_state.cp_hqd_pq_doorbell_control &=
4781 ~(DOORBELL_SOURCE | DOORBELL_HIT);
4782
4783 } else {
4784 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
4785 }
4786 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4787 mqd->queue_state.cp_hqd_pq_doorbell_control);
4788
4789 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4790 rdev->ring[idx].wptr = 0;
4791 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
4792 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4793 mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR);
4794
4795 /* set the vmid for the queue */
4796 mqd->queue_state.cp_hqd_vmid = 0;
4797 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
4798
4799 /* activate the queue */
4800 mqd->queue_state.cp_hqd_active = 1;
4801 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
4802
4803 cik_srbm_select(rdev, 0, 0, 0, 0);
4804 mutex_unlock(&rdev->srbm_mutex);
4805
4806 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
4807 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4808
4809 rdev->ring[idx].ready = true;
4810 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
4811 if (r)
4812 rdev->ring[idx].ready = false;
4813 }
4814
4815 return 0;
4816 }
4817
4818 static void cik_cp_enable(struct radeon_device *rdev, bool enable)
4819 {
4820 cik_cp_gfx_enable(rdev, enable);
4821 cik_cp_compute_enable(rdev, enable);
4822 }
4823
4824 static int cik_cp_load_microcode(struct radeon_device *rdev)
4825 {
4826 int r;
4827
4828 r = cik_cp_gfx_load_microcode(rdev);
4829 if (r)
4830 return r;
4831 r = cik_cp_compute_load_microcode(rdev);
4832 if (r)
4833 return r;
4834
4835 return 0;
4836 }
4837
4838 static void cik_cp_fini(struct radeon_device *rdev)
4839 {
4840 cik_cp_gfx_fini(rdev);
4841 cik_cp_compute_fini(rdev);
4842 }
4843
4844 static int cik_cp_resume(struct radeon_device *rdev)
4845 {
4846 int r;
4847
4848 cik_enable_gui_idle_interrupt(rdev, false);
4849
4850 r = cik_cp_load_microcode(rdev);
4851 if (r)
4852 return r;
4853
4854 r = cik_cp_gfx_resume(rdev);
4855 if (r)
4856 return r;
4857 r = cik_cp_compute_resume(rdev);
4858 if (r)
4859 return r;
4860
4861 cik_enable_gui_idle_interrupt(rdev, true);
4862
4863 return 0;
4864 }
4865
4866 static void cik_print_gpu_status_regs(struct radeon_device *rdev)
4867 {
4868 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
4869 RREG32(GRBM_STATUS));
4870 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
4871 RREG32(GRBM_STATUS2));
4872 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
4873 RREG32(GRBM_STATUS_SE0));
4874 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
4875 RREG32(GRBM_STATUS_SE1));
4876 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
4877 RREG32(GRBM_STATUS_SE2));
4878 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
4879 RREG32(GRBM_STATUS_SE3));
4880 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
4881 RREG32(SRBM_STATUS));
4882 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
4883 RREG32(SRBM_STATUS2));
4884 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
4885 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
4886 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
4887 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
4888 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
4889 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
4890 RREG32(CP_STALLED_STAT1));
4891 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
4892 RREG32(CP_STALLED_STAT2));
4893 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
4894 RREG32(CP_STALLED_STAT3));
4895 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
4896 RREG32(CP_CPF_BUSY_STAT));
4897 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
4898 RREG32(CP_CPF_STALLED_STAT1));
4899 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
4900 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
4901 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
4902 RREG32(CP_CPC_STALLED_STAT1));
4903 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
4904 }
4905
4906 /**
4907 * cik_gpu_check_soft_reset - check which blocks are busy
4908 *
4909 * @rdev: radeon_device pointer
4910 *
4911 * Check which blocks are busy and return the relevant reset
4912 * mask to be used by cik_gpu_soft_reset().
4913 * Returns a mask of the blocks to be reset.
4914 */
4915 u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
4916 {
4917 u32 reset_mask = 0;
4918 u32 tmp;
4919
4920 /* GRBM_STATUS */
4921 tmp = RREG32(GRBM_STATUS);
4922 if (tmp & (PA_BUSY | SC_BUSY |
4923 BCI_BUSY | SX_BUSY |
4924 TA_BUSY | VGT_BUSY |
4925 DB_BUSY | CB_BUSY |
4926 GDS_BUSY | SPI_BUSY |
4927 IA_BUSY | IA_BUSY_NO_DMA))
4928 reset_mask |= RADEON_RESET_GFX;
4929
4930 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
4931 reset_mask |= RADEON_RESET_CP;
4932
4933 /* GRBM_STATUS2 */
4934 tmp = RREG32(GRBM_STATUS2);
4935 if (tmp & RLC_BUSY)
4936 reset_mask |= RADEON_RESET_RLC;
4937
4938 /* SDMA0_STATUS_REG */
4939 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
4940 if (!(tmp & SDMA_IDLE))
4941 reset_mask |= RADEON_RESET_DMA;
4942
4943 /* SDMA1_STATUS_REG */
4944 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
4945 if (!(tmp & SDMA_IDLE))
4946 reset_mask |= RADEON_RESET_DMA1;
4947
4948 /* SRBM_STATUS2 */
4949 tmp = RREG32(SRBM_STATUS2);
4950 if (tmp & SDMA_BUSY)
4951 reset_mask |= RADEON_RESET_DMA;
4952
4953 if (tmp & SDMA1_BUSY)
4954 reset_mask |= RADEON_RESET_DMA1;
4955
4956 /* SRBM_STATUS */
4957 tmp = RREG32(SRBM_STATUS);
4958
4959 if (tmp & IH_BUSY)
4960 reset_mask |= RADEON_RESET_IH;
4961
4962 if (tmp & SEM_BUSY)
4963 reset_mask |= RADEON_RESET_SEM;
4964
4965 if (tmp & GRBM_RQ_PENDING)
4966 reset_mask |= RADEON_RESET_GRBM;
4967
4968 if (tmp & VMC_BUSY)
4969 reset_mask |= RADEON_RESET_VMC;
4970
4971 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
4972 MCC_BUSY | MCD_BUSY))
4973 reset_mask |= RADEON_RESET_MC;
4974
4975 if (evergreen_is_display_hung(rdev))
4976 reset_mask |= RADEON_RESET_DISPLAY;
4977
4978 /* Skip MC reset as it's mostly likely not hung, just busy */
4979 if (reset_mask & RADEON_RESET_MC) {
4980 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
4981 reset_mask &= ~RADEON_RESET_MC;
4982 }
4983
4984 return reset_mask;
4985 }
4986
4987 /**
4988 * cik_gpu_soft_reset - soft reset GPU
4989 *
4990 * @rdev: radeon_device pointer
4991 * @reset_mask: mask of which blocks to reset
4992 *
4993 * Soft reset the blocks specified in @reset_mask.
4994 */
4995 static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
4996 {
4997 struct evergreen_mc_save save;
4998 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4999 u32 tmp;
5000
5001 if (reset_mask == 0)
5002 return;
5003
5004 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
5005
5006 cik_print_gpu_status_regs(rdev);
5007 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
5008 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
5009 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5010 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
5011
5012 /* disable CG/PG */
5013 cik_fini_pg(rdev);
5014 cik_fini_cg(rdev);
5015
5016 /* stop the rlc */
5017 cik_rlc_stop(rdev);
5018
5019 /* Disable GFX parsing/prefetching */
5020 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5021
5022 /* Disable MEC parsing/prefetching */
5023 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5024
5025 if (reset_mask & RADEON_RESET_DMA) {
5026 /* sdma0 */
5027 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5028 tmp |= SDMA_HALT;
5029 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5030 }
5031 if (reset_mask & RADEON_RESET_DMA1) {
5032 /* sdma1 */
5033 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5034 tmp |= SDMA_HALT;
5035 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5036 }
5037
5038 evergreen_mc_stop(rdev, &save);
5039 if (evergreen_mc_wait_for_idle(rdev)) {
5040 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5041 }
5042
5043 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
5044 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
5045
5046 if (reset_mask & RADEON_RESET_CP) {
5047 grbm_soft_reset |= SOFT_RESET_CP;
5048
5049 srbm_soft_reset |= SOFT_RESET_GRBM;
5050 }
5051
5052 if (reset_mask & RADEON_RESET_DMA)
5053 srbm_soft_reset |= SOFT_RESET_SDMA;
5054
5055 if (reset_mask & RADEON_RESET_DMA1)
5056 srbm_soft_reset |= SOFT_RESET_SDMA1;
5057
5058 if (reset_mask & RADEON_RESET_DISPLAY)
5059 srbm_soft_reset |= SOFT_RESET_DC;
5060
5061 if (reset_mask & RADEON_RESET_RLC)
5062 grbm_soft_reset |= SOFT_RESET_RLC;
5063
5064 if (reset_mask & RADEON_RESET_SEM)
5065 srbm_soft_reset |= SOFT_RESET_SEM;
5066
5067 if (reset_mask & RADEON_RESET_IH)
5068 srbm_soft_reset |= SOFT_RESET_IH;
5069
5070 if (reset_mask & RADEON_RESET_GRBM)
5071 srbm_soft_reset |= SOFT_RESET_GRBM;
5072
5073 if (reset_mask & RADEON_RESET_VMC)
5074 srbm_soft_reset |= SOFT_RESET_VMC;
5075
5076 if (!(rdev->flags & RADEON_IS_IGP)) {
5077 if (reset_mask & RADEON_RESET_MC)
5078 srbm_soft_reset |= SOFT_RESET_MC;
5079 }
5080
5081 if (grbm_soft_reset) {
5082 tmp = RREG32(GRBM_SOFT_RESET);
5083 tmp |= grbm_soft_reset;
5084 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
5085 WREG32(GRBM_SOFT_RESET, tmp);
5086 tmp = RREG32(GRBM_SOFT_RESET);
5087
5088 udelay(50);
5089
5090 tmp &= ~grbm_soft_reset;
5091 WREG32(GRBM_SOFT_RESET, tmp);
5092 tmp = RREG32(GRBM_SOFT_RESET);
5093 }
5094
5095 if (srbm_soft_reset) {
5096 tmp = RREG32(SRBM_SOFT_RESET);
5097 tmp |= srbm_soft_reset;
5098 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
5099 WREG32(SRBM_SOFT_RESET, tmp);
5100 tmp = RREG32(SRBM_SOFT_RESET);
5101
5102 udelay(50);
5103
5104 tmp &= ~srbm_soft_reset;
5105 WREG32(SRBM_SOFT_RESET, tmp);
5106 tmp = RREG32(SRBM_SOFT_RESET);
5107 }
5108
5109 /* Wait a little for things to settle down */
5110 udelay(50);
5111
5112 evergreen_mc_resume(rdev, &save);
5113 udelay(50);
5114
5115 cik_print_gpu_status_regs(rdev);
5116 }
5117
5118 struct kv_reset_save_regs {
5119 u32 gmcon_reng_execute;
5120 u32 gmcon_misc;
5121 u32 gmcon_misc3;
5122 };
5123
5124 static void kv_save_regs_for_reset(struct radeon_device *rdev,
5125 struct kv_reset_save_regs *save)
5126 {
5127 save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE);
5128 save->gmcon_misc = RREG32(GMCON_MISC);
5129 save->gmcon_misc3 = RREG32(GMCON_MISC3);
5130
5131 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP);
5132 WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE |
5133 STCTRL_STUTTER_EN));
5134 }
5135
5136 static void kv_restore_regs_for_reset(struct radeon_device *rdev,
5137 struct kv_reset_save_regs *save)
5138 {
5139 int i;
5140
5141 WREG32(GMCON_PGFSM_WRITE, 0);
5142 WREG32(GMCON_PGFSM_CONFIG, 0x200010ff);
5143
5144 for (i = 0; i < 5; i++)
5145 WREG32(GMCON_PGFSM_WRITE, 0);
5146
5147 WREG32(GMCON_PGFSM_WRITE, 0);
5148 WREG32(GMCON_PGFSM_CONFIG, 0x300010ff);
5149
5150 for (i = 0; i < 5; i++)
5151 WREG32(GMCON_PGFSM_WRITE, 0);
5152
5153 WREG32(GMCON_PGFSM_WRITE, 0x210000);
5154 WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff);
5155
5156 for (i = 0; i < 5; i++)
5157 WREG32(GMCON_PGFSM_WRITE, 0);
5158
5159 WREG32(GMCON_PGFSM_WRITE, 0x21003);
5160 WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff);
5161
5162 for (i = 0; i < 5; i++)
5163 WREG32(GMCON_PGFSM_WRITE, 0);
5164
5165 WREG32(GMCON_PGFSM_WRITE, 0x2b00);
5166 WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff);
5167
5168 for (i = 0; i < 5; i++)
5169 WREG32(GMCON_PGFSM_WRITE, 0);
5170
5171 WREG32(GMCON_PGFSM_WRITE, 0);
5172 WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff);
5173
5174 for (i = 0; i < 5; i++)
5175 WREG32(GMCON_PGFSM_WRITE, 0);
5176
5177 WREG32(GMCON_PGFSM_WRITE, 0x420000);
5178 WREG32(GMCON_PGFSM_CONFIG, 0x100010ff);
5179
5180 for (i = 0; i < 5; i++)
5181 WREG32(GMCON_PGFSM_WRITE, 0);
5182
5183 WREG32(GMCON_PGFSM_WRITE, 0x120202);
5184 WREG32(GMCON_PGFSM_CONFIG, 0x500010ff);
5185
5186 for (i = 0; i < 5; i++)
5187 WREG32(GMCON_PGFSM_WRITE, 0);
5188
5189 WREG32(GMCON_PGFSM_WRITE, 0x3e3e36);
5190 WREG32(GMCON_PGFSM_CONFIG, 0x600010ff);
5191
5192 for (i = 0; i < 5; i++)
5193 WREG32(GMCON_PGFSM_WRITE, 0);
5194
5195 WREG32(GMCON_PGFSM_WRITE, 0x373f3e);
5196 WREG32(GMCON_PGFSM_CONFIG, 0x700010ff);
5197
5198 for (i = 0; i < 5; i++)
5199 WREG32(GMCON_PGFSM_WRITE, 0);
5200
5201 WREG32(GMCON_PGFSM_WRITE, 0x3e1332);
5202 WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff);
5203
5204 WREG32(GMCON_MISC3, save->gmcon_misc3);
5205 WREG32(GMCON_MISC, save->gmcon_misc);
5206 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute);
5207 }
5208
5209 static void cik_gpu_pci_config_reset(struct radeon_device *rdev)
5210 {
5211 struct evergreen_mc_save save;
5212 struct kv_reset_save_regs kv_save = { 0 };
5213 u32 tmp, i;
5214
5215 dev_info(rdev->dev, "GPU pci config reset\n");
5216
5217 /* disable dpm? */
5218
5219 /* disable cg/pg */
5220 cik_fini_pg(rdev);
5221 cik_fini_cg(rdev);
5222
5223 /* Disable GFX parsing/prefetching */
5224 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5225
5226 /* Disable MEC parsing/prefetching */
5227 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5228
5229 /* sdma0 */
5230 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5231 tmp |= SDMA_HALT;
5232 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5233 /* sdma1 */
5234 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5235 tmp |= SDMA_HALT;
5236 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5237 /* XXX other engines? */
5238
5239 /* halt the rlc, disable cp internal ints */
5240 cik_rlc_stop(rdev);
5241
5242 udelay(50);
5243
5244 /* disable mem access */
5245 evergreen_mc_stop(rdev, &save);
5246 if (evergreen_mc_wait_for_idle(rdev)) {
5247 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
5248 }
5249
5250 if (rdev->flags & RADEON_IS_IGP)
5251 kv_save_regs_for_reset(rdev, &kv_save);
5252
5253 /* disable BM */
5254 pci_clear_master(rdev->pdev);
5255 /* reset */
5256 radeon_pci_config_reset(rdev);
5257
5258 udelay(100);
5259
5260 /* wait for asic to come out of reset */
5261 for (i = 0; i < rdev->usec_timeout; i++) {
5262 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
5263 break;
5264 udelay(1);
5265 }
5266
5267 /* does asic init need to be run first??? */
5268 if (rdev->flags & RADEON_IS_IGP)
5269 kv_restore_regs_for_reset(rdev, &kv_save);
5270 }
5271
5272 /**
5273 * cik_asic_reset - soft reset GPU
5274 *
5275 * @rdev: radeon_device pointer
5276 * @hard: force hard reset
5277 *
5278 * Look up which blocks are hung and attempt
5279 * to reset them.
5280 * Returns 0 for success.
5281 */
5282 int cik_asic_reset(struct radeon_device *rdev, bool hard)
5283 {
5284 u32 reset_mask;
5285
5286 if (hard) {
5287 cik_gpu_pci_config_reset(rdev);
5288 return 0;
5289 }
5290
5291 reset_mask = cik_gpu_check_soft_reset(rdev);
5292
5293 if (reset_mask)
5294 r600_set_bios_scratch_engine_hung(rdev, true);
5295
5296 /* try soft reset */
5297 cik_gpu_soft_reset(rdev, reset_mask);
5298
5299 reset_mask = cik_gpu_check_soft_reset(rdev);
5300
5301 /* try pci config reset */
5302 if (reset_mask && radeon_hard_reset)
5303 cik_gpu_pci_config_reset(rdev);
5304
5305 reset_mask = cik_gpu_check_soft_reset(rdev);
5306
5307 if (!reset_mask)
5308 r600_set_bios_scratch_engine_hung(rdev, false);
5309
5310 return 0;
5311 }
5312
5313 /**
5314 * cik_gfx_is_lockup - check if the 3D engine is locked up
5315 *
5316 * @rdev: radeon_device pointer
5317 * @ring: radeon_ring structure holding ring information
5318 *
5319 * Check if the 3D engine is locked up (CIK).
5320 * Returns true if the engine is locked, false if not.
5321 */
5322 bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
5323 {
5324 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
5325
5326 if (!(reset_mask & (RADEON_RESET_GFX |
5327 RADEON_RESET_COMPUTE |
5328 RADEON_RESET_CP))) {
5329 radeon_ring_lockup_update(rdev, ring);
5330 return false;
5331 }
5332 return radeon_ring_test_lockup(rdev, ring);
5333 }
5334
5335 /* MC */
5336 /**
5337 * cik_mc_program - program the GPU memory controller
5338 *
5339 * @rdev: radeon_device pointer
5340 *
5341 * Set the location of vram, gart, and AGP in the GPU's
5342 * physical address space (CIK).
5343 */
5344 static void cik_mc_program(struct radeon_device *rdev)
5345 {
5346 struct evergreen_mc_save save;
5347 u32 tmp;
5348 int i, j;
5349
5350 /* Initialize HDP */
5351 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
5352 WREG32((0x2c14 + j), 0x00000000);
5353 WREG32((0x2c18 + j), 0x00000000);
5354 WREG32((0x2c1c + j), 0x00000000);
5355 WREG32((0x2c20 + j), 0x00000000);
5356 WREG32((0x2c24 + j), 0x00000000);
5357 }
5358 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
5359
5360 evergreen_mc_stop(rdev, &save);
5361 if (radeon_mc_wait_for_idle(rdev)) {
5362 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5363 }
5364 /* Lockout access through VGA aperture*/
5365 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
5366 /* Update configuration */
5367 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
5368 rdev->mc.vram_start >> 12);
5369 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
5370 rdev->mc.vram_end >> 12);
5371 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
5372 rdev->vram_scratch.gpu_addr >> 12);
5373 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
5374 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
5375 WREG32(MC_VM_FB_LOCATION, tmp);
5376 /* XXX double check these! */
5377 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
5378 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
5379 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
5380 WREG32(MC_VM_AGP_BASE, 0);
5381 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
5382 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
5383 if (radeon_mc_wait_for_idle(rdev)) {
5384 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5385 }
5386 evergreen_mc_resume(rdev, &save);
5387 /* we need to own VRAM, so turn off the VGA renderer here
5388 * to stop it overwriting our objects */
5389 rv515_vga_render_disable(rdev);
5390 }
5391
5392 /**
5393 * cik_mc_init - initialize the memory controller driver params
5394 *
5395 * @rdev: radeon_device pointer
5396 *
5397 * Look up the amount of vram, vram width, and decide how to place
5398 * vram and gart within the GPU's physical address space (CIK).
5399 * Returns 0 for success.
5400 */
5401 static int cik_mc_init(struct radeon_device *rdev)
5402 {
5403 u32 tmp;
5404 int chansize, numchan;
5405
5406 /* Get VRAM informations */
5407 rdev->mc.vram_is_ddr = true;
5408 tmp = RREG32(MC_ARB_RAMCFG);
5409 if (tmp & CHANSIZE_MASK) {
5410 chansize = 64;
5411 } else {
5412 chansize = 32;
5413 }
5414 tmp = RREG32(MC_SHARED_CHMAP);
5415 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
5416 case 0:
5417 default:
5418 numchan = 1;
5419 break;
5420 case 1:
5421 numchan = 2;
5422 break;
5423 case 2:
5424 numchan = 4;
5425 break;
5426 case 3:
5427 numchan = 8;
5428 break;
5429 case 4:
5430 numchan = 3;
5431 break;
5432 case 5:
5433 numchan = 6;
5434 break;
5435 case 6:
5436 numchan = 10;
5437 break;
5438 case 7:
5439 numchan = 12;
5440 break;
5441 case 8:
5442 numchan = 16;
5443 break;
5444 }
5445 rdev->mc.vram_width = numchan * chansize;
5446 /* Could aper size report 0 ? */
5447 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5448 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5449 /* size in MB on si */
5450 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5451 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5452 rdev->mc.visible_vram_size = rdev->mc.aper_size;
5453 si_vram_gtt_location(rdev, &rdev->mc);
5454 radeon_update_bandwidth_info(rdev);
5455
5456 return 0;
5457 }
5458
5459 /*
5460 * GART
5461 * VMID 0 is the physical GPU addresses as used by the kernel.
5462 * VMIDs 1-15 are used for userspace clients and are handled
5463 * by the radeon vm/hsa code.
5464 */
5465 /**
5466 * cik_pcie_gart_tlb_flush - gart tlb flush callback
5467 *
5468 * @rdev: radeon_device pointer
5469 *
5470 * Flush the TLB for the VMID 0 page table (CIK).
5471 */
5472 void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5473 {
5474 /* flush hdp cache */
5475 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5476
5477 /* bits 0-15 are the VM contexts0-15 */
5478 WREG32(VM_INVALIDATE_REQUEST, 0x1);
5479 }
5480
5481 static void cik_pcie_init_compute_vmid(struct radeon_device *rdev)
5482 {
5483 int i;
5484 uint32_t sh_mem_bases, sh_mem_config;
5485
5486 sh_mem_bases = 0x6000 | 0x6000 << 16;
5487 sh_mem_config = ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED);
5488 sh_mem_config |= DEFAULT_MTYPE(MTYPE_NONCACHED);
5489
5490 mutex_lock(&rdev->srbm_mutex);
5491 for (i = 8; i < 16; i++) {
5492 cik_srbm_select(rdev, 0, 0, 0, i);
5493 /* CP and shaders */
5494 WREG32(SH_MEM_CONFIG, sh_mem_config);
5495 WREG32(SH_MEM_APE1_BASE, 1);
5496 WREG32(SH_MEM_APE1_LIMIT, 0);
5497 WREG32(SH_MEM_BASES, sh_mem_bases);
5498 }
5499 cik_srbm_select(rdev, 0, 0, 0, 0);
5500 mutex_unlock(&rdev->srbm_mutex);
5501 }
5502
5503 /**
5504 * cik_pcie_gart_enable - gart enable
5505 *
5506 * @rdev: radeon_device pointer
5507 *
5508 * This sets up the TLBs, programs the page tables for VMID0,
5509 * sets up the hw for VMIDs 1-15 which are allocated on
5510 * demand, and sets up the global locations for the LDS, GDS,
5511 * and GPUVM for FSA64 clients (CIK).
5512 * Returns 0 for success, errors for failure.
5513 */
5514 static int cik_pcie_gart_enable(struct radeon_device *rdev)
5515 {
5516 int r, i;
5517
5518 if (rdev->gart.robj == NULL) {
5519 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5520 return -EINVAL;
5521 }
5522 r = radeon_gart_table_vram_pin(rdev);
5523 if (r)
5524 return r;
5525 /* Setup TLB control */
5526 WREG32(MC_VM_MX_L1_TLB_CNTL,
5527 (0xA << 7) |
5528 ENABLE_L1_TLB |
5529 ENABLE_L1_FRAGMENT_PROCESSING |
5530 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5531 ENABLE_ADVANCED_DRIVER_MODEL |
5532 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5533 /* Setup L2 cache */
5534 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5535 ENABLE_L2_FRAGMENT_PROCESSING |
5536 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5537 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5538 EFFECTIVE_L2_QUEUE_SIZE(7) |
5539 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5540 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5541 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5542 BANK_SELECT(4) |
5543 L2_CACHE_BIGK_FRAGMENT_SIZE(4));
5544 /* setup context0 */
5545 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5546 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5547 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5548 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5549 (u32)(rdev->dummy_page.addr >> 12));
5550 WREG32(VM_CONTEXT0_CNTL2, 0);
5551 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5552 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5553
5554 WREG32(0x15D4, 0);
5555 WREG32(0x15D8, 0);
5556 WREG32(0x15DC, 0);
5557
5558 /* restore context1-15 */
5559 /* set vm size, must be a multiple of 4 */
5560 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5561 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn - 1);
5562 for (i = 1; i < 16; i++) {
5563 if (i < 8)
5564 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5565 rdev->vm_manager.saved_table_addr[i]);
5566 else
5567 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5568 rdev->vm_manager.saved_table_addr[i]);
5569 }
5570
5571 /* enable context1-15 */
5572 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5573 (u32)(rdev->dummy_page.addr >> 12));
5574 WREG32(VM_CONTEXT1_CNTL2, 4);
5575 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
5576 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
5577 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5578 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5579 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5580 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5581 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5582 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5583 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5584 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5585 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5586 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5587 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5588 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
5589
5590 if (rdev->family == CHIP_KAVERI) {
5591 u32 tmp = RREG32(CHUB_CONTROL);
5592 tmp &= ~BYPASS_VM;
5593 WREG32(CHUB_CONTROL, tmp);
5594 }
5595
5596 /* XXX SH_MEM regs */
5597 /* where to put LDS, scratch, GPUVM in FSA64 space */
5598 mutex_lock(&rdev->srbm_mutex);
5599 for (i = 0; i < 16; i++) {
5600 cik_srbm_select(rdev, 0, 0, 0, i);
5601 /* CP and shaders */
5602 WREG32(SH_MEM_CONFIG, 0);
5603 WREG32(SH_MEM_APE1_BASE, 1);
5604 WREG32(SH_MEM_APE1_LIMIT, 0);
5605 WREG32(SH_MEM_BASES, 0);
5606 /* SDMA GFX */
5607 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5608 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5609 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5610 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5611 /* XXX SDMA RLC - todo */
5612 }
5613 cik_srbm_select(rdev, 0, 0, 0, 0);
5614 mutex_unlock(&rdev->srbm_mutex);
5615
5616 cik_pcie_init_compute_vmid(rdev);
5617
5618 cik_pcie_gart_tlb_flush(rdev);
5619 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5620 (unsigned)(rdev->mc.gtt_size >> 20),
5621 (unsigned long long)rdev->gart.table_addr);
5622 rdev->gart.ready = true;
5623 return 0;
5624 }
5625
5626 /**
5627 * cik_pcie_gart_disable - gart disable
5628 *
5629 * @rdev: radeon_device pointer
5630 *
5631 * This disables all VM page table (CIK).
5632 */
5633 static void cik_pcie_gart_disable(struct radeon_device *rdev)
5634 {
5635 unsigned i;
5636
5637 for (i = 1; i < 16; ++i) {
5638 uint32_t reg;
5639 if (i < 8)
5640 reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2);
5641 else
5642 reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2);
5643 rdev->vm_manager.saved_table_addr[i] = RREG32(reg);
5644 }
5645
5646 /* Disable all tables */
5647 WREG32(VM_CONTEXT0_CNTL, 0);
5648 WREG32(VM_CONTEXT1_CNTL, 0);
5649 /* Setup TLB control */
5650 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5651 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5652 /* Setup L2 cache */
5653 WREG32(VM_L2_CNTL,
5654 ENABLE_L2_FRAGMENT_PROCESSING |
5655 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5656 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5657 EFFECTIVE_L2_QUEUE_SIZE(7) |
5658 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5659 WREG32(VM_L2_CNTL2, 0);
5660 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5661 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5662 radeon_gart_table_vram_unpin(rdev);
5663 }
5664
5665 /**
5666 * cik_pcie_gart_fini - vm fini callback
5667 *
5668 * @rdev: radeon_device pointer
5669 *
5670 * Tears down the driver GART/VM setup (CIK).
5671 */
5672 static void cik_pcie_gart_fini(struct radeon_device *rdev)
5673 {
5674 cik_pcie_gart_disable(rdev);
5675 radeon_gart_table_vram_free(rdev);
5676 radeon_gart_fini(rdev);
5677 }
5678
5679 /* vm parser */
5680 /**
5681 * cik_ib_parse - vm ib_parse callback
5682 *
5683 * @rdev: radeon_device pointer
5684 * @ib: indirect buffer pointer
5685 *
5686 * CIK uses hw IB checking so this is a nop (CIK).
5687 */
5688 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5689 {
5690 return 0;
5691 }
5692
5693 /*
5694 * vm
5695 * VMID 0 is the physical GPU addresses as used by the kernel.
5696 * VMIDs 1-15 are used for userspace clients and are handled
5697 * by the radeon vm/hsa code.
5698 */
5699 /**
5700 * cik_vm_init - cik vm init callback
5701 *
5702 * @rdev: radeon_device pointer
5703 *
5704 * Inits cik specific vm parameters (number of VMs, base of vram for
5705 * VMIDs 1-15) (CIK).
5706 * Returns 0 for success.
5707 */
5708 int cik_vm_init(struct radeon_device *rdev)
5709 {
5710 /*
5711 * number of VMs
5712 * VMID 0 is reserved for System
5713 * radeon graphics/compute will use VMIDs 1-7
5714 * amdkfd will use VMIDs 8-15
5715 */
5716 rdev->vm_manager.nvm = RADEON_NUM_OF_VMIDS;
5717 /* base offset of vram pages */
5718 if (rdev->flags & RADEON_IS_IGP) {
5719 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5720 tmp <<= 22;
5721 rdev->vm_manager.vram_base_offset = tmp;
5722 } else
5723 rdev->vm_manager.vram_base_offset = 0;
5724
5725 return 0;
5726 }
5727
5728 /**
5729 * cik_vm_fini - cik vm fini callback
5730 *
5731 * @rdev: radeon_device pointer
5732 *
5733 * Tear down any asic specific VM setup (CIK).
5734 */
5735 void cik_vm_fini(struct radeon_device *rdev)
5736 {
5737 }
5738
5739 /**
5740 * cik_vm_decode_fault - print human readable fault info
5741 *
5742 * @rdev: radeon_device pointer
5743 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5744 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5745 *
5746 * Print human readable fault information (CIK).
5747 */
5748 static void cik_vm_decode_fault(struct radeon_device *rdev,
5749 u32 status, u32 addr, u32 mc_client)
5750 {
5751 u32 mc_id;
5752 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5753 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
5754 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5755 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
5756
5757 if (rdev->family == CHIP_HAWAII)
5758 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5759 else
5760 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5761
5762 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
5763 protections, vmid, addr,
5764 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
5765 block, mc_client, mc_id);
5766 }
5767
5768 /**
5769 * cik_vm_flush - cik vm flush using the CP
5770 *
5771 * @rdev: radeon_device pointer
5772 *
5773 * Update the page table base and flush the VM TLB
5774 * using the CP (CIK).
5775 */
5776 void cik_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
5777 unsigned vm_id, uint64_t pd_addr)
5778 {
5779 int usepfp = (ring->idx == RADEON_RING_TYPE_GFX_INDEX);
5780
5781 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5782 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5783 WRITE_DATA_DST_SEL(0)));
5784 if (vm_id < 8) {
5785 radeon_ring_write(ring,
5786 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2);
5787 } else {
5788 radeon_ring_write(ring,
5789 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2);
5790 }
5791 radeon_ring_write(ring, 0);
5792 radeon_ring_write(ring, pd_addr >> 12);
5793
5794 /* update SH_MEM_* regs */
5795 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5796 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5797 WRITE_DATA_DST_SEL(0)));
5798 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5799 radeon_ring_write(ring, 0);
5800 radeon_ring_write(ring, VMID(vm_id));
5801
5802 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
5803 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5804 WRITE_DATA_DST_SEL(0)));
5805 radeon_ring_write(ring, SH_MEM_BASES >> 2);
5806 radeon_ring_write(ring, 0);
5807
5808 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
5809 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
5810 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
5811 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
5812
5813 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5814 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5815 WRITE_DATA_DST_SEL(0)));
5816 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5817 radeon_ring_write(ring, 0);
5818 radeon_ring_write(ring, VMID(0));
5819
5820 /* HDP flush */
5821 cik_hdp_flush_cp_ring_emit(rdev, ring->idx);
5822
5823 /* bits 0-15 are the VM contexts0-15 */
5824 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5825 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5826 WRITE_DATA_DST_SEL(0)));
5827 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5828 radeon_ring_write(ring, 0);
5829 radeon_ring_write(ring, 1 << vm_id);
5830
5831 /* wait for the invalidate to complete */
5832 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
5833 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(0) | /* wait */
5834 WAIT_REG_MEM_FUNCTION(0) | /* always */
5835 WAIT_REG_MEM_ENGINE(0))); /* me */
5836 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5837 radeon_ring_write(ring, 0);
5838 radeon_ring_write(ring, 0); /* ref */
5839 radeon_ring_write(ring, 0); /* mask */
5840 radeon_ring_write(ring, 0x20); /* poll interval */
5841
5842 /* compute doesn't have PFP */
5843 if (usepfp) {
5844 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5845 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5846 radeon_ring_write(ring, 0x0);
5847 }
5848 }
5849
5850 /*
5851 * RLC
5852 * The RLC is a multi-purpose microengine that handles a
5853 * variety of functions, the most important of which is
5854 * the interrupt controller.
5855 */
5856 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
5857 bool enable)
5858 {
5859 u32 tmp = RREG32(CP_INT_CNTL_RING0);
5860
5861 if (enable)
5862 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5863 else
5864 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5865 WREG32(CP_INT_CNTL_RING0, tmp);
5866 }
5867
5868 static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
5869 {
5870 u32 tmp;
5871
5872 tmp = RREG32(RLC_LB_CNTL);
5873 if (enable)
5874 tmp |= LOAD_BALANCE_ENABLE;
5875 else
5876 tmp &= ~LOAD_BALANCE_ENABLE;
5877 WREG32(RLC_LB_CNTL, tmp);
5878 }
5879
5880 static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
5881 {
5882 u32 i, j, k;
5883 u32 mask;
5884
5885 mutex_lock(&rdev->grbm_idx_mutex);
5886 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
5887 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
5888 cik_select_se_sh(rdev, i, j);
5889 for (k = 0; k < rdev->usec_timeout; k++) {
5890 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
5891 break;
5892 udelay(1);
5893 }
5894 }
5895 }
5896 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5897 mutex_unlock(&rdev->grbm_idx_mutex);
5898
5899 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
5900 for (k = 0; k < rdev->usec_timeout; k++) {
5901 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
5902 break;
5903 udelay(1);
5904 }
5905 }
5906
5907 static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
5908 {
5909 u32 tmp;
5910
5911 tmp = RREG32(RLC_CNTL);
5912 if (tmp != rlc)
5913 WREG32(RLC_CNTL, rlc);
5914 }
5915
5916 static u32 cik_halt_rlc(struct radeon_device *rdev)
5917 {
5918 u32 data, orig;
5919
5920 orig = data = RREG32(RLC_CNTL);
5921
5922 if (data & RLC_ENABLE) {
5923 u32 i;
5924
5925 data &= ~RLC_ENABLE;
5926 WREG32(RLC_CNTL, data);
5927
5928 for (i = 0; i < rdev->usec_timeout; i++) {
5929 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
5930 break;
5931 udelay(1);
5932 }
5933
5934 cik_wait_for_rlc_serdes(rdev);
5935 }
5936
5937 return orig;
5938 }
5939
5940 void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
5941 {
5942 u32 tmp, i, mask;
5943
5944 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
5945 WREG32(RLC_GPR_REG2, tmp);
5946
5947 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
5948 for (i = 0; i < rdev->usec_timeout; i++) {
5949 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
5950 break;
5951 udelay(1);
5952 }
5953
5954 for (i = 0; i < rdev->usec_timeout; i++) {
5955 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
5956 break;
5957 udelay(1);
5958 }
5959 }
5960
5961 void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
5962 {
5963 u32 tmp;
5964
5965 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
5966 WREG32(RLC_GPR_REG2, tmp);
5967 }
5968
5969 /**
5970 * cik_rlc_stop - stop the RLC ME
5971 *
5972 * @rdev: radeon_device pointer
5973 *
5974 * Halt the RLC ME (MicroEngine) (CIK).
5975 */
5976 static void cik_rlc_stop(struct radeon_device *rdev)
5977 {
5978 WREG32(RLC_CNTL, 0);
5979
5980 cik_enable_gui_idle_interrupt(rdev, false);
5981
5982 cik_wait_for_rlc_serdes(rdev);
5983 }
5984
5985 /**
5986 * cik_rlc_start - start the RLC ME
5987 *
5988 * @rdev: radeon_device pointer
5989 *
5990 * Unhalt the RLC ME (MicroEngine) (CIK).
5991 */
5992 static void cik_rlc_start(struct radeon_device *rdev)
5993 {
5994 WREG32(RLC_CNTL, RLC_ENABLE);
5995
5996 cik_enable_gui_idle_interrupt(rdev, true);
5997
5998 udelay(50);
5999 }
6000
6001 /**
6002 * cik_rlc_resume - setup the RLC hw
6003 *
6004 * @rdev: radeon_device pointer
6005 *
6006 * Initialize the RLC registers, load the ucode,
6007 * and start the RLC (CIK).
6008 * Returns 0 for success, -EINVAL if the ucode is not available.
6009 */
6010 static int cik_rlc_resume(struct radeon_device *rdev)
6011 {
6012 u32 i, size, tmp;
6013
6014 if (!rdev->rlc_fw)
6015 return -EINVAL;
6016
6017 cik_rlc_stop(rdev);
6018
6019 /* disable CG */
6020 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
6021 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
6022
6023 si_rlc_reset(rdev);
6024
6025 cik_init_pg(rdev);
6026
6027 cik_init_cg(rdev);
6028
6029 WREG32(RLC_LB_CNTR_INIT, 0);
6030 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
6031
6032 mutex_lock(&rdev->grbm_idx_mutex);
6033 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6034 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
6035 WREG32(RLC_LB_PARAMS, 0x00600408);
6036 WREG32(RLC_LB_CNTL, 0x80000004);
6037 mutex_unlock(&rdev->grbm_idx_mutex);
6038
6039 WREG32(RLC_MC_CNTL, 0);
6040 WREG32(RLC_UCODE_CNTL, 0);
6041
6042 if (rdev->new_fw) {
6043 const struct rlc_firmware_header_v1_0 *hdr =
6044 (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
6045 const __le32 *fw_data = (const __le32 *)
6046 (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6047
6048 radeon_ucode_print_rlc_hdr(&hdr->header);
6049
6050 size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
6051 WREG32(RLC_GPM_UCODE_ADDR, 0);
6052 for (i = 0; i < size; i++)
6053 WREG32(RLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
6054 WREG32(RLC_GPM_UCODE_ADDR, le32_to_cpu(hdr->header.ucode_version));
6055 } else {
6056 const __be32 *fw_data;
6057
6058 switch (rdev->family) {
6059 case CHIP_BONAIRE:
6060 case CHIP_HAWAII:
6061 default:
6062 size = BONAIRE_RLC_UCODE_SIZE;
6063 break;
6064 case CHIP_KAVERI:
6065 size = KV_RLC_UCODE_SIZE;
6066 break;
6067 case CHIP_KABINI:
6068 size = KB_RLC_UCODE_SIZE;
6069 break;
6070 case CHIP_MULLINS:
6071 size = ML_RLC_UCODE_SIZE;
6072 break;
6073 }
6074
6075 fw_data = (const __be32 *)rdev->rlc_fw->data;
6076 WREG32(RLC_GPM_UCODE_ADDR, 0);
6077 for (i = 0; i < size; i++)
6078 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
6079 WREG32(RLC_GPM_UCODE_ADDR, 0);
6080 }
6081
6082 /* XXX - find out what chips support lbpw */
6083 cik_enable_lbpw(rdev, false);
6084
6085 if (rdev->family == CHIP_BONAIRE)
6086 WREG32(RLC_DRIVER_DMA_STATUS, 0);
6087
6088 cik_rlc_start(rdev);
6089
6090 return 0;
6091 }
6092
6093 static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
6094 {
6095 u32 data, orig, tmp, tmp2;
6096
6097 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
6098
6099 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
6100 cik_enable_gui_idle_interrupt(rdev, true);
6101
6102 tmp = cik_halt_rlc(rdev);
6103
6104 mutex_lock(&rdev->grbm_idx_mutex);
6105 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6106 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6107 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6108 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
6109 WREG32(RLC_SERDES_WR_CTRL, tmp2);
6110 mutex_unlock(&rdev->grbm_idx_mutex);
6111
6112 cik_update_rlc(rdev, tmp);
6113
6114 data |= CGCG_EN | CGLS_EN;
6115 } else {
6116 cik_enable_gui_idle_interrupt(rdev, false);
6117
6118 RREG32(CB_CGTT_SCLK_CTRL);
6119 RREG32(CB_CGTT_SCLK_CTRL);
6120 RREG32(CB_CGTT_SCLK_CTRL);
6121 RREG32(CB_CGTT_SCLK_CTRL);
6122
6123 data &= ~(CGCG_EN | CGLS_EN);
6124 }
6125
6126 if (orig != data)
6127 WREG32(RLC_CGCG_CGLS_CTRL, data);
6128
6129 }
6130
6131 static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
6132 {
6133 u32 data, orig, tmp = 0;
6134
6135 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
6136 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
6137 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
6138 orig = data = RREG32(CP_MEM_SLP_CNTL);
6139 data |= CP_MEM_LS_EN;
6140 if (orig != data)
6141 WREG32(CP_MEM_SLP_CNTL, data);
6142 }
6143 }
6144
6145 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
6146 data |= 0x00000001;
6147 data &= 0xfffffffd;
6148 if (orig != data)
6149 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6150
6151 tmp = cik_halt_rlc(rdev);
6152
6153 mutex_lock(&rdev->grbm_idx_mutex);
6154 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6155 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6156 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6157 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
6158 WREG32(RLC_SERDES_WR_CTRL, data);
6159 mutex_unlock(&rdev->grbm_idx_mutex);
6160
6161 cik_update_rlc(rdev, tmp);
6162
6163 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
6164 orig = data = RREG32(CGTS_SM_CTRL_REG);
6165 data &= ~SM_MODE_MASK;
6166 data |= SM_MODE(0x2);
6167 data |= SM_MODE_ENABLE;
6168 data &= ~CGTS_OVERRIDE;
6169 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
6170 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
6171 data &= ~CGTS_LS_OVERRIDE;
6172 data &= ~ON_MONITOR_ADD_MASK;
6173 data |= ON_MONITOR_ADD_EN;
6174 data |= ON_MONITOR_ADD(0x96);
6175 if (orig != data)
6176 WREG32(CGTS_SM_CTRL_REG, data);
6177 }
6178 } else {
6179 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
6180 data |= 0x00000003;
6181 if (orig != data)
6182 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6183
6184 data = RREG32(RLC_MEM_SLP_CNTL);
6185 if (data & RLC_MEM_LS_EN) {
6186 data &= ~RLC_MEM_LS_EN;
6187 WREG32(RLC_MEM_SLP_CNTL, data);
6188 }
6189
6190 data = RREG32(CP_MEM_SLP_CNTL);
6191 if (data & CP_MEM_LS_EN) {
6192 data &= ~CP_MEM_LS_EN;
6193 WREG32(CP_MEM_SLP_CNTL, data);
6194 }
6195
6196 orig = data = RREG32(CGTS_SM_CTRL_REG);
6197 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
6198 if (orig != data)
6199 WREG32(CGTS_SM_CTRL_REG, data);
6200
6201 tmp = cik_halt_rlc(rdev);
6202
6203 mutex_lock(&rdev->grbm_idx_mutex);
6204 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6205 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6206 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6207 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
6208 WREG32(RLC_SERDES_WR_CTRL, data);
6209 mutex_unlock(&rdev->grbm_idx_mutex);
6210
6211 cik_update_rlc(rdev, tmp);
6212 }
6213 }
6214
6215 static const u32 mc_cg_registers[] =
6216 {
6217 MC_HUB_MISC_HUB_CG,
6218 MC_HUB_MISC_SIP_CG,
6219 MC_HUB_MISC_VM_CG,
6220 MC_XPB_CLK_GAT,
6221 ATC_MISC_CG,
6222 MC_CITF_MISC_WR_CG,
6223 MC_CITF_MISC_RD_CG,
6224 MC_CITF_MISC_VM_CG,
6225 VM_L2_CG,
6226 };
6227
6228 static void cik_enable_mc_ls(struct radeon_device *rdev,
6229 bool enable)
6230 {
6231 int i;
6232 u32 orig, data;
6233
6234 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6235 orig = data = RREG32(mc_cg_registers[i]);
6236 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
6237 data |= MC_LS_ENABLE;
6238 else
6239 data &= ~MC_LS_ENABLE;
6240 if (data != orig)
6241 WREG32(mc_cg_registers[i], data);
6242 }
6243 }
6244
6245 static void cik_enable_mc_mgcg(struct radeon_device *rdev,
6246 bool enable)
6247 {
6248 int i;
6249 u32 orig, data;
6250
6251 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6252 orig = data = RREG32(mc_cg_registers[i]);
6253 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
6254 data |= MC_CG_ENABLE;
6255 else
6256 data &= ~MC_CG_ENABLE;
6257 if (data != orig)
6258 WREG32(mc_cg_registers[i], data);
6259 }
6260 }
6261
6262 static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
6263 bool enable)
6264 {
6265 u32 orig, data;
6266
6267 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
6268 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
6269 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
6270 } else {
6271 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
6272 data |= 0xff000000;
6273 if (data != orig)
6274 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
6275
6276 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
6277 data |= 0xff000000;
6278 if (data != orig)
6279 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
6280 }
6281 }
6282
6283 static void cik_enable_sdma_mgls(struct radeon_device *rdev,
6284 bool enable)
6285 {
6286 u32 orig, data;
6287
6288 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
6289 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6290 data |= 0x100;
6291 if (orig != data)
6292 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6293
6294 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6295 data |= 0x100;
6296 if (orig != data)
6297 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6298 } else {
6299 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6300 data &= ~0x100;
6301 if (orig != data)
6302 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6303
6304 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6305 data &= ~0x100;
6306 if (orig != data)
6307 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6308 }
6309 }
6310
6311 static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
6312 bool enable)
6313 {
6314 u32 orig, data;
6315
6316 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
6317 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6318 data = 0xfff;
6319 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6320
6321 orig = data = RREG32(UVD_CGC_CTRL);
6322 data |= DCM;
6323 if (orig != data)
6324 WREG32(UVD_CGC_CTRL, data);
6325 } else {
6326 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6327 data &= ~0xfff;
6328 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6329
6330 orig = data = RREG32(UVD_CGC_CTRL);
6331 data &= ~DCM;
6332 if (orig != data)
6333 WREG32(UVD_CGC_CTRL, data);
6334 }
6335 }
6336
6337 static void cik_enable_bif_mgls(struct radeon_device *rdev,
6338 bool enable)
6339 {
6340 u32 orig, data;
6341
6342 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
6343
6344 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
6345 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
6346 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
6347 else
6348 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
6349 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
6350
6351 if (orig != data)
6352 WREG32_PCIE_PORT(PCIE_CNTL2, data);
6353 }
6354
6355 static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
6356 bool enable)
6357 {
6358 u32 orig, data;
6359
6360 orig = data = RREG32(HDP_HOST_PATH_CNTL);
6361
6362 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
6363 data &= ~CLOCK_GATING_DIS;
6364 else
6365 data |= CLOCK_GATING_DIS;
6366
6367 if (orig != data)
6368 WREG32(HDP_HOST_PATH_CNTL, data);
6369 }
6370
6371 static void cik_enable_hdp_ls(struct radeon_device *rdev,
6372 bool enable)
6373 {
6374 u32 orig, data;
6375
6376 orig = data = RREG32(HDP_MEM_POWER_LS);
6377
6378 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
6379 data |= HDP_LS_ENABLE;
6380 else
6381 data &= ~HDP_LS_ENABLE;
6382
6383 if (orig != data)
6384 WREG32(HDP_MEM_POWER_LS, data);
6385 }
6386
6387 void cik_update_cg(struct radeon_device *rdev,
6388 u32 block, bool enable)
6389 {
6390
6391 if (block & RADEON_CG_BLOCK_GFX) {
6392 cik_enable_gui_idle_interrupt(rdev, false);
6393 /* order matters! */
6394 if (enable) {
6395 cik_enable_mgcg(rdev, true);
6396 cik_enable_cgcg(rdev, true);
6397 } else {
6398 cik_enable_cgcg(rdev, false);
6399 cik_enable_mgcg(rdev, false);
6400 }
6401 cik_enable_gui_idle_interrupt(rdev, true);
6402 }
6403
6404 if (block & RADEON_CG_BLOCK_MC) {
6405 if (!(rdev->flags & RADEON_IS_IGP)) {
6406 cik_enable_mc_mgcg(rdev, enable);
6407 cik_enable_mc_ls(rdev, enable);
6408 }
6409 }
6410
6411 if (block & RADEON_CG_BLOCK_SDMA) {
6412 cik_enable_sdma_mgcg(rdev, enable);
6413 cik_enable_sdma_mgls(rdev, enable);
6414 }
6415
6416 if (block & RADEON_CG_BLOCK_BIF) {
6417 cik_enable_bif_mgls(rdev, enable);
6418 }
6419
6420 if (block & RADEON_CG_BLOCK_UVD) {
6421 if (rdev->has_uvd)
6422 cik_enable_uvd_mgcg(rdev, enable);
6423 }
6424
6425 if (block & RADEON_CG_BLOCK_HDP) {
6426 cik_enable_hdp_mgcg(rdev, enable);
6427 cik_enable_hdp_ls(rdev, enable);
6428 }
6429
6430 if (block & RADEON_CG_BLOCK_VCE) {
6431 vce_v2_0_enable_mgcg(rdev, enable);
6432 }
6433 }
6434
6435 static void cik_init_cg(struct radeon_device *rdev)
6436 {
6437
6438 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
6439
6440 if (rdev->has_uvd)
6441 si_init_uvd_internal_cg(rdev);
6442
6443 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6444 RADEON_CG_BLOCK_SDMA |
6445 RADEON_CG_BLOCK_BIF |
6446 RADEON_CG_BLOCK_UVD |
6447 RADEON_CG_BLOCK_HDP), true);
6448 }
6449
6450 static void cik_fini_cg(struct radeon_device *rdev)
6451 {
6452 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6453 RADEON_CG_BLOCK_SDMA |
6454 RADEON_CG_BLOCK_BIF |
6455 RADEON_CG_BLOCK_UVD |
6456 RADEON_CG_BLOCK_HDP), false);
6457
6458 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
6459 }
6460
6461 static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
6462 bool enable)
6463 {
6464 u32 data, orig;
6465
6466 orig = data = RREG32(RLC_PG_CNTL);
6467 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6468 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6469 else
6470 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6471 if (orig != data)
6472 WREG32(RLC_PG_CNTL, data);
6473 }
6474
6475 static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
6476 bool enable)
6477 {
6478 u32 data, orig;
6479
6480 orig = data = RREG32(RLC_PG_CNTL);
6481 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6482 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6483 else
6484 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6485 if (orig != data)
6486 WREG32(RLC_PG_CNTL, data);
6487 }
6488
6489 static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
6490 {
6491 u32 data, orig;
6492
6493 orig = data = RREG32(RLC_PG_CNTL);
6494 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
6495 data &= ~DISABLE_CP_PG;
6496 else
6497 data |= DISABLE_CP_PG;
6498 if (orig != data)
6499 WREG32(RLC_PG_CNTL, data);
6500 }
6501
6502 static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
6503 {
6504 u32 data, orig;
6505
6506 orig = data = RREG32(RLC_PG_CNTL);
6507 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
6508 data &= ~DISABLE_GDS_PG;
6509 else
6510 data |= DISABLE_GDS_PG;
6511 if (orig != data)
6512 WREG32(RLC_PG_CNTL, data);
6513 }
6514
6515 #define CP_ME_TABLE_SIZE 96
6516 #define CP_ME_TABLE_OFFSET 2048
6517 #define CP_MEC_TABLE_OFFSET 4096
6518
6519 void cik_init_cp_pg_table(struct radeon_device *rdev)
6520 {
6521 volatile u32 *dst_ptr;
6522 int me, i, max_me = 4;
6523 u32 bo_offset = 0;
6524 u32 table_offset, table_size;
6525
6526 if (rdev->family == CHIP_KAVERI)
6527 max_me = 5;
6528
6529 if (rdev->rlc.cp_table_ptr == NULL)
6530 return;
6531
6532 /* write the cp table buffer */
6533 dst_ptr = rdev->rlc.cp_table_ptr;
6534 for (me = 0; me < max_me; me++) {
6535 if (rdev->new_fw) {
6536 const __le32 *fw_data;
6537 const struct gfx_firmware_header_v1_0 *hdr;
6538
6539 if (me == 0) {
6540 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
6541 fw_data = (const __le32 *)
6542 (rdev->ce_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6543 table_offset = le32_to_cpu(hdr->jt_offset);
6544 table_size = le32_to_cpu(hdr->jt_size);
6545 } else if (me == 1) {
6546 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
6547 fw_data = (const __le32 *)
6548 (rdev->pfp_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6549 table_offset = le32_to_cpu(hdr->jt_offset);
6550 table_size = le32_to_cpu(hdr->jt_size);
6551 } else if (me == 2) {
6552 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
6553 fw_data = (const __le32 *)
6554 (rdev->me_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6555 table_offset = le32_to_cpu(hdr->jt_offset);
6556 table_size = le32_to_cpu(hdr->jt_size);
6557 } else if (me == 3) {
6558 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
6559 fw_data = (const __le32 *)
6560 (rdev->mec_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6561 table_offset = le32_to_cpu(hdr->jt_offset);
6562 table_size = le32_to_cpu(hdr->jt_size);
6563 } else {
6564 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
6565 fw_data = (const __le32 *)
6566 (rdev->mec2_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6567 table_offset = le32_to_cpu(hdr->jt_offset);
6568 table_size = le32_to_cpu(hdr->jt_size);
6569 }
6570
6571 for (i = 0; i < table_size; i ++) {
6572 dst_ptr[bo_offset + i] =
6573 cpu_to_le32(le32_to_cpu(fw_data[table_offset + i]));
6574 }
6575 bo_offset += table_size;
6576 } else {
6577 const __be32 *fw_data;
6578 table_size = CP_ME_TABLE_SIZE;
6579
6580 if (me == 0) {
6581 fw_data = (const __be32 *)rdev->ce_fw->data;
6582 table_offset = CP_ME_TABLE_OFFSET;
6583 } else if (me == 1) {
6584 fw_data = (const __be32 *)rdev->pfp_fw->data;
6585 table_offset = CP_ME_TABLE_OFFSET;
6586 } else if (me == 2) {
6587 fw_data = (const __be32 *)rdev->me_fw->data;
6588 table_offset = CP_ME_TABLE_OFFSET;
6589 } else {
6590 fw_data = (const __be32 *)rdev->mec_fw->data;
6591 table_offset = CP_MEC_TABLE_OFFSET;
6592 }
6593
6594 for (i = 0; i < table_size; i ++) {
6595 dst_ptr[bo_offset + i] =
6596 cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
6597 }
6598 bo_offset += table_size;
6599 }
6600 }
6601 }
6602
6603 static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6604 bool enable)
6605 {
6606 u32 data, orig;
6607
6608 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
6609 orig = data = RREG32(RLC_PG_CNTL);
6610 data |= GFX_PG_ENABLE;
6611 if (orig != data)
6612 WREG32(RLC_PG_CNTL, data);
6613
6614 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6615 data |= AUTO_PG_EN;
6616 if (orig != data)
6617 WREG32(RLC_AUTO_PG_CTRL, data);
6618 } else {
6619 orig = data = RREG32(RLC_PG_CNTL);
6620 data &= ~GFX_PG_ENABLE;
6621 if (orig != data)
6622 WREG32(RLC_PG_CNTL, data);
6623
6624 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6625 data &= ~AUTO_PG_EN;
6626 if (orig != data)
6627 WREG32(RLC_AUTO_PG_CTRL, data);
6628
6629 data = RREG32(DB_RENDER_CONTROL);
6630 }
6631 }
6632
6633 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6634 {
6635 u32 mask = 0, tmp, tmp1;
6636 int i;
6637
6638 mutex_lock(&rdev->grbm_idx_mutex);
6639 cik_select_se_sh(rdev, se, sh);
6640 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6641 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6642 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6643 mutex_unlock(&rdev->grbm_idx_mutex);
6644
6645 tmp &= 0xffff0000;
6646
6647 tmp |= tmp1;
6648 tmp >>= 16;
6649
6650 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6651 mask <<= 1;
6652 mask |= 1;
6653 }
6654
6655 return (~tmp) & mask;
6656 }
6657
6658 static void cik_init_ao_cu_mask(struct radeon_device *rdev)
6659 {
6660 u32 i, j, k, active_cu_number = 0;
6661 u32 mask, counter, cu_bitmap;
6662 u32 tmp = 0;
6663
6664 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6665 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6666 mask = 1;
6667 cu_bitmap = 0;
6668 counter = 0;
6669 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6670 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6671 if (counter < 2)
6672 cu_bitmap |= mask;
6673 counter ++;
6674 }
6675 mask <<= 1;
6676 }
6677
6678 active_cu_number += counter;
6679 tmp |= (cu_bitmap << (i * 16 + j * 8));
6680 }
6681 }
6682
6683 WREG32(RLC_PG_AO_CU_MASK, tmp);
6684
6685 tmp = RREG32(RLC_MAX_PG_CU);
6686 tmp &= ~MAX_PU_CU_MASK;
6687 tmp |= MAX_PU_CU(active_cu_number);
6688 WREG32(RLC_MAX_PG_CU, tmp);
6689 }
6690
6691 static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6692 bool enable)
6693 {
6694 u32 data, orig;
6695
6696 orig = data = RREG32(RLC_PG_CNTL);
6697 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
6698 data |= STATIC_PER_CU_PG_ENABLE;
6699 else
6700 data &= ~STATIC_PER_CU_PG_ENABLE;
6701 if (orig != data)
6702 WREG32(RLC_PG_CNTL, data);
6703 }
6704
6705 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6706 bool enable)
6707 {
6708 u32 data, orig;
6709
6710 orig = data = RREG32(RLC_PG_CNTL);
6711 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
6712 data |= DYN_PER_CU_PG_ENABLE;
6713 else
6714 data &= ~DYN_PER_CU_PG_ENABLE;
6715 if (orig != data)
6716 WREG32(RLC_PG_CNTL, data);
6717 }
6718
6719 #define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6720 #define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
6721
6722 static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6723 {
6724 u32 data, orig;
6725 u32 i;
6726
6727 if (rdev->rlc.cs_data) {
6728 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6729 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
6730 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
6731 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
6732 } else {
6733 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6734 for (i = 0; i < 3; i++)
6735 WREG32(RLC_GPM_SCRATCH_DATA, 0);
6736 }
6737 if (rdev->rlc.reg_list) {
6738 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6739 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6740 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
6741 }
6742
6743 orig = data = RREG32(RLC_PG_CNTL);
6744 data |= GFX_PG_SRC;
6745 if (orig != data)
6746 WREG32(RLC_PG_CNTL, data);
6747
6748 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6749 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
6750
6751 data = RREG32(CP_RB_WPTR_POLL_CNTL);
6752 data &= ~IDLE_POLL_COUNT_MASK;
6753 data |= IDLE_POLL_COUNT(0x60);
6754 WREG32(CP_RB_WPTR_POLL_CNTL, data);
6755
6756 data = 0x10101010;
6757 WREG32(RLC_PG_DELAY, data);
6758
6759 data = RREG32(RLC_PG_DELAY_2);
6760 data &= ~0xff;
6761 data |= 0x3;
6762 WREG32(RLC_PG_DELAY_2, data);
6763
6764 data = RREG32(RLC_AUTO_PG_CTRL);
6765 data &= ~GRBM_REG_SGIT_MASK;
6766 data |= GRBM_REG_SGIT(0x700);
6767 WREG32(RLC_AUTO_PG_CTRL, data);
6768
6769 }
6770
6771 static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
6772 {
6773 cik_enable_gfx_cgpg(rdev, enable);
6774 cik_enable_gfx_static_mgpg(rdev, enable);
6775 cik_enable_gfx_dynamic_mgpg(rdev, enable);
6776 }
6777
6778 u32 cik_get_csb_size(struct radeon_device *rdev)
6779 {
6780 u32 count = 0;
6781 const struct cs_section_def *sect = NULL;
6782 const struct cs_extent_def *ext = NULL;
6783
6784 if (rdev->rlc.cs_data == NULL)
6785 return 0;
6786
6787 /* begin clear state */
6788 count += 2;
6789 /* context control state */
6790 count += 3;
6791
6792 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6793 for (ext = sect->section; ext->extent != NULL; ++ext) {
6794 if (sect->id == SECT_CONTEXT)
6795 count += 2 + ext->reg_count;
6796 else
6797 return 0;
6798 }
6799 }
6800 /* pa_sc_raster_config/pa_sc_raster_config1 */
6801 count += 4;
6802 /* end clear state */
6803 count += 2;
6804 /* clear state */
6805 count += 2;
6806
6807 return count;
6808 }
6809
6810 void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
6811 {
6812 u32 count = 0, i;
6813 const struct cs_section_def *sect = NULL;
6814 const struct cs_extent_def *ext = NULL;
6815
6816 if (rdev->rlc.cs_data == NULL)
6817 return;
6818 if (buffer == NULL)
6819 return;
6820
6821 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6822 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
6823
6824 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6825 buffer[count++] = cpu_to_le32(0x80000000);
6826 buffer[count++] = cpu_to_le32(0x80000000);
6827
6828 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6829 for (ext = sect->section; ext->extent != NULL; ++ext) {
6830 if (sect->id == SECT_CONTEXT) {
6831 buffer[count++] =
6832 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
6833 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
6834 for (i = 0; i < ext->reg_count; i++)
6835 buffer[count++] = cpu_to_le32(ext->extent[i]);
6836 } else {
6837 return;
6838 }
6839 }
6840 }
6841
6842 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
6843 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
6844 switch (rdev->family) {
6845 case CHIP_BONAIRE:
6846 buffer[count++] = cpu_to_le32(0x16000012);
6847 buffer[count++] = cpu_to_le32(0x00000000);
6848 break;
6849 case CHIP_KAVERI:
6850 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6851 buffer[count++] = cpu_to_le32(0x00000000);
6852 break;
6853 case CHIP_KABINI:
6854 case CHIP_MULLINS:
6855 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6856 buffer[count++] = cpu_to_le32(0x00000000);
6857 break;
6858 case CHIP_HAWAII:
6859 buffer[count++] = cpu_to_le32(0x3a00161a);
6860 buffer[count++] = cpu_to_le32(0x0000002e);
6861 break;
6862 default:
6863 buffer[count++] = cpu_to_le32(0x00000000);
6864 buffer[count++] = cpu_to_le32(0x00000000);
6865 break;
6866 }
6867
6868 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6869 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
6870
6871 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
6872 buffer[count++] = cpu_to_le32(0);
6873 }
6874
6875 static void cik_init_pg(struct radeon_device *rdev)
6876 {
6877 if (rdev->pg_flags) {
6878 cik_enable_sck_slowdown_on_pu(rdev, true);
6879 cik_enable_sck_slowdown_on_pd(rdev, true);
6880 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
6881 cik_init_gfx_cgpg(rdev);
6882 cik_enable_cp_pg(rdev, true);
6883 cik_enable_gds_pg(rdev, true);
6884 }
6885 cik_init_ao_cu_mask(rdev);
6886 cik_update_gfx_pg(rdev, true);
6887 }
6888 }
6889
6890 static void cik_fini_pg(struct radeon_device *rdev)
6891 {
6892 if (rdev->pg_flags) {
6893 cik_update_gfx_pg(rdev, false);
6894 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
6895 cik_enable_cp_pg(rdev, false);
6896 cik_enable_gds_pg(rdev, false);
6897 }
6898 }
6899 }
6900
6901 /*
6902 * Interrupts
6903 * Starting with r6xx, interrupts are handled via a ring buffer.
6904 * Ring buffers are areas of GPU accessible memory that the GPU
6905 * writes interrupt vectors into and the host reads vectors out of.
6906 * There is a rptr (read pointer) that determines where the
6907 * host is currently reading, and a wptr (write pointer)
6908 * which determines where the GPU has written. When the
6909 * pointers are equal, the ring is idle. When the GPU
6910 * writes vectors to the ring buffer, it increments the
6911 * wptr. When there is an interrupt, the host then starts
6912 * fetching commands and processing them until the pointers are
6913 * equal again at which point it updates the rptr.
6914 */
6915
6916 /**
6917 * cik_enable_interrupts - Enable the interrupt ring buffer
6918 *
6919 * @rdev: radeon_device pointer
6920 *
6921 * Enable the interrupt ring buffer (CIK).
6922 */
6923 static void cik_enable_interrupts(struct radeon_device *rdev)
6924 {
6925 u32 ih_cntl = RREG32(IH_CNTL);
6926 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6927
6928 ih_cntl |= ENABLE_INTR;
6929 ih_rb_cntl |= IH_RB_ENABLE;
6930 WREG32(IH_CNTL, ih_cntl);
6931 WREG32(IH_RB_CNTL, ih_rb_cntl);
6932 rdev->ih.enabled = true;
6933 }
6934
6935 /**
6936 * cik_disable_interrupts - Disable the interrupt ring buffer
6937 *
6938 * @rdev: radeon_device pointer
6939 *
6940 * Disable the interrupt ring buffer (CIK).
6941 */
6942 static void cik_disable_interrupts(struct radeon_device *rdev)
6943 {
6944 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6945 u32 ih_cntl = RREG32(IH_CNTL);
6946
6947 ih_rb_cntl &= ~IH_RB_ENABLE;
6948 ih_cntl &= ~ENABLE_INTR;
6949 WREG32(IH_RB_CNTL, ih_rb_cntl);
6950 WREG32(IH_CNTL, ih_cntl);
6951 /* set rptr, wptr to 0 */
6952 WREG32(IH_RB_RPTR, 0);
6953 WREG32(IH_RB_WPTR, 0);
6954 rdev->ih.enabled = false;
6955 rdev->ih.rptr = 0;
6956 }
6957
6958 /**
6959 * cik_disable_interrupt_state - Disable all interrupt sources
6960 *
6961 * @rdev: radeon_device pointer
6962 *
6963 * Clear all interrupt enable bits used by the driver (CIK).
6964 */
6965 static void cik_disable_interrupt_state(struct radeon_device *rdev)
6966 {
6967 u32 tmp;
6968
6969 /* gfx ring */
6970 tmp = RREG32(CP_INT_CNTL_RING0) &
6971 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6972 WREG32(CP_INT_CNTL_RING0, tmp);
6973 /* sdma */
6974 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6975 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
6976 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6977 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
6978 /* compute queues */
6979 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
6980 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
6981 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
6982 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
6983 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
6984 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
6985 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
6986 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
6987 /* grbm */
6988 WREG32(GRBM_INT_CNTL, 0);
6989 /* SRBM */
6990 WREG32(SRBM_INT_CNTL, 0);
6991 /* vline/vblank, etc. */
6992 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6993 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6994 if (rdev->num_crtc >= 4) {
6995 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6996 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6997 }
6998 if (rdev->num_crtc >= 6) {
6999 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
7000 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
7001 }
7002 /* pflip */
7003 if (rdev->num_crtc >= 2) {
7004 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
7005 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
7006 }
7007 if (rdev->num_crtc >= 4) {
7008 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
7009 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
7010 }
7011 if (rdev->num_crtc >= 6) {
7012 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
7013 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
7014 }
7015
7016 /* dac hotplug */
7017 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
7018
7019 /* digital hotplug */
7020 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7021 WREG32(DC_HPD1_INT_CONTROL, tmp);
7022 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7023 WREG32(DC_HPD2_INT_CONTROL, tmp);
7024 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7025 WREG32(DC_HPD3_INT_CONTROL, tmp);
7026 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7027 WREG32(DC_HPD4_INT_CONTROL, tmp);
7028 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7029 WREG32(DC_HPD5_INT_CONTROL, tmp);
7030 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7031 WREG32(DC_HPD6_INT_CONTROL, tmp);
7032
7033 }
7034
7035 /**
7036 * cik_irq_init - init and enable the interrupt ring
7037 *
7038 * @rdev: radeon_device pointer
7039 *
7040 * Allocate a ring buffer for the interrupt controller,
7041 * enable the RLC, disable interrupts, enable the IH
7042 * ring buffer and enable it (CIK).
7043 * Called at device load and reume.
7044 * Returns 0 for success, errors for failure.
7045 */
7046 static int cik_irq_init(struct radeon_device *rdev)
7047 {
7048 int ret = 0;
7049 int rb_bufsz;
7050 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
7051
7052 /* allocate ring */
7053 ret = r600_ih_ring_alloc(rdev);
7054 if (ret)
7055 return ret;
7056
7057 /* disable irqs */
7058 cik_disable_interrupts(rdev);
7059
7060 /* init rlc */
7061 ret = cik_rlc_resume(rdev);
7062 if (ret) {
7063 r600_ih_ring_fini(rdev);
7064 return ret;
7065 }
7066
7067 /* setup interrupt control */
7068 /* XXX this should actually be a bus address, not an MC address. same on older asics */
7069 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
7070 interrupt_cntl = RREG32(INTERRUPT_CNTL);
7071 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
7072 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
7073 */
7074 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
7075 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
7076 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
7077 WREG32(INTERRUPT_CNTL, interrupt_cntl);
7078
7079 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
7080 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
7081
7082 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
7083 IH_WPTR_OVERFLOW_CLEAR |
7084 (rb_bufsz << 1));
7085
7086 if (rdev->wb.enabled)
7087 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
7088
7089 /* set the writeback address whether it's enabled or not */
7090 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
7091 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
7092
7093 WREG32(IH_RB_CNTL, ih_rb_cntl);
7094
7095 /* set rptr, wptr to 0 */
7096 WREG32(IH_RB_RPTR, 0);
7097 WREG32(IH_RB_WPTR, 0);
7098
7099 /* Default settings for IH_CNTL (disabled at first) */
7100 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
7101 /* RPTR_REARM only works if msi's are enabled */
7102 if (rdev->msi_enabled)
7103 ih_cntl |= RPTR_REARM;
7104 WREG32(IH_CNTL, ih_cntl);
7105
7106 /* force the active interrupt state to all disabled */
7107 cik_disable_interrupt_state(rdev);
7108
7109 pci_set_master(rdev->pdev);
7110
7111 /* enable irqs */
7112 cik_enable_interrupts(rdev);
7113
7114 return ret;
7115 }
7116
7117 /**
7118 * cik_irq_set - enable/disable interrupt sources
7119 *
7120 * @rdev: radeon_device pointer
7121 *
7122 * Enable interrupt sources on the GPU (vblanks, hpd,
7123 * etc.) (CIK).
7124 * Returns 0 for success, errors for failure.
7125 */
7126 int cik_irq_set(struct radeon_device *rdev)
7127 {
7128 u32 cp_int_cntl;
7129 u32 cp_m1p0;
7130 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
7131 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
7132 u32 grbm_int_cntl = 0;
7133 u32 dma_cntl, dma_cntl1;
7134
7135 if (!rdev->irq.installed) {
7136 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
7137 return -EINVAL;
7138 }
7139 /* don't enable anything if the ih is disabled */
7140 if (!rdev->ih.enabled) {
7141 cik_disable_interrupts(rdev);
7142 /* force the active interrupt state to all disabled */
7143 cik_disable_interrupt_state(rdev);
7144 return 0;
7145 }
7146
7147 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
7148 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
7149 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
7150
7151 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7152 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7153 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7154 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7155 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7156 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7157
7158 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
7159 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7160
7161 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7162
7163 /* enable CP interrupts on all rings */
7164 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
7165 DRM_DEBUG("cik_irq_set: sw int gfx\n");
7166 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
7167 }
7168 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
7169 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7170 DRM_DEBUG("si_irq_set: sw int cp1\n");
7171 if (ring->me == 1) {
7172 switch (ring->pipe) {
7173 case 0:
7174 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7175 break;
7176 default:
7177 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7178 break;
7179 }
7180 } else {
7181 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
7182 }
7183 }
7184 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
7185 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7186 DRM_DEBUG("si_irq_set: sw int cp2\n");
7187 if (ring->me == 1) {
7188 switch (ring->pipe) {
7189 case 0:
7190 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7191 break;
7192 default:
7193 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7194 break;
7195 }
7196 } else {
7197 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
7198 }
7199 }
7200
7201 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
7202 DRM_DEBUG("cik_irq_set: sw int dma\n");
7203 dma_cntl |= TRAP_ENABLE;
7204 }
7205
7206 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
7207 DRM_DEBUG("cik_irq_set: sw int dma1\n");
7208 dma_cntl1 |= TRAP_ENABLE;
7209 }
7210
7211 if (rdev->irq.crtc_vblank_int[0] ||
7212 atomic_read(&rdev->irq.pflip[0])) {
7213 DRM_DEBUG("cik_irq_set: vblank 0\n");
7214 crtc1 |= VBLANK_INTERRUPT_MASK;
7215 }
7216 if (rdev->irq.crtc_vblank_int[1] ||
7217 atomic_read(&rdev->irq.pflip[1])) {
7218 DRM_DEBUG("cik_irq_set: vblank 1\n");
7219 crtc2 |= VBLANK_INTERRUPT_MASK;
7220 }
7221 if (rdev->irq.crtc_vblank_int[2] ||
7222 atomic_read(&rdev->irq.pflip[2])) {
7223 DRM_DEBUG("cik_irq_set: vblank 2\n");
7224 crtc3 |= VBLANK_INTERRUPT_MASK;
7225 }
7226 if (rdev->irq.crtc_vblank_int[3] ||
7227 atomic_read(&rdev->irq.pflip[3])) {
7228 DRM_DEBUG("cik_irq_set: vblank 3\n");
7229 crtc4 |= VBLANK_INTERRUPT_MASK;
7230 }
7231 if (rdev->irq.crtc_vblank_int[4] ||
7232 atomic_read(&rdev->irq.pflip[4])) {
7233 DRM_DEBUG("cik_irq_set: vblank 4\n");
7234 crtc5 |= VBLANK_INTERRUPT_MASK;
7235 }
7236 if (rdev->irq.crtc_vblank_int[5] ||
7237 atomic_read(&rdev->irq.pflip[5])) {
7238 DRM_DEBUG("cik_irq_set: vblank 5\n");
7239 crtc6 |= VBLANK_INTERRUPT_MASK;
7240 }
7241 if (rdev->irq.hpd[0]) {
7242 DRM_DEBUG("cik_irq_set: hpd 1\n");
7243 hpd1 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
7244 }
7245 if (rdev->irq.hpd[1]) {
7246 DRM_DEBUG("cik_irq_set: hpd 2\n");
7247 hpd2 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
7248 }
7249 if (rdev->irq.hpd[2]) {
7250 DRM_DEBUG("cik_irq_set: hpd 3\n");
7251 hpd3 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
7252 }
7253 if (rdev->irq.hpd[3]) {
7254 DRM_DEBUG("cik_irq_set: hpd 4\n");
7255 hpd4 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
7256 }
7257 if (rdev->irq.hpd[4]) {
7258 DRM_DEBUG("cik_irq_set: hpd 5\n");
7259 hpd5 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
7260 }
7261 if (rdev->irq.hpd[5]) {
7262 DRM_DEBUG("cik_irq_set: hpd 6\n");
7263 hpd6 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
7264 }
7265
7266 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
7267
7268 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
7269 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
7270
7271 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
7272
7273 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
7274
7275 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
7276 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
7277 if (rdev->num_crtc >= 4) {
7278 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
7279 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
7280 }
7281 if (rdev->num_crtc >= 6) {
7282 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
7283 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
7284 }
7285
7286 if (rdev->num_crtc >= 2) {
7287 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
7288 GRPH_PFLIP_INT_MASK);
7289 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
7290 GRPH_PFLIP_INT_MASK);
7291 }
7292 if (rdev->num_crtc >= 4) {
7293 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
7294 GRPH_PFLIP_INT_MASK);
7295 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
7296 GRPH_PFLIP_INT_MASK);
7297 }
7298 if (rdev->num_crtc >= 6) {
7299 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
7300 GRPH_PFLIP_INT_MASK);
7301 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
7302 GRPH_PFLIP_INT_MASK);
7303 }
7304
7305 WREG32(DC_HPD1_INT_CONTROL, hpd1);
7306 WREG32(DC_HPD2_INT_CONTROL, hpd2);
7307 WREG32(DC_HPD3_INT_CONTROL, hpd3);
7308 WREG32(DC_HPD4_INT_CONTROL, hpd4);
7309 WREG32(DC_HPD5_INT_CONTROL, hpd5);
7310 WREG32(DC_HPD6_INT_CONTROL, hpd6);
7311
7312 /* posting read */
7313 RREG32(SRBM_STATUS);
7314
7315 return 0;
7316 }
7317
7318 /**
7319 * cik_irq_ack - ack interrupt sources
7320 *
7321 * @rdev: radeon_device pointer
7322 *
7323 * Ack interrupt sources on the GPU (vblanks, hpd,
7324 * etc.) (CIK). Certain interrupts sources are sw
7325 * generated and do not require an explicit ack.
7326 */
7327 static inline void cik_irq_ack(struct radeon_device *rdev)
7328 {
7329 u32 tmp;
7330
7331 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
7332 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
7333 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
7334 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
7335 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
7336 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
7337 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
7338
7339 rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS +
7340 EVERGREEN_CRTC0_REGISTER_OFFSET);
7341 rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS +
7342 EVERGREEN_CRTC1_REGISTER_OFFSET);
7343 if (rdev->num_crtc >= 4) {
7344 rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS +
7345 EVERGREEN_CRTC2_REGISTER_OFFSET);
7346 rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS +
7347 EVERGREEN_CRTC3_REGISTER_OFFSET);
7348 }
7349 if (rdev->num_crtc >= 6) {
7350 rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS +
7351 EVERGREEN_CRTC4_REGISTER_OFFSET);
7352 rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS +
7353 EVERGREEN_CRTC5_REGISTER_OFFSET);
7354 }
7355
7356 if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
7357 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET,
7358 GRPH_PFLIP_INT_CLEAR);
7359 if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
7360 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET,
7361 GRPH_PFLIP_INT_CLEAR);
7362 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
7363 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
7364 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
7365 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
7366 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
7367 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
7368 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
7369 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
7370
7371 if (rdev->num_crtc >= 4) {
7372 if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
7373 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET,
7374 GRPH_PFLIP_INT_CLEAR);
7375 if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
7376 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET,
7377 GRPH_PFLIP_INT_CLEAR);
7378 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
7379 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
7380 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
7381 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
7382 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
7383 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
7384 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
7385 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
7386 }
7387
7388 if (rdev->num_crtc >= 6) {
7389 if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
7390 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET,
7391 GRPH_PFLIP_INT_CLEAR);
7392 if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
7393 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET,
7394 GRPH_PFLIP_INT_CLEAR);
7395 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
7396 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
7397 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
7398 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
7399 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
7400 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
7401 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
7402 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
7403 }
7404
7405 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7406 tmp = RREG32(DC_HPD1_INT_CONTROL);
7407 tmp |= DC_HPDx_INT_ACK;
7408 WREG32(DC_HPD1_INT_CONTROL, tmp);
7409 }
7410 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7411 tmp = RREG32(DC_HPD2_INT_CONTROL);
7412 tmp |= DC_HPDx_INT_ACK;
7413 WREG32(DC_HPD2_INT_CONTROL, tmp);
7414 }
7415 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7416 tmp = RREG32(DC_HPD3_INT_CONTROL);
7417 tmp |= DC_HPDx_INT_ACK;
7418 WREG32(DC_HPD3_INT_CONTROL, tmp);
7419 }
7420 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7421 tmp = RREG32(DC_HPD4_INT_CONTROL);
7422 tmp |= DC_HPDx_INT_ACK;
7423 WREG32(DC_HPD4_INT_CONTROL, tmp);
7424 }
7425 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7426 tmp = RREG32(DC_HPD5_INT_CONTROL);
7427 tmp |= DC_HPDx_INT_ACK;
7428 WREG32(DC_HPD5_INT_CONTROL, tmp);
7429 }
7430 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7431 tmp = RREG32(DC_HPD5_INT_CONTROL);
7432 tmp |= DC_HPDx_INT_ACK;
7433 WREG32(DC_HPD6_INT_CONTROL, tmp);
7434 }
7435 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_RX_INTERRUPT) {
7436 tmp = RREG32(DC_HPD1_INT_CONTROL);
7437 tmp |= DC_HPDx_RX_INT_ACK;
7438 WREG32(DC_HPD1_INT_CONTROL, tmp);
7439 }
7440 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_RX_INTERRUPT) {
7441 tmp = RREG32(DC_HPD2_INT_CONTROL);
7442 tmp |= DC_HPDx_RX_INT_ACK;
7443 WREG32(DC_HPD2_INT_CONTROL, tmp);
7444 }
7445 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) {
7446 tmp = RREG32(DC_HPD3_INT_CONTROL);
7447 tmp |= DC_HPDx_RX_INT_ACK;
7448 WREG32(DC_HPD3_INT_CONTROL, tmp);
7449 }
7450 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) {
7451 tmp = RREG32(DC_HPD4_INT_CONTROL);
7452 tmp |= DC_HPDx_RX_INT_ACK;
7453 WREG32(DC_HPD4_INT_CONTROL, tmp);
7454 }
7455 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) {
7456 tmp = RREG32(DC_HPD5_INT_CONTROL);
7457 tmp |= DC_HPDx_RX_INT_ACK;
7458 WREG32(DC_HPD5_INT_CONTROL, tmp);
7459 }
7460 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) {
7461 tmp = RREG32(DC_HPD5_INT_CONTROL);
7462 tmp |= DC_HPDx_RX_INT_ACK;
7463 WREG32(DC_HPD6_INT_CONTROL, tmp);
7464 }
7465 }
7466
7467 /**
7468 * cik_irq_disable - disable interrupts
7469 *
7470 * @rdev: radeon_device pointer
7471 *
7472 * Disable interrupts on the hw (CIK).
7473 */
7474 static void cik_irq_disable(struct radeon_device *rdev)
7475 {
7476 cik_disable_interrupts(rdev);
7477 /* Wait and acknowledge irq */
7478 mdelay(1);
7479 cik_irq_ack(rdev);
7480 cik_disable_interrupt_state(rdev);
7481 }
7482
7483 /**
7484 * cik_irq_disable - disable interrupts for suspend
7485 *
7486 * @rdev: radeon_device pointer
7487 *
7488 * Disable interrupts and stop the RLC (CIK).
7489 * Used for suspend.
7490 */
7491 static void cik_irq_suspend(struct radeon_device *rdev)
7492 {
7493 cik_irq_disable(rdev);
7494 cik_rlc_stop(rdev);
7495 }
7496
7497 /**
7498 * cik_irq_fini - tear down interrupt support
7499 *
7500 * @rdev: radeon_device pointer
7501 *
7502 * Disable interrupts on the hw and free the IH ring
7503 * buffer (CIK).
7504 * Used for driver unload.
7505 */
7506 static void cik_irq_fini(struct radeon_device *rdev)
7507 {
7508 cik_irq_suspend(rdev);
7509 r600_ih_ring_fini(rdev);
7510 }
7511
7512 /**
7513 * cik_get_ih_wptr - get the IH ring buffer wptr
7514 *
7515 * @rdev: radeon_device pointer
7516 *
7517 * Get the IH ring buffer wptr from either the register
7518 * or the writeback memory buffer (CIK). Also check for
7519 * ring buffer overflow and deal with it.
7520 * Used by cik_irq_process().
7521 * Returns the value of the wptr.
7522 */
7523 static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
7524 {
7525 u32 wptr, tmp;
7526
7527 if (rdev->wb.enabled)
7528 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
7529 else
7530 wptr = RREG32(IH_RB_WPTR);
7531
7532 if (wptr & RB_OVERFLOW) {
7533 wptr &= ~RB_OVERFLOW;
7534 /* When a ring buffer overflow happen start parsing interrupt
7535 * from the last not overwritten vector (wptr + 16). Hopefully
7536 * this should allow us to catchup.
7537 */
7538 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
7539 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
7540 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
7541 tmp = RREG32(IH_RB_CNTL);
7542 tmp |= IH_WPTR_OVERFLOW_CLEAR;
7543 WREG32(IH_RB_CNTL, tmp);
7544 }
7545 return (wptr & rdev->ih.ptr_mask);
7546 }
7547
7548 /* CIK IV Ring
7549 * Each IV ring entry is 128 bits:
7550 * [7:0] - interrupt source id
7551 * [31:8] - reserved
7552 * [59:32] - interrupt source data
7553 * [63:60] - reserved
7554 * [71:64] - RINGID
7555 * CP:
7556 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
7557 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7558 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7559 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7560 * PIPE_ID - ME0 0=3D
7561 * - ME1&2 compute dispatcher (4 pipes each)
7562 * SDMA:
7563 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
7564 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
7565 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
7566 * [79:72] - VMID
7567 * [95:80] - PASID
7568 * [127:96] - reserved
7569 */
7570 /**
7571 * cik_irq_process - interrupt handler
7572 *
7573 * @rdev: radeon_device pointer
7574 *
7575 * Interrupt hander (CIK). Walk the IH ring,
7576 * ack interrupts and schedule work to handle
7577 * interrupt events.
7578 * Returns irq process return code.
7579 */
7580 int cik_irq_process(struct radeon_device *rdev)
7581 {
7582 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7583 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7584 u32 wptr;
7585 u32 rptr;
7586 u32 src_id, src_data, ring_id;
7587 u8 me_id, pipe_id, queue_id;
7588 u32 ring_index;
7589 bool queue_hotplug = false;
7590 bool queue_dp = false;
7591 bool queue_reset = false;
7592 u32 addr, status, mc_client;
7593 bool queue_thermal = false;
7594
7595 if (!rdev->ih.enabled || rdev->shutdown)
7596 return IRQ_NONE;
7597
7598 wptr = cik_get_ih_wptr(rdev);
7599
7600 restart_ih:
7601 /* is somebody else already processing irqs? */
7602 if (atomic_xchg(&rdev->ih.lock, 1))
7603 return IRQ_NONE;
7604
7605 rptr = rdev->ih.rptr;
7606 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7607
7608 /* Order reading of wptr vs. reading of IH ring data */
7609 rmb();
7610
7611 /* display interrupts */
7612 cik_irq_ack(rdev);
7613
7614 while (rptr != wptr) {
7615 /* wptr/rptr are in bytes! */
7616 ring_index = rptr / 4;
7617
7618 radeon_kfd_interrupt(rdev,
7619 (const void *) &rdev->ih.ring[ring_index]);
7620
7621 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7622 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7623 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
7624
7625 switch (src_id) {
7626 case 1: /* D1 vblank/vline */
7627 switch (src_data) {
7628 case 0: /* D1 vblank */
7629 if (!(rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT))
7630 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7631
7632 if (rdev->irq.crtc_vblank_int[0]) {
7633 drm_handle_vblank(rdev->ddev, 0);
7634 rdev->pm.vblank_sync = true;
7635 wake_up(&rdev->irq.vblank_queue);
7636 }
7637 if (atomic_read(&rdev->irq.pflip[0]))
7638 radeon_crtc_handle_vblank(rdev, 0);
7639 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7640 DRM_DEBUG("IH: D1 vblank\n");
7641
7642 break;
7643 case 1: /* D1 vline */
7644 if (!(rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT))
7645 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7646
7647 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7648 DRM_DEBUG("IH: D1 vline\n");
7649
7650 break;
7651 default:
7652 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7653 break;
7654 }
7655 break;
7656 case 2: /* D2 vblank/vline */
7657 switch (src_data) {
7658 case 0: /* D2 vblank */
7659 if (!(rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT))
7660 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7661
7662 if (rdev->irq.crtc_vblank_int[1]) {
7663 drm_handle_vblank(rdev->ddev, 1);
7664 rdev->pm.vblank_sync = true;
7665 wake_up(&rdev->irq.vblank_queue);
7666 }
7667 if (atomic_read(&rdev->irq.pflip[1]))
7668 radeon_crtc_handle_vblank(rdev, 1);
7669 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7670 DRM_DEBUG("IH: D2 vblank\n");
7671
7672 break;
7673 case 1: /* D2 vline */
7674 if (!(rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT))
7675 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7676
7677 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7678 DRM_DEBUG("IH: D2 vline\n");
7679
7680 break;
7681 default:
7682 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7683 break;
7684 }
7685 break;
7686 case 3: /* D3 vblank/vline */
7687 switch (src_data) {
7688 case 0: /* D3 vblank */
7689 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT))
7690 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7691
7692 if (rdev->irq.crtc_vblank_int[2]) {
7693 drm_handle_vblank(rdev->ddev, 2);
7694 rdev->pm.vblank_sync = true;
7695 wake_up(&rdev->irq.vblank_queue);
7696 }
7697 if (atomic_read(&rdev->irq.pflip[2]))
7698 radeon_crtc_handle_vblank(rdev, 2);
7699 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7700 DRM_DEBUG("IH: D3 vblank\n");
7701
7702 break;
7703 case 1: /* D3 vline */
7704 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT))
7705 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7706
7707 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7708 DRM_DEBUG("IH: D3 vline\n");
7709
7710 break;
7711 default:
7712 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7713 break;
7714 }
7715 break;
7716 case 4: /* D4 vblank/vline */
7717 switch (src_data) {
7718 case 0: /* D4 vblank */
7719 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT))
7720 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7721
7722 if (rdev->irq.crtc_vblank_int[3]) {
7723 drm_handle_vblank(rdev->ddev, 3);
7724 rdev->pm.vblank_sync = true;
7725 wake_up(&rdev->irq.vblank_queue);
7726 }
7727 if (atomic_read(&rdev->irq.pflip[3]))
7728 radeon_crtc_handle_vblank(rdev, 3);
7729 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7730 DRM_DEBUG("IH: D4 vblank\n");
7731
7732 break;
7733 case 1: /* D4 vline */
7734 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT))
7735 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7736
7737 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7738 DRM_DEBUG("IH: D4 vline\n");
7739
7740 break;
7741 default:
7742 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7743 break;
7744 }
7745 break;
7746 case 5: /* D5 vblank/vline */
7747 switch (src_data) {
7748 case 0: /* D5 vblank */
7749 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT))
7750 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7751
7752 if (rdev->irq.crtc_vblank_int[4]) {
7753 drm_handle_vblank(rdev->ddev, 4);
7754 rdev->pm.vblank_sync = true;
7755 wake_up(&rdev->irq.vblank_queue);
7756 }
7757 if (atomic_read(&rdev->irq.pflip[4]))
7758 radeon_crtc_handle_vblank(rdev, 4);
7759 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7760 DRM_DEBUG("IH: D5 vblank\n");
7761
7762 break;
7763 case 1: /* D5 vline */
7764 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT))
7765 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7766
7767 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7768 DRM_DEBUG("IH: D5 vline\n");
7769
7770 break;
7771 default:
7772 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7773 break;
7774 }
7775 break;
7776 case 6: /* D6 vblank/vline */
7777 switch (src_data) {
7778 case 0: /* D6 vblank */
7779 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT))
7780 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7781
7782 if (rdev->irq.crtc_vblank_int[5]) {
7783 drm_handle_vblank(rdev->ddev, 5);
7784 rdev->pm.vblank_sync = true;
7785 wake_up(&rdev->irq.vblank_queue);
7786 }
7787 if (atomic_read(&rdev->irq.pflip[5]))
7788 radeon_crtc_handle_vblank(rdev, 5);
7789 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7790 DRM_DEBUG("IH: D6 vblank\n");
7791
7792 break;
7793 case 1: /* D6 vline */
7794 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT))
7795 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7796
7797 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7798 DRM_DEBUG("IH: D6 vline\n");
7799
7800 break;
7801 default:
7802 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7803 break;
7804 }
7805 break;
7806 case 8: /* D1 page flip */
7807 case 10: /* D2 page flip */
7808 case 12: /* D3 page flip */
7809 case 14: /* D4 page flip */
7810 case 16: /* D5 page flip */
7811 case 18: /* D6 page flip */
7812 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
7813 if (radeon_use_pflipirq > 0)
7814 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
7815 break;
7816 case 42: /* HPD hotplug */
7817 switch (src_data) {
7818 case 0:
7819 if (!(rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT))
7820 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7821
7822 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
7823 queue_hotplug = true;
7824 DRM_DEBUG("IH: HPD1\n");
7825
7826 break;
7827 case 1:
7828 if (!(rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT))
7829 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7830
7831 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
7832 queue_hotplug = true;
7833 DRM_DEBUG("IH: HPD2\n");
7834
7835 break;
7836 case 2:
7837 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT))
7838 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7839
7840 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
7841 queue_hotplug = true;
7842 DRM_DEBUG("IH: HPD3\n");
7843
7844 break;
7845 case 3:
7846 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT))
7847 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7848
7849 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
7850 queue_hotplug = true;
7851 DRM_DEBUG("IH: HPD4\n");
7852
7853 break;
7854 case 4:
7855 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT))
7856 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7857
7858 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
7859 queue_hotplug = true;
7860 DRM_DEBUG("IH: HPD5\n");
7861
7862 break;
7863 case 5:
7864 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT))
7865 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7866
7867 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
7868 queue_hotplug = true;
7869 DRM_DEBUG("IH: HPD6\n");
7870
7871 break;
7872 case 6:
7873 if (!(rdev->irq.stat_regs.cik.disp_int & DC_HPD1_RX_INTERRUPT))
7874 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7875
7876 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_RX_INTERRUPT;
7877 queue_dp = true;
7878 DRM_DEBUG("IH: HPD_RX 1\n");
7879
7880 break;
7881 case 7:
7882 if (!(rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_RX_INTERRUPT))
7883 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7884
7885 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT;
7886 queue_dp = true;
7887 DRM_DEBUG("IH: HPD_RX 2\n");
7888
7889 break;
7890 case 8:
7891 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_RX_INTERRUPT))
7892 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7893
7894 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT;
7895 queue_dp = true;
7896 DRM_DEBUG("IH: HPD_RX 3\n");
7897
7898 break;
7899 case 9:
7900 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_RX_INTERRUPT))
7901 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7902
7903 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT;
7904 queue_dp = true;
7905 DRM_DEBUG("IH: HPD_RX 4\n");
7906
7907 break;
7908 case 10:
7909 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_RX_INTERRUPT))
7910 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7911
7912 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT;
7913 queue_dp = true;
7914 DRM_DEBUG("IH: HPD_RX 5\n");
7915
7916 break;
7917 case 11:
7918 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_RX_INTERRUPT))
7919 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7920
7921 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT;
7922 queue_dp = true;
7923 DRM_DEBUG("IH: HPD_RX 6\n");
7924
7925 break;
7926 default:
7927 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7928 break;
7929 }
7930 break;
7931 case 96:
7932 DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
7933 WREG32(SRBM_INT_ACK, 0x1);
7934 break;
7935 case 124: /* UVD */
7936 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
7937 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
7938 break;
7939 case 146:
7940 case 147:
7941 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
7942 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
7943 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
7944 /* reset addr and status */
7945 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
7946 if (addr == 0x0 && status == 0x0)
7947 break;
7948 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
7949 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
7950 addr);
7951 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
7952 status);
7953 cik_vm_decode_fault(rdev, status, addr, mc_client);
7954 break;
7955 case 167: /* VCE */
7956 DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data);
7957 switch (src_data) {
7958 case 0:
7959 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX);
7960 break;
7961 case 1:
7962 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX);
7963 break;
7964 default:
7965 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
7966 break;
7967 }
7968 break;
7969 case 176: /* GFX RB CP_INT */
7970 case 177: /* GFX IB CP_INT */
7971 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7972 break;
7973 case 181: /* CP EOP event */
7974 DRM_DEBUG("IH: CP EOP\n");
7975 /* XXX check the bitfield order! */
7976 me_id = (ring_id & 0x60) >> 5;
7977 pipe_id = (ring_id & 0x18) >> 3;
7978 queue_id = (ring_id & 0x7) >> 0;
7979 switch (me_id) {
7980 case 0:
7981 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7982 break;
7983 case 1:
7984 case 2:
7985 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
7986 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7987 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
7988 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7989 break;
7990 }
7991 break;
7992 case 184: /* CP Privileged reg access */
7993 DRM_ERROR("Illegal register access in command stream\n");
7994 /* XXX check the bitfield order! */
7995 me_id = (ring_id & 0x60) >> 5;
7996 pipe_id = (ring_id & 0x18) >> 3;
7997 queue_id = (ring_id & 0x7) >> 0;
7998 switch (me_id) {
7999 case 0:
8000 /* This results in a full GPU reset, but all we need to do is soft
8001 * reset the CP for gfx
8002 */
8003 queue_reset = true;
8004 break;
8005 case 1:
8006 /* XXX compute */
8007 queue_reset = true;
8008 break;
8009 case 2:
8010 /* XXX compute */
8011 queue_reset = true;
8012 break;
8013 }
8014 break;
8015 case 185: /* CP Privileged inst */
8016 DRM_ERROR("Illegal instruction in command stream\n");
8017 /* XXX check the bitfield order! */
8018 me_id = (ring_id & 0x60) >> 5;
8019 pipe_id = (ring_id & 0x18) >> 3;
8020 queue_id = (ring_id & 0x7) >> 0;
8021 switch (me_id) {
8022 case 0:
8023 /* This results in a full GPU reset, but all we need to do is soft
8024 * reset the CP for gfx
8025 */
8026 queue_reset = true;
8027 break;
8028 case 1:
8029 /* XXX compute */
8030 queue_reset = true;
8031 break;
8032 case 2:
8033 /* XXX compute */
8034 queue_reset = true;
8035 break;
8036 }
8037 break;
8038 case 224: /* SDMA trap event */
8039 /* XXX check the bitfield order! */
8040 me_id = (ring_id & 0x3) >> 0;
8041 queue_id = (ring_id & 0xc) >> 2;
8042 DRM_DEBUG("IH: SDMA trap\n");
8043 switch (me_id) {
8044 case 0:
8045 switch (queue_id) {
8046 case 0:
8047 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
8048 break;
8049 case 1:
8050 /* XXX compute */
8051 break;
8052 case 2:
8053 /* XXX compute */
8054 break;
8055 }
8056 break;
8057 case 1:
8058 switch (queue_id) {
8059 case 0:
8060 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8061 break;
8062 case 1:
8063 /* XXX compute */
8064 break;
8065 case 2:
8066 /* XXX compute */
8067 break;
8068 }
8069 break;
8070 }
8071 break;
8072 case 230: /* thermal low to high */
8073 DRM_DEBUG("IH: thermal low to high\n");
8074 rdev->pm.dpm.thermal.high_to_low = false;
8075 queue_thermal = true;
8076 break;
8077 case 231: /* thermal high to low */
8078 DRM_DEBUG("IH: thermal high to low\n");
8079 rdev->pm.dpm.thermal.high_to_low = true;
8080 queue_thermal = true;
8081 break;
8082 case 233: /* GUI IDLE */
8083 DRM_DEBUG("IH: GUI idle\n");
8084 break;
8085 case 241: /* SDMA Privileged inst */
8086 case 247: /* SDMA Privileged inst */
8087 DRM_ERROR("Illegal instruction in SDMA command stream\n");
8088 /* XXX check the bitfield order! */
8089 me_id = (ring_id & 0x3) >> 0;
8090 queue_id = (ring_id & 0xc) >> 2;
8091 switch (me_id) {
8092 case 0:
8093 switch (queue_id) {
8094 case 0:
8095 queue_reset = true;
8096 break;
8097 case 1:
8098 /* XXX compute */
8099 queue_reset = true;
8100 break;
8101 case 2:
8102 /* XXX compute */
8103 queue_reset = true;
8104 break;
8105 }
8106 break;
8107 case 1:
8108 switch (queue_id) {
8109 case 0:
8110 queue_reset = true;
8111 break;
8112 case 1:
8113 /* XXX compute */
8114 queue_reset = true;
8115 break;
8116 case 2:
8117 /* XXX compute */
8118 queue_reset = true;
8119 break;
8120 }
8121 break;
8122 }
8123 break;
8124 default:
8125 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8126 break;
8127 }
8128
8129 /* wptr/rptr are in bytes! */
8130 rptr += 16;
8131 rptr &= rdev->ih.ptr_mask;
8132 WREG32(IH_RB_RPTR, rptr);
8133 }
8134 if (queue_dp)
8135 schedule_work(&rdev->dp_work);
8136 if (queue_hotplug)
8137 schedule_delayed_work(&rdev->hotplug_work, 0);
8138 if (queue_reset) {
8139 rdev->needs_reset = true;
8140 wake_up_all(&rdev->fence_queue);
8141 }
8142 if (queue_thermal)
8143 schedule_work(&rdev->pm.dpm.thermal.work);
8144 rdev->ih.rptr = rptr;
8145 atomic_set(&rdev->ih.lock, 0);
8146
8147 /* make sure wptr hasn't changed while processing */
8148 wptr = cik_get_ih_wptr(rdev);
8149 if (wptr != rptr)
8150 goto restart_ih;
8151
8152 return IRQ_HANDLED;
8153 }
8154
8155 /*
8156 * startup/shutdown callbacks
8157 */
8158 static void cik_uvd_init(struct radeon_device *rdev)
8159 {
8160 int r;
8161
8162 if (!rdev->has_uvd)
8163 return;
8164
8165 r = radeon_uvd_init(rdev);
8166 if (r) {
8167 dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
8168 /*
8169 * At this point rdev->uvd.vcpu_bo is NULL which trickles down
8170 * to early fails cik_uvd_start() and thus nothing happens
8171 * there. So it is pointless to try to go through that code
8172 * hence why we disable uvd here.
8173 */
8174 rdev->has_uvd = 0;
8175 return;
8176 }
8177 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
8178 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
8179 }
8180
8181 static void cik_uvd_start(struct radeon_device *rdev)
8182 {
8183 int r;
8184
8185 if (!rdev->has_uvd)
8186 return;
8187
8188 r = radeon_uvd_resume(rdev);
8189 if (r) {
8190 dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
8191 goto error;
8192 }
8193 r = uvd_v4_2_resume(rdev);
8194 if (r) {
8195 dev_err(rdev->dev, "failed UVD 4.2 resume (%d).\n", r);
8196 goto error;
8197 }
8198 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
8199 if (r) {
8200 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
8201 goto error;
8202 }
8203 return;
8204
8205 error:
8206 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
8207 }
8208
8209 static void cik_uvd_resume(struct radeon_device *rdev)
8210 {
8211 struct radeon_ring *ring;
8212 int r;
8213
8214 if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
8215 return;
8216
8217 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8218 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, RADEON_CP_PACKET2);
8219 if (r) {
8220 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
8221 return;
8222 }
8223 r = uvd_v1_0_init(rdev);
8224 if (r) {
8225 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
8226 return;
8227 }
8228 }
8229
8230 static void cik_vce_init(struct radeon_device *rdev)
8231 {
8232 int r;
8233
8234 if (!rdev->has_vce)
8235 return;
8236
8237 r = radeon_vce_init(rdev);
8238 if (r) {
8239 dev_err(rdev->dev, "failed VCE (%d) init.\n", r);
8240 /*
8241 * At this point rdev->vce.vcpu_bo is NULL which trickles down
8242 * to early fails cik_vce_start() and thus nothing happens
8243 * there. So it is pointless to try to go through that code
8244 * hence why we disable vce here.
8245 */
8246 rdev->has_vce = 0;
8247 return;
8248 }
8249 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_obj = NULL;
8250 r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE1_INDEX], 4096);
8251 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_obj = NULL;
8252 r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE2_INDEX], 4096);
8253 }
8254
8255 static void cik_vce_start(struct radeon_device *rdev)
8256 {
8257 int r;
8258
8259 if (!rdev->has_vce)
8260 return;
8261
8262 r = radeon_vce_resume(rdev);
8263 if (r) {
8264 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
8265 goto error;
8266 }
8267 r = vce_v2_0_resume(rdev);
8268 if (r) {
8269 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
8270 goto error;
8271 }
8272 r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE1_INDEX);
8273 if (r) {
8274 dev_err(rdev->dev, "failed initializing VCE1 fences (%d).\n", r);
8275 goto error;
8276 }
8277 r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE2_INDEX);
8278 if (r) {
8279 dev_err(rdev->dev, "failed initializing VCE2 fences (%d).\n", r);
8280 goto error;
8281 }
8282 return;
8283
8284 error:
8285 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
8286 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
8287 }
8288
8289 static void cik_vce_resume(struct radeon_device *rdev)
8290 {
8291 struct radeon_ring *ring;
8292 int r;
8293
8294 if (!rdev->has_vce || !rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size)
8295 return;
8296
8297 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8298 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP);
8299 if (r) {
8300 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
8301 return;
8302 }
8303 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8304 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP);
8305 if (r) {
8306 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
8307 return;
8308 }
8309 r = vce_v1_0_init(rdev);
8310 if (r) {
8311 dev_err(rdev->dev, "failed initializing VCE (%d).\n", r);
8312 return;
8313 }
8314 }
8315
8316 /**
8317 * cik_startup - program the asic to a functional state
8318 *
8319 * @rdev: radeon_device pointer
8320 *
8321 * Programs the asic to a functional state (CIK).
8322 * Called by cik_init() and cik_resume().
8323 * Returns 0 for success, error for failure.
8324 */
8325 static int cik_startup(struct radeon_device *rdev)
8326 {
8327 struct radeon_ring *ring;
8328 u32 nop;
8329 int r;
8330
8331 /* enable pcie gen2/3 link */
8332 cik_pcie_gen3_enable(rdev);
8333 /* enable aspm */
8334 cik_program_aspm(rdev);
8335
8336 /* scratch needs to be initialized before MC */
8337 r = r600_vram_scratch_init(rdev);
8338 if (r)
8339 return r;
8340
8341 cik_mc_program(rdev);
8342
8343 if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
8344 r = ci_mc_load_microcode(rdev);
8345 if (r) {
8346 DRM_ERROR("Failed to load MC firmware!\n");
8347 return r;
8348 }
8349 }
8350
8351 r = cik_pcie_gart_enable(rdev);
8352 if (r)
8353 return r;
8354 cik_gpu_init(rdev);
8355
8356 /* allocate rlc buffers */
8357 if (rdev->flags & RADEON_IS_IGP) {
8358 if (rdev->family == CHIP_KAVERI) {
8359 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
8360 rdev->rlc.reg_list_size =
8361 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
8362 } else {
8363 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
8364 rdev->rlc.reg_list_size =
8365 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
8366 }
8367 }
8368 rdev->rlc.cs_data = ci_cs_data;
8369 rdev->rlc.cp_table_size = ALIGN(CP_ME_TABLE_SIZE * 5 * 4, 2048); /* CP JT */
8370 rdev->rlc.cp_table_size += 64 * 1024; /* GDS */
8371 r = sumo_rlc_init(rdev);
8372 if (r) {
8373 DRM_ERROR("Failed to init rlc BOs!\n");
8374 return r;
8375 }
8376
8377 /* allocate wb buffer */
8378 r = radeon_wb_init(rdev);
8379 if (r)
8380 return r;
8381
8382 /* allocate mec buffers */
8383 r = cik_mec_init(rdev);
8384 if (r) {
8385 DRM_ERROR("Failed to init MEC BOs!\n");
8386 return r;
8387 }
8388
8389 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
8390 if (r) {
8391 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8392 return r;
8393 }
8394
8395 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
8396 if (r) {
8397 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8398 return r;
8399 }
8400
8401 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
8402 if (r) {
8403 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8404 return r;
8405 }
8406
8407 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
8408 if (r) {
8409 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8410 return r;
8411 }
8412
8413 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8414 if (r) {
8415 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8416 return r;
8417 }
8418
8419 cik_uvd_start(rdev);
8420 cik_vce_start(rdev);
8421
8422 /* Enable IRQ */
8423 if (!rdev->irq.installed) {
8424 r = radeon_irq_kms_init(rdev);
8425 if (r)
8426 return r;
8427 }
8428
8429 r = cik_irq_init(rdev);
8430 if (r) {
8431 DRM_ERROR("radeon: IH init failed (%d).\n", r);
8432 radeon_irq_kms_fini(rdev);
8433 return r;
8434 }
8435 cik_irq_set(rdev);
8436
8437 if (rdev->family == CHIP_HAWAII) {
8438 if (rdev->new_fw)
8439 nop = PACKET3(PACKET3_NOP, 0x3FFF);
8440 else
8441 nop = RADEON_CP_PACKET2;
8442 } else {
8443 nop = PACKET3(PACKET3_NOP, 0x3FFF);
8444 }
8445
8446 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8447 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
8448 nop);
8449 if (r)
8450 return r;
8451
8452 /* set up the compute queues */
8453 /* type-2 packets are deprecated on MEC, use type-3 instead */
8454 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8455 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
8456 nop);
8457 if (r)
8458 return r;
8459 ring->me = 1; /* first MEC */
8460 ring->pipe = 0; /* first pipe */
8461 ring->queue = 0; /* first queue */
8462 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
8463
8464 /* type-2 packets are deprecated on MEC, use type-3 instead */
8465 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8466 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
8467 nop);
8468 if (r)
8469 return r;
8470 /* dGPU only have 1 MEC */
8471 ring->me = 1; /* first MEC */
8472 ring->pipe = 0; /* first pipe */
8473 ring->queue = 1; /* second queue */
8474 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
8475
8476 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8477 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
8478 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
8479 if (r)
8480 return r;
8481
8482 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8483 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
8484 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
8485 if (r)
8486 return r;
8487
8488 r = cik_cp_resume(rdev);
8489 if (r)
8490 return r;
8491
8492 r = cik_sdma_resume(rdev);
8493 if (r)
8494 return r;
8495
8496 cik_uvd_resume(rdev);
8497 cik_vce_resume(rdev);
8498
8499 r = radeon_ib_pool_init(rdev);
8500 if (r) {
8501 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
8502 return r;
8503 }
8504
8505 r = radeon_vm_manager_init(rdev);
8506 if (r) {
8507 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
8508 return r;
8509 }
8510
8511 r = radeon_audio_init(rdev);
8512 if (r)
8513 return r;
8514
8515 r = radeon_kfd_resume(rdev);
8516 if (r)
8517 return r;
8518
8519 return 0;
8520 }
8521
8522 /**
8523 * cik_resume - resume the asic to a functional state
8524 *
8525 * @rdev: radeon_device pointer
8526 *
8527 * Programs the asic to a functional state (CIK).
8528 * Called at resume.
8529 * Returns 0 for success, error for failure.
8530 */
8531 int cik_resume(struct radeon_device *rdev)
8532 {
8533 int r;
8534
8535 /* post card */
8536 atom_asic_init(rdev->mode_info.atom_context);
8537
8538 /* init golden registers */
8539 cik_init_golden_registers(rdev);
8540
8541 if (rdev->pm.pm_method == PM_METHOD_DPM)
8542 radeon_pm_resume(rdev);
8543
8544 rdev->accel_working = true;
8545 r = cik_startup(rdev);
8546 if (r) {
8547 DRM_ERROR("cik startup failed on resume\n");
8548 rdev->accel_working = false;
8549 return r;
8550 }
8551
8552 return r;
8553
8554 }
8555
8556 /**
8557 * cik_suspend - suspend the asic
8558 *
8559 * @rdev: radeon_device pointer
8560 *
8561 * Bring the chip into a state suitable for suspend (CIK).
8562 * Called at suspend.
8563 * Returns 0 for success.
8564 */
8565 int cik_suspend(struct radeon_device *rdev)
8566 {
8567 radeon_kfd_suspend(rdev);
8568 radeon_pm_suspend(rdev);
8569 radeon_audio_fini(rdev);
8570 radeon_vm_manager_fini(rdev);
8571 cik_cp_enable(rdev, false);
8572 cik_sdma_enable(rdev, false);
8573 if (rdev->has_uvd) {
8574 uvd_v1_0_fini(rdev);
8575 radeon_uvd_suspend(rdev);
8576 }
8577 if (rdev->has_vce)
8578 radeon_vce_suspend(rdev);
8579 cik_fini_pg(rdev);
8580 cik_fini_cg(rdev);
8581 cik_irq_suspend(rdev);
8582 radeon_wb_disable(rdev);
8583 cik_pcie_gart_disable(rdev);
8584 return 0;
8585 }
8586
8587 /* Plan is to move initialization in that function and use
8588 * helper function so that radeon_device_init pretty much
8589 * do nothing more than calling asic specific function. This
8590 * should also allow to remove a bunch of callback function
8591 * like vram_info.
8592 */
8593 /**
8594 * cik_init - asic specific driver and hw init
8595 *
8596 * @rdev: radeon_device pointer
8597 *
8598 * Setup asic specific driver variables and program the hw
8599 * to a functional state (CIK).
8600 * Called at driver startup.
8601 * Returns 0 for success, errors for failure.
8602 */
8603 int cik_init(struct radeon_device *rdev)
8604 {
8605 struct radeon_ring *ring;
8606 int r;
8607
8608 /* Read BIOS */
8609 if (!radeon_get_bios(rdev)) {
8610 if (ASIC_IS_AVIVO(rdev))
8611 return -EINVAL;
8612 }
8613 /* Must be an ATOMBIOS */
8614 if (!rdev->is_atom_bios) {
8615 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
8616 return -EINVAL;
8617 }
8618 r = radeon_atombios_init(rdev);
8619 if (r)
8620 return r;
8621
8622 /* Post card if necessary */
8623 if (!radeon_card_posted(rdev)) {
8624 if (!rdev->bios) {
8625 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
8626 return -EINVAL;
8627 }
8628 DRM_INFO("GPU not posted. posting now...\n");
8629 atom_asic_init(rdev->mode_info.atom_context);
8630 }
8631 /* init golden registers */
8632 cik_init_golden_registers(rdev);
8633 /* Initialize scratch registers */
8634 cik_scratch_init(rdev);
8635 /* Initialize surface registers */
8636 radeon_surface_init(rdev);
8637 /* Initialize clocks */
8638 radeon_get_clock_info(rdev->ddev);
8639
8640 /* Fence driver */
8641 r = radeon_fence_driver_init(rdev);
8642 if (r)
8643 return r;
8644
8645 /* initialize memory controller */
8646 r = cik_mc_init(rdev);
8647 if (r)
8648 return r;
8649 /* Memory manager */
8650 r = radeon_bo_init(rdev);
8651 if (r)
8652 return r;
8653
8654 if (rdev->flags & RADEON_IS_IGP) {
8655 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8656 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
8657 r = cik_init_microcode(rdev);
8658 if (r) {
8659 DRM_ERROR("Failed to load firmware!\n");
8660 return r;
8661 }
8662 }
8663 } else {
8664 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8665 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
8666 !rdev->mc_fw) {
8667 r = cik_init_microcode(rdev);
8668 if (r) {
8669 DRM_ERROR("Failed to load firmware!\n");
8670 return r;
8671 }
8672 }
8673 }
8674
8675 /* Initialize power management */
8676 radeon_pm_init(rdev);
8677
8678 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8679 ring->ring_obj = NULL;
8680 r600_ring_init(rdev, ring, 1024 * 1024);
8681
8682 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8683 ring->ring_obj = NULL;
8684 r600_ring_init(rdev, ring, 1024 * 1024);
8685 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8686 if (r)
8687 return r;
8688
8689 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8690 ring->ring_obj = NULL;
8691 r600_ring_init(rdev, ring, 1024 * 1024);
8692 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8693 if (r)
8694 return r;
8695
8696 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8697 ring->ring_obj = NULL;
8698 r600_ring_init(rdev, ring, 256 * 1024);
8699
8700 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8701 ring->ring_obj = NULL;
8702 r600_ring_init(rdev, ring, 256 * 1024);
8703
8704 cik_uvd_init(rdev);
8705 cik_vce_init(rdev);
8706
8707 rdev->ih.ring_obj = NULL;
8708 r600_ih_ring_init(rdev, 64 * 1024);
8709
8710 r = r600_pcie_gart_init(rdev);
8711 if (r)
8712 return r;
8713
8714 rdev->accel_working = true;
8715 r = cik_startup(rdev);
8716 if (r) {
8717 dev_err(rdev->dev, "disabling GPU acceleration\n");
8718 cik_cp_fini(rdev);
8719 cik_sdma_fini(rdev);
8720 cik_irq_fini(rdev);
8721 sumo_rlc_fini(rdev);
8722 cik_mec_fini(rdev);
8723 radeon_wb_fini(rdev);
8724 radeon_ib_pool_fini(rdev);
8725 radeon_vm_manager_fini(rdev);
8726 radeon_irq_kms_fini(rdev);
8727 cik_pcie_gart_fini(rdev);
8728 rdev->accel_working = false;
8729 }
8730
8731 /* Don't start up if the MC ucode is missing.
8732 * The default clocks and voltages before the MC ucode
8733 * is loaded are not suffient for advanced operations.
8734 */
8735 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
8736 DRM_ERROR("radeon: MC ucode required for NI+.\n");
8737 return -EINVAL;
8738 }
8739
8740 return 0;
8741 }
8742
8743 /**
8744 * cik_fini - asic specific driver and hw fini
8745 *
8746 * @rdev: radeon_device pointer
8747 *
8748 * Tear down the asic specific driver variables and program the hw
8749 * to an idle state (CIK).
8750 * Called at driver unload.
8751 */
8752 void cik_fini(struct radeon_device *rdev)
8753 {
8754 radeon_pm_fini(rdev);
8755 cik_cp_fini(rdev);
8756 cik_sdma_fini(rdev);
8757 cik_fini_pg(rdev);
8758 cik_fini_cg(rdev);
8759 cik_irq_fini(rdev);
8760 sumo_rlc_fini(rdev);
8761 cik_mec_fini(rdev);
8762 radeon_wb_fini(rdev);
8763 radeon_vm_manager_fini(rdev);
8764 radeon_ib_pool_fini(rdev);
8765 radeon_irq_kms_fini(rdev);
8766 uvd_v1_0_fini(rdev);
8767 radeon_uvd_fini(rdev);
8768 radeon_vce_fini(rdev);
8769 cik_pcie_gart_fini(rdev);
8770 r600_vram_scratch_fini(rdev);
8771 radeon_gem_fini(rdev);
8772 radeon_fence_driver_fini(rdev);
8773 radeon_bo_fini(rdev);
8774 radeon_atombios_fini(rdev);
8775 kfree(rdev->bios);
8776 rdev->bios = NULL;
8777 }
8778
8779 void dce8_program_fmt(struct drm_encoder *encoder)
8780 {
8781 struct drm_device *dev = encoder->dev;
8782 struct radeon_device *rdev = dev->dev_private;
8783 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
8784 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
8785 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
8786 int bpc = 0;
8787 u32 tmp = 0;
8788 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
8789
8790 if (connector) {
8791 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
8792 bpc = radeon_get_monitor_bpc(connector);
8793 dither = radeon_connector->dither;
8794 }
8795
8796 /* LVDS/eDP FMT is set up by atom */
8797 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
8798 return;
8799
8800 /* not needed for analog */
8801 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
8802 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
8803 return;
8804
8805 if (bpc == 0)
8806 return;
8807
8808 switch (bpc) {
8809 case 6:
8810 if (dither == RADEON_FMT_DITHER_ENABLE)
8811 /* XXX sort out optimal dither settings */
8812 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8813 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
8814 else
8815 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
8816 break;
8817 case 8:
8818 if (dither == RADEON_FMT_DITHER_ENABLE)
8819 /* XXX sort out optimal dither settings */
8820 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8821 FMT_RGB_RANDOM_ENABLE |
8822 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
8823 else
8824 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
8825 break;
8826 case 10:
8827 if (dither == RADEON_FMT_DITHER_ENABLE)
8828 /* XXX sort out optimal dither settings */
8829 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8830 FMT_RGB_RANDOM_ENABLE |
8831 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
8832 else
8833 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
8834 break;
8835 default:
8836 /* not needed */
8837 break;
8838 }
8839
8840 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8841 }
8842
8843 /* display watermark setup */
8844 /**
8845 * dce8_line_buffer_adjust - Set up the line buffer
8846 *
8847 * @rdev: radeon_device pointer
8848 * @radeon_crtc: the selected display controller
8849 * @mode: the current display mode on the selected display
8850 * controller
8851 *
8852 * Setup up the line buffer allocation for
8853 * the selected display controller (CIK).
8854 * Returns the line buffer size in pixels.
8855 */
8856 static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8857 struct radeon_crtc *radeon_crtc,
8858 struct drm_display_mode *mode)
8859 {
8860 u32 tmp, buffer_alloc, i;
8861 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
8862 /*
8863 * Line Buffer Setup
8864 * There are 6 line buffers, one for each display controllers.
8865 * There are 3 partitions per LB. Select the number of partitions
8866 * to enable based on the display width. For display widths larger
8867 * than 4096, you need use to use 2 display controllers and combine
8868 * them using the stereo blender.
8869 */
8870 if (radeon_crtc->base.enabled && mode) {
8871 if (mode->crtc_hdisplay < 1920) {
8872 tmp = 1;
8873 buffer_alloc = 2;
8874 } else if (mode->crtc_hdisplay < 2560) {
8875 tmp = 2;
8876 buffer_alloc = 2;
8877 } else if (mode->crtc_hdisplay < 4096) {
8878 tmp = 0;
8879 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8880 } else {
8881 DRM_DEBUG_KMS("Mode too big for LB!\n");
8882 tmp = 0;
8883 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8884 }
8885 } else {
8886 tmp = 1;
8887 buffer_alloc = 0;
8888 }
8889
8890 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8891 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8892
8893 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8894 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8895 for (i = 0; i < rdev->usec_timeout; i++) {
8896 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8897 DMIF_BUFFERS_ALLOCATED_COMPLETED)
8898 break;
8899 udelay(1);
8900 }
8901
8902 if (radeon_crtc->base.enabled && mode) {
8903 switch (tmp) {
8904 case 0:
8905 default:
8906 return 4096 * 2;
8907 case 1:
8908 return 1920 * 2;
8909 case 2:
8910 return 2560 * 2;
8911 }
8912 }
8913
8914 /* controller not enabled, so no lb used */
8915 return 0;
8916 }
8917
8918 /**
8919 * cik_get_number_of_dram_channels - get the number of dram channels
8920 *
8921 * @rdev: radeon_device pointer
8922 *
8923 * Look up the number of video ram channels (CIK).
8924 * Used for display watermark bandwidth calculations
8925 * Returns the number of dram channels
8926 */
8927 static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8928 {
8929 u32 tmp = RREG32(MC_SHARED_CHMAP);
8930
8931 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8932 case 0:
8933 default:
8934 return 1;
8935 case 1:
8936 return 2;
8937 case 2:
8938 return 4;
8939 case 3:
8940 return 8;
8941 case 4:
8942 return 3;
8943 case 5:
8944 return 6;
8945 case 6:
8946 return 10;
8947 case 7:
8948 return 12;
8949 case 8:
8950 return 16;
8951 }
8952 }
8953
8954 struct dce8_wm_params {
8955 u32 dram_channels; /* number of dram channels */
8956 u32 yclk; /* bandwidth per dram data pin in kHz */
8957 u32 sclk; /* engine clock in kHz */
8958 u32 disp_clk; /* display clock in kHz */
8959 u32 src_width; /* viewport width */
8960 u32 active_time; /* active display time in ns */
8961 u32 blank_time; /* blank time in ns */
8962 bool interlaced; /* mode is interlaced */
8963 fixed20_12 vsc; /* vertical scale ratio */
8964 u32 num_heads; /* number of active crtcs */
8965 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8966 u32 lb_size; /* line buffer allocated to pipe */
8967 u32 vtaps; /* vertical scaler taps */
8968 };
8969
8970 /**
8971 * dce8_dram_bandwidth - get the dram bandwidth
8972 *
8973 * @wm: watermark calculation data
8974 *
8975 * Calculate the raw dram bandwidth (CIK).
8976 * Used for display watermark bandwidth calculations
8977 * Returns the dram bandwidth in MBytes/s
8978 */
8979 static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
8980 {
8981 /* Calculate raw DRAM Bandwidth */
8982 fixed20_12 dram_efficiency; /* 0.7 */
8983 fixed20_12 yclk, dram_channels, bandwidth;
8984 fixed20_12 a;
8985
8986 a.full = dfixed_const(1000);
8987 yclk.full = dfixed_const(wm->yclk);
8988 yclk.full = dfixed_div(yclk, a);
8989 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8990 a.full = dfixed_const(10);
8991 dram_efficiency.full = dfixed_const(7);
8992 dram_efficiency.full = dfixed_div(dram_efficiency, a);
8993 bandwidth.full = dfixed_mul(dram_channels, yclk);
8994 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
8995
8996 return dfixed_trunc(bandwidth);
8997 }
8998
8999 /**
9000 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
9001 *
9002 * @wm: watermark calculation data
9003 *
9004 * Calculate the dram bandwidth used for display (CIK).
9005 * Used for display watermark bandwidth calculations
9006 * Returns the dram bandwidth for display in MBytes/s
9007 */
9008 static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
9009 {
9010 /* Calculate DRAM Bandwidth and the part allocated to display. */
9011 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
9012 fixed20_12 yclk, dram_channels, bandwidth;
9013 fixed20_12 a;
9014
9015 a.full = dfixed_const(1000);
9016 yclk.full = dfixed_const(wm->yclk);
9017 yclk.full = dfixed_div(yclk, a);
9018 dram_channels.full = dfixed_const(wm->dram_channels * 4);
9019 a.full = dfixed_const(10);
9020 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
9021 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
9022 bandwidth.full = dfixed_mul(dram_channels, yclk);
9023 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
9024
9025 return dfixed_trunc(bandwidth);
9026 }
9027
9028 /**
9029 * dce8_data_return_bandwidth - get the data return bandwidth
9030 *
9031 * @wm: watermark calculation data
9032 *
9033 * Calculate the data return bandwidth used for display (CIK).
9034 * Used for display watermark bandwidth calculations
9035 * Returns the data return bandwidth in MBytes/s
9036 */
9037 static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
9038 {
9039 /* Calculate the display Data return Bandwidth */
9040 fixed20_12 return_efficiency; /* 0.8 */
9041 fixed20_12 sclk, bandwidth;
9042 fixed20_12 a;
9043
9044 a.full = dfixed_const(1000);
9045 sclk.full = dfixed_const(wm->sclk);
9046 sclk.full = dfixed_div(sclk, a);
9047 a.full = dfixed_const(10);
9048 return_efficiency.full = dfixed_const(8);
9049 return_efficiency.full = dfixed_div(return_efficiency, a);
9050 a.full = dfixed_const(32);
9051 bandwidth.full = dfixed_mul(a, sclk);
9052 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
9053
9054 return dfixed_trunc(bandwidth);
9055 }
9056
9057 /**
9058 * dce8_dmif_request_bandwidth - get the dmif bandwidth
9059 *
9060 * @wm: watermark calculation data
9061 *
9062 * Calculate the dmif bandwidth used for display (CIK).
9063 * Used for display watermark bandwidth calculations
9064 * Returns the dmif bandwidth in MBytes/s
9065 */
9066 static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
9067 {
9068 /* Calculate the DMIF Request Bandwidth */
9069 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
9070 fixed20_12 disp_clk, bandwidth;
9071 fixed20_12 a, b;
9072
9073 a.full = dfixed_const(1000);
9074 disp_clk.full = dfixed_const(wm->disp_clk);
9075 disp_clk.full = dfixed_div(disp_clk, a);
9076 a.full = dfixed_const(32);
9077 b.full = dfixed_mul(a, disp_clk);
9078
9079 a.full = dfixed_const(10);
9080 disp_clk_request_efficiency.full = dfixed_const(8);
9081 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
9082
9083 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
9084
9085 return dfixed_trunc(bandwidth);
9086 }
9087
9088 /**
9089 * dce8_available_bandwidth - get the min available bandwidth
9090 *
9091 * @wm: watermark calculation data
9092 *
9093 * Calculate the min available bandwidth used for display (CIK).
9094 * Used for display watermark bandwidth calculations
9095 * Returns the min available bandwidth in MBytes/s
9096 */
9097 static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
9098 {
9099 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
9100 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
9101 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
9102 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
9103
9104 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
9105 }
9106
9107 /**
9108 * dce8_average_bandwidth - get the average available bandwidth
9109 *
9110 * @wm: watermark calculation data
9111 *
9112 * Calculate the average available bandwidth used for display (CIK).
9113 * Used for display watermark bandwidth calculations
9114 * Returns the average available bandwidth in MBytes/s
9115 */
9116 static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
9117 {
9118 /* Calculate the display mode Average Bandwidth
9119 * DisplayMode should contain the source and destination dimensions,
9120 * timing, etc.
9121 */
9122 fixed20_12 bpp;
9123 fixed20_12 line_time;
9124 fixed20_12 src_width;
9125 fixed20_12 bandwidth;
9126 fixed20_12 a;
9127
9128 a.full = dfixed_const(1000);
9129 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
9130 line_time.full = dfixed_div(line_time, a);
9131 bpp.full = dfixed_const(wm->bytes_per_pixel);
9132 src_width.full = dfixed_const(wm->src_width);
9133 bandwidth.full = dfixed_mul(src_width, bpp);
9134 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
9135 bandwidth.full = dfixed_div(bandwidth, line_time);
9136
9137 return dfixed_trunc(bandwidth);
9138 }
9139
9140 /**
9141 * dce8_latency_watermark - get the latency watermark
9142 *
9143 * @wm: watermark calculation data
9144 *
9145 * Calculate the latency watermark (CIK).
9146 * Used for display watermark bandwidth calculations
9147 * Returns the latency watermark in ns
9148 */
9149 static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
9150 {
9151 /* First calculate the latency in ns */
9152 u32 mc_latency = 2000; /* 2000 ns. */
9153 u32 available_bandwidth = dce8_available_bandwidth(wm);
9154 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
9155 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
9156 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
9157 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
9158 (wm->num_heads * cursor_line_pair_return_time);
9159 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
9160 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
9161 u32 tmp, dmif_size = 12288;
9162 fixed20_12 a, b, c;
9163
9164 if (wm->num_heads == 0)
9165 return 0;
9166
9167 a.full = dfixed_const(2);
9168 b.full = dfixed_const(1);
9169 if ((wm->vsc.full > a.full) ||
9170 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
9171 (wm->vtaps >= 5) ||
9172 ((wm->vsc.full >= a.full) && wm->interlaced))
9173 max_src_lines_per_dst_line = 4;
9174 else
9175 max_src_lines_per_dst_line = 2;
9176
9177 a.full = dfixed_const(available_bandwidth);
9178 b.full = dfixed_const(wm->num_heads);
9179 a.full = dfixed_div(a, b);
9180
9181 b.full = dfixed_const(mc_latency + 512);
9182 c.full = dfixed_const(wm->disp_clk);
9183 b.full = dfixed_div(b, c);
9184
9185 c.full = dfixed_const(dmif_size);
9186 b.full = dfixed_div(c, b);
9187
9188 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
9189
9190 b.full = dfixed_const(1000);
9191 c.full = dfixed_const(wm->disp_clk);
9192 b.full = dfixed_div(c, b);
9193 c.full = dfixed_const(wm->bytes_per_pixel);
9194 b.full = dfixed_mul(b, c);
9195
9196 lb_fill_bw = min(tmp, dfixed_trunc(b));
9197
9198 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
9199 b.full = dfixed_const(1000);
9200 c.full = dfixed_const(lb_fill_bw);
9201 b.full = dfixed_div(c, b);
9202 a.full = dfixed_div(a, b);
9203 line_fill_time = dfixed_trunc(a);
9204
9205 if (line_fill_time < wm->active_time)
9206 return latency;
9207 else
9208 return latency + (line_fill_time - wm->active_time);
9209
9210 }
9211
9212 /**
9213 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
9214 * average and available dram bandwidth
9215 *
9216 * @wm: watermark calculation data
9217 *
9218 * Check if the display average bandwidth fits in the display
9219 * dram bandwidth (CIK).
9220 * Used for display watermark bandwidth calculations
9221 * Returns true if the display fits, false if not.
9222 */
9223 static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
9224 {
9225 if (dce8_average_bandwidth(wm) <=
9226 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
9227 return true;
9228 else
9229 return false;
9230 }
9231
9232 /**
9233 * dce8_average_bandwidth_vs_available_bandwidth - check
9234 * average and available bandwidth
9235 *
9236 * @wm: watermark calculation data
9237 *
9238 * Check if the display average bandwidth fits in the display
9239 * available bandwidth (CIK).
9240 * Used for display watermark bandwidth calculations
9241 * Returns true if the display fits, false if not.
9242 */
9243 static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
9244 {
9245 if (dce8_average_bandwidth(wm) <=
9246 (dce8_available_bandwidth(wm) / wm->num_heads))
9247 return true;
9248 else
9249 return false;
9250 }
9251
9252 /**
9253 * dce8_check_latency_hiding - check latency hiding
9254 *
9255 * @wm: watermark calculation data
9256 *
9257 * Check latency hiding (CIK).
9258 * Used for display watermark bandwidth calculations
9259 * Returns true if the display fits, false if not.
9260 */
9261 static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
9262 {
9263 u32 lb_partitions = wm->lb_size / wm->src_width;
9264 u32 line_time = wm->active_time + wm->blank_time;
9265 u32 latency_tolerant_lines;
9266 u32 latency_hiding;
9267 fixed20_12 a;
9268
9269 a.full = dfixed_const(1);
9270 if (wm->vsc.full > a.full)
9271 latency_tolerant_lines = 1;
9272 else {
9273 if (lb_partitions <= (wm->vtaps + 1))
9274 latency_tolerant_lines = 1;
9275 else
9276 latency_tolerant_lines = 2;
9277 }
9278
9279 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
9280
9281 if (dce8_latency_watermark(wm) <= latency_hiding)
9282 return true;
9283 else
9284 return false;
9285 }
9286
9287 /**
9288 * dce8_program_watermarks - program display watermarks
9289 *
9290 * @rdev: radeon_device pointer
9291 * @radeon_crtc: the selected display controller
9292 * @lb_size: line buffer size
9293 * @num_heads: number of display controllers in use
9294 *
9295 * Calculate and program the display watermarks for the
9296 * selected display controller (CIK).
9297 */
9298 static void dce8_program_watermarks(struct radeon_device *rdev,
9299 struct radeon_crtc *radeon_crtc,
9300 u32 lb_size, u32 num_heads)
9301 {
9302 struct drm_display_mode *mode = &radeon_crtc->base.mode;
9303 struct dce8_wm_params wm_low, wm_high;
9304 u32 pixel_period;
9305 u32 line_time = 0;
9306 u32 latency_watermark_a = 0, latency_watermark_b = 0;
9307 u32 tmp, wm_mask;
9308
9309 if (radeon_crtc->base.enabled && num_heads && mode) {
9310 pixel_period = 1000000 / (u32)mode->clock;
9311 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
9312
9313 /* watermark for high clocks */
9314 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9315 rdev->pm.dpm_enabled) {
9316 wm_high.yclk =
9317 radeon_dpm_get_mclk(rdev, false) * 10;
9318 wm_high.sclk =
9319 radeon_dpm_get_sclk(rdev, false) * 10;
9320 } else {
9321 wm_high.yclk = rdev->pm.current_mclk * 10;
9322 wm_high.sclk = rdev->pm.current_sclk * 10;
9323 }
9324
9325 wm_high.disp_clk = mode->clock;
9326 wm_high.src_width = mode->crtc_hdisplay;
9327 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
9328 wm_high.blank_time = line_time - wm_high.active_time;
9329 wm_high.interlaced = false;
9330 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
9331 wm_high.interlaced = true;
9332 wm_high.vsc = radeon_crtc->vsc;
9333 wm_high.vtaps = 1;
9334 if (radeon_crtc->rmx_type != RMX_OFF)
9335 wm_high.vtaps = 2;
9336 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
9337 wm_high.lb_size = lb_size;
9338 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
9339 wm_high.num_heads = num_heads;
9340
9341 /* set for high clocks */
9342 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
9343
9344 /* possibly force display priority to high */
9345 /* should really do this at mode validation time... */
9346 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
9347 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
9348 !dce8_check_latency_hiding(&wm_high) ||
9349 (rdev->disp_priority == 2)) {
9350 DRM_DEBUG_KMS("force priority to high\n");
9351 }
9352
9353 /* watermark for low clocks */
9354 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9355 rdev->pm.dpm_enabled) {
9356 wm_low.yclk =
9357 radeon_dpm_get_mclk(rdev, true) * 10;
9358 wm_low.sclk =
9359 radeon_dpm_get_sclk(rdev, true) * 10;
9360 } else {
9361 wm_low.yclk = rdev->pm.current_mclk * 10;
9362 wm_low.sclk = rdev->pm.current_sclk * 10;
9363 }
9364
9365 wm_low.disp_clk = mode->clock;
9366 wm_low.src_width = mode->crtc_hdisplay;
9367 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
9368 wm_low.blank_time = line_time - wm_low.active_time;
9369 wm_low.interlaced = false;
9370 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
9371 wm_low.interlaced = true;
9372 wm_low.vsc = radeon_crtc->vsc;
9373 wm_low.vtaps = 1;
9374 if (radeon_crtc->rmx_type != RMX_OFF)
9375 wm_low.vtaps = 2;
9376 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
9377 wm_low.lb_size = lb_size;
9378 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
9379 wm_low.num_heads = num_heads;
9380
9381 /* set for low clocks */
9382 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
9383
9384 /* possibly force display priority to high */
9385 /* should really do this at mode validation time... */
9386 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
9387 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
9388 !dce8_check_latency_hiding(&wm_low) ||
9389 (rdev->disp_priority == 2)) {
9390 DRM_DEBUG_KMS("force priority to high\n");
9391 }
9392
9393 /* Save number of lines the linebuffer leads before the scanout */
9394 radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
9395 }
9396
9397 /* select wm A */
9398 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9399 tmp = wm_mask;
9400 tmp &= ~LATENCY_WATERMARK_MASK(3);
9401 tmp |= LATENCY_WATERMARK_MASK(1);
9402 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9403 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9404 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
9405 LATENCY_HIGH_WATERMARK(line_time)));
9406 /* select wm B */
9407 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9408 tmp &= ~LATENCY_WATERMARK_MASK(3);
9409 tmp |= LATENCY_WATERMARK_MASK(2);
9410 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9411 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9412 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
9413 LATENCY_HIGH_WATERMARK(line_time)));
9414 /* restore original selection */
9415 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
9416
9417 /* save values for DPM */
9418 radeon_crtc->line_time = line_time;
9419 radeon_crtc->wm_high = latency_watermark_a;
9420 radeon_crtc->wm_low = latency_watermark_b;
9421 }
9422
9423 /**
9424 * dce8_bandwidth_update - program display watermarks
9425 *
9426 * @rdev: radeon_device pointer
9427 *
9428 * Calculate and program the display watermarks and line
9429 * buffer allocation (CIK).
9430 */
9431 void dce8_bandwidth_update(struct radeon_device *rdev)
9432 {
9433 struct drm_display_mode *mode = NULL;
9434 u32 num_heads = 0, lb_size;
9435 int i;
9436
9437 if (!rdev->mode_info.mode_config_initialized)
9438 return;
9439
9440 radeon_update_display_priority(rdev);
9441
9442 for (i = 0; i < rdev->num_crtc; i++) {
9443 if (rdev->mode_info.crtcs[i]->base.enabled)
9444 num_heads++;
9445 }
9446 for (i = 0; i < rdev->num_crtc; i++) {
9447 mode = &rdev->mode_info.crtcs[i]->base.mode;
9448 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
9449 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
9450 }
9451 }
9452
9453 /**
9454 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
9455 *
9456 * @rdev: radeon_device pointer
9457 *
9458 * Fetches a GPU clock counter snapshot (SI).
9459 * Returns the 64 bit clock counter snapshot.
9460 */
9461 uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
9462 {
9463 uint64_t clock;
9464
9465 mutex_lock(&rdev->gpu_clock_mutex);
9466 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
9467 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
9468 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
9469 mutex_unlock(&rdev->gpu_clock_mutex);
9470 return clock;
9471 }
9472
9473 static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
9474 u32 cntl_reg, u32 status_reg)
9475 {
9476 int r, i;
9477 struct atom_clock_dividers dividers;
9478 uint32_t tmp;
9479
9480 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9481 clock, false, &dividers);
9482 if (r)
9483 return r;
9484
9485 tmp = RREG32_SMC(cntl_reg);
9486 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
9487 tmp |= dividers.post_divider;
9488 WREG32_SMC(cntl_reg, tmp);
9489
9490 for (i = 0; i < 100; i++) {
9491 if (RREG32_SMC(status_reg) & DCLK_STATUS)
9492 break;
9493 mdelay(10);
9494 }
9495 if (i == 100)
9496 return -ETIMEDOUT;
9497
9498 return 0;
9499 }
9500
9501 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
9502 {
9503 int r = 0;
9504
9505 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
9506 if (r)
9507 return r;
9508
9509 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
9510 return r;
9511 }
9512
9513 int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
9514 {
9515 int r, i;
9516 struct atom_clock_dividers dividers;
9517 u32 tmp;
9518
9519 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9520 ecclk, false, &dividers);
9521 if (r)
9522 return r;
9523
9524 for (i = 0; i < 100; i++) {
9525 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9526 break;
9527 mdelay(10);
9528 }
9529 if (i == 100)
9530 return -ETIMEDOUT;
9531
9532 tmp = RREG32_SMC(CG_ECLK_CNTL);
9533 tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK);
9534 tmp |= dividers.post_divider;
9535 WREG32_SMC(CG_ECLK_CNTL, tmp);
9536
9537 for (i = 0; i < 100; i++) {
9538 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9539 break;
9540 mdelay(10);
9541 }
9542 if (i == 100)
9543 return -ETIMEDOUT;
9544
9545 return 0;
9546 }
9547
9548 static void cik_pcie_gen3_enable(struct radeon_device *rdev)
9549 {
9550 struct pci_dev *root = rdev->pdev->bus->self;
9551 int bridge_pos, gpu_pos;
9552 u32 speed_cntl, mask, current_data_rate;
9553 int ret, i;
9554 u16 tmp16;
9555
9556 if (pci_is_root_bus(rdev->pdev->bus))
9557 return;
9558
9559 if (radeon_pcie_gen2 == 0)
9560 return;
9561
9562 if (rdev->flags & RADEON_IS_IGP)
9563 return;
9564
9565 if (!(rdev->flags & RADEON_IS_PCIE))
9566 return;
9567
9568 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
9569 if (ret != 0)
9570 return;
9571
9572 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
9573 return;
9574
9575 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9576 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
9577 LC_CURRENT_DATA_RATE_SHIFT;
9578 if (mask & DRM_PCIE_SPEED_80) {
9579 if (current_data_rate == 2) {
9580 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
9581 return;
9582 }
9583 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
9584 } else if (mask & DRM_PCIE_SPEED_50) {
9585 if (current_data_rate == 1) {
9586 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
9587 return;
9588 }
9589 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
9590 }
9591
9592 bridge_pos = pci_pcie_cap(root);
9593 if (!bridge_pos)
9594 return;
9595
9596 gpu_pos = pci_pcie_cap(rdev->pdev);
9597 if (!gpu_pos)
9598 return;
9599
9600 if (mask & DRM_PCIE_SPEED_80) {
9601 /* re-try equalization if gen3 is not already enabled */
9602 if (current_data_rate != 2) {
9603 u16 bridge_cfg, gpu_cfg;
9604 u16 bridge_cfg2, gpu_cfg2;
9605 u32 max_lw, current_lw, tmp;
9606
9607 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9608 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9609
9610 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
9611 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9612
9613 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
9614 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9615
9616 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9617 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
9618 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
9619
9620 if (current_lw < max_lw) {
9621 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9622 if (tmp & LC_RENEGOTIATION_SUPPORT) {
9623 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
9624 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
9625 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
9626 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
9627 }
9628 }
9629
9630 for (i = 0; i < 10; i++) {
9631 /* check status */
9632 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
9633 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
9634 break;
9635
9636 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9637 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9638
9639 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
9640 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
9641
9642 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9643 tmp |= LC_SET_QUIESCE;
9644 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9645
9646 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9647 tmp |= LC_REDO_EQ;
9648 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9649
9650 mdelay(100);
9651
9652 /* linkctl */
9653 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
9654 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9655 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
9656 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9657
9658 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
9659 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9660 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
9661 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9662
9663 /* linkctl2 */
9664 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
9665 tmp16 &= ~((1 << 4) | (7 << 9));
9666 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
9667 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
9668
9669 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9670 tmp16 &= ~((1 << 4) | (7 << 9));
9671 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
9672 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9673
9674 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9675 tmp &= ~LC_SET_QUIESCE;
9676 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9677 }
9678 }
9679 }
9680
9681 /* set the link speed */
9682 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
9683 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
9684 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9685
9686 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9687 tmp16 &= ~0xf;
9688 if (mask & DRM_PCIE_SPEED_80)
9689 tmp16 |= 3; /* gen3 */
9690 else if (mask & DRM_PCIE_SPEED_50)
9691 tmp16 |= 2; /* gen2 */
9692 else
9693 tmp16 |= 1; /* gen1 */
9694 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9695
9696 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9697 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
9698 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9699
9700 for (i = 0; i < rdev->usec_timeout; i++) {
9701 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9702 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
9703 break;
9704 udelay(1);
9705 }
9706 }
9707
9708 static void cik_program_aspm(struct radeon_device *rdev)
9709 {
9710 u32 data, orig;
9711 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
9712 bool disable_clkreq = false;
9713
9714 if (radeon_aspm == 0)
9715 return;
9716
9717 /* XXX double check IGPs */
9718 if (rdev->flags & RADEON_IS_IGP)
9719 return;
9720
9721 if (!(rdev->flags & RADEON_IS_PCIE))
9722 return;
9723
9724 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9725 data &= ~LC_XMIT_N_FTS_MASK;
9726 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
9727 if (orig != data)
9728 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
9729
9730 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
9731 data |= LC_GO_TO_RECOVERY;
9732 if (orig != data)
9733 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
9734
9735 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
9736 data |= P_IGNORE_EDB_ERR;
9737 if (orig != data)
9738 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
9739
9740 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9741 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
9742 data |= LC_PMI_TO_L1_DIS;
9743 if (!disable_l0s)
9744 data |= LC_L0S_INACTIVITY(7);
9745
9746 if (!disable_l1) {
9747 data |= LC_L1_INACTIVITY(7);
9748 data &= ~LC_PMI_TO_L1_DIS;
9749 if (orig != data)
9750 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9751
9752 if (!disable_plloff_in_l1) {
9753 bool clk_req_support;
9754
9755 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
9756 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9757 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9758 if (orig != data)
9759 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
9760
9761 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
9762 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9763 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9764 if (orig != data)
9765 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
9766
9767 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
9768 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9769 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9770 if (orig != data)
9771 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
9772
9773 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
9774 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9775 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9776 if (orig != data)
9777 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
9778
9779 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9780 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
9781 data |= LC_DYN_LANES_PWR_STATE(3);
9782 if (orig != data)
9783 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
9784
9785 if (!disable_clkreq &&
9786 !pci_is_root_bus(rdev->pdev->bus)) {
9787 struct pci_dev *root = rdev->pdev->bus->self;
9788 u32 lnkcap;
9789
9790 clk_req_support = false;
9791 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
9792 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
9793 clk_req_support = true;
9794 } else {
9795 clk_req_support = false;
9796 }
9797
9798 if (clk_req_support) {
9799 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
9800 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
9801 if (orig != data)
9802 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
9803
9804 orig = data = RREG32_SMC(THM_CLK_CNTL);
9805 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
9806 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
9807 if (orig != data)
9808 WREG32_SMC(THM_CLK_CNTL, data);
9809
9810 orig = data = RREG32_SMC(MISC_CLK_CTRL);
9811 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
9812 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
9813 if (orig != data)
9814 WREG32_SMC(MISC_CLK_CTRL, data);
9815
9816 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
9817 data &= ~BCLK_AS_XCLK;
9818 if (orig != data)
9819 WREG32_SMC(CG_CLKPIN_CNTL, data);
9820
9821 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
9822 data &= ~FORCE_BIF_REFCLK_EN;
9823 if (orig != data)
9824 WREG32_SMC(CG_CLKPIN_CNTL_2, data);
9825
9826 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
9827 data &= ~MPLL_CLKOUT_SEL_MASK;
9828 data |= MPLL_CLKOUT_SEL(4);
9829 if (orig != data)
9830 WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
9831 }
9832 }
9833 } else {
9834 if (orig != data)
9835 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9836 }
9837
9838 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
9839 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
9840 if (orig != data)
9841 WREG32_PCIE_PORT(PCIE_CNTL2, data);
9842
9843 if (!disable_l0s) {
9844 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9845 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
9846 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9847 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
9848 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9849 data &= ~LC_L0S_INACTIVITY_MASK;
9850 if (orig != data)
9851 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9852 }
9853 }
9854 }
9855 }
Linux with added FPC-III support