?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge branch 'x86-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel...
[cris-mirror.git] / drivers / gpu / drm / radeon / cik.c
blobd3045a371a557261776ff32a88aca99b8836c538
1 /*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
27 #include <drm/drmP.h>
28 #include "radeon.h"
29 #include "radeon_asic.h"
30 #include "radeon_audio.h"
31 #include "cikd.h"
32 #include "atom.h"
33 #include "cik_blit_shaders.h"
34 #include "radeon_ucode.h"
35 #include "clearstate_ci.h"
36
37 #define SH_MEM_CONFIG_GFX_DEFAULT \
38 ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED)
39
40 MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
41 MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
42 MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
43 MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
44 MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
45 MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin");
46 MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
47 MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
48 MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
49
50 MODULE_FIRMWARE("radeon/bonaire_pfp.bin");
51 MODULE_FIRMWARE("radeon/bonaire_me.bin");
52 MODULE_FIRMWARE("radeon/bonaire_ce.bin");
53 MODULE_FIRMWARE("radeon/bonaire_mec.bin");
54 MODULE_FIRMWARE("radeon/bonaire_mc.bin");
55 MODULE_FIRMWARE("radeon/bonaire_rlc.bin");
56 MODULE_FIRMWARE("radeon/bonaire_sdma.bin");
57 MODULE_FIRMWARE("radeon/bonaire_smc.bin");
58 MODULE_FIRMWARE("radeon/bonaire_k_smc.bin");
59
60 MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
61 MODULE_FIRMWARE("radeon/HAWAII_me.bin");
62 MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
63 MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
64 MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
65 MODULE_FIRMWARE("radeon/HAWAII_mc2.bin");
66 MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
67 MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
68 MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
69
70 MODULE_FIRMWARE("radeon/hawaii_pfp.bin");
71 MODULE_FIRMWARE("radeon/hawaii_me.bin");
72 MODULE_FIRMWARE("radeon/hawaii_ce.bin");
73 MODULE_FIRMWARE("radeon/hawaii_mec.bin");
74 MODULE_FIRMWARE("radeon/hawaii_mc.bin");
75 MODULE_FIRMWARE("radeon/hawaii_rlc.bin");
76 MODULE_FIRMWARE("radeon/hawaii_sdma.bin");
77 MODULE_FIRMWARE("radeon/hawaii_smc.bin");
78 MODULE_FIRMWARE("radeon/hawaii_k_smc.bin");
79
80 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
81 MODULE_FIRMWARE("radeon/KAVERI_me.bin");
82 MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
83 MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
84 MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
85 MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
86
87 MODULE_FIRMWARE("radeon/kaveri_pfp.bin");
88 MODULE_FIRMWARE("radeon/kaveri_me.bin");
89 MODULE_FIRMWARE("radeon/kaveri_ce.bin");
90 MODULE_FIRMWARE("radeon/kaveri_mec.bin");
91 MODULE_FIRMWARE("radeon/kaveri_mec2.bin");
92 MODULE_FIRMWARE("radeon/kaveri_rlc.bin");
93 MODULE_FIRMWARE("radeon/kaveri_sdma.bin");
94
95 MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
96 MODULE_FIRMWARE("radeon/KABINI_me.bin");
97 MODULE_FIRMWARE("radeon/KABINI_ce.bin");
98 MODULE_FIRMWARE("radeon/KABINI_mec.bin");
99 MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
100 MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
101
102 MODULE_FIRMWARE("radeon/kabini_pfp.bin");
103 MODULE_FIRMWARE("radeon/kabini_me.bin");
104 MODULE_FIRMWARE("radeon/kabini_ce.bin");
105 MODULE_FIRMWARE("radeon/kabini_mec.bin");
106 MODULE_FIRMWARE("radeon/kabini_rlc.bin");
107 MODULE_FIRMWARE("radeon/kabini_sdma.bin");
108
109 MODULE_FIRMWARE("radeon/MULLINS_pfp.bin");
110 MODULE_FIRMWARE("radeon/MULLINS_me.bin");
111 MODULE_FIRMWARE("radeon/MULLINS_ce.bin");
112 MODULE_FIRMWARE("radeon/MULLINS_mec.bin");
113 MODULE_FIRMWARE("radeon/MULLINS_rlc.bin");
114 MODULE_FIRMWARE("radeon/MULLINS_sdma.bin");
115
116 MODULE_FIRMWARE("radeon/mullins_pfp.bin");
117 MODULE_FIRMWARE("radeon/mullins_me.bin");
118 MODULE_FIRMWARE("radeon/mullins_ce.bin");
119 MODULE_FIRMWARE("radeon/mullins_mec.bin");
120 MODULE_FIRMWARE("radeon/mullins_rlc.bin");
121 MODULE_FIRMWARE("radeon/mullins_sdma.bin");
122
123 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
124 extern void r600_ih_ring_fini(struct radeon_device *rdev);
125 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
126 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
127 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
128 extern void sumo_rlc_fini(struct radeon_device *rdev);
129 extern int sumo_rlc_init(struct radeon_device *rdev);
130 extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
131 extern void si_rlc_reset(struct radeon_device *rdev);
132 extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
133 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
134 extern int cik_sdma_resume(struct radeon_device *rdev);
135 extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
136 extern void cik_sdma_fini(struct radeon_device *rdev);
137 extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable);
138 static void cik_rlc_stop(struct radeon_device *rdev);
139 static void cik_pcie_gen3_enable(struct radeon_device *rdev);
140 static void cik_program_aspm(struct radeon_device *rdev);
141 static void cik_init_pg(struct radeon_device *rdev);
142 static void cik_init_cg(struct radeon_device *rdev);
143 static void cik_fini_pg(struct radeon_device *rdev);
144 static void cik_fini_cg(struct radeon_device *rdev);
145 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
146 bool enable);
147
148 /**
149 * cik_get_allowed_info_register - fetch the register for the info ioctl
150 *
151 * @rdev: radeon_device pointer
152 * @reg: register offset in bytes
153 * @val: register value
154 *
155 * Returns 0 for success or -EINVAL for an invalid register
156 *
157 */
158 int cik_get_allowed_info_register(struct radeon_device *rdev,
159 u32 reg, u32 *val)
160 {
161 switch (reg) {
162 case GRBM_STATUS:
163 case GRBM_STATUS2:
164 case GRBM_STATUS_SE0:
165 case GRBM_STATUS_SE1:
166 case GRBM_STATUS_SE2:
167 case GRBM_STATUS_SE3:
168 case SRBM_STATUS:
169 case SRBM_STATUS2:
170 case (SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET):
171 case (SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET):
172 case UVD_STATUS:
173 /* TODO VCE */
174 *val = RREG32(reg);
175 return 0;
176 default:
177 return -EINVAL;
178 }
179 }
180
181 /*
182 * Indirect registers accessor
183 */
184 u32 cik_didt_rreg(struct radeon_device *rdev, u32 reg)
185 {
186 unsigned long flags;
187 u32 r;
188
189 spin_lock_irqsave(&rdev->didt_idx_lock, flags);
190 WREG32(CIK_DIDT_IND_INDEX, (reg));
191 r = RREG32(CIK_DIDT_IND_DATA);
192 spin_unlock_irqrestore(&rdev->didt_idx_lock, flags);
193 return r;
194 }
195
196 void cik_didt_wreg(struct radeon_device *rdev, u32 reg, u32 v)
197 {
198 unsigned long flags;
199
200 spin_lock_irqsave(&rdev->didt_idx_lock, flags);
201 WREG32(CIK_DIDT_IND_INDEX, (reg));
202 WREG32(CIK_DIDT_IND_DATA, (v));
203 spin_unlock_irqrestore(&rdev->didt_idx_lock, flags);
204 }
205
206 /* get temperature in millidegrees */
207 int ci_get_temp(struct radeon_device *rdev)
208 {
209 u32 temp;
210 int actual_temp = 0;
211
212 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
213 CTF_TEMP_SHIFT;
214
215 if (temp & 0x200)
216 actual_temp = 255;
217 else
218 actual_temp = temp & 0x1ff;
219
220 actual_temp = actual_temp * 1000;
221
222 return actual_temp;
223 }
224
225 /* get temperature in millidegrees */
226 int kv_get_temp(struct radeon_device *rdev)
227 {
228 u32 temp;
229 int actual_temp = 0;
230
231 temp = RREG32_SMC(0xC0300E0C);
232
233 if (temp)
234 actual_temp = (temp / 8) - 49;
235 else
236 actual_temp = 0;
237
238 actual_temp = actual_temp * 1000;
239
240 return actual_temp;
241 }
242
243 /*
244 * Indirect registers accessor
245 */
246 u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
247 {
248 unsigned long flags;
249 u32 r;
250
251 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
252 WREG32(PCIE_INDEX, reg);
253 (void)RREG32(PCIE_INDEX);
254 r = RREG32(PCIE_DATA);
255 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
256 return r;
257 }
258
259 void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
260 {
261 unsigned long flags;
262
263 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
264 WREG32(PCIE_INDEX, reg);
265 (void)RREG32(PCIE_INDEX);
266 WREG32(PCIE_DATA, v);
267 (void)RREG32(PCIE_DATA);
268 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
269 }
270
271 static const u32 spectre_rlc_save_restore_register_list[] =
272 {
273 (0x0e00 << 16) | (0xc12c >> 2),
274 0x00000000,
275 (0x0e00 << 16) | (0xc140 >> 2),
276 0x00000000,
277 (0x0e00 << 16) | (0xc150 >> 2),
278 0x00000000,
279 (0x0e00 << 16) | (0xc15c >> 2),
280 0x00000000,
281 (0x0e00 << 16) | (0xc168 >> 2),
282 0x00000000,
283 (0x0e00 << 16) | (0xc170 >> 2),
284 0x00000000,
285 (0x0e00 << 16) | (0xc178 >> 2),
286 0x00000000,
287 (0x0e00 << 16) | (0xc204 >> 2),
288 0x00000000,
289 (0x0e00 << 16) | (0xc2b4 >> 2),
290 0x00000000,
291 (0x0e00 << 16) | (0xc2b8 >> 2),
292 0x00000000,
293 (0x0e00 << 16) | (0xc2bc >> 2),
294 0x00000000,
295 (0x0e00 << 16) | (0xc2c0 >> 2),
296 0x00000000,
297 (0x0e00 << 16) | (0x8228 >> 2),
298 0x00000000,
299 (0x0e00 << 16) | (0x829c >> 2),
300 0x00000000,
301 (0x0e00 << 16) | (0x869c >> 2),
302 0x00000000,
303 (0x0600 << 16) | (0x98f4 >> 2),
304 0x00000000,
305 (0x0e00 << 16) | (0x98f8 >> 2),
306 0x00000000,
307 (0x0e00 << 16) | (0x9900 >> 2),
308 0x00000000,
309 (0x0e00 << 16) | (0xc260 >> 2),
310 0x00000000,
311 (0x0e00 << 16) | (0x90e8 >> 2),
312 0x00000000,
313 (0x0e00 << 16) | (0x3c000 >> 2),
314 0x00000000,
315 (0x0e00 << 16) | (0x3c00c >> 2),
316 0x00000000,
317 (0x0e00 << 16) | (0x8c1c >> 2),
318 0x00000000,
319 (0x0e00 << 16) | (0x9700 >> 2),
320 0x00000000,
321 (0x0e00 << 16) | (0xcd20 >> 2),
322 0x00000000,
323 (0x4e00 << 16) | (0xcd20 >> 2),
324 0x00000000,
325 (0x5e00 << 16) | (0xcd20 >> 2),
326 0x00000000,
327 (0x6e00 << 16) | (0xcd20 >> 2),
328 0x00000000,
329 (0x7e00 << 16) | (0xcd20 >> 2),
330 0x00000000,
331 (0x8e00 << 16) | (0xcd20 >> 2),
332 0x00000000,
333 (0x9e00 << 16) | (0xcd20 >> 2),
334 0x00000000,
335 (0xae00 << 16) | (0xcd20 >> 2),
336 0x00000000,
337 (0xbe00 << 16) | (0xcd20 >> 2),
338 0x00000000,
339 (0x0e00 << 16) | (0x89bc >> 2),
340 0x00000000,
341 (0x0e00 << 16) | (0x8900 >> 2),
342 0x00000000,
343 0x3,
344 (0x0e00 << 16) | (0xc130 >> 2),
345 0x00000000,
346 (0x0e00 << 16) | (0xc134 >> 2),
347 0x00000000,
348 (0x0e00 << 16) | (0xc1fc >> 2),
349 0x00000000,
350 (0x0e00 << 16) | (0xc208 >> 2),
351 0x00000000,
352 (0x0e00 << 16) | (0xc264 >> 2),
353 0x00000000,
354 (0x0e00 << 16) | (0xc268 >> 2),
355 0x00000000,
356 (0x0e00 << 16) | (0xc26c >> 2),
357 0x00000000,
358 (0x0e00 << 16) | (0xc270 >> 2),
359 0x00000000,
360 (0x0e00 << 16) | (0xc274 >> 2),
361 0x00000000,
362 (0x0e00 << 16) | (0xc278 >> 2),
363 0x00000000,
364 (0x0e00 << 16) | (0xc27c >> 2),
365 0x00000000,
366 (0x0e00 << 16) | (0xc280 >> 2),
367 0x00000000,
368 (0x0e00 << 16) | (0xc284 >> 2),
369 0x00000000,
370 (0x0e00 << 16) | (0xc288 >> 2),
371 0x00000000,
372 (0x0e00 << 16) | (0xc28c >> 2),
373 0x00000000,
374 (0x0e00 << 16) | (0xc290 >> 2),
375 0x00000000,
376 (0x0e00 << 16) | (0xc294 >> 2),
377 0x00000000,
378 (0x0e00 << 16) | (0xc298 >> 2),
379 0x00000000,
380 (0x0e00 << 16) | (0xc29c >> 2),
381 0x00000000,
382 (0x0e00 << 16) | (0xc2a0 >> 2),
383 0x00000000,
384 (0x0e00 << 16) | (0xc2a4 >> 2),
385 0x00000000,
386 (0x0e00 << 16) | (0xc2a8 >> 2),
387 0x00000000,
388 (0x0e00 << 16) | (0xc2ac >> 2),
389 0x00000000,
390 (0x0e00 << 16) | (0xc2b0 >> 2),
391 0x00000000,
392 (0x0e00 << 16) | (0x301d0 >> 2),
393 0x00000000,
394 (0x0e00 << 16) | (0x30238 >> 2),
395 0x00000000,
396 (0x0e00 << 16) | (0x30250 >> 2),
397 0x00000000,
398 (0x0e00 << 16) | (0x30254 >> 2),
399 0x00000000,
400 (0x0e00 << 16) | (0x30258 >> 2),
401 0x00000000,
402 (0x0e00 << 16) | (0x3025c >> 2),
403 0x00000000,
404 (0x4e00 << 16) | (0xc900 >> 2),
405 0x00000000,
406 (0x5e00 << 16) | (0xc900 >> 2),
407 0x00000000,
408 (0x6e00 << 16) | (0xc900 >> 2),
409 0x00000000,
410 (0x7e00 << 16) | (0xc900 >> 2),
411 0x00000000,
412 (0x8e00 << 16) | (0xc900 >> 2),
413 0x00000000,
414 (0x9e00 << 16) | (0xc900 >> 2),
415 0x00000000,
416 (0xae00 << 16) | (0xc900 >> 2),
417 0x00000000,
418 (0xbe00 << 16) | (0xc900 >> 2),
419 0x00000000,
420 (0x4e00 << 16) | (0xc904 >> 2),
421 0x00000000,
422 (0x5e00 << 16) | (0xc904 >> 2),
423 0x00000000,
424 (0x6e00 << 16) | (0xc904 >> 2),
425 0x00000000,
426 (0x7e00 << 16) | (0xc904 >> 2),
427 0x00000000,
428 (0x8e00 << 16) | (0xc904 >> 2),
429 0x00000000,
430 (0x9e00 << 16) | (0xc904 >> 2),
431 0x00000000,
432 (0xae00 << 16) | (0xc904 >> 2),
433 0x00000000,
434 (0xbe00 << 16) | (0xc904 >> 2),
435 0x00000000,
436 (0x4e00 << 16) | (0xc908 >> 2),
437 0x00000000,
438 (0x5e00 << 16) | (0xc908 >> 2),
439 0x00000000,
440 (0x6e00 << 16) | (0xc908 >> 2),
441 0x00000000,
442 (0x7e00 << 16) | (0xc908 >> 2),
443 0x00000000,
444 (0x8e00 << 16) | (0xc908 >> 2),
445 0x00000000,
446 (0x9e00 << 16) | (0xc908 >> 2),
447 0x00000000,
448 (0xae00 << 16) | (0xc908 >> 2),
449 0x00000000,
450 (0xbe00 << 16) | (0xc908 >> 2),
451 0x00000000,
452 (0x4e00 << 16) | (0xc90c >> 2),
453 0x00000000,
454 (0x5e00 << 16) | (0xc90c >> 2),
455 0x00000000,
456 (0x6e00 << 16) | (0xc90c >> 2),
457 0x00000000,
458 (0x7e00 << 16) | (0xc90c >> 2),
459 0x00000000,
460 (0x8e00 << 16) | (0xc90c >> 2),
461 0x00000000,
462 (0x9e00 << 16) | (0xc90c >> 2),
463 0x00000000,
464 (0xae00 << 16) | (0xc90c >> 2),
465 0x00000000,
466 (0xbe00 << 16) | (0xc90c >> 2),
467 0x00000000,
468 (0x4e00 << 16) | (0xc910 >> 2),
469 0x00000000,
470 (0x5e00 << 16) | (0xc910 >> 2),
471 0x00000000,
472 (0x6e00 << 16) | (0xc910 >> 2),
473 0x00000000,
474 (0x7e00 << 16) | (0xc910 >> 2),
475 0x00000000,
476 (0x8e00 << 16) | (0xc910 >> 2),
477 0x00000000,
478 (0x9e00 << 16) | (0xc910 >> 2),
479 0x00000000,
480 (0xae00 << 16) | (0xc910 >> 2),
481 0x00000000,
482 (0xbe00 << 16) | (0xc910 >> 2),
483 0x00000000,
484 (0x0e00 << 16) | (0xc99c >> 2),
485 0x00000000,
486 (0x0e00 << 16) | (0x9834 >> 2),
487 0x00000000,
488 (0x0000 << 16) | (0x30f00 >> 2),
489 0x00000000,
490 (0x0001 << 16) | (0x30f00 >> 2),
491 0x00000000,
492 (0x0000 << 16) | (0x30f04 >> 2),
493 0x00000000,
494 (0x0001 << 16) | (0x30f04 >> 2),
495 0x00000000,
496 (0x0000 << 16) | (0x30f08 >> 2),
497 0x00000000,
498 (0x0001 << 16) | (0x30f08 >> 2),
499 0x00000000,
500 (0x0000 << 16) | (0x30f0c >> 2),
501 0x00000000,
502 (0x0001 << 16) | (0x30f0c >> 2),
503 0x00000000,
504 (0x0600 << 16) | (0x9b7c >> 2),
505 0x00000000,
506 (0x0e00 << 16) | (0x8a14 >> 2),
507 0x00000000,
508 (0x0e00 << 16) | (0x8a18 >> 2),
509 0x00000000,
510 (0x0600 << 16) | (0x30a00 >> 2),
511 0x00000000,
512 (0x0e00 << 16) | (0x8bf0 >> 2),
513 0x00000000,
514 (0x0e00 << 16) | (0x8bcc >> 2),
515 0x00000000,
516 (0x0e00 << 16) | (0x8b24 >> 2),
517 0x00000000,
518 (0x0e00 << 16) | (0x30a04 >> 2),
519 0x00000000,
520 (0x0600 << 16) | (0x30a10 >> 2),
521 0x00000000,
522 (0x0600 << 16) | (0x30a14 >> 2),
523 0x00000000,
524 (0x0600 << 16) | (0x30a18 >> 2),
525 0x00000000,
526 (0x0600 << 16) | (0x30a2c >> 2),
527 0x00000000,
528 (0x0e00 << 16) | (0xc700 >> 2),
529 0x00000000,
530 (0x0e00 << 16) | (0xc704 >> 2),
531 0x00000000,
532 (0x0e00 << 16) | (0xc708 >> 2),
533 0x00000000,
534 (0x0e00 << 16) | (0xc768 >> 2),
535 0x00000000,
536 (0x0400 << 16) | (0xc770 >> 2),
537 0x00000000,
538 (0x0400 << 16) | (0xc774 >> 2),
539 0x00000000,
540 (0x0400 << 16) | (0xc778 >> 2),
541 0x00000000,
542 (0x0400 << 16) | (0xc77c >> 2),
543 0x00000000,
544 (0x0400 << 16) | (0xc780 >> 2),
545 0x00000000,
546 (0x0400 << 16) | (0xc784 >> 2),
547 0x00000000,
548 (0x0400 << 16) | (0xc788 >> 2),
549 0x00000000,
550 (0x0400 << 16) | (0xc78c >> 2),
551 0x00000000,
552 (0x0400 << 16) | (0xc798 >> 2),
553 0x00000000,
554 (0x0400 << 16) | (0xc79c >> 2),
555 0x00000000,
556 (0x0400 << 16) | (0xc7a0 >> 2),
557 0x00000000,
558 (0x0400 << 16) | (0xc7a4 >> 2),
559 0x00000000,
560 (0x0400 << 16) | (0xc7a8 >> 2),
561 0x00000000,
562 (0x0400 << 16) | (0xc7ac >> 2),
563 0x00000000,
564 (0x0400 << 16) | (0xc7b0 >> 2),
565 0x00000000,
566 (0x0400 << 16) | (0xc7b4 >> 2),
567 0x00000000,
568 (0x0e00 << 16) | (0x9100 >> 2),
569 0x00000000,
570 (0x0e00 << 16) | (0x3c010 >> 2),
571 0x00000000,
572 (0x0e00 << 16) | (0x92a8 >> 2),
573 0x00000000,
574 (0x0e00 << 16) | (0x92ac >> 2),
575 0x00000000,
576 (0x0e00 << 16) | (0x92b4 >> 2),
577 0x00000000,
578 (0x0e00 << 16) | (0x92b8 >> 2),
579 0x00000000,
580 (0x0e00 << 16) | (0x92bc >> 2),
581 0x00000000,
582 (0x0e00 << 16) | (0x92c0 >> 2),
583 0x00000000,
584 (0x0e00 << 16) | (0x92c4 >> 2),
585 0x00000000,
586 (0x0e00 << 16) | (0x92c8 >> 2),
587 0x00000000,
588 (0x0e00 << 16) | (0x92cc >> 2),
589 0x00000000,
590 (0x0e00 << 16) | (0x92d0 >> 2),
591 0x00000000,
592 (0x0e00 << 16) | (0x8c00 >> 2),
593 0x00000000,
594 (0x0e00 << 16) | (0x8c04 >> 2),
595 0x00000000,
596 (0x0e00 << 16) | (0x8c20 >> 2),
597 0x00000000,
598 (0x0e00 << 16) | (0x8c38 >> 2),
599 0x00000000,
600 (0x0e00 << 16) | (0x8c3c >> 2),
601 0x00000000,
602 (0x0e00 << 16) | (0xae00 >> 2),
603 0x00000000,
604 (0x0e00 << 16) | (0x9604 >> 2),
605 0x00000000,
606 (0x0e00 << 16) | (0xac08 >> 2),
607 0x00000000,
608 (0x0e00 << 16) | (0xac0c >> 2),
609 0x00000000,
610 (0x0e00 << 16) | (0xac10 >> 2),
611 0x00000000,
612 (0x0e00 << 16) | (0xac14 >> 2),
613 0x00000000,
614 (0x0e00 << 16) | (0xac58 >> 2),
615 0x00000000,
616 (0x0e00 << 16) | (0xac68 >> 2),
617 0x00000000,
618 (0x0e00 << 16) | (0xac6c >> 2),
619 0x00000000,
620 (0x0e00 << 16) | (0xac70 >> 2),
621 0x00000000,
622 (0x0e00 << 16) | (0xac74 >> 2),
623 0x00000000,
624 (0x0e00 << 16) | (0xac78 >> 2),
625 0x00000000,
626 (0x0e00 << 16) | (0xac7c >> 2),
627 0x00000000,
628 (0x0e00 << 16) | (0xac80 >> 2),
629 0x00000000,
630 (0x0e00 << 16) | (0xac84 >> 2),
631 0x00000000,
632 (0x0e00 << 16) | (0xac88 >> 2),
633 0x00000000,
634 (0x0e00 << 16) | (0xac8c >> 2),
635 0x00000000,
636 (0x0e00 << 16) | (0x970c >> 2),
637 0x00000000,
638 (0x0e00 << 16) | (0x9714 >> 2),
639 0x00000000,
640 (0x0e00 << 16) | (0x9718 >> 2),
641 0x00000000,
642 (0x0e00 << 16) | (0x971c >> 2),
643 0x00000000,
644 (0x0e00 << 16) | (0x31068 >> 2),
645 0x00000000,
646 (0x4e00 << 16) | (0x31068 >> 2),
647 0x00000000,
648 (0x5e00 << 16) | (0x31068 >> 2),
649 0x00000000,
650 (0x6e00 << 16) | (0x31068 >> 2),
651 0x00000000,
652 (0x7e00 << 16) | (0x31068 >> 2),
653 0x00000000,
654 (0x8e00 << 16) | (0x31068 >> 2),
655 0x00000000,
656 (0x9e00 << 16) | (0x31068 >> 2),
657 0x00000000,
658 (0xae00 << 16) | (0x31068 >> 2),
659 0x00000000,
660 (0xbe00 << 16) | (0x31068 >> 2),
661 0x00000000,
662 (0x0e00 << 16) | (0xcd10 >> 2),
663 0x00000000,
664 (0x0e00 << 16) | (0xcd14 >> 2),
665 0x00000000,
666 (0x0e00 << 16) | (0x88b0 >> 2),
667 0x00000000,
668 (0x0e00 << 16) | (0x88b4 >> 2),
669 0x00000000,
670 (0x0e00 << 16) | (0x88b8 >> 2),
671 0x00000000,
672 (0x0e00 << 16) | (0x88bc >> 2),
673 0x00000000,
674 (0x0400 << 16) | (0x89c0 >> 2),
675 0x00000000,
676 (0x0e00 << 16) | (0x88c4 >> 2),
677 0x00000000,
678 (0x0e00 << 16) | (0x88c8 >> 2),
679 0x00000000,
680 (0x0e00 << 16) | (0x88d0 >> 2),
681 0x00000000,
682 (0x0e00 << 16) | (0x88d4 >> 2),
683 0x00000000,
684 (0x0e00 << 16) | (0x88d8 >> 2),
685 0x00000000,
686 (0x0e00 << 16) | (0x8980 >> 2),
687 0x00000000,
688 (0x0e00 << 16) | (0x30938 >> 2),
689 0x00000000,
690 (0x0e00 << 16) | (0x3093c >> 2),
691 0x00000000,
692 (0x0e00 << 16) | (0x30940 >> 2),
693 0x00000000,
694 (0x0e00 << 16) | (0x89a0 >> 2),
695 0x00000000,
696 (0x0e00 << 16) | (0x30900 >> 2),
697 0x00000000,
698 (0x0e00 << 16) | (0x30904 >> 2),
699 0x00000000,
700 (0x0e00 << 16) | (0x89b4 >> 2),
701 0x00000000,
702 (0x0e00 << 16) | (0x3c210 >> 2),
703 0x00000000,
704 (0x0e00 << 16) | (0x3c214 >> 2),
705 0x00000000,
706 (0x0e00 << 16) | (0x3c218 >> 2),
707 0x00000000,
708 (0x0e00 << 16) | (0x8904 >> 2),
709 0x00000000,
710 0x5,
711 (0x0e00 << 16) | (0x8c28 >> 2),
712 (0x0e00 << 16) | (0x8c2c >> 2),
713 (0x0e00 << 16) | (0x8c30 >> 2),
714 (0x0e00 << 16) | (0x8c34 >> 2),
715 (0x0e00 << 16) | (0x9600 >> 2),
716 };
717
718 static const u32 kalindi_rlc_save_restore_register_list[] =
719 {
720 (0x0e00 << 16) | (0xc12c >> 2),
721 0x00000000,
722 (0x0e00 << 16) | (0xc140 >> 2),
723 0x00000000,
724 (0x0e00 << 16) | (0xc150 >> 2),
725 0x00000000,
726 (0x0e00 << 16) | (0xc15c >> 2),
727 0x00000000,
728 (0x0e00 << 16) | (0xc168 >> 2),
729 0x00000000,
730 (0x0e00 << 16) | (0xc170 >> 2),
731 0x00000000,
732 (0x0e00 << 16) | (0xc204 >> 2),
733 0x00000000,
734 (0x0e00 << 16) | (0xc2b4 >> 2),
735 0x00000000,
736 (0x0e00 << 16) | (0xc2b8 >> 2),
737 0x00000000,
738 (0x0e00 << 16) | (0xc2bc >> 2),
739 0x00000000,
740 (0x0e00 << 16) | (0xc2c0 >> 2),
741 0x00000000,
742 (0x0e00 << 16) | (0x8228 >> 2),
743 0x00000000,
744 (0x0e00 << 16) | (0x829c >> 2),
745 0x00000000,
746 (0x0e00 << 16) | (0x869c >> 2),
747 0x00000000,
748 (0x0600 << 16) | (0x98f4 >> 2),
749 0x00000000,
750 (0x0e00 << 16) | (0x98f8 >> 2),
751 0x00000000,
752 (0x0e00 << 16) | (0x9900 >> 2),
753 0x00000000,
754 (0x0e00 << 16) | (0xc260 >> 2),
755 0x00000000,
756 (0x0e00 << 16) | (0x90e8 >> 2),
757 0x00000000,
758 (0x0e00 << 16) | (0x3c000 >> 2),
759 0x00000000,
760 (0x0e00 << 16) | (0x3c00c >> 2),
761 0x00000000,
762 (0x0e00 << 16) | (0x8c1c >> 2),
763 0x00000000,
764 (0x0e00 << 16) | (0x9700 >> 2),
765 0x00000000,
766 (0x0e00 << 16) | (0xcd20 >> 2),
767 0x00000000,
768 (0x4e00 << 16) | (0xcd20 >> 2),
769 0x00000000,
770 (0x5e00 << 16) | (0xcd20 >> 2),
771 0x00000000,
772 (0x6e00 << 16) | (0xcd20 >> 2),
773 0x00000000,
774 (0x7e00 << 16) | (0xcd20 >> 2),
775 0x00000000,
776 (0x0e00 << 16) | (0x89bc >> 2),
777 0x00000000,
778 (0x0e00 << 16) | (0x8900 >> 2),
779 0x00000000,
780 0x3,
781 (0x0e00 << 16) | (0xc130 >> 2),
782 0x00000000,
783 (0x0e00 << 16) | (0xc134 >> 2),
784 0x00000000,
785 (0x0e00 << 16) | (0xc1fc >> 2),
786 0x00000000,
787 (0x0e00 << 16) | (0xc208 >> 2),
788 0x00000000,
789 (0x0e00 << 16) | (0xc264 >> 2),
790 0x00000000,
791 (0x0e00 << 16) | (0xc268 >> 2),
792 0x00000000,
793 (0x0e00 << 16) | (0xc26c >> 2),
794 0x00000000,
795 (0x0e00 << 16) | (0xc270 >> 2),
796 0x00000000,
797 (0x0e00 << 16) | (0xc274 >> 2),
798 0x00000000,
799 (0x0e00 << 16) | (0xc28c >> 2),
800 0x00000000,
801 (0x0e00 << 16) | (0xc290 >> 2),
802 0x00000000,
803 (0x0e00 << 16) | (0xc294 >> 2),
804 0x00000000,
805 (0x0e00 << 16) | (0xc298 >> 2),
806 0x00000000,
807 (0x0e00 << 16) | (0xc2a0 >> 2),
808 0x00000000,
809 (0x0e00 << 16) | (0xc2a4 >> 2),
810 0x00000000,
811 (0x0e00 << 16) | (0xc2a8 >> 2),
812 0x00000000,
813 (0x0e00 << 16) | (0xc2ac >> 2),
814 0x00000000,
815 (0x0e00 << 16) | (0x301d0 >> 2),
816 0x00000000,
817 (0x0e00 << 16) | (0x30238 >> 2),
818 0x00000000,
819 (0x0e00 << 16) | (0x30250 >> 2),
820 0x00000000,
821 (0x0e00 << 16) | (0x30254 >> 2),
822 0x00000000,
823 (0x0e00 << 16) | (0x30258 >> 2),
824 0x00000000,
825 (0x0e00 << 16) | (0x3025c >> 2),
826 0x00000000,
827 (0x4e00 << 16) | (0xc900 >> 2),
828 0x00000000,
829 (0x5e00 << 16) | (0xc900 >> 2),
830 0x00000000,
831 (0x6e00 << 16) | (0xc900 >> 2),
832 0x00000000,
833 (0x7e00 << 16) | (0xc900 >> 2),
834 0x00000000,
835 (0x4e00 << 16) | (0xc904 >> 2),
836 0x00000000,
837 (0x5e00 << 16) | (0xc904 >> 2),
838 0x00000000,
839 (0x6e00 << 16) | (0xc904 >> 2),
840 0x00000000,
841 (0x7e00 << 16) | (0xc904 >> 2),
842 0x00000000,
843 (0x4e00 << 16) | (0xc908 >> 2),
844 0x00000000,
845 (0x5e00 << 16) | (0xc908 >> 2),
846 0x00000000,
847 (0x6e00 << 16) | (0xc908 >> 2),
848 0x00000000,
849 (0x7e00 << 16) | (0xc908 >> 2),
850 0x00000000,
851 (0x4e00 << 16) | (0xc90c >> 2),
852 0x00000000,
853 (0x5e00 << 16) | (0xc90c >> 2),
854 0x00000000,
855 (0x6e00 << 16) | (0xc90c >> 2),
856 0x00000000,
857 (0x7e00 << 16) | (0xc90c >> 2),
858 0x00000000,
859 (0x4e00 << 16) | (0xc910 >> 2),
860 0x00000000,
861 (0x5e00 << 16) | (0xc910 >> 2),
862 0x00000000,
863 (0x6e00 << 16) | (0xc910 >> 2),
864 0x00000000,
865 (0x7e00 << 16) | (0xc910 >> 2),
866 0x00000000,
867 (0x0e00 << 16) | (0xc99c >> 2),
868 0x00000000,
869 (0x0e00 << 16) | (0x9834 >> 2),
870 0x00000000,
871 (0x0000 << 16) | (0x30f00 >> 2),
872 0x00000000,
873 (0x0000 << 16) | (0x30f04 >> 2),
874 0x00000000,
875 (0x0000 << 16) | (0x30f08 >> 2),
876 0x00000000,
877 (0x0000 << 16) | (0x30f0c >> 2),
878 0x00000000,
879 (0x0600 << 16) | (0x9b7c >> 2),
880 0x00000000,
881 (0x0e00 << 16) | (0x8a14 >> 2),
882 0x00000000,
883 (0x0e00 << 16) | (0x8a18 >> 2),
884 0x00000000,
885 (0x0600 << 16) | (0x30a00 >> 2),
886 0x00000000,
887 (0x0e00 << 16) | (0x8bf0 >> 2),
888 0x00000000,
889 (0x0e00 << 16) | (0x8bcc >> 2),
890 0x00000000,
891 (0x0e00 << 16) | (0x8b24 >> 2),
892 0x00000000,
893 (0x0e00 << 16) | (0x30a04 >> 2),
894 0x00000000,
895 (0x0600 << 16) | (0x30a10 >> 2),
896 0x00000000,
897 (0x0600 << 16) | (0x30a14 >> 2),
898 0x00000000,
899 (0x0600 << 16) | (0x30a18 >> 2),
900 0x00000000,
901 (0x0600 << 16) | (0x30a2c >> 2),
902 0x00000000,
903 (0x0e00 << 16) | (0xc700 >> 2),
904 0x00000000,
905 (0x0e00 << 16) | (0xc704 >> 2),
906 0x00000000,
907 (0x0e00 << 16) | (0xc708 >> 2),
908 0x00000000,
909 (0x0e00 << 16) | (0xc768 >> 2),
910 0x00000000,
911 (0x0400 << 16) | (0xc770 >> 2),
912 0x00000000,
913 (0x0400 << 16) | (0xc774 >> 2),
914 0x00000000,
915 (0x0400 << 16) | (0xc798 >> 2),
916 0x00000000,
917 (0x0400 << 16) | (0xc79c >> 2),
918 0x00000000,
919 (0x0e00 << 16) | (0x9100 >> 2),
920 0x00000000,
921 (0x0e00 << 16) | (0x3c010 >> 2),
922 0x00000000,
923 (0x0e00 << 16) | (0x8c00 >> 2),
924 0x00000000,
925 (0x0e00 << 16) | (0x8c04 >> 2),
926 0x00000000,
927 (0x0e00 << 16) | (0x8c20 >> 2),
928 0x00000000,
929 (0x0e00 << 16) | (0x8c38 >> 2),
930 0x00000000,
931 (0x0e00 << 16) | (0x8c3c >> 2),
932 0x00000000,
933 (0x0e00 << 16) | (0xae00 >> 2),
934 0x00000000,
935 (0x0e00 << 16) | (0x9604 >> 2),
936 0x00000000,
937 (0x0e00 << 16) | (0xac08 >> 2),
938 0x00000000,
939 (0x0e00 << 16) | (0xac0c >> 2),
940 0x00000000,
941 (0x0e00 << 16) | (0xac10 >> 2),
942 0x00000000,
943 (0x0e00 << 16) | (0xac14 >> 2),
944 0x00000000,
945 (0x0e00 << 16) | (0xac58 >> 2),
946 0x00000000,
947 (0x0e00 << 16) | (0xac68 >> 2),
948 0x00000000,
949 (0x0e00 << 16) | (0xac6c >> 2),
950 0x00000000,
951 (0x0e00 << 16) | (0xac70 >> 2),
952 0x00000000,
953 (0x0e00 << 16) | (0xac74 >> 2),
954 0x00000000,
955 (0x0e00 << 16) | (0xac78 >> 2),
956 0x00000000,
957 (0x0e00 << 16) | (0xac7c >> 2),
958 0x00000000,
959 (0x0e00 << 16) | (0xac80 >> 2),
960 0x00000000,
961 (0x0e00 << 16) | (0xac84 >> 2),
962 0x00000000,
963 (0x0e00 << 16) | (0xac88 >> 2),
964 0x00000000,
965 (0x0e00 << 16) | (0xac8c >> 2),
966 0x00000000,
967 (0x0e00 << 16) | (0x970c >> 2),
968 0x00000000,
969 (0x0e00 << 16) | (0x9714 >> 2),
970 0x00000000,
971 (0x0e00 << 16) | (0x9718 >> 2),
972 0x00000000,
973 (0x0e00 << 16) | (0x971c >> 2),
974 0x00000000,
975 (0x0e00 << 16) | (0x31068 >> 2),
976 0x00000000,
977 (0x4e00 << 16) | (0x31068 >> 2),
978 0x00000000,
979 (0x5e00 << 16) | (0x31068 >> 2),
980 0x00000000,
981 (0x6e00 << 16) | (0x31068 >> 2),
982 0x00000000,
983 (0x7e00 << 16) | (0x31068 >> 2),
984 0x00000000,
985 (0x0e00 << 16) | (0xcd10 >> 2),
986 0x00000000,
987 (0x0e00 << 16) | (0xcd14 >> 2),
988 0x00000000,
989 (0x0e00 << 16) | (0x88b0 >> 2),
990 0x00000000,
991 (0x0e00 << 16) | (0x88b4 >> 2),
992 0x00000000,
993 (0x0e00 << 16) | (0x88b8 >> 2),
994 0x00000000,
995 (0x0e00 << 16) | (0x88bc >> 2),
996 0x00000000,
997 (0x0400 << 16) | (0x89c0 >> 2),
998 0x00000000,
999 (0x0e00 << 16) | (0x88c4 >> 2),
1000 0x00000000,
1001 (0x0e00 << 16) | (0x88c8 >> 2),
1002 0x00000000,
1003 (0x0e00 << 16) | (0x88d0 >> 2),
1004 0x00000000,
1005 (0x0e00 << 16) | (0x88d4 >> 2),
1006 0x00000000,
1007 (0x0e00 << 16) | (0x88d8 >> 2),
1008 0x00000000,
1009 (0x0e00 << 16) | (0x8980 >> 2),
1010 0x00000000,
1011 (0x0e00 << 16) | (0x30938 >> 2),
1012 0x00000000,
1013 (0x0e00 << 16) | (0x3093c >> 2),
1014 0x00000000,
1015 (0x0e00 << 16) | (0x30940 >> 2),
1016 0x00000000,
1017 (0x0e00 << 16) | (0x89a0 >> 2),
1018 0x00000000,
1019 (0x0e00 << 16) | (0x30900 >> 2),
1020 0x00000000,
1021 (0x0e00 << 16) | (0x30904 >> 2),
1022 0x00000000,
1023 (0x0e00 << 16) | (0x89b4 >> 2),
1024 0x00000000,
1025 (0x0e00 << 16) | (0x3e1fc >> 2),
1026 0x00000000,
1027 (0x0e00 << 16) | (0x3c210 >> 2),
1028 0x00000000,
1029 (0x0e00 << 16) | (0x3c214 >> 2),
1030 0x00000000,
1031 (0x0e00 << 16) | (0x3c218 >> 2),
1032 0x00000000,
1033 (0x0e00 << 16) | (0x8904 >> 2),
1034 0x00000000,
1035 0x5,
1036 (0x0e00 << 16) | (0x8c28 >> 2),
1037 (0x0e00 << 16) | (0x8c2c >> 2),
1038 (0x0e00 << 16) | (0x8c30 >> 2),
1039 (0x0e00 << 16) | (0x8c34 >> 2),
1040 (0x0e00 << 16) | (0x9600 >> 2),
1041 };
1042
1043 static const u32 bonaire_golden_spm_registers[] =
1044 {
1045 0x30800, 0xe0ffffff, 0xe0000000
1046 };
1047
1048 static const u32 bonaire_golden_common_registers[] =
1049 {
1050 0xc770, 0xffffffff, 0x00000800,
1051 0xc774, 0xffffffff, 0x00000800,
1052 0xc798, 0xffffffff, 0x00007fbf,
1053 0xc79c, 0xffffffff, 0x00007faf
1054 };
1055
1056 static const u32 bonaire_golden_registers[] =
1057 {
1058 0x3354, 0x00000333, 0x00000333,
1059 0x3350, 0x000c0fc0, 0x00040200,
1060 0x9a10, 0x00010000, 0x00058208,
1061 0x3c000, 0xffff1fff, 0x00140000,
1062 0x3c200, 0xfdfc0fff, 0x00000100,
1063 0x3c234, 0x40000000, 0x40000200,
1064 0x9830, 0xffffffff, 0x00000000,
1065 0x9834, 0xf00fffff, 0x00000400,
1066 0x9838, 0x0002021c, 0x00020200,
1067 0xc78, 0x00000080, 0x00000000,
1068 0x5bb0, 0x000000f0, 0x00000070,
1069 0x5bc0, 0xf0311fff, 0x80300000,
1070 0x98f8, 0x73773777, 0x12010001,
1071 0x350c, 0x00810000, 0x408af000,
1072 0x7030, 0x31000111, 0x00000011,
1073 0x2f48, 0x73773777, 0x12010001,
1074 0x220c, 0x00007fb6, 0x0021a1b1,
1075 0x2210, 0x00007fb6, 0x002021b1,
1076 0x2180, 0x00007fb6, 0x00002191,
1077 0x2218, 0x00007fb6, 0x002121b1,
1078 0x221c, 0x00007fb6, 0x002021b1,
1079 0x21dc, 0x00007fb6, 0x00002191,
1080 0x21e0, 0x00007fb6, 0x00002191,
1081 0x3628, 0x0000003f, 0x0000000a,
1082 0x362c, 0x0000003f, 0x0000000a,
1083 0x2ae4, 0x00073ffe, 0x000022a2,
1084 0x240c, 0x000007ff, 0x00000000,
1085 0x8a14, 0xf000003f, 0x00000007,
1086 0x8bf0, 0x00002001, 0x00000001,
1087 0x8b24, 0xffffffff, 0x00ffffff,
1088 0x30a04, 0x0000ff0f, 0x00000000,
1089 0x28a4c, 0x07ffffff, 0x06000000,
1090 0x4d8, 0x00000fff, 0x00000100,
1091 0x3e78, 0x00000001, 0x00000002,
1092 0x9100, 0x03000000, 0x0362c688,
1093 0x8c00, 0x000000ff, 0x00000001,
1094 0xe40, 0x00001fff, 0x00001fff,
1095 0x9060, 0x0000007f, 0x00000020,
1096 0x9508, 0x00010000, 0x00010000,
1097 0xac14, 0x000003ff, 0x000000f3,
1098 0xac0c, 0xffffffff, 0x00001032
1099 };
1100
1101 static const u32 bonaire_mgcg_cgcg_init[] =
1102 {
1103 0xc420, 0xffffffff, 0xfffffffc,
1104 0x30800, 0xffffffff, 0xe0000000,
1105 0x3c2a0, 0xffffffff, 0x00000100,
1106 0x3c208, 0xffffffff, 0x00000100,
1107 0x3c2c0, 0xffffffff, 0xc0000100,
1108 0x3c2c8, 0xffffffff, 0xc0000100,
1109 0x3c2c4, 0xffffffff, 0xc0000100,
1110 0x55e4, 0xffffffff, 0x00600100,
1111 0x3c280, 0xffffffff, 0x00000100,
1112 0x3c214, 0xffffffff, 0x06000100,
1113 0x3c220, 0xffffffff, 0x00000100,
1114 0x3c218, 0xffffffff, 0x06000100,
1115 0x3c204, 0xffffffff, 0x00000100,
1116 0x3c2e0, 0xffffffff, 0x00000100,
1117 0x3c224, 0xffffffff, 0x00000100,
1118 0x3c200, 0xffffffff, 0x00000100,
1119 0x3c230, 0xffffffff, 0x00000100,
1120 0x3c234, 0xffffffff, 0x00000100,
1121 0x3c250, 0xffffffff, 0x00000100,
1122 0x3c254, 0xffffffff, 0x00000100,
1123 0x3c258, 0xffffffff, 0x00000100,
1124 0x3c25c, 0xffffffff, 0x00000100,
1125 0x3c260, 0xffffffff, 0x00000100,
1126 0x3c27c, 0xffffffff, 0x00000100,
1127 0x3c278, 0xffffffff, 0x00000100,
1128 0x3c210, 0xffffffff, 0x06000100,
1129 0x3c290, 0xffffffff, 0x00000100,
1130 0x3c274, 0xffffffff, 0x00000100,
1131 0x3c2b4, 0xffffffff, 0x00000100,
1132 0x3c2b0, 0xffffffff, 0x00000100,
1133 0x3c270, 0xffffffff, 0x00000100,
1134 0x30800, 0xffffffff, 0xe0000000,
1135 0x3c020, 0xffffffff, 0x00010000,
1136 0x3c024, 0xffffffff, 0x00030002,
1137 0x3c028, 0xffffffff, 0x00040007,
1138 0x3c02c, 0xffffffff, 0x00060005,
1139 0x3c030, 0xffffffff, 0x00090008,
1140 0x3c034, 0xffffffff, 0x00010000,
1141 0x3c038, 0xffffffff, 0x00030002,
1142 0x3c03c, 0xffffffff, 0x00040007,
1143 0x3c040, 0xffffffff, 0x00060005,
1144 0x3c044, 0xffffffff, 0x00090008,
1145 0x3c048, 0xffffffff, 0x00010000,
1146 0x3c04c, 0xffffffff, 0x00030002,
1147 0x3c050, 0xffffffff, 0x00040007,
1148 0x3c054, 0xffffffff, 0x00060005,
1149 0x3c058, 0xffffffff, 0x00090008,
1150 0x3c05c, 0xffffffff, 0x00010000,
1151 0x3c060, 0xffffffff, 0x00030002,
1152 0x3c064, 0xffffffff, 0x00040007,
1153 0x3c068, 0xffffffff, 0x00060005,
1154 0x3c06c, 0xffffffff, 0x00090008,
1155 0x3c070, 0xffffffff, 0x00010000,
1156 0x3c074, 0xffffffff, 0x00030002,
1157 0x3c078, 0xffffffff, 0x00040007,
1158 0x3c07c, 0xffffffff, 0x00060005,
1159 0x3c080, 0xffffffff, 0x00090008,
1160 0x3c084, 0xffffffff, 0x00010000,
1161 0x3c088, 0xffffffff, 0x00030002,
1162 0x3c08c, 0xffffffff, 0x00040007,
1163 0x3c090, 0xffffffff, 0x00060005,
1164 0x3c094, 0xffffffff, 0x00090008,
1165 0x3c098, 0xffffffff, 0x00010000,
1166 0x3c09c, 0xffffffff, 0x00030002,
1167 0x3c0a0, 0xffffffff, 0x00040007,
1168 0x3c0a4, 0xffffffff, 0x00060005,
1169 0x3c0a8, 0xffffffff, 0x00090008,
1170 0x3c000, 0xffffffff, 0x96e00200,
1171 0x8708, 0xffffffff, 0x00900100,
1172 0xc424, 0xffffffff, 0x0020003f,
1173 0x38, 0xffffffff, 0x0140001c,
1174 0x3c, 0x000f0000, 0x000f0000,
1175 0x220, 0xffffffff, 0xC060000C,
1176 0x224, 0xc0000fff, 0x00000100,
1177 0xf90, 0xffffffff, 0x00000100,
1178 0xf98, 0x00000101, 0x00000000,
1179 0x20a8, 0xffffffff, 0x00000104,
1180 0x55e4, 0xff000fff, 0x00000100,
1181 0x30cc, 0xc0000fff, 0x00000104,
1182 0xc1e4, 0x00000001, 0x00000001,
1183 0xd00c, 0xff000ff0, 0x00000100,
1184 0xd80c, 0xff000ff0, 0x00000100
1185 };
1186
1187 static const u32 spectre_golden_spm_registers[] =
1188 {
1189 0x30800, 0xe0ffffff, 0xe0000000
1190 };
1191
1192 static const u32 spectre_golden_common_registers[] =
1193 {
1194 0xc770, 0xffffffff, 0x00000800,
1195 0xc774, 0xffffffff, 0x00000800,
1196 0xc798, 0xffffffff, 0x00007fbf,
1197 0xc79c, 0xffffffff, 0x00007faf
1198 };
1199
1200 static const u32 spectre_golden_registers[] =
1201 {
1202 0x3c000, 0xffff1fff, 0x96940200,
1203 0x3c00c, 0xffff0001, 0xff000000,
1204 0x3c200, 0xfffc0fff, 0x00000100,
1205 0x6ed8, 0x00010101, 0x00010000,
1206 0x9834, 0xf00fffff, 0x00000400,
1207 0x9838, 0xfffffffc, 0x00020200,
1208 0x5bb0, 0x000000f0, 0x00000070,
1209 0x5bc0, 0xf0311fff, 0x80300000,
1210 0x98f8, 0x73773777, 0x12010001,
1211 0x9b7c, 0x00ff0000, 0x00fc0000,
1212 0x2f48, 0x73773777, 0x12010001,
1213 0x8a14, 0xf000003f, 0x00000007,
1214 0x8b24, 0xffffffff, 0x00ffffff,
1215 0x28350, 0x3f3f3fff, 0x00000082,
1216 0x28354, 0x0000003f, 0x00000000,
1217 0x3e78, 0x00000001, 0x00000002,
1218 0x913c, 0xffff03df, 0x00000004,
1219 0xc768, 0x00000008, 0x00000008,
1220 0x8c00, 0x000008ff, 0x00000800,
1221 0x9508, 0x00010000, 0x00010000,
1222 0xac0c, 0xffffffff, 0x54763210,
1223 0x214f8, 0x01ff01ff, 0x00000002,
1224 0x21498, 0x007ff800, 0x00200000,
1225 0x2015c, 0xffffffff, 0x00000f40,
1226 0x30934, 0xffffffff, 0x00000001
1227 };
1228
1229 static const u32 spectre_mgcg_cgcg_init[] =
1230 {
1231 0xc420, 0xffffffff, 0xfffffffc,
1232 0x30800, 0xffffffff, 0xe0000000,
1233 0x3c2a0, 0xffffffff, 0x00000100,
1234 0x3c208, 0xffffffff, 0x00000100,
1235 0x3c2c0, 0xffffffff, 0x00000100,
1236 0x3c2c8, 0xffffffff, 0x00000100,
1237 0x3c2c4, 0xffffffff, 0x00000100,
1238 0x55e4, 0xffffffff, 0x00600100,
1239 0x3c280, 0xffffffff, 0x00000100,
1240 0x3c214, 0xffffffff, 0x06000100,
1241 0x3c220, 0xffffffff, 0x00000100,
1242 0x3c218, 0xffffffff, 0x06000100,
1243 0x3c204, 0xffffffff, 0x00000100,
1244 0x3c2e0, 0xffffffff, 0x00000100,
1245 0x3c224, 0xffffffff, 0x00000100,
1246 0x3c200, 0xffffffff, 0x00000100,
1247 0x3c230, 0xffffffff, 0x00000100,
1248 0x3c234, 0xffffffff, 0x00000100,
1249 0x3c250, 0xffffffff, 0x00000100,
1250 0x3c254, 0xffffffff, 0x00000100,
1251 0x3c258, 0xffffffff, 0x00000100,
1252 0x3c25c, 0xffffffff, 0x00000100,
1253 0x3c260, 0xffffffff, 0x00000100,
1254 0x3c27c, 0xffffffff, 0x00000100,
1255 0x3c278, 0xffffffff, 0x00000100,
1256 0x3c210, 0xffffffff, 0x06000100,
1257 0x3c290, 0xffffffff, 0x00000100,
1258 0x3c274, 0xffffffff, 0x00000100,
1259 0x3c2b4, 0xffffffff, 0x00000100,
1260 0x3c2b0, 0xffffffff, 0x00000100,
1261 0x3c270, 0xffffffff, 0x00000100,
1262 0x30800, 0xffffffff, 0xe0000000,
1263 0x3c020, 0xffffffff, 0x00010000,
1264 0x3c024, 0xffffffff, 0x00030002,
1265 0x3c028, 0xffffffff, 0x00040007,
1266 0x3c02c, 0xffffffff, 0x00060005,
1267 0x3c030, 0xffffffff, 0x00090008,
1268 0x3c034, 0xffffffff, 0x00010000,
1269 0x3c038, 0xffffffff, 0x00030002,
1270 0x3c03c, 0xffffffff, 0x00040007,
1271 0x3c040, 0xffffffff, 0x00060005,
1272 0x3c044, 0xffffffff, 0x00090008,
1273 0x3c048, 0xffffffff, 0x00010000,
1274 0x3c04c, 0xffffffff, 0x00030002,
1275 0x3c050, 0xffffffff, 0x00040007,
1276 0x3c054, 0xffffffff, 0x00060005,
1277 0x3c058, 0xffffffff, 0x00090008,
1278 0x3c05c, 0xffffffff, 0x00010000,
1279 0x3c060, 0xffffffff, 0x00030002,
1280 0x3c064, 0xffffffff, 0x00040007,
1281 0x3c068, 0xffffffff, 0x00060005,
1282 0x3c06c, 0xffffffff, 0x00090008,
1283 0x3c070, 0xffffffff, 0x00010000,
1284 0x3c074, 0xffffffff, 0x00030002,
1285 0x3c078, 0xffffffff, 0x00040007,
1286 0x3c07c, 0xffffffff, 0x00060005,
1287 0x3c080, 0xffffffff, 0x00090008,
1288 0x3c084, 0xffffffff, 0x00010000,
1289 0x3c088, 0xffffffff, 0x00030002,
1290 0x3c08c, 0xffffffff, 0x00040007,
1291 0x3c090, 0xffffffff, 0x00060005,
1292 0x3c094, 0xffffffff, 0x00090008,
1293 0x3c098, 0xffffffff, 0x00010000,
1294 0x3c09c, 0xffffffff, 0x00030002,
1295 0x3c0a0, 0xffffffff, 0x00040007,
1296 0x3c0a4, 0xffffffff, 0x00060005,
1297 0x3c0a8, 0xffffffff, 0x00090008,
1298 0x3c0ac, 0xffffffff, 0x00010000,
1299 0x3c0b0, 0xffffffff, 0x00030002,
1300 0x3c0b4, 0xffffffff, 0x00040007,
1301 0x3c0b8, 0xffffffff, 0x00060005,
1302 0x3c0bc, 0xffffffff, 0x00090008,
1303 0x3c000, 0xffffffff, 0x96e00200,
1304 0x8708, 0xffffffff, 0x00900100,
1305 0xc424, 0xffffffff, 0x0020003f,
1306 0x38, 0xffffffff, 0x0140001c,
1307 0x3c, 0x000f0000, 0x000f0000,
1308 0x220, 0xffffffff, 0xC060000C,
1309 0x224, 0xc0000fff, 0x00000100,
1310 0xf90, 0xffffffff, 0x00000100,
1311 0xf98, 0x00000101, 0x00000000,
1312 0x20a8, 0xffffffff, 0x00000104,
1313 0x55e4, 0xff000fff, 0x00000100,
1314 0x30cc, 0xc0000fff, 0x00000104,
1315 0xc1e4, 0x00000001, 0x00000001,
1316 0xd00c, 0xff000ff0, 0x00000100,
1317 0xd80c, 0xff000ff0, 0x00000100
1318 };
1319
1320 static const u32 kalindi_golden_spm_registers[] =
1321 {
1322 0x30800, 0xe0ffffff, 0xe0000000
1323 };
1324
1325 static const u32 kalindi_golden_common_registers[] =
1326 {
1327 0xc770, 0xffffffff, 0x00000800,
1328 0xc774, 0xffffffff, 0x00000800,
1329 0xc798, 0xffffffff, 0x00007fbf,
1330 0xc79c, 0xffffffff, 0x00007faf
1331 };
1332
1333 static const u32 kalindi_golden_registers[] =
1334 {
1335 0x3c000, 0xffffdfff, 0x6e944040,
1336 0x55e4, 0xff607fff, 0xfc000100,
1337 0x3c220, 0xff000fff, 0x00000100,
1338 0x3c224, 0xff000fff, 0x00000100,
1339 0x3c200, 0xfffc0fff, 0x00000100,
1340 0x6ed8, 0x00010101, 0x00010000,
1341 0x9830, 0xffffffff, 0x00000000,
1342 0x9834, 0xf00fffff, 0x00000400,
1343 0x5bb0, 0x000000f0, 0x00000070,
1344 0x5bc0, 0xf0311fff, 0x80300000,
1345 0x98f8, 0x73773777, 0x12010001,
1346 0x98fc, 0xffffffff, 0x00000010,
1347 0x9b7c, 0x00ff0000, 0x00fc0000,
1348 0x8030, 0x00001f0f, 0x0000100a,
1349 0x2f48, 0x73773777, 0x12010001,
1350 0x2408, 0x000fffff, 0x000c007f,
1351 0x8a14, 0xf000003f, 0x00000007,
1352 0x8b24, 0x3fff3fff, 0x00ffcfff,
1353 0x30a04, 0x0000ff0f, 0x00000000,
1354 0x28a4c, 0x07ffffff, 0x06000000,
1355 0x4d8, 0x00000fff, 0x00000100,
1356 0x3e78, 0x00000001, 0x00000002,
1357 0xc768, 0x00000008, 0x00000008,
1358 0x8c00, 0x000000ff, 0x00000003,
1359 0x214f8, 0x01ff01ff, 0x00000002,
1360 0x21498, 0x007ff800, 0x00200000,
1361 0x2015c, 0xffffffff, 0x00000f40,
1362 0x88c4, 0x001f3ae3, 0x00000082,
1363 0x88d4, 0x0000001f, 0x00000010,
1364 0x30934, 0xffffffff, 0x00000000
1365 };
1366
1367 static const u32 kalindi_mgcg_cgcg_init[] =
1368 {
1369 0xc420, 0xffffffff, 0xfffffffc,
1370 0x30800, 0xffffffff, 0xe0000000,
1371 0x3c2a0, 0xffffffff, 0x00000100,
1372 0x3c208, 0xffffffff, 0x00000100,
1373 0x3c2c0, 0xffffffff, 0x00000100,
1374 0x3c2c8, 0xffffffff, 0x00000100,
1375 0x3c2c4, 0xffffffff, 0x00000100,
1376 0x55e4, 0xffffffff, 0x00600100,
1377 0x3c280, 0xffffffff, 0x00000100,
1378 0x3c214, 0xffffffff, 0x06000100,
1379 0x3c220, 0xffffffff, 0x00000100,
1380 0x3c218, 0xffffffff, 0x06000100,
1381 0x3c204, 0xffffffff, 0x00000100,
1382 0x3c2e0, 0xffffffff, 0x00000100,
1383 0x3c224, 0xffffffff, 0x00000100,
1384 0x3c200, 0xffffffff, 0x00000100,
1385 0x3c230, 0xffffffff, 0x00000100,
1386 0x3c234, 0xffffffff, 0x00000100,
1387 0x3c250, 0xffffffff, 0x00000100,
1388 0x3c254, 0xffffffff, 0x00000100,
1389 0x3c258, 0xffffffff, 0x00000100,
1390 0x3c25c, 0xffffffff, 0x00000100,
1391 0x3c260, 0xffffffff, 0x00000100,
1392 0x3c27c, 0xffffffff, 0x00000100,
1393 0x3c278, 0xffffffff, 0x00000100,
1394 0x3c210, 0xffffffff, 0x06000100,
1395 0x3c290, 0xffffffff, 0x00000100,
1396 0x3c274, 0xffffffff, 0x00000100,
1397 0x3c2b4, 0xffffffff, 0x00000100,
1398 0x3c2b0, 0xffffffff, 0x00000100,
1399 0x3c270, 0xffffffff, 0x00000100,
1400 0x30800, 0xffffffff, 0xe0000000,
1401 0x3c020, 0xffffffff, 0x00010000,
1402 0x3c024, 0xffffffff, 0x00030002,
1403 0x3c028, 0xffffffff, 0x00040007,
1404 0x3c02c, 0xffffffff, 0x00060005,
1405 0x3c030, 0xffffffff, 0x00090008,
1406 0x3c034, 0xffffffff, 0x00010000,
1407 0x3c038, 0xffffffff, 0x00030002,
1408 0x3c03c, 0xffffffff, 0x00040007,
1409 0x3c040, 0xffffffff, 0x00060005,
1410 0x3c044, 0xffffffff, 0x00090008,
1411 0x3c000, 0xffffffff, 0x96e00200,
1412 0x8708, 0xffffffff, 0x00900100,
1413 0xc424, 0xffffffff, 0x0020003f,
1414 0x38, 0xffffffff, 0x0140001c,
1415 0x3c, 0x000f0000, 0x000f0000,
1416 0x220, 0xffffffff, 0xC060000C,
1417 0x224, 0xc0000fff, 0x00000100,
1418 0x20a8, 0xffffffff, 0x00000104,
1419 0x55e4, 0xff000fff, 0x00000100,
1420 0x30cc, 0xc0000fff, 0x00000104,
1421 0xc1e4, 0x00000001, 0x00000001,
1422 0xd00c, 0xff000ff0, 0x00000100,
1423 0xd80c, 0xff000ff0, 0x00000100
1424 };
1425
1426 static const u32 hawaii_golden_spm_registers[] =
1427 {
1428 0x30800, 0xe0ffffff, 0xe0000000
1429 };
1430
1431 static const u32 hawaii_golden_common_registers[] =
1432 {
1433 0x30800, 0xffffffff, 0xe0000000,
1434 0x28350, 0xffffffff, 0x3a00161a,
1435 0x28354, 0xffffffff, 0x0000002e,
1436 0x9a10, 0xffffffff, 0x00018208,
1437 0x98f8, 0xffffffff, 0x12011003
1438 };
1439
1440 static const u32 hawaii_golden_registers[] =
1441 {
1442 0x3354, 0x00000333, 0x00000333,
1443 0x9a10, 0x00010000, 0x00058208,
1444 0x9830, 0xffffffff, 0x00000000,
1445 0x9834, 0xf00fffff, 0x00000400,
1446 0x9838, 0x0002021c, 0x00020200,
1447 0xc78, 0x00000080, 0x00000000,
1448 0x5bb0, 0x000000f0, 0x00000070,
1449 0x5bc0, 0xf0311fff, 0x80300000,
1450 0x350c, 0x00810000, 0x408af000,
1451 0x7030, 0x31000111, 0x00000011,
1452 0x2f48, 0x73773777, 0x12010001,
1453 0x2120, 0x0000007f, 0x0000001b,
1454 0x21dc, 0x00007fb6, 0x00002191,
1455 0x3628, 0x0000003f, 0x0000000a,
1456 0x362c, 0x0000003f, 0x0000000a,
1457 0x2ae4, 0x00073ffe, 0x000022a2,
1458 0x240c, 0x000007ff, 0x00000000,
1459 0x8bf0, 0x00002001, 0x00000001,
1460 0x8b24, 0xffffffff, 0x00ffffff,
1461 0x30a04, 0x0000ff0f, 0x00000000,
1462 0x28a4c, 0x07ffffff, 0x06000000,
1463 0x3e78, 0x00000001, 0x00000002,
1464 0xc768, 0x00000008, 0x00000008,
1465 0xc770, 0x00000f00, 0x00000800,
1466 0xc774, 0x00000f00, 0x00000800,
1467 0xc798, 0x00ffffff, 0x00ff7fbf,
1468 0xc79c, 0x00ffffff, 0x00ff7faf,
1469 0x8c00, 0x000000ff, 0x00000800,
1470 0xe40, 0x00001fff, 0x00001fff,
1471 0x9060, 0x0000007f, 0x00000020,
1472 0x9508, 0x00010000, 0x00010000,
1473 0xae00, 0x00100000, 0x000ff07c,
1474 0xac14, 0x000003ff, 0x0000000f,
1475 0xac10, 0xffffffff, 0x7564fdec,
1476 0xac0c, 0xffffffff, 0x3120b9a8,
1477 0xac08, 0x20000000, 0x0f9c0000
1478 };
1479
1480 static const u32 hawaii_mgcg_cgcg_init[] =
1481 {
1482 0xc420, 0xffffffff, 0xfffffffd,
1483 0x30800, 0xffffffff, 0xe0000000,
1484 0x3c2a0, 0xffffffff, 0x00000100,
1485 0x3c208, 0xffffffff, 0x00000100,
1486 0x3c2c0, 0xffffffff, 0x00000100,
1487 0x3c2c8, 0xffffffff, 0x00000100,
1488 0x3c2c4, 0xffffffff, 0x00000100,
1489 0x55e4, 0xffffffff, 0x00200100,
1490 0x3c280, 0xffffffff, 0x00000100,
1491 0x3c214, 0xffffffff, 0x06000100,
1492 0x3c220, 0xffffffff, 0x00000100,
1493 0x3c218, 0xffffffff, 0x06000100,
1494 0x3c204, 0xffffffff, 0x00000100,
1495 0x3c2e0, 0xffffffff, 0x00000100,
1496 0x3c224, 0xffffffff, 0x00000100,
1497 0x3c200, 0xffffffff, 0x00000100,
1498 0x3c230, 0xffffffff, 0x00000100,
1499 0x3c234, 0xffffffff, 0x00000100,
1500 0x3c250, 0xffffffff, 0x00000100,
1501 0x3c254, 0xffffffff, 0x00000100,
1502 0x3c258, 0xffffffff, 0x00000100,
1503 0x3c25c, 0xffffffff, 0x00000100,
1504 0x3c260, 0xffffffff, 0x00000100,
1505 0x3c27c, 0xffffffff, 0x00000100,
1506 0x3c278, 0xffffffff, 0x00000100,
1507 0x3c210, 0xffffffff, 0x06000100,
1508 0x3c290, 0xffffffff, 0x00000100,
1509 0x3c274, 0xffffffff, 0x00000100,
1510 0x3c2b4, 0xffffffff, 0x00000100,
1511 0x3c2b0, 0xffffffff, 0x00000100,
1512 0x3c270, 0xffffffff, 0x00000100,
1513 0x30800, 0xffffffff, 0xe0000000,
1514 0x3c020, 0xffffffff, 0x00010000,
1515 0x3c024, 0xffffffff, 0x00030002,
1516 0x3c028, 0xffffffff, 0x00040007,
1517 0x3c02c, 0xffffffff, 0x00060005,
1518 0x3c030, 0xffffffff, 0x00090008,
1519 0x3c034, 0xffffffff, 0x00010000,
1520 0x3c038, 0xffffffff, 0x00030002,
1521 0x3c03c, 0xffffffff, 0x00040007,
1522 0x3c040, 0xffffffff, 0x00060005,
1523 0x3c044, 0xffffffff, 0x00090008,
1524 0x3c048, 0xffffffff, 0x00010000,
1525 0x3c04c, 0xffffffff, 0x00030002,
1526 0x3c050, 0xffffffff, 0x00040007,
1527 0x3c054, 0xffffffff, 0x00060005,
1528 0x3c058, 0xffffffff, 0x00090008,
1529 0x3c05c, 0xffffffff, 0x00010000,
1530 0x3c060, 0xffffffff, 0x00030002,
1531 0x3c064, 0xffffffff, 0x00040007,
1532 0x3c068, 0xffffffff, 0x00060005,
1533 0x3c06c, 0xffffffff, 0x00090008,
1534 0x3c070, 0xffffffff, 0x00010000,
1535 0x3c074, 0xffffffff, 0x00030002,
1536 0x3c078, 0xffffffff, 0x00040007,
1537 0x3c07c, 0xffffffff, 0x00060005,
1538 0x3c080, 0xffffffff, 0x00090008,
1539 0x3c084, 0xffffffff, 0x00010000,
1540 0x3c088, 0xffffffff, 0x00030002,
1541 0x3c08c, 0xffffffff, 0x00040007,
1542 0x3c090, 0xffffffff, 0x00060005,
1543 0x3c094, 0xffffffff, 0x00090008,
1544 0x3c098, 0xffffffff, 0x00010000,
1545 0x3c09c, 0xffffffff, 0x00030002,
1546 0x3c0a0, 0xffffffff, 0x00040007,
1547 0x3c0a4, 0xffffffff, 0x00060005,
1548 0x3c0a8, 0xffffffff, 0x00090008,
1549 0x3c0ac, 0xffffffff, 0x00010000,
1550 0x3c0b0, 0xffffffff, 0x00030002,
1551 0x3c0b4, 0xffffffff, 0x00040007,
1552 0x3c0b8, 0xffffffff, 0x00060005,
1553 0x3c0bc, 0xffffffff, 0x00090008,
1554 0x3c0c0, 0xffffffff, 0x00010000,
1555 0x3c0c4, 0xffffffff, 0x00030002,
1556 0x3c0c8, 0xffffffff, 0x00040007,
1557 0x3c0cc, 0xffffffff, 0x00060005,
1558 0x3c0d0, 0xffffffff, 0x00090008,
1559 0x3c0d4, 0xffffffff, 0x00010000,
1560 0x3c0d8, 0xffffffff, 0x00030002,
1561 0x3c0dc, 0xffffffff, 0x00040007,
1562 0x3c0e0, 0xffffffff, 0x00060005,
1563 0x3c0e4, 0xffffffff, 0x00090008,
1564 0x3c0e8, 0xffffffff, 0x00010000,
1565 0x3c0ec, 0xffffffff, 0x00030002,
1566 0x3c0f0, 0xffffffff, 0x00040007,
1567 0x3c0f4, 0xffffffff, 0x00060005,
1568 0x3c0f8, 0xffffffff, 0x00090008,
1569 0xc318, 0xffffffff, 0x00020200,
1570 0x3350, 0xffffffff, 0x00000200,
1571 0x15c0, 0xffffffff, 0x00000400,
1572 0x55e8, 0xffffffff, 0x00000000,
1573 0x2f50, 0xffffffff, 0x00000902,
1574 0x3c000, 0xffffffff, 0x96940200,
1575 0x8708, 0xffffffff, 0x00900100,
1576 0xc424, 0xffffffff, 0x0020003f,
1577 0x38, 0xffffffff, 0x0140001c,
1578 0x3c, 0x000f0000, 0x000f0000,
1579 0x220, 0xffffffff, 0xc060000c,
1580 0x224, 0xc0000fff, 0x00000100,
1581 0xf90, 0xffffffff, 0x00000100,
1582 0xf98, 0x00000101, 0x00000000,
1583 0x20a8, 0xffffffff, 0x00000104,
1584 0x55e4, 0xff000fff, 0x00000100,
1585 0x30cc, 0xc0000fff, 0x00000104,
1586 0xc1e4, 0x00000001, 0x00000001,
1587 0xd00c, 0xff000ff0, 0x00000100,
1588 0xd80c, 0xff000ff0, 0x00000100
1589 };
1590
1591 static const u32 godavari_golden_registers[] =
1592 {
1593 0x55e4, 0xff607fff, 0xfc000100,
1594 0x6ed8, 0x00010101, 0x00010000,
1595 0x9830, 0xffffffff, 0x00000000,
1596 0x98302, 0xf00fffff, 0x00000400,
1597 0x6130, 0xffffffff, 0x00010000,
1598 0x5bb0, 0x000000f0, 0x00000070,
1599 0x5bc0, 0xf0311fff, 0x80300000,
1600 0x98f8, 0x73773777, 0x12010001,
1601 0x98fc, 0xffffffff, 0x00000010,
1602 0x8030, 0x00001f0f, 0x0000100a,
1603 0x2f48, 0x73773777, 0x12010001,
1604 0x2408, 0x000fffff, 0x000c007f,
1605 0x8a14, 0xf000003f, 0x00000007,
1606 0x8b24, 0xffffffff, 0x00ff0fff,
1607 0x30a04, 0x0000ff0f, 0x00000000,
1608 0x28a4c, 0x07ffffff, 0x06000000,
1609 0x4d8, 0x00000fff, 0x00000100,
1610 0xd014, 0x00010000, 0x00810001,
1611 0xd814, 0x00010000, 0x00810001,
1612 0x3e78, 0x00000001, 0x00000002,
1613 0xc768, 0x00000008, 0x00000008,
1614 0xc770, 0x00000f00, 0x00000800,
1615 0xc774, 0x00000f00, 0x00000800,
1616 0xc798, 0x00ffffff, 0x00ff7fbf,
1617 0xc79c, 0x00ffffff, 0x00ff7faf,
1618 0x8c00, 0x000000ff, 0x00000001,
1619 0x214f8, 0x01ff01ff, 0x00000002,
1620 0x21498, 0x007ff800, 0x00200000,
1621 0x2015c, 0xffffffff, 0x00000f40,
1622 0x88c4, 0x001f3ae3, 0x00000082,
1623 0x88d4, 0x0000001f, 0x00000010,
1624 0x30934, 0xffffffff, 0x00000000
1625 };
1626
1627
1628 static void cik_init_golden_registers(struct radeon_device *rdev)
1629 {
1630 switch (rdev->family) {
1631 case CHIP_BONAIRE:
1632 radeon_program_register_sequence(rdev,
1633 bonaire_mgcg_cgcg_init,
1634 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1635 radeon_program_register_sequence(rdev,
1636 bonaire_golden_registers,
1637 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1638 radeon_program_register_sequence(rdev,
1639 bonaire_golden_common_registers,
1640 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1641 radeon_program_register_sequence(rdev,
1642 bonaire_golden_spm_registers,
1643 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1644 break;
1645 case CHIP_KABINI:
1646 radeon_program_register_sequence(rdev,
1647 kalindi_mgcg_cgcg_init,
1648 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1649 radeon_program_register_sequence(rdev,
1650 kalindi_golden_registers,
1651 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1652 radeon_program_register_sequence(rdev,
1653 kalindi_golden_common_registers,
1654 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1655 radeon_program_register_sequence(rdev,
1656 kalindi_golden_spm_registers,
1657 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1658 break;
1659 case CHIP_MULLINS:
1660 radeon_program_register_sequence(rdev,
1661 kalindi_mgcg_cgcg_init,
1662 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1663 radeon_program_register_sequence(rdev,
1664 godavari_golden_registers,
1665 (const u32)ARRAY_SIZE(godavari_golden_registers));
1666 radeon_program_register_sequence(rdev,
1667 kalindi_golden_common_registers,
1668 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1669 radeon_program_register_sequence(rdev,
1670 kalindi_golden_spm_registers,
1671 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1672 break;
1673 case CHIP_KAVERI:
1674 radeon_program_register_sequence(rdev,
1675 spectre_mgcg_cgcg_init,
1676 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1677 radeon_program_register_sequence(rdev,
1678 spectre_golden_registers,
1679 (const u32)ARRAY_SIZE(spectre_golden_registers));
1680 radeon_program_register_sequence(rdev,
1681 spectre_golden_common_registers,
1682 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1683 radeon_program_register_sequence(rdev,
1684 spectre_golden_spm_registers,
1685 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1686 break;
1687 case CHIP_HAWAII:
1688 radeon_program_register_sequence(rdev,
1689 hawaii_mgcg_cgcg_init,
1690 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1691 radeon_program_register_sequence(rdev,
1692 hawaii_golden_registers,
1693 (const u32)ARRAY_SIZE(hawaii_golden_registers));
1694 radeon_program_register_sequence(rdev,
1695 hawaii_golden_common_registers,
1696 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1697 radeon_program_register_sequence(rdev,
1698 hawaii_golden_spm_registers,
1699 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1700 break;
1701 default:
1702 break;
1703 }
1704 }
1705
1706 /**
1707 * cik_get_xclk - get the xclk
1708 *
1709 * @rdev: radeon_device pointer
1710 *
1711 * Returns the reference clock used by the gfx engine
1712 * (CIK).
1713 */
1714 u32 cik_get_xclk(struct radeon_device *rdev)
1715 {
1716 u32 reference_clock = rdev->clock.spll.reference_freq;
1717
1718 if (rdev->flags & RADEON_IS_IGP) {
1719 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1720 return reference_clock / 2;
1721 } else {
1722 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1723 return reference_clock / 4;
1724 }
1725 return reference_clock;
1726 }
1727
1728 /**
1729 * cik_mm_rdoorbell - read a doorbell dword
1730 *
1731 * @rdev: radeon_device pointer
1732 * @index: doorbell index
1733 *
1734 * Returns the value in the doorbell aperture at the
1735 * requested doorbell index (CIK).
1736 */
1737 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
1738 {
1739 if (index < rdev->doorbell.num_doorbells) {
1740 return readl(rdev->doorbell.ptr + index);
1741 } else {
1742 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
1743 return 0;
1744 }
1745 }
1746
1747 /**
1748 * cik_mm_wdoorbell - write a doorbell dword
1749 *
1750 * @rdev: radeon_device pointer
1751 * @index: doorbell index
1752 * @v: value to write
1753 *
1754 * Writes @v to the doorbell aperture at the
1755 * requested doorbell index (CIK).
1756 */
1757 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
1758 {
1759 if (index < rdev->doorbell.num_doorbells) {
1760 writel(v, rdev->doorbell.ptr + index);
1761 } else {
1762 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
1763 }
1764 }
1765
1766 #define BONAIRE_IO_MC_REGS_SIZE 36
1767
1768 static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1769 {
1770 {0x00000070, 0x04400000},
1771 {0x00000071, 0x80c01803},
1772 {0x00000072, 0x00004004},
1773 {0x00000073, 0x00000100},
1774 {0x00000074, 0x00ff0000},
1775 {0x00000075, 0x34000000},
1776 {0x00000076, 0x08000014},
1777 {0x00000077, 0x00cc08ec},
1778 {0x00000078, 0x00000400},
1779 {0x00000079, 0x00000000},
1780 {0x0000007a, 0x04090000},
1781 {0x0000007c, 0x00000000},
1782 {0x0000007e, 0x4408a8e8},
1783 {0x0000007f, 0x00000304},
1784 {0x00000080, 0x00000000},
1785 {0x00000082, 0x00000001},
1786 {0x00000083, 0x00000002},
1787 {0x00000084, 0xf3e4f400},
1788 {0x00000085, 0x052024e3},
1789 {0x00000087, 0x00000000},
1790 {0x00000088, 0x01000000},
1791 {0x0000008a, 0x1c0a0000},
1792 {0x0000008b, 0xff010000},
1793 {0x0000008d, 0xffffefff},
1794 {0x0000008e, 0xfff3efff},
1795 {0x0000008f, 0xfff3efbf},
1796 {0x00000092, 0xf7ffffff},
1797 {0x00000093, 0xffffff7f},
1798 {0x00000095, 0x00101101},
1799 {0x00000096, 0x00000fff},
1800 {0x00000097, 0x00116fff},
1801 {0x00000098, 0x60010000},
1802 {0x00000099, 0x10010000},
1803 {0x0000009a, 0x00006000},
1804 {0x0000009b, 0x00001000},
1805 {0x0000009f, 0x00b48000}
1806 };
1807
1808 #define HAWAII_IO_MC_REGS_SIZE 22
1809
1810 static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1811 {
1812 {0x0000007d, 0x40000000},
1813 {0x0000007e, 0x40180304},
1814 {0x0000007f, 0x0000ff00},
1815 {0x00000081, 0x00000000},
1816 {0x00000083, 0x00000800},
1817 {0x00000086, 0x00000000},
1818 {0x00000087, 0x00000100},
1819 {0x00000088, 0x00020100},
1820 {0x00000089, 0x00000000},
1821 {0x0000008b, 0x00040000},
1822 {0x0000008c, 0x00000100},
1823 {0x0000008e, 0xff010000},
1824 {0x00000090, 0xffffefff},
1825 {0x00000091, 0xfff3efff},
1826 {0x00000092, 0xfff3efbf},
1827 {0x00000093, 0xf7ffffff},
1828 {0x00000094, 0xffffff7f},
1829 {0x00000095, 0x00000fff},
1830 {0x00000096, 0x00116fff},
1831 {0x00000097, 0x60010000},
1832 {0x00000098, 0x10010000},
1833 {0x0000009f, 0x00c79000}
1834 };
1835
1836
1837 /**
1838 * cik_srbm_select - select specific register instances
1839 *
1840 * @rdev: radeon_device pointer
1841 * @me: selected ME (micro engine)
1842 * @pipe: pipe
1843 * @queue: queue
1844 * @vmid: VMID
1845 *
1846 * Switches the currently active registers instances. Some
1847 * registers are instanced per VMID, others are instanced per
1848 * me/pipe/queue combination.
1849 */
1850 static void cik_srbm_select(struct radeon_device *rdev,
1851 u32 me, u32 pipe, u32 queue, u32 vmid)
1852 {
1853 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1854 MEID(me & 0x3) |
1855 VMID(vmid & 0xf) |
1856 QUEUEID(queue & 0x7));
1857 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1858 }
1859
1860 /* ucode loading */
1861 /**
1862 * ci_mc_load_microcode - load MC ucode into the hw
1863 *
1864 * @rdev: radeon_device pointer
1865 *
1866 * Load the GDDR MC ucode into the hw (CIK).
1867 * Returns 0 on success, error on failure.
1868 */
1869 int ci_mc_load_microcode(struct radeon_device *rdev)
1870 {
1871 const __be32 *fw_data = NULL;
1872 const __le32 *new_fw_data = NULL;
1873 u32 running, tmp;
1874 u32 *io_mc_regs = NULL;
1875 const __le32 *new_io_mc_regs = NULL;
1876 int i, regs_size, ucode_size;
1877
1878 if (!rdev->mc_fw)
1879 return -EINVAL;
1880
1881 if (rdev->new_fw) {
1882 const struct mc_firmware_header_v1_0 *hdr =
1883 (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
1884
1885 radeon_ucode_print_mc_hdr(&hdr->header);
1886
1887 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1888 new_io_mc_regs = (const __le32 *)
1889 (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1890 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1891 new_fw_data = (const __le32 *)
1892 (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1893 } else {
1894 ucode_size = rdev->mc_fw->size / 4;
1895
1896 switch (rdev->family) {
1897 case CHIP_BONAIRE:
1898 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1899 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1900 break;
1901 case CHIP_HAWAII:
1902 io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1903 regs_size = HAWAII_IO_MC_REGS_SIZE;
1904 break;
1905 default:
1906 return -EINVAL;
1907 }
1908 fw_data = (const __be32 *)rdev->mc_fw->data;
1909 }
1910
1911 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1912
1913 if (running == 0) {
1914 /* reset the engine and set to writable */
1915 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1916 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1917
1918 /* load mc io regs */
1919 for (i = 0; i < regs_size; i++) {
1920 if (rdev->new_fw) {
1921 WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1922 WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1923 } else {
1924 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1925 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1926 }
1927 }
1928
1929 tmp = RREG32(MC_SEQ_MISC0);
1930 if ((rdev->pdev->device == 0x6649) && ((tmp & 0xff00) == 0x5600)) {
1931 WREG32(MC_SEQ_IO_DEBUG_INDEX, 5);
1932 WREG32(MC_SEQ_IO_DEBUG_DATA, 0x00000023);
1933 WREG32(MC_SEQ_IO_DEBUG_INDEX, 9);
1934 WREG32(MC_SEQ_IO_DEBUG_DATA, 0x000001f0);
1935 }
1936
1937 /* load the MC ucode */
1938 for (i = 0; i < ucode_size; i++) {
1939 if (rdev->new_fw)
1940 WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1941 else
1942 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1943 }
1944
1945 /* put the engine back into the active state */
1946 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1947 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1948 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1949
1950 /* wait for training to complete */
1951 for (i = 0; i < rdev->usec_timeout; i++) {
1952 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1953 break;
1954 udelay(1);
1955 }
1956 for (i = 0; i < rdev->usec_timeout; i++) {
1957 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1958 break;
1959 udelay(1);
1960 }
1961 }
1962
1963 return 0;
1964 }
1965
1966 /**
1967 * cik_init_microcode - load ucode images from disk
1968 *
1969 * @rdev: radeon_device pointer
1970 *
1971 * Use the firmware interface to load the ucode images into
1972 * the driver (not loaded into hw).
1973 * Returns 0 on success, error on failure.
1974 */
1975 static int cik_init_microcode(struct radeon_device *rdev)
1976 {
1977 const char *chip_name;
1978 const char *new_chip_name;
1979 size_t pfp_req_size, me_req_size, ce_req_size,
1980 mec_req_size, rlc_req_size, mc_req_size = 0,
1981 sdma_req_size, smc_req_size = 0, mc2_req_size = 0;
1982 char fw_name[30];
1983 int new_fw = 0;
1984 int err;
1985 int num_fw;
1986 bool new_smc = false;
1987
1988 DRM_DEBUG("\n");
1989
1990 switch (rdev->family) {
1991 case CHIP_BONAIRE:
1992 chip_name = "BONAIRE";
1993 if ((rdev->pdev->revision == 0x80) ||
1994 (rdev->pdev->revision == 0x81) ||
1995 (rdev->pdev->device == 0x665f))
1996 new_smc = true;
1997 new_chip_name = "bonaire";
1998 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1999 me_req_size = CIK_ME_UCODE_SIZE * 4;
2000 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2001 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2002 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
2003 mc_req_size = BONAIRE_MC_UCODE_SIZE * 4;
2004 mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4;
2005 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2006 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
2007 num_fw = 8;
2008 break;
2009 case CHIP_HAWAII:
2010 chip_name = "HAWAII";
2011 if (rdev->pdev->revision == 0x80)
2012 new_smc = true;
2013 new_chip_name = "hawaii";
2014 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2015 me_req_size = CIK_ME_UCODE_SIZE * 4;
2016 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2017 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2018 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
2019 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
2020 mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4;
2021 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2022 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
2023 num_fw = 8;
2024 break;
2025 case CHIP_KAVERI:
2026 chip_name = "KAVERI";
2027 new_chip_name = "kaveri";
2028 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2029 me_req_size = CIK_ME_UCODE_SIZE * 4;
2030 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2031 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2032 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
2033 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2034 num_fw = 7;
2035 break;
2036 case CHIP_KABINI:
2037 chip_name = "KABINI";
2038 new_chip_name = "kabini";
2039 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2040 me_req_size = CIK_ME_UCODE_SIZE * 4;
2041 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2042 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2043 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
2044 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2045 num_fw = 6;
2046 break;
2047 case CHIP_MULLINS:
2048 chip_name = "MULLINS";
2049 new_chip_name = "mullins";
2050 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2051 me_req_size = CIK_ME_UCODE_SIZE * 4;
2052 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2053 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2054 rlc_req_size = ML_RLC_UCODE_SIZE * 4;
2055 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2056 num_fw = 6;
2057 break;
2058 default: BUG();
2059 }
2060
2061 DRM_INFO("Loading %s Microcode\n", new_chip_name);
2062
2063 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name);
2064 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
2065 if (err) {
2066 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
2067 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
2068 if (err)
2069 goto out;
2070 if (rdev->pfp_fw->size != pfp_req_size) {
2071 pr_err("cik_cp: Bogus length %zu in firmware \"%s\"\n",
2072 rdev->pfp_fw->size, fw_name);
2073 err = -EINVAL;
2074 goto out;
2075 }
2076 } else {
2077 err = radeon_ucode_validate(rdev->pfp_fw);
2078 if (err) {
2079 pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2080 fw_name);
2081 goto out;
2082 } else {
2083 new_fw++;
2084 }
2085 }
2086
2087 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name);
2088 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2089 if (err) {
2090 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
2091 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2092 if (err)
2093 goto out;
2094 if (rdev->me_fw->size != me_req_size) {
2095 pr_err("cik_cp: Bogus length %zu in firmware \"%s\"\n",
2096 rdev->me_fw->size, fw_name);
2097 err = -EINVAL;
2098 }
2099 } else {
2100 err = radeon_ucode_validate(rdev->me_fw);
2101 if (err) {
2102 pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2103 fw_name);
2104 goto out;
2105 } else {
2106 new_fw++;
2107 }
2108 }
2109
2110 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name);
2111 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2112 if (err) {
2113 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
2114 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2115 if (err)
2116 goto out;
2117 if (rdev->ce_fw->size != ce_req_size) {
2118 pr_err("cik_cp: Bogus length %zu in firmware \"%s\"\n",
2119 rdev->ce_fw->size, fw_name);
2120 err = -EINVAL;
2121 }
2122 } else {
2123 err = radeon_ucode_validate(rdev->ce_fw);
2124 if (err) {
2125 pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2126 fw_name);
2127 goto out;
2128 } else {
2129 new_fw++;
2130 }
2131 }
2132
2133 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", new_chip_name);
2134 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2135 if (err) {
2136 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
2137 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2138 if (err)
2139 goto out;
2140 if (rdev->mec_fw->size != mec_req_size) {
2141 pr_err("cik_cp: Bogus length %zu in firmware \"%s\"\n",
2142 rdev->mec_fw->size, fw_name);
2143 err = -EINVAL;
2144 }
2145 } else {
2146 err = radeon_ucode_validate(rdev->mec_fw);
2147 if (err) {
2148 pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2149 fw_name);
2150 goto out;
2151 } else {
2152 new_fw++;
2153 }
2154 }
2155
2156 if (rdev->family == CHIP_KAVERI) {
2157 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec2.bin", new_chip_name);
2158 err = request_firmware(&rdev->mec2_fw, fw_name, rdev->dev);
2159 if (err) {
2160 goto out;
2161 } else {
2162 err = radeon_ucode_validate(rdev->mec2_fw);
2163 if (err) {
2164 goto out;
2165 } else {
2166 new_fw++;
2167 }
2168 }
2169 }
2170
2171 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name);
2172 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2173 if (err) {
2174 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
2175 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2176 if (err)
2177 goto out;
2178 if (rdev->rlc_fw->size != rlc_req_size) {
2179 pr_err("cik_rlc: Bogus length %zu in firmware \"%s\"\n",
2180 rdev->rlc_fw->size, fw_name);
2181 err = -EINVAL;
2182 }
2183 } else {
2184 err = radeon_ucode_validate(rdev->rlc_fw);
2185 if (err) {
2186 pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2187 fw_name);
2188 goto out;
2189 } else {
2190 new_fw++;
2191 }
2192 }
2193
2194 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", new_chip_name);
2195 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2196 if (err) {
2197 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
2198 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2199 if (err)
2200 goto out;
2201 if (rdev->sdma_fw->size != sdma_req_size) {
2202 pr_err("cik_sdma: Bogus length %zu in firmware \"%s\"\n",
2203 rdev->sdma_fw->size, fw_name);
2204 err = -EINVAL;
2205 }
2206 } else {
2207 err = radeon_ucode_validate(rdev->sdma_fw);
2208 if (err) {
2209 pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2210 fw_name);
2211 goto out;
2212 } else {
2213 new_fw++;
2214 }
2215 }
2216
2217 /* No SMC, MC ucode on APUs */
2218 if (!(rdev->flags & RADEON_IS_IGP)) {
2219 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name);
2220 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2221 if (err) {
2222 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
2223 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2224 if (err) {
2225 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
2226 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2227 if (err)
2228 goto out;
2229 }
2230 if ((rdev->mc_fw->size != mc_req_size) &&
2231 (rdev->mc_fw->size != mc2_req_size)){
2232 pr_err("cik_mc: Bogus length %zu in firmware \"%s\"\n",
2233 rdev->mc_fw->size, fw_name);
2234 err = -EINVAL;
2235 }
2236 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
2237 } else {
2238 err = radeon_ucode_validate(rdev->mc_fw);
2239 if (err) {
2240 pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2241 fw_name);
2242 goto out;
2243 } else {
2244 new_fw++;
2245 }
2246 }
2247
2248 if (new_smc)
2249 snprintf(fw_name, sizeof(fw_name), "radeon/%s_k_smc.bin", new_chip_name);
2250 else
2251 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
2252 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2253 if (err) {
2254 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
2255 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2256 if (err) {
2257 pr_err("smc: error loading firmware \"%s\"\n",
2258 fw_name);
2259 release_firmware(rdev->smc_fw);
2260 rdev->smc_fw = NULL;
2261 err = 0;
2262 } else if (rdev->smc_fw->size != smc_req_size) {
2263 pr_err("cik_smc: Bogus length %zu in firmware \"%s\"\n",
2264 rdev->smc_fw->size, fw_name);
2265 err = -EINVAL;
2266 }
2267 } else {
2268 err = radeon_ucode_validate(rdev->smc_fw);
2269 if (err) {
2270 pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2271 fw_name);
2272 goto out;
2273 } else {
2274 new_fw++;
2275 }
2276 }
2277 }
2278
2279 if (new_fw == 0) {
2280 rdev->new_fw = false;
2281 } else if (new_fw < num_fw) {
2282 pr_err("ci_fw: mixing new and old firmware!\n");
2283 err = -EINVAL;
2284 } else {
2285 rdev->new_fw = true;
2286 }
2287
2288 out:
2289 if (err) {
2290 if (err != -EINVAL)
2291 pr_err("cik_cp: Failed to load firmware \"%s\"\n",
2292 fw_name);
2293 release_firmware(rdev->pfp_fw);
2294 rdev->pfp_fw = NULL;
2295 release_firmware(rdev->me_fw);
2296 rdev->me_fw = NULL;
2297 release_firmware(rdev->ce_fw);
2298 rdev->ce_fw = NULL;
2299 release_firmware(rdev->mec_fw);
2300 rdev->mec_fw = NULL;
2301 release_firmware(rdev->mec2_fw);
2302 rdev->mec2_fw = NULL;
2303 release_firmware(rdev->rlc_fw);
2304 rdev->rlc_fw = NULL;
2305 release_firmware(rdev->sdma_fw);
2306 rdev->sdma_fw = NULL;
2307 release_firmware(rdev->mc_fw);
2308 rdev->mc_fw = NULL;
2309 release_firmware(rdev->smc_fw);
2310 rdev->smc_fw = NULL;
2311 }
2312 return err;
2313 }
2314
2315 /*
2316 * Core functions
2317 */
2318 /**
2319 * cik_tiling_mode_table_init - init the hw tiling table
2320 *
2321 * @rdev: radeon_device pointer
2322 *
2323 * Starting with SI, the tiling setup is done globally in a
2324 * set of 32 tiling modes. Rather than selecting each set of
2325 * parameters per surface as on older asics, we just select
2326 * which index in the tiling table we want to use, and the
2327 * surface uses those parameters (CIK).
2328 */
2329 static void cik_tiling_mode_table_init(struct radeon_device *rdev)
2330 {
2331 u32 *tile = rdev->config.cik.tile_mode_array;
2332 u32 *macrotile = rdev->config.cik.macrotile_mode_array;
2333 const u32 num_tile_mode_states =
2334 ARRAY_SIZE(rdev->config.cik.tile_mode_array);
2335 const u32 num_secondary_tile_mode_states =
2336 ARRAY_SIZE(rdev->config.cik.macrotile_mode_array);
2337 u32 reg_offset, split_equal_to_row_size;
2338 u32 num_pipe_configs;
2339 u32 num_rbs = rdev->config.cik.max_backends_per_se *
2340 rdev->config.cik.max_shader_engines;
2341
2342 switch (rdev->config.cik.mem_row_size_in_kb) {
2343 case 1:
2344 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2345 break;
2346 case 2:
2347 default:
2348 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2349 break;
2350 case 4:
2351 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2352 break;
2353 }
2354
2355 num_pipe_configs = rdev->config.cik.max_tile_pipes;
2356 if (num_pipe_configs > 8)
2357 num_pipe_configs = 16;
2358
2359 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2360 tile[reg_offset] = 0;
2361 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2362 macrotile[reg_offset] = 0;
2363
2364 switch(num_pipe_configs) {
2365 case 16:
2366 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2367 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2368 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2369 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2370 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2371 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2372 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2373 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2374 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2375 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2376 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2377 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2378 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2379 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2380 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2381 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2382 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2383 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2384 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2385 TILE_SPLIT(split_equal_to_row_size));
2386 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2387 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2388 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2389 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2390 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2391 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2392 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2393 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2394 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2395 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2396 TILE_SPLIT(split_equal_to_row_size));
2397 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2398 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2399 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2400 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2401 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2402 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2403 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2404 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2405 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2406 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2407 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2408 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2409 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2410 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2411 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2412 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2413 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2414 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2415 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2416 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2417 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2418 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2419 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2420 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2421 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2422 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2423 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2424 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2425 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2426 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2427 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2428 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2429 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2430 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2431 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2432 tile[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2433 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2434 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2435 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2436 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2437 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2438 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2439 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2440 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2441 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2442 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2443 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2444
2445 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2446 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2447 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2448 NUM_BANKS(ADDR_SURF_16_BANK));
2449 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2450 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2451 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2452 NUM_BANKS(ADDR_SURF_16_BANK));
2453 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2454 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2455 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2456 NUM_BANKS(ADDR_SURF_16_BANK));
2457 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2458 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2459 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2460 NUM_BANKS(ADDR_SURF_16_BANK));
2461 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2462 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2463 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2464 NUM_BANKS(ADDR_SURF_8_BANK));
2465 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2466 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2467 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2468 NUM_BANKS(ADDR_SURF_4_BANK));
2469 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2470 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2471 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2472 NUM_BANKS(ADDR_SURF_2_BANK));
2473 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2474 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2475 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2476 NUM_BANKS(ADDR_SURF_16_BANK));
2477 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2478 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2479 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2480 NUM_BANKS(ADDR_SURF_16_BANK));
2481 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2482 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2483 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2484 NUM_BANKS(ADDR_SURF_16_BANK));
2485 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2486 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2487 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2488 NUM_BANKS(ADDR_SURF_8_BANK));
2489 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2490 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2491 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2492 NUM_BANKS(ADDR_SURF_4_BANK));
2493 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2494 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2495 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2496 NUM_BANKS(ADDR_SURF_2_BANK));
2497 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2498 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2499 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2500 NUM_BANKS(ADDR_SURF_2_BANK));
2501
2502 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2503 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2504 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2505 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
2506 break;
2507
2508 case 8:
2509 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2510 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2511 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2512 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2513 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2514 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2515 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2516 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2517 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2518 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2519 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2520 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2521 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2522 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2523 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2524 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2525 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2526 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2527 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2528 TILE_SPLIT(split_equal_to_row_size));
2529 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2530 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2531 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2532 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2533 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2534 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2535 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2536 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2537 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2538 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2539 TILE_SPLIT(split_equal_to_row_size));
2540 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2541 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2542 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2543 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2544 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2545 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2546 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2547 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2548 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2549 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2550 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2551 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2552 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2553 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2554 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2555 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2556 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2557 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2558 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2559 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2560 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2561 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2562 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2563 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2564 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2565 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2566 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2567 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2568 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2569 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2570 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2571 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2572 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2573 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2574 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2575 tile[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2576 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2577 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2578 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2579 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2580 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2581 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2582 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2583 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2584 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2585 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2586 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2587
2588 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2589 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2590 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2591 NUM_BANKS(ADDR_SURF_16_BANK));
2592 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2593 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2594 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2595 NUM_BANKS(ADDR_SURF_16_BANK));
2596 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2597 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2598 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2599 NUM_BANKS(ADDR_SURF_16_BANK));
2600 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2601 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2602 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2603 NUM_BANKS(ADDR_SURF_16_BANK));
2604 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2605 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2606 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2607 NUM_BANKS(ADDR_SURF_8_BANK));
2608 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2609 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2610 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2611 NUM_BANKS(ADDR_SURF_4_BANK));
2612 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2613 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2614 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2615 NUM_BANKS(ADDR_SURF_2_BANK));
2616 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2617 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2618 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2619 NUM_BANKS(ADDR_SURF_16_BANK));
2620 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2621 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2622 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2623 NUM_BANKS(ADDR_SURF_16_BANK));
2624 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2625 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2626 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2627 NUM_BANKS(ADDR_SURF_16_BANK));
2628 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2629 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2630 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2631 NUM_BANKS(ADDR_SURF_16_BANK));
2632 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2633 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2634 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2635 NUM_BANKS(ADDR_SURF_8_BANK));
2636 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2637 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2638 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2639 NUM_BANKS(ADDR_SURF_4_BANK));
2640 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2641 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2642 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2643 NUM_BANKS(ADDR_SURF_2_BANK));
2644
2645 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2646 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2647 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2648 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
2649 break;
2650
2651 case 4:
2652 if (num_rbs == 4) {
2653 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2654 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2655 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2656 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2657 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2658 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2659 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2660 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2661 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2662 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2663 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2664 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2665 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2666 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2667 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2668 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2669 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2670 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2671 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2672 TILE_SPLIT(split_equal_to_row_size));
2673 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2674 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2675 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2676 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2677 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2678 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2679 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2680 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2681 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2682 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2683 TILE_SPLIT(split_equal_to_row_size));
2684 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2685 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2686 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2687 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2688 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2689 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2690 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2691 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2692 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2693 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2694 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2695 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2696 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2697 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2698 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2699 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2700 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2701 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2702 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2703 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2704 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2705 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2706 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2707 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2708 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2709 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2710 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2711 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2712 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2713 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2714 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2715 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2716 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2717 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2718 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2719 tile[28] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2720 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2721 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2722 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2723 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2724 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2725 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2726 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2727 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2728 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2729 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2730 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2731
2732 } else if (num_rbs < 4) {
2733 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2734 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2735 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2736 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2737 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2738 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2739 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2740 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2741 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2742 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2743 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2744 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2745 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2746 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2747 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2748 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2749 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2750 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2751 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2752 TILE_SPLIT(split_equal_to_row_size));
2753 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2754 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2755 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2756 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2757 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2758 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2759 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2760 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2761 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2762 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2763 TILE_SPLIT(split_equal_to_row_size));
2764 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2765 PIPE_CONFIG(ADDR_SURF_P4_8x16));
2766 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2767 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2768 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2769 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2770 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2771 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2772 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2773 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2774 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2775 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2776 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2777 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2778 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2779 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2780 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2781 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2782 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2783 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2784 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2785 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2786 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2787 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2788 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2789 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2790 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2791 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2792 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2793 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2794 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2795 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2796 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2797 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2798 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2799 tile[28] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2800 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2801 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2802 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2803 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2804 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2805 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2806 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2807 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2808 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2809 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2810 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2811 }
2812
2813 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2814 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2815 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2816 NUM_BANKS(ADDR_SURF_16_BANK));
2817 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2818 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2819 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2820 NUM_BANKS(ADDR_SURF_16_BANK));
2821 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2822 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2823 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2824 NUM_BANKS(ADDR_SURF_16_BANK));
2825 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2826 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2827 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2828 NUM_BANKS(ADDR_SURF_16_BANK));
2829 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2830 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2831 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2832 NUM_BANKS(ADDR_SURF_16_BANK));
2833 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2834 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2835 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2836 NUM_BANKS(ADDR_SURF_8_BANK));
2837 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2838 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2839 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2840 NUM_BANKS(ADDR_SURF_4_BANK));
2841 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2842 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2843 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2844 NUM_BANKS(ADDR_SURF_16_BANK));
2845 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2846 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2847 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2848 NUM_BANKS(ADDR_SURF_16_BANK));
2849 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2850 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2851 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2852 NUM_BANKS(ADDR_SURF_16_BANK));
2853 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2854 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2855 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2856 NUM_BANKS(ADDR_SURF_16_BANK));
2857 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2858 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2859 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2860 NUM_BANKS(ADDR_SURF_16_BANK));
2861 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2862 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2863 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2864 NUM_BANKS(ADDR_SURF_8_BANK));
2865 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2866 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2867 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2868 NUM_BANKS(ADDR_SURF_4_BANK));
2869
2870 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2871 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2872 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2873 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
2874 break;
2875
2876 case 2:
2877 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2878 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2879 PIPE_CONFIG(ADDR_SURF_P2) |
2880 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2881 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2882 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2883 PIPE_CONFIG(ADDR_SURF_P2) |
2884 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2885 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2886 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2887 PIPE_CONFIG(ADDR_SURF_P2) |
2888 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2889 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2890 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2891 PIPE_CONFIG(ADDR_SURF_P2) |
2892 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2893 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2894 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2895 PIPE_CONFIG(ADDR_SURF_P2) |
2896 TILE_SPLIT(split_equal_to_row_size));
2897 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2898 PIPE_CONFIG(ADDR_SURF_P2) |
2899 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2900 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2901 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2902 PIPE_CONFIG(ADDR_SURF_P2) |
2903 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2904 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2905 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2906 PIPE_CONFIG(ADDR_SURF_P2) |
2907 TILE_SPLIT(split_equal_to_row_size));
2908 tile[8] = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2909 PIPE_CONFIG(ADDR_SURF_P2);
2910 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2911 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2912 PIPE_CONFIG(ADDR_SURF_P2));
2913 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2914 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2915 PIPE_CONFIG(ADDR_SURF_P2) |
2916 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2917 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2918 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2919 PIPE_CONFIG(ADDR_SURF_P2) |
2920 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2921 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2922 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2923 PIPE_CONFIG(ADDR_SURF_P2) |
2924 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2925 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2926 PIPE_CONFIG(ADDR_SURF_P2) |
2927 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2928 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2929 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2930 PIPE_CONFIG(ADDR_SURF_P2) |
2931 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2932 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2933 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2934 PIPE_CONFIG(ADDR_SURF_P2) |
2935 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2936 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2937 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2938 PIPE_CONFIG(ADDR_SURF_P2) |
2939 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2940 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2941 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2942 PIPE_CONFIG(ADDR_SURF_P2));
2943 tile[28] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2944 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2945 PIPE_CONFIG(ADDR_SURF_P2) |
2946 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2947 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2948 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2949 PIPE_CONFIG(ADDR_SURF_P2) |
2950 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2951 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2952 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2953 PIPE_CONFIG(ADDR_SURF_P2) |
2954 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2955
2956 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2957 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2958 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2959 NUM_BANKS(ADDR_SURF_16_BANK));
2960 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2961 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2962 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2963 NUM_BANKS(ADDR_SURF_16_BANK));
2964 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2965 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2966 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2967 NUM_BANKS(ADDR_SURF_16_BANK));
2968 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2969 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2970 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2971 NUM_BANKS(ADDR_SURF_16_BANK));
2972 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2973 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2974 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2975 NUM_BANKS(ADDR_SURF_16_BANK));
2976 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2977 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2978 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2979 NUM_BANKS(ADDR_SURF_16_BANK));
2980 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2981 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2982 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2983 NUM_BANKS(ADDR_SURF_8_BANK));
2984 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2985 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2986 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2987 NUM_BANKS(ADDR_SURF_16_BANK));
2988 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2989 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2990 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2991 NUM_BANKS(ADDR_SURF_16_BANK));
2992 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2993 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2994 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2995 NUM_BANKS(ADDR_SURF_16_BANK));
2996 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2997 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2998 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2999 NUM_BANKS(ADDR_SURF_16_BANK));
3000 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3001 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3002 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3003 NUM_BANKS(ADDR_SURF_16_BANK));
3004 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3005 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3006 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3007 NUM_BANKS(ADDR_SURF_16_BANK));
3008 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3009 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3010 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3011 NUM_BANKS(ADDR_SURF_8_BANK));
3012
3013 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
3014 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
3015 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
3016 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
3017 break;
3018
3019 default:
3020 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
3021 }
3022 }
3023
3024 /**
3025 * cik_select_se_sh - select which SE, SH to address
3026 *
3027 * @rdev: radeon_device pointer
3028 * @se_num: shader engine to address
3029 * @sh_num: sh block to address
3030 *
3031 * Select which SE, SH combinations to address. Certain
3032 * registers are instanced per SE or SH. 0xffffffff means
3033 * broadcast to all SEs or SHs (CIK).
3034 */
3035 static void cik_select_se_sh(struct radeon_device *rdev,
3036 u32 se_num, u32 sh_num)
3037 {
3038 u32 data = INSTANCE_BROADCAST_WRITES;
3039
3040 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
3041 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
3042 else if (se_num == 0xffffffff)
3043 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3044 else if (sh_num == 0xffffffff)
3045 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3046 else
3047 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3048 WREG32(GRBM_GFX_INDEX, data);
3049 }
3050
3051 /**
3052 * cik_create_bitmask - create a bitmask
3053 *
3054 * @bit_width: length of the mask
3055 *
3056 * create a variable length bit mask (CIK).
3057 * Returns the bitmask.
3058 */
3059 static u32 cik_create_bitmask(u32 bit_width)
3060 {
3061 u32 i, mask = 0;
3062
3063 for (i = 0; i < bit_width; i++) {
3064 mask <<= 1;
3065 mask |= 1;
3066 }
3067 return mask;
3068 }
3069
3070 /**
3071 * cik_get_rb_disabled - computes the mask of disabled RBs
3072 *
3073 * @rdev: radeon_device pointer
3074 * @max_rb_num: max RBs (render backends) for the asic
3075 * @se_num: number of SEs (shader engines) for the asic
3076 * @sh_per_se: number of SH blocks per SE for the asic
3077 *
3078 * Calculates the bitmask of disabled RBs (CIK).
3079 * Returns the disabled RB bitmask.
3080 */
3081 static u32 cik_get_rb_disabled(struct radeon_device *rdev,
3082 u32 max_rb_num_per_se,
3083 u32 sh_per_se)
3084 {
3085 u32 data, mask;
3086
3087 data = RREG32(CC_RB_BACKEND_DISABLE);
3088 if (data & 1)
3089 data &= BACKEND_DISABLE_MASK;
3090 else
3091 data = 0;
3092 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3093
3094 data >>= BACKEND_DISABLE_SHIFT;
3095
3096 mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
3097
3098 return data & mask;
3099 }
3100
3101 /**
3102 * cik_setup_rb - setup the RBs on the asic
3103 *
3104 * @rdev: radeon_device pointer
3105 * @se_num: number of SEs (shader engines) for the asic
3106 * @sh_per_se: number of SH blocks per SE for the asic
3107 * @max_rb_num: max RBs (render backends) for the asic
3108 *
3109 * Configures per-SE/SH RB registers (CIK).
3110 */
3111 static void cik_setup_rb(struct radeon_device *rdev,
3112 u32 se_num, u32 sh_per_se,
3113 u32 max_rb_num_per_se)
3114 {
3115 int i, j;
3116 u32 data, mask;
3117 u32 disabled_rbs = 0;
3118 u32 enabled_rbs = 0;
3119
3120 for (i = 0; i < se_num; i++) {
3121 for (j = 0; j < sh_per_se; j++) {
3122 cik_select_se_sh(rdev, i, j);
3123 data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
3124 if (rdev->family == CHIP_HAWAII)
3125 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3126 else
3127 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
3128 }
3129 }
3130 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3131
3132 mask = 1;
3133 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
3134 if (!(disabled_rbs & mask))
3135 enabled_rbs |= mask;
3136 mask <<= 1;
3137 }
3138
3139 rdev->config.cik.backend_enable_mask = enabled_rbs;
3140
3141 for (i = 0; i < se_num; i++) {
3142 cik_select_se_sh(rdev, i, 0xffffffff);
3143 data = 0;
3144 for (j = 0; j < sh_per_se; j++) {
3145 switch (enabled_rbs & 3) {
3146 case 0:
3147 if (j == 0)
3148 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3149 else
3150 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3151 break;
3152 case 1:
3153 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3154 break;
3155 case 2:
3156 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3157 break;
3158 case 3:
3159 default:
3160 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3161 break;
3162 }
3163 enabled_rbs >>= 2;
3164 }
3165 WREG32(PA_SC_RASTER_CONFIG, data);
3166 }
3167 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3168 }
3169
3170 /**
3171 * cik_gpu_init - setup the 3D engine
3172 *
3173 * @rdev: radeon_device pointer
3174 *
3175 * Configures the 3D engine and tiling configuration
3176 * registers so that the 3D engine is usable.
3177 */
3178 static void cik_gpu_init(struct radeon_device *rdev)
3179 {
3180 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3181 u32 mc_shared_chmap, mc_arb_ramcfg;
3182 u32 hdp_host_path_cntl;
3183 u32 tmp;
3184 int i, j;
3185
3186 switch (rdev->family) {
3187 case CHIP_BONAIRE:
3188 rdev->config.cik.max_shader_engines = 2;
3189 rdev->config.cik.max_tile_pipes = 4;
3190 rdev->config.cik.max_cu_per_sh = 7;
3191 rdev->config.cik.max_sh_per_se = 1;
3192 rdev->config.cik.max_backends_per_se = 2;
3193 rdev->config.cik.max_texture_channel_caches = 4;
3194 rdev->config.cik.max_gprs = 256;
3195 rdev->config.cik.max_gs_threads = 32;
3196 rdev->config.cik.max_hw_contexts = 8;
3197
3198 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3199 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3200 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3201 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3202 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3203 break;
3204 case CHIP_HAWAII:
3205 rdev->config.cik.max_shader_engines = 4;
3206 rdev->config.cik.max_tile_pipes = 16;
3207 rdev->config.cik.max_cu_per_sh = 11;
3208 rdev->config.cik.max_sh_per_se = 1;
3209 rdev->config.cik.max_backends_per_se = 4;
3210 rdev->config.cik.max_texture_channel_caches = 16;
3211 rdev->config.cik.max_gprs = 256;
3212 rdev->config.cik.max_gs_threads = 32;
3213 rdev->config.cik.max_hw_contexts = 8;
3214
3215 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3216 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3217 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3218 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3219 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3220 break;
3221 case CHIP_KAVERI:
3222 rdev->config.cik.max_shader_engines = 1;
3223 rdev->config.cik.max_tile_pipes = 4;
3224 if ((rdev->pdev->device == 0x1304) ||
3225 (rdev->pdev->device == 0x1305) ||
3226 (rdev->pdev->device == 0x130C) ||
3227 (rdev->pdev->device == 0x130F) ||
3228 (rdev->pdev->device == 0x1310) ||
3229 (rdev->pdev->device == 0x1311) ||
3230 (rdev->pdev->device == 0x131C)) {
3231 rdev->config.cik.max_cu_per_sh = 8;
3232 rdev->config.cik.max_backends_per_se = 2;
3233 } else if ((rdev->pdev->device == 0x1309) ||
3234 (rdev->pdev->device == 0x130A) ||
3235 (rdev->pdev->device == 0x130D) ||
3236 (rdev->pdev->device == 0x1313) ||
3237 (rdev->pdev->device == 0x131D)) {
3238 rdev->config.cik.max_cu_per_sh = 6;
3239 rdev->config.cik.max_backends_per_se = 2;
3240 } else if ((rdev->pdev->device == 0x1306) ||
3241 (rdev->pdev->device == 0x1307) ||
3242 (rdev->pdev->device == 0x130B) ||
3243 (rdev->pdev->device == 0x130E) ||
3244 (rdev->pdev->device == 0x1315) ||
3245 (rdev->pdev->device == 0x1318) ||
3246 (rdev->pdev->device == 0x131B)) {
3247 rdev->config.cik.max_cu_per_sh = 4;
3248 rdev->config.cik.max_backends_per_se = 1;
3249 } else {
3250 rdev->config.cik.max_cu_per_sh = 3;
3251 rdev->config.cik.max_backends_per_se = 1;
3252 }
3253 rdev->config.cik.max_sh_per_se = 1;
3254 rdev->config.cik.max_texture_channel_caches = 4;
3255 rdev->config.cik.max_gprs = 256;
3256 rdev->config.cik.max_gs_threads = 16;
3257 rdev->config.cik.max_hw_contexts = 8;
3258
3259 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3260 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3261 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3262 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3263 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3264 break;
3265 case CHIP_KABINI:
3266 case CHIP_MULLINS:
3267 default:
3268 rdev->config.cik.max_shader_engines = 1;
3269 rdev->config.cik.max_tile_pipes = 2;
3270 rdev->config.cik.max_cu_per_sh = 2;
3271 rdev->config.cik.max_sh_per_se = 1;
3272 rdev->config.cik.max_backends_per_se = 1;
3273 rdev->config.cik.max_texture_channel_caches = 2;
3274 rdev->config.cik.max_gprs = 256;
3275 rdev->config.cik.max_gs_threads = 16;
3276 rdev->config.cik.max_hw_contexts = 8;
3277
3278 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3279 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3280 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3281 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3282 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3283 break;
3284 }
3285
3286 /* Initialize HDP */
3287 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3288 WREG32((0x2c14 + j), 0x00000000);
3289 WREG32((0x2c18 + j), 0x00000000);
3290 WREG32((0x2c1c + j), 0x00000000);
3291 WREG32((0x2c20 + j), 0x00000000);
3292 WREG32((0x2c24 + j), 0x00000000);
3293 }
3294
3295 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3296 WREG32(SRBM_INT_CNTL, 0x1);
3297 WREG32(SRBM_INT_ACK, 0x1);
3298
3299 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3300
3301 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3302 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3303
3304 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3305 rdev->config.cik.mem_max_burst_length_bytes = 256;
3306 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3307 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3308 if (rdev->config.cik.mem_row_size_in_kb > 4)
3309 rdev->config.cik.mem_row_size_in_kb = 4;
3310 /* XXX use MC settings? */
3311 rdev->config.cik.shader_engine_tile_size = 32;
3312 rdev->config.cik.num_gpus = 1;
3313 rdev->config.cik.multi_gpu_tile_size = 64;
3314
3315 /* fix up row size */
3316 gb_addr_config &= ~ROW_SIZE_MASK;
3317 switch (rdev->config.cik.mem_row_size_in_kb) {
3318 case 1:
3319 default:
3320 gb_addr_config |= ROW_SIZE(0);
3321 break;
3322 case 2:
3323 gb_addr_config |= ROW_SIZE(1);
3324 break;
3325 case 4:
3326 gb_addr_config |= ROW_SIZE(2);
3327 break;
3328 }
3329
3330 /* setup tiling info dword. gb_addr_config is not adequate since it does
3331 * not have bank info, so create a custom tiling dword.
3332 * bits 3:0 num_pipes
3333 * bits 7:4 num_banks
3334 * bits 11:8 group_size
3335 * bits 15:12 row_size
3336 */
3337 rdev->config.cik.tile_config = 0;
3338 switch (rdev->config.cik.num_tile_pipes) {
3339 case 1:
3340 rdev->config.cik.tile_config |= (0 << 0);
3341 break;
3342 case 2:
3343 rdev->config.cik.tile_config |= (1 << 0);
3344 break;
3345 case 4:
3346 rdev->config.cik.tile_config |= (2 << 0);
3347 break;
3348 case 8:
3349 default:
3350 /* XXX what about 12? */
3351 rdev->config.cik.tile_config |= (3 << 0);
3352 break;
3353 }
3354 rdev->config.cik.tile_config |=
3355 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
3356 rdev->config.cik.tile_config |=
3357 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3358 rdev->config.cik.tile_config |=
3359 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3360
3361 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3362 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3363 WREG32(DMIF_ADDR_CALC, gb_addr_config);
3364 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3365 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
3366 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3367 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3368 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3369
3370 cik_tiling_mode_table_init(rdev);
3371
3372 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3373 rdev->config.cik.max_sh_per_se,
3374 rdev->config.cik.max_backends_per_se);
3375
3376 rdev->config.cik.active_cus = 0;
3377 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
3378 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
3379 rdev->config.cik.active_cus +=
3380 hweight32(cik_get_cu_active_bitmap(rdev, i, j));
3381 }
3382 }
3383
3384 /* set HW defaults for 3D engine */
3385 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3386
3387 WREG32(SX_DEBUG_1, 0x20);
3388
3389 WREG32(TA_CNTL_AUX, 0x00010000);
3390
3391 tmp = RREG32(SPI_CONFIG_CNTL);
3392 tmp |= 0x03000000;
3393 WREG32(SPI_CONFIG_CNTL, tmp);
3394
3395 WREG32(SQ_CONFIG, 1);
3396
3397 WREG32(DB_DEBUG, 0);
3398
3399 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3400 tmp |= 0x00000400;
3401 WREG32(DB_DEBUG2, tmp);
3402
3403 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3404 tmp |= 0x00020200;
3405 WREG32(DB_DEBUG3, tmp);
3406
3407 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3408 tmp |= 0x00018208;
3409 WREG32(CB_HW_CONTROL, tmp);
3410
3411 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3412
3413 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3414 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3415 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3416 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3417
3418 WREG32(VGT_NUM_INSTANCES, 1);
3419
3420 WREG32(CP_PERFMON_CNTL, 0);
3421
3422 WREG32(SQ_CONFIG, 0);
3423
3424 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3425 FORCE_EOV_MAX_REZ_CNT(255)));
3426
3427 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3428 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3429
3430 WREG32(VGT_GS_VERTEX_REUSE, 16);
3431 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3432
3433 tmp = RREG32(HDP_MISC_CNTL);
3434 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3435 WREG32(HDP_MISC_CNTL, tmp);
3436
3437 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3438 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3439
3440 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3441 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3442
3443 udelay(50);
3444 }
3445
3446 /*
3447 * GPU scratch registers helpers function.
3448 */
3449 /**
3450 * cik_scratch_init - setup driver info for CP scratch regs
3451 *
3452 * @rdev: radeon_device pointer
3453 *
3454 * Set up the number and offset of the CP scratch registers.
3455 * NOTE: use of CP scratch registers is a legacy inferface and
3456 * is not used by default on newer asics (r6xx+). On newer asics,
3457 * memory buffers are used for fences rather than scratch regs.
3458 */
3459 static void cik_scratch_init(struct radeon_device *rdev)
3460 {
3461 int i;
3462
3463 rdev->scratch.num_reg = 7;
3464 rdev->scratch.reg_base = SCRATCH_REG0;
3465 for (i = 0; i < rdev->scratch.num_reg; i++) {
3466 rdev->scratch.free[i] = true;
3467 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3468 }
3469 }
3470
3471 /**
3472 * cik_ring_test - basic gfx ring test
3473 *
3474 * @rdev: radeon_device pointer
3475 * @ring: radeon_ring structure holding ring information
3476 *
3477 * Allocate a scratch register and write to it using the gfx ring (CIK).
3478 * Provides a basic gfx ring test to verify that the ring is working.
3479 * Used by cik_cp_gfx_resume();
3480 * Returns 0 on success, error on failure.
3481 */
3482 int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3483 {
3484 uint32_t scratch;
3485 uint32_t tmp = 0;
3486 unsigned i;
3487 int r;
3488
3489 r = radeon_scratch_get(rdev, &scratch);
3490 if (r) {
3491 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3492 return r;
3493 }
3494 WREG32(scratch, 0xCAFEDEAD);
3495 r = radeon_ring_lock(rdev, ring, 3);
3496 if (r) {
3497 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3498 radeon_scratch_free(rdev, scratch);
3499 return r;
3500 }
3501 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3502 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3503 radeon_ring_write(ring, 0xDEADBEEF);
3504 radeon_ring_unlock_commit(rdev, ring, false);
3505
3506 for (i = 0; i < rdev->usec_timeout; i++) {
3507 tmp = RREG32(scratch);
3508 if (tmp == 0xDEADBEEF)
3509 break;
3510 DRM_UDELAY(1);
3511 }
3512 if (i < rdev->usec_timeout) {
3513 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3514 } else {
3515 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3516 ring->idx, scratch, tmp);
3517 r = -EINVAL;
3518 }
3519 radeon_scratch_free(rdev, scratch);
3520 return r;
3521 }
3522
3523 /**
3524 * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3525 *
3526 * @rdev: radeon_device pointer
3527 * @ridx: radeon ring index
3528 *
3529 * Emits an hdp flush on the cp.
3530 */
3531 static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,
3532 int ridx)
3533 {
3534 struct radeon_ring *ring = &rdev->ring[ridx];
3535 u32 ref_and_mask;
3536
3537 switch (ring->idx) {
3538 case CAYMAN_RING_TYPE_CP1_INDEX:
3539 case CAYMAN_RING_TYPE_CP2_INDEX:
3540 default:
3541 switch (ring->me) {
3542 case 0:
3543 ref_and_mask = CP2 << ring->pipe;
3544 break;
3545 case 1:
3546 ref_and_mask = CP6 << ring->pipe;
3547 break;
3548 default:
3549 return;
3550 }
3551 break;
3552 case RADEON_RING_TYPE_GFX_INDEX:
3553 ref_and_mask = CP0;
3554 break;
3555 }
3556
3557 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3558 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
3559 WAIT_REG_MEM_FUNCTION(3) | /* == */
3560 WAIT_REG_MEM_ENGINE(1))); /* pfp */
3561 radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2);
3562 radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2);
3563 radeon_ring_write(ring, ref_and_mask);
3564 radeon_ring_write(ring, ref_and_mask);
3565 radeon_ring_write(ring, 0x20); /* poll interval */
3566 }
3567
3568 /**
3569 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
3570 *
3571 * @rdev: radeon_device pointer
3572 * @fence: radeon fence object
3573 *
3574 * Emits a fence sequnce number on the gfx ring and flushes
3575 * GPU caches.
3576 */
3577 void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3578 struct radeon_fence *fence)
3579 {
3580 struct radeon_ring *ring = &rdev->ring[fence->ring];
3581 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3582
3583 /* Workaround for cache flush problems. First send a dummy EOP
3584 * event down the pipe with seq one below.
3585 */
3586 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3587 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3588 EOP_TC_ACTION_EN |
3589 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3590 EVENT_INDEX(5)));
3591 radeon_ring_write(ring, addr & 0xfffffffc);
3592 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) |
3593 DATA_SEL(1) | INT_SEL(0));
3594 radeon_ring_write(ring, fence->seq - 1);
3595 radeon_ring_write(ring, 0);
3596
3597 /* Then send the real EOP event down the pipe. */
3598 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3599 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3600 EOP_TC_ACTION_EN |
3601 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3602 EVENT_INDEX(5)));
3603 radeon_ring_write(ring, addr & 0xfffffffc);
3604 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3605 radeon_ring_write(ring, fence->seq);
3606 radeon_ring_write(ring, 0);
3607 }
3608
3609 /**
3610 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3611 *
3612 * @rdev: radeon_device pointer
3613 * @fence: radeon fence object
3614 *
3615 * Emits a fence sequnce number on the compute ring and flushes
3616 * GPU caches.
3617 */
3618 void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3619 struct radeon_fence *fence)
3620 {
3621 struct radeon_ring *ring = &rdev->ring[fence->ring];
3622 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3623
3624 /* RELEASE_MEM - flush caches, send int */
3625 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3626 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3627 EOP_TC_ACTION_EN |
3628 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3629 EVENT_INDEX(5)));
3630 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3631 radeon_ring_write(ring, addr & 0xfffffffc);
3632 radeon_ring_write(ring, upper_32_bits(addr));
3633 radeon_ring_write(ring, fence->seq);
3634 radeon_ring_write(ring, 0);
3635 }
3636
3637 /**
3638 * cik_semaphore_ring_emit - emit a semaphore on the CP ring
3639 *
3640 * @rdev: radeon_device pointer
3641 * @ring: radeon ring buffer object
3642 * @semaphore: radeon semaphore object
3643 * @emit_wait: Is this a sempahore wait?
3644 *
3645 * Emits a semaphore signal/wait packet to the CP ring and prevents the PFP
3646 * from running ahead of semaphore waits.
3647 */
3648 bool cik_semaphore_ring_emit(struct radeon_device *rdev,
3649 struct radeon_ring *ring,
3650 struct radeon_semaphore *semaphore,
3651 bool emit_wait)
3652 {
3653 uint64_t addr = semaphore->gpu_addr;
3654 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3655
3656 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3657 radeon_ring_write(ring, lower_32_bits(addr));
3658 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
3659
3660 if (emit_wait && ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
3661 /* Prevent the PFP from running ahead of the semaphore wait */
3662 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
3663 radeon_ring_write(ring, 0x0);
3664 }
3665
3666 return true;
3667 }
3668
3669 /**
3670 * cik_copy_cpdma - copy pages using the CP DMA engine
3671 *
3672 * @rdev: radeon_device pointer
3673 * @src_offset: src GPU address
3674 * @dst_offset: dst GPU address
3675 * @num_gpu_pages: number of GPU pages to xfer
3676 * @resv: reservation object to sync to
3677 *
3678 * Copy GPU paging using the CP DMA engine (CIK+).
3679 * Used by the radeon ttm implementation to move pages if
3680 * registered as the asic copy callback.
3681 */
3682 struct radeon_fence *cik_copy_cpdma(struct radeon_device *rdev,
3683 uint64_t src_offset, uint64_t dst_offset,
3684 unsigned num_gpu_pages,
3685 struct reservation_object *resv)
3686 {
3687 struct radeon_fence *fence;
3688 struct radeon_sync sync;
3689 int ring_index = rdev->asic->copy.blit_ring_index;
3690 struct radeon_ring *ring = &rdev->ring[ring_index];
3691 u32 size_in_bytes, cur_size_in_bytes, control;
3692 int i, num_loops;
3693 int r = 0;
3694
3695 radeon_sync_create(&sync);
3696
3697 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3698 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3699 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3700 if (r) {
3701 DRM_ERROR("radeon: moving bo (%d).\n", r);
3702 radeon_sync_free(rdev, &sync, NULL);
3703 return ERR_PTR(r);
3704 }
3705
3706 radeon_sync_resv(rdev, &sync, resv, false);
3707 radeon_sync_rings(rdev, &sync, ring->idx);
3708
3709 for (i = 0; i < num_loops; i++) {
3710 cur_size_in_bytes = size_in_bytes;
3711 if (cur_size_in_bytes > 0x1fffff)
3712 cur_size_in_bytes = 0x1fffff;
3713 size_in_bytes -= cur_size_in_bytes;
3714 control = 0;
3715 if (size_in_bytes == 0)
3716 control |= PACKET3_DMA_DATA_CP_SYNC;
3717 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
3718 radeon_ring_write(ring, control);
3719 radeon_ring_write(ring, lower_32_bits(src_offset));
3720 radeon_ring_write(ring, upper_32_bits(src_offset));
3721 radeon_ring_write(ring, lower_32_bits(dst_offset));
3722 radeon_ring_write(ring, upper_32_bits(dst_offset));
3723 radeon_ring_write(ring, cur_size_in_bytes);
3724 src_offset += cur_size_in_bytes;
3725 dst_offset += cur_size_in_bytes;
3726 }
3727
3728 r = radeon_fence_emit(rdev, &fence, ring->idx);
3729 if (r) {
3730 radeon_ring_unlock_undo(rdev, ring);
3731 radeon_sync_free(rdev, &sync, NULL);
3732 return ERR_PTR(r);
3733 }
3734
3735 radeon_ring_unlock_commit(rdev, ring, false);
3736 radeon_sync_free(rdev, &sync, fence);
3737
3738 return fence;
3739 }
3740
3741 /*
3742 * IB stuff
3743 */
3744 /**
3745 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3746 *
3747 * @rdev: radeon_device pointer
3748 * @ib: radeon indirect buffer object
3749 *
3750 * Emits a DE (drawing engine) or CE (constant engine) IB
3751 * on the gfx ring. IBs are usually generated by userspace
3752 * acceleration drivers and submitted to the kernel for
3753 * scheduling on the ring. This function schedules the IB
3754 * on the gfx ring for execution by the GPU.
3755 */
3756 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3757 {
3758 struct radeon_ring *ring = &rdev->ring[ib->ring];
3759 unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
3760 u32 header, control = INDIRECT_BUFFER_VALID;
3761
3762 if (ib->is_const_ib) {
3763 /* set switch buffer packet before const IB */
3764 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3765 radeon_ring_write(ring, 0);
3766
3767 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3768 } else {
3769 u32 next_rptr;
3770 if (ring->rptr_save_reg) {
3771 next_rptr = ring->wptr + 3 + 4;
3772 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3773 radeon_ring_write(ring, ((ring->rptr_save_reg -
3774 PACKET3_SET_UCONFIG_REG_START) >> 2));
3775 radeon_ring_write(ring, next_rptr);
3776 } else if (rdev->wb.enabled) {
3777 next_rptr = ring->wptr + 5 + 4;
3778 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3779 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
3780 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3781 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
3782 radeon_ring_write(ring, next_rptr);
3783 }
3784
3785 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3786 }
3787
3788 control |= ib->length_dw | (vm_id << 24);
3789
3790 radeon_ring_write(ring, header);
3791 radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFFC));
3792 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3793 radeon_ring_write(ring, control);
3794 }
3795
3796 /**
3797 * cik_ib_test - basic gfx ring IB test
3798 *
3799 * @rdev: radeon_device pointer
3800 * @ring: radeon_ring structure holding ring information
3801 *
3802 * Allocate an IB and execute it on the gfx ring (CIK).
3803 * Provides a basic gfx ring test to verify that IBs are working.
3804 * Returns 0 on success, error on failure.
3805 */
3806 int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3807 {
3808 struct radeon_ib ib;
3809 uint32_t scratch;
3810 uint32_t tmp = 0;
3811 unsigned i;
3812 int r;
3813
3814 r = radeon_scratch_get(rdev, &scratch);
3815 if (r) {
3816 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3817 return r;
3818 }
3819 WREG32(scratch, 0xCAFEDEAD);
3820 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3821 if (r) {
3822 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
3823 radeon_scratch_free(rdev, scratch);
3824 return r;
3825 }
3826 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
3827 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
3828 ib.ptr[2] = 0xDEADBEEF;
3829 ib.length_dw = 3;
3830 r = radeon_ib_schedule(rdev, &ib, NULL, false);
3831 if (r) {
3832 radeon_scratch_free(rdev, scratch);
3833 radeon_ib_free(rdev, &ib);
3834 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3835 return r;
3836 }
3837 r = radeon_fence_wait_timeout(ib.fence, false, usecs_to_jiffies(
3838 RADEON_USEC_IB_TEST_TIMEOUT));
3839 if (r < 0) {
3840 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3841 radeon_scratch_free(rdev, scratch);
3842 radeon_ib_free(rdev, &ib);
3843 return r;
3844 } else if (r == 0) {
3845 DRM_ERROR("radeon: fence wait timed out.\n");
3846 radeon_scratch_free(rdev, scratch);
3847 radeon_ib_free(rdev, &ib);
3848 return -ETIMEDOUT;
3849 }
3850 r = 0;
3851 for (i = 0; i < rdev->usec_timeout; i++) {
3852 tmp = RREG32(scratch);
3853 if (tmp == 0xDEADBEEF)
3854 break;
3855 DRM_UDELAY(1);
3856 }
3857 if (i < rdev->usec_timeout) {
3858 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3859 } else {
3860 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3861 scratch, tmp);
3862 r = -EINVAL;
3863 }
3864 radeon_scratch_free(rdev, scratch);
3865 radeon_ib_free(rdev, &ib);
3866 return r;
3867 }
3868
3869 /*
3870 * CP.
3871 * On CIK, gfx and compute now have independant command processors.
3872 *
3873 * GFX
3874 * Gfx consists of a single ring and can process both gfx jobs and
3875 * compute jobs. The gfx CP consists of three microengines (ME):
3876 * PFP - Pre-Fetch Parser
3877 * ME - Micro Engine
3878 * CE - Constant Engine
3879 * The PFP and ME make up what is considered the Drawing Engine (DE).
3880 * The CE is an asynchronous engine used for updating buffer desciptors
3881 * used by the DE so that they can be loaded into cache in parallel
3882 * while the DE is processing state update packets.
3883 *
3884 * Compute
3885 * The compute CP consists of two microengines (ME):
3886 * MEC1 - Compute MicroEngine 1
3887 * MEC2 - Compute MicroEngine 2
3888 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3889 * The queues are exposed to userspace and are programmed directly
3890 * by the compute runtime.
3891 */
3892 /**
3893 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3894 *
3895 * @rdev: radeon_device pointer
3896 * @enable: enable or disable the MEs
3897 *
3898 * Halts or unhalts the gfx MEs.
3899 */
3900 static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
3901 {
3902 if (enable)
3903 WREG32(CP_ME_CNTL, 0);
3904 else {
3905 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3906 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3907 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3908 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3909 }
3910 udelay(50);
3911 }
3912
3913 /**
3914 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3915 *
3916 * @rdev: radeon_device pointer
3917 *
3918 * Loads the gfx PFP, ME, and CE ucode.
3919 * Returns 0 for success, -EINVAL if the ucode is not available.
3920 */
3921 static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
3922 {
3923 int i;
3924
3925 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3926 return -EINVAL;
3927
3928 cik_cp_gfx_enable(rdev, false);
3929
3930 if (rdev->new_fw) {
3931 const struct gfx_firmware_header_v1_0 *pfp_hdr =
3932 (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
3933 const struct gfx_firmware_header_v1_0 *ce_hdr =
3934 (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
3935 const struct gfx_firmware_header_v1_0 *me_hdr =
3936 (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
3937 const __le32 *fw_data;
3938 u32 fw_size;
3939
3940 radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
3941 radeon_ucode_print_gfx_hdr(&ce_hdr->header);
3942 radeon_ucode_print_gfx_hdr(&me_hdr->header);
3943
3944 /* PFP */
3945 fw_data = (const __le32 *)
3946 (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3947 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3948 WREG32(CP_PFP_UCODE_ADDR, 0);
3949 for (i = 0; i < fw_size; i++)
3950 WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
3951 WREG32(CP_PFP_UCODE_ADDR, le32_to_cpu(pfp_hdr->header.ucode_version));
3952
3953 /* CE */
3954 fw_data = (const __le32 *)
3955 (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3956 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3957 WREG32(CP_CE_UCODE_ADDR, 0);
3958 for (i = 0; i < fw_size; i++)
3959 WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
3960 WREG32(CP_CE_UCODE_ADDR, le32_to_cpu(ce_hdr->header.ucode_version));
3961
3962 /* ME */
3963 fw_data = (const __be32 *)
3964 (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3965 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3966 WREG32(CP_ME_RAM_WADDR, 0);
3967 for (i = 0; i < fw_size; i++)
3968 WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
3969 WREG32(CP_ME_RAM_WADDR, le32_to_cpu(me_hdr->header.ucode_version));
3970 WREG32(CP_ME_RAM_RADDR, le32_to_cpu(me_hdr->header.ucode_version));
3971 } else {
3972 const __be32 *fw_data;
3973
3974 /* PFP */
3975 fw_data = (const __be32 *)rdev->pfp_fw->data;
3976 WREG32(CP_PFP_UCODE_ADDR, 0);
3977 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
3978 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3979 WREG32(CP_PFP_UCODE_ADDR, 0);
3980
3981 /* CE */
3982 fw_data = (const __be32 *)rdev->ce_fw->data;
3983 WREG32(CP_CE_UCODE_ADDR, 0);
3984 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
3985 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3986 WREG32(CP_CE_UCODE_ADDR, 0);
3987
3988 /* ME */
3989 fw_data = (const __be32 *)rdev->me_fw->data;
3990 WREG32(CP_ME_RAM_WADDR, 0);
3991 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
3992 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3993 WREG32(CP_ME_RAM_WADDR, 0);
3994 }
3995
3996 return 0;
3997 }
3998
3999 /**
4000 * cik_cp_gfx_start - start the gfx ring
4001 *
4002 * @rdev: radeon_device pointer
4003 *
4004 * Enables the ring and loads the clear state context and other
4005 * packets required to init the ring.
4006 * Returns 0 for success, error for failure.
4007 */
4008 static int cik_cp_gfx_start(struct radeon_device *rdev)
4009 {
4010 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4011 int r, i;
4012
4013 /* init the CP */
4014 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
4015 WREG32(CP_ENDIAN_SWAP, 0);
4016 WREG32(CP_DEVICE_ID, 1);
4017
4018 cik_cp_gfx_enable(rdev, true);
4019
4020 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
4021 if (r) {
4022 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
4023 return r;
4024 }
4025
4026 /* init the CE partitions. CE only used for gfx on CIK */
4027 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
4028 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
4029 radeon_ring_write(ring, 0x8000);
4030 radeon_ring_write(ring, 0x8000);
4031
4032 /* setup clear context state */
4033 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4034 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
4035
4036 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
4037 radeon_ring_write(ring, 0x80000000);
4038 radeon_ring_write(ring, 0x80000000);
4039
4040 for (i = 0; i < cik_default_size; i++)
4041 radeon_ring_write(ring, cik_default_state[i]);
4042
4043 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4044 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
4045
4046 /* set clear context state */
4047 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
4048 radeon_ring_write(ring, 0);
4049
4050 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
4051 radeon_ring_write(ring, 0x00000316);
4052 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
4053 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
4054
4055 radeon_ring_unlock_commit(rdev, ring, false);
4056
4057 return 0;
4058 }
4059
4060 /**
4061 * cik_cp_gfx_fini - stop the gfx ring
4062 *
4063 * @rdev: radeon_device pointer
4064 *
4065 * Stop the gfx ring and tear down the driver ring
4066 * info.
4067 */
4068 static void cik_cp_gfx_fini(struct radeon_device *rdev)
4069 {
4070 cik_cp_gfx_enable(rdev, false);
4071 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4072 }
4073
4074 /**
4075 * cik_cp_gfx_resume - setup the gfx ring buffer registers
4076 *
4077 * @rdev: radeon_device pointer
4078 *
4079 * Program the location and size of the gfx ring buffer
4080 * and test it to make sure it's working.
4081 * Returns 0 for success, error for failure.
4082 */
4083 static int cik_cp_gfx_resume(struct radeon_device *rdev)
4084 {
4085 struct radeon_ring *ring;
4086 u32 tmp;
4087 u32 rb_bufsz;
4088 u64 rb_addr;
4089 int r;
4090
4091 WREG32(CP_SEM_WAIT_TIMER, 0x0);
4092 if (rdev->family != CHIP_HAWAII)
4093 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
4094
4095 /* Set the write pointer delay */
4096 WREG32(CP_RB_WPTR_DELAY, 0);
4097
4098 /* set the RB to use vmid 0 */
4099 WREG32(CP_RB_VMID, 0);
4100
4101 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
4102
4103 /* ring 0 - compute and gfx */
4104 /* Set ring buffer size */
4105 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4106 rb_bufsz = order_base_2(ring->ring_size / 8);
4107 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
4108 #ifdef __BIG_ENDIAN
4109 tmp |= BUF_SWAP_32BIT;
4110 #endif
4111 WREG32(CP_RB0_CNTL, tmp);
4112
4113 /* Initialize the ring buffer's read and write pointers */
4114 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
4115 ring->wptr = 0;
4116 WREG32(CP_RB0_WPTR, ring->wptr);
4117
4118 /* set the wb address wether it's enabled or not */
4119 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
4120 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
4121
4122 /* scratch register shadowing is no longer supported */
4123 WREG32(SCRATCH_UMSK, 0);
4124
4125 if (!rdev->wb.enabled)
4126 tmp |= RB_NO_UPDATE;
4127
4128 mdelay(1);
4129 WREG32(CP_RB0_CNTL, tmp);
4130
4131 rb_addr = ring->gpu_addr >> 8;
4132 WREG32(CP_RB0_BASE, rb_addr);
4133 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4134
4135 /* start the ring */
4136 cik_cp_gfx_start(rdev);
4137 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4138 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4139 if (r) {
4140 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4141 return r;
4142 }
4143
4144 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4145 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
4146
4147 return 0;
4148 }
4149
4150 u32 cik_gfx_get_rptr(struct radeon_device *rdev,
4151 struct radeon_ring *ring)
4152 {
4153 u32 rptr;
4154
4155 if (rdev->wb.enabled)
4156 rptr = rdev->wb.wb[ring->rptr_offs/4];
4157 else
4158 rptr = RREG32(CP_RB0_RPTR);
4159
4160 return rptr;
4161 }
4162
4163 u32 cik_gfx_get_wptr(struct radeon_device *rdev,
4164 struct radeon_ring *ring)
4165 {
4166 return RREG32(CP_RB0_WPTR);
4167 }
4168
4169 void cik_gfx_set_wptr(struct radeon_device *rdev,
4170 struct radeon_ring *ring)
4171 {
4172 WREG32(CP_RB0_WPTR, ring->wptr);
4173 (void)RREG32(CP_RB0_WPTR);
4174 }
4175
4176 u32 cik_compute_get_rptr(struct radeon_device *rdev,
4177 struct radeon_ring *ring)
4178 {
4179 u32 rptr;
4180
4181 if (rdev->wb.enabled) {
4182 rptr = rdev->wb.wb[ring->rptr_offs/4];
4183 } else {
4184 mutex_lock(&rdev->srbm_mutex);
4185 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4186 rptr = RREG32(CP_HQD_PQ_RPTR);
4187 cik_srbm_select(rdev, 0, 0, 0, 0);
4188 mutex_unlock(&rdev->srbm_mutex);
4189 }
4190
4191 return rptr;
4192 }
4193
4194 u32 cik_compute_get_wptr(struct radeon_device *rdev,
4195 struct radeon_ring *ring)
4196 {
4197 u32 wptr;
4198
4199 if (rdev->wb.enabled) {
4200 /* XXX check if swapping is necessary on BE */
4201 wptr = rdev->wb.wb[ring->wptr_offs/4];
4202 } else {
4203 mutex_lock(&rdev->srbm_mutex);
4204 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4205 wptr = RREG32(CP_HQD_PQ_WPTR);
4206 cik_srbm_select(rdev, 0, 0, 0, 0);
4207 mutex_unlock(&rdev->srbm_mutex);
4208 }
4209
4210 return wptr;
4211 }
4212
4213 void cik_compute_set_wptr(struct radeon_device *rdev,
4214 struct radeon_ring *ring)
4215 {
4216 /* XXX check if swapping is necessary on BE */
4217 rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;
4218 WDOORBELL32(ring->doorbell_index, ring->wptr);
4219 }
4220
4221 static void cik_compute_stop(struct radeon_device *rdev,
4222 struct radeon_ring *ring)
4223 {
4224 u32 j, tmp;
4225
4226 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4227 /* Disable wptr polling. */
4228 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4229 tmp &= ~WPTR_POLL_EN;
4230 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4231 /* Disable HQD. */
4232 if (RREG32(CP_HQD_ACTIVE) & 1) {
4233 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4234 for (j = 0; j < rdev->usec_timeout; j++) {
4235 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4236 break;
4237 udelay(1);
4238 }
4239 WREG32(CP_HQD_DEQUEUE_REQUEST, 0);
4240 WREG32(CP_HQD_PQ_RPTR, 0);
4241 WREG32(CP_HQD_PQ_WPTR, 0);
4242 }
4243 cik_srbm_select(rdev, 0, 0, 0, 0);
4244 }
4245
4246 /**
4247 * cik_cp_compute_enable - enable/disable the compute CP MEs
4248 *
4249 * @rdev: radeon_device pointer
4250 * @enable: enable or disable the MEs
4251 *
4252 * Halts or unhalts the compute MEs.
4253 */
4254 static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4255 {
4256 if (enable)
4257 WREG32(CP_MEC_CNTL, 0);
4258 else {
4259 /*
4260 * To make hibernation reliable we need to clear compute ring
4261 * configuration before halting the compute ring.
4262 */
4263 mutex_lock(&rdev->srbm_mutex);
4264 cik_compute_stop(rdev,&rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
4265 cik_compute_stop(rdev,&rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
4266 mutex_unlock(&rdev->srbm_mutex);
4267
4268 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
4269 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
4270 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
4271 }
4272 udelay(50);
4273 }
4274
4275 /**
4276 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4277 *
4278 * @rdev: radeon_device pointer
4279 *
4280 * Loads the compute MEC1&2 ucode.
4281 * Returns 0 for success, -EINVAL if the ucode is not available.
4282 */
4283 static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4284 {
4285 int i;
4286
4287 if (!rdev->mec_fw)
4288 return -EINVAL;
4289
4290 cik_cp_compute_enable(rdev, false);
4291
4292 if (rdev->new_fw) {
4293 const struct gfx_firmware_header_v1_0 *mec_hdr =
4294 (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
4295 const __le32 *fw_data;
4296 u32 fw_size;
4297
4298 radeon_ucode_print_gfx_hdr(&mec_hdr->header);
4299
4300 /* MEC1 */
4301 fw_data = (const __le32 *)
4302 (rdev->mec_fw->data + le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
4303 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4;
4304 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4305 for (i = 0; i < fw_size; i++)
4306 WREG32(CP_MEC_ME1_UCODE_DATA, le32_to_cpup(fw_data++));
4307 WREG32(CP_MEC_ME1_UCODE_ADDR, le32_to_cpu(mec_hdr->header.ucode_version));
4308
4309 /* MEC2 */
4310 if (rdev->family == CHIP_KAVERI) {
4311 const struct gfx_firmware_header_v1_0 *mec2_hdr =
4312 (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
4313
4314 fw_data = (const __le32 *)
4315 (rdev->mec2_fw->data +
4316 le32_to_cpu(mec2_hdr->header.ucode_array_offset_bytes));
4317 fw_size = le32_to_cpu(mec2_hdr->header.ucode_size_bytes) / 4;
4318 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4319 for (i = 0; i < fw_size; i++)
4320 WREG32(CP_MEC_ME2_UCODE_DATA, le32_to_cpup(fw_data++));
4321 WREG32(CP_MEC_ME2_UCODE_ADDR, le32_to_cpu(mec2_hdr->header.ucode_version));
4322 }
4323 } else {
4324 const __be32 *fw_data;
4325
4326 /* MEC1 */
4327 fw_data = (const __be32 *)rdev->mec_fw->data;
4328 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4329 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4330 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4331 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4332
4333 if (rdev->family == CHIP_KAVERI) {
4334 /* MEC2 */
4335 fw_data = (const __be32 *)rdev->mec_fw->data;
4336 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4337 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4338 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4339 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4340 }
4341 }
4342
4343 return 0;
4344 }
4345
4346 /**
4347 * cik_cp_compute_start - start the compute queues
4348 *
4349 * @rdev: radeon_device pointer
4350 *
4351 * Enable the compute queues.
4352 * Returns 0 for success, error for failure.
4353 */
4354 static int cik_cp_compute_start(struct radeon_device *rdev)
4355 {
4356 cik_cp_compute_enable(rdev, true);
4357
4358 return 0;
4359 }
4360
4361 /**
4362 * cik_cp_compute_fini - stop the compute queues
4363 *
4364 * @rdev: radeon_device pointer
4365 *
4366 * Stop the compute queues and tear down the driver queue
4367 * info.
4368 */
4369 static void cik_cp_compute_fini(struct radeon_device *rdev)
4370 {
4371 int i, idx, r;
4372
4373 cik_cp_compute_enable(rdev, false);
4374
4375 for (i = 0; i < 2; i++) {
4376 if (i == 0)
4377 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4378 else
4379 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4380
4381 if (rdev->ring[idx].mqd_obj) {
4382 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4383 if (unlikely(r != 0))
4384 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4385
4386 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4387 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4388
4389 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4390 rdev->ring[idx].mqd_obj = NULL;
4391 }
4392 }
4393 }
4394
4395 static void cik_mec_fini(struct radeon_device *rdev)
4396 {
4397 int r;
4398
4399 if (rdev->mec.hpd_eop_obj) {
4400 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4401 if (unlikely(r != 0))
4402 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4403 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4404 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4405
4406 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4407 rdev->mec.hpd_eop_obj = NULL;
4408 }
4409 }
4410
4411 #define MEC_HPD_SIZE 2048
4412
4413 static int cik_mec_init(struct radeon_device *rdev)
4414 {
4415 int r;
4416 u32 *hpd;
4417
4418 /*
4419 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4420 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4421 */
4422 if (rdev->family == CHIP_KAVERI)
4423 rdev->mec.num_mec = 2;
4424 else
4425 rdev->mec.num_mec = 1;
4426 rdev->mec.num_pipe = 4;
4427 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4428
4429 if (rdev->mec.hpd_eop_obj == NULL) {
4430 r = radeon_bo_create(rdev,
4431 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4432 PAGE_SIZE, true,
4433 RADEON_GEM_DOMAIN_GTT, 0, NULL, NULL,
4434 &rdev->mec.hpd_eop_obj);
4435 if (r) {
4436 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4437 return r;
4438 }
4439 }
4440
4441 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4442 if (unlikely(r != 0)) {
4443 cik_mec_fini(rdev);
4444 return r;
4445 }
4446 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4447 &rdev->mec.hpd_eop_gpu_addr);
4448 if (r) {
4449 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4450 cik_mec_fini(rdev);
4451 return r;
4452 }
4453 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4454 if (r) {
4455 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4456 cik_mec_fini(rdev);
4457 return r;
4458 }
4459
4460 /* clear memory. Not sure if this is required or not */
4461 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4462
4463 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4464 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4465
4466 return 0;
4467 }
4468
4469 struct hqd_registers
4470 {
4471 u32 cp_mqd_base_addr;
4472 u32 cp_mqd_base_addr_hi;
4473 u32 cp_hqd_active;
4474 u32 cp_hqd_vmid;
4475 u32 cp_hqd_persistent_state;
4476 u32 cp_hqd_pipe_priority;
4477 u32 cp_hqd_queue_priority;
4478 u32 cp_hqd_quantum;
4479 u32 cp_hqd_pq_base;
4480 u32 cp_hqd_pq_base_hi;
4481 u32 cp_hqd_pq_rptr;
4482 u32 cp_hqd_pq_rptr_report_addr;
4483 u32 cp_hqd_pq_rptr_report_addr_hi;
4484 u32 cp_hqd_pq_wptr_poll_addr;
4485 u32 cp_hqd_pq_wptr_poll_addr_hi;
4486 u32 cp_hqd_pq_doorbell_control;
4487 u32 cp_hqd_pq_wptr;
4488 u32 cp_hqd_pq_control;
4489 u32 cp_hqd_ib_base_addr;
4490 u32 cp_hqd_ib_base_addr_hi;
4491 u32 cp_hqd_ib_rptr;
4492 u32 cp_hqd_ib_control;
4493 u32 cp_hqd_iq_timer;
4494 u32 cp_hqd_iq_rptr;
4495 u32 cp_hqd_dequeue_request;
4496 u32 cp_hqd_dma_offload;
4497 u32 cp_hqd_sema_cmd;
4498 u32 cp_hqd_msg_type;
4499 u32 cp_hqd_atomic0_preop_lo;
4500 u32 cp_hqd_atomic0_preop_hi;
4501 u32 cp_hqd_atomic1_preop_lo;
4502 u32 cp_hqd_atomic1_preop_hi;
4503 u32 cp_hqd_hq_scheduler0;
4504 u32 cp_hqd_hq_scheduler1;
4505 u32 cp_mqd_control;
4506 };
4507
4508 struct bonaire_mqd
4509 {
4510 u32 header;
4511 u32 dispatch_initiator;
4512 u32 dimensions[3];
4513 u32 start_idx[3];
4514 u32 num_threads[3];
4515 u32 pipeline_stat_enable;
4516 u32 perf_counter_enable;
4517 u32 pgm[2];
4518 u32 tba[2];
4519 u32 tma[2];
4520 u32 pgm_rsrc[2];
4521 u32 vmid;
4522 u32 resource_limits;
4523 u32 static_thread_mgmt01[2];
4524 u32 tmp_ring_size;
4525 u32 static_thread_mgmt23[2];
4526 u32 restart[3];
4527 u32 thread_trace_enable;
4528 u32 reserved1;
4529 u32 user_data[16];
4530 u32 vgtcs_invoke_count[2];
4531 struct hqd_registers queue_state;
4532 u32 dequeue_cntr;
4533 u32 interrupt_queue[64];
4534 };
4535
4536 /**
4537 * cik_cp_compute_resume - setup the compute queue registers
4538 *
4539 * @rdev: radeon_device pointer
4540 *
4541 * Program the compute queues and test them to make sure they
4542 * are working.
4543 * Returns 0 for success, error for failure.
4544 */
4545 static int cik_cp_compute_resume(struct radeon_device *rdev)
4546 {
4547 int r, i, j, idx;
4548 u32 tmp;
4549 bool use_doorbell = true;
4550 u64 hqd_gpu_addr;
4551 u64 mqd_gpu_addr;
4552 u64 eop_gpu_addr;
4553 u64 wb_gpu_addr;
4554 u32 *buf;
4555 struct bonaire_mqd *mqd;
4556
4557 r = cik_cp_compute_start(rdev);
4558 if (r)
4559 return r;
4560
4561 /* fix up chicken bits */
4562 tmp = RREG32(CP_CPF_DEBUG);
4563 tmp |= (1 << 23);
4564 WREG32(CP_CPF_DEBUG, tmp);
4565
4566 /* init the pipes */
4567 mutex_lock(&rdev->srbm_mutex);
4568
4569 for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); ++i) {
4570 int me = (i < 4) ? 1 : 2;
4571 int pipe = (i < 4) ? i : (i - 4);
4572
4573 cik_srbm_select(rdev, me, pipe, 0, 0);
4574
4575 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2) ;
4576 /* write the EOP addr */
4577 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4578 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4579
4580 /* set the VMID assigned */
4581 WREG32(CP_HPD_EOP_VMID, 0);
4582
4583 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4584 tmp = RREG32(CP_HPD_EOP_CONTROL);
4585 tmp &= ~EOP_SIZE_MASK;
4586 tmp |= order_base_2(MEC_HPD_SIZE / 8);
4587 WREG32(CP_HPD_EOP_CONTROL, tmp);
4588
4589 }
4590 cik_srbm_select(rdev, 0, 0, 0, 0);
4591 mutex_unlock(&rdev->srbm_mutex);
4592
4593 /* init the queues. Just two for now. */
4594 for (i = 0; i < 2; i++) {
4595 if (i == 0)
4596 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4597 else
4598 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4599
4600 if (rdev->ring[idx].mqd_obj == NULL) {
4601 r = radeon_bo_create(rdev,
4602 sizeof(struct bonaire_mqd),
4603 PAGE_SIZE, true,
4604 RADEON_GEM_DOMAIN_GTT, 0, NULL,
4605 NULL, &rdev->ring[idx].mqd_obj);
4606 if (r) {
4607 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4608 return r;
4609 }
4610 }
4611
4612 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4613 if (unlikely(r != 0)) {
4614 cik_cp_compute_fini(rdev);
4615 return r;
4616 }
4617 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4618 &mqd_gpu_addr);
4619 if (r) {
4620 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4621 cik_cp_compute_fini(rdev);
4622 return r;
4623 }
4624 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4625 if (r) {
4626 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4627 cik_cp_compute_fini(rdev);
4628 return r;
4629 }
4630
4631 /* init the mqd struct */
4632 memset(buf, 0, sizeof(struct bonaire_mqd));
4633
4634 mqd = (struct bonaire_mqd *)buf;
4635 mqd->header = 0xC0310800;
4636 mqd->static_thread_mgmt01[0] = 0xffffffff;
4637 mqd->static_thread_mgmt01[1] = 0xffffffff;
4638 mqd->static_thread_mgmt23[0] = 0xffffffff;
4639 mqd->static_thread_mgmt23[1] = 0xffffffff;
4640
4641 mutex_lock(&rdev->srbm_mutex);
4642 cik_srbm_select(rdev, rdev->ring[idx].me,
4643 rdev->ring[idx].pipe,
4644 rdev->ring[idx].queue, 0);
4645
4646 /* disable wptr polling */
4647 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4648 tmp &= ~WPTR_POLL_EN;
4649 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4650
4651 /* enable doorbell? */
4652 mqd->queue_state.cp_hqd_pq_doorbell_control =
4653 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4654 if (use_doorbell)
4655 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4656 else
4657 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4658 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4659 mqd->queue_state.cp_hqd_pq_doorbell_control);
4660
4661 /* disable the queue if it's active */
4662 mqd->queue_state.cp_hqd_dequeue_request = 0;
4663 mqd->queue_state.cp_hqd_pq_rptr = 0;
4664 mqd->queue_state.cp_hqd_pq_wptr= 0;
4665 if (RREG32(CP_HQD_ACTIVE) & 1) {
4666 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4667 for (j = 0; j < rdev->usec_timeout; j++) {
4668 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4669 break;
4670 udelay(1);
4671 }
4672 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4673 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4674 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4675 }
4676
4677 /* set the pointer to the MQD */
4678 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4679 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4680 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4681 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4682 /* set MQD vmid to 0 */
4683 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4684 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4685 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4686
4687 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4688 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4689 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4690 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4691 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4692 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4693
4694 /* set up the HQD, this is similar to CP_RB0_CNTL */
4695 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4696 mqd->queue_state.cp_hqd_pq_control &=
4697 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4698
4699 mqd->queue_state.cp_hqd_pq_control |=
4700 order_base_2(rdev->ring[idx].ring_size / 8);
4701 mqd->queue_state.cp_hqd_pq_control |=
4702 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
4703 #ifdef __BIG_ENDIAN
4704 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4705 #endif
4706 mqd->queue_state.cp_hqd_pq_control &=
4707 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4708 mqd->queue_state.cp_hqd_pq_control |=
4709 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4710 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4711
4712 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4713 if (i == 0)
4714 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4715 else
4716 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4717 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4718 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4719 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4720 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4721 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4722
4723 /* set the wb address wether it's enabled or not */
4724 if (i == 0)
4725 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4726 else
4727 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4728 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4729 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4730 upper_32_bits(wb_gpu_addr) & 0xffff;
4731 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4732 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4733 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4734 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4735
4736 /* enable the doorbell if requested */
4737 if (use_doorbell) {
4738 mqd->queue_state.cp_hqd_pq_doorbell_control =
4739 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4740 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
4741 mqd->queue_state.cp_hqd_pq_doorbell_control |=
4742 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
4743 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4744 mqd->queue_state.cp_hqd_pq_doorbell_control &=
4745 ~(DOORBELL_SOURCE | DOORBELL_HIT);
4746
4747 } else {
4748 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
4749 }
4750 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4751 mqd->queue_state.cp_hqd_pq_doorbell_control);
4752
4753 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4754 rdev->ring[idx].wptr = 0;
4755 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
4756 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4757 mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR);
4758
4759 /* set the vmid for the queue */
4760 mqd->queue_state.cp_hqd_vmid = 0;
4761 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
4762
4763 /* activate the queue */
4764 mqd->queue_state.cp_hqd_active = 1;
4765 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
4766
4767 cik_srbm_select(rdev, 0, 0, 0, 0);
4768 mutex_unlock(&rdev->srbm_mutex);
4769
4770 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
4771 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4772
4773 rdev->ring[idx].ready = true;
4774 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
4775 if (r)
4776 rdev->ring[idx].ready = false;
4777 }
4778
4779 return 0;
4780 }
4781
4782 static void cik_cp_enable(struct radeon_device *rdev, bool enable)
4783 {
4784 cik_cp_gfx_enable(rdev, enable);
4785 cik_cp_compute_enable(rdev, enable);
4786 }
4787
4788 static int cik_cp_load_microcode(struct radeon_device *rdev)
4789 {
4790 int r;
4791
4792 r = cik_cp_gfx_load_microcode(rdev);
4793 if (r)
4794 return r;
4795 r = cik_cp_compute_load_microcode(rdev);
4796 if (r)
4797 return r;
4798
4799 return 0;
4800 }
4801
4802 static void cik_cp_fini(struct radeon_device *rdev)
4803 {
4804 cik_cp_gfx_fini(rdev);
4805 cik_cp_compute_fini(rdev);
4806 }
4807
4808 static int cik_cp_resume(struct radeon_device *rdev)
4809 {
4810 int r;
4811
4812 cik_enable_gui_idle_interrupt(rdev, false);
4813
4814 r = cik_cp_load_microcode(rdev);
4815 if (r)
4816 return r;
4817
4818 r = cik_cp_gfx_resume(rdev);
4819 if (r)
4820 return r;
4821 r = cik_cp_compute_resume(rdev);
4822 if (r)
4823 return r;
4824
4825 cik_enable_gui_idle_interrupt(rdev, true);
4826
4827 return 0;
4828 }
4829
4830 static void cik_print_gpu_status_regs(struct radeon_device *rdev)
4831 {
4832 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
4833 RREG32(GRBM_STATUS));
4834 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
4835 RREG32(GRBM_STATUS2));
4836 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
4837 RREG32(GRBM_STATUS_SE0));
4838 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
4839 RREG32(GRBM_STATUS_SE1));
4840 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
4841 RREG32(GRBM_STATUS_SE2));
4842 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
4843 RREG32(GRBM_STATUS_SE3));
4844 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
4845 RREG32(SRBM_STATUS));
4846 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
4847 RREG32(SRBM_STATUS2));
4848 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
4849 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
4850 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
4851 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
4852 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
4853 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
4854 RREG32(CP_STALLED_STAT1));
4855 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
4856 RREG32(CP_STALLED_STAT2));
4857 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
4858 RREG32(CP_STALLED_STAT3));
4859 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
4860 RREG32(CP_CPF_BUSY_STAT));
4861 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
4862 RREG32(CP_CPF_STALLED_STAT1));
4863 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
4864 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
4865 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
4866 RREG32(CP_CPC_STALLED_STAT1));
4867 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
4868 }
4869
4870 /**
4871 * cik_gpu_check_soft_reset - check which blocks are busy
4872 *
4873 * @rdev: radeon_device pointer
4874 *
4875 * Check which blocks are busy and return the relevant reset
4876 * mask to be used by cik_gpu_soft_reset().
4877 * Returns a mask of the blocks to be reset.
4878 */
4879 u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
4880 {
4881 u32 reset_mask = 0;
4882 u32 tmp;
4883
4884 /* GRBM_STATUS */
4885 tmp = RREG32(GRBM_STATUS);
4886 if (tmp & (PA_BUSY | SC_BUSY |
4887 BCI_BUSY | SX_BUSY |
4888 TA_BUSY | VGT_BUSY |
4889 DB_BUSY | CB_BUSY |
4890 GDS_BUSY | SPI_BUSY |
4891 IA_BUSY | IA_BUSY_NO_DMA))
4892 reset_mask |= RADEON_RESET_GFX;
4893
4894 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
4895 reset_mask |= RADEON_RESET_CP;
4896
4897 /* GRBM_STATUS2 */
4898 tmp = RREG32(GRBM_STATUS2);
4899 if (tmp & RLC_BUSY)
4900 reset_mask |= RADEON_RESET_RLC;
4901
4902 /* SDMA0_STATUS_REG */
4903 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
4904 if (!(tmp & SDMA_IDLE))
4905 reset_mask |= RADEON_RESET_DMA;
4906
4907 /* SDMA1_STATUS_REG */
4908 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
4909 if (!(tmp & SDMA_IDLE))
4910 reset_mask |= RADEON_RESET_DMA1;
4911
4912 /* SRBM_STATUS2 */
4913 tmp = RREG32(SRBM_STATUS2);
4914 if (tmp & SDMA_BUSY)
4915 reset_mask |= RADEON_RESET_DMA;
4916
4917 if (tmp & SDMA1_BUSY)
4918 reset_mask |= RADEON_RESET_DMA1;
4919
4920 /* SRBM_STATUS */
4921 tmp = RREG32(SRBM_STATUS);
4922
4923 if (tmp & IH_BUSY)
4924 reset_mask |= RADEON_RESET_IH;
4925
4926 if (tmp & SEM_BUSY)
4927 reset_mask |= RADEON_RESET_SEM;
4928
4929 if (tmp & GRBM_RQ_PENDING)
4930 reset_mask |= RADEON_RESET_GRBM;
4931
4932 if (tmp & VMC_BUSY)
4933 reset_mask |= RADEON_RESET_VMC;
4934
4935 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
4936 MCC_BUSY | MCD_BUSY))
4937 reset_mask |= RADEON_RESET_MC;
4938
4939 if (evergreen_is_display_hung(rdev))
4940 reset_mask |= RADEON_RESET_DISPLAY;
4941
4942 /* Skip MC reset as it's mostly likely not hung, just busy */
4943 if (reset_mask & RADEON_RESET_MC) {
4944 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
4945 reset_mask &= ~RADEON_RESET_MC;
4946 }
4947
4948 return reset_mask;
4949 }
4950
4951 /**
4952 * cik_gpu_soft_reset - soft reset GPU
4953 *
4954 * @rdev: radeon_device pointer
4955 * @reset_mask: mask of which blocks to reset
4956 *
4957 * Soft reset the blocks specified in @reset_mask.
4958 */
4959 static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
4960 {
4961 struct evergreen_mc_save save;
4962 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4963 u32 tmp;
4964
4965 if (reset_mask == 0)
4966 return;
4967
4968 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
4969
4970 cik_print_gpu_status_regs(rdev);
4971 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4972 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4973 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4974 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
4975
4976 /* disable CG/PG */
4977 cik_fini_pg(rdev);
4978 cik_fini_cg(rdev);
4979
4980 /* stop the rlc */
4981 cik_rlc_stop(rdev);
4982
4983 /* Disable GFX parsing/prefetching */
4984 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4985
4986 /* Disable MEC parsing/prefetching */
4987 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
4988
4989 if (reset_mask & RADEON_RESET_DMA) {
4990 /* sdma0 */
4991 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
4992 tmp |= SDMA_HALT;
4993 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
4994 }
4995 if (reset_mask & RADEON_RESET_DMA1) {
4996 /* sdma1 */
4997 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
4998 tmp |= SDMA_HALT;
4999 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5000 }
5001
5002 evergreen_mc_stop(rdev, &save);
5003 if (evergreen_mc_wait_for_idle(rdev)) {
5004 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5005 }
5006
5007 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
5008 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
5009
5010 if (reset_mask & RADEON_RESET_CP) {
5011 grbm_soft_reset |= SOFT_RESET_CP;
5012
5013 srbm_soft_reset |= SOFT_RESET_GRBM;
5014 }
5015
5016 if (reset_mask & RADEON_RESET_DMA)
5017 srbm_soft_reset |= SOFT_RESET_SDMA;
5018
5019 if (reset_mask & RADEON_RESET_DMA1)
5020 srbm_soft_reset |= SOFT_RESET_SDMA1;
5021
5022 if (reset_mask & RADEON_RESET_DISPLAY)
5023 srbm_soft_reset |= SOFT_RESET_DC;
5024
5025 if (reset_mask & RADEON_RESET_RLC)
5026 grbm_soft_reset |= SOFT_RESET_RLC;
5027
5028 if (reset_mask & RADEON_RESET_SEM)
5029 srbm_soft_reset |= SOFT_RESET_SEM;
5030
5031 if (reset_mask & RADEON_RESET_IH)
5032 srbm_soft_reset |= SOFT_RESET_IH;
5033
5034 if (reset_mask & RADEON_RESET_GRBM)
5035 srbm_soft_reset |= SOFT_RESET_GRBM;
5036
5037 if (reset_mask & RADEON_RESET_VMC)
5038 srbm_soft_reset |= SOFT_RESET_VMC;
5039
5040 if (!(rdev->flags & RADEON_IS_IGP)) {
5041 if (reset_mask & RADEON_RESET_MC)
5042 srbm_soft_reset |= SOFT_RESET_MC;
5043 }
5044
5045 if (grbm_soft_reset) {
5046 tmp = RREG32(GRBM_SOFT_RESET);
5047 tmp |= grbm_soft_reset;
5048 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
5049 WREG32(GRBM_SOFT_RESET, tmp);
5050 tmp = RREG32(GRBM_SOFT_RESET);
5051
5052 udelay(50);
5053
5054 tmp &= ~grbm_soft_reset;
5055 WREG32(GRBM_SOFT_RESET, tmp);
5056 tmp = RREG32(GRBM_SOFT_RESET);
5057 }
5058
5059 if (srbm_soft_reset) {
5060 tmp = RREG32(SRBM_SOFT_RESET);
5061 tmp |= srbm_soft_reset;
5062 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
5063 WREG32(SRBM_SOFT_RESET, tmp);
5064 tmp = RREG32(SRBM_SOFT_RESET);
5065
5066 udelay(50);
5067
5068 tmp &= ~srbm_soft_reset;
5069 WREG32(SRBM_SOFT_RESET, tmp);
5070 tmp = RREG32(SRBM_SOFT_RESET);
5071 }
5072
5073 /* Wait a little for things to settle down */
5074 udelay(50);
5075
5076 evergreen_mc_resume(rdev, &save);
5077 udelay(50);
5078
5079 cik_print_gpu_status_regs(rdev);
5080 }
5081
5082 struct kv_reset_save_regs {
5083 u32 gmcon_reng_execute;
5084 u32 gmcon_misc;
5085 u32 gmcon_misc3;
5086 };
5087
5088 static void kv_save_regs_for_reset(struct radeon_device *rdev,
5089 struct kv_reset_save_regs *save)
5090 {
5091 save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE);
5092 save->gmcon_misc = RREG32(GMCON_MISC);
5093 save->gmcon_misc3 = RREG32(GMCON_MISC3);
5094
5095 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP);
5096 WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE |
5097 STCTRL_STUTTER_EN));
5098 }
5099
5100 static void kv_restore_regs_for_reset(struct radeon_device *rdev,
5101 struct kv_reset_save_regs *save)
5102 {
5103 int i;
5104
5105 WREG32(GMCON_PGFSM_WRITE, 0);
5106 WREG32(GMCON_PGFSM_CONFIG, 0x200010ff);
5107
5108 for (i = 0; i < 5; i++)
5109 WREG32(GMCON_PGFSM_WRITE, 0);
5110
5111 WREG32(GMCON_PGFSM_WRITE, 0);
5112 WREG32(GMCON_PGFSM_CONFIG, 0x300010ff);
5113
5114 for (i = 0; i < 5; i++)
5115 WREG32(GMCON_PGFSM_WRITE, 0);
5116
5117 WREG32(GMCON_PGFSM_WRITE, 0x210000);
5118 WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff);
5119
5120 for (i = 0; i < 5; i++)
5121 WREG32(GMCON_PGFSM_WRITE, 0);
5122
5123 WREG32(GMCON_PGFSM_WRITE, 0x21003);
5124 WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff);
5125
5126 for (i = 0; i < 5; i++)
5127 WREG32(GMCON_PGFSM_WRITE, 0);
5128
5129 WREG32(GMCON_PGFSM_WRITE, 0x2b00);
5130 WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff);
5131
5132 for (i = 0; i < 5; i++)
5133 WREG32(GMCON_PGFSM_WRITE, 0);
5134
5135 WREG32(GMCON_PGFSM_WRITE, 0);
5136 WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff);
5137
5138 for (i = 0; i < 5; i++)
5139 WREG32(GMCON_PGFSM_WRITE, 0);
5140
5141 WREG32(GMCON_PGFSM_WRITE, 0x420000);
5142 WREG32(GMCON_PGFSM_CONFIG, 0x100010ff);
5143
5144 for (i = 0; i < 5; i++)
5145 WREG32(GMCON_PGFSM_WRITE, 0);
5146
5147 WREG32(GMCON_PGFSM_WRITE, 0x120202);
5148 WREG32(GMCON_PGFSM_CONFIG, 0x500010ff);
5149
5150 for (i = 0; i < 5; i++)
5151 WREG32(GMCON_PGFSM_WRITE, 0);
5152
5153 WREG32(GMCON_PGFSM_WRITE, 0x3e3e36);
5154 WREG32(GMCON_PGFSM_CONFIG, 0x600010ff);
5155
5156 for (i = 0; i < 5; i++)
5157 WREG32(GMCON_PGFSM_WRITE, 0);
5158
5159 WREG32(GMCON_PGFSM_WRITE, 0x373f3e);
5160 WREG32(GMCON_PGFSM_CONFIG, 0x700010ff);
5161
5162 for (i = 0; i < 5; i++)
5163 WREG32(GMCON_PGFSM_WRITE, 0);
5164
5165 WREG32(GMCON_PGFSM_WRITE, 0x3e1332);
5166 WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff);
5167
5168 WREG32(GMCON_MISC3, save->gmcon_misc3);
5169 WREG32(GMCON_MISC, save->gmcon_misc);
5170 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute);
5171 }
5172
5173 static void cik_gpu_pci_config_reset(struct radeon_device *rdev)
5174 {
5175 struct evergreen_mc_save save;
5176 struct kv_reset_save_regs kv_save = { 0 };
5177 u32 tmp, i;
5178
5179 dev_info(rdev->dev, "GPU pci config reset\n");
5180
5181 /* disable dpm? */
5182
5183 /* disable cg/pg */
5184 cik_fini_pg(rdev);
5185 cik_fini_cg(rdev);
5186
5187 /* Disable GFX parsing/prefetching */
5188 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5189
5190 /* Disable MEC parsing/prefetching */
5191 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5192
5193 /* sdma0 */
5194 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5195 tmp |= SDMA_HALT;
5196 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5197 /* sdma1 */
5198 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5199 tmp |= SDMA_HALT;
5200 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5201 /* XXX other engines? */
5202
5203 /* halt the rlc, disable cp internal ints */
5204 cik_rlc_stop(rdev);
5205
5206 udelay(50);
5207
5208 /* disable mem access */
5209 evergreen_mc_stop(rdev, &save);
5210 if (evergreen_mc_wait_for_idle(rdev)) {
5211 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
5212 }
5213
5214 if (rdev->flags & RADEON_IS_IGP)
5215 kv_save_regs_for_reset(rdev, &kv_save);
5216
5217 /* disable BM */
5218 pci_clear_master(rdev->pdev);
5219 /* reset */
5220 radeon_pci_config_reset(rdev);
5221
5222 udelay(100);
5223
5224 /* wait for asic to come out of reset */
5225 for (i = 0; i < rdev->usec_timeout; i++) {
5226 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
5227 break;
5228 udelay(1);
5229 }
5230
5231 /* does asic init need to be run first??? */
5232 if (rdev->flags & RADEON_IS_IGP)
5233 kv_restore_regs_for_reset(rdev, &kv_save);
5234 }
5235
5236 /**
5237 * cik_asic_reset - soft reset GPU
5238 *
5239 * @rdev: radeon_device pointer
5240 * @hard: force hard reset
5241 *
5242 * Look up which blocks are hung and attempt
5243 * to reset them.
5244 * Returns 0 for success.
5245 */
5246 int cik_asic_reset(struct radeon_device *rdev, bool hard)
5247 {
5248 u32 reset_mask;
5249
5250 if (hard) {
5251 cik_gpu_pci_config_reset(rdev);
5252 return 0;
5253 }
5254
5255 reset_mask = cik_gpu_check_soft_reset(rdev);
5256
5257 if (reset_mask)
5258 r600_set_bios_scratch_engine_hung(rdev, true);
5259
5260 /* try soft reset */
5261 cik_gpu_soft_reset(rdev, reset_mask);
5262
5263 reset_mask = cik_gpu_check_soft_reset(rdev);
5264
5265 /* try pci config reset */
5266 if (reset_mask && radeon_hard_reset)
5267 cik_gpu_pci_config_reset(rdev);
5268
5269 reset_mask = cik_gpu_check_soft_reset(rdev);
5270
5271 if (!reset_mask)
5272 r600_set_bios_scratch_engine_hung(rdev, false);
5273
5274 return 0;
5275 }
5276
5277 /**
5278 * cik_gfx_is_lockup - check if the 3D engine is locked up
5279 *
5280 * @rdev: radeon_device pointer
5281 * @ring: radeon_ring structure holding ring information
5282 *
5283 * Check if the 3D engine is locked up (CIK).
5284 * Returns true if the engine is locked, false if not.
5285 */
5286 bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
5287 {
5288 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
5289
5290 if (!(reset_mask & (RADEON_RESET_GFX |
5291 RADEON_RESET_COMPUTE |
5292 RADEON_RESET_CP))) {
5293 radeon_ring_lockup_update(rdev, ring);
5294 return false;
5295 }
5296 return radeon_ring_test_lockup(rdev, ring);
5297 }
5298
5299 /* MC */
5300 /**
5301 * cik_mc_program - program the GPU memory controller
5302 *
5303 * @rdev: radeon_device pointer
5304 *
5305 * Set the location of vram, gart, and AGP in the GPU's
5306 * physical address space (CIK).
5307 */
5308 static void cik_mc_program(struct radeon_device *rdev)
5309 {
5310 struct evergreen_mc_save save;
5311 u32 tmp;
5312 int i, j;
5313
5314 /* Initialize HDP */
5315 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
5316 WREG32((0x2c14 + j), 0x00000000);
5317 WREG32((0x2c18 + j), 0x00000000);
5318 WREG32((0x2c1c + j), 0x00000000);
5319 WREG32((0x2c20 + j), 0x00000000);
5320 WREG32((0x2c24 + j), 0x00000000);
5321 }
5322 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
5323
5324 evergreen_mc_stop(rdev, &save);
5325 if (radeon_mc_wait_for_idle(rdev)) {
5326 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5327 }
5328 /* Lockout access through VGA aperture*/
5329 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
5330 /* Update configuration */
5331 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
5332 rdev->mc.vram_start >> 12);
5333 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
5334 rdev->mc.vram_end >> 12);
5335 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
5336 rdev->vram_scratch.gpu_addr >> 12);
5337 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
5338 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
5339 WREG32(MC_VM_FB_LOCATION, tmp);
5340 /* XXX double check these! */
5341 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
5342 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
5343 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
5344 WREG32(MC_VM_AGP_BASE, 0);
5345 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
5346 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
5347 if (radeon_mc_wait_for_idle(rdev)) {
5348 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5349 }
5350 evergreen_mc_resume(rdev, &save);
5351 /* we need to own VRAM, so turn off the VGA renderer here
5352 * to stop it overwriting our objects */
5353 rv515_vga_render_disable(rdev);
5354 }
5355
5356 /**
5357 * cik_mc_init - initialize the memory controller driver params
5358 *
5359 * @rdev: radeon_device pointer
5360 *
5361 * Look up the amount of vram, vram width, and decide how to place
5362 * vram and gart within the GPU's physical address space (CIK).
5363 * Returns 0 for success.
5364 */
5365 static int cik_mc_init(struct radeon_device *rdev)
5366 {
5367 u32 tmp;
5368 int chansize, numchan;
5369
5370 /* Get VRAM informations */
5371 rdev->mc.vram_is_ddr = true;
5372 tmp = RREG32(MC_ARB_RAMCFG);
5373 if (tmp & CHANSIZE_MASK) {
5374 chansize = 64;
5375 } else {
5376 chansize = 32;
5377 }
5378 tmp = RREG32(MC_SHARED_CHMAP);
5379 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
5380 case 0:
5381 default:
5382 numchan = 1;
5383 break;
5384 case 1:
5385 numchan = 2;
5386 break;
5387 case 2:
5388 numchan = 4;
5389 break;
5390 case 3:
5391 numchan = 8;
5392 break;
5393 case 4:
5394 numchan = 3;
5395 break;
5396 case 5:
5397 numchan = 6;
5398 break;
5399 case 6:
5400 numchan = 10;
5401 break;
5402 case 7:
5403 numchan = 12;
5404 break;
5405 case 8:
5406 numchan = 16;
5407 break;
5408 }
5409 rdev->mc.vram_width = numchan * chansize;
5410 /* Could aper size report 0 ? */
5411 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5412 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5413 /* size in MB on si */
5414 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5415 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5416 rdev->mc.visible_vram_size = rdev->mc.aper_size;
5417 si_vram_gtt_location(rdev, &rdev->mc);
5418 radeon_update_bandwidth_info(rdev);
5419
5420 return 0;
5421 }
5422
5423 /*
5424 * GART
5425 * VMID 0 is the physical GPU addresses as used by the kernel.
5426 * VMIDs 1-15 are used for userspace clients and are handled
5427 * by the radeon vm/hsa code.
5428 */
5429 /**
5430 * cik_pcie_gart_tlb_flush - gart tlb flush callback
5431 *
5432 * @rdev: radeon_device pointer
5433 *
5434 * Flush the TLB for the VMID 0 page table (CIK).
5435 */
5436 void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5437 {
5438 /* flush hdp cache */
5439 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5440
5441 /* bits 0-15 are the VM contexts0-15 */
5442 WREG32(VM_INVALIDATE_REQUEST, 0x1);
5443 }
5444
5445 /**
5446 * cik_pcie_gart_enable - gart enable
5447 *
5448 * @rdev: radeon_device pointer
5449 *
5450 * This sets up the TLBs, programs the page tables for VMID0,
5451 * sets up the hw for VMIDs 1-15 which are allocated on
5452 * demand, and sets up the global locations for the LDS, GDS,
5453 * and GPUVM for FSA64 clients (CIK).
5454 * Returns 0 for success, errors for failure.
5455 */
5456 static int cik_pcie_gart_enable(struct radeon_device *rdev)
5457 {
5458 int r, i;
5459
5460 if (rdev->gart.robj == NULL) {
5461 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5462 return -EINVAL;
5463 }
5464 r = radeon_gart_table_vram_pin(rdev);
5465 if (r)
5466 return r;
5467 /* Setup TLB control */
5468 WREG32(MC_VM_MX_L1_TLB_CNTL,
5469 (0xA << 7) |
5470 ENABLE_L1_TLB |
5471 ENABLE_L1_FRAGMENT_PROCESSING |
5472 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5473 ENABLE_ADVANCED_DRIVER_MODEL |
5474 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5475 /* Setup L2 cache */
5476 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5477 ENABLE_L2_FRAGMENT_PROCESSING |
5478 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5479 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5480 EFFECTIVE_L2_QUEUE_SIZE(7) |
5481 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5482 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5483 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5484 BANK_SELECT(4) |
5485 L2_CACHE_BIGK_FRAGMENT_SIZE(4));
5486 /* setup context0 */
5487 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5488 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5489 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5490 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5491 (u32)(rdev->dummy_page.addr >> 12));
5492 WREG32(VM_CONTEXT0_CNTL2, 0);
5493 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5494 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5495
5496 WREG32(0x15D4, 0);
5497 WREG32(0x15D8, 0);
5498 WREG32(0x15DC, 0);
5499
5500 /* restore context1-15 */
5501 /* set vm size, must be a multiple of 4 */
5502 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5503 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn - 1);
5504 for (i = 1; i < 16; i++) {
5505 if (i < 8)
5506 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5507 rdev->vm_manager.saved_table_addr[i]);
5508 else
5509 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5510 rdev->vm_manager.saved_table_addr[i]);
5511 }
5512
5513 /* enable context1-15 */
5514 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5515 (u32)(rdev->dummy_page.addr >> 12));
5516 WREG32(VM_CONTEXT1_CNTL2, 4);
5517 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
5518 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
5519 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5520 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5521 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5522 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5523 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5524 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5525 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5526 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5527 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5528 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5529 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5530 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
5531
5532 if (rdev->family == CHIP_KAVERI) {
5533 u32 tmp = RREG32(CHUB_CONTROL);
5534 tmp &= ~BYPASS_VM;
5535 WREG32(CHUB_CONTROL, tmp);
5536 }
5537
5538 /* XXX SH_MEM regs */
5539 /* where to put LDS, scratch, GPUVM in FSA64 space */
5540 mutex_lock(&rdev->srbm_mutex);
5541 for (i = 0; i < 16; i++) {
5542 cik_srbm_select(rdev, 0, 0, 0, i);
5543 /* CP and shaders */
5544 WREG32(SH_MEM_CONFIG, SH_MEM_CONFIG_GFX_DEFAULT);
5545 WREG32(SH_MEM_APE1_BASE, 1);
5546 WREG32(SH_MEM_APE1_LIMIT, 0);
5547 WREG32(SH_MEM_BASES, 0);
5548 /* SDMA GFX */
5549 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5550 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5551 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5552 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5553 /* XXX SDMA RLC - todo */
5554 }
5555 cik_srbm_select(rdev, 0, 0, 0, 0);
5556 mutex_unlock(&rdev->srbm_mutex);
5557
5558 cik_pcie_gart_tlb_flush(rdev);
5559 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5560 (unsigned)(rdev->mc.gtt_size >> 20),
5561 (unsigned long long)rdev->gart.table_addr);
5562 rdev->gart.ready = true;
5563 return 0;
5564 }
5565
5566 /**
5567 * cik_pcie_gart_disable - gart disable
5568 *
5569 * @rdev: radeon_device pointer
5570 *
5571 * This disables all VM page table (CIK).
5572 */
5573 static void cik_pcie_gart_disable(struct radeon_device *rdev)
5574 {
5575 unsigned i;
5576
5577 for (i = 1; i < 16; ++i) {
5578 uint32_t reg;
5579 if (i < 8)
5580 reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2);
5581 else
5582 reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2);
5583 rdev->vm_manager.saved_table_addr[i] = RREG32(reg);
5584 }
5585
5586 /* Disable all tables */
5587 WREG32(VM_CONTEXT0_CNTL, 0);
5588 WREG32(VM_CONTEXT1_CNTL, 0);
5589 /* Setup TLB control */
5590 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5591 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5592 /* Setup L2 cache */
5593 WREG32(VM_L2_CNTL,
5594 ENABLE_L2_FRAGMENT_PROCESSING |
5595 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5596 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5597 EFFECTIVE_L2_QUEUE_SIZE(7) |
5598 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5599 WREG32(VM_L2_CNTL2, 0);
5600 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5601 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5602 radeon_gart_table_vram_unpin(rdev);
5603 }
5604
5605 /**
5606 * cik_pcie_gart_fini - vm fini callback
5607 *
5608 * @rdev: radeon_device pointer
5609 *
5610 * Tears down the driver GART/VM setup (CIK).
5611 */
5612 static void cik_pcie_gart_fini(struct radeon_device *rdev)
5613 {
5614 cik_pcie_gart_disable(rdev);
5615 radeon_gart_table_vram_free(rdev);
5616 radeon_gart_fini(rdev);
5617 }
5618
5619 /* vm parser */
5620 /**
5621 * cik_ib_parse - vm ib_parse callback
5622 *
5623 * @rdev: radeon_device pointer
5624 * @ib: indirect buffer pointer
5625 *
5626 * CIK uses hw IB checking so this is a nop (CIK).
5627 */
5628 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5629 {
5630 return 0;
5631 }
5632
5633 /*
5634 * vm
5635 * VMID 0 is the physical GPU addresses as used by the kernel.
5636 * VMIDs 1-15 are used for userspace clients and are handled
5637 * by the radeon vm/hsa code.
5638 */
5639 /**
5640 * cik_vm_init - cik vm init callback
5641 *
5642 * @rdev: radeon_device pointer
5643 *
5644 * Inits cik specific vm parameters (number of VMs, base of vram for
5645 * VMIDs 1-15) (CIK).
5646 * Returns 0 for success.
5647 */
5648 int cik_vm_init(struct radeon_device *rdev)
5649 {
5650 /*
5651 * number of VMs
5652 * VMID 0 is reserved for System
5653 * radeon graphics/compute will use VMIDs 1-15
5654 */
5655 rdev->vm_manager.nvm = 16;
5656 /* base offset of vram pages */
5657 if (rdev->flags & RADEON_IS_IGP) {
5658 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5659 tmp <<= 22;
5660 rdev->vm_manager.vram_base_offset = tmp;
5661 } else
5662 rdev->vm_manager.vram_base_offset = 0;
5663
5664 return 0;
5665 }
5666
5667 /**
5668 * cik_vm_fini - cik vm fini callback
5669 *
5670 * @rdev: radeon_device pointer
5671 *
5672 * Tear down any asic specific VM setup (CIK).
5673 */
5674 void cik_vm_fini(struct radeon_device *rdev)
5675 {
5676 }
5677
5678 /**
5679 * cik_vm_decode_fault - print human readable fault info
5680 *
5681 * @rdev: radeon_device pointer
5682 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5683 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5684 *
5685 * Print human readable fault information (CIK).
5686 */
5687 static void cik_vm_decode_fault(struct radeon_device *rdev,
5688 u32 status, u32 addr, u32 mc_client)
5689 {
5690 u32 mc_id;
5691 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5692 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
5693 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5694 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
5695
5696 if (rdev->family == CHIP_HAWAII)
5697 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5698 else
5699 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5700
5701 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
5702 protections, vmid, addr,
5703 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
5704 block, mc_client, mc_id);
5705 }
5706
5707 /**
5708 * cik_vm_flush - cik vm flush using the CP
5709 *
5710 * @rdev: radeon_device pointer
5711 *
5712 * Update the page table base and flush the VM TLB
5713 * using the CP (CIK).
5714 */
5715 void cik_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
5716 unsigned vm_id, uint64_t pd_addr)
5717 {
5718 int usepfp = (ring->idx == RADEON_RING_TYPE_GFX_INDEX);
5719
5720 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5721 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5722 WRITE_DATA_DST_SEL(0)));
5723 if (vm_id < 8) {
5724 radeon_ring_write(ring,
5725 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2);
5726 } else {
5727 radeon_ring_write(ring,
5728 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2);
5729 }
5730 radeon_ring_write(ring, 0);
5731 radeon_ring_write(ring, pd_addr >> 12);
5732
5733 /* update SH_MEM_* regs */
5734 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5735 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5736 WRITE_DATA_DST_SEL(0)));
5737 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5738 radeon_ring_write(ring, 0);
5739 radeon_ring_write(ring, VMID(vm_id));
5740
5741 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
5742 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5743 WRITE_DATA_DST_SEL(0)));
5744 radeon_ring_write(ring, SH_MEM_BASES >> 2);
5745 radeon_ring_write(ring, 0);
5746
5747 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
5748 radeon_ring_write(ring, SH_MEM_CONFIG_GFX_DEFAULT); /* SH_MEM_CONFIG */
5749 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
5750 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
5751
5752 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5753 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5754 WRITE_DATA_DST_SEL(0)));
5755 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5756 radeon_ring_write(ring, 0);
5757 radeon_ring_write(ring, VMID(0));
5758
5759 /* HDP flush */
5760 cik_hdp_flush_cp_ring_emit(rdev, ring->idx);
5761
5762 /* bits 0-15 are the VM contexts0-15 */
5763 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5764 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5765 WRITE_DATA_DST_SEL(0)));
5766 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5767 radeon_ring_write(ring, 0);
5768 radeon_ring_write(ring, 1 << vm_id);
5769
5770 /* wait for the invalidate to complete */
5771 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
5772 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(0) | /* wait */
5773 WAIT_REG_MEM_FUNCTION(0) | /* always */
5774 WAIT_REG_MEM_ENGINE(0))); /* me */
5775 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5776 radeon_ring_write(ring, 0);
5777 radeon_ring_write(ring, 0); /* ref */
5778 radeon_ring_write(ring, 0); /* mask */
5779 radeon_ring_write(ring, 0x20); /* poll interval */
5780
5781 /* compute doesn't have PFP */
5782 if (usepfp) {
5783 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5784 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5785 radeon_ring_write(ring, 0x0);
5786 }
5787 }
5788
5789 /*
5790 * RLC
5791 * The RLC is a multi-purpose microengine that handles a
5792 * variety of functions, the most important of which is
5793 * the interrupt controller.
5794 */
5795 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
5796 bool enable)
5797 {
5798 u32 tmp = RREG32(CP_INT_CNTL_RING0);
5799
5800 if (enable)
5801 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5802 else
5803 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5804 WREG32(CP_INT_CNTL_RING0, tmp);
5805 }
5806
5807 static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
5808 {
5809 u32 tmp;
5810
5811 tmp = RREG32(RLC_LB_CNTL);
5812 if (enable)
5813 tmp |= LOAD_BALANCE_ENABLE;
5814 else
5815 tmp &= ~LOAD_BALANCE_ENABLE;
5816 WREG32(RLC_LB_CNTL, tmp);
5817 }
5818
5819 static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
5820 {
5821 u32 i, j, k;
5822 u32 mask;
5823
5824 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
5825 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
5826 cik_select_se_sh(rdev, i, j);
5827 for (k = 0; k < rdev->usec_timeout; k++) {
5828 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
5829 break;
5830 udelay(1);
5831 }
5832 }
5833 }
5834 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5835
5836 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
5837 for (k = 0; k < rdev->usec_timeout; k++) {
5838 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
5839 break;
5840 udelay(1);
5841 }
5842 }
5843
5844 static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
5845 {
5846 u32 tmp;
5847
5848 tmp = RREG32(RLC_CNTL);
5849 if (tmp != rlc)
5850 WREG32(RLC_CNTL, rlc);
5851 }
5852
5853 static u32 cik_halt_rlc(struct radeon_device *rdev)
5854 {
5855 u32 data, orig;
5856
5857 orig = data = RREG32(RLC_CNTL);
5858
5859 if (data & RLC_ENABLE) {
5860 u32 i;
5861
5862 data &= ~RLC_ENABLE;
5863 WREG32(RLC_CNTL, data);
5864
5865 for (i = 0; i < rdev->usec_timeout; i++) {
5866 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
5867 break;
5868 udelay(1);
5869 }
5870
5871 cik_wait_for_rlc_serdes(rdev);
5872 }
5873
5874 return orig;
5875 }
5876
5877 void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
5878 {
5879 u32 tmp, i, mask;
5880
5881 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
5882 WREG32(RLC_GPR_REG2, tmp);
5883
5884 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
5885 for (i = 0; i < rdev->usec_timeout; i++) {
5886 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
5887 break;
5888 udelay(1);
5889 }
5890
5891 for (i = 0; i < rdev->usec_timeout; i++) {
5892 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
5893 break;
5894 udelay(1);
5895 }
5896 }
5897
5898 void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
5899 {
5900 u32 tmp;
5901
5902 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
5903 WREG32(RLC_GPR_REG2, tmp);
5904 }
5905
5906 /**
5907 * cik_rlc_stop - stop the RLC ME
5908 *
5909 * @rdev: radeon_device pointer
5910 *
5911 * Halt the RLC ME (MicroEngine) (CIK).
5912 */
5913 static void cik_rlc_stop(struct radeon_device *rdev)
5914 {
5915 WREG32(RLC_CNTL, 0);
5916
5917 cik_enable_gui_idle_interrupt(rdev, false);
5918
5919 cik_wait_for_rlc_serdes(rdev);
5920 }
5921
5922 /**
5923 * cik_rlc_start - start the RLC ME
5924 *
5925 * @rdev: radeon_device pointer
5926 *
5927 * Unhalt the RLC ME (MicroEngine) (CIK).
5928 */
5929 static void cik_rlc_start(struct radeon_device *rdev)
5930 {
5931 WREG32(RLC_CNTL, RLC_ENABLE);
5932
5933 cik_enable_gui_idle_interrupt(rdev, true);
5934
5935 udelay(50);
5936 }
5937
5938 /**
5939 * cik_rlc_resume - setup the RLC hw
5940 *
5941 * @rdev: radeon_device pointer
5942 *
5943 * Initialize the RLC registers, load the ucode,
5944 * and start the RLC (CIK).
5945 * Returns 0 for success, -EINVAL if the ucode is not available.
5946 */
5947 static int cik_rlc_resume(struct radeon_device *rdev)
5948 {
5949 u32 i, size, tmp;
5950
5951 if (!rdev->rlc_fw)
5952 return -EINVAL;
5953
5954 cik_rlc_stop(rdev);
5955
5956 /* disable CG */
5957 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
5958 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
5959
5960 si_rlc_reset(rdev);
5961
5962 cik_init_pg(rdev);
5963
5964 cik_init_cg(rdev);
5965
5966 WREG32(RLC_LB_CNTR_INIT, 0);
5967 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
5968
5969 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5970 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5971 WREG32(RLC_LB_PARAMS, 0x00600408);
5972 WREG32(RLC_LB_CNTL, 0x80000004);
5973
5974 WREG32(RLC_MC_CNTL, 0);
5975 WREG32(RLC_UCODE_CNTL, 0);
5976
5977 if (rdev->new_fw) {
5978 const struct rlc_firmware_header_v1_0 *hdr =
5979 (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
5980 const __le32 *fw_data = (const __le32 *)
5981 (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
5982
5983 radeon_ucode_print_rlc_hdr(&hdr->header);
5984
5985 size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
5986 WREG32(RLC_GPM_UCODE_ADDR, 0);
5987 for (i = 0; i < size; i++)
5988 WREG32(RLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
5989 WREG32(RLC_GPM_UCODE_ADDR, le32_to_cpu(hdr->header.ucode_version));
5990 } else {
5991 const __be32 *fw_data;
5992
5993 switch (rdev->family) {
5994 case CHIP_BONAIRE:
5995 case CHIP_HAWAII:
5996 default:
5997 size = BONAIRE_RLC_UCODE_SIZE;
5998 break;
5999 case CHIP_KAVERI:
6000 size = KV_RLC_UCODE_SIZE;
6001 break;
6002 case CHIP_KABINI:
6003 size = KB_RLC_UCODE_SIZE;
6004 break;
6005 case CHIP_MULLINS:
6006 size = ML_RLC_UCODE_SIZE;
6007 break;
6008 }
6009
6010 fw_data = (const __be32 *)rdev->rlc_fw->data;
6011 WREG32(RLC_GPM_UCODE_ADDR, 0);
6012 for (i = 0; i < size; i++)
6013 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
6014 WREG32(RLC_GPM_UCODE_ADDR, 0);
6015 }
6016
6017 /* XXX - find out what chips support lbpw */
6018 cik_enable_lbpw(rdev, false);
6019
6020 if (rdev->family == CHIP_BONAIRE)
6021 WREG32(RLC_DRIVER_DMA_STATUS, 0);
6022
6023 cik_rlc_start(rdev);
6024
6025 return 0;
6026 }
6027
6028 static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
6029 {
6030 u32 data, orig, tmp, tmp2;
6031
6032 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
6033
6034 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
6035 cik_enable_gui_idle_interrupt(rdev, true);
6036
6037 tmp = cik_halt_rlc(rdev);
6038
6039 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6040 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6041 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6042 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
6043 WREG32(RLC_SERDES_WR_CTRL, tmp2);
6044
6045 cik_update_rlc(rdev, tmp);
6046
6047 data |= CGCG_EN | CGLS_EN;
6048 } else {
6049 cik_enable_gui_idle_interrupt(rdev, false);
6050
6051 RREG32(CB_CGTT_SCLK_CTRL);
6052 RREG32(CB_CGTT_SCLK_CTRL);
6053 RREG32(CB_CGTT_SCLK_CTRL);
6054 RREG32(CB_CGTT_SCLK_CTRL);
6055
6056 data &= ~(CGCG_EN | CGLS_EN);
6057 }
6058
6059 if (orig != data)
6060 WREG32(RLC_CGCG_CGLS_CTRL, data);
6061
6062 }
6063
6064 static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
6065 {
6066 u32 data, orig, tmp = 0;
6067
6068 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
6069 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
6070 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
6071 orig = data = RREG32(CP_MEM_SLP_CNTL);
6072 data |= CP_MEM_LS_EN;
6073 if (orig != data)
6074 WREG32(CP_MEM_SLP_CNTL, data);
6075 }
6076 }
6077
6078 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
6079 data |= 0x00000001;
6080 data &= 0xfffffffd;
6081 if (orig != data)
6082 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6083
6084 tmp = cik_halt_rlc(rdev);
6085
6086 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6087 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6088 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6089 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
6090 WREG32(RLC_SERDES_WR_CTRL, data);
6091
6092 cik_update_rlc(rdev, tmp);
6093
6094 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
6095 orig = data = RREG32(CGTS_SM_CTRL_REG);
6096 data &= ~SM_MODE_MASK;
6097 data |= SM_MODE(0x2);
6098 data |= SM_MODE_ENABLE;
6099 data &= ~CGTS_OVERRIDE;
6100 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
6101 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
6102 data &= ~CGTS_LS_OVERRIDE;
6103 data &= ~ON_MONITOR_ADD_MASK;
6104 data |= ON_MONITOR_ADD_EN;
6105 data |= ON_MONITOR_ADD(0x96);
6106 if (orig != data)
6107 WREG32(CGTS_SM_CTRL_REG, data);
6108 }
6109 } else {
6110 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
6111 data |= 0x00000003;
6112 if (orig != data)
6113 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6114
6115 data = RREG32(RLC_MEM_SLP_CNTL);
6116 if (data & RLC_MEM_LS_EN) {
6117 data &= ~RLC_MEM_LS_EN;
6118 WREG32(RLC_MEM_SLP_CNTL, data);
6119 }
6120
6121 data = RREG32(CP_MEM_SLP_CNTL);
6122 if (data & CP_MEM_LS_EN) {
6123 data &= ~CP_MEM_LS_EN;
6124 WREG32(CP_MEM_SLP_CNTL, data);
6125 }
6126
6127 orig = data = RREG32(CGTS_SM_CTRL_REG);
6128 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
6129 if (orig != data)
6130 WREG32(CGTS_SM_CTRL_REG, data);
6131
6132 tmp = cik_halt_rlc(rdev);
6133
6134 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6135 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6136 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6137 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
6138 WREG32(RLC_SERDES_WR_CTRL, data);
6139
6140 cik_update_rlc(rdev, tmp);
6141 }
6142 }
6143
6144 static const u32 mc_cg_registers[] =
6145 {
6146 MC_HUB_MISC_HUB_CG,
6147 MC_HUB_MISC_SIP_CG,
6148 MC_HUB_MISC_VM_CG,
6149 MC_XPB_CLK_GAT,
6150 ATC_MISC_CG,
6151 MC_CITF_MISC_WR_CG,
6152 MC_CITF_MISC_RD_CG,
6153 MC_CITF_MISC_VM_CG,
6154 VM_L2_CG,
6155 };
6156
6157 static void cik_enable_mc_ls(struct radeon_device *rdev,
6158 bool enable)
6159 {
6160 int i;
6161 u32 orig, data;
6162
6163 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6164 orig = data = RREG32(mc_cg_registers[i]);
6165 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
6166 data |= MC_LS_ENABLE;
6167 else
6168 data &= ~MC_LS_ENABLE;
6169 if (data != orig)
6170 WREG32(mc_cg_registers[i], data);
6171 }
6172 }
6173
6174 static void cik_enable_mc_mgcg(struct radeon_device *rdev,
6175 bool enable)
6176 {
6177 int i;
6178 u32 orig, data;
6179
6180 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6181 orig = data = RREG32(mc_cg_registers[i]);
6182 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
6183 data |= MC_CG_ENABLE;
6184 else
6185 data &= ~MC_CG_ENABLE;
6186 if (data != orig)
6187 WREG32(mc_cg_registers[i], data);
6188 }
6189 }
6190
6191 static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
6192 bool enable)
6193 {
6194 u32 orig, data;
6195
6196 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
6197 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
6198 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
6199 } else {
6200 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
6201 data |= 0xff000000;
6202 if (data != orig)
6203 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
6204
6205 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
6206 data |= 0xff000000;
6207 if (data != orig)
6208 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
6209 }
6210 }
6211
6212 static void cik_enable_sdma_mgls(struct radeon_device *rdev,
6213 bool enable)
6214 {
6215 u32 orig, data;
6216
6217 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
6218 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6219 data |= 0x100;
6220 if (orig != data)
6221 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6222
6223 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6224 data |= 0x100;
6225 if (orig != data)
6226 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6227 } else {
6228 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6229 data &= ~0x100;
6230 if (orig != data)
6231 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6232
6233 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6234 data &= ~0x100;
6235 if (orig != data)
6236 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6237 }
6238 }
6239
6240 static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
6241 bool enable)
6242 {
6243 u32 orig, data;
6244
6245 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
6246 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6247 data = 0xfff;
6248 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6249
6250 orig = data = RREG32(UVD_CGC_CTRL);
6251 data |= DCM;
6252 if (orig != data)
6253 WREG32(UVD_CGC_CTRL, data);
6254 } else {
6255 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6256 data &= ~0xfff;
6257 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6258
6259 orig = data = RREG32(UVD_CGC_CTRL);
6260 data &= ~DCM;
6261 if (orig != data)
6262 WREG32(UVD_CGC_CTRL, data);
6263 }
6264 }
6265
6266 static void cik_enable_bif_mgls(struct radeon_device *rdev,
6267 bool enable)
6268 {
6269 u32 orig, data;
6270
6271 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
6272
6273 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
6274 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
6275 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
6276 else
6277 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
6278 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
6279
6280 if (orig != data)
6281 WREG32_PCIE_PORT(PCIE_CNTL2, data);
6282 }
6283
6284 static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
6285 bool enable)
6286 {
6287 u32 orig, data;
6288
6289 orig = data = RREG32(HDP_HOST_PATH_CNTL);
6290
6291 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
6292 data &= ~CLOCK_GATING_DIS;
6293 else
6294 data |= CLOCK_GATING_DIS;
6295
6296 if (orig != data)
6297 WREG32(HDP_HOST_PATH_CNTL, data);
6298 }
6299
6300 static void cik_enable_hdp_ls(struct radeon_device *rdev,
6301 bool enable)
6302 {
6303 u32 orig, data;
6304
6305 orig = data = RREG32(HDP_MEM_POWER_LS);
6306
6307 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
6308 data |= HDP_LS_ENABLE;
6309 else
6310 data &= ~HDP_LS_ENABLE;
6311
6312 if (orig != data)
6313 WREG32(HDP_MEM_POWER_LS, data);
6314 }
6315
6316 void cik_update_cg(struct radeon_device *rdev,
6317 u32 block, bool enable)
6318 {
6319
6320 if (block & RADEON_CG_BLOCK_GFX) {
6321 cik_enable_gui_idle_interrupt(rdev, false);
6322 /* order matters! */
6323 if (enable) {
6324 cik_enable_mgcg(rdev, true);
6325 cik_enable_cgcg(rdev, true);
6326 } else {
6327 cik_enable_cgcg(rdev, false);
6328 cik_enable_mgcg(rdev, false);
6329 }
6330 cik_enable_gui_idle_interrupt(rdev, true);
6331 }
6332
6333 if (block & RADEON_CG_BLOCK_MC) {
6334 if (!(rdev->flags & RADEON_IS_IGP)) {
6335 cik_enable_mc_mgcg(rdev, enable);
6336 cik_enable_mc_ls(rdev, enable);
6337 }
6338 }
6339
6340 if (block & RADEON_CG_BLOCK_SDMA) {
6341 cik_enable_sdma_mgcg(rdev, enable);
6342 cik_enable_sdma_mgls(rdev, enable);
6343 }
6344
6345 if (block & RADEON_CG_BLOCK_BIF) {
6346 cik_enable_bif_mgls(rdev, enable);
6347 }
6348
6349 if (block & RADEON_CG_BLOCK_UVD) {
6350 if (rdev->has_uvd)
6351 cik_enable_uvd_mgcg(rdev, enable);
6352 }
6353
6354 if (block & RADEON_CG_BLOCK_HDP) {
6355 cik_enable_hdp_mgcg(rdev, enable);
6356 cik_enable_hdp_ls(rdev, enable);
6357 }
6358
6359 if (block & RADEON_CG_BLOCK_VCE) {
6360 vce_v2_0_enable_mgcg(rdev, enable);
6361 }
6362 }
6363
6364 static void cik_init_cg(struct radeon_device *rdev)
6365 {
6366
6367 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
6368
6369 if (rdev->has_uvd)
6370 si_init_uvd_internal_cg(rdev);
6371
6372 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6373 RADEON_CG_BLOCK_SDMA |
6374 RADEON_CG_BLOCK_BIF |
6375 RADEON_CG_BLOCK_UVD |
6376 RADEON_CG_BLOCK_HDP), true);
6377 }
6378
6379 static void cik_fini_cg(struct radeon_device *rdev)
6380 {
6381 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6382 RADEON_CG_BLOCK_SDMA |
6383 RADEON_CG_BLOCK_BIF |
6384 RADEON_CG_BLOCK_UVD |
6385 RADEON_CG_BLOCK_HDP), false);
6386
6387 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
6388 }
6389
6390 static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
6391 bool enable)
6392 {
6393 u32 data, orig;
6394
6395 orig = data = RREG32(RLC_PG_CNTL);
6396 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6397 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6398 else
6399 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6400 if (orig != data)
6401 WREG32(RLC_PG_CNTL, data);
6402 }
6403
6404 static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
6405 bool enable)
6406 {
6407 u32 data, orig;
6408
6409 orig = data = RREG32(RLC_PG_CNTL);
6410 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6411 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6412 else
6413 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6414 if (orig != data)
6415 WREG32(RLC_PG_CNTL, data);
6416 }
6417
6418 static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
6419 {
6420 u32 data, orig;
6421
6422 orig = data = RREG32(RLC_PG_CNTL);
6423 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
6424 data &= ~DISABLE_CP_PG;
6425 else
6426 data |= DISABLE_CP_PG;
6427 if (orig != data)
6428 WREG32(RLC_PG_CNTL, data);
6429 }
6430
6431 static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
6432 {
6433 u32 data, orig;
6434
6435 orig = data = RREG32(RLC_PG_CNTL);
6436 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
6437 data &= ~DISABLE_GDS_PG;
6438 else
6439 data |= DISABLE_GDS_PG;
6440 if (orig != data)
6441 WREG32(RLC_PG_CNTL, data);
6442 }
6443
6444 #define CP_ME_TABLE_SIZE 96
6445 #define CP_ME_TABLE_OFFSET 2048
6446 #define CP_MEC_TABLE_OFFSET 4096
6447
6448 void cik_init_cp_pg_table(struct radeon_device *rdev)
6449 {
6450 volatile u32 *dst_ptr;
6451 int me, i, max_me = 4;
6452 u32 bo_offset = 0;
6453 u32 table_offset, table_size;
6454
6455 if (rdev->family == CHIP_KAVERI)
6456 max_me = 5;
6457
6458 if (rdev->rlc.cp_table_ptr == NULL)
6459 return;
6460
6461 /* write the cp table buffer */
6462 dst_ptr = rdev->rlc.cp_table_ptr;
6463 for (me = 0; me < max_me; me++) {
6464 if (rdev->new_fw) {
6465 const __le32 *fw_data;
6466 const struct gfx_firmware_header_v1_0 *hdr;
6467
6468 if (me == 0) {
6469 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
6470 fw_data = (const __le32 *)
6471 (rdev->ce_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6472 table_offset = le32_to_cpu(hdr->jt_offset);
6473 table_size = le32_to_cpu(hdr->jt_size);
6474 } else if (me == 1) {
6475 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
6476 fw_data = (const __le32 *)
6477 (rdev->pfp_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6478 table_offset = le32_to_cpu(hdr->jt_offset);
6479 table_size = le32_to_cpu(hdr->jt_size);
6480 } else if (me == 2) {
6481 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
6482 fw_data = (const __le32 *)
6483 (rdev->me_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6484 table_offset = le32_to_cpu(hdr->jt_offset);
6485 table_size = le32_to_cpu(hdr->jt_size);
6486 } else if (me == 3) {
6487 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
6488 fw_data = (const __le32 *)
6489 (rdev->mec_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6490 table_offset = le32_to_cpu(hdr->jt_offset);
6491 table_size = le32_to_cpu(hdr->jt_size);
6492 } else {
6493 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
6494 fw_data = (const __le32 *)
6495 (rdev->mec2_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6496 table_offset = le32_to_cpu(hdr->jt_offset);
6497 table_size = le32_to_cpu(hdr->jt_size);
6498 }
6499
6500 for (i = 0; i < table_size; i ++) {
6501 dst_ptr[bo_offset + i] =
6502 cpu_to_le32(le32_to_cpu(fw_data[table_offset + i]));
6503 }
6504 bo_offset += table_size;
6505 } else {
6506 const __be32 *fw_data;
6507 table_size = CP_ME_TABLE_SIZE;
6508
6509 if (me == 0) {
6510 fw_data = (const __be32 *)rdev->ce_fw->data;
6511 table_offset = CP_ME_TABLE_OFFSET;
6512 } else if (me == 1) {
6513 fw_data = (const __be32 *)rdev->pfp_fw->data;
6514 table_offset = CP_ME_TABLE_OFFSET;
6515 } else if (me == 2) {
6516 fw_data = (const __be32 *)rdev->me_fw->data;
6517 table_offset = CP_ME_TABLE_OFFSET;
6518 } else {
6519 fw_data = (const __be32 *)rdev->mec_fw->data;
6520 table_offset = CP_MEC_TABLE_OFFSET;
6521 }
6522
6523 for (i = 0; i < table_size; i ++) {
6524 dst_ptr[bo_offset + i] =
6525 cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
6526 }
6527 bo_offset += table_size;
6528 }
6529 }
6530 }
6531
6532 static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6533 bool enable)
6534 {
6535 u32 data, orig;
6536
6537 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
6538 orig = data = RREG32(RLC_PG_CNTL);
6539 data |= GFX_PG_ENABLE;
6540 if (orig != data)
6541 WREG32(RLC_PG_CNTL, data);
6542
6543 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6544 data |= AUTO_PG_EN;
6545 if (orig != data)
6546 WREG32(RLC_AUTO_PG_CTRL, data);
6547 } else {
6548 orig = data = RREG32(RLC_PG_CNTL);
6549 data &= ~GFX_PG_ENABLE;
6550 if (orig != data)
6551 WREG32(RLC_PG_CNTL, data);
6552
6553 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6554 data &= ~AUTO_PG_EN;
6555 if (orig != data)
6556 WREG32(RLC_AUTO_PG_CTRL, data);
6557
6558 data = RREG32(DB_RENDER_CONTROL);
6559 }
6560 }
6561
6562 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6563 {
6564 u32 mask = 0, tmp, tmp1;
6565 int i;
6566
6567 cik_select_se_sh(rdev, se, sh);
6568 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6569 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6570 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6571
6572 tmp &= 0xffff0000;
6573
6574 tmp |= tmp1;
6575 tmp >>= 16;
6576
6577 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6578 mask <<= 1;
6579 mask |= 1;
6580 }
6581
6582 return (~tmp) & mask;
6583 }
6584
6585 static void cik_init_ao_cu_mask(struct radeon_device *rdev)
6586 {
6587 u32 i, j, k, active_cu_number = 0;
6588 u32 mask, counter, cu_bitmap;
6589 u32 tmp = 0;
6590
6591 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6592 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6593 mask = 1;
6594 cu_bitmap = 0;
6595 counter = 0;
6596 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6597 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6598 if (counter < 2)
6599 cu_bitmap |= mask;
6600 counter ++;
6601 }
6602 mask <<= 1;
6603 }
6604
6605 active_cu_number += counter;
6606 tmp |= (cu_bitmap << (i * 16 + j * 8));
6607 }
6608 }
6609
6610 WREG32(RLC_PG_AO_CU_MASK, tmp);
6611
6612 tmp = RREG32(RLC_MAX_PG_CU);
6613 tmp &= ~MAX_PU_CU_MASK;
6614 tmp |= MAX_PU_CU(active_cu_number);
6615 WREG32(RLC_MAX_PG_CU, tmp);
6616 }
6617
6618 static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6619 bool enable)
6620 {
6621 u32 data, orig;
6622
6623 orig = data = RREG32(RLC_PG_CNTL);
6624 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
6625 data |= STATIC_PER_CU_PG_ENABLE;
6626 else
6627 data &= ~STATIC_PER_CU_PG_ENABLE;
6628 if (orig != data)
6629 WREG32(RLC_PG_CNTL, data);
6630 }
6631
6632 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6633 bool enable)
6634 {
6635 u32 data, orig;
6636
6637 orig = data = RREG32(RLC_PG_CNTL);
6638 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
6639 data |= DYN_PER_CU_PG_ENABLE;
6640 else
6641 data &= ~DYN_PER_CU_PG_ENABLE;
6642 if (orig != data)
6643 WREG32(RLC_PG_CNTL, data);
6644 }
6645
6646 #define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6647 #define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
6648
6649 static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6650 {
6651 u32 data, orig;
6652 u32 i;
6653
6654 if (rdev->rlc.cs_data) {
6655 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6656 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
6657 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
6658 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
6659 } else {
6660 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6661 for (i = 0; i < 3; i++)
6662 WREG32(RLC_GPM_SCRATCH_DATA, 0);
6663 }
6664 if (rdev->rlc.reg_list) {
6665 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6666 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6667 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
6668 }
6669
6670 orig = data = RREG32(RLC_PG_CNTL);
6671 data |= GFX_PG_SRC;
6672 if (orig != data)
6673 WREG32(RLC_PG_CNTL, data);
6674
6675 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6676 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
6677
6678 data = RREG32(CP_RB_WPTR_POLL_CNTL);
6679 data &= ~IDLE_POLL_COUNT_MASK;
6680 data |= IDLE_POLL_COUNT(0x60);
6681 WREG32(CP_RB_WPTR_POLL_CNTL, data);
6682
6683 data = 0x10101010;
6684 WREG32(RLC_PG_DELAY, data);
6685
6686 data = RREG32(RLC_PG_DELAY_2);
6687 data &= ~0xff;
6688 data |= 0x3;
6689 WREG32(RLC_PG_DELAY_2, data);
6690
6691 data = RREG32(RLC_AUTO_PG_CTRL);
6692 data &= ~GRBM_REG_SGIT_MASK;
6693 data |= GRBM_REG_SGIT(0x700);
6694 WREG32(RLC_AUTO_PG_CTRL, data);
6695
6696 }
6697
6698 static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
6699 {
6700 cik_enable_gfx_cgpg(rdev, enable);
6701 cik_enable_gfx_static_mgpg(rdev, enable);
6702 cik_enable_gfx_dynamic_mgpg(rdev, enable);
6703 }
6704
6705 u32 cik_get_csb_size(struct radeon_device *rdev)
6706 {
6707 u32 count = 0;
6708 const struct cs_section_def *sect = NULL;
6709 const struct cs_extent_def *ext = NULL;
6710
6711 if (rdev->rlc.cs_data == NULL)
6712 return 0;
6713
6714 /* begin clear state */
6715 count += 2;
6716 /* context control state */
6717 count += 3;
6718
6719 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6720 for (ext = sect->section; ext->extent != NULL; ++ext) {
6721 if (sect->id == SECT_CONTEXT)
6722 count += 2 + ext->reg_count;
6723 else
6724 return 0;
6725 }
6726 }
6727 /* pa_sc_raster_config/pa_sc_raster_config1 */
6728 count += 4;
6729 /* end clear state */
6730 count += 2;
6731 /* clear state */
6732 count += 2;
6733
6734 return count;
6735 }
6736
6737 void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
6738 {
6739 u32 count = 0, i;
6740 const struct cs_section_def *sect = NULL;
6741 const struct cs_extent_def *ext = NULL;
6742
6743 if (rdev->rlc.cs_data == NULL)
6744 return;
6745 if (buffer == NULL)
6746 return;
6747
6748 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6749 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
6750
6751 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6752 buffer[count++] = cpu_to_le32(0x80000000);
6753 buffer[count++] = cpu_to_le32(0x80000000);
6754
6755 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6756 for (ext = sect->section; ext->extent != NULL; ++ext) {
6757 if (sect->id == SECT_CONTEXT) {
6758 buffer[count++] =
6759 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
6760 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
6761 for (i = 0; i < ext->reg_count; i++)
6762 buffer[count++] = cpu_to_le32(ext->extent[i]);
6763 } else {
6764 return;
6765 }
6766 }
6767 }
6768
6769 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
6770 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
6771 switch (rdev->family) {
6772 case CHIP_BONAIRE:
6773 buffer[count++] = cpu_to_le32(0x16000012);
6774 buffer[count++] = cpu_to_le32(0x00000000);
6775 break;
6776 case CHIP_KAVERI:
6777 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6778 buffer[count++] = cpu_to_le32(0x00000000);
6779 break;
6780 case CHIP_KABINI:
6781 case CHIP_MULLINS:
6782 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6783 buffer[count++] = cpu_to_le32(0x00000000);
6784 break;
6785 case CHIP_HAWAII:
6786 buffer[count++] = cpu_to_le32(0x3a00161a);
6787 buffer[count++] = cpu_to_le32(0x0000002e);
6788 break;
6789 default:
6790 buffer[count++] = cpu_to_le32(0x00000000);
6791 buffer[count++] = cpu_to_le32(0x00000000);
6792 break;
6793 }
6794
6795 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6796 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
6797
6798 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
6799 buffer[count++] = cpu_to_le32(0);
6800 }
6801
6802 static void cik_init_pg(struct radeon_device *rdev)
6803 {
6804 if (rdev->pg_flags) {
6805 cik_enable_sck_slowdown_on_pu(rdev, true);
6806 cik_enable_sck_slowdown_on_pd(rdev, true);
6807 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
6808 cik_init_gfx_cgpg(rdev);
6809 cik_enable_cp_pg(rdev, true);
6810 cik_enable_gds_pg(rdev, true);
6811 }
6812 cik_init_ao_cu_mask(rdev);
6813 cik_update_gfx_pg(rdev, true);
6814 }
6815 }
6816
6817 static void cik_fini_pg(struct radeon_device *rdev)
6818 {
6819 if (rdev->pg_flags) {
6820 cik_update_gfx_pg(rdev, false);
6821 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
6822 cik_enable_cp_pg(rdev, false);
6823 cik_enable_gds_pg(rdev, false);
6824 }
6825 }
6826 }
6827
6828 /*
6829 * Interrupts
6830 * Starting with r6xx, interrupts are handled via a ring buffer.
6831 * Ring buffers are areas of GPU accessible memory that the GPU
6832 * writes interrupt vectors into and the host reads vectors out of.
6833 * There is a rptr (read pointer) that determines where the
6834 * host is currently reading, and a wptr (write pointer)
6835 * which determines where the GPU has written. When the
6836 * pointers are equal, the ring is idle. When the GPU
6837 * writes vectors to the ring buffer, it increments the
6838 * wptr. When there is an interrupt, the host then starts
6839 * fetching commands and processing them until the pointers are
6840 * equal again at which point it updates the rptr.
6841 */
6842
6843 /**
6844 * cik_enable_interrupts - Enable the interrupt ring buffer
6845 *
6846 * @rdev: radeon_device pointer
6847 *
6848 * Enable the interrupt ring buffer (CIK).
6849 */
6850 static void cik_enable_interrupts(struct radeon_device *rdev)
6851 {
6852 u32 ih_cntl = RREG32(IH_CNTL);
6853 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6854
6855 ih_cntl |= ENABLE_INTR;
6856 ih_rb_cntl |= IH_RB_ENABLE;
6857 WREG32(IH_CNTL, ih_cntl);
6858 WREG32(IH_RB_CNTL, ih_rb_cntl);
6859 rdev->ih.enabled = true;
6860 }
6861
6862 /**
6863 * cik_disable_interrupts - Disable the interrupt ring buffer
6864 *
6865 * @rdev: radeon_device pointer
6866 *
6867 * Disable the interrupt ring buffer (CIK).
6868 */
6869 static void cik_disable_interrupts(struct radeon_device *rdev)
6870 {
6871 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6872 u32 ih_cntl = RREG32(IH_CNTL);
6873
6874 ih_rb_cntl &= ~IH_RB_ENABLE;
6875 ih_cntl &= ~ENABLE_INTR;
6876 WREG32(IH_RB_CNTL, ih_rb_cntl);
6877 WREG32(IH_CNTL, ih_cntl);
6878 /* set rptr, wptr to 0 */
6879 WREG32(IH_RB_RPTR, 0);
6880 WREG32(IH_RB_WPTR, 0);
6881 rdev->ih.enabled = false;
6882 rdev->ih.rptr = 0;
6883 }
6884
6885 /**
6886 * cik_disable_interrupt_state - Disable all interrupt sources
6887 *
6888 * @rdev: radeon_device pointer
6889 *
6890 * Clear all interrupt enable bits used by the driver (CIK).
6891 */
6892 static void cik_disable_interrupt_state(struct radeon_device *rdev)
6893 {
6894 u32 tmp;
6895
6896 /* gfx ring */
6897 tmp = RREG32(CP_INT_CNTL_RING0) &
6898 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6899 WREG32(CP_INT_CNTL_RING0, tmp);
6900 /* sdma */
6901 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6902 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
6903 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6904 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
6905 /* compute queues */
6906 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
6907 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
6908 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
6909 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
6910 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
6911 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
6912 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
6913 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
6914 /* grbm */
6915 WREG32(GRBM_INT_CNTL, 0);
6916 /* SRBM */
6917 WREG32(SRBM_INT_CNTL, 0);
6918 /* vline/vblank, etc. */
6919 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6920 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6921 if (rdev->num_crtc >= 4) {
6922 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6923 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6924 }
6925 if (rdev->num_crtc >= 6) {
6926 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6927 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6928 }
6929 /* pflip */
6930 if (rdev->num_crtc >= 2) {
6931 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6932 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6933 }
6934 if (rdev->num_crtc >= 4) {
6935 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6936 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6937 }
6938 if (rdev->num_crtc >= 6) {
6939 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6940 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6941 }
6942
6943 /* dac hotplug */
6944 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
6945
6946 /* digital hotplug */
6947 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6948 WREG32(DC_HPD1_INT_CONTROL, tmp);
6949 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6950 WREG32(DC_HPD2_INT_CONTROL, tmp);
6951 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6952 WREG32(DC_HPD3_INT_CONTROL, tmp);
6953 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6954 WREG32(DC_HPD4_INT_CONTROL, tmp);
6955 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6956 WREG32(DC_HPD5_INT_CONTROL, tmp);
6957 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6958 WREG32(DC_HPD6_INT_CONTROL, tmp);
6959
6960 }
6961
6962 /**
6963 * cik_irq_init - init and enable the interrupt ring
6964 *
6965 * @rdev: radeon_device pointer
6966 *
6967 * Allocate a ring buffer for the interrupt controller,
6968 * enable the RLC, disable interrupts, enable the IH
6969 * ring buffer and enable it (CIK).
6970 * Called at device load and reume.
6971 * Returns 0 for success, errors for failure.
6972 */
6973 static int cik_irq_init(struct radeon_device *rdev)
6974 {
6975 int ret = 0;
6976 int rb_bufsz;
6977 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
6978
6979 /* allocate ring */
6980 ret = r600_ih_ring_alloc(rdev);
6981 if (ret)
6982 return ret;
6983
6984 /* disable irqs */
6985 cik_disable_interrupts(rdev);
6986
6987 /* init rlc */
6988 ret = cik_rlc_resume(rdev);
6989 if (ret) {
6990 r600_ih_ring_fini(rdev);
6991 return ret;
6992 }
6993
6994 /* setup interrupt control */
6995 /* XXX this should actually be a bus address, not an MC address. same on older asics */
6996 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
6997 interrupt_cntl = RREG32(INTERRUPT_CNTL);
6998 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
6999 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
7000 */
7001 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
7002 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
7003 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
7004 WREG32(INTERRUPT_CNTL, interrupt_cntl);
7005
7006 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
7007 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
7008
7009 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
7010 IH_WPTR_OVERFLOW_CLEAR |
7011 (rb_bufsz << 1));
7012
7013 if (rdev->wb.enabled)
7014 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
7015
7016 /* set the writeback address whether it's enabled or not */
7017 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
7018 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
7019
7020 WREG32(IH_RB_CNTL, ih_rb_cntl);
7021
7022 /* set rptr, wptr to 0 */
7023 WREG32(IH_RB_RPTR, 0);
7024 WREG32(IH_RB_WPTR, 0);
7025
7026 /* Default settings for IH_CNTL (disabled at first) */
7027 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
7028 /* RPTR_REARM only works if msi's are enabled */
7029 if (rdev->msi_enabled)
7030 ih_cntl |= RPTR_REARM;
7031 WREG32(IH_CNTL, ih_cntl);
7032
7033 /* force the active interrupt state to all disabled */
7034 cik_disable_interrupt_state(rdev);
7035
7036 pci_set_master(rdev->pdev);
7037
7038 /* enable irqs */
7039 cik_enable_interrupts(rdev);
7040
7041 return ret;
7042 }
7043
7044 /**
7045 * cik_irq_set - enable/disable interrupt sources
7046 *
7047 * @rdev: radeon_device pointer
7048 *
7049 * Enable interrupt sources on the GPU (vblanks, hpd,
7050 * etc.) (CIK).
7051 * Returns 0 for success, errors for failure.
7052 */
7053 int cik_irq_set(struct radeon_device *rdev)
7054 {
7055 u32 cp_int_cntl;
7056 u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
7057 u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
7058 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
7059 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
7060 u32 grbm_int_cntl = 0;
7061 u32 dma_cntl, dma_cntl1;
7062
7063 if (!rdev->irq.installed) {
7064 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
7065 return -EINVAL;
7066 }
7067 /* don't enable anything if the ih is disabled */
7068 if (!rdev->ih.enabled) {
7069 cik_disable_interrupts(rdev);
7070 /* force the active interrupt state to all disabled */
7071 cik_disable_interrupt_state(rdev);
7072 return 0;
7073 }
7074
7075 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
7076 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
7077 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
7078
7079 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7080 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7081 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7082 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7083 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7084 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7085
7086 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
7087 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7088
7089 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7090 cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7091 cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7092 cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7093 cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7094 cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7095 cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7096 cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7097
7098 /* enable CP interrupts on all rings */
7099 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
7100 DRM_DEBUG("cik_irq_set: sw int gfx\n");
7101 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
7102 }
7103 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
7104 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7105 DRM_DEBUG("si_irq_set: sw int cp1\n");
7106 if (ring->me == 1) {
7107 switch (ring->pipe) {
7108 case 0:
7109 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7110 break;
7111 case 1:
7112 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
7113 break;
7114 case 2:
7115 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7116 break;
7117 case 3:
7118 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7119 break;
7120 default:
7121 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7122 break;
7123 }
7124 } else if (ring->me == 2) {
7125 switch (ring->pipe) {
7126 case 0:
7127 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
7128 break;
7129 case 1:
7130 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
7131 break;
7132 case 2:
7133 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7134 break;
7135 case 3:
7136 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7137 break;
7138 default:
7139 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7140 break;
7141 }
7142 } else {
7143 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
7144 }
7145 }
7146 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
7147 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7148 DRM_DEBUG("si_irq_set: sw int cp2\n");
7149 if (ring->me == 1) {
7150 switch (ring->pipe) {
7151 case 0:
7152 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7153 break;
7154 case 1:
7155 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
7156 break;
7157 case 2:
7158 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7159 break;
7160 case 3:
7161 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7162 break;
7163 default:
7164 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7165 break;
7166 }
7167 } else if (ring->me == 2) {
7168 switch (ring->pipe) {
7169 case 0:
7170 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
7171 break;
7172 case 1:
7173 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
7174 break;
7175 case 2:
7176 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7177 break;
7178 case 3:
7179 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7180 break;
7181 default:
7182 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7183 break;
7184 }
7185 } else {
7186 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
7187 }
7188 }
7189
7190 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
7191 DRM_DEBUG("cik_irq_set: sw int dma\n");
7192 dma_cntl |= TRAP_ENABLE;
7193 }
7194
7195 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
7196 DRM_DEBUG("cik_irq_set: sw int dma1\n");
7197 dma_cntl1 |= TRAP_ENABLE;
7198 }
7199
7200 if (rdev->irq.crtc_vblank_int[0] ||
7201 atomic_read(&rdev->irq.pflip[0])) {
7202 DRM_DEBUG("cik_irq_set: vblank 0\n");
7203 crtc1 |= VBLANK_INTERRUPT_MASK;
7204 }
7205 if (rdev->irq.crtc_vblank_int[1] ||
7206 atomic_read(&rdev->irq.pflip[1])) {
7207 DRM_DEBUG("cik_irq_set: vblank 1\n");
7208 crtc2 |= VBLANK_INTERRUPT_MASK;
7209 }
7210 if (rdev->irq.crtc_vblank_int[2] ||
7211 atomic_read(&rdev->irq.pflip[2])) {
7212 DRM_DEBUG("cik_irq_set: vblank 2\n");
7213 crtc3 |= VBLANK_INTERRUPT_MASK;
7214 }
7215 if (rdev->irq.crtc_vblank_int[3] ||
7216 atomic_read(&rdev->irq.pflip[3])) {
7217 DRM_DEBUG("cik_irq_set: vblank 3\n");
7218 crtc4 |= VBLANK_INTERRUPT_MASK;
7219 }
7220 if (rdev->irq.crtc_vblank_int[4] ||
7221 atomic_read(&rdev->irq.pflip[4])) {
7222 DRM_DEBUG("cik_irq_set: vblank 4\n");
7223 crtc5 |= VBLANK_INTERRUPT_MASK;
7224 }
7225 if (rdev->irq.crtc_vblank_int[5] ||
7226 atomic_read(&rdev->irq.pflip[5])) {
7227 DRM_DEBUG("cik_irq_set: vblank 5\n");
7228 crtc6 |= VBLANK_INTERRUPT_MASK;
7229 }
7230 if (rdev->irq.hpd[0]) {
7231 DRM_DEBUG("cik_irq_set: hpd 1\n");
7232 hpd1 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
7233 }
7234 if (rdev->irq.hpd[1]) {
7235 DRM_DEBUG("cik_irq_set: hpd 2\n");
7236 hpd2 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
7237 }
7238 if (rdev->irq.hpd[2]) {
7239 DRM_DEBUG("cik_irq_set: hpd 3\n");
7240 hpd3 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
7241 }
7242 if (rdev->irq.hpd[3]) {
7243 DRM_DEBUG("cik_irq_set: hpd 4\n");
7244 hpd4 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
7245 }
7246 if (rdev->irq.hpd[4]) {
7247 DRM_DEBUG("cik_irq_set: hpd 5\n");
7248 hpd5 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
7249 }
7250 if (rdev->irq.hpd[5]) {
7251 DRM_DEBUG("cik_irq_set: hpd 6\n");
7252 hpd6 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
7253 }
7254
7255 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
7256
7257 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
7258 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
7259
7260 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
7261 WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
7262 WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
7263 WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
7264 WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
7265 WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
7266 WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
7267 WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
7268
7269 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
7270
7271 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
7272 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
7273 if (rdev->num_crtc >= 4) {
7274 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
7275 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
7276 }
7277 if (rdev->num_crtc >= 6) {
7278 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
7279 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
7280 }
7281
7282 if (rdev->num_crtc >= 2) {
7283 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
7284 GRPH_PFLIP_INT_MASK);
7285 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
7286 GRPH_PFLIP_INT_MASK);
7287 }
7288 if (rdev->num_crtc >= 4) {
7289 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
7290 GRPH_PFLIP_INT_MASK);
7291 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
7292 GRPH_PFLIP_INT_MASK);
7293 }
7294 if (rdev->num_crtc >= 6) {
7295 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
7296 GRPH_PFLIP_INT_MASK);
7297 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
7298 GRPH_PFLIP_INT_MASK);
7299 }
7300
7301 WREG32(DC_HPD1_INT_CONTROL, hpd1);
7302 WREG32(DC_HPD2_INT_CONTROL, hpd2);
7303 WREG32(DC_HPD3_INT_CONTROL, hpd3);
7304 WREG32(DC_HPD4_INT_CONTROL, hpd4);
7305 WREG32(DC_HPD5_INT_CONTROL, hpd5);
7306 WREG32(DC_HPD6_INT_CONTROL, hpd6);
7307
7308 /* posting read */
7309 RREG32(SRBM_STATUS);
7310
7311 return 0;
7312 }
7313
7314 /**
7315 * cik_irq_ack - ack interrupt sources
7316 *
7317 * @rdev: radeon_device pointer
7318 *
7319 * Ack interrupt sources on the GPU (vblanks, hpd,
7320 * etc.) (CIK). Certain interrupts sources are sw
7321 * generated and do not require an explicit ack.
7322 */
7323 static inline void cik_irq_ack(struct radeon_device *rdev)
7324 {
7325 u32 tmp;
7326
7327 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
7328 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
7329 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
7330 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
7331 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
7332 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
7333 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
7334
7335 rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS +
7336 EVERGREEN_CRTC0_REGISTER_OFFSET);
7337 rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS +
7338 EVERGREEN_CRTC1_REGISTER_OFFSET);
7339 if (rdev->num_crtc >= 4) {
7340 rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS +
7341 EVERGREEN_CRTC2_REGISTER_OFFSET);
7342 rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS +
7343 EVERGREEN_CRTC3_REGISTER_OFFSET);
7344 }
7345 if (rdev->num_crtc >= 6) {
7346 rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS +
7347 EVERGREEN_CRTC4_REGISTER_OFFSET);
7348 rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS +
7349 EVERGREEN_CRTC5_REGISTER_OFFSET);
7350 }
7351
7352 if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
7353 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET,
7354 GRPH_PFLIP_INT_CLEAR);
7355 if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
7356 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET,
7357 GRPH_PFLIP_INT_CLEAR);
7358 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
7359 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
7360 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
7361 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
7362 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
7363 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
7364 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
7365 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
7366
7367 if (rdev->num_crtc >= 4) {
7368 if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
7369 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET,
7370 GRPH_PFLIP_INT_CLEAR);
7371 if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
7372 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET,
7373 GRPH_PFLIP_INT_CLEAR);
7374 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
7375 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
7376 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
7377 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
7378 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
7379 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
7380 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
7381 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
7382 }
7383
7384 if (rdev->num_crtc >= 6) {
7385 if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
7386 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET,
7387 GRPH_PFLIP_INT_CLEAR);
7388 if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
7389 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET,
7390 GRPH_PFLIP_INT_CLEAR);
7391 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
7392 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
7393 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
7394 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
7395 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
7396 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
7397 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
7398 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
7399 }
7400
7401 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7402 tmp = RREG32(DC_HPD1_INT_CONTROL);
7403 tmp |= DC_HPDx_INT_ACK;
7404 WREG32(DC_HPD1_INT_CONTROL, tmp);
7405 }
7406 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7407 tmp = RREG32(DC_HPD2_INT_CONTROL);
7408 tmp |= DC_HPDx_INT_ACK;
7409 WREG32(DC_HPD2_INT_CONTROL, tmp);
7410 }
7411 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7412 tmp = RREG32(DC_HPD3_INT_CONTROL);
7413 tmp |= DC_HPDx_INT_ACK;
7414 WREG32(DC_HPD3_INT_CONTROL, tmp);
7415 }
7416 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7417 tmp = RREG32(DC_HPD4_INT_CONTROL);
7418 tmp |= DC_HPDx_INT_ACK;
7419 WREG32(DC_HPD4_INT_CONTROL, tmp);
7420 }
7421 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7422 tmp = RREG32(DC_HPD5_INT_CONTROL);
7423 tmp |= DC_HPDx_INT_ACK;
7424 WREG32(DC_HPD5_INT_CONTROL, tmp);
7425 }
7426 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7427 tmp = RREG32(DC_HPD6_INT_CONTROL);
7428 tmp |= DC_HPDx_INT_ACK;
7429 WREG32(DC_HPD6_INT_CONTROL, tmp);
7430 }
7431 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_RX_INTERRUPT) {
7432 tmp = RREG32(DC_HPD1_INT_CONTROL);
7433 tmp |= DC_HPDx_RX_INT_ACK;
7434 WREG32(DC_HPD1_INT_CONTROL, tmp);
7435 }
7436 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_RX_INTERRUPT) {
7437 tmp = RREG32(DC_HPD2_INT_CONTROL);
7438 tmp |= DC_HPDx_RX_INT_ACK;
7439 WREG32(DC_HPD2_INT_CONTROL, tmp);
7440 }
7441 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) {
7442 tmp = RREG32(DC_HPD3_INT_CONTROL);
7443 tmp |= DC_HPDx_RX_INT_ACK;
7444 WREG32(DC_HPD3_INT_CONTROL, tmp);
7445 }
7446 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) {
7447 tmp = RREG32(DC_HPD4_INT_CONTROL);
7448 tmp |= DC_HPDx_RX_INT_ACK;
7449 WREG32(DC_HPD4_INT_CONTROL, tmp);
7450 }
7451 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) {
7452 tmp = RREG32(DC_HPD5_INT_CONTROL);
7453 tmp |= DC_HPDx_RX_INT_ACK;
7454 WREG32(DC_HPD5_INT_CONTROL, tmp);
7455 }
7456 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) {
7457 tmp = RREG32(DC_HPD6_INT_CONTROL);
7458 tmp |= DC_HPDx_RX_INT_ACK;
7459 WREG32(DC_HPD6_INT_CONTROL, tmp);
7460 }
7461 }
7462
7463 /**
7464 * cik_irq_disable - disable interrupts
7465 *
7466 * @rdev: radeon_device pointer
7467 *
7468 * Disable interrupts on the hw (CIK).
7469 */
7470 static void cik_irq_disable(struct radeon_device *rdev)
7471 {
7472 cik_disable_interrupts(rdev);
7473 /* Wait and acknowledge irq */
7474 mdelay(1);
7475 cik_irq_ack(rdev);
7476 cik_disable_interrupt_state(rdev);
7477 }
7478
7479 /**
7480 * cik_irq_disable - disable interrupts for suspend
7481 *
7482 * @rdev: radeon_device pointer
7483 *
7484 * Disable interrupts and stop the RLC (CIK).
7485 * Used for suspend.
7486 */
7487 static void cik_irq_suspend(struct radeon_device *rdev)
7488 {
7489 cik_irq_disable(rdev);
7490 cik_rlc_stop(rdev);
7491 }
7492
7493 /**
7494 * cik_irq_fini - tear down interrupt support
7495 *
7496 * @rdev: radeon_device pointer
7497 *
7498 * Disable interrupts on the hw and free the IH ring
7499 * buffer (CIK).
7500 * Used for driver unload.
7501 */
7502 static void cik_irq_fini(struct radeon_device *rdev)
7503 {
7504 cik_irq_suspend(rdev);
7505 r600_ih_ring_fini(rdev);
7506 }
7507
7508 /**
7509 * cik_get_ih_wptr - get the IH ring buffer wptr
7510 *
7511 * @rdev: radeon_device pointer
7512 *
7513 * Get the IH ring buffer wptr from either the register
7514 * or the writeback memory buffer (CIK). Also check for
7515 * ring buffer overflow and deal with it.
7516 * Used by cik_irq_process().
7517 * Returns the value of the wptr.
7518 */
7519 static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
7520 {
7521 u32 wptr, tmp;
7522
7523 if (rdev->wb.enabled)
7524 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
7525 else
7526 wptr = RREG32(IH_RB_WPTR);
7527
7528 if (wptr & RB_OVERFLOW) {
7529 wptr &= ~RB_OVERFLOW;
7530 /* When a ring buffer overflow happen start parsing interrupt
7531 * from the last not overwritten vector (wptr + 16). Hopefully
7532 * this should allow us to catchup.
7533 */
7534 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
7535 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
7536 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
7537 tmp = RREG32(IH_RB_CNTL);
7538 tmp |= IH_WPTR_OVERFLOW_CLEAR;
7539 WREG32(IH_RB_CNTL, tmp);
7540 }
7541 return (wptr & rdev->ih.ptr_mask);
7542 }
7543
7544 /* CIK IV Ring
7545 * Each IV ring entry is 128 bits:
7546 * [7:0] - interrupt source id
7547 * [31:8] - reserved
7548 * [59:32] - interrupt source data
7549 * [63:60] - reserved
7550 * [71:64] - RINGID
7551 * CP:
7552 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
7553 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7554 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7555 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7556 * PIPE_ID - ME0 0=3D
7557 * - ME1&2 compute dispatcher (4 pipes each)
7558 * SDMA:
7559 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
7560 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
7561 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
7562 * [79:72] - VMID
7563 * [95:80] - PASID
7564 * [127:96] - reserved
7565 */
7566 /**
7567 * cik_irq_process - interrupt handler
7568 *
7569 * @rdev: radeon_device pointer
7570 *
7571 * Interrupt hander (CIK). Walk the IH ring,
7572 * ack interrupts and schedule work to handle
7573 * interrupt events.
7574 * Returns irq process return code.
7575 */
7576 int cik_irq_process(struct radeon_device *rdev)
7577 {
7578 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7579 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7580 u32 wptr;
7581 u32 rptr;
7582 u32 src_id, src_data, ring_id;
7583 u8 me_id, pipe_id, queue_id;
7584 u32 ring_index;
7585 bool queue_hotplug = false;
7586 bool queue_dp = false;
7587 bool queue_reset = false;
7588 u32 addr, status, mc_client;
7589 bool queue_thermal = false;
7590
7591 if (!rdev->ih.enabled || rdev->shutdown)
7592 return IRQ_NONE;
7593
7594 wptr = cik_get_ih_wptr(rdev);
7595
7596 restart_ih:
7597 /* is somebody else already processing irqs? */
7598 if (atomic_xchg(&rdev->ih.lock, 1))
7599 return IRQ_NONE;
7600
7601 rptr = rdev->ih.rptr;
7602 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7603
7604 /* Order reading of wptr vs. reading of IH ring data */
7605 rmb();
7606
7607 /* display interrupts */
7608 cik_irq_ack(rdev);
7609
7610 while (rptr != wptr) {
7611 /* wptr/rptr are in bytes! */
7612 ring_index = rptr / 4;
7613
7614 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7615 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7616 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
7617
7618 switch (src_id) {
7619 case 1: /* D1 vblank/vline */
7620 switch (src_data) {
7621 case 0: /* D1 vblank */
7622 if (!(rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT))
7623 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7624
7625 if (rdev->irq.crtc_vblank_int[0]) {
7626 drm_handle_vblank(rdev->ddev, 0);
7627 rdev->pm.vblank_sync = true;
7628 wake_up(&rdev->irq.vblank_queue);
7629 }
7630 if (atomic_read(&rdev->irq.pflip[0]))
7631 radeon_crtc_handle_vblank(rdev, 0);
7632 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7633 DRM_DEBUG("IH: D1 vblank\n");
7634
7635 break;
7636 case 1: /* D1 vline */
7637 if (!(rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT))
7638 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7639
7640 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7641 DRM_DEBUG("IH: D1 vline\n");
7642
7643 break;
7644 default:
7645 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7646 break;
7647 }
7648 break;
7649 case 2: /* D2 vblank/vline */
7650 switch (src_data) {
7651 case 0: /* D2 vblank */
7652 if (!(rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT))
7653 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7654
7655 if (rdev->irq.crtc_vblank_int[1]) {
7656 drm_handle_vblank(rdev->ddev, 1);
7657 rdev->pm.vblank_sync = true;
7658 wake_up(&rdev->irq.vblank_queue);
7659 }
7660 if (atomic_read(&rdev->irq.pflip[1]))
7661 radeon_crtc_handle_vblank(rdev, 1);
7662 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7663 DRM_DEBUG("IH: D2 vblank\n");
7664
7665 break;
7666 case 1: /* D2 vline */
7667 if (!(rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT))
7668 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7669
7670 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7671 DRM_DEBUG("IH: D2 vline\n");
7672
7673 break;
7674 default:
7675 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7676 break;
7677 }
7678 break;
7679 case 3: /* D3 vblank/vline */
7680 switch (src_data) {
7681 case 0: /* D3 vblank */
7682 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT))
7683 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7684
7685 if (rdev->irq.crtc_vblank_int[2]) {
7686 drm_handle_vblank(rdev->ddev, 2);
7687 rdev->pm.vblank_sync = true;
7688 wake_up(&rdev->irq.vblank_queue);
7689 }
7690 if (atomic_read(&rdev->irq.pflip[2]))
7691 radeon_crtc_handle_vblank(rdev, 2);
7692 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7693 DRM_DEBUG("IH: D3 vblank\n");
7694
7695 break;
7696 case 1: /* D3 vline */
7697 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT))
7698 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7699
7700 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7701 DRM_DEBUG("IH: D3 vline\n");
7702
7703 break;
7704 default:
7705 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7706 break;
7707 }
7708 break;
7709 case 4: /* D4 vblank/vline */
7710 switch (src_data) {
7711 case 0: /* D4 vblank */
7712 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT))
7713 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7714
7715 if (rdev->irq.crtc_vblank_int[3]) {
7716 drm_handle_vblank(rdev->ddev, 3);
7717 rdev->pm.vblank_sync = true;
7718 wake_up(&rdev->irq.vblank_queue);
7719 }
7720 if (atomic_read(&rdev->irq.pflip[3]))
7721 radeon_crtc_handle_vblank(rdev, 3);
7722 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7723 DRM_DEBUG("IH: D4 vblank\n");
7724
7725 break;
7726 case 1: /* D4 vline */
7727 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT))
7728 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7729
7730 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7731 DRM_DEBUG("IH: D4 vline\n");
7732
7733 break;
7734 default:
7735 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7736 break;
7737 }
7738 break;
7739 case 5: /* D5 vblank/vline */
7740 switch (src_data) {
7741 case 0: /* D5 vblank */
7742 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT))
7743 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7744
7745 if (rdev->irq.crtc_vblank_int[4]) {
7746 drm_handle_vblank(rdev->ddev, 4);
7747 rdev->pm.vblank_sync = true;
7748 wake_up(&rdev->irq.vblank_queue);
7749 }
7750 if (atomic_read(&rdev->irq.pflip[4]))
7751 radeon_crtc_handle_vblank(rdev, 4);
7752 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7753 DRM_DEBUG("IH: D5 vblank\n");
7754
7755 break;
7756 case 1: /* D5 vline */
7757 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT))
7758 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7759
7760 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7761 DRM_DEBUG("IH: D5 vline\n");
7762
7763 break;
7764 default:
7765 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7766 break;
7767 }
7768 break;
7769 case 6: /* D6 vblank/vline */
7770 switch (src_data) {
7771 case 0: /* D6 vblank */
7772 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT))
7773 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7774
7775 if (rdev->irq.crtc_vblank_int[5]) {
7776 drm_handle_vblank(rdev->ddev, 5);
7777 rdev->pm.vblank_sync = true;
7778 wake_up(&rdev->irq.vblank_queue);
7779 }
7780 if (atomic_read(&rdev->irq.pflip[5]))
7781 radeon_crtc_handle_vblank(rdev, 5);
7782 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7783 DRM_DEBUG("IH: D6 vblank\n");
7784
7785 break;
7786 case 1: /* D6 vline */
7787 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT))
7788 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7789
7790 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7791 DRM_DEBUG("IH: D6 vline\n");
7792
7793 break;
7794 default:
7795 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7796 break;
7797 }
7798 break;
7799 case 8: /* D1 page flip */
7800 case 10: /* D2 page flip */
7801 case 12: /* D3 page flip */
7802 case 14: /* D4 page flip */
7803 case 16: /* D5 page flip */
7804 case 18: /* D6 page flip */
7805 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
7806 if (radeon_use_pflipirq > 0)
7807 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
7808 break;
7809 case 42: /* HPD hotplug */
7810 switch (src_data) {
7811 case 0:
7812 if (!(rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT))
7813 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7814
7815 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
7816 queue_hotplug = true;
7817 DRM_DEBUG("IH: HPD1\n");
7818
7819 break;
7820 case 1:
7821 if (!(rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT))
7822 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7823
7824 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
7825 queue_hotplug = true;
7826 DRM_DEBUG("IH: HPD2\n");
7827
7828 break;
7829 case 2:
7830 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT))
7831 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7832
7833 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
7834 queue_hotplug = true;
7835 DRM_DEBUG("IH: HPD3\n");
7836
7837 break;
7838 case 3:
7839 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT))
7840 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7841
7842 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
7843 queue_hotplug = true;
7844 DRM_DEBUG("IH: HPD4\n");
7845
7846 break;
7847 case 4:
7848 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT))
7849 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7850
7851 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
7852 queue_hotplug = true;
7853 DRM_DEBUG("IH: HPD5\n");
7854
7855 break;
7856 case 5:
7857 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT))
7858 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7859
7860 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
7861 queue_hotplug = true;
7862 DRM_DEBUG("IH: HPD6\n");
7863
7864 break;
7865 case 6:
7866 if (!(rdev->irq.stat_regs.cik.disp_int & DC_HPD1_RX_INTERRUPT))
7867 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7868
7869 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_RX_INTERRUPT;
7870 queue_dp = true;
7871 DRM_DEBUG("IH: HPD_RX 1\n");
7872
7873 break;
7874 case 7:
7875 if (!(rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_RX_INTERRUPT))
7876 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7877
7878 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT;
7879 queue_dp = true;
7880 DRM_DEBUG("IH: HPD_RX 2\n");
7881
7882 break;
7883 case 8:
7884 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_RX_INTERRUPT))
7885 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7886
7887 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT;
7888 queue_dp = true;
7889 DRM_DEBUG("IH: HPD_RX 3\n");
7890
7891 break;
7892 case 9:
7893 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_RX_INTERRUPT))
7894 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7895
7896 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT;
7897 queue_dp = true;
7898 DRM_DEBUG("IH: HPD_RX 4\n");
7899
7900 break;
7901 case 10:
7902 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_RX_INTERRUPT))
7903 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7904
7905 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT;
7906 queue_dp = true;
7907 DRM_DEBUG("IH: HPD_RX 5\n");
7908
7909 break;
7910 case 11:
7911 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_RX_INTERRUPT))
7912 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7913
7914 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT;
7915 queue_dp = true;
7916 DRM_DEBUG("IH: HPD_RX 6\n");
7917
7918 break;
7919 default:
7920 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7921 break;
7922 }
7923 break;
7924 case 96:
7925 DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
7926 WREG32(SRBM_INT_ACK, 0x1);
7927 break;
7928 case 124: /* UVD */
7929 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
7930 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
7931 break;
7932 case 146:
7933 case 147:
7934 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
7935 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
7936 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
7937 /* reset addr and status */
7938 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
7939 if (addr == 0x0 && status == 0x0)
7940 break;
7941 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
7942 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
7943 addr);
7944 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
7945 status);
7946 cik_vm_decode_fault(rdev, status, addr, mc_client);
7947 break;
7948 case 167: /* VCE */
7949 DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data);
7950 switch (src_data) {
7951 case 0:
7952 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX);
7953 break;
7954 case 1:
7955 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX);
7956 break;
7957 default:
7958 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
7959 break;
7960 }
7961 break;
7962 case 176: /* GFX RB CP_INT */
7963 case 177: /* GFX IB CP_INT */
7964 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7965 break;
7966 case 181: /* CP EOP event */
7967 DRM_DEBUG("IH: CP EOP\n");
7968 /* XXX check the bitfield order! */
7969 me_id = (ring_id & 0x60) >> 5;
7970 pipe_id = (ring_id & 0x18) >> 3;
7971 queue_id = (ring_id & 0x7) >> 0;
7972 switch (me_id) {
7973 case 0:
7974 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7975 break;
7976 case 1:
7977 case 2:
7978 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
7979 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7980 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
7981 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7982 break;
7983 }
7984 break;
7985 case 184: /* CP Privileged reg access */
7986 DRM_ERROR("Illegal register access in command stream\n");
7987 /* XXX check the bitfield order! */
7988 me_id = (ring_id & 0x60) >> 5;
7989 pipe_id = (ring_id & 0x18) >> 3;
7990 queue_id = (ring_id & 0x7) >> 0;
7991 switch (me_id) {
7992 case 0:
7993 /* This results in a full GPU reset, but all we need to do is soft
7994 * reset the CP for gfx
7995 */
7996 queue_reset = true;
7997 break;
7998 case 1:
7999 /* XXX compute */
8000 queue_reset = true;
8001 break;
8002 case 2:
8003 /* XXX compute */
8004 queue_reset = true;
8005 break;
8006 }
8007 break;
8008 case 185: /* CP Privileged inst */
8009 DRM_ERROR("Illegal instruction in command stream\n");
8010 /* XXX check the bitfield order! */
8011 me_id = (ring_id & 0x60) >> 5;
8012 pipe_id = (ring_id & 0x18) >> 3;
8013 queue_id = (ring_id & 0x7) >> 0;
8014 switch (me_id) {
8015 case 0:
8016 /* This results in a full GPU reset, but all we need to do is soft
8017 * reset the CP for gfx
8018 */
8019 queue_reset = true;
8020 break;
8021 case 1:
8022 /* XXX compute */
8023 queue_reset = true;
8024 break;
8025 case 2:
8026 /* XXX compute */
8027 queue_reset = true;
8028 break;
8029 }
8030 break;
8031 case 224: /* SDMA trap event */
8032 /* XXX check the bitfield order! */
8033 me_id = (ring_id & 0x3) >> 0;
8034 queue_id = (ring_id & 0xc) >> 2;
8035 DRM_DEBUG("IH: SDMA trap\n");
8036 switch (me_id) {
8037 case 0:
8038 switch (queue_id) {
8039 case 0:
8040 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
8041 break;
8042 case 1:
8043 /* XXX compute */
8044 break;
8045 case 2:
8046 /* XXX compute */
8047 break;
8048 }
8049 break;
8050 case 1:
8051 switch (queue_id) {
8052 case 0:
8053 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8054 break;
8055 case 1:
8056 /* XXX compute */
8057 break;
8058 case 2:
8059 /* XXX compute */
8060 break;
8061 }
8062 break;
8063 }
8064 break;
8065 case 230: /* thermal low to high */
8066 DRM_DEBUG("IH: thermal low to high\n");
8067 rdev->pm.dpm.thermal.high_to_low = false;
8068 queue_thermal = true;
8069 break;
8070 case 231: /* thermal high to low */
8071 DRM_DEBUG("IH: thermal high to low\n");
8072 rdev->pm.dpm.thermal.high_to_low = true;
8073 queue_thermal = true;
8074 break;
8075 case 233: /* GUI IDLE */
8076 DRM_DEBUG("IH: GUI idle\n");
8077 break;
8078 case 241: /* SDMA Privileged inst */
8079 case 247: /* SDMA Privileged inst */
8080 DRM_ERROR("Illegal instruction in SDMA command stream\n");
8081 /* XXX check the bitfield order! */
8082 me_id = (ring_id & 0x3) >> 0;
8083 queue_id = (ring_id & 0xc) >> 2;
8084 switch (me_id) {
8085 case 0:
8086 switch (queue_id) {
8087 case 0:
8088 queue_reset = true;
8089 break;
8090 case 1:
8091 /* XXX compute */
8092 queue_reset = true;
8093 break;
8094 case 2:
8095 /* XXX compute */
8096 queue_reset = true;
8097 break;
8098 }
8099 break;
8100 case 1:
8101 switch (queue_id) {
8102 case 0:
8103 queue_reset = true;
8104 break;
8105 case 1:
8106 /* XXX compute */
8107 queue_reset = true;
8108 break;
8109 case 2:
8110 /* XXX compute */
8111 queue_reset = true;
8112 break;
8113 }
8114 break;
8115 }
8116 break;
8117 default:
8118 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8119 break;
8120 }
8121
8122 /* wptr/rptr are in bytes! */
8123 rptr += 16;
8124 rptr &= rdev->ih.ptr_mask;
8125 WREG32(IH_RB_RPTR, rptr);
8126 }
8127 if (queue_dp)
8128 schedule_work(&rdev->dp_work);
8129 if (queue_hotplug)
8130 schedule_delayed_work(&rdev->hotplug_work, 0);
8131 if (queue_reset) {
8132 rdev->needs_reset = true;
8133 wake_up_all(&rdev->fence_queue);
8134 }
8135 if (queue_thermal)
8136 schedule_work(&rdev->pm.dpm.thermal.work);
8137 rdev->ih.rptr = rptr;
8138 atomic_set(&rdev->ih.lock, 0);
8139
8140 /* make sure wptr hasn't changed while processing */
8141 wptr = cik_get_ih_wptr(rdev);
8142 if (wptr != rptr)
8143 goto restart_ih;
8144
8145 return IRQ_HANDLED;
8146 }
8147
8148 /*
8149 * startup/shutdown callbacks
8150 */
8151 static void cik_uvd_init(struct radeon_device *rdev)
8152 {
8153 int r;
8154
8155 if (!rdev->has_uvd)
8156 return;
8157
8158 r = radeon_uvd_init(rdev);
8159 if (r) {
8160 dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
8161 /*
8162 * At this point rdev->uvd.vcpu_bo is NULL which trickles down
8163 * to early fails cik_uvd_start() and thus nothing happens
8164 * there. So it is pointless to try to go through that code
8165 * hence why we disable uvd here.
8166 */
8167 rdev->has_uvd = 0;
8168 return;
8169 }
8170 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
8171 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
8172 }
8173
8174 static void cik_uvd_start(struct radeon_device *rdev)
8175 {
8176 int r;
8177
8178 if (!rdev->has_uvd)
8179 return;
8180
8181 r = radeon_uvd_resume(rdev);
8182 if (r) {
8183 dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
8184 goto error;
8185 }
8186 r = uvd_v4_2_resume(rdev);
8187 if (r) {
8188 dev_err(rdev->dev, "failed UVD 4.2 resume (%d).\n", r);
8189 goto error;
8190 }
8191 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
8192 if (r) {
8193 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
8194 goto error;
8195 }
8196 return;
8197
8198 error:
8199 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
8200 }
8201
8202 static void cik_uvd_resume(struct radeon_device *rdev)
8203 {
8204 struct radeon_ring *ring;
8205 int r;
8206
8207 if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
8208 return;
8209
8210 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8211 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
8212 if (r) {
8213 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
8214 return;
8215 }
8216 r = uvd_v1_0_init(rdev);
8217 if (r) {
8218 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
8219 return;
8220 }
8221 }
8222
8223 static void cik_vce_init(struct radeon_device *rdev)
8224 {
8225 int r;
8226
8227 if (!rdev->has_vce)
8228 return;
8229
8230 r = radeon_vce_init(rdev);
8231 if (r) {
8232 dev_err(rdev->dev, "failed VCE (%d) init.\n", r);
8233 /*
8234 * At this point rdev->vce.vcpu_bo is NULL which trickles down
8235 * to early fails cik_vce_start() and thus nothing happens
8236 * there. So it is pointless to try to go through that code
8237 * hence why we disable vce here.
8238 */
8239 rdev->has_vce = 0;
8240 return;
8241 }
8242 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_obj = NULL;
8243 r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE1_INDEX], 4096);
8244 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_obj = NULL;
8245 r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE2_INDEX], 4096);
8246 }
8247
8248 static void cik_vce_start(struct radeon_device *rdev)
8249 {
8250 int r;
8251
8252 if (!rdev->has_vce)
8253 return;
8254
8255 r = radeon_vce_resume(rdev);
8256 if (r) {
8257 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
8258 goto error;
8259 }
8260 r = vce_v2_0_resume(rdev);
8261 if (r) {
8262 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
8263 goto error;
8264 }
8265 r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE1_INDEX);
8266 if (r) {
8267 dev_err(rdev->dev, "failed initializing VCE1 fences (%d).\n", r);
8268 goto error;
8269 }
8270 r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE2_INDEX);
8271 if (r) {
8272 dev_err(rdev->dev, "failed initializing VCE2 fences (%d).\n", r);
8273 goto error;
8274 }
8275 return;
8276
8277 error:
8278 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
8279 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
8280 }
8281
8282 static void cik_vce_resume(struct radeon_device *rdev)
8283 {
8284 struct radeon_ring *ring;
8285 int r;
8286
8287 if (!rdev->has_vce || !rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size)
8288 return;
8289
8290 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8291 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP);
8292 if (r) {
8293 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
8294 return;
8295 }
8296 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8297 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP);
8298 if (r) {
8299 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
8300 return;
8301 }
8302 r = vce_v1_0_init(rdev);
8303 if (r) {
8304 dev_err(rdev->dev, "failed initializing VCE (%d).\n", r);
8305 return;
8306 }
8307 }
8308
8309 /**
8310 * cik_startup - program the asic to a functional state
8311 *
8312 * @rdev: radeon_device pointer
8313 *
8314 * Programs the asic to a functional state (CIK).
8315 * Called by cik_init() and cik_resume().
8316 * Returns 0 for success, error for failure.
8317 */
8318 static int cik_startup(struct radeon_device *rdev)
8319 {
8320 struct radeon_ring *ring;
8321 u32 nop;
8322 int r;
8323
8324 /* enable pcie gen2/3 link */
8325 cik_pcie_gen3_enable(rdev);
8326 /* enable aspm */
8327 cik_program_aspm(rdev);
8328
8329 /* scratch needs to be initialized before MC */
8330 r = r600_vram_scratch_init(rdev);
8331 if (r)
8332 return r;
8333
8334 cik_mc_program(rdev);
8335
8336 if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
8337 r = ci_mc_load_microcode(rdev);
8338 if (r) {
8339 DRM_ERROR("Failed to load MC firmware!\n");
8340 return r;
8341 }
8342 }
8343
8344 r = cik_pcie_gart_enable(rdev);
8345 if (r)
8346 return r;
8347 cik_gpu_init(rdev);
8348
8349 /* allocate rlc buffers */
8350 if (rdev->flags & RADEON_IS_IGP) {
8351 if (rdev->family == CHIP_KAVERI) {
8352 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
8353 rdev->rlc.reg_list_size =
8354 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
8355 } else {
8356 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
8357 rdev->rlc.reg_list_size =
8358 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
8359 }
8360 }
8361 rdev->rlc.cs_data = ci_cs_data;
8362 rdev->rlc.cp_table_size = ALIGN(CP_ME_TABLE_SIZE * 5 * 4, 2048); /* CP JT */
8363 rdev->rlc.cp_table_size += 64 * 1024; /* GDS */
8364 r = sumo_rlc_init(rdev);
8365 if (r) {
8366 DRM_ERROR("Failed to init rlc BOs!\n");
8367 return r;
8368 }
8369
8370 /* allocate wb buffer */
8371 r = radeon_wb_init(rdev);
8372 if (r)
8373 return r;
8374
8375 /* allocate mec buffers */
8376 r = cik_mec_init(rdev);
8377 if (r) {
8378 DRM_ERROR("Failed to init MEC BOs!\n");
8379 return r;
8380 }
8381
8382 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
8383 if (r) {
8384 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8385 return r;
8386 }
8387
8388 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
8389 if (r) {
8390 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8391 return r;
8392 }
8393
8394 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
8395 if (r) {
8396 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8397 return r;
8398 }
8399
8400 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
8401 if (r) {
8402 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8403 return r;
8404 }
8405
8406 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8407 if (r) {
8408 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8409 return r;
8410 }
8411
8412 cik_uvd_start(rdev);
8413 cik_vce_start(rdev);
8414
8415 /* Enable IRQ */
8416 if (!rdev->irq.installed) {
8417 r = radeon_irq_kms_init(rdev);
8418 if (r)
8419 return r;
8420 }
8421
8422 r = cik_irq_init(rdev);
8423 if (r) {
8424 DRM_ERROR("radeon: IH init failed (%d).\n", r);
8425 radeon_irq_kms_fini(rdev);
8426 return r;
8427 }
8428 cik_irq_set(rdev);
8429
8430 if (rdev->family == CHIP_HAWAII) {
8431 if (rdev->new_fw)
8432 nop = PACKET3(PACKET3_NOP, 0x3FFF);
8433 else
8434 nop = RADEON_CP_PACKET2;
8435 } else {
8436 nop = PACKET3(PACKET3_NOP, 0x3FFF);
8437 }
8438
8439 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8440 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
8441 nop);
8442 if (r)
8443 return r;
8444
8445 /* set up the compute queues */
8446 /* type-2 packets are deprecated on MEC, use type-3 instead */
8447 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8448 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
8449 nop);
8450 if (r)
8451 return r;
8452 ring->me = 1; /* first MEC */
8453 ring->pipe = 0; /* first pipe */
8454 ring->queue = 0; /* first queue */
8455 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
8456
8457 /* type-2 packets are deprecated on MEC, use type-3 instead */
8458 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8459 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
8460 nop);
8461 if (r)
8462 return r;
8463 /* dGPU only have 1 MEC */
8464 ring->me = 1; /* first MEC */
8465 ring->pipe = 0; /* first pipe */
8466 ring->queue = 1; /* second queue */
8467 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
8468
8469 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8470 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
8471 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
8472 if (r)
8473 return r;
8474
8475 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8476 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
8477 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
8478 if (r)
8479 return r;
8480
8481 r = cik_cp_resume(rdev);
8482 if (r)
8483 return r;
8484
8485 r = cik_sdma_resume(rdev);
8486 if (r)
8487 return r;
8488
8489 cik_uvd_resume(rdev);
8490 cik_vce_resume(rdev);
8491
8492 r = radeon_ib_pool_init(rdev);
8493 if (r) {
8494 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
8495 return r;
8496 }
8497
8498 r = radeon_vm_manager_init(rdev);
8499 if (r) {
8500 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
8501 return r;
8502 }
8503
8504 r = radeon_audio_init(rdev);
8505 if (r)
8506 return r;
8507
8508 return 0;
8509 }
8510
8511 /**
8512 * cik_resume - resume the asic to a functional state
8513 *
8514 * @rdev: radeon_device pointer
8515 *
8516 * Programs the asic to a functional state (CIK).
8517 * Called at resume.
8518 * Returns 0 for success, error for failure.
8519 */
8520 int cik_resume(struct radeon_device *rdev)
8521 {
8522 int r;
8523
8524 /* post card */
8525 atom_asic_init(rdev->mode_info.atom_context);
8526
8527 /* init golden registers */
8528 cik_init_golden_registers(rdev);
8529
8530 if (rdev->pm.pm_method == PM_METHOD_DPM)
8531 radeon_pm_resume(rdev);
8532
8533 rdev->accel_working = true;
8534 r = cik_startup(rdev);
8535 if (r) {
8536 DRM_ERROR("cik startup failed on resume\n");
8537 rdev->accel_working = false;
8538 return r;
8539 }
8540
8541 return r;
8542
8543 }
8544
8545 /**
8546 * cik_suspend - suspend the asic
8547 *
8548 * @rdev: radeon_device pointer
8549 *
8550 * Bring the chip into a state suitable for suspend (CIK).
8551 * Called at suspend.
8552 * Returns 0 for success.
8553 */
8554 int cik_suspend(struct radeon_device *rdev)
8555 {
8556 radeon_pm_suspend(rdev);
8557 radeon_audio_fini(rdev);
8558 radeon_vm_manager_fini(rdev);
8559 cik_cp_enable(rdev, false);
8560 cik_sdma_enable(rdev, false);
8561 if (rdev->has_uvd) {
8562 uvd_v1_0_fini(rdev);
8563 radeon_uvd_suspend(rdev);
8564 }
8565 if (rdev->has_vce)
8566 radeon_vce_suspend(rdev);
8567 cik_fini_pg(rdev);
8568 cik_fini_cg(rdev);
8569 cik_irq_suspend(rdev);
8570 radeon_wb_disable(rdev);
8571 cik_pcie_gart_disable(rdev);
8572 return 0;
8573 }
8574
8575 /* Plan is to move initialization in that function and use
8576 * helper function so that radeon_device_init pretty much
8577 * do nothing more than calling asic specific function. This
8578 * should also allow to remove a bunch of callback function
8579 * like vram_info.
8580 */
8581 /**
8582 * cik_init - asic specific driver and hw init
8583 *
8584 * @rdev: radeon_device pointer
8585 *
8586 * Setup asic specific driver variables and program the hw
8587 * to a functional state (CIK).
8588 * Called at driver startup.
8589 * Returns 0 for success, errors for failure.
8590 */
8591 int cik_init(struct radeon_device *rdev)
8592 {
8593 struct radeon_ring *ring;
8594 int r;
8595
8596 /* Read BIOS */
8597 if (!radeon_get_bios(rdev)) {
8598 if (ASIC_IS_AVIVO(rdev))
8599 return -EINVAL;
8600 }
8601 /* Must be an ATOMBIOS */
8602 if (!rdev->is_atom_bios) {
8603 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
8604 return -EINVAL;
8605 }
8606 r = radeon_atombios_init(rdev);
8607 if (r)
8608 return r;
8609
8610 /* Post card if necessary */
8611 if (!radeon_card_posted(rdev)) {
8612 if (!rdev->bios) {
8613 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
8614 return -EINVAL;
8615 }
8616 DRM_INFO("GPU not posted. posting now...\n");
8617 atom_asic_init(rdev->mode_info.atom_context);
8618 }
8619 /* init golden registers */
8620 cik_init_golden_registers(rdev);
8621 /* Initialize scratch registers */
8622 cik_scratch_init(rdev);
8623 /* Initialize surface registers */
8624 radeon_surface_init(rdev);
8625 /* Initialize clocks */
8626 radeon_get_clock_info(rdev->ddev);
8627
8628 /* Fence driver */
8629 r = radeon_fence_driver_init(rdev);
8630 if (r)
8631 return r;
8632
8633 /* initialize memory controller */
8634 r = cik_mc_init(rdev);
8635 if (r)
8636 return r;
8637 /* Memory manager */
8638 r = radeon_bo_init(rdev);
8639 if (r)
8640 return r;
8641
8642 if (rdev->flags & RADEON_IS_IGP) {
8643 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8644 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
8645 r = cik_init_microcode(rdev);
8646 if (r) {
8647 DRM_ERROR("Failed to load firmware!\n");
8648 return r;
8649 }
8650 }
8651 } else {
8652 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8653 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
8654 !rdev->mc_fw) {
8655 r = cik_init_microcode(rdev);
8656 if (r) {
8657 DRM_ERROR("Failed to load firmware!\n");
8658 return r;
8659 }
8660 }
8661 }
8662
8663 /* Initialize power management */
8664 radeon_pm_init(rdev);
8665
8666 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8667 ring->ring_obj = NULL;
8668 r600_ring_init(rdev, ring, 1024 * 1024);
8669
8670 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8671 ring->ring_obj = NULL;
8672 r600_ring_init(rdev, ring, 1024 * 1024);
8673 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8674 if (r)
8675 return r;
8676
8677 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8678 ring->ring_obj = NULL;
8679 r600_ring_init(rdev, ring, 1024 * 1024);
8680 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8681 if (r)
8682 return r;
8683
8684 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8685 ring->ring_obj = NULL;
8686 r600_ring_init(rdev, ring, 256 * 1024);
8687
8688 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8689 ring->ring_obj = NULL;
8690 r600_ring_init(rdev, ring, 256 * 1024);
8691
8692 cik_uvd_init(rdev);
8693 cik_vce_init(rdev);
8694
8695 rdev->ih.ring_obj = NULL;
8696 r600_ih_ring_init(rdev, 64 * 1024);
8697
8698 r = r600_pcie_gart_init(rdev);
8699 if (r)
8700 return r;
8701
8702 rdev->accel_working = true;
8703 r = cik_startup(rdev);
8704 if (r) {
8705 dev_err(rdev->dev, "disabling GPU acceleration\n");
8706 cik_cp_fini(rdev);
8707 cik_sdma_fini(rdev);
8708 cik_irq_fini(rdev);
8709 sumo_rlc_fini(rdev);
8710 cik_mec_fini(rdev);
8711 radeon_wb_fini(rdev);
8712 radeon_ib_pool_fini(rdev);
8713 radeon_vm_manager_fini(rdev);
8714 radeon_irq_kms_fini(rdev);
8715 cik_pcie_gart_fini(rdev);
8716 rdev->accel_working = false;
8717 }
8718
8719 /* Don't start up if the MC ucode is missing.
8720 * The default clocks and voltages before the MC ucode
8721 * is loaded are not suffient for advanced operations.
8722 */
8723 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
8724 DRM_ERROR("radeon: MC ucode required for NI+.\n");
8725 return -EINVAL;
8726 }
8727
8728 return 0;
8729 }
8730
8731 /**
8732 * cik_fini - asic specific driver and hw fini
8733 *
8734 * @rdev: radeon_device pointer
8735 *
8736 * Tear down the asic specific driver variables and program the hw
8737 * to an idle state (CIK).
8738 * Called at driver unload.
8739 */
8740 void cik_fini(struct radeon_device *rdev)
8741 {
8742 radeon_pm_fini(rdev);
8743 cik_cp_fini(rdev);
8744 cik_sdma_fini(rdev);
8745 cik_fini_pg(rdev);
8746 cik_fini_cg(rdev);
8747 cik_irq_fini(rdev);
8748 sumo_rlc_fini(rdev);
8749 cik_mec_fini(rdev);
8750 radeon_wb_fini(rdev);
8751 radeon_vm_manager_fini(rdev);
8752 radeon_ib_pool_fini(rdev);
8753 radeon_irq_kms_fini(rdev);
8754 uvd_v1_0_fini(rdev);
8755 radeon_uvd_fini(rdev);
8756 radeon_vce_fini(rdev);
8757 cik_pcie_gart_fini(rdev);
8758 r600_vram_scratch_fini(rdev);
8759 radeon_gem_fini(rdev);
8760 radeon_fence_driver_fini(rdev);
8761 radeon_bo_fini(rdev);
8762 radeon_atombios_fini(rdev);
8763 kfree(rdev->bios);
8764 rdev->bios = NULL;
8765 }
8766
8767 void dce8_program_fmt(struct drm_encoder *encoder)
8768 {
8769 struct drm_device *dev = encoder->dev;
8770 struct radeon_device *rdev = dev->dev_private;
8771 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
8772 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
8773 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
8774 int bpc = 0;
8775 u32 tmp = 0;
8776 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
8777
8778 if (connector) {
8779 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
8780 bpc = radeon_get_monitor_bpc(connector);
8781 dither = radeon_connector->dither;
8782 }
8783
8784 /* LVDS/eDP FMT is set up by atom */
8785 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
8786 return;
8787
8788 /* not needed for analog */
8789 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
8790 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
8791 return;
8792
8793 if (bpc == 0)
8794 return;
8795
8796 switch (bpc) {
8797 case 6:
8798 if (dither == RADEON_FMT_DITHER_ENABLE)
8799 /* XXX sort out optimal dither settings */
8800 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8801 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
8802 else
8803 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
8804 break;
8805 case 8:
8806 if (dither == RADEON_FMT_DITHER_ENABLE)
8807 /* XXX sort out optimal dither settings */
8808 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8809 FMT_RGB_RANDOM_ENABLE |
8810 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
8811 else
8812 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
8813 break;
8814 case 10:
8815 if (dither == RADEON_FMT_DITHER_ENABLE)
8816 /* XXX sort out optimal dither settings */
8817 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8818 FMT_RGB_RANDOM_ENABLE |
8819 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
8820 else
8821 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
8822 break;
8823 default:
8824 /* not needed */
8825 break;
8826 }
8827
8828 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8829 }
8830
8831 /* display watermark setup */
8832 /**
8833 * dce8_line_buffer_adjust - Set up the line buffer
8834 *
8835 * @rdev: radeon_device pointer
8836 * @radeon_crtc: the selected display controller
8837 * @mode: the current display mode on the selected display
8838 * controller
8839 *
8840 * Setup up the line buffer allocation for
8841 * the selected display controller (CIK).
8842 * Returns the line buffer size in pixels.
8843 */
8844 static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8845 struct radeon_crtc *radeon_crtc,
8846 struct drm_display_mode *mode)
8847 {
8848 u32 tmp, buffer_alloc, i;
8849 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
8850 /*
8851 * Line Buffer Setup
8852 * There are 6 line buffers, one for each display controllers.
8853 * There are 3 partitions per LB. Select the number of partitions
8854 * to enable based on the display width. For display widths larger
8855 * than 4096, you need use to use 2 display controllers and combine
8856 * them using the stereo blender.
8857 */
8858 if (radeon_crtc->base.enabled && mode) {
8859 if (mode->crtc_hdisplay < 1920) {
8860 tmp = 1;
8861 buffer_alloc = 2;
8862 } else if (mode->crtc_hdisplay < 2560) {
8863 tmp = 2;
8864 buffer_alloc = 2;
8865 } else if (mode->crtc_hdisplay < 4096) {
8866 tmp = 0;
8867 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8868 } else {
8869 DRM_DEBUG_KMS("Mode too big for LB!\n");
8870 tmp = 0;
8871 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8872 }
8873 } else {
8874 tmp = 1;
8875 buffer_alloc = 0;
8876 }
8877
8878 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8879 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8880
8881 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8882 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8883 for (i = 0; i < rdev->usec_timeout; i++) {
8884 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8885 DMIF_BUFFERS_ALLOCATED_COMPLETED)
8886 break;
8887 udelay(1);
8888 }
8889
8890 if (radeon_crtc->base.enabled && mode) {
8891 switch (tmp) {
8892 case 0:
8893 default:
8894 return 4096 * 2;
8895 case 1:
8896 return 1920 * 2;
8897 case 2:
8898 return 2560 * 2;
8899 }
8900 }
8901
8902 /* controller not enabled, so no lb used */
8903 return 0;
8904 }
8905
8906 /**
8907 * cik_get_number_of_dram_channels - get the number of dram channels
8908 *
8909 * @rdev: radeon_device pointer
8910 *
8911 * Look up the number of video ram channels (CIK).
8912 * Used for display watermark bandwidth calculations
8913 * Returns the number of dram channels
8914 */
8915 static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8916 {
8917 u32 tmp = RREG32(MC_SHARED_CHMAP);
8918
8919 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8920 case 0:
8921 default:
8922 return 1;
8923 case 1:
8924 return 2;
8925 case 2:
8926 return 4;
8927 case 3:
8928 return 8;
8929 case 4:
8930 return 3;
8931 case 5:
8932 return 6;
8933 case 6:
8934 return 10;
8935 case 7:
8936 return 12;
8937 case 8:
8938 return 16;
8939 }
8940 }
8941
8942 struct dce8_wm_params {
8943 u32 dram_channels; /* number of dram channels */
8944 u32 yclk; /* bandwidth per dram data pin in kHz */
8945 u32 sclk; /* engine clock in kHz */
8946 u32 disp_clk; /* display clock in kHz */
8947 u32 src_width; /* viewport width */
8948 u32 active_time; /* active display time in ns */
8949 u32 blank_time; /* blank time in ns */
8950 bool interlaced; /* mode is interlaced */
8951 fixed20_12 vsc; /* vertical scale ratio */
8952 u32 num_heads; /* number of active crtcs */
8953 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8954 u32 lb_size; /* line buffer allocated to pipe */
8955 u32 vtaps; /* vertical scaler taps */
8956 };
8957
8958 /**
8959 * dce8_dram_bandwidth - get the dram bandwidth
8960 *
8961 * @wm: watermark calculation data
8962 *
8963 * Calculate the raw dram bandwidth (CIK).
8964 * Used for display watermark bandwidth calculations
8965 * Returns the dram bandwidth in MBytes/s
8966 */
8967 static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
8968 {
8969 /* Calculate raw DRAM Bandwidth */
8970 fixed20_12 dram_efficiency; /* 0.7 */
8971 fixed20_12 yclk, dram_channels, bandwidth;
8972 fixed20_12 a;
8973
8974 a.full = dfixed_const(1000);
8975 yclk.full = dfixed_const(wm->yclk);
8976 yclk.full = dfixed_div(yclk, a);
8977 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8978 a.full = dfixed_const(10);
8979 dram_efficiency.full = dfixed_const(7);
8980 dram_efficiency.full = dfixed_div(dram_efficiency, a);
8981 bandwidth.full = dfixed_mul(dram_channels, yclk);
8982 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
8983
8984 return dfixed_trunc(bandwidth);
8985 }
8986
8987 /**
8988 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
8989 *
8990 * @wm: watermark calculation data
8991 *
8992 * Calculate the dram bandwidth used for display (CIK).
8993 * Used for display watermark bandwidth calculations
8994 * Returns the dram bandwidth for display in MBytes/s
8995 */
8996 static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8997 {
8998 /* Calculate DRAM Bandwidth and the part allocated to display. */
8999 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
9000 fixed20_12 yclk, dram_channels, bandwidth;
9001 fixed20_12 a;
9002
9003 a.full = dfixed_const(1000);
9004 yclk.full = dfixed_const(wm->yclk);
9005 yclk.full = dfixed_div(yclk, a);
9006 dram_channels.full = dfixed_const(wm->dram_channels * 4);
9007 a.full = dfixed_const(10);
9008 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
9009 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
9010 bandwidth.full = dfixed_mul(dram_channels, yclk);
9011 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
9012
9013 return dfixed_trunc(bandwidth);
9014 }
9015
9016 /**
9017 * dce8_data_return_bandwidth - get the data return bandwidth
9018 *
9019 * @wm: watermark calculation data
9020 *
9021 * Calculate the data return bandwidth used for display (CIK).
9022 * Used for display watermark bandwidth calculations
9023 * Returns the data return bandwidth in MBytes/s
9024 */
9025 static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
9026 {
9027 /* Calculate the display Data return Bandwidth */
9028 fixed20_12 return_efficiency; /* 0.8 */
9029 fixed20_12 sclk, bandwidth;
9030 fixed20_12 a;
9031
9032 a.full = dfixed_const(1000);
9033 sclk.full = dfixed_const(wm->sclk);
9034 sclk.full = dfixed_div(sclk, a);
9035 a.full = dfixed_const(10);
9036 return_efficiency.full = dfixed_const(8);
9037 return_efficiency.full = dfixed_div(return_efficiency, a);
9038 a.full = dfixed_const(32);
9039 bandwidth.full = dfixed_mul(a, sclk);
9040 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
9041
9042 return dfixed_trunc(bandwidth);
9043 }
9044
9045 /**
9046 * dce8_dmif_request_bandwidth - get the dmif bandwidth
9047 *
9048 * @wm: watermark calculation data
9049 *
9050 * Calculate the dmif bandwidth used for display (CIK).
9051 * Used for display watermark bandwidth calculations
9052 * Returns the dmif bandwidth in MBytes/s
9053 */
9054 static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
9055 {
9056 /* Calculate the DMIF Request Bandwidth */
9057 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
9058 fixed20_12 disp_clk, bandwidth;
9059 fixed20_12 a, b;
9060
9061 a.full = dfixed_const(1000);
9062 disp_clk.full = dfixed_const(wm->disp_clk);
9063 disp_clk.full = dfixed_div(disp_clk, a);
9064 a.full = dfixed_const(32);
9065 b.full = dfixed_mul(a, disp_clk);
9066
9067 a.full = dfixed_const(10);
9068 disp_clk_request_efficiency.full = dfixed_const(8);
9069 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
9070
9071 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
9072
9073 return dfixed_trunc(bandwidth);
9074 }
9075
9076 /**
9077 * dce8_available_bandwidth - get the min available bandwidth
9078 *
9079 * @wm: watermark calculation data
9080 *
9081 * Calculate the min available bandwidth used for display (CIK).
9082 * Used for display watermark bandwidth calculations
9083 * Returns the min available bandwidth in MBytes/s
9084 */
9085 static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
9086 {
9087 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
9088 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
9089 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
9090 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
9091
9092 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
9093 }
9094
9095 /**
9096 * dce8_average_bandwidth - get the average available bandwidth
9097 *
9098 * @wm: watermark calculation data
9099 *
9100 * Calculate the average available bandwidth used for display (CIK).
9101 * Used for display watermark bandwidth calculations
9102 * Returns the average available bandwidth in MBytes/s
9103 */
9104 static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
9105 {
9106 /* Calculate the display mode Average Bandwidth
9107 * DisplayMode should contain the source and destination dimensions,
9108 * timing, etc.
9109 */
9110 fixed20_12 bpp;
9111 fixed20_12 line_time;
9112 fixed20_12 src_width;
9113 fixed20_12 bandwidth;
9114 fixed20_12 a;
9115
9116 a.full = dfixed_const(1000);
9117 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
9118 line_time.full = dfixed_div(line_time, a);
9119 bpp.full = dfixed_const(wm->bytes_per_pixel);
9120 src_width.full = dfixed_const(wm->src_width);
9121 bandwidth.full = dfixed_mul(src_width, bpp);
9122 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
9123 bandwidth.full = dfixed_div(bandwidth, line_time);
9124
9125 return dfixed_trunc(bandwidth);
9126 }
9127
9128 /**
9129 * dce8_latency_watermark - get the latency watermark
9130 *
9131 * @wm: watermark calculation data
9132 *
9133 * Calculate the latency watermark (CIK).
9134 * Used for display watermark bandwidth calculations
9135 * Returns the latency watermark in ns
9136 */
9137 static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
9138 {
9139 /* First calculate the latency in ns */
9140 u32 mc_latency = 2000; /* 2000 ns. */
9141 u32 available_bandwidth = dce8_available_bandwidth(wm);
9142 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
9143 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
9144 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
9145 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
9146 (wm->num_heads * cursor_line_pair_return_time);
9147 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
9148 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
9149 u32 tmp, dmif_size = 12288;
9150 fixed20_12 a, b, c;
9151
9152 if (wm->num_heads == 0)
9153 return 0;
9154
9155 a.full = dfixed_const(2);
9156 b.full = dfixed_const(1);
9157 if ((wm->vsc.full > a.full) ||
9158 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
9159 (wm->vtaps >= 5) ||
9160 ((wm->vsc.full >= a.full) && wm->interlaced))
9161 max_src_lines_per_dst_line = 4;
9162 else
9163 max_src_lines_per_dst_line = 2;
9164
9165 a.full = dfixed_const(available_bandwidth);
9166 b.full = dfixed_const(wm->num_heads);
9167 a.full = dfixed_div(a, b);
9168 tmp = div_u64((u64) dmif_size * (u64) wm->disp_clk, mc_latency + 512);
9169 tmp = min(dfixed_trunc(a), tmp);
9170
9171 lb_fill_bw = min(tmp, wm->disp_clk * wm->bytes_per_pixel / 1000);
9172
9173 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
9174 b.full = dfixed_const(1000);
9175 c.full = dfixed_const(lb_fill_bw);
9176 b.full = dfixed_div(c, b);
9177 a.full = dfixed_div(a, b);
9178 line_fill_time = dfixed_trunc(a);
9179
9180 if (line_fill_time < wm->active_time)
9181 return latency;
9182 else
9183 return latency + (line_fill_time - wm->active_time);
9184
9185 }
9186
9187 /**
9188 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
9189 * average and available dram bandwidth
9190 *
9191 * @wm: watermark calculation data
9192 *
9193 * Check if the display average bandwidth fits in the display
9194 * dram bandwidth (CIK).
9195 * Used for display watermark bandwidth calculations
9196 * Returns true if the display fits, false if not.
9197 */
9198 static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
9199 {
9200 if (dce8_average_bandwidth(wm) <=
9201 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
9202 return true;
9203 else
9204 return false;
9205 }
9206
9207 /**
9208 * dce8_average_bandwidth_vs_available_bandwidth - check
9209 * average and available bandwidth
9210 *
9211 * @wm: watermark calculation data
9212 *
9213 * Check if the display average bandwidth fits in the display
9214 * available bandwidth (CIK).
9215 * Used for display watermark bandwidth calculations
9216 * Returns true if the display fits, false if not.
9217 */
9218 static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
9219 {
9220 if (dce8_average_bandwidth(wm) <=
9221 (dce8_available_bandwidth(wm) / wm->num_heads))
9222 return true;
9223 else
9224 return false;
9225 }
9226
9227 /**
9228 * dce8_check_latency_hiding - check latency hiding
9229 *
9230 * @wm: watermark calculation data
9231 *
9232 * Check latency hiding (CIK).
9233 * Used for display watermark bandwidth calculations
9234 * Returns true if the display fits, false if not.
9235 */
9236 static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
9237 {
9238 u32 lb_partitions = wm->lb_size / wm->src_width;
9239 u32 line_time = wm->active_time + wm->blank_time;
9240 u32 latency_tolerant_lines;
9241 u32 latency_hiding;
9242 fixed20_12 a;
9243
9244 a.full = dfixed_const(1);
9245 if (wm->vsc.full > a.full)
9246 latency_tolerant_lines = 1;
9247 else {
9248 if (lb_partitions <= (wm->vtaps + 1))
9249 latency_tolerant_lines = 1;
9250 else
9251 latency_tolerant_lines = 2;
9252 }
9253
9254 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
9255
9256 if (dce8_latency_watermark(wm) <= latency_hiding)
9257 return true;
9258 else
9259 return false;
9260 }
9261
9262 /**
9263 * dce8_program_watermarks - program display watermarks
9264 *
9265 * @rdev: radeon_device pointer
9266 * @radeon_crtc: the selected display controller
9267 * @lb_size: line buffer size
9268 * @num_heads: number of display controllers in use
9269 *
9270 * Calculate and program the display watermarks for the
9271 * selected display controller (CIK).
9272 */
9273 static void dce8_program_watermarks(struct radeon_device *rdev,
9274 struct radeon_crtc *radeon_crtc,
9275 u32 lb_size, u32 num_heads)
9276 {
9277 struct drm_display_mode *mode = &radeon_crtc->base.mode;
9278 struct dce8_wm_params wm_low, wm_high;
9279 u32 active_time;
9280 u32 line_time = 0;
9281 u32 latency_watermark_a = 0, latency_watermark_b = 0;
9282 u32 tmp, wm_mask;
9283
9284 if (radeon_crtc->base.enabled && num_heads && mode) {
9285 active_time = (u32) div_u64((u64)mode->crtc_hdisplay * 1000000,
9286 (u32)mode->clock);
9287 line_time = (u32) div_u64((u64)mode->crtc_htotal * 1000000,
9288 (u32)mode->clock);
9289 line_time = min(line_time, (u32)65535);
9290
9291 /* watermark for high clocks */
9292 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9293 rdev->pm.dpm_enabled) {
9294 wm_high.yclk =
9295 radeon_dpm_get_mclk(rdev, false) * 10;
9296 wm_high.sclk =
9297 radeon_dpm_get_sclk(rdev, false) * 10;
9298 } else {
9299 wm_high.yclk = rdev->pm.current_mclk * 10;
9300 wm_high.sclk = rdev->pm.current_sclk * 10;
9301 }
9302
9303 wm_high.disp_clk = mode->clock;
9304 wm_high.src_width = mode->crtc_hdisplay;
9305 wm_high.active_time = active_time;
9306 wm_high.blank_time = line_time - wm_high.active_time;
9307 wm_high.interlaced = false;
9308 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
9309 wm_high.interlaced = true;
9310 wm_high.vsc = radeon_crtc->vsc;
9311 wm_high.vtaps = 1;
9312 if (radeon_crtc->rmx_type != RMX_OFF)
9313 wm_high.vtaps = 2;
9314 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
9315 wm_high.lb_size = lb_size;
9316 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
9317 wm_high.num_heads = num_heads;
9318
9319 /* set for high clocks */
9320 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
9321
9322 /* possibly force display priority to high */
9323 /* should really do this at mode validation time... */
9324 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
9325 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
9326 !dce8_check_latency_hiding(&wm_high) ||
9327 (rdev->disp_priority == 2)) {
9328 DRM_DEBUG_KMS("force priority to high\n");
9329 }
9330
9331 /* watermark for low clocks */
9332 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9333 rdev->pm.dpm_enabled) {
9334 wm_low.yclk =
9335 radeon_dpm_get_mclk(rdev, true) * 10;
9336 wm_low.sclk =
9337 radeon_dpm_get_sclk(rdev, true) * 10;
9338 } else {
9339 wm_low.yclk = rdev->pm.current_mclk * 10;
9340 wm_low.sclk = rdev->pm.current_sclk * 10;
9341 }
9342
9343 wm_low.disp_clk = mode->clock;
9344 wm_low.src_width = mode->crtc_hdisplay;
9345 wm_low.active_time = active_time;
9346 wm_low.blank_time = line_time - wm_low.active_time;
9347 wm_low.interlaced = false;
9348 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
9349 wm_low.interlaced = true;
9350 wm_low.vsc = radeon_crtc->vsc;
9351 wm_low.vtaps = 1;
9352 if (radeon_crtc->rmx_type != RMX_OFF)
9353 wm_low.vtaps = 2;
9354 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
9355 wm_low.lb_size = lb_size;
9356 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
9357 wm_low.num_heads = num_heads;
9358
9359 /* set for low clocks */
9360 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
9361
9362 /* possibly force display priority to high */
9363 /* should really do this at mode validation time... */
9364 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
9365 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
9366 !dce8_check_latency_hiding(&wm_low) ||
9367 (rdev->disp_priority == 2)) {
9368 DRM_DEBUG_KMS("force priority to high\n");
9369 }
9370
9371 /* Save number of lines the linebuffer leads before the scanout */
9372 radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
9373 }
9374
9375 /* select wm A */
9376 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9377 tmp = wm_mask;
9378 tmp &= ~LATENCY_WATERMARK_MASK(3);
9379 tmp |= LATENCY_WATERMARK_MASK(1);
9380 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9381 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9382 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
9383 LATENCY_HIGH_WATERMARK(line_time)));
9384 /* select wm B */
9385 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9386 tmp &= ~LATENCY_WATERMARK_MASK(3);
9387 tmp |= LATENCY_WATERMARK_MASK(2);
9388 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9389 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9390 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
9391 LATENCY_HIGH_WATERMARK(line_time)));
9392 /* restore original selection */
9393 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
9394
9395 /* save values for DPM */
9396 radeon_crtc->line_time = line_time;
9397 radeon_crtc->wm_high = latency_watermark_a;
9398 radeon_crtc->wm_low = latency_watermark_b;
9399 }
9400
9401 /**
9402 * dce8_bandwidth_update - program display watermarks
9403 *
9404 * @rdev: radeon_device pointer
9405 *
9406 * Calculate and program the display watermarks and line
9407 * buffer allocation (CIK).
9408 */
9409 void dce8_bandwidth_update(struct radeon_device *rdev)
9410 {
9411 struct drm_display_mode *mode = NULL;
9412 u32 num_heads = 0, lb_size;
9413 int i;
9414
9415 if (!rdev->mode_info.mode_config_initialized)
9416 return;
9417
9418 radeon_update_display_priority(rdev);
9419
9420 for (i = 0; i < rdev->num_crtc; i++) {
9421 if (rdev->mode_info.crtcs[i]->base.enabled)
9422 num_heads++;
9423 }
9424 for (i = 0; i < rdev->num_crtc; i++) {
9425 mode = &rdev->mode_info.crtcs[i]->base.mode;
9426 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
9427 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
9428 }
9429 }
9430
9431 /**
9432 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
9433 *
9434 * @rdev: radeon_device pointer
9435 *
9436 * Fetches a GPU clock counter snapshot (SI).
9437 * Returns the 64 bit clock counter snapshot.
9438 */
9439 uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
9440 {
9441 uint64_t clock;
9442
9443 mutex_lock(&rdev->gpu_clock_mutex);
9444 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
9445 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
9446 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
9447 mutex_unlock(&rdev->gpu_clock_mutex);
9448 return clock;
9449 }
9450
9451 static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
9452 u32 cntl_reg, u32 status_reg)
9453 {
9454 int r, i;
9455 struct atom_clock_dividers dividers;
9456 uint32_t tmp;
9457
9458 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9459 clock, false, &dividers);
9460 if (r)
9461 return r;
9462
9463 tmp = RREG32_SMC(cntl_reg);
9464 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
9465 tmp |= dividers.post_divider;
9466 WREG32_SMC(cntl_reg, tmp);
9467
9468 for (i = 0; i < 100; i++) {
9469 if (RREG32_SMC(status_reg) & DCLK_STATUS)
9470 break;
9471 mdelay(10);
9472 }
9473 if (i == 100)
9474 return -ETIMEDOUT;
9475
9476 return 0;
9477 }
9478
9479 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
9480 {
9481 int r = 0;
9482
9483 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
9484 if (r)
9485 return r;
9486
9487 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
9488 return r;
9489 }
9490
9491 int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
9492 {
9493 int r, i;
9494 struct atom_clock_dividers dividers;
9495 u32 tmp;
9496
9497 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9498 ecclk, false, &dividers);
9499 if (r)
9500 return r;
9501
9502 for (i = 0; i < 100; i++) {
9503 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9504 break;
9505 mdelay(10);
9506 }
9507 if (i == 100)
9508 return -ETIMEDOUT;
9509
9510 tmp = RREG32_SMC(CG_ECLK_CNTL);
9511 tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK);
9512 tmp |= dividers.post_divider;
9513 WREG32_SMC(CG_ECLK_CNTL, tmp);
9514
9515 for (i = 0; i < 100; i++) {
9516 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9517 break;
9518 mdelay(10);
9519 }
9520 if (i == 100)
9521 return -ETIMEDOUT;
9522
9523 return 0;
9524 }
9525
9526 static void cik_pcie_gen3_enable(struct radeon_device *rdev)
9527 {
9528 struct pci_dev *root = rdev->pdev->bus->self;
9529 int bridge_pos, gpu_pos;
9530 u32 speed_cntl, mask, current_data_rate;
9531 int ret, i;
9532 u16 tmp16;
9533
9534 if (pci_is_root_bus(rdev->pdev->bus))
9535 return;
9536
9537 if (radeon_pcie_gen2 == 0)
9538 return;
9539
9540 if (rdev->flags & RADEON_IS_IGP)
9541 return;
9542
9543 if (!(rdev->flags & RADEON_IS_PCIE))
9544 return;
9545
9546 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
9547 if (ret != 0)
9548 return;
9549
9550 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
9551 return;
9552
9553 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9554 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
9555 LC_CURRENT_DATA_RATE_SHIFT;
9556 if (mask & DRM_PCIE_SPEED_80) {
9557 if (current_data_rate == 2) {
9558 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
9559 return;
9560 }
9561 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
9562 } else if (mask & DRM_PCIE_SPEED_50) {
9563 if (current_data_rate == 1) {
9564 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
9565 return;
9566 }
9567 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
9568 }
9569
9570 bridge_pos = pci_pcie_cap(root);
9571 if (!bridge_pos)
9572 return;
9573
9574 gpu_pos = pci_pcie_cap(rdev->pdev);
9575 if (!gpu_pos)
9576 return;
9577
9578 if (mask & DRM_PCIE_SPEED_80) {
9579 /* re-try equalization if gen3 is not already enabled */
9580 if (current_data_rate != 2) {
9581 u16 bridge_cfg, gpu_cfg;
9582 u16 bridge_cfg2, gpu_cfg2;
9583 u32 max_lw, current_lw, tmp;
9584
9585 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9586 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9587
9588 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
9589 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9590
9591 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
9592 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9593
9594 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9595 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
9596 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
9597
9598 if (current_lw < max_lw) {
9599 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9600 if (tmp & LC_RENEGOTIATION_SUPPORT) {
9601 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
9602 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
9603 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
9604 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
9605 }
9606 }
9607
9608 for (i = 0; i < 10; i++) {
9609 /* check status */
9610 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
9611 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
9612 break;
9613
9614 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9615 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9616
9617 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
9618 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
9619
9620 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9621 tmp |= LC_SET_QUIESCE;
9622 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9623
9624 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9625 tmp |= LC_REDO_EQ;
9626 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9627
9628 mdelay(100);
9629
9630 /* linkctl */
9631 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
9632 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9633 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
9634 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9635
9636 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
9637 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9638 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
9639 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9640
9641 /* linkctl2 */
9642 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
9643 tmp16 &= ~((1 << 4) | (7 << 9));
9644 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
9645 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
9646
9647 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9648 tmp16 &= ~((1 << 4) | (7 << 9));
9649 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
9650 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9651
9652 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9653 tmp &= ~LC_SET_QUIESCE;
9654 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9655 }
9656 }
9657 }
9658
9659 /* set the link speed */
9660 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
9661 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
9662 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9663
9664 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9665 tmp16 &= ~0xf;
9666 if (mask & DRM_PCIE_SPEED_80)
9667 tmp16 |= 3; /* gen3 */
9668 else if (mask & DRM_PCIE_SPEED_50)
9669 tmp16 |= 2; /* gen2 */
9670 else
9671 tmp16 |= 1; /* gen1 */
9672 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9673
9674 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9675 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
9676 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9677
9678 for (i = 0; i < rdev->usec_timeout; i++) {
9679 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9680 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
9681 break;
9682 udelay(1);
9683 }
9684 }
9685
9686 static void cik_program_aspm(struct radeon_device *rdev)
9687 {
9688 u32 data, orig;
9689 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
9690 bool disable_clkreq = false;
9691
9692 if (radeon_aspm == 0)
9693 return;
9694
9695 /* XXX double check IGPs */
9696 if (rdev->flags & RADEON_IS_IGP)
9697 return;
9698
9699 if (!(rdev->flags & RADEON_IS_PCIE))
9700 return;
9701
9702 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9703 data &= ~LC_XMIT_N_FTS_MASK;
9704 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
9705 if (orig != data)
9706 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
9707
9708 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
9709 data |= LC_GO_TO_RECOVERY;
9710 if (orig != data)
9711 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
9712
9713 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
9714 data |= P_IGNORE_EDB_ERR;
9715 if (orig != data)
9716 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
9717
9718 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9719 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
9720 data |= LC_PMI_TO_L1_DIS;
9721 if (!disable_l0s)
9722 data |= LC_L0S_INACTIVITY(7);
9723
9724 if (!disable_l1) {
9725 data |= LC_L1_INACTIVITY(7);
9726 data &= ~LC_PMI_TO_L1_DIS;
9727 if (orig != data)
9728 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9729
9730 if (!disable_plloff_in_l1) {
9731 bool clk_req_support;
9732
9733 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
9734 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9735 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9736 if (orig != data)
9737 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
9738
9739 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
9740 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9741 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9742 if (orig != data)
9743 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
9744
9745 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
9746 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9747 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9748 if (orig != data)
9749 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
9750
9751 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
9752 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9753 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9754 if (orig != data)
9755 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
9756
9757 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9758 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
9759 data |= LC_DYN_LANES_PWR_STATE(3);
9760 if (orig != data)
9761 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
9762
9763 if (!disable_clkreq &&
9764 !pci_is_root_bus(rdev->pdev->bus)) {
9765 struct pci_dev *root = rdev->pdev->bus->self;
9766 u32 lnkcap;
9767
9768 clk_req_support = false;
9769 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
9770 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
9771 clk_req_support = true;
9772 } else {
9773 clk_req_support = false;
9774 }
9775
9776 if (clk_req_support) {
9777 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
9778 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
9779 if (orig != data)
9780 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
9781
9782 orig = data = RREG32_SMC(THM_CLK_CNTL);
9783 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
9784 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
9785 if (orig != data)
9786 WREG32_SMC(THM_CLK_CNTL, data);
9787
9788 orig = data = RREG32_SMC(MISC_CLK_CTRL);
9789 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
9790 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
9791 if (orig != data)
9792 WREG32_SMC(MISC_CLK_CTRL, data);
9793
9794 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
9795 data &= ~BCLK_AS_XCLK;
9796 if (orig != data)
9797 WREG32_SMC(CG_CLKPIN_CNTL, data);
9798
9799 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
9800 data &= ~FORCE_BIF_REFCLK_EN;
9801 if (orig != data)
9802 WREG32_SMC(CG_CLKPIN_CNTL_2, data);
9803
9804 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
9805 data &= ~MPLL_CLKOUT_SEL_MASK;
9806 data |= MPLL_CLKOUT_SEL(4);
9807 if (orig != data)
9808 WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
9809 }
9810 }
9811 } else {
9812 if (orig != data)
9813 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9814 }
9815
9816 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
9817 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
9818 if (orig != data)
9819 WREG32_PCIE_PORT(PCIE_CNTL2, data);
9820
9821 if (!disable_l0s) {
9822 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9823 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
9824 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9825 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
9826 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9827 data &= ~LC_L0S_INACTIVITY_MASK;
9828 if (orig != data)
9829 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9830 }
9831 }
9832 }
9833 }
CRIS linux kernel tree