?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
sched: Remove double_rq_lock() from __migrate_task()
[linux/fpc-iii.git] / drivers / gpu / drm / radeon / cik.c
blobc0ea66192fe03a8fe977ac0aea977a07204530c4
1 /*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
27 #include "drmP.h"
28 #include "radeon.h"
29 #include "radeon_asic.h"
30 #include "cikd.h"
31 #include "atom.h"
32 #include "cik_blit_shaders.h"
33 #include "radeon_ucode.h"
34 #include "clearstate_ci.h"
35
36 MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
37 MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
38 MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
39 MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
40 MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
41 MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin");
42 MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
43 MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
44 MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
45 MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
46 MODULE_FIRMWARE("radeon/HAWAII_me.bin");
47 MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
48 MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
49 MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
50 MODULE_FIRMWARE("radeon/HAWAII_mc2.bin");
51 MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
52 MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
53 MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
54 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
55 MODULE_FIRMWARE("radeon/KAVERI_me.bin");
56 MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
57 MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
58 MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
59 MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
60 MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
61 MODULE_FIRMWARE("radeon/KABINI_me.bin");
62 MODULE_FIRMWARE("radeon/KABINI_ce.bin");
63 MODULE_FIRMWARE("radeon/KABINI_mec.bin");
64 MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
65 MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
66 MODULE_FIRMWARE("radeon/MULLINS_pfp.bin");
67 MODULE_FIRMWARE("radeon/MULLINS_me.bin");
68 MODULE_FIRMWARE("radeon/MULLINS_ce.bin");
69 MODULE_FIRMWARE("radeon/MULLINS_mec.bin");
70 MODULE_FIRMWARE("radeon/MULLINS_rlc.bin");
71 MODULE_FIRMWARE("radeon/MULLINS_sdma.bin");
72
73 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
74 extern void r600_ih_ring_fini(struct radeon_device *rdev);
75 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
76 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
77 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
78 extern void sumo_rlc_fini(struct radeon_device *rdev);
79 extern int sumo_rlc_init(struct radeon_device *rdev);
80 extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
81 extern void si_rlc_reset(struct radeon_device *rdev);
82 extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
83 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
84 extern int cik_sdma_resume(struct radeon_device *rdev);
85 extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
86 extern void cik_sdma_fini(struct radeon_device *rdev);
87 extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable);
88 static void cik_rlc_stop(struct radeon_device *rdev);
89 static void cik_pcie_gen3_enable(struct radeon_device *rdev);
90 static void cik_program_aspm(struct radeon_device *rdev);
91 static void cik_init_pg(struct radeon_device *rdev);
92 static void cik_init_cg(struct radeon_device *rdev);
93 static void cik_fini_pg(struct radeon_device *rdev);
94 static void cik_fini_cg(struct radeon_device *rdev);
95 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
96 bool enable);
97
98 /* get temperature in millidegrees */
99 int ci_get_temp(struct radeon_device *rdev)
100 {
101 u32 temp;
102 int actual_temp = 0;
103
104 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
105 CTF_TEMP_SHIFT;
106
107 if (temp & 0x200)
108 actual_temp = 255;
109 else
110 actual_temp = temp & 0x1ff;
111
112 actual_temp = actual_temp * 1000;
113
114 return actual_temp;
115 }
116
117 /* get temperature in millidegrees */
118 int kv_get_temp(struct radeon_device *rdev)
119 {
120 u32 temp;
121 int actual_temp = 0;
122
123 temp = RREG32_SMC(0xC0300E0C);
124
125 if (temp)
126 actual_temp = (temp / 8) - 49;
127 else
128 actual_temp = 0;
129
130 actual_temp = actual_temp * 1000;
131
132 return actual_temp;
133 }
134
135 /*
136 * Indirect registers accessor
137 */
138 u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
139 {
140 unsigned long flags;
141 u32 r;
142
143 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
144 WREG32(PCIE_INDEX, reg);
145 (void)RREG32(PCIE_INDEX);
146 r = RREG32(PCIE_DATA);
147 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
148 return r;
149 }
150
151 void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
152 {
153 unsigned long flags;
154
155 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
156 WREG32(PCIE_INDEX, reg);
157 (void)RREG32(PCIE_INDEX);
158 WREG32(PCIE_DATA, v);
159 (void)RREG32(PCIE_DATA);
160 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
161 }
162
163 static const u32 spectre_rlc_save_restore_register_list[] =
164 {
165 (0x0e00 << 16) | (0xc12c >> 2),
166 0x00000000,
167 (0x0e00 << 16) | (0xc140 >> 2),
168 0x00000000,
169 (0x0e00 << 16) | (0xc150 >> 2),
170 0x00000000,
171 (0x0e00 << 16) | (0xc15c >> 2),
172 0x00000000,
173 (0x0e00 << 16) | (0xc168 >> 2),
174 0x00000000,
175 (0x0e00 << 16) | (0xc170 >> 2),
176 0x00000000,
177 (0x0e00 << 16) | (0xc178 >> 2),
178 0x00000000,
179 (0x0e00 << 16) | (0xc204 >> 2),
180 0x00000000,
181 (0x0e00 << 16) | (0xc2b4 >> 2),
182 0x00000000,
183 (0x0e00 << 16) | (0xc2b8 >> 2),
184 0x00000000,
185 (0x0e00 << 16) | (0xc2bc >> 2),
186 0x00000000,
187 (0x0e00 << 16) | (0xc2c0 >> 2),
188 0x00000000,
189 (0x0e00 << 16) | (0x8228 >> 2),
190 0x00000000,
191 (0x0e00 << 16) | (0x829c >> 2),
192 0x00000000,
193 (0x0e00 << 16) | (0x869c >> 2),
194 0x00000000,
195 (0x0600 << 16) | (0x98f4 >> 2),
196 0x00000000,
197 (0x0e00 << 16) | (0x98f8 >> 2),
198 0x00000000,
199 (0x0e00 << 16) | (0x9900 >> 2),
200 0x00000000,
201 (0x0e00 << 16) | (0xc260 >> 2),
202 0x00000000,
203 (0x0e00 << 16) | (0x90e8 >> 2),
204 0x00000000,
205 (0x0e00 << 16) | (0x3c000 >> 2),
206 0x00000000,
207 (0x0e00 << 16) | (0x3c00c >> 2),
208 0x00000000,
209 (0x0e00 << 16) | (0x8c1c >> 2),
210 0x00000000,
211 (0x0e00 << 16) | (0x9700 >> 2),
212 0x00000000,
213 (0x0e00 << 16) | (0xcd20 >> 2),
214 0x00000000,
215 (0x4e00 << 16) | (0xcd20 >> 2),
216 0x00000000,
217 (0x5e00 << 16) | (0xcd20 >> 2),
218 0x00000000,
219 (0x6e00 << 16) | (0xcd20 >> 2),
220 0x00000000,
221 (0x7e00 << 16) | (0xcd20 >> 2),
222 0x00000000,
223 (0x8e00 << 16) | (0xcd20 >> 2),
224 0x00000000,
225 (0x9e00 << 16) | (0xcd20 >> 2),
226 0x00000000,
227 (0xae00 << 16) | (0xcd20 >> 2),
228 0x00000000,
229 (0xbe00 << 16) | (0xcd20 >> 2),
230 0x00000000,
231 (0x0e00 << 16) | (0x89bc >> 2),
232 0x00000000,
233 (0x0e00 << 16) | (0x8900 >> 2),
234 0x00000000,
235 0x3,
236 (0x0e00 << 16) | (0xc130 >> 2),
237 0x00000000,
238 (0x0e00 << 16) | (0xc134 >> 2),
239 0x00000000,
240 (0x0e00 << 16) | (0xc1fc >> 2),
241 0x00000000,
242 (0x0e00 << 16) | (0xc208 >> 2),
243 0x00000000,
244 (0x0e00 << 16) | (0xc264 >> 2),
245 0x00000000,
246 (0x0e00 << 16) | (0xc268 >> 2),
247 0x00000000,
248 (0x0e00 << 16) | (0xc26c >> 2),
249 0x00000000,
250 (0x0e00 << 16) | (0xc270 >> 2),
251 0x00000000,
252 (0x0e00 << 16) | (0xc274 >> 2),
253 0x00000000,
254 (0x0e00 << 16) | (0xc278 >> 2),
255 0x00000000,
256 (0x0e00 << 16) | (0xc27c >> 2),
257 0x00000000,
258 (0x0e00 << 16) | (0xc280 >> 2),
259 0x00000000,
260 (0x0e00 << 16) | (0xc284 >> 2),
261 0x00000000,
262 (0x0e00 << 16) | (0xc288 >> 2),
263 0x00000000,
264 (0x0e00 << 16) | (0xc28c >> 2),
265 0x00000000,
266 (0x0e00 << 16) | (0xc290 >> 2),
267 0x00000000,
268 (0x0e00 << 16) | (0xc294 >> 2),
269 0x00000000,
270 (0x0e00 << 16) | (0xc298 >> 2),
271 0x00000000,
272 (0x0e00 << 16) | (0xc29c >> 2),
273 0x00000000,
274 (0x0e00 << 16) | (0xc2a0 >> 2),
275 0x00000000,
276 (0x0e00 << 16) | (0xc2a4 >> 2),
277 0x00000000,
278 (0x0e00 << 16) | (0xc2a8 >> 2),
279 0x00000000,
280 (0x0e00 << 16) | (0xc2ac >> 2),
281 0x00000000,
282 (0x0e00 << 16) | (0xc2b0 >> 2),
283 0x00000000,
284 (0x0e00 << 16) | (0x301d0 >> 2),
285 0x00000000,
286 (0x0e00 << 16) | (0x30238 >> 2),
287 0x00000000,
288 (0x0e00 << 16) | (0x30250 >> 2),
289 0x00000000,
290 (0x0e00 << 16) | (0x30254 >> 2),
291 0x00000000,
292 (0x0e00 << 16) | (0x30258 >> 2),
293 0x00000000,
294 (0x0e00 << 16) | (0x3025c >> 2),
295 0x00000000,
296 (0x4e00 << 16) | (0xc900 >> 2),
297 0x00000000,
298 (0x5e00 << 16) | (0xc900 >> 2),
299 0x00000000,
300 (0x6e00 << 16) | (0xc900 >> 2),
301 0x00000000,
302 (0x7e00 << 16) | (0xc900 >> 2),
303 0x00000000,
304 (0x8e00 << 16) | (0xc900 >> 2),
305 0x00000000,
306 (0x9e00 << 16) | (0xc900 >> 2),
307 0x00000000,
308 (0xae00 << 16) | (0xc900 >> 2),
309 0x00000000,
310 (0xbe00 << 16) | (0xc900 >> 2),
311 0x00000000,
312 (0x4e00 << 16) | (0xc904 >> 2),
313 0x00000000,
314 (0x5e00 << 16) | (0xc904 >> 2),
315 0x00000000,
316 (0x6e00 << 16) | (0xc904 >> 2),
317 0x00000000,
318 (0x7e00 << 16) | (0xc904 >> 2),
319 0x00000000,
320 (0x8e00 << 16) | (0xc904 >> 2),
321 0x00000000,
322 (0x9e00 << 16) | (0xc904 >> 2),
323 0x00000000,
324 (0xae00 << 16) | (0xc904 >> 2),
325 0x00000000,
326 (0xbe00 << 16) | (0xc904 >> 2),
327 0x00000000,
328 (0x4e00 << 16) | (0xc908 >> 2),
329 0x00000000,
330 (0x5e00 << 16) | (0xc908 >> 2),
331 0x00000000,
332 (0x6e00 << 16) | (0xc908 >> 2),
333 0x00000000,
334 (0x7e00 << 16) | (0xc908 >> 2),
335 0x00000000,
336 (0x8e00 << 16) | (0xc908 >> 2),
337 0x00000000,
338 (0x9e00 << 16) | (0xc908 >> 2),
339 0x00000000,
340 (0xae00 << 16) | (0xc908 >> 2),
341 0x00000000,
342 (0xbe00 << 16) | (0xc908 >> 2),
343 0x00000000,
344 (0x4e00 << 16) | (0xc90c >> 2),
345 0x00000000,
346 (0x5e00 << 16) | (0xc90c >> 2),
347 0x00000000,
348 (0x6e00 << 16) | (0xc90c >> 2),
349 0x00000000,
350 (0x7e00 << 16) | (0xc90c >> 2),
351 0x00000000,
352 (0x8e00 << 16) | (0xc90c >> 2),
353 0x00000000,
354 (0x9e00 << 16) | (0xc90c >> 2),
355 0x00000000,
356 (0xae00 << 16) | (0xc90c >> 2),
357 0x00000000,
358 (0xbe00 << 16) | (0xc90c >> 2),
359 0x00000000,
360 (0x4e00 << 16) | (0xc910 >> 2),
361 0x00000000,
362 (0x5e00 << 16) | (0xc910 >> 2),
363 0x00000000,
364 (0x6e00 << 16) | (0xc910 >> 2),
365 0x00000000,
366 (0x7e00 << 16) | (0xc910 >> 2),
367 0x00000000,
368 (0x8e00 << 16) | (0xc910 >> 2),
369 0x00000000,
370 (0x9e00 << 16) | (0xc910 >> 2),
371 0x00000000,
372 (0xae00 << 16) | (0xc910 >> 2),
373 0x00000000,
374 (0xbe00 << 16) | (0xc910 >> 2),
375 0x00000000,
376 (0x0e00 << 16) | (0xc99c >> 2),
377 0x00000000,
378 (0x0e00 << 16) | (0x9834 >> 2),
379 0x00000000,
380 (0x0000 << 16) | (0x30f00 >> 2),
381 0x00000000,
382 (0x0001 << 16) | (0x30f00 >> 2),
383 0x00000000,
384 (0x0000 << 16) | (0x30f04 >> 2),
385 0x00000000,
386 (0x0001 << 16) | (0x30f04 >> 2),
387 0x00000000,
388 (0x0000 << 16) | (0x30f08 >> 2),
389 0x00000000,
390 (0x0001 << 16) | (0x30f08 >> 2),
391 0x00000000,
392 (0x0000 << 16) | (0x30f0c >> 2),
393 0x00000000,
394 (0x0001 << 16) | (0x30f0c >> 2),
395 0x00000000,
396 (0x0600 << 16) | (0x9b7c >> 2),
397 0x00000000,
398 (0x0e00 << 16) | (0x8a14 >> 2),
399 0x00000000,
400 (0x0e00 << 16) | (0x8a18 >> 2),
401 0x00000000,
402 (0x0600 << 16) | (0x30a00 >> 2),
403 0x00000000,
404 (0x0e00 << 16) | (0x8bf0 >> 2),
405 0x00000000,
406 (0x0e00 << 16) | (0x8bcc >> 2),
407 0x00000000,
408 (0x0e00 << 16) | (0x8b24 >> 2),
409 0x00000000,
410 (0x0e00 << 16) | (0x30a04 >> 2),
411 0x00000000,
412 (0x0600 << 16) | (0x30a10 >> 2),
413 0x00000000,
414 (0x0600 << 16) | (0x30a14 >> 2),
415 0x00000000,
416 (0x0600 << 16) | (0x30a18 >> 2),
417 0x00000000,
418 (0x0600 << 16) | (0x30a2c >> 2),
419 0x00000000,
420 (0x0e00 << 16) | (0xc700 >> 2),
421 0x00000000,
422 (0x0e00 << 16) | (0xc704 >> 2),
423 0x00000000,
424 (0x0e00 << 16) | (0xc708 >> 2),
425 0x00000000,
426 (0x0e00 << 16) | (0xc768 >> 2),
427 0x00000000,
428 (0x0400 << 16) | (0xc770 >> 2),
429 0x00000000,
430 (0x0400 << 16) | (0xc774 >> 2),
431 0x00000000,
432 (0x0400 << 16) | (0xc778 >> 2),
433 0x00000000,
434 (0x0400 << 16) | (0xc77c >> 2),
435 0x00000000,
436 (0x0400 << 16) | (0xc780 >> 2),
437 0x00000000,
438 (0x0400 << 16) | (0xc784 >> 2),
439 0x00000000,
440 (0x0400 << 16) | (0xc788 >> 2),
441 0x00000000,
442 (0x0400 << 16) | (0xc78c >> 2),
443 0x00000000,
444 (0x0400 << 16) | (0xc798 >> 2),
445 0x00000000,
446 (0x0400 << 16) | (0xc79c >> 2),
447 0x00000000,
448 (0x0400 << 16) | (0xc7a0 >> 2),
449 0x00000000,
450 (0x0400 << 16) | (0xc7a4 >> 2),
451 0x00000000,
452 (0x0400 << 16) | (0xc7a8 >> 2),
453 0x00000000,
454 (0x0400 << 16) | (0xc7ac >> 2),
455 0x00000000,
456 (0x0400 << 16) | (0xc7b0 >> 2),
457 0x00000000,
458 (0x0400 << 16) | (0xc7b4 >> 2),
459 0x00000000,
460 (0x0e00 << 16) | (0x9100 >> 2),
461 0x00000000,
462 (0x0e00 << 16) | (0x3c010 >> 2),
463 0x00000000,
464 (0x0e00 << 16) | (0x92a8 >> 2),
465 0x00000000,
466 (0x0e00 << 16) | (0x92ac >> 2),
467 0x00000000,
468 (0x0e00 << 16) | (0x92b4 >> 2),
469 0x00000000,
470 (0x0e00 << 16) | (0x92b8 >> 2),
471 0x00000000,
472 (0x0e00 << 16) | (0x92bc >> 2),
473 0x00000000,
474 (0x0e00 << 16) | (0x92c0 >> 2),
475 0x00000000,
476 (0x0e00 << 16) | (0x92c4 >> 2),
477 0x00000000,
478 (0x0e00 << 16) | (0x92c8 >> 2),
479 0x00000000,
480 (0x0e00 << 16) | (0x92cc >> 2),
481 0x00000000,
482 (0x0e00 << 16) | (0x92d0 >> 2),
483 0x00000000,
484 (0x0e00 << 16) | (0x8c00 >> 2),
485 0x00000000,
486 (0x0e00 << 16) | (0x8c04 >> 2),
487 0x00000000,
488 (0x0e00 << 16) | (0x8c20 >> 2),
489 0x00000000,
490 (0x0e00 << 16) | (0x8c38 >> 2),
491 0x00000000,
492 (0x0e00 << 16) | (0x8c3c >> 2),
493 0x00000000,
494 (0x0e00 << 16) | (0xae00 >> 2),
495 0x00000000,
496 (0x0e00 << 16) | (0x9604 >> 2),
497 0x00000000,
498 (0x0e00 << 16) | (0xac08 >> 2),
499 0x00000000,
500 (0x0e00 << 16) | (0xac0c >> 2),
501 0x00000000,
502 (0x0e00 << 16) | (0xac10 >> 2),
503 0x00000000,
504 (0x0e00 << 16) | (0xac14 >> 2),
505 0x00000000,
506 (0x0e00 << 16) | (0xac58 >> 2),
507 0x00000000,
508 (0x0e00 << 16) | (0xac68 >> 2),
509 0x00000000,
510 (0x0e00 << 16) | (0xac6c >> 2),
511 0x00000000,
512 (0x0e00 << 16) | (0xac70 >> 2),
513 0x00000000,
514 (0x0e00 << 16) | (0xac74 >> 2),
515 0x00000000,
516 (0x0e00 << 16) | (0xac78 >> 2),
517 0x00000000,
518 (0x0e00 << 16) | (0xac7c >> 2),
519 0x00000000,
520 (0x0e00 << 16) | (0xac80 >> 2),
521 0x00000000,
522 (0x0e00 << 16) | (0xac84 >> 2),
523 0x00000000,
524 (0x0e00 << 16) | (0xac88 >> 2),
525 0x00000000,
526 (0x0e00 << 16) | (0xac8c >> 2),
527 0x00000000,
528 (0x0e00 << 16) | (0x970c >> 2),
529 0x00000000,
530 (0x0e00 << 16) | (0x9714 >> 2),
531 0x00000000,
532 (0x0e00 << 16) | (0x9718 >> 2),
533 0x00000000,
534 (0x0e00 << 16) | (0x971c >> 2),
535 0x00000000,
536 (0x0e00 << 16) | (0x31068 >> 2),
537 0x00000000,
538 (0x4e00 << 16) | (0x31068 >> 2),
539 0x00000000,
540 (0x5e00 << 16) | (0x31068 >> 2),
541 0x00000000,
542 (0x6e00 << 16) | (0x31068 >> 2),
543 0x00000000,
544 (0x7e00 << 16) | (0x31068 >> 2),
545 0x00000000,
546 (0x8e00 << 16) | (0x31068 >> 2),
547 0x00000000,
548 (0x9e00 << 16) | (0x31068 >> 2),
549 0x00000000,
550 (0xae00 << 16) | (0x31068 >> 2),
551 0x00000000,
552 (0xbe00 << 16) | (0x31068 >> 2),
553 0x00000000,
554 (0x0e00 << 16) | (0xcd10 >> 2),
555 0x00000000,
556 (0x0e00 << 16) | (0xcd14 >> 2),
557 0x00000000,
558 (0x0e00 << 16) | (0x88b0 >> 2),
559 0x00000000,
560 (0x0e00 << 16) | (0x88b4 >> 2),
561 0x00000000,
562 (0x0e00 << 16) | (0x88b8 >> 2),
563 0x00000000,
564 (0x0e00 << 16) | (0x88bc >> 2),
565 0x00000000,
566 (0x0400 << 16) | (0x89c0 >> 2),
567 0x00000000,
568 (0x0e00 << 16) | (0x88c4 >> 2),
569 0x00000000,
570 (0x0e00 << 16) | (0x88c8 >> 2),
571 0x00000000,
572 (0x0e00 << 16) | (0x88d0 >> 2),
573 0x00000000,
574 (0x0e00 << 16) | (0x88d4 >> 2),
575 0x00000000,
576 (0x0e00 << 16) | (0x88d8 >> 2),
577 0x00000000,
578 (0x0e00 << 16) | (0x8980 >> 2),
579 0x00000000,
580 (0x0e00 << 16) | (0x30938 >> 2),
581 0x00000000,
582 (0x0e00 << 16) | (0x3093c >> 2),
583 0x00000000,
584 (0x0e00 << 16) | (0x30940 >> 2),
585 0x00000000,
586 (0x0e00 << 16) | (0x89a0 >> 2),
587 0x00000000,
588 (0x0e00 << 16) | (0x30900 >> 2),
589 0x00000000,
590 (0x0e00 << 16) | (0x30904 >> 2),
591 0x00000000,
592 (0x0e00 << 16) | (0x89b4 >> 2),
593 0x00000000,
594 (0x0e00 << 16) | (0x3c210 >> 2),
595 0x00000000,
596 (0x0e00 << 16) | (0x3c214 >> 2),
597 0x00000000,
598 (0x0e00 << 16) | (0x3c218 >> 2),
599 0x00000000,
600 (0x0e00 << 16) | (0x8904 >> 2),
601 0x00000000,
602 0x5,
603 (0x0e00 << 16) | (0x8c28 >> 2),
604 (0x0e00 << 16) | (0x8c2c >> 2),
605 (0x0e00 << 16) | (0x8c30 >> 2),
606 (0x0e00 << 16) | (0x8c34 >> 2),
607 (0x0e00 << 16) | (0x9600 >> 2),
608 };
609
610 static const u32 kalindi_rlc_save_restore_register_list[] =
611 {
612 (0x0e00 << 16) | (0xc12c >> 2),
613 0x00000000,
614 (0x0e00 << 16) | (0xc140 >> 2),
615 0x00000000,
616 (0x0e00 << 16) | (0xc150 >> 2),
617 0x00000000,
618 (0x0e00 << 16) | (0xc15c >> 2),
619 0x00000000,
620 (0x0e00 << 16) | (0xc168 >> 2),
621 0x00000000,
622 (0x0e00 << 16) | (0xc170 >> 2),
623 0x00000000,
624 (0x0e00 << 16) | (0xc204 >> 2),
625 0x00000000,
626 (0x0e00 << 16) | (0xc2b4 >> 2),
627 0x00000000,
628 (0x0e00 << 16) | (0xc2b8 >> 2),
629 0x00000000,
630 (0x0e00 << 16) | (0xc2bc >> 2),
631 0x00000000,
632 (0x0e00 << 16) | (0xc2c0 >> 2),
633 0x00000000,
634 (0x0e00 << 16) | (0x8228 >> 2),
635 0x00000000,
636 (0x0e00 << 16) | (0x829c >> 2),
637 0x00000000,
638 (0x0e00 << 16) | (0x869c >> 2),
639 0x00000000,
640 (0x0600 << 16) | (0x98f4 >> 2),
641 0x00000000,
642 (0x0e00 << 16) | (0x98f8 >> 2),
643 0x00000000,
644 (0x0e00 << 16) | (0x9900 >> 2),
645 0x00000000,
646 (0x0e00 << 16) | (0xc260 >> 2),
647 0x00000000,
648 (0x0e00 << 16) | (0x90e8 >> 2),
649 0x00000000,
650 (0x0e00 << 16) | (0x3c000 >> 2),
651 0x00000000,
652 (0x0e00 << 16) | (0x3c00c >> 2),
653 0x00000000,
654 (0x0e00 << 16) | (0x8c1c >> 2),
655 0x00000000,
656 (0x0e00 << 16) | (0x9700 >> 2),
657 0x00000000,
658 (0x0e00 << 16) | (0xcd20 >> 2),
659 0x00000000,
660 (0x4e00 << 16) | (0xcd20 >> 2),
661 0x00000000,
662 (0x5e00 << 16) | (0xcd20 >> 2),
663 0x00000000,
664 (0x6e00 << 16) | (0xcd20 >> 2),
665 0x00000000,
666 (0x7e00 << 16) | (0xcd20 >> 2),
667 0x00000000,
668 (0x0e00 << 16) | (0x89bc >> 2),
669 0x00000000,
670 (0x0e00 << 16) | (0x8900 >> 2),
671 0x00000000,
672 0x3,
673 (0x0e00 << 16) | (0xc130 >> 2),
674 0x00000000,
675 (0x0e00 << 16) | (0xc134 >> 2),
676 0x00000000,
677 (0x0e00 << 16) | (0xc1fc >> 2),
678 0x00000000,
679 (0x0e00 << 16) | (0xc208 >> 2),
680 0x00000000,
681 (0x0e00 << 16) | (0xc264 >> 2),
682 0x00000000,
683 (0x0e00 << 16) | (0xc268 >> 2),
684 0x00000000,
685 (0x0e00 << 16) | (0xc26c >> 2),
686 0x00000000,
687 (0x0e00 << 16) | (0xc270 >> 2),
688 0x00000000,
689 (0x0e00 << 16) | (0xc274 >> 2),
690 0x00000000,
691 (0x0e00 << 16) | (0xc28c >> 2),
692 0x00000000,
693 (0x0e00 << 16) | (0xc290 >> 2),
694 0x00000000,
695 (0x0e00 << 16) | (0xc294 >> 2),
696 0x00000000,
697 (0x0e00 << 16) | (0xc298 >> 2),
698 0x00000000,
699 (0x0e00 << 16) | (0xc2a0 >> 2),
700 0x00000000,
701 (0x0e00 << 16) | (0xc2a4 >> 2),
702 0x00000000,
703 (0x0e00 << 16) | (0xc2a8 >> 2),
704 0x00000000,
705 (0x0e00 << 16) | (0xc2ac >> 2),
706 0x00000000,
707 (0x0e00 << 16) | (0x301d0 >> 2),
708 0x00000000,
709 (0x0e00 << 16) | (0x30238 >> 2),
710 0x00000000,
711 (0x0e00 << 16) | (0x30250 >> 2),
712 0x00000000,
713 (0x0e00 << 16) | (0x30254 >> 2),
714 0x00000000,
715 (0x0e00 << 16) | (0x30258 >> 2),
716 0x00000000,
717 (0x0e00 << 16) | (0x3025c >> 2),
718 0x00000000,
719 (0x4e00 << 16) | (0xc900 >> 2),
720 0x00000000,
721 (0x5e00 << 16) | (0xc900 >> 2),
722 0x00000000,
723 (0x6e00 << 16) | (0xc900 >> 2),
724 0x00000000,
725 (0x7e00 << 16) | (0xc900 >> 2),
726 0x00000000,
727 (0x4e00 << 16) | (0xc904 >> 2),
728 0x00000000,
729 (0x5e00 << 16) | (0xc904 >> 2),
730 0x00000000,
731 (0x6e00 << 16) | (0xc904 >> 2),
732 0x00000000,
733 (0x7e00 << 16) | (0xc904 >> 2),
734 0x00000000,
735 (0x4e00 << 16) | (0xc908 >> 2),
736 0x00000000,
737 (0x5e00 << 16) | (0xc908 >> 2),
738 0x00000000,
739 (0x6e00 << 16) | (0xc908 >> 2),
740 0x00000000,
741 (0x7e00 << 16) | (0xc908 >> 2),
742 0x00000000,
743 (0x4e00 << 16) | (0xc90c >> 2),
744 0x00000000,
745 (0x5e00 << 16) | (0xc90c >> 2),
746 0x00000000,
747 (0x6e00 << 16) | (0xc90c >> 2),
748 0x00000000,
749 (0x7e00 << 16) | (0xc90c >> 2),
750 0x00000000,
751 (0x4e00 << 16) | (0xc910 >> 2),
752 0x00000000,
753 (0x5e00 << 16) | (0xc910 >> 2),
754 0x00000000,
755 (0x6e00 << 16) | (0xc910 >> 2),
756 0x00000000,
757 (0x7e00 << 16) | (0xc910 >> 2),
758 0x00000000,
759 (0x0e00 << 16) | (0xc99c >> 2),
760 0x00000000,
761 (0x0e00 << 16) | (0x9834 >> 2),
762 0x00000000,
763 (0x0000 << 16) | (0x30f00 >> 2),
764 0x00000000,
765 (0x0000 << 16) | (0x30f04 >> 2),
766 0x00000000,
767 (0x0000 << 16) | (0x30f08 >> 2),
768 0x00000000,
769 (0x0000 << 16) | (0x30f0c >> 2),
770 0x00000000,
771 (0x0600 << 16) | (0x9b7c >> 2),
772 0x00000000,
773 (0x0e00 << 16) | (0x8a14 >> 2),
774 0x00000000,
775 (0x0e00 << 16) | (0x8a18 >> 2),
776 0x00000000,
777 (0x0600 << 16) | (0x30a00 >> 2),
778 0x00000000,
779 (0x0e00 << 16) | (0x8bf0 >> 2),
780 0x00000000,
781 (0x0e00 << 16) | (0x8bcc >> 2),
782 0x00000000,
783 (0x0e00 << 16) | (0x8b24 >> 2),
784 0x00000000,
785 (0x0e00 << 16) | (0x30a04 >> 2),
786 0x00000000,
787 (0x0600 << 16) | (0x30a10 >> 2),
788 0x00000000,
789 (0x0600 << 16) | (0x30a14 >> 2),
790 0x00000000,
791 (0x0600 << 16) | (0x30a18 >> 2),
792 0x00000000,
793 (0x0600 << 16) | (0x30a2c >> 2),
794 0x00000000,
795 (0x0e00 << 16) | (0xc700 >> 2),
796 0x00000000,
797 (0x0e00 << 16) | (0xc704 >> 2),
798 0x00000000,
799 (0x0e00 << 16) | (0xc708 >> 2),
800 0x00000000,
801 (0x0e00 << 16) | (0xc768 >> 2),
802 0x00000000,
803 (0x0400 << 16) | (0xc770 >> 2),
804 0x00000000,
805 (0x0400 << 16) | (0xc774 >> 2),
806 0x00000000,
807 (0x0400 << 16) | (0xc798 >> 2),
808 0x00000000,
809 (0x0400 << 16) | (0xc79c >> 2),
810 0x00000000,
811 (0x0e00 << 16) | (0x9100 >> 2),
812 0x00000000,
813 (0x0e00 << 16) | (0x3c010 >> 2),
814 0x00000000,
815 (0x0e00 << 16) | (0x8c00 >> 2),
816 0x00000000,
817 (0x0e00 << 16) | (0x8c04 >> 2),
818 0x00000000,
819 (0x0e00 << 16) | (0x8c20 >> 2),
820 0x00000000,
821 (0x0e00 << 16) | (0x8c38 >> 2),
822 0x00000000,
823 (0x0e00 << 16) | (0x8c3c >> 2),
824 0x00000000,
825 (0x0e00 << 16) | (0xae00 >> 2),
826 0x00000000,
827 (0x0e00 << 16) | (0x9604 >> 2),
828 0x00000000,
829 (0x0e00 << 16) | (0xac08 >> 2),
830 0x00000000,
831 (0x0e00 << 16) | (0xac0c >> 2),
832 0x00000000,
833 (0x0e00 << 16) | (0xac10 >> 2),
834 0x00000000,
835 (0x0e00 << 16) | (0xac14 >> 2),
836 0x00000000,
837 (0x0e00 << 16) | (0xac58 >> 2),
838 0x00000000,
839 (0x0e00 << 16) | (0xac68 >> 2),
840 0x00000000,
841 (0x0e00 << 16) | (0xac6c >> 2),
842 0x00000000,
843 (0x0e00 << 16) | (0xac70 >> 2),
844 0x00000000,
845 (0x0e00 << 16) | (0xac74 >> 2),
846 0x00000000,
847 (0x0e00 << 16) | (0xac78 >> 2),
848 0x00000000,
849 (0x0e00 << 16) | (0xac7c >> 2),
850 0x00000000,
851 (0x0e00 << 16) | (0xac80 >> 2),
852 0x00000000,
853 (0x0e00 << 16) | (0xac84 >> 2),
854 0x00000000,
855 (0x0e00 << 16) | (0xac88 >> 2),
856 0x00000000,
857 (0x0e00 << 16) | (0xac8c >> 2),
858 0x00000000,
859 (0x0e00 << 16) | (0x970c >> 2),
860 0x00000000,
861 (0x0e00 << 16) | (0x9714 >> 2),
862 0x00000000,
863 (0x0e00 << 16) | (0x9718 >> 2),
864 0x00000000,
865 (0x0e00 << 16) | (0x971c >> 2),
866 0x00000000,
867 (0x0e00 << 16) | (0x31068 >> 2),
868 0x00000000,
869 (0x4e00 << 16) | (0x31068 >> 2),
870 0x00000000,
871 (0x5e00 << 16) | (0x31068 >> 2),
872 0x00000000,
873 (0x6e00 << 16) | (0x31068 >> 2),
874 0x00000000,
875 (0x7e00 << 16) | (0x31068 >> 2),
876 0x00000000,
877 (0x0e00 << 16) | (0xcd10 >> 2),
878 0x00000000,
879 (0x0e00 << 16) | (0xcd14 >> 2),
880 0x00000000,
881 (0x0e00 << 16) | (0x88b0 >> 2),
882 0x00000000,
883 (0x0e00 << 16) | (0x88b4 >> 2),
884 0x00000000,
885 (0x0e00 << 16) | (0x88b8 >> 2),
886 0x00000000,
887 (0x0e00 << 16) | (0x88bc >> 2),
888 0x00000000,
889 (0x0400 << 16) | (0x89c0 >> 2),
890 0x00000000,
891 (0x0e00 << 16) | (0x88c4 >> 2),
892 0x00000000,
893 (0x0e00 << 16) | (0x88c8 >> 2),
894 0x00000000,
895 (0x0e00 << 16) | (0x88d0 >> 2),
896 0x00000000,
897 (0x0e00 << 16) | (0x88d4 >> 2),
898 0x00000000,
899 (0x0e00 << 16) | (0x88d8 >> 2),
900 0x00000000,
901 (0x0e00 << 16) | (0x8980 >> 2),
902 0x00000000,
903 (0x0e00 << 16) | (0x30938 >> 2),
904 0x00000000,
905 (0x0e00 << 16) | (0x3093c >> 2),
906 0x00000000,
907 (0x0e00 << 16) | (0x30940 >> 2),
908 0x00000000,
909 (0x0e00 << 16) | (0x89a0 >> 2),
910 0x00000000,
911 (0x0e00 << 16) | (0x30900 >> 2),
912 0x00000000,
913 (0x0e00 << 16) | (0x30904 >> 2),
914 0x00000000,
915 (0x0e00 << 16) | (0x89b4 >> 2),
916 0x00000000,
917 (0x0e00 << 16) | (0x3e1fc >> 2),
918 0x00000000,
919 (0x0e00 << 16) | (0x3c210 >> 2),
920 0x00000000,
921 (0x0e00 << 16) | (0x3c214 >> 2),
922 0x00000000,
923 (0x0e00 << 16) | (0x3c218 >> 2),
924 0x00000000,
925 (0x0e00 << 16) | (0x8904 >> 2),
926 0x00000000,
927 0x5,
928 (0x0e00 << 16) | (0x8c28 >> 2),
929 (0x0e00 << 16) | (0x8c2c >> 2),
930 (0x0e00 << 16) | (0x8c30 >> 2),
931 (0x0e00 << 16) | (0x8c34 >> 2),
932 (0x0e00 << 16) | (0x9600 >> 2),
933 };
934
935 static const u32 bonaire_golden_spm_registers[] =
936 {
937 0x30800, 0xe0ffffff, 0xe0000000
938 };
939
940 static const u32 bonaire_golden_common_registers[] =
941 {
942 0xc770, 0xffffffff, 0x00000800,
943 0xc774, 0xffffffff, 0x00000800,
944 0xc798, 0xffffffff, 0x00007fbf,
945 0xc79c, 0xffffffff, 0x00007faf
946 };
947
948 static const u32 bonaire_golden_registers[] =
949 {
950 0x3354, 0x00000333, 0x00000333,
951 0x3350, 0x000c0fc0, 0x00040200,
952 0x9a10, 0x00010000, 0x00058208,
953 0x3c000, 0xffff1fff, 0x00140000,
954 0x3c200, 0xfdfc0fff, 0x00000100,
955 0x3c234, 0x40000000, 0x40000200,
956 0x9830, 0xffffffff, 0x00000000,
957 0x9834, 0xf00fffff, 0x00000400,
958 0x9838, 0x0002021c, 0x00020200,
959 0xc78, 0x00000080, 0x00000000,
960 0x5bb0, 0x000000f0, 0x00000070,
961 0x5bc0, 0xf0311fff, 0x80300000,
962 0x98f8, 0x73773777, 0x12010001,
963 0x350c, 0x00810000, 0x408af000,
964 0x7030, 0x31000111, 0x00000011,
965 0x2f48, 0x73773777, 0x12010001,
966 0x220c, 0x00007fb6, 0x0021a1b1,
967 0x2210, 0x00007fb6, 0x002021b1,
968 0x2180, 0x00007fb6, 0x00002191,
969 0x2218, 0x00007fb6, 0x002121b1,
970 0x221c, 0x00007fb6, 0x002021b1,
971 0x21dc, 0x00007fb6, 0x00002191,
972 0x21e0, 0x00007fb6, 0x00002191,
973 0x3628, 0x0000003f, 0x0000000a,
974 0x362c, 0x0000003f, 0x0000000a,
975 0x2ae4, 0x00073ffe, 0x000022a2,
976 0x240c, 0x000007ff, 0x00000000,
977 0x8a14, 0xf000003f, 0x00000007,
978 0x8bf0, 0x00002001, 0x00000001,
979 0x8b24, 0xffffffff, 0x00ffffff,
980 0x30a04, 0x0000ff0f, 0x00000000,
981 0x28a4c, 0x07ffffff, 0x06000000,
982 0x4d8, 0x00000fff, 0x00000100,
983 0x3e78, 0x00000001, 0x00000002,
984 0x9100, 0x03000000, 0x0362c688,
985 0x8c00, 0x000000ff, 0x00000001,
986 0xe40, 0x00001fff, 0x00001fff,
987 0x9060, 0x0000007f, 0x00000020,
988 0x9508, 0x00010000, 0x00010000,
989 0xac14, 0x000003ff, 0x000000f3,
990 0xac0c, 0xffffffff, 0x00001032
991 };
992
993 static const u32 bonaire_mgcg_cgcg_init[] =
994 {
995 0xc420, 0xffffffff, 0xfffffffc,
996 0x30800, 0xffffffff, 0xe0000000,
997 0x3c2a0, 0xffffffff, 0x00000100,
998 0x3c208, 0xffffffff, 0x00000100,
999 0x3c2c0, 0xffffffff, 0xc0000100,
1000 0x3c2c8, 0xffffffff, 0xc0000100,
1001 0x3c2c4, 0xffffffff, 0xc0000100,
1002 0x55e4, 0xffffffff, 0x00600100,
1003 0x3c280, 0xffffffff, 0x00000100,
1004 0x3c214, 0xffffffff, 0x06000100,
1005 0x3c220, 0xffffffff, 0x00000100,
1006 0x3c218, 0xffffffff, 0x06000100,
1007 0x3c204, 0xffffffff, 0x00000100,
1008 0x3c2e0, 0xffffffff, 0x00000100,
1009 0x3c224, 0xffffffff, 0x00000100,
1010 0x3c200, 0xffffffff, 0x00000100,
1011 0x3c230, 0xffffffff, 0x00000100,
1012 0x3c234, 0xffffffff, 0x00000100,
1013 0x3c250, 0xffffffff, 0x00000100,
1014 0x3c254, 0xffffffff, 0x00000100,
1015 0x3c258, 0xffffffff, 0x00000100,
1016 0x3c25c, 0xffffffff, 0x00000100,
1017 0x3c260, 0xffffffff, 0x00000100,
1018 0x3c27c, 0xffffffff, 0x00000100,
1019 0x3c278, 0xffffffff, 0x00000100,
1020 0x3c210, 0xffffffff, 0x06000100,
1021 0x3c290, 0xffffffff, 0x00000100,
1022 0x3c274, 0xffffffff, 0x00000100,
1023 0x3c2b4, 0xffffffff, 0x00000100,
1024 0x3c2b0, 0xffffffff, 0x00000100,
1025 0x3c270, 0xffffffff, 0x00000100,
1026 0x30800, 0xffffffff, 0xe0000000,
1027 0x3c020, 0xffffffff, 0x00010000,
1028 0x3c024, 0xffffffff, 0x00030002,
1029 0x3c028, 0xffffffff, 0x00040007,
1030 0x3c02c, 0xffffffff, 0x00060005,
1031 0x3c030, 0xffffffff, 0x00090008,
1032 0x3c034, 0xffffffff, 0x00010000,
1033 0x3c038, 0xffffffff, 0x00030002,
1034 0x3c03c, 0xffffffff, 0x00040007,
1035 0x3c040, 0xffffffff, 0x00060005,
1036 0x3c044, 0xffffffff, 0x00090008,
1037 0x3c048, 0xffffffff, 0x00010000,
1038 0x3c04c, 0xffffffff, 0x00030002,
1039 0x3c050, 0xffffffff, 0x00040007,
1040 0x3c054, 0xffffffff, 0x00060005,
1041 0x3c058, 0xffffffff, 0x00090008,
1042 0x3c05c, 0xffffffff, 0x00010000,
1043 0x3c060, 0xffffffff, 0x00030002,
1044 0x3c064, 0xffffffff, 0x00040007,
1045 0x3c068, 0xffffffff, 0x00060005,
1046 0x3c06c, 0xffffffff, 0x00090008,
1047 0x3c070, 0xffffffff, 0x00010000,
1048 0x3c074, 0xffffffff, 0x00030002,
1049 0x3c078, 0xffffffff, 0x00040007,
1050 0x3c07c, 0xffffffff, 0x00060005,
1051 0x3c080, 0xffffffff, 0x00090008,
1052 0x3c084, 0xffffffff, 0x00010000,
1053 0x3c088, 0xffffffff, 0x00030002,
1054 0x3c08c, 0xffffffff, 0x00040007,
1055 0x3c090, 0xffffffff, 0x00060005,
1056 0x3c094, 0xffffffff, 0x00090008,
1057 0x3c098, 0xffffffff, 0x00010000,
1058 0x3c09c, 0xffffffff, 0x00030002,
1059 0x3c0a0, 0xffffffff, 0x00040007,
1060 0x3c0a4, 0xffffffff, 0x00060005,
1061 0x3c0a8, 0xffffffff, 0x00090008,
1062 0x3c000, 0xffffffff, 0x96e00200,
1063 0x8708, 0xffffffff, 0x00900100,
1064 0xc424, 0xffffffff, 0x0020003f,
1065 0x38, 0xffffffff, 0x0140001c,
1066 0x3c, 0x000f0000, 0x000f0000,
1067 0x220, 0xffffffff, 0xC060000C,
1068 0x224, 0xc0000fff, 0x00000100,
1069 0xf90, 0xffffffff, 0x00000100,
1070 0xf98, 0x00000101, 0x00000000,
1071 0x20a8, 0xffffffff, 0x00000104,
1072 0x55e4, 0xff000fff, 0x00000100,
1073 0x30cc, 0xc0000fff, 0x00000104,
1074 0xc1e4, 0x00000001, 0x00000001,
1075 0xd00c, 0xff000ff0, 0x00000100,
1076 0xd80c, 0xff000ff0, 0x00000100
1077 };
1078
1079 static const u32 spectre_golden_spm_registers[] =
1080 {
1081 0x30800, 0xe0ffffff, 0xe0000000
1082 };
1083
1084 static const u32 spectre_golden_common_registers[] =
1085 {
1086 0xc770, 0xffffffff, 0x00000800,
1087 0xc774, 0xffffffff, 0x00000800,
1088 0xc798, 0xffffffff, 0x00007fbf,
1089 0xc79c, 0xffffffff, 0x00007faf
1090 };
1091
1092 static const u32 spectre_golden_registers[] =
1093 {
1094 0x3c000, 0xffff1fff, 0x96940200,
1095 0x3c00c, 0xffff0001, 0xff000000,
1096 0x3c200, 0xfffc0fff, 0x00000100,
1097 0x6ed8, 0x00010101, 0x00010000,
1098 0x9834, 0xf00fffff, 0x00000400,
1099 0x9838, 0xfffffffc, 0x00020200,
1100 0x5bb0, 0x000000f0, 0x00000070,
1101 0x5bc0, 0xf0311fff, 0x80300000,
1102 0x98f8, 0x73773777, 0x12010001,
1103 0x9b7c, 0x00ff0000, 0x00fc0000,
1104 0x2f48, 0x73773777, 0x12010001,
1105 0x8a14, 0xf000003f, 0x00000007,
1106 0x8b24, 0xffffffff, 0x00ffffff,
1107 0x28350, 0x3f3f3fff, 0x00000082,
1108 0x28354, 0x0000003f, 0x00000000,
1109 0x3e78, 0x00000001, 0x00000002,
1110 0x913c, 0xffff03df, 0x00000004,
1111 0xc768, 0x00000008, 0x00000008,
1112 0x8c00, 0x000008ff, 0x00000800,
1113 0x9508, 0x00010000, 0x00010000,
1114 0xac0c, 0xffffffff, 0x54763210,
1115 0x214f8, 0x01ff01ff, 0x00000002,
1116 0x21498, 0x007ff800, 0x00200000,
1117 0x2015c, 0xffffffff, 0x00000f40,
1118 0x30934, 0xffffffff, 0x00000001
1119 };
1120
1121 static const u32 spectre_mgcg_cgcg_init[] =
1122 {
1123 0xc420, 0xffffffff, 0xfffffffc,
1124 0x30800, 0xffffffff, 0xe0000000,
1125 0x3c2a0, 0xffffffff, 0x00000100,
1126 0x3c208, 0xffffffff, 0x00000100,
1127 0x3c2c0, 0xffffffff, 0x00000100,
1128 0x3c2c8, 0xffffffff, 0x00000100,
1129 0x3c2c4, 0xffffffff, 0x00000100,
1130 0x55e4, 0xffffffff, 0x00600100,
1131 0x3c280, 0xffffffff, 0x00000100,
1132 0x3c214, 0xffffffff, 0x06000100,
1133 0x3c220, 0xffffffff, 0x00000100,
1134 0x3c218, 0xffffffff, 0x06000100,
1135 0x3c204, 0xffffffff, 0x00000100,
1136 0x3c2e0, 0xffffffff, 0x00000100,
1137 0x3c224, 0xffffffff, 0x00000100,
1138 0x3c200, 0xffffffff, 0x00000100,
1139 0x3c230, 0xffffffff, 0x00000100,
1140 0x3c234, 0xffffffff, 0x00000100,
1141 0x3c250, 0xffffffff, 0x00000100,
1142 0x3c254, 0xffffffff, 0x00000100,
1143 0x3c258, 0xffffffff, 0x00000100,
1144 0x3c25c, 0xffffffff, 0x00000100,
1145 0x3c260, 0xffffffff, 0x00000100,
1146 0x3c27c, 0xffffffff, 0x00000100,
1147 0x3c278, 0xffffffff, 0x00000100,
1148 0x3c210, 0xffffffff, 0x06000100,
1149 0x3c290, 0xffffffff, 0x00000100,
1150 0x3c274, 0xffffffff, 0x00000100,
1151 0x3c2b4, 0xffffffff, 0x00000100,
1152 0x3c2b0, 0xffffffff, 0x00000100,
1153 0x3c270, 0xffffffff, 0x00000100,
1154 0x30800, 0xffffffff, 0xe0000000,
1155 0x3c020, 0xffffffff, 0x00010000,
1156 0x3c024, 0xffffffff, 0x00030002,
1157 0x3c028, 0xffffffff, 0x00040007,
1158 0x3c02c, 0xffffffff, 0x00060005,
1159 0x3c030, 0xffffffff, 0x00090008,
1160 0x3c034, 0xffffffff, 0x00010000,
1161 0x3c038, 0xffffffff, 0x00030002,
1162 0x3c03c, 0xffffffff, 0x00040007,
1163 0x3c040, 0xffffffff, 0x00060005,
1164 0x3c044, 0xffffffff, 0x00090008,
1165 0x3c048, 0xffffffff, 0x00010000,
1166 0x3c04c, 0xffffffff, 0x00030002,
1167 0x3c050, 0xffffffff, 0x00040007,
1168 0x3c054, 0xffffffff, 0x00060005,
1169 0x3c058, 0xffffffff, 0x00090008,
1170 0x3c05c, 0xffffffff, 0x00010000,
1171 0x3c060, 0xffffffff, 0x00030002,
1172 0x3c064, 0xffffffff, 0x00040007,
1173 0x3c068, 0xffffffff, 0x00060005,
1174 0x3c06c, 0xffffffff, 0x00090008,
1175 0x3c070, 0xffffffff, 0x00010000,
1176 0x3c074, 0xffffffff, 0x00030002,
1177 0x3c078, 0xffffffff, 0x00040007,
1178 0x3c07c, 0xffffffff, 0x00060005,
1179 0x3c080, 0xffffffff, 0x00090008,
1180 0x3c084, 0xffffffff, 0x00010000,
1181 0x3c088, 0xffffffff, 0x00030002,
1182 0x3c08c, 0xffffffff, 0x00040007,
1183 0x3c090, 0xffffffff, 0x00060005,
1184 0x3c094, 0xffffffff, 0x00090008,
1185 0x3c098, 0xffffffff, 0x00010000,
1186 0x3c09c, 0xffffffff, 0x00030002,
1187 0x3c0a0, 0xffffffff, 0x00040007,
1188 0x3c0a4, 0xffffffff, 0x00060005,
1189 0x3c0a8, 0xffffffff, 0x00090008,
1190 0x3c0ac, 0xffffffff, 0x00010000,
1191 0x3c0b0, 0xffffffff, 0x00030002,
1192 0x3c0b4, 0xffffffff, 0x00040007,
1193 0x3c0b8, 0xffffffff, 0x00060005,
1194 0x3c0bc, 0xffffffff, 0x00090008,
1195 0x3c000, 0xffffffff, 0x96e00200,
1196 0x8708, 0xffffffff, 0x00900100,
1197 0xc424, 0xffffffff, 0x0020003f,
1198 0x38, 0xffffffff, 0x0140001c,
1199 0x3c, 0x000f0000, 0x000f0000,
1200 0x220, 0xffffffff, 0xC060000C,
1201 0x224, 0xc0000fff, 0x00000100,
1202 0xf90, 0xffffffff, 0x00000100,
1203 0xf98, 0x00000101, 0x00000000,
1204 0x20a8, 0xffffffff, 0x00000104,
1205 0x55e4, 0xff000fff, 0x00000100,
1206 0x30cc, 0xc0000fff, 0x00000104,
1207 0xc1e4, 0x00000001, 0x00000001,
1208 0xd00c, 0xff000ff0, 0x00000100,
1209 0xd80c, 0xff000ff0, 0x00000100
1210 };
1211
1212 static const u32 kalindi_golden_spm_registers[] =
1213 {
1214 0x30800, 0xe0ffffff, 0xe0000000
1215 };
1216
1217 static const u32 kalindi_golden_common_registers[] =
1218 {
1219 0xc770, 0xffffffff, 0x00000800,
1220 0xc774, 0xffffffff, 0x00000800,
1221 0xc798, 0xffffffff, 0x00007fbf,
1222 0xc79c, 0xffffffff, 0x00007faf
1223 };
1224
1225 static const u32 kalindi_golden_registers[] =
1226 {
1227 0x3c000, 0xffffdfff, 0x6e944040,
1228 0x55e4, 0xff607fff, 0xfc000100,
1229 0x3c220, 0xff000fff, 0x00000100,
1230 0x3c224, 0xff000fff, 0x00000100,
1231 0x3c200, 0xfffc0fff, 0x00000100,
1232 0x6ed8, 0x00010101, 0x00010000,
1233 0x9830, 0xffffffff, 0x00000000,
1234 0x9834, 0xf00fffff, 0x00000400,
1235 0x5bb0, 0x000000f0, 0x00000070,
1236 0x5bc0, 0xf0311fff, 0x80300000,
1237 0x98f8, 0x73773777, 0x12010001,
1238 0x98fc, 0xffffffff, 0x00000010,
1239 0x9b7c, 0x00ff0000, 0x00fc0000,
1240 0x8030, 0x00001f0f, 0x0000100a,
1241 0x2f48, 0x73773777, 0x12010001,
1242 0x2408, 0x000fffff, 0x000c007f,
1243 0x8a14, 0xf000003f, 0x00000007,
1244 0x8b24, 0x3fff3fff, 0x00ffcfff,
1245 0x30a04, 0x0000ff0f, 0x00000000,
1246 0x28a4c, 0x07ffffff, 0x06000000,
1247 0x4d8, 0x00000fff, 0x00000100,
1248 0x3e78, 0x00000001, 0x00000002,
1249 0xc768, 0x00000008, 0x00000008,
1250 0x8c00, 0x000000ff, 0x00000003,
1251 0x214f8, 0x01ff01ff, 0x00000002,
1252 0x21498, 0x007ff800, 0x00200000,
1253 0x2015c, 0xffffffff, 0x00000f40,
1254 0x88c4, 0x001f3ae3, 0x00000082,
1255 0x88d4, 0x0000001f, 0x00000010,
1256 0x30934, 0xffffffff, 0x00000000
1257 };
1258
1259 static const u32 kalindi_mgcg_cgcg_init[] =
1260 {
1261 0xc420, 0xffffffff, 0xfffffffc,
1262 0x30800, 0xffffffff, 0xe0000000,
1263 0x3c2a0, 0xffffffff, 0x00000100,
1264 0x3c208, 0xffffffff, 0x00000100,
1265 0x3c2c0, 0xffffffff, 0x00000100,
1266 0x3c2c8, 0xffffffff, 0x00000100,
1267 0x3c2c4, 0xffffffff, 0x00000100,
1268 0x55e4, 0xffffffff, 0x00600100,
1269 0x3c280, 0xffffffff, 0x00000100,
1270 0x3c214, 0xffffffff, 0x06000100,
1271 0x3c220, 0xffffffff, 0x00000100,
1272 0x3c218, 0xffffffff, 0x06000100,
1273 0x3c204, 0xffffffff, 0x00000100,
1274 0x3c2e0, 0xffffffff, 0x00000100,
1275 0x3c224, 0xffffffff, 0x00000100,
1276 0x3c200, 0xffffffff, 0x00000100,
1277 0x3c230, 0xffffffff, 0x00000100,
1278 0x3c234, 0xffffffff, 0x00000100,
1279 0x3c250, 0xffffffff, 0x00000100,
1280 0x3c254, 0xffffffff, 0x00000100,
1281 0x3c258, 0xffffffff, 0x00000100,
1282 0x3c25c, 0xffffffff, 0x00000100,
1283 0x3c260, 0xffffffff, 0x00000100,
1284 0x3c27c, 0xffffffff, 0x00000100,
1285 0x3c278, 0xffffffff, 0x00000100,
1286 0x3c210, 0xffffffff, 0x06000100,
1287 0x3c290, 0xffffffff, 0x00000100,
1288 0x3c274, 0xffffffff, 0x00000100,
1289 0x3c2b4, 0xffffffff, 0x00000100,
1290 0x3c2b0, 0xffffffff, 0x00000100,
1291 0x3c270, 0xffffffff, 0x00000100,
1292 0x30800, 0xffffffff, 0xe0000000,
1293 0x3c020, 0xffffffff, 0x00010000,
1294 0x3c024, 0xffffffff, 0x00030002,
1295 0x3c028, 0xffffffff, 0x00040007,
1296 0x3c02c, 0xffffffff, 0x00060005,
1297 0x3c030, 0xffffffff, 0x00090008,
1298 0x3c034, 0xffffffff, 0x00010000,
1299 0x3c038, 0xffffffff, 0x00030002,
1300 0x3c03c, 0xffffffff, 0x00040007,
1301 0x3c040, 0xffffffff, 0x00060005,
1302 0x3c044, 0xffffffff, 0x00090008,
1303 0x3c000, 0xffffffff, 0x96e00200,
1304 0x8708, 0xffffffff, 0x00900100,
1305 0xc424, 0xffffffff, 0x0020003f,
1306 0x38, 0xffffffff, 0x0140001c,
1307 0x3c, 0x000f0000, 0x000f0000,
1308 0x220, 0xffffffff, 0xC060000C,
1309 0x224, 0xc0000fff, 0x00000100,
1310 0x20a8, 0xffffffff, 0x00000104,
1311 0x55e4, 0xff000fff, 0x00000100,
1312 0x30cc, 0xc0000fff, 0x00000104,
1313 0xc1e4, 0x00000001, 0x00000001,
1314 0xd00c, 0xff000ff0, 0x00000100,
1315 0xd80c, 0xff000ff0, 0x00000100
1316 };
1317
1318 static const u32 hawaii_golden_spm_registers[] =
1319 {
1320 0x30800, 0xe0ffffff, 0xe0000000
1321 };
1322
1323 static const u32 hawaii_golden_common_registers[] =
1324 {
1325 0x30800, 0xffffffff, 0xe0000000,
1326 0x28350, 0xffffffff, 0x3a00161a,
1327 0x28354, 0xffffffff, 0x0000002e,
1328 0x9a10, 0xffffffff, 0x00018208,
1329 0x98f8, 0xffffffff, 0x12011003
1330 };
1331
1332 static const u32 hawaii_golden_registers[] =
1333 {
1334 0x3354, 0x00000333, 0x00000333,
1335 0x9a10, 0x00010000, 0x00058208,
1336 0x9830, 0xffffffff, 0x00000000,
1337 0x9834, 0xf00fffff, 0x00000400,
1338 0x9838, 0x0002021c, 0x00020200,
1339 0xc78, 0x00000080, 0x00000000,
1340 0x5bb0, 0x000000f0, 0x00000070,
1341 0x5bc0, 0xf0311fff, 0x80300000,
1342 0x350c, 0x00810000, 0x408af000,
1343 0x7030, 0x31000111, 0x00000011,
1344 0x2f48, 0x73773777, 0x12010001,
1345 0x2120, 0x0000007f, 0x0000001b,
1346 0x21dc, 0x00007fb6, 0x00002191,
1347 0x3628, 0x0000003f, 0x0000000a,
1348 0x362c, 0x0000003f, 0x0000000a,
1349 0x2ae4, 0x00073ffe, 0x000022a2,
1350 0x240c, 0x000007ff, 0x00000000,
1351 0x8bf0, 0x00002001, 0x00000001,
1352 0x8b24, 0xffffffff, 0x00ffffff,
1353 0x30a04, 0x0000ff0f, 0x00000000,
1354 0x28a4c, 0x07ffffff, 0x06000000,
1355 0x3e78, 0x00000001, 0x00000002,
1356 0xc768, 0x00000008, 0x00000008,
1357 0xc770, 0x00000f00, 0x00000800,
1358 0xc774, 0x00000f00, 0x00000800,
1359 0xc798, 0x00ffffff, 0x00ff7fbf,
1360 0xc79c, 0x00ffffff, 0x00ff7faf,
1361 0x8c00, 0x000000ff, 0x00000800,
1362 0xe40, 0x00001fff, 0x00001fff,
1363 0x9060, 0x0000007f, 0x00000020,
1364 0x9508, 0x00010000, 0x00010000,
1365 0xae00, 0x00100000, 0x000ff07c,
1366 0xac14, 0x000003ff, 0x0000000f,
1367 0xac10, 0xffffffff, 0x7564fdec,
1368 0xac0c, 0xffffffff, 0x3120b9a8,
1369 0xac08, 0x20000000, 0x0f9c0000
1370 };
1371
1372 static const u32 hawaii_mgcg_cgcg_init[] =
1373 {
1374 0xc420, 0xffffffff, 0xfffffffd,
1375 0x30800, 0xffffffff, 0xe0000000,
1376 0x3c2a0, 0xffffffff, 0x00000100,
1377 0x3c208, 0xffffffff, 0x00000100,
1378 0x3c2c0, 0xffffffff, 0x00000100,
1379 0x3c2c8, 0xffffffff, 0x00000100,
1380 0x3c2c4, 0xffffffff, 0x00000100,
1381 0x55e4, 0xffffffff, 0x00200100,
1382 0x3c280, 0xffffffff, 0x00000100,
1383 0x3c214, 0xffffffff, 0x06000100,
1384 0x3c220, 0xffffffff, 0x00000100,
1385 0x3c218, 0xffffffff, 0x06000100,
1386 0x3c204, 0xffffffff, 0x00000100,
1387 0x3c2e0, 0xffffffff, 0x00000100,
1388 0x3c224, 0xffffffff, 0x00000100,
1389 0x3c200, 0xffffffff, 0x00000100,
1390 0x3c230, 0xffffffff, 0x00000100,
1391 0x3c234, 0xffffffff, 0x00000100,
1392 0x3c250, 0xffffffff, 0x00000100,
1393 0x3c254, 0xffffffff, 0x00000100,
1394 0x3c258, 0xffffffff, 0x00000100,
1395 0x3c25c, 0xffffffff, 0x00000100,
1396 0x3c260, 0xffffffff, 0x00000100,
1397 0x3c27c, 0xffffffff, 0x00000100,
1398 0x3c278, 0xffffffff, 0x00000100,
1399 0x3c210, 0xffffffff, 0x06000100,
1400 0x3c290, 0xffffffff, 0x00000100,
1401 0x3c274, 0xffffffff, 0x00000100,
1402 0x3c2b4, 0xffffffff, 0x00000100,
1403 0x3c2b0, 0xffffffff, 0x00000100,
1404 0x3c270, 0xffffffff, 0x00000100,
1405 0x30800, 0xffffffff, 0xe0000000,
1406 0x3c020, 0xffffffff, 0x00010000,
1407 0x3c024, 0xffffffff, 0x00030002,
1408 0x3c028, 0xffffffff, 0x00040007,
1409 0x3c02c, 0xffffffff, 0x00060005,
1410 0x3c030, 0xffffffff, 0x00090008,
1411 0x3c034, 0xffffffff, 0x00010000,
1412 0x3c038, 0xffffffff, 0x00030002,
1413 0x3c03c, 0xffffffff, 0x00040007,
1414 0x3c040, 0xffffffff, 0x00060005,
1415 0x3c044, 0xffffffff, 0x00090008,
1416 0x3c048, 0xffffffff, 0x00010000,
1417 0x3c04c, 0xffffffff, 0x00030002,
1418 0x3c050, 0xffffffff, 0x00040007,
1419 0x3c054, 0xffffffff, 0x00060005,
1420 0x3c058, 0xffffffff, 0x00090008,
1421 0x3c05c, 0xffffffff, 0x00010000,
1422 0x3c060, 0xffffffff, 0x00030002,
1423 0x3c064, 0xffffffff, 0x00040007,
1424 0x3c068, 0xffffffff, 0x00060005,
1425 0x3c06c, 0xffffffff, 0x00090008,
1426 0x3c070, 0xffffffff, 0x00010000,
1427 0x3c074, 0xffffffff, 0x00030002,
1428 0x3c078, 0xffffffff, 0x00040007,
1429 0x3c07c, 0xffffffff, 0x00060005,
1430 0x3c080, 0xffffffff, 0x00090008,
1431 0x3c084, 0xffffffff, 0x00010000,
1432 0x3c088, 0xffffffff, 0x00030002,
1433 0x3c08c, 0xffffffff, 0x00040007,
1434 0x3c090, 0xffffffff, 0x00060005,
1435 0x3c094, 0xffffffff, 0x00090008,
1436 0x3c098, 0xffffffff, 0x00010000,
1437 0x3c09c, 0xffffffff, 0x00030002,
1438 0x3c0a0, 0xffffffff, 0x00040007,
1439 0x3c0a4, 0xffffffff, 0x00060005,
1440 0x3c0a8, 0xffffffff, 0x00090008,
1441 0x3c0ac, 0xffffffff, 0x00010000,
1442 0x3c0b0, 0xffffffff, 0x00030002,
1443 0x3c0b4, 0xffffffff, 0x00040007,
1444 0x3c0b8, 0xffffffff, 0x00060005,
1445 0x3c0bc, 0xffffffff, 0x00090008,
1446 0x3c0c0, 0xffffffff, 0x00010000,
1447 0x3c0c4, 0xffffffff, 0x00030002,
1448 0x3c0c8, 0xffffffff, 0x00040007,
1449 0x3c0cc, 0xffffffff, 0x00060005,
1450 0x3c0d0, 0xffffffff, 0x00090008,
1451 0x3c0d4, 0xffffffff, 0x00010000,
1452 0x3c0d8, 0xffffffff, 0x00030002,
1453 0x3c0dc, 0xffffffff, 0x00040007,
1454 0x3c0e0, 0xffffffff, 0x00060005,
1455 0x3c0e4, 0xffffffff, 0x00090008,
1456 0x3c0e8, 0xffffffff, 0x00010000,
1457 0x3c0ec, 0xffffffff, 0x00030002,
1458 0x3c0f0, 0xffffffff, 0x00040007,
1459 0x3c0f4, 0xffffffff, 0x00060005,
1460 0x3c0f8, 0xffffffff, 0x00090008,
1461 0xc318, 0xffffffff, 0x00020200,
1462 0x3350, 0xffffffff, 0x00000200,
1463 0x15c0, 0xffffffff, 0x00000400,
1464 0x55e8, 0xffffffff, 0x00000000,
1465 0x2f50, 0xffffffff, 0x00000902,
1466 0x3c000, 0xffffffff, 0x96940200,
1467 0x8708, 0xffffffff, 0x00900100,
1468 0xc424, 0xffffffff, 0x0020003f,
1469 0x38, 0xffffffff, 0x0140001c,
1470 0x3c, 0x000f0000, 0x000f0000,
1471 0x220, 0xffffffff, 0xc060000c,
1472 0x224, 0xc0000fff, 0x00000100,
1473 0xf90, 0xffffffff, 0x00000100,
1474 0xf98, 0x00000101, 0x00000000,
1475 0x20a8, 0xffffffff, 0x00000104,
1476 0x55e4, 0xff000fff, 0x00000100,
1477 0x30cc, 0xc0000fff, 0x00000104,
1478 0xc1e4, 0x00000001, 0x00000001,
1479 0xd00c, 0xff000ff0, 0x00000100,
1480 0xd80c, 0xff000ff0, 0x00000100
1481 };
1482
1483 static const u32 godavari_golden_registers[] =
1484 {
1485 0x55e4, 0xff607fff, 0xfc000100,
1486 0x6ed8, 0x00010101, 0x00010000,
1487 0x9830, 0xffffffff, 0x00000000,
1488 0x98302, 0xf00fffff, 0x00000400,
1489 0x6130, 0xffffffff, 0x00010000,
1490 0x5bb0, 0x000000f0, 0x00000070,
1491 0x5bc0, 0xf0311fff, 0x80300000,
1492 0x98f8, 0x73773777, 0x12010001,
1493 0x98fc, 0xffffffff, 0x00000010,
1494 0x8030, 0x00001f0f, 0x0000100a,
1495 0x2f48, 0x73773777, 0x12010001,
1496 0x2408, 0x000fffff, 0x000c007f,
1497 0x8a14, 0xf000003f, 0x00000007,
1498 0x8b24, 0xffffffff, 0x00ff0fff,
1499 0x30a04, 0x0000ff0f, 0x00000000,
1500 0x28a4c, 0x07ffffff, 0x06000000,
1501 0x4d8, 0x00000fff, 0x00000100,
1502 0xd014, 0x00010000, 0x00810001,
1503 0xd814, 0x00010000, 0x00810001,
1504 0x3e78, 0x00000001, 0x00000002,
1505 0xc768, 0x00000008, 0x00000008,
1506 0xc770, 0x00000f00, 0x00000800,
1507 0xc774, 0x00000f00, 0x00000800,
1508 0xc798, 0x00ffffff, 0x00ff7fbf,
1509 0xc79c, 0x00ffffff, 0x00ff7faf,
1510 0x8c00, 0x000000ff, 0x00000001,
1511 0x214f8, 0x01ff01ff, 0x00000002,
1512 0x21498, 0x007ff800, 0x00200000,
1513 0x2015c, 0xffffffff, 0x00000f40,
1514 0x88c4, 0x001f3ae3, 0x00000082,
1515 0x88d4, 0x0000001f, 0x00000010,
1516 0x30934, 0xffffffff, 0x00000000
1517 };
1518
1519
1520 static void cik_init_golden_registers(struct radeon_device *rdev)
1521 {
1522 switch (rdev->family) {
1523 case CHIP_BONAIRE:
1524 radeon_program_register_sequence(rdev,
1525 bonaire_mgcg_cgcg_init,
1526 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1527 radeon_program_register_sequence(rdev,
1528 bonaire_golden_registers,
1529 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1530 radeon_program_register_sequence(rdev,
1531 bonaire_golden_common_registers,
1532 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1533 radeon_program_register_sequence(rdev,
1534 bonaire_golden_spm_registers,
1535 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1536 break;
1537 case CHIP_KABINI:
1538 radeon_program_register_sequence(rdev,
1539 kalindi_mgcg_cgcg_init,
1540 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1541 radeon_program_register_sequence(rdev,
1542 kalindi_golden_registers,
1543 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1544 radeon_program_register_sequence(rdev,
1545 kalindi_golden_common_registers,
1546 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1547 radeon_program_register_sequence(rdev,
1548 kalindi_golden_spm_registers,
1549 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1550 break;
1551 case CHIP_MULLINS:
1552 radeon_program_register_sequence(rdev,
1553 kalindi_mgcg_cgcg_init,
1554 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1555 radeon_program_register_sequence(rdev,
1556 godavari_golden_registers,
1557 (const u32)ARRAY_SIZE(godavari_golden_registers));
1558 radeon_program_register_sequence(rdev,
1559 kalindi_golden_common_registers,
1560 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1561 radeon_program_register_sequence(rdev,
1562 kalindi_golden_spm_registers,
1563 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1564 break;
1565 case CHIP_KAVERI:
1566 radeon_program_register_sequence(rdev,
1567 spectre_mgcg_cgcg_init,
1568 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1569 radeon_program_register_sequence(rdev,
1570 spectre_golden_registers,
1571 (const u32)ARRAY_SIZE(spectre_golden_registers));
1572 radeon_program_register_sequence(rdev,
1573 spectre_golden_common_registers,
1574 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1575 radeon_program_register_sequence(rdev,
1576 spectre_golden_spm_registers,
1577 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1578 break;
1579 case CHIP_HAWAII:
1580 radeon_program_register_sequence(rdev,
1581 hawaii_mgcg_cgcg_init,
1582 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1583 radeon_program_register_sequence(rdev,
1584 hawaii_golden_registers,
1585 (const u32)ARRAY_SIZE(hawaii_golden_registers));
1586 radeon_program_register_sequence(rdev,
1587 hawaii_golden_common_registers,
1588 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1589 radeon_program_register_sequence(rdev,
1590 hawaii_golden_spm_registers,
1591 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1592 break;
1593 default:
1594 break;
1595 }
1596 }
1597
1598 /**
1599 * cik_get_xclk - get the xclk
1600 *
1601 * @rdev: radeon_device pointer
1602 *
1603 * Returns the reference clock used by the gfx engine
1604 * (CIK).
1605 */
1606 u32 cik_get_xclk(struct radeon_device *rdev)
1607 {
1608 u32 reference_clock = rdev->clock.spll.reference_freq;
1609
1610 if (rdev->flags & RADEON_IS_IGP) {
1611 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1612 return reference_clock / 2;
1613 } else {
1614 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1615 return reference_clock / 4;
1616 }
1617 return reference_clock;
1618 }
1619
1620 /**
1621 * cik_mm_rdoorbell - read a doorbell dword
1622 *
1623 * @rdev: radeon_device pointer
1624 * @index: doorbell index
1625 *
1626 * Returns the value in the doorbell aperture at the
1627 * requested doorbell index (CIK).
1628 */
1629 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
1630 {
1631 if (index < rdev->doorbell.num_doorbells) {
1632 return readl(rdev->doorbell.ptr + index);
1633 } else {
1634 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
1635 return 0;
1636 }
1637 }
1638
1639 /**
1640 * cik_mm_wdoorbell - write a doorbell dword
1641 *
1642 * @rdev: radeon_device pointer
1643 * @index: doorbell index
1644 * @v: value to write
1645 *
1646 * Writes @v to the doorbell aperture at the
1647 * requested doorbell index (CIK).
1648 */
1649 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
1650 {
1651 if (index < rdev->doorbell.num_doorbells) {
1652 writel(v, rdev->doorbell.ptr + index);
1653 } else {
1654 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
1655 }
1656 }
1657
1658 #define BONAIRE_IO_MC_REGS_SIZE 36
1659
1660 static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1661 {
1662 {0x00000070, 0x04400000},
1663 {0x00000071, 0x80c01803},
1664 {0x00000072, 0x00004004},
1665 {0x00000073, 0x00000100},
1666 {0x00000074, 0x00ff0000},
1667 {0x00000075, 0x34000000},
1668 {0x00000076, 0x08000014},
1669 {0x00000077, 0x00cc08ec},
1670 {0x00000078, 0x00000400},
1671 {0x00000079, 0x00000000},
1672 {0x0000007a, 0x04090000},
1673 {0x0000007c, 0x00000000},
1674 {0x0000007e, 0x4408a8e8},
1675 {0x0000007f, 0x00000304},
1676 {0x00000080, 0x00000000},
1677 {0x00000082, 0x00000001},
1678 {0x00000083, 0x00000002},
1679 {0x00000084, 0xf3e4f400},
1680 {0x00000085, 0x052024e3},
1681 {0x00000087, 0x00000000},
1682 {0x00000088, 0x01000000},
1683 {0x0000008a, 0x1c0a0000},
1684 {0x0000008b, 0xff010000},
1685 {0x0000008d, 0xffffefff},
1686 {0x0000008e, 0xfff3efff},
1687 {0x0000008f, 0xfff3efbf},
1688 {0x00000092, 0xf7ffffff},
1689 {0x00000093, 0xffffff7f},
1690 {0x00000095, 0x00101101},
1691 {0x00000096, 0x00000fff},
1692 {0x00000097, 0x00116fff},
1693 {0x00000098, 0x60010000},
1694 {0x00000099, 0x10010000},
1695 {0x0000009a, 0x00006000},
1696 {0x0000009b, 0x00001000},
1697 {0x0000009f, 0x00b48000}
1698 };
1699
1700 #define HAWAII_IO_MC_REGS_SIZE 22
1701
1702 static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1703 {
1704 {0x0000007d, 0x40000000},
1705 {0x0000007e, 0x40180304},
1706 {0x0000007f, 0x0000ff00},
1707 {0x00000081, 0x00000000},
1708 {0x00000083, 0x00000800},
1709 {0x00000086, 0x00000000},
1710 {0x00000087, 0x00000100},
1711 {0x00000088, 0x00020100},
1712 {0x00000089, 0x00000000},
1713 {0x0000008b, 0x00040000},
1714 {0x0000008c, 0x00000100},
1715 {0x0000008e, 0xff010000},
1716 {0x00000090, 0xffffefff},
1717 {0x00000091, 0xfff3efff},
1718 {0x00000092, 0xfff3efbf},
1719 {0x00000093, 0xf7ffffff},
1720 {0x00000094, 0xffffff7f},
1721 {0x00000095, 0x00000fff},
1722 {0x00000096, 0x00116fff},
1723 {0x00000097, 0x60010000},
1724 {0x00000098, 0x10010000},
1725 {0x0000009f, 0x00c79000}
1726 };
1727
1728
1729 /**
1730 * cik_srbm_select - select specific register instances
1731 *
1732 * @rdev: radeon_device pointer
1733 * @me: selected ME (micro engine)
1734 * @pipe: pipe
1735 * @queue: queue
1736 * @vmid: VMID
1737 *
1738 * Switches the currently active registers instances. Some
1739 * registers are instanced per VMID, others are instanced per
1740 * me/pipe/queue combination.
1741 */
1742 static void cik_srbm_select(struct radeon_device *rdev,
1743 u32 me, u32 pipe, u32 queue, u32 vmid)
1744 {
1745 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1746 MEID(me & 0x3) |
1747 VMID(vmid & 0xf) |
1748 QUEUEID(queue & 0x7));
1749 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1750 }
1751
1752 /* ucode loading */
1753 /**
1754 * ci_mc_load_microcode - load MC ucode into the hw
1755 *
1756 * @rdev: radeon_device pointer
1757 *
1758 * Load the GDDR MC ucode into the hw (CIK).
1759 * Returns 0 on success, error on failure.
1760 */
1761 int ci_mc_load_microcode(struct radeon_device *rdev)
1762 {
1763 const __be32 *fw_data;
1764 u32 running, blackout = 0;
1765 u32 *io_mc_regs;
1766 int i, regs_size, ucode_size;
1767
1768 if (!rdev->mc_fw)
1769 return -EINVAL;
1770
1771 ucode_size = rdev->mc_fw->size / 4;
1772
1773 switch (rdev->family) {
1774 case CHIP_BONAIRE:
1775 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1776 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1777 break;
1778 case CHIP_HAWAII:
1779 io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1780 regs_size = HAWAII_IO_MC_REGS_SIZE;
1781 break;
1782 default:
1783 return -EINVAL;
1784 }
1785
1786 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1787
1788 if (running == 0) {
1789 if (running) {
1790 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1791 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1792 }
1793
1794 /* reset the engine and set to writable */
1795 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1796 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1797
1798 /* load mc io regs */
1799 for (i = 0; i < regs_size; i++) {
1800 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1801 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1802 }
1803 /* load the MC ucode */
1804 fw_data = (const __be32 *)rdev->mc_fw->data;
1805 for (i = 0; i < ucode_size; i++)
1806 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1807
1808 /* put the engine back into the active state */
1809 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1810 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1811 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1812
1813 /* wait for training to complete */
1814 for (i = 0; i < rdev->usec_timeout; i++) {
1815 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1816 break;
1817 udelay(1);
1818 }
1819 for (i = 0; i < rdev->usec_timeout; i++) {
1820 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1821 break;
1822 udelay(1);
1823 }
1824
1825 if (running)
1826 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1827 }
1828
1829 return 0;
1830 }
1831
1832 /**
1833 * cik_init_microcode - load ucode images from disk
1834 *
1835 * @rdev: radeon_device pointer
1836 *
1837 * Use the firmware interface to load the ucode images into
1838 * the driver (not loaded into hw).
1839 * Returns 0 on success, error on failure.
1840 */
1841 static int cik_init_microcode(struct radeon_device *rdev)
1842 {
1843 const char *chip_name;
1844 size_t pfp_req_size, me_req_size, ce_req_size,
1845 mec_req_size, rlc_req_size, mc_req_size = 0,
1846 sdma_req_size, smc_req_size = 0, mc2_req_size = 0;
1847 char fw_name[30];
1848 int err;
1849
1850 DRM_DEBUG("\n");
1851
1852 switch (rdev->family) {
1853 case CHIP_BONAIRE:
1854 chip_name = "BONAIRE";
1855 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1856 me_req_size = CIK_ME_UCODE_SIZE * 4;
1857 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1858 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1859 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1860 mc_req_size = BONAIRE_MC_UCODE_SIZE * 4;
1861 mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4;
1862 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1863 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
1864 break;
1865 case CHIP_HAWAII:
1866 chip_name = "HAWAII";
1867 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1868 me_req_size = CIK_ME_UCODE_SIZE * 4;
1869 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1870 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1871 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1872 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
1873 mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4;
1874 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1875 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
1876 break;
1877 case CHIP_KAVERI:
1878 chip_name = "KAVERI";
1879 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1880 me_req_size = CIK_ME_UCODE_SIZE * 4;
1881 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1882 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1883 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
1884 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1885 break;
1886 case CHIP_KABINI:
1887 chip_name = "KABINI";
1888 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1889 me_req_size = CIK_ME_UCODE_SIZE * 4;
1890 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1891 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1892 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
1893 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1894 break;
1895 case CHIP_MULLINS:
1896 chip_name = "MULLINS";
1897 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1898 me_req_size = CIK_ME_UCODE_SIZE * 4;
1899 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1900 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1901 rlc_req_size = ML_RLC_UCODE_SIZE * 4;
1902 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1903 break;
1904 default: BUG();
1905 }
1906
1907 DRM_INFO("Loading %s Microcode\n", chip_name);
1908
1909 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1910 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1911 if (err)
1912 goto out;
1913 if (rdev->pfp_fw->size != pfp_req_size) {
1914 printk(KERN_ERR
1915 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1916 rdev->pfp_fw->size, fw_name);
1917 err = -EINVAL;
1918 goto out;
1919 }
1920
1921 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1922 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1923 if (err)
1924 goto out;
1925 if (rdev->me_fw->size != me_req_size) {
1926 printk(KERN_ERR
1927 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1928 rdev->me_fw->size, fw_name);
1929 err = -EINVAL;
1930 }
1931
1932 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1933 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1934 if (err)
1935 goto out;
1936 if (rdev->ce_fw->size != ce_req_size) {
1937 printk(KERN_ERR
1938 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1939 rdev->ce_fw->size, fw_name);
1940 err = -EINVAL;
1941 }
1942
1943 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
1944 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
1945 if (err)
1946 goto out;
1947 if (rdev->mec_fw->size != mec_req_size) {
1948 printk(KERN_ERR
1949 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1950 rdev->mec_fw->size, fw_name);
1951 err = -EINVAL;
1952 }
1953
1954 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
1955 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1956 if (err)
1957 goto out;
1958 if (rdev->rlc_fw->size != rlc_req_size) {
1959 printk(KERN_ERR
1960 "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
1961 rdev->rlc_fw->size, fw_name);
1962 err = -EINVAL;
1963 }
1964
1965 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
1966 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
1967 if (err)
1968 goto out;
1969 if (rdev->sdma_fw->size != sdma_req_size) {
1970 printk(KERN_ERR
1971 "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
1972 rdev->sdma_fw->size, fw_name);
1973 err = -EINVAL;
1974 }
1975
1976 /* No SMC, MC ucode on APUs */
1977 if (!(rdev->flags & RADEON_IS_IGP)) {
1978 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
1979 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1980 if (err) {
1981 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1982 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1983 if (err)
1984 goto out;
1985 }
1986 if ((rdev->mc_fw->size != mc_req_size) &&
1987 (rdev->mc_fw->size != mc2_req_size)){
1988 printk(KERN_ERR
1989 "cik_mc: Bogus length %zu in firmware \"%s\"\n",
1990 rdev->mc_fw->size, fw_name);
1991 err = -EINVAL;
1992 }
1993 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
1994
1995 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1996 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1997 if (err) {
1998 printk(KERN_ERR
1999 "smc: error loading firmware \"%s\"\n",
2000 fw_name);
2001 release_firmware(rdev->smc_fw);
2002 rdev->smc_fw = NULL;
2003 err = 0;
2004 } else if (rdev->smc_fw->size != smc_req_size) {
2005 printk(KERN_ERR
2006 "cik_smc: Bogus length %zu in firmware \"%s\"\n",
2007 rdev->smc_fw->size, fw_name);
2008 err = -EINVAL;
2009 }
2010 }
2011
2012 out:
2013 if (err) {
2014 if (err != -EINVAL)
2015 printk(KERN_ERR
2016 "cik_cp: Failed to load firmware \"%s\"\n",
2017 fw_name);
2018 release_firmware(rdev->pfp_fw);
2019 rdev->pfp_fw = NULL;
2020 release_firmware(rdev->me_fw);
2021 rdev->me_fw = NULL;
2022 release_firmware(rdev->ce_fw);
2023 rdev->ce_fw = NULL;
2024 release_firmware(rdev->rlc_fw);
2025 rdev->rlc_fw = NULL;
2026 release_firmware(rdev->mc_fw);
2027 rdev->mc_fw = NULL;
2028 release_firmware(rdev->smc_fw);
2029 rdev->smc_fw = NULL;
2030 }
2031 return err;
2032 }
2033
2034 /*
2035 * Core functions
2036 */
2037 /**
2038 * cik_tiling_mode_table_init - init the hw tiling table
2039 *
2040 * @rdev: radeon_device pointer
2041 *
2042 * Starting with SI, the tiling setup is done globally in a
2043 * set of 32 tiling modes. Rather than selecting each set of
2044 * parameters per surface as on older asics, we just select
2045 * which index in the tiling table we want to use, and the
2046 * surface uses those parameters (CIK).
2047 */
2048 static void cik_tiling_mode_table_init(struct radeon_device *rdev)
2049 {
2050 const u32 num_tile_mode_states = 32;
2051 const u32 num_secondary_tile_mode_states = 16;
2052 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2053 u32 num_pipe_configs;
2054 u32 num_rbs = rdev->config.cik.max_backends_per_se *
2055 rdev->config.cik.max_shader_engines;
2056
2057 switch (rdev->config.cik.mem_row_size_in_kb) {
2058 case 1:
2059 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2060 break;
2061 case 2:
2062 default:
2063 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2064 break;
2065 case 4:
2066 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2067 break;
2068 }
2069
2070 num_pipe_configs = rdev->config.cik.max_tile_pipes;
2071 if (num_pipe_configs > 8)
2072 num_pipe_configs = 16;
2073
2074 if (num_pipe_configs == 16) {
2075 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2076 switch (reg_offset) {
2077 case 0:
2078 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2079 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2080 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2081 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2082 break;
2083 case 1:
2084 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2085 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2086 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2087 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2088 break;
2089 case 2:
2090 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2091 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2092 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2093 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2094 break;
2095 case 3:
2096 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2097 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2098 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2099 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2100 break;
2101 case 4:
2102 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2103 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2104 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2105 TILE_SPLIT(split_equal_to_row_size));
2106 break;
2107 case 5:
2108 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2109 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2110 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2111 break;
2112 case 6:
2113 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2114 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2115 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2116 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2117 break;
2118 case 7:
2119 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2120 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2121 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2122 TILE_SPLIT(split_equal_to_row_size));
2123 break;
2124 case 8:
2125 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2126 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2127 break;
2128 case 9:
2129 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2130 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2131 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2132 break;
2133 case 10:
2134 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2135 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2136 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2137 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2138 break;
2139 case 11:
2140 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2141 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2142 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2143 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2144 break;
2145 case 12:
2146 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2147 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2148 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2149 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2150 break;
2151 case 13:
2152 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2153 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2154 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2155 break;
2156 case 14:
2157 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2158 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2159 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2160 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2161 break;
2162 case 16:
2163 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2164 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2165 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2166 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2167 break;
2168 case 17:
2169 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2170 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2171 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2172 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2173 break;
2174 case 27:
2175 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2176 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2177 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2178 break;
2179 case 28:
2180 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2181 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2182 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2183 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2184 break;
2185 case 29:
2186 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2187 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2188 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2189 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2190 break;
2191 case 30:
2192 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2193 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2194 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2195 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2196 break;
2197 default:
2198 gb_tile_moden = 0;
2199 break;
2200 }
2201 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2202 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2203 }
2204 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2205 switch (reg_offset) {
2206 case 0:
2207 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2208 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2209 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2210 NUM_BANKS(ADDR_SURF_16_BANK));
2211 break;
2212 case 1:
2213 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2214 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2215 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2216 NUM_BANKS(ADDR_SURF_16_BANK));
2217 break;
2218 case 2:
2219 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2220 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2221 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2222 NUM_BANKS(ADDR_SURF_16_BANK));
2223 break;
2224 case 3:
2225 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2226 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2227 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2228 NUM_BANKS(ADDR_SURF_16_BANK));
2229 break;
2230 case 4:
2231 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2232 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2233 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2234 NUM_BANKS(ADDR_SURF_8_BANK));
2235 break;
2236 case 5:
2237 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2238 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2239 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2240 NUM_BANKS(ADDR_SURF_4_BANK));
2241 break;
2242 case 6:
2243 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2244 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2245 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2246 NUM_BANKS(ADDR_SURF_2_BANK));
2247 break;
2248 case 8:
2249 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2250 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2251 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2252 NUM_BANKS(ADDR_SURF_16_BANK));
2253 break;
2254 case 9:
2255 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2256 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2257 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2258 NUM_BANKS(ADDR_SURF_16_BANK));
2259 break;
2260 case 10:
2261 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2262 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2263 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2264 NUM_BANKS(ADDR_SURF_16_BANK));
2265 break;
2266 case 11:
2267 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2268 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2269 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2270 NUM_BANKS(ADDR_SURF_8_BANK));
2271 break;
2272 case 12:
2273 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2274 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2275 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2276 NUM_BANKS(ADDR_SURF_4_BANK));
2277 break;
2278 case 13:
2279 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2280 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2281 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2282 NUM_BANKS(ADDR_SURF_2_BANK));
2283 break;
2284 case 14:
2285 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2286 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2287 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2288 NUM_BANKS(ADDR_SURF_2_BANK));
2289 break;
2290 default:
2291 gb_tile_moden = 0;
2292 break;
2293 }
2294 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2295 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2296 }
2297 } else if (num_pipe_configs == 8) {
2298 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2299 switch (reg_offset) {
2300 case 0:
2301 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2302 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2303 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2304 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2305 break;
2306 case 1:
2307 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2308 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2309 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2310 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2311 break;
2312 case 2:
2313 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2314 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2315 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2316 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2317 break;
2318 case 3:
2319 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2320 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2321 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2322 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2323 break;
2324 case 4:
2325 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2326 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2327 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2328 TILE_SPLIT(split_equal_to_row_size));
2329 break;
2330 case 5:
2331 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2332 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2333 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2334 break;
2335 case 6:
2336 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2337 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2338 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2339 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2340 break;
2341 case 7:
2342 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2343 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2344 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2345 TILE_SPLIT(split_equal_to_row_size));
2346 break;
2347 case 8:
2348 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2349 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2350 break;
2351 case 9:
2352 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2353 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2354 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2355 break;
2356 case 10:
2357 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2358 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2359 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2360 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2361 break;
2362 case 11:
2363 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2364 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2365 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2366 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2367 break;
2368 case 12:
2369 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2370 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2371 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2372 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2373 break;
2374 case 13:
2375 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2376 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2377 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2378 break;
2379 case 14:
2380 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2381 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2382 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2383 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2384 break;
2385 case 16:
2386 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2387 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2388 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2389 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2390 break;
2391 case 17:
2392 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2393 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2394 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2395 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2396 break;
2397 case 27:
2398 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2399 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2400 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2401 break;
2402 case 28:
2403 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2404 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2405 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2406 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2407 break;
2408 case 29:
2409 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2410 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2411 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2412 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2413 break;
2414 case 30:
2415 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2416 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2417 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2418 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2419 break;
2420 default:
2421 gb_tile_moden = 0;
2422 break;
2423 }
2424 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2425 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2426 }
2427 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2428 switch (reg_offset) {
2429 case 0:
2430 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2431 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2432 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2433 NUM_BANKS(ADDR_SURF_16_BANK));
2434 break;
2435 case 1:
2436 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2437 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2438 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2439 NUM_BANKS(ADDR_SURF_16_BANK));
2440 break;
2441 case 2:
2442 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2443 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2444 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2445 NUM_BANKS(ADDR_SURF_16_BANK));
2446 break;
2447 case 3:
2448 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2449 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2450 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2451 NUM_BANKS(ADDR_SURF_16_BANK));
2452 break;
2453 case 4:
2454 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2455 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2456 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2457 NUM_BANKS(ADDR_SURF_8_BANK));
2458 break;
2459 case 5:
2460 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2461 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2462 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2463 NUM_BANKS(ADDR_SURF_4_BANK));
2464 break;
2465 case 6:
2466 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2467 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2468 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2469 NUM_BANKS(ADDR_SURF_2_BANK));
2470 break;
2471 case 8:
2472 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2473 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2474 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2475 NUM_BANKS(ADDR_SURF_16_BANK));
2476 break;
2477 case 9:
2478 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2479 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2480 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2481 NUM_BANKS(ADDR_SURF_16_BANK));
2482 break;
2483 case 10:
2484 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2485 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2486 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2487 NUM_BANKS(ADDR_SURF_16_BANK));
2488 break;
2489 case 11:
2490 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2491 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2492 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2493 NUM_BANKS(ADDR_SURF_16_BANK));
2494 break;
2495 case 12:
2496 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2497 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2498 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2499 NUM_BANKS(ADDR_SURF_8_BANK));
2500 break;
2501 case 13:
2502 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2503 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2504 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2505 NUM_BANKS(ADDR_SURF_4_BANK));
2506 break;
2507 case 14:
2508 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2509 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2510 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2511 NUM_BANKS(ADDR_SURF_2_BANK));
2512 break;
2513 default:
2514 gb_tile_moden = 0;
2515 break;
2516 }
2517 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2518 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2519 }
2520 } else if (num_pipe_configs == 4) {
2521 if (num_rbs == 4) {
2522 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2523 switch (reg_offset) {
2524 case 0:
2525 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2526 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2527 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2528 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2529 break;
2530 case 1:
2531 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2532 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2533 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2534 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2535 break;
2536 case 2:
2537 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2538 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2539 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2540 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2541 break;
2542 case 3:
2543 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2544 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2545 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2546 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2547 break;
2548 case 4:
2549 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2550 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2551 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2552 TILE_SPLIT(split_equal_to_row_size));
2553 break;
2554 case 5:
2555 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2556 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2557 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2558 break;
2559 case 6:
2560 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2561 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2562 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2563 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2564 break;
2565 case 7:
2566 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2567 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2568 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2569 TILE_SPLIT(split_equal_to_row_size));
2570 break;
2571 case 8:
2572 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2573 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2574 break;
2575 case 9:
2576 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2577 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2578 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2579 break;
2580 case 10:
2581 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2582 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2583 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2584 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2585 break;
2586 case 11:
2587 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2588 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2589 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2590 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2591 break;
2592 case 12:
2593 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2594 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2595 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2596 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2597 break;
2598 case 13:
2599 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2600 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2601 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2602 break;
2603 case 14:
2604 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2605 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2606 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2607 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2608 break;
2609 case 16:
2610 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2611 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2612 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2613 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2614 break;
2615 case 17:
2616 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2617 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2618 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2619 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2620 break;
2621 case 27:
2622 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2623 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2624 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2625 break;
2626 case 28:
2627 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2628 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2629 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2630 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2631 break;
2632 case 29:
2633 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2634 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2635 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2636 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2637 break;
2638 case 30:
2639 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2640 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2641 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2642 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2643 break;
2644 default:
2645 gb_tile_moden = 0;
2646 break;
2647 }
2648 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2649 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2650 }
2651 } else if (num_rbs < 4) {
2652 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2653 switch (reg_offset) {
2654 case 0:
2655 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2656 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2657 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2658 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2659 break;
2660 case 1:
2661 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2662 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2663 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2664 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2665 break;
2666 case 2:
2667 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2668 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2669 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2670 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2671 break;
2672 case 3:
2673 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2674 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2675 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2676 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2677 break;
2678 case 4:
2679 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2680 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2681 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2682 TILE_SPLIT(split_equal_to_row_size));
2683 break;
2684 case 5:
2685 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2686 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2687 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2688 break;
2689 case 6:
2690 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2691 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2692 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2693 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2694 break;
2695 case 7:
2696 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2697 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2698 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2699 TILE_SPLIT(split_equal_to_row_size));
2700 break;
2701 case 8:
2702 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2703 PIPE_CONFIG(ADDR_SURF_P4_8x16));
2704 break;
2705 case 9:
2706 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2707 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2708 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2709 break;
2710 case 10:
2711 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2712 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2713 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2714 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2715 break;
2716 case 11:
2717 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2718 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2719 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2720 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2721 break;
2722 case 12:
2723 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2724 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2725 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2726 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2727 break;
2728 case 13:
2729 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2730 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2731 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2732 break;
2733 case 14:
2734 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2735 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2736 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2737 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2738 break;
2739 case 16:
2740 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2741 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2742 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2743 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2744 break;
2745 case 17:
2746 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2747 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2748 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2749 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2750 break;
2751 case 27:
2752 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2753 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2754 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2755 break;
2756 case 28:
2757 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2758 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2759 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2760 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2761 break;
2762 case 29:
2763 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2764 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2765 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2766 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2767 break;
2768 case 30:
2769 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2770 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2771 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2772 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2773 break;
2774 default:
2775 gb_tile_moden = 0;
2776 break;
2777 }
2778 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2779 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2780 }
2781 }
2782 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2783 switch (reg_offset) {
2784 case 0:
2785 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2786 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2787 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2788 NUM_BANKS(ADDR_SURF_16_BANK));
2789 break;
2790 case 1:
2791 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2792 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2793 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2794 NUM_BANKS(ADDR_SURF_16_BANK));
2795 break;
2796 case 2:
2797 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2798 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2799 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2800 NUM_BANKS(ADDR_SURF_16_BANK));
2801 break;
2802 case 3:
2803 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2804 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2805 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2806 NUM_BANKS(ADDR_SURF_16_BANK));
2807 break;
2808 case 4:
2809 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2810 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2811 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2812 NUM_BANKS(ADDR_SURF_16_BANK));
2813 break;
2814 case 5:
2815 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2816 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2817 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2818 NUM_BANKS(ADDR_SURF_8_BANK));
2819 break;
2820 case 6:
2821 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2822 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2823 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2824 NUM_BANKS(ADDR_SURF_4_BANK));
2825 break;
2826 case 8:
2827 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2828 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2829 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2830 NUM_BANKS(ADDR_SURF_16_BANK));
2831 break;
2832 case 9:
2833 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2834 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2835 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2836 NUM_BANKS(ADDR_SURF_16_BANK));
2837 break;
2838 case 10:
2839 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2840 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2841 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2842 NUM_BANKS(ADDR_SURF_16_BANK));
2843 break;
2844 case 11:
2845 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2846 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2847 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2848 NUM_BANKS(ADDR_SURF_16_BANK));
2849 break;
2850 case 12:
2851 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2852 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2853 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2854 NUM_BANKS(ADDR_SURF_16_BANK));
2855 break;
2856 case 13:
2857 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2858 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2859 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2860 NUM_BANKS(ADDR_SURF_8_BANK));
2861 break;
2862 case 14:
2863 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2864 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2865 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2866 NUM_BANKS(ADDR_SURF_4_BANK));
2867 break;
2868 default:
2869 gb_tile_moden = 0;
2870 break;
2871 }
2872 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2873 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2874 }
2875 } else if (num_pipe_configs == 2) {
2876 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2877 switch (reg_offset) {
2878 case 0:
2879 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2880 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2881 PIPE_CONFIG(ADDR_SURF_P2) |
2882 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2883 break;
2884 case 1:
2885 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2886 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2887 PIPE_CONFIG(ADDR_SURF_P2) |
2888 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2889 break;
2890 case 2:
2891 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2892 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2893 PIPE_CONFIG(ADDR_SURF_P2) |
2894 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2895 break;
2896 case 3:
2897 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2898 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2899 PIPE_CONFIG(ADDR_SURF_P2) |
2900 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2901 break;
2902 case 4:
2903 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2904 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2905 PIPE_CONFIG(ADDR_SURF_P2) |
2906 TILE_SPLIT(split_equal_to_row_size));
2907 break;
2908 case 5:
2909 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2910 PIPE_CONFIG(ADDR_SURF_P2) |
2911 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2912 break;
2913 case 6:
2914 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2915 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2916 PIPE_CONFIG(ADDR_SURF_P2) |
2917 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2918 break;
2919 case 7:
2920 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2921 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2922 PIPE_CONFIG(ADDR_SURF_P2) |
2923 TILE_SPLIT(split_equal_to_row_size));
2924 break;
2925 case 8:
2926 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2927 PIPE_CONFIG(ADDR_SURF_P2);
2928 break;
2929 case 9:
2930 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2931 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2932 PIPE_CONFIG(ADDR_SURF_P2));
2933 break;
2934 case 10:
2935 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2936 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2937 PIPE_CONFIG(ADDR_SURF_P2) |
2938 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2939 break;
2940 case 11:
2941 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2942 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2943 PIPE_CONFIG(ADDR_SURF_P2) |
2944 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2945 break;
2946 case 12:
2947 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2948 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2949 PIPE_CONFIG(ADDR_SURF_P2) |
2950 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2951 break;
2952 case 13:
2953 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2954 PIPE_CONFIG(ADDR_SURF_P2) |
2955 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2956 break;
2957 case 14:
2958 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2959 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2960 PIPE_CONFIG(ADDR_SURF_P2) |
2961 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2962 break;
2963 case 16:
2964 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2965 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2966 PIPE_CONFIG(ADDR_SURF_P2) |
2967 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2968 break;
2969 case 17:
2970 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2971 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2972 PIPE_CONFIG(ADDR_SURF_P2) |
2973 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2974 break;
2975 case 27:
2976 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2977 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2978 PIPE_CONFIG(ADDR_SURF_P2));
2979 break;
2980 case 28:
2981 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2982 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2983 PIPE_CONFIG(ADDR_SURF_P2) |
2984 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2985 break;
2986 case 29:
2987 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2988 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2989 PIPE_CONFIG(ADDR_SURF_P2) |
2990 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2991 break;
2992 case 30:
2993 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2994 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2995 PIPE_CONFIG(ADDR_SURF_P2) |
2996 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2997 break;
2998 default:
2999 gb_tile_moden = 0;
3000 break;
3001 }
3002 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
3003 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3004 }
3005 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
3006 switch (reg_offset) {
3007 case 0:
3008 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3009 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3010 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3011 NUM_BANKS(ADDR_SURF_16_BANK));
3012 break;
3013 case 1:
3014 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3015 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3016 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3017 NUM_BANKS(ADDR_SURF_16_BANK));
3018 break;
3019 case 2:
3020 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3021 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3022 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3023 NUM_BANKS(ADDR_SURF_16_BANK));
3024 break;
3025 case 3:
3026 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3027 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3028 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3029 NUM_BANKS(ADDR_SURF_16_BANK));
3030 break;
3031 case 4:
3032 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3033 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3034 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3035 NUM_BANKS(ADDR_SURF_16_BANK));
3036 break;
3037 case 5:
3038 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3039 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3040 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3041 NUM_BANKS(ADDR_SURF_16_BANK));
3042 break;
3043 case 6:
3044 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3045 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3046 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3047 NUM_BANKS(ADDR_SURF_8_BANK));
3048 break;
3049 case 8:
3050 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3051 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3052 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3053 NUM_BANKS(ADDR_SURF_16_BANK));
3054 break;
3055 case 9:
3056 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3057 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3058 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3059 NUM_BANKS(ADDR_SURF_16_BANK));
3060 break;
3061 case 10:
3062 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3063 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3064 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3065 NUM_BANKS(ADDR_SURF_16_BANK));
3066 break;
3067 case 11:
3068 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3069 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3070 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3071 NUM_BANKS(ADDR_SURF_16_BANK));
3072 break;
3073 case 12:
3074 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3075 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3076 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3077 NUM_BANKS(ADDR_SURF_16_BANK));
3078 break;
3079 case 13:
3080 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3081 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3082 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3083 NUM_BANKS(ADDR_SURF_16_BANK));
3084 break;
3085 case 14:
3086 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3087 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3088 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3089 NUM_BANKS(ADDR_SURF_8_BANK));
3090 break;
3091 default:
3092 gb_tile_moden = 0;
3093 break;
3094 }
3095 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
3096 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3097 }
3098 } else
3099 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
3100 }
3101
3102 /**
3103 * cik_select_se_sh - select which SE, SH to address
3104 *
3105 * @rdev: radeon_device pointer
3106 * @se_num: shader engine to address
3107 * @sh_num: sh block to address
3108 *
3109 * Select which SE, SH combinations to address. Certain
3110 * registers are instanced per SE or SH. 0xffffffff means
3111 * broadcast to all SEs or SHs (CIK).
3112 */
3113 static void cik_select_se_sh(struct radeon_device *rdev,
3114 u32 se_num, u32 sh_num)
3115 {
3116 u32 data = INSTANCE_BROADCAST_WRITES;
3117
3118 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
3119 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
3120 else if (se_num == 0xffffffff)
3121 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3122 else if (sh_num == 0xffffffff)
3123 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3124 else
3125 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3126 WREG32(GRBM_GFX_INDEX, data);
3127 }
3128
3129 /**
3130 * cik_create_bitmask - create a bitmask
3131 *
3132 * @bit_width: length of the mask
3133 *
3134 * create a variable length bit mask (CIK).
3135 * Returns the bitmask.
3136 */
3137 static u32 cik_create_bitmask(u32 bit_width)
3138 {
3139 u32 i, mask = 0;
3140
3141 for (i = 0; i < bit_width; i++) {
3142 mask <<= 1;
3143 mask |= 1;
3144 }
3145 return mask;
3146 }
3147
3148 /**
3149 * cik_get_rb_disabled - computes the mask of disabled RBs
3150 *
3151 * @rdev: radeon_device pointer
3152 * @max_rb_num: max RBs (render backends) for the asic
3153 * @se_num: number of SEs (shader engines) for the asic
3154 * @sh_per_se: number of SH blocks per SE for the asic
3155 *
3156 * Calculates the bitmask of disabled RBs (CIK).
3157 * Returns the disabled RB bitmask.
3158 */
3159 static u32 cik_get_rb_disabled(struct radeon_device *rdev,
3160 u32 max_rb_num_per_se,
3161 u32 sh_per_se)
3162 {
3163 u32 data, mask;
3164
3165 data = RREG32(CC_RB_BACKEND_DISABLE);
3166 if (data & 1)
3167 data &= BACKEND_DISABLE_MASK;
3168 else
3169 data = 0;
3170 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3171
3172 data >>= BACKEND_DISABLE_SHIFT;
3173
3174 mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
3175
3176 return data & mask;
3177 }
3178
3179 /**
3180 * cik_setup_rb - setup the RBs on the asic
3181 *
3182 * @rdev: radeon_device pointer
3183 * @se_num: number of SEs (shader engines) for the asic
3184 * @sh_per_se: number of SH blocks per SE for the asic
3185 * @max_rb_num: max RBs (render backends) for the asic
3186 *
3187 * Configures per-SE/SH RB registers (CIK).
3188 */
3189 static void cik_setup_rb(struct radeon_device *rdev,
3190 u32 se_num, u32 sh_per_se,
3191 u32 max_rb_num_per_se)
3192 {
3193 int i, j;
3194 u32 data, mask;
3195 u32 disabled_rbs = 0;
3196 u32 enabled_rbs = 0;
3197
3198 for (i = 0; i < se_num; i++) {
3199 for (j = 0; j < sh_per_se; j++) {
3200 cik_select_se_sh(rdev, i, j);
3201 data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
3202 if (rdev->family == CHIP_HAWAII)
3203 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3204 else
3205 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
3206 }
3207 }
3208 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3209
3210 mask = 1;
3211 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
3212 if (!(disabled_rbs & mask))
3213 enabled_rbs |= mask;
3214 mask <<= 1;
3215 }
3216
3217 rdev->config.cik.backend_enable_mask = enabled_rbs;
3218
3219 for (i = 0; i < se_num; i++) {
3220 cik_select_se_sh(rdev, i, 0xffffffff);
3221 data = 0;
3222 for (j = 0; j < sh_per_se; j++) {
3223 switch (enabled_rbs & 3) {
3224 case 0:
3225 if (j == 0)
3226 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3227 else
3228 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3229 break;
3230 case 1:
3231 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3232 break;
3233 case 2:
3234 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3235 break;
3236 case 3:
3237 default:
3238 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3239 break;
3240 }
3241 enabled_rbs >>= 2;
3242 }
3243 WREG32(PA_SC_RASTER_CONFIG, data);
3244 }
3245 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3246 }
3247
3248 /**
3249 * cik_gpu_init - setup the 3D engine
3250 *
3251 * @rdev: radeon_device pointer
3252 *
3253 * Configures the 3D engine and tiling configuration
3254 * registers so that the 3D engine is usable.
3255 */
3256 static void cik_gpu_init(struct radeon_device *rdev)
3257 {
3258 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3259 u32 mc_shared_chmap, mc_arb_ramcfg;
3260 u32 hdp_host_path_cntl;
3261 u32 tmp;
3262 int i, j, k;
3263
3264 switch (rdev->family) {
3265 case CHIP_BONAIRE:
3266 rdev->config.cik.max_shader_engines = 2;
3267 rdev->config.cik.max_tile_pipes = 4;
3268 rdev->config.cik.max_cu_per_sh = 7;
3269 rdev->config.cik.max_sh_per_se = 1;
3270 rdev->config.cik.max_backends_per_se = 2;
3271 rdev->config.cik.max_texture_channel_caches = 4;
3272 rdev->config.cik.max_gprs = 256;
3273 rdev->config.cik.max_gs_threads = 32;
3274 rdev->config.cik.max_hw_contexts = 8;
3275
3276 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3277 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3278 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3279 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3280 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3281 break;
3282 case CHIP_HAWAII:
3283 rdev->config.cik.max_shader_engines = 4;
3284 rdev->config.cik.max_tile_pipes = 16;
3285 rdev->config.cik.max_cu_per_sh = 11;
3286 rdev->config.cik.max_sh_per_se = 1;
3287 rdev->config.cik.max_backends_per_se = 4;
3288 rdev->config.cik.max_texture_channel_caches = 16;
3289 rdev->config.cik.max_gprs = 256;
3290 rdev->config.cik.max_gs_threads = 32;
3291 rdev->config.cik.max_hw_contexts = 8;
3292
3293 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3294 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3295 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3296 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3297 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3298 break;
3299 case CHIP_KAVERI:
3300 rdev->config.cik.max_shader_engines = 1;
3301 rdev->config.cik.max_tile_pipes = 4;
3302 if ((rdev->pdev->device == 0x1304) ||
3303 (rdev->pdev->device == 0x1305) ||
3304 (rdev->pdev->device == 0x130C) ||
3305 (rdev->pdev->device == 0x130F) ||
3306 (rdev->pdev->device == 0x1310) ||
3307 (rdev->pdev->device == 0x1311) ||
3308 (rdev->pdev->device == 0x131C)) {
3309 rdev->config.cik.max_cu_per_sh = 8;
3310 rdev->config.cik.max_backends_per_se = 2;
3311 } else if ((rdev->pdev->device == 0x1309) ||
3312 (rdev->pdev->device == 0x130A) ||
3313 (rdev->pdev->device == 0x130D) ||
3314 (rdev->pdev->device == 0x1313) ||
3315 (rdev->pdev->device == 0x131D)) {
3316 rdev->config.cik.max_cu_per_sh = 6;
3317 rdev->config.cik.max_backends_per_se = 2;
3318 } else if ((rdev->pdev->device == 0x1306) ||
3319 (rdev->pdev->device == 0x1307) ||
3320 (rdev->pdev->device == 0x130B) ||
3321 (rdev->pdev->device == 0x130E) ||
3322 (rdev->pdev->device == 0x1315) ||
3323 (rdev->pdev->device == 0x131B)) {
3324 rdev->config.cik.max_cu_per_sh = 4;
3325 rdev->config.cik.max_backends_per_se = 1;
3326 } else {
3327 rdev->config.cik.max_cu_per_sh = 3;
3328 rdev->config.cik.max_backends_per_se = 1;
3329 }
3330 rdev->config.cik.max_sh_per_se = 1;
3331 rdev->config.cik.max_texture_channel_caches = 4;
3332 rdev->config.cik.max_gprs = 256;
3333 rdev->config.cik.max_gs_threads = 16;
3334 rdev->config.cik.max_hw_contexts = 8;
3335
3336 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3337 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3338 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3339 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3340 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3341 break;
3342 case CHIP_KABINI:
3343 case CHIP_MULLINS:
3344 default:
3345 rdev->config.cik.max_shader_engines = 1;
3346 rdev->config.cik.max_tile_pipes = 2;
3347 rdev->config.cik.max_cu_per_sh = 2;
3348 rdev->config.cik.max_sh_per_se = 1;
3349 rdev->config.cik.max_backends_per_se = 1;
3350 rdev->config.cik.max_texture_channel_caches = 2;
3351 rdev->config.cik.max_gprs = 256;
3352 rdev->config.cik.max_gs_threads = 16;
3353 rdev->config.cik.max_hw_contexts = 8;
3354
3355 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3356 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3357 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3358 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3359 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3360 break;
3361 }
3362
3363 /* Initialize HDP */
3364 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3365 WREG32((0x2c14 + j), 0x00000000);
3366 WREG32((0x2c18 + j), 0x00000000);
3367 WREG32((0x2c1c + j), 0x00000000);
3368 WREG32((0x2c20 + j), 0x00000000);
3369 WREG32((0x2c24 + j), 0x00000000);
3370 }
3371
3372 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3373
3374 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3375
3376 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3377 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3378
3379 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3380 rdev->config.cik.mem_max_burst_length_bytes = 256;
3381 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3382 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3383 if (rdev->config.cik.mem_row_size_in_kb > 4)
3384 rdev->config.cik.mem_row_size_in_kb = 4;
3385 /* XXX use MC settings? */
3386 rdev->config.cik.shader_engine_tile_size = 32;
3387 rdev->config.cik.num_gpus = 1;
3388 rdev->config.cik.multi_gpu_tile_size = 64;
3389
3390 /* fix up row size */
3391 gb_addr_config &= ~ROW_SIZE_MASK;
3392 switch (rdev->config.cik.mem_row_size_in_kb) {
3393 case 1:
3394 default:
3395 gb_addr_config |= ROW_SIZE(0);
3396 break;
3397 case 2:
3398 gb_addr_config |= ROW_SIZE(1);
3399 break;
3400 case 4:
3401 gb_addr_config |= ROW_SIZE(2);
3402 break;
3403 }
3404
3405 /* setup tiling info dword. gb_addr_config is not adequate since it does
3406 * not have bank info, so create a custom tiling dword.
3407 * bits 3:0 num_pipes
3408 * bits 7:4 num_banks
3409 * bits 11:8 group_size
3410 * bits 15:12 row_size
3411 */
3412 rdev->config.cik.tile_config = 0;
3413 switch (rdev->config.cik.num_tile_pipes) {
3414 case 1:
3415 rdev->config.cik.tile_config |= (0 << 0);
3416 break;
3417 case 2:
3418 rdev->config.cik.tile_config |= (1 << 0);
3419 break;
3420 case 4:
3421 rdev->config.cik.tile_config |= (2 << 0);
3422 break;
3423 case 8:
3424 default:
3425 /* XXX what about 12? */
3426 rdev->config.cik.tile_config |= (3 << 0);
3427 break;
3428 }
3429 rdev->config.cik.tile_config |=
3430 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
3431 rdev->config.cik.tile_config |=
3432 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3433 rdev->config.cik.tile_config |=
3434 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3435
3436 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3437 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3438 WREG32(DMIF_ADDR_CALC, gb_addr_config);
3439 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3440 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
3441 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3442 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3443 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3444
3445 cik_tiling_mode_table_init(rdev);
3446
3447 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3448 rdev->config.cik.max_sh_per_se,
3449 rdev->config.cik.max_backends_per_se);
3450
3451 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
3452 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
3453 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k++) {
3454 rdev->config.cik.active_cus +=
3455 hweight32(cik_get_cu_active_bitmap(rdev, i, j));
3456 }
3457 }
3458 }
3459
3460 /* set HW defaults for 3D engine */
3461 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3462
3463 WREG32(SX_DEBUG_1, 0x20);
3464
3465 WREG32(TA_CNTL_AUX, 0x00010000);
3466
3467 tmp = RREG32(SPI_CONFIG_CNTL);
3468 tmp |= 0x03000000;
3469 WREG32(SPI_CONFIG_CNTL, tmp);
3470
3471 WREG32(SQ_CONFIG, 1);
3472
3473 WREG32(DB_DEBUG, 0);
3474
3475 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3476 tmp |= 0x00000400;
3477 WREG32(DB_DEBUG2, tmp);
3478
3479 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3480 tmp |= 0x00020200;
3481 WREG32(DB_DEBUG3, tmp);
3482
3483 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3484 tmp |= 0x00018208;
3485 WREG32(CB_HW_CONTROL, tmp);
3486
3487 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3488
3489 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3490 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3491 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3492 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3493
3494 WREG32(VGT_NUM_INSTANCES, 1);
3495
3496 WREG32(CP_PERFMON_CNTL, 0);
3497
3498 WREG32(SQ_CONFIG, 0);
3499
3500 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3501 FORCE_EOV_MAX_REZ_CNT(255)));
3502
3503 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3504 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3505
3506 WREG32(VGT_GS_VERTEX_REUSE, 16);
3507 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3508
3509 tmp = RREG32(HDP_MISC_CNTL);
3510 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3511 WREG32(HDP_MISC_CNTL, tmp);
3512
3513 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3514 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3515
3516 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3517 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3518
3519 udelay(50);
3520 }
3521
3522 /*
3523 * GPU scratch registers helpers function.
3524 */
3525 /**
3526 * cik_scratch_init - setup driver info for CP scratch regs
3527 *
3528 * @rdev: radeon_device pointer
3529 *
3530 * Set up the number and offset of the CP scratch registers.
3531 * NOTE: use of CP scratch registers is a legacy inferface and
3532 * is not used by default on newer asics (r6xx+). On newer asics,
3533 * memory buffers are used for fences rather than scratch regs.
3534 */
3535 static void cik_scratch_init(struct radeon_device *rdev)
3536 {
3537 int i;
3538
3539 rdev->scratch.num_reg = 7;
3540 rdev->scratch.reg_base = SCRATCH_REG0;
3541 for (i = 0; i < rdev->scratch.num_reg; i++) {
3542 rdev->scratch.free[i] = true;
3543 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3544 }
3545 }
3546
3547 /**
3548 * cik_ring_test - basic gfx ring test
3549 *
3550 * @rdev: radeon_device pointer
3551 * @ring: radeon_ring structure holding ring information
3552 *
3553 * Allocate a scratch register and write to it using the gfx ring (CIK).
3554 * Provides a basic gfx ring test to verify that the ring is working.
3555 * Used by cik_cp_gfx_resume();
3556 * Returns 0 on success, error on failure.
3557 */
3558 int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3559 {
3560 uint32_t scratch;
3561 uint32_t tmp = 0;
3562 unsigned i;
3563 int r;
3564
3565 r = radeon_scratch_get(rdev, &scratch);
3566 if (r) {
3567 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3568 return r;
3569 }
3570 WREG32(scratch, 0xCAFEDEAD);
3571 r = radeon_ring_lock(rdev, ring, 3);
3572 if (r) {
3573 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3574 radeon_scratch_free(rdev, scratch);
3575 return r;
3576 }
3577 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3578 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3579 radeon_ring_write(ring, 0xDEADBEEF);
3580 radeon_ring_unlock_commit(rdev, ring);
3581
3582 for (i = 0; i < rdev->usec_timeout; i++) {
3583 tmp = RREG32(scratch);
3584 if (tmp == 0xDEADBEEF)
3585 break;
3586 DRM_UDELAY(1);
3587 }
3588 if (i < rdev->usec_timeout) {
3589 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3590 } else {
3591 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3592 ring->idx, scratch, tmp);
3593 r = -EINVAL;
3594 }
3595 radeon_scratch_free(rdev, scratch);
3596 return r;
3597 }
3598
3599 /**
3600 * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3601 *
3602 * @rdev: radeon_device pointer
3603 * @ridx: radeon ring index
3604 *
3605 * Emits an hdp flush on the cp.
3606 */
3607 static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,
3608 int ridx)
3609 {
3610 struct radeon_ring *ring = &rdev->ring[ridx];
3611 u32 ref_and_mask;
3612
3613 switch (ring->idx) {
3614 case CAYMAN_RING_TYPE_CP1_INDEX:
3615 case CAYMAN_RING_TYPE_CP2_INDEX:
3616 default:
3617 switch (ring->me) {
3618 case 0:
3619 ref_and_mask = CP2 << ring->pipe;
3620 break;
3621 case 1:
3622 ref_and_mask = CP6 << ring->pipe;
3623 break;
3624 default:
3625 return;
3626 }
3627 break;
3628 case RADEON_RING_TYPE_GFX_INDEX:
3629 ref_and_mask = CP0;
3630 break;
3631 }
3632
3633 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3634 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
3635 WAIT_REG_MEM_FUNCTION(3) | /* == */
3636 WAIT_REG_MEM_ENGINE(1))); /* pfp */
3637 radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2);
3638 radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2);
3639 radeon_ring_write(ring, ref_and_mask);
3640 radeon_ring_write(ring, ref_and_mask);
3641 radeon_ring_write(ring, 0x20); /* poll interval */
3642 }
3643
3644 /**
3645 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
3646 *
3647 * @rdev: radeon_device pointer
3648 * @fence: radeon fence object
3649 *
3650 * Emits a fence sequnce number on the gfx ring and flushes
3651 * GPU caches.
3652 */
3653 void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3654 struct radeon_fence *fence)
3655 {
3656 struct radeon_ring *ring = &rdev->ring[fence->ring];
3657 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3658
3659 /* EVENT_WRITE_EOP - flush caches, send int */
3660 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3661 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3662 EOP_TC_ACTION_EN |
3663 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3664 EVENT_INDEX(5)));
3665 radeon_ring_write(ring, addr & 0xfffffffc);
3666 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3667 radeon_ring_write(ring, fence->seq);
3668 radeon_ring_write(ring, 0);
3669 /* HDP flush */
3670 cik_hdp_flush_cp_ring_emit(rdev, fence->ring);
3671 }
3672
3673 /**
3674 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3675 *
3676 * @rdev: radeon_device pointer
3677 * @fence: radeon fence object
3678 *
3679 * Emits a fence sequnce number on the compute ring and flushes
3680 * GPU caches.
3681 */
3682 void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3683 struct radeon_fence *fence)
3684 {
3685 struct radeon_ring *ring = &rdev->ring[fence->ring];
3686 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3687
3688 /* RELEASE_MEM - flush caches, send int */
3689 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3690 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3691 EOP_TC_ACTION_EN |
3692 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3693 EVENT_INDEX(5)));
3694 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3695 radeon_ring_write(ring, addr & 0xfffffffc);
3696 radeon_ring_write(ring, upper_32_bits(addr));
3697 radeon_ring_write(ring, fence->seq);
3698 radeon_ring_write(ring, 0);
3699 /* HDP flush */
3700 cik_hdp_flush_cp_ring_emit(rdev, fence->ring);
3701 }
3702
3703 bool cik_semaphore_ring_emit(struct radeon_device *rdev,
3704 struct radeon_ring *ring,
3705 struct radeon_semaphore *semaphore,
3706 bool emit_wait)
3707 {
3708 uint64_t addr = semaphore->gpu_addr;
3709 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3710
3711 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3712 radeon_ring_write(ring, lower_32_bits(addr));
3713 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
3714
3715 return true;
3716 }
3717
3718 /**
3719 * cik_copy_cpdma - copy pages using the CP DMA engine
3720 *
3721 * @rdev: radeon_device pointer
3722 * @src_offset: src GPU address
3723 * @dst_offset: dst GPU address
3724 * @num_gpu_pages: number of GPU pages to xfer
3725 * @fence: radeon fence object
3726 *
3727 * Copy GPU paging using the CP DMA engine (CIK+).
3728 * Used by the radeon ttm implementation to move pages if
3729 * registered as the asic copy callback.
3730 */
3731 int cik_copy_cpdma(struct radeon_device *rdev,
3732 uint64_t src_offset, uint64_t dst_offset,
3733 unsigned num_gpu_pages,
3734 struct radeon_fence **fence)
3735 {
3736 struct radeon_semaphore *sem = NULL;
3737 int ring_index = rdev->asic->copy.blit_ring_index;
3738 struct radeon_ring *ring = &rdev->ring[ring_index];
3739 u32 size_in_bytes, cur_size_in_bytes, control;
3740 int i, num_loops;
3741 int r = 0;
3742
3743 r = radeon_semaphore_create(rdev, &sem);
3744 if (r) {
3745 DRM_ERROR("radeon: moving bo (%d).\n", r);
3746 return r;
3747 }
3748
3749 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3750 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3751 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3752 if (r) {
3753 DRM_ERROR("radeon: moving bo (%d).\n", r);
3754 radeon_semaphore_free(rdev, &sem, NULL);
3755 return r;
3756 }
3757
3758 radeon_semaphore_sync_to(sem, *fence);
3759 radeon_semaphore_sync_rings(rdev, sem, ring->idx);
3760
3761 for (i = 0; i < num_loops; i++) {
3762 cur_size_in_bytes = size_in_bytes;
3763 if (cur_size_in_bytes > 0x1fffff)
3764 cur_size_in_bytes = 0x1fffff;
3765 size_in_bytes -= cur_size_in_bytes;
3766 control = 0;
3767 if (size_in_bytes == 0)
3768 control |= PACKET3_DMA_DATA_CP_SYNC;
3769 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
3770 radeon_ring_write(ring, control);
3771 radeon_ring_write(ring, lower_32_bits(src_offset));
3772 radeon_ring_write(ring, upper_32_bits(src_offset));
3773 radeon_ring_write(ring, lower_32_bits(dst_offset));
3774 radeon_ring_write(ring, upper_32_bits(dst_offset));
3775 radeon_ring_write(ring, cur_size_in_bytes);
3776 src_offset += cur_size_in_bytes;
3777 dst_offset += cur_size_in_bytes;
3778 }
3779
3780 r = radeon_fence_emit(rdev, fence, ring->idx);
3781 if (r) {
3782 radeon_ring_unlock_undo(rdev, ring);
3783 radeon_semaphore_free(rdev, &sem, NULL);
3784 return r;
3785 }
3786
3787 radeon_ring_unlock_commit(rdev, ring);
3788 radeon_semaphore_free(rdev, &sem, *fence);
3789
3790 return r;
3791 }
3792
3793 /*
3794 * IB stuff
3795 */
3796 /**
3797 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3798 *
3799 * @rdev: radeon_device pointer
3800 * @ib: radeon indirect buffer object
3801 *
3802 * Emits an DE (drawing engine) or CE (constant engine) IB
3803 * on the gfx ring. IBs are usually generated by userspace
3804 * acceleration drivers and submitted to the kernel for
3805 * sheduling on the ring. This function schedules the IB
3806 * on the gfx ring for execution by the GPU.
3807 */
3808 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3809 {
3810 struct radeon_ring *ring = &rdev->ring[ib->ring];
3811 u32 header, control = INDIRECT_BUFFER_VALID;
3812
3813 if (ib->is_const_ib) {
3814 /* set switch buffer packet before const IB */
3815 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3816 radeon_ring_write(ring, 0);
3817
3818 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3819 } else {
3820 u32 next_rptr;
3821 if (ring->rptr_save_reg) {
3822 next_rptr = ring->wptr + 3 + 4;
3823 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3824 radeon_ring_write(ring, ((ring->rptr_save_reg -
3825 PACKET3_SET_UCONFIG_REG_START) >> 2));
3826 radeon_ring_write(ring, next_rptr);
3827 } else if (rdev->wb.enabled) {
3828 next_rptr = ring->wptr + 5 + 4;
3829 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3830 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
3831 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3832 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
3833 radeon_ring_write(ring, next_rptr);
3834 }
3835
3836 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3837 }
3838
3839 control |= ib->length_dw |
3840 (ib->vm ? (ib->vm->id << 24) : 0);
3841
3842 radeon_ring_write(ring, header);
3843 radeon_ring_write(ring,
3844 #ifdef __BIG_ENDIAN
3845 (2 << 0) |
3846 #endif
3847 (ib->gpu_addr & 0xFFFFFFFC));
3848 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3849 radeon_ring_write(ring, control);
3850 }
3851
3852 /**
3853 * cik_ib_test - basic gfx ring IB test
3854 *
3855 * @rdev: radeon_device pointer
3856 * @ring: radeon_ring structure holding ring information
3857 *
3858 * Allocate an IB and execute it on the gfx ring (CIK).
3859 * Provides a basic gfx ring test to verify that IBs are working.
3860 * Returns 0 on success, error on failure.
3861 */
3862 int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3863 {
3864 struct radeon_ib ib;
3865 uint32_t scratch;
3866 uint32_t tmp = 0;
3867 unsigned i;
3868 int r;
3869
3870 r = radeon_scratch_get(rdev, &scratch);
3871 if (r) {
3872 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3873 return r;
3874 }
3875 WREG32(scratch, 0xCAFEDEAD);
3876 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3877 if (r) {
3878 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
3879 radeon_scratch_free(rdev, scratch);
3880 return r;
3881 }
3882 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
3883 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
3884 ib.ptr[2] = 0xDEADBEEF;
3885 ib.length_dw = 3;
3886 r = radeon_ib_schedule(rdev, &ib, NULL);
3887 if (r) {
3888 radeon_scratch_free(rdev, scratch);
3889 radeon_ib_free(rdev, &ib);
3890 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3891 return r;
3892 }
3893 r = radeon_fence_wait(ib.fence, false);
3894 if (r) {
3895 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3896 radeon_scratch_free(rdev, scratch);
3897 radeon_ib_free(rdev, &ib);
3898 return r;
3899 }
3900 for (i = 0; i < rdev->usec_timeout; i++) {
3901 tmp = RREG32(scratch);
3902 if (tmp == 0xDEADBEEF)
3903 break;
3904 DRM_UDELAY(1);
3905 }
3906 if (i < rdev->usec_timeout) {
3907 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3908 } else {
3909 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3910 scratch, tmp);
3911 r = -EINVAL;
3912 }
3913 radeon_scratch_free(rdev, scratch);
3914 radeon_ib_free(rdev, &ib);
3915 return r;
3916 }
3917
3918 /*
3919 * CP.
3920 * On CIK, gfx and compute now have independant command processors.
3921 *
3922 * GFX
3923 * Gfx consists of a single ring and can process both gfx jobs and
3924 * compute jobs. The gfx CP consists of three microengines (ME):
3925 * PFP - Pre-Fetch Parser
3926 * ME - Micro Engine
3927 * CE - Constant Engine
3928 * The PFP and ME make up what is considered the Drawing Engine (DE).
3929 * The CE is an asynchronous engine used for updating buffer desciptors
3930 * used by the DE so that they can be loaded into cache in parallel
3931 * while the DE is processing state update packets.
3932 *
3933 * Compute
3934 * The compute CP consists of two microengines (ME):
3935 * MEC1 - Compute MicroEngine 1
3936 * MEC2 - Compute MicroEngine 2
3937 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3938 * The queues are exposed to userspace and are programmed directly
3939 * by the compute runtime.
3940 */
3941 /**
3942 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3943 *
3944 * @rdev: radeon_device pointer
3945 * @enable: enable or disable the MEs
3946 *
3947 * Halts or unhalts the gfx MEs.
3948 */
3949 static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
3950 {
3951 if (enable)
3952 WREG32(CP_ME_CNTL, 0);
3953 else {
3954 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3955 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3956 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3957 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3958 }
3959 udelay(50);
3960 }
3961
3962 /**
3963 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3964 *
3965 * @rdev: radeon_device pointer
3966 *
3967 * Loads the gfx PFP, ME, and CE ucode.
3968 * Returns 0 for success, -EINVAL if the ucode is not available.
3969 */
3970 static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
3971 {
3972 const __be32 *fw_data;
3973 int i;
3974
3975 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3976 return -EINVAL;
3977
3978 cik_cp_gfx_enable(rdev, false);
3979
3980 /* PFP */
3981 fw_data = (const __be32 *)rdev->pfp_fw->data;
3982 WREG32(CP_PFP_UCODE_ADDR, 0);
3983 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
3984 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3985 WREG32(CP_PFP_UCODE_ADDR, 0);
3986
3987 /* CE */
3988 fw_data = (const __be32 *)rdev->ce_fw->data;
3989 WREG32(CP_CE_UCODE_ADDR, 0);
3990 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
3991 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3992 WREG32(CP_CE_UCODE_ADDR, 0);
3993
3994 /* ME */
3995 fw_data = (const __be32 *)rdev->me_fw->data;
3996 WREG32(CP_ME_RAM_WADDR, 0);
3997 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
3998 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3999 WREG32(CP_ME_RAM_WADDR, 0);
4000
4001 WREG32(CP_PFP_UCODE_ADDR, 0);
4002 WREG32(CP_CE_UCODE_ADDR, 0);
4003 WREG32(CP_ME_RAM_WADDR, 0);
4004 WREG32(CP_ME_RAM_RADDR, 0);
4005 return 0;
4006 }
4007
4008 /**
4009 * cik_cp_gfx_start - start the gfx ring
4010 *
4011 * @rdev: radeon_device pointer
4012 *
4013 * Enables the ring and loads the clear state context and other
4014 * packets required to init the ring.
4015 * Returns 0 for success, error for failure.
4016 */
4017 static int cik_cp_gfx_start(struct radeon_device *rdev)
4018 {
4019 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4020 int r, i;
4021
4022 /* init the CP */
4023 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
4024 WREG32(CP_ENDIAN_SWAP, 0);
4025 WREG32(CP_DEVICE_ID, 1);
4026
4027 cik_cp_gfx_enable(rdev, true);
4028
4029 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
4030 if (r) {
4031 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
4032 return r;
4033 }
4034
4035 /* init the CE partitions. CE only used for gfx on CIK */
4036 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
4037 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
4038 radeon_ring_write(ring, 0xc000);
4039 radeon_ring_write(ring, 0xc000);
4040
4041 /* setup clear context state */
4042 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4043 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
4044
4045 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
4046 radeon_ring_write(ring, 0x80000000);
4047 radeon_ring_write(ring, 0x80000000);
4048
4049 for (i = 0; i < cik_default_size; i++)
4050 radeon_ring_write(ring, cik_default_state[i]);
4051
4052 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4053 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
4054
4055 /* set clear context state */
4056 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
4057 radeon_ring_write(ring, 0);
4058
4059 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
4060 radeon_ring_write(ring, 0x00000316);
4061 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
4062 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
4063
4064 radeon_ring_unlock_commit(rdev, ring);
4065
4066 return 0;
4067 }
4068
4069 /**
4070 * cik_cp_gfx_fini - stop the gfx ring
4071 *
4072 * @rdev: radeon_device pointer
4073 *
4074 * Stop the gfx ring and tear down the driver ring
4075 * info.
4076 */
4077 static void cik_cp_gfx_fini(struct radeon_device *rdev)
4078 {
4079 cik_cp_gfx_enable(rdev, false);
4080 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4081 }
4082
4083 /**
4084 * cik_cp_gfx_resume - setup the gfx ring buffer registers
4085 *
4086 * @rdev: radeon_device pointer
4087 *
4088 * Program the location and size of the gfx ring buffer
4089 * and test it to make sure it's working.
4090 * Returns 0 for success, error for failure.
4091 */
4092 static int cik_cp_gfx_resume(struct radeon_device *rdev)
4093 {
4094 struct radeon_ring *ring;
4095 u32 tmp;
4096 u32 rb_bufsz;
4097 u64 rb_addr;
4098 int r;
4099
4100 WREG32(CP_SEM_WAIT_TIMER, 0x0);
4101 if (rdev->family != CHIP_HAWAII)
4102 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
4103
4104 /* Set the write pointer delay */
4105 WREG32(CP_RB_WPTR_DELAY, 0);
4106
4107 /* set the RB to use vmid 0 */
4108 WREG32(CP_RB_VMID, 0);
4109
4110 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
4111
4112 /* ring 0 - compute and gfx */
4113 /* Set ring buffer size */
4114 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4115 rb_bufsz = order_base_2(ring->ring_size / 8);
4116 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
4117 #ifdef __BIG_ENDIAN
4118 tmp |= BUF_SWAP_32BIT;
4119 #endif
4120 WREG32(CP_RB0_CNTL, tmp);
4121
4122 /* Initialize the ring buffer's read and write pointers */
4123 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
4124 ring->wptr = 0;
4125 WREG32(CP_RB0_WPTR, ring->wptr);
4126
4127 /* set the wb address wether it's enabled or not */
4128 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
4129 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
4130
4131 /* scratch register shadowing is no longer supported */
4132 WREG32(SCRATCH_UMSK, 0);
4133
4134 if (!rdev->wb.enabled)
4135 tmp |= RB_NO_UPDATE;
4136
4137 mdelay(1);
4138 WREG32(CP_RB0_CNTL, tmp);
4139
4140 rb_addr = ring->gpu_addr >> 8;
4141 WREG32(CP_RB0_BASE, rb_addr);
4142 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4143
4144 /* start the ring */
4145 cik_cp_gfx_start(rdev);
4146 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4147 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4148 if (r) {
4149 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4150 return r;
4151 }
4152
4153 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4154 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
4155
4156 return 0;
4157 }
4158
4159 u32 cik_gfx_get_rptr(struct radeon_device *rdev,
4160 struct radeon_ring *ring)
4161 {
4162 u32 rptr;
4163
4164 if (rdev->wb.enabled)
4165 rptr = rdev->wb.wb[ring->rptr_offs/4];
4166 else
4167 rptr = RREG32(CP_RB0_RPTR);
4168
4169 return rptr;
4170 }
4171
4172 u32 cik_gfx_get_wptr(struct radeon_device *rdev,
4173 struct radeon_ring *ring)
4174 {
4175 u32 wptr;
4176
4177 wptr = RREG32(CP_RB0_WPTR);
4178
4179 return wptr;
4180 }
4181
4182 void cik_gfx_set_wptr(struct radeon_device *rdev,
4183 struct radeon_ring *ring)
4184 {
4185 WREG32(CP_RB0_WPTR, ring->wptr);
4186 (void)RREG32(CP_RB0_WPTR);
4187 }
4188
4189 u32 cik_compute_get_rptr(struct radeon_device *rdev,
4190 struct radeon_ring *ring)
4191 {
4192 u32 rptr;
4193
4194 if (rdev->wb.enabled) {
4195 rptr = rdev->wb.wb[ring->rptr_offs/4];
4196 } else {
4197 mutex_lock(&rdev->srbm_mutex);
4198 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4199 rptr = RREG32(CP_HQD_PQ_RPTR);
4200 cik_srbm_select(rdev, 0, 0, 0, 0);
4201 mutex_unlock(&rdev->srbm_mutex);
4202 }
4203
4204 return rptr;
4205 }
4206
4207 u32 cik_compute_get_wptr(struct radeon_device *rdev,
4208 struct radeon_ring *ring)
4209 {
4210 u32 wptr;
4211
4212 if (rdev->wb.enabled) {
4213 /* XXX check if swapping is necessary on BE */
4214 wptr = rdev->wb.wb[ring->wptr_offs/4];
4215 } else {
4216 mutex_lock(&rdev->srbm_mutex);
4217 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4218 wptr = RREG32(CP_HQD_PQ_WPTR);
4219 cik_srbm_select(rdev, 0, 0, 0, 0);
4220 mutex_unlock(&rdev->srbm_mutex);
4221 }
4222
4223 return wptr;
4224 }
4225
4226 void cik_compute_set_wptr(struct radeon_device *rdev,
4227 struct radeon_ring *ring)
4228 {
4229 /* XXX check if swapping is necessary on BE */
4230 rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;
4231 WDOORBELL32(ring->doorbell_index, ring->wptr);
4232 }
4233
4234 /**
4235 * cik_cp_compute_enable - enable/disable the compute CP MEs
4236 *
4237 * @rdev: radeon_device pointer
4238 * @enable: enable or disable the MEs
4239 *
4240 * Halts or unhalts the compute MEs.
4241 */
4242 static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4243 {
4244 if (enable)
4245 WREG32(CP_MEC_CNTL, 0);
4246 else {
4247 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
4248 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
4249 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
4250 }
4251 udelay(50);
4252 }
4253
4254 /**
4255 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4256 *
4257 * @rdev: radeon_device pointer
4258 *
4259 * Loads the compute MEC1&2 ucode.
4260 * Returns 0 for success, -EINVAL if the ucode is not available.
4261 */
4262 static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4263 {
4264 const __be32 *fw_data;
4265 int i;
4266
4267 if (!rdev->mec_fw)
4268 return -EINVAL;
4269
4270 cik_cp_compute_enable(rdev, false);
4271
4272 /* MEC1 */
4273 fw_data = (const __be32 *)rdev->mec_fw->data;
4274 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4275 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4276 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4277 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4278
4279 if (rdev->family == CHIP_KAVERI) {
4280 /* MEC2 */
4281 fw_data = (const __be32 *)rdev->mec_fw->data;
4282 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4283 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4284 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4285 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4286 }
4287
4288 return 0;
4289 }
4290
4291 /**
4292 * cik_cp_compute_start - start the compute queues
4293 *
4294 * @rdev: radeon_device pointer
4295 *
4296 * Enable the compute queues.
4297 * Returns 0 for success, error for failure.
4298 */
4299 static int cik_cp_compute_start(struct radeon_device *rdev)
4300 {
4301 cik_cp_compute_enable(rdev, true);
4302
4303 return 0;
4304 }
4305
4306 /**
4307 * cik_cp_compute_fini - stop the compute queues
4308 *
4309 * @rdev: radeon_device pointer
4310 *
4311 * Stop the compute queues and tear down the driver queue
4312 * info.
4313 */
4314 static void cik_cp_compute_fini(struct radeon_device *rdev)
4315 {
4316 int i, idx, r;
4317
4318 cik_cp_compute_enable(rdev, false);
4319
4320 for (i = 0; i < 2; i++) {
4321 if (i == 0)
4322 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4323 else
4324 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4325
4326 if (rdev->ring[idx].mqd_obj) {
4327 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4328 if (unlikely(r != 0))
4329 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4330
4331 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4332 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4333
4334 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4335 rdev->ring[idx].mqd_obj = NULL;
4336 }
4337 }
4338 }
4339
4340 static void cik_mec_fini(struct radeon_device *rdev)
4341 {
4342 int r;
4343
4344 if (rdev->mec.hpd_eop_obj) {
4345 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4346 if (unlikely(r != 0))
4347 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4348 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4349 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4350
4351 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4352 rdev->mec.hpd_eop_obj = NULL;
4353 }
4354 }
4355
4356 #define MEC_HPD_SIZE 2048
4357
4358 static int cik_mec_init(struct radeon_device *rdev)
4359 {
4360 int r;
4361 u32 *hpd;
4362
4363 /*
4364 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4365 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4366 */
4367 if (rdev->family == CHIP_KAVERI)
4368 rdev->mec.num_mec = 2;
4369 else
4370 rdev->mec.num_mec = 1;
4371 rdev->mec.num_pipe = 4;
4372 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4373
4374 if (rdev->mec.hpd_eop_obj == NULL) {
4375 r = radeon_bo_create(rdev,
4376 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4377 PAGE_SIZE, true,
4378 RADEON_GEM_DOMAIN_GTT, NULL,
4379 &rdev->mec.hpd_eop_obj);
4380 if (r) {
4381 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4382 return r;
4383 }
4384 }
4385
4386 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4387 if (unlikely(r != 0)) {
4388 cik_mec_fini(rdev);
4389 return r;
4390 }
4391 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4392 &rdev->mec.hpd_eop_gpu_addr);
4393 if (r) {
4394 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4395 cik_mec_fini(rdev);
4396 return r;
4397 }
4398 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4399 if (r) {
4400 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4401 cik_mec_fini(rdev);
4402 return r;
4403 }
4404
4405 /* clear memory. Not sure if this is required or not */
4406 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4407
4408 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4409 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4410
4411 return 0;
4412 }
4413
4414 struct hqd_registers
4415 {
4416 u32 cp_mqd_base_addr;
4417 u32 cp_mqd_base_addr_hi;
4418 u32 cp_hqd_active;
4419 u32 cp_hqd_vmid;
4420 u32 cp_hqd_persistent_state;
4421 u32 cp_hqd_pipe_priority;
4422 u32 cp_hqd_queue_priority;
4423 u32 cp_hqd_quantum;
4424 u32 cp_hqd_pq_base;
4425 u32 cp_hqd_pq_base_hi;
4426 u32 cp_hqd_pq_rptr;
4427 u32 cp_hqd_pq_rptr_report_addr;
4428 u32 cp_hqd_pq_rptr_report_addr_hi;
4429 u32 cp_hqd_pq_wptr_poll_addr;
4430 u32 cp_hqd_pq_wptr_poll_addr_hi;
4431 u32 cp_hqd_pq_doorbell_control;
4432 u32 cp_hqd_pq_wptr;
4433 u32 cp_hqd_pq_control;
4434 u32 cp_hqd_ib_base_addr;
4435 u32 cp_hqd_ib_base_addr_hi;
4436 u32 cp_hqd_ib_rptr;
4437 u32 cp_hqd_ib_control;
4438 u32 cp_hqd_iq_timer;
4439 u32 cp_hqd_iq_rptr;
4440 u32 cp_hqd_dequeue_request;
4441 u32 cp_hqd_dma_offload;
4442 u32 cp_hqd_sema_cmd;
4443 u32 cp_hqd_msg_type;
4444 u32 cp_hqd_atomic0_preop_lo;
4445 u32 cp_hqd_atomic0_preop_hi;
4446 u32 cp_hqd_atomic1_preop_lo;
4447 u32 cp_hqd_atomic1_preop_hi;
4448 u32 cp_hqd_hq_scheduler0;
4449 u32 cp_hqd_hq_scheduler1;
4450 u32 cp_mqd_control;
4451 };
4452
4453 struct bonaire_mqd
4454 {
4455 u32 header;
4456 u32 dispatch_initiator;
4457 u32 dimensions[3];
4458 u32 start_idx[3];
4459 u32 num_threads[3];
4460 u32 pipeline_stat_enable;
4461 u32 perf_counter_enable;
4462 u32 pgm[2];
4463 u32 tba[2];
4464 u32 tma[2];
4465 u32 pgm_rsrc[2];
4466 u32 vmid;
4467 u32 resource_limits;
4468 u32 static_thread_mgmt01[2];
4469 u32 tmp_ring_size;
4470 u32 static_thread_mgmt23[2];
4471 u32 restart[3];
4472 u32 thread_trace_enable;
4473 u32 reserved1;
4474 u32 user_data[16];
4475 u32 vgtcs_invoke_count[2];
4476 struct hqd_registers queue_state;
4477 u32 dequeue_cntr;
4478 u32 interrupt_queue[64];
4479 };
4480
4481 /**
4482 * cik_cp_compute_resume - setup the compute queue registers
4483 *
4484 * @rdev: radeon_device pointer
4485 *
4486 * Program the compute queues and test them to make sure they
4487 * are working.
4488 * Returns 0 for success, error for failure.
4489 */
4490 static int cik_cp_compute_resume(struct radeon_device *rdev)
4491 {
4492 int r, i, idx;
4493 u32 tmp;
4494 bool use_doorbell = true;
4495 u64 hqd_gpu_addr;
4496 u64 mqd_gpu_addr;
4497 u64 eop_gpu_addr;
4498 u64 wb_gpu_addr;
4499 u32 *buf;
4500 struct bonaire_mqd *mqd;
4501
4502 r = cik_cp_compute_start(rdev);
4503 if (r)
4504 return r;
4505
4506 /* fix up chicken bits */
4507 tmp = RREG32(CP_CPF_DEBUG);
4508 tmp |= (1 << 23);
4509 WREG32(CP_CPF_DEBUG, tmp);
4510
4511 /* init the pipes */
4512 mutex_lock(&rdev->srbm_mutex);
4513 for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
4514 int me = (i < 4) ? 1 : 2;
4515 int pipe = (i < 4) ? i : (i - 4);
4516
4517 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);
4518
4519 cik_srbm_select(rdev, me, pipe, 0, 0);
4520
4521 /* write the EOP addr */
4522 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4523 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4524
4525 /* set the VMID assigned */
4526 WREG32(CP_HPD_EOP_VMID, 0);
4527
4528 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4529 tmp = RREG32(CP_HPD_EOP_CONTROL);
4530 tmp &= ~EOP_SIZE_MASK;
4531 tmp |= order_base_2(MEC_HPD_SIZE / 8);
4532 WREG32(CP_HPD_EOP_CONTROL, tmp);
4533 }
4534 cik_srbm_select(rdev, 0, 0, 0, 0);
4535 mutex_unlock(&rdev->srbm_mutex);
4536
4537 /* init the queues. Just two for now. */
4538 for (i = 0; i < 2; i++) {
4539 if (i == 0)
4540 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4541 else
4542 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4543
4544 if (rdev->ring[idx].mqd_obj == NULL) {
4545 r = radeon_bo_create(rdev,
4546 sizeof(struct bonaire_mqd),
4547 PAGE_SIZE, true,
4548 RADEON_GEM_DOMAIN_GTT, NULL,
4549 &rdev->ring[idx].mqd_obj);
4550 if (r) {
4551 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4552 return r;
4553 }
4554 }
4555
4556 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4557 if (unlikely(r != 0)) {
4558 cik_cp_compute_fini(rdev);
4559 return r;
4560 }
4561 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4562 &mqd_gpu_addr);
4563 if (r) {
4564 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4565 cik_cp_compute_fini(rdev);
4566 return r;
4567 }
4568 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4569 if (r) {
4570 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4571 cik_cp_compute_fini(rdev);
4572 return r;
4573 }
4574
4575 /* init the mqd struct */
4576 memset(buf, 0, sizeof(struct bonaire_mqd));
4577
4578 mqd = (struct bonaire_mqd *)buf;
4579 mqd->header = 0xC0310800;
4580 mqd->static_thread_mgmt01[0] = 0xffffffff;
4581 mqd->static_thread_mgmt01[1] = 0xffffffff;
4582 mqd->static_thread_mgmt23[0] = 0xffffffff;
4583 mqd->static_thread_mgmt23[1] = 0xffffffff;
4584
4585 mutex_lock(&rdev->srbm_mutex);
4586 cik_srbm_select(rdev, rdev->ring[idx].me,
4587 rdev->ring[idx].pipe,
4588 rdev->ring[idx].queue, 0);
4589
4590 /* disable wptr polling */
4591 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4592 tmp &= ~WPTR_POLL_EN;
4593 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4594
4595 /* enable doorbell? */
4596 mqd->queue_state.cp_hqd_pq_doorbell_control =
4597 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4598 if (use_doorbell)
4599 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4600 else
4601 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4602 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4603 mqd->queue_state.cp_hqd_pq_doorbell_control);
4604
4605 /* disable the queue if it's active */
4606 mqd->queue_state.cp_hqd_dequeue_request = 0;
4607 mqd->queue_state.cp_hqd_pq_rptr = 0;
4608 mqd->queue_state.cp_hqd_pq_wptr= 0;
4609 if (RREG32(CP_HQD_ACTIVE) & 1) {
4610 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4611 for (i = 0; i < rdev->usec_timeout; i++) {
4612 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4613 break;
4614 udelay(1);
4615 }
4616 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4617 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4618 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4619 }
4620
4621 /* set the pointer to the MQD */
4622 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4623 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4624 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4625 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4626 /* set MQD vmid to 0 */
4627 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4628 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4629 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4630
4631 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4632 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4633 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4634 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4635 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4636 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4637
4638 /* set up the HQD, this is similar to CP_RB0_CNTL */
4639 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4640 mqd->queue_state.cp_hqd_pq_control &=
4641 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4642
4643 mqd->queue_state.cp_hqd_pq_control |=
4644 order_base_2(rdev->ring[idx].ring_size / 8);
4645 mqd->queue_state.cp_hqd_pq_control |=
4646 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
4647 #ifdef __BIG_ENDIAN
4648 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4649 #endif
4650 mqd->queue_state.cp_hqd_pq_control &=
4651 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4652 mqd->queue_state.cp_hqd_pq_control |=
4653 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4654 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4655
4656 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4657 if (i == 0)
4658 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4659 else
4660 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4661 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4662 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4663 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4664 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4665 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4666
4667 /* set the wb address wether it's enabled or not */
4668 if (i == 0)
4669 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4670 else
4671 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4672 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4673 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4674 upper_32_bits(wb_gpu_addr) & 0xffff;
4675 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4676 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4677 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4678 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4679
4680 /* enable the doorbell if requested */
4681 if (use_doorbell) {
4682 mqd->queue_state.cp_hqd_pq_doorbell_control =
4683 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4684 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
4685 mqd->queue_state.cp_hqd_pq_doorbell_control |=
4686 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
4687 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4688 mqd->queue_state.cp_hqd_pq_doorbell_control &=
4689 ~(DOORBELL_SOURCE | DOORBELL_HIT);
4690
4691 } else {
4692 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
4693 }
4694 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4695 mqd->queue_state.cp_hqd_pq_doorbell_control);
4696
4697 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4698 rdev->ring[idx].wptr = 0;
4699 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
4700 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4701 mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR);
4702
4703 /* set the vmid for the queue */
4704 mqd->queue_state.cp_hqd_vmid = 0;
4705 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
4706
4707 /* activate the queue */
4708 mqd->queue_state.cp_hqd_active = 1;
4709 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
4710
4711 cik_srbm_select(rdev, 0, 0, 0, 0);
4712 mutex_unlock(&rdev->srbm_mutex);
4713
4714 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
4715 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4716
4717 rdev->ring[idx].ready = true;
4718 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
4719 if (r)
4720 rdev->ring[idx].ready = false;
4721 }
4722
4723 return 0;
4724 }
4725
4726 static void cik_cp_enable(struct radeon_device *rdev, bool enable)
4727 {
4728 cik_cp_gfx_enable(rdev, enable);
4729 cik_cp_compute_enable(rdev, enable);
4730 }
4731
4732 static int cik_cp_load_microcode(struct radeon_device *rdev)
4733 {
4734 int r;
4735
4736 r = cik_cp_gfx_load_microcode(rdev);
4737 if (r)
4738 return r;
4739 r = cik_cp_compute_load_microcode(rdev);
4740 if (r)
4741 return r;
4742
4743 return 0;
4744 }
4745
4746 static void cik_cp_fini(struct radeon_device *rdev)
4747 {
4748 cik_cp_gfx_fini(rdev);
4749 cik_cp_compute_fini(rdev);
4750 }
4751
4752 static int cik_cp_resume(struct radeon_device *rdev)
4753 {
4754 int r;
4755
4756 cik_enable_gui_idle_interrupt(rdev, false);
4757
4758 r = cik_cp_load_microcode(rdev);
4759 if (r)
4760 return r;
4761
4762 r = cik_cp_gfx_resume(rdev);
4763 if (r)
4764 return r;
4765 r = cik_cp_compute_resume(rdev);
4766 if (r)
4767 return r;
4768
4769 cik_enable_gui_idle_interrupt(rdev, true);
4770
4771 return 0;
4772 }
4773
4774 static void cik_print_gpu_status_regs(struct radeon_device *rdev)
4775 {
4776 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
4777 RREG32(GRBM_STATUS));
4778 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
4779 RREG32(GRBM_STATUS2));
4780 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
4781 RREG32(GRBM_STATUS_SE0));
4782 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
4783 RREG32(GRBM_STATUS_SE1));
4784 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
4785 RREG32(GRBM_STATUS_SE2));
4786 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
4787 RREG32(GRBM_STATUS_SE3));
4788 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
4789 RREG32(SRBM_STATUS));
4790 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
4791 RREG32(SRBM_STATUS2));
4792 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
4793 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
4794 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
4795 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
4796 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
4797 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
4798 RREG32(CP_STALLED_STAT1));
4799 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
4800 RREG32(CP_STALLED_STAT2));
4801 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
4802 RREG32(CP_STALLED_STAT3));
4803 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
4804 RREG32(CP_CPF_BUSY_STAT));
4805 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
4806 RREG32(CP_CPF_STALLED_STAT1));
4807 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
4808 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
4809 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
4810 RREG32(CP_CPC_STALLED_STAT1));
4811 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
4812 }
4813
4814 /**
4815 * cik_gpu_check_soft_reset - check which blocks are busy
4816 *
4817 * @rdev: radeon_device pointer
4818 *
4819 * Check which blocks are busy and return the relevant reset
4820 * mask to be used by cik_gpu_soft_reset().
4821 * Returns a mask of the blocks to be reset.
4822 */
4823 u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
4824 {
4825 u32 reset_mask = 0;
4826 u32 tmp;
4827
4828 /* GRBM_STATUS */
4829 tmp = RREG32(GRBM_STATUS);
4830 if (tmp & (PA_BUSY | SC_BUSY |
4831 BCI_BUSY | SX_BUSY |
4832 TA_BUSY | VGT_BUSY |
4833 DB_BUSY | CB_BUSY |
4834 GDS_BUSY | SPI_BUSY |
4835 IA_BUSY | IA_BUSY_NO_DMA))
4836 reset_mask |= RADEON_RESET_GFX;
4837
4838 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
4839 reset_mask |= RADEON_RESET_CP;
4840
4841 /* GRBM_STATUS2 */
4842 tmp = RREG32(GRBM_STATUS2);
4843 if (tmp & RLC_BUSY)
4844 reset_mask |= RADEON_RESET_RLC;
4845
4846 /* SDMA0_STATUS_REG */
4847 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
4848 if (!(tmp & SDMA_IDLE))
4849 reset_mask |= RADEON_RESET_DMA;
4850
4851 /* SDMA1_STATUS_REG */
4852 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
4853 if (!(tmp & SDMA_IDLE))
4854 reset_mask |= RADEON_RESET_DMA1;
4855
4856 /* SRBM_STATUS2 */
4857 tmp = RREG32(SRBM_STATUS2);
4858 if (tmp & SDMA_BUSY)
4859 reset_mask |= RADEON_RESET_DMA;
4860
4861 if (tmp & SDMA1_BUSY)
4862 reset_mask |= RADEON_RESET_DMA1;
4863
4864 /* SRBM_STATUS */
4865 tmp = RREG32(SRBM_STATUS);
4866
4867 if (tmp & IH_BUSY)
4868 reset_mask |= RADEON_RESET_IH;
4869
4870 if (tmp & SEM_BUSY)
4871 reset_mask |= RADEON_RESET_SEM;
4872
4873 if (tmp & GRBM_RQ_PENDING)
4874 reset_mask |= RADEON_RESET_GRBM;
4875
4876 if (tmp & VMC_BUSY)
4877 reset_mask |= RADEON_RESET_VMC;
4878
4879 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
4880 MCC_BUSY | MCD_BUSY))
4881 reset_mask |= RADEON_RESET_MC;
4882
4883 if (evergreen_is_display_hung(rdev))
4884 reset_mask |= RADEON_RESET_DISPLAY;
4885
4886 /* Skip MC reset as it's mostly likely not hung, just busy */
4887 if (reset_mask & RADEON_RESET_MC) {
4888 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
4889 reset_mask &= ~RADEON_RESET_MC;
4890 }
4891
4892 return reset_mask;
4893 }
4894
4895 /**
4896 * cik_gpu_soft_reset - soft reset GPU
4897 *
4898 * @rdev: radeon_device pointer
4899 * @reset_mask: mask of which blocks to reset
4900 *
4901 * Soft reset the blocks specified in @reset_mask.
4902 */
4903 static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
4904 {
4905 struct evergreen_mc_save save;
4906 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4907 u32 tmp;
4908
4909 if (reset_mask == 0)
4910 return;
4911
4912 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
4913
4914 cik_print_gpu_status_regs(rdev);
4915 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4916 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4917 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4918 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
4919
4920 /* disable CG/PG */
4921 cik_fini_pg(rdev);
4922 cik_fini_cg(rdev);
4923
4924 /* stop the rlc */
4925 cik_rlc_stop(rdev);
4926
4927 /* Disable GFX parsing/prefetching */
4928 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4929
4930 /* Disable MEC parsing/prefetching */
4931 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
4932
4933 if (reset_mask & RADEON_RESET_DMA) {
4934 /* sdma0 */
4935 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
4936 tmp |= SDMA_HALT;
4937 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
4938 }
4939 if (reset_mask & RADEON_RESET_DMA1) {
4940 /* sdma1 */
4941 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
4942 tmp |= SDMA_HALT;
4943 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
4944 }
4945
4946 evergreen_mc_stop(rdev, &save);
4947 if (evergreen_mc_wait_for_idle(rdev)) {
4948 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4949 }
4950
4951 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
4952 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
4953
4954 if (reset_mask & RADEON_RESET_CP) {
4955 grbm_soft_reset |= SOFT_RESET_CP;
4956
4957 srbm_soft_reset |= SOFT_RESET_GRBM;
4958 }
4959
4960 if (reset_mask & RADEON_RESET_DMA)
4961 srbm_soft_reset |= SOFT_RESET_SDMA;
4962
4963 if (reset_mask & RADEON_RESET_DMA1)
4964 srbm_soft_reset |= SOFT_RESET_SDMA1;
4965
4966 if (reset_mask & RADEON_RESET_DISPLAY)
4967 srbm_soft_reset |= SOFT_RESET_DC;
4968
4969 if (reset_mask & RADEON_RESET_RLC)
4970 grbm_soft_reset |= SOFT_RESET_RLC;
4971
4972 if (reset_mask & RADEON_RESET_SEM)
4973 srbm_soft_reset |= SOFT_RESET_SEM;
4974
4975 if (reset_mask & RADEON_RESET_IH)
4976 srbm_soft_reset |= SOFT_RESET_IH;
4977
4978 if (reset_mask & RADEON_RESET_GRBM)
4979 srbm_soft_reset |= SOFT_RESET_GRBM;
4980
4981 if (reset_mask & RADEON_RESET_VMC)
4982 srbm_soft_reset |= SOFT_RESET_VMC;
4983
4984 if (!(rdev->flags & RADEON_IS_IGP)) {
4985 if (reset_mask & RADEON_RESET_MC)
4986 srbm_soft_reset |= SOFT_RESET_MC;
4987 }
4988
4989 if (grbm_soft_reset) {
4990 tmp = RREG32(GRBM_SOFT_RESET);
4991 tmp |= grbm_soft_reset;
4992 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4993 WREG32(GRBM_SOFT_RESET, tmp);
4994 tmp = RREG32(GRBM_SOFT_RESET);
4995
4996 udelay(50);
4997
4998 tmp &= ~grbm_soft_reset;
4999 WREG32(GRBM_SOFT_RESET, tmp);
5000 tmp = RREG32(GRBM_SOFT_RESET);
5001 }
5002
5003 if (srbm_soft_reset) {
5004 tmp = RREG32(SRBM_SOFT_RESET);
5005 tmp |= srbm_soft_reset;
5006 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
5007 WREG32(SRBM_SOFT_RESET, tmp);
5008 tmp = RREG32(SRBM_SOFT_RESET);
5009
5010 udelay(50);
5011
5012 tmp &= ~srbm_soft_reset;
5013 WREG32(SRBM_SOFT_RESET, tmp);
5014 tmp = RREG32(SRBM_SOFT_RESET);
5015 }
5016
5017 /* Wait a little for things to settle down */
5018 udelay(50);
5019
5020 evergreen_mc_resume(rdev, &save);
5021 udelay(50);
5022
5023 cik_print_gpu_status_regs(rdev);
5024 }
5025
5026 struct kv_reset_save_regs {
5027 u32 gmcon_reng_execute;
5028 u32 gmcon_misc;
5029 u32 gmcon_misc3;
5030 };
5031
5032 static void kv_save_regs_for_reset(struct radeon_device *rdev,
5033 struct kv_reset_save_regs *save)
5034 {
5035 save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE);
5036 save->gmcon_misc = RREG32(GMCON_MISC);
5037 save->gmcon_misc3 = RREG32(GMCON_MISC3);
5038
5039 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP);
5040 WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE |
5041 STCTRL_STUTTER_EN));
5042 }
5043
5044 static void kv_restore_regs_for_reset(struct radeon_device *rdev,
5045 struct kv_reset_save_regs *save)
5046 {
5047 int i;
5048
5049 WREG32(GMCON_PGFSM_WRITE, 0);
5050 WREG32(GMCON_PGFSM_CONFIG, 0x200010ff);
5051
5052 for (i = 0; i < 5; i++)
5053 WREG32(GMCON_PGFSM_WRITE, 0);
5054
5055 WREG32(GMCON_PGFSM_WRITE, 0);
5056 WREG32(GMCON_PGFSM_CONFIG, 0x300010ff);
5057
5058 for (i = 0; i < 5; i++)
5059 WREG32(GMCON_PGFSM_WRITE, 0);
5060
5061 WREG32(GMCON_PGFSM_WRITE, 0x210000);
5062 WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff);
5063
5064 for (i = 0; i < 5; i++)
5065 WREG32(GMCON_PGFSM_WRITE, 0);
5066
5067 WREG32(GMCON_PGFSM_WRITE, 0x21003);
5068 WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff);
5069
5070 for (i = 0; i < 5; i++)
5071 WREG32(GMCON_PGFSM_WRITE, 0);
5072
5073 WREG32(GMCON_PGFSM_WRITE, 0x2b00);
5074 WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff);
5075
5076 for (i = 0; i < 5; i++)
5077 WREG32(GMCON_PGFSM_WRITE, 0);
5078
5079 WREG32(GMCON_PGFSM_WRITE, 0);
5080 WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff);
5081
5082 for (i = 0; i < 5; i++)
5083 WREG32(GMCON_PGFSM_WRITE, 0);
5084
5085 WREG32(GMCON_PGFSM_WRITE, 0x420000);
5086 WREG32(GMCON_PGFSM_CONFIG, 0x100010ff);
5087
5088 for (i = 0; i < 5; i++)
5089 WREG32(GMCON_PGFSM_WRITE, 0);
5090
5091 WREG32(GMCON_PGFSM_WRITE, 0x120202);
5092 WREG32(GMCON_PGFSM_CONFIG, 0x500010ff);
5093
5094 for (i = 0; i < 5; i++)
5095 WREG32(GMCON_PGFSM_WRITE, 0);
5096
5097 WREG32(GMCON_PGFSM_WRITE, 0x3e3e36);
5098 WREG32(GMCON_PGFSM_CONFIG, 0x600010ff);
5099
5100 for (i = 0; i < 5; i++)
5101 WREG32(GMCON_PGFSM_WRITE, 0);
5102
5103 WREG32(GMCON_PGFSM_WRITE, 0x373f3e);
5104 WREG32(GMCON_PGFSM_CONFIG, 0x700010ff);
5105
5106 for (i = 0; i < 5; i++)
5107 WREG32(GMCON_PGFSM_WRITE, 0);
5108
5109 WREG32(GMCON_PGFSM_WRITE, 0x3e1332);
5110 WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff);
5111
5112 WREG32(GMCON_MISC3, save->gmcon_misc3);
5113 WREG32(GMCON_MISC, save->gmcon_misc);
5114 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute);
5115 }
5116
5117 static void cik_gpu_pci_config_reset(struct radeon_device *rdev)
5118 {
5119 struct evergreen_mc_save save;
5120 struct kv_reset_save_regs kv_save = { 0 };
5121 u32 tmp, i;
5122
5123 dev_info(rdev->dev, "GPU pci config reset\n");
5124
5125 /* disable dpm? */
5126
5127 /* disable cg/pg */
5128 cik_fini_pg(rdev);
5129 cik_fini_cg(rdev);
5130
5131 /* Disable GFX parsing/prefetching */
5132 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5133
5134 /* Disable MEC parsing/prefetching */
5135 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5136
5137 /* sdma0 */
5138 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5139 tmp |= SDMA_HALT;
5140 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5141 /* sdma1 */
5142 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5143 tmp |= SDMA_HALT;
5144 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5145 /* XXX other engines? */
5146
5147 /* halt the rlc, disable cp internal ints */
5148 cik_rlc_stop(rdev);
5149
5150 udelay(50);
5151
5152 /* disable mem access */
5153 evergreen_mc_stop(rdev, &save);
5154 if (evergreen_mc_wait_for_idle(rdev)) {
5155 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
5156 }
5157
5158 if (rdev->flags & RADEON_IS_IGP)
5159 kv_save_regs_for_reset(rdev, &kv_save);
5160
5161 /* disable BM */
5162 pci_clear_master(rdev->pdev);
5163 /* reset */
5164 radeon_pci_config_reset(rdev);
5165
5166 udelay(100);
5167
5168 /* wait for asic to come out of reset */
5169 for (i = 0; i < rdev->usec_timeout; i++) {
5170 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
5171 break;
5172 udelay(1);
5173 }
5174
5175 /* does asic init need to be run first??? */
5176 if (rdev->flags & RADEON_IS_IGP)
5177 kv_restore_regs_for_reset(rdev, &kv_save);
5178 }
5179
5180 /**
5181 * cik_asic_reset - soft reset GPU
5182 *
5183 * @rdev: radeon_device pointer
5184 *
5185 * Look up which blocks are hung and attempt
5186 * to reset them.
5187 * Returns 0 for success.
5188 */
5189 int cik_asic_reset(struct radeon_device *rdev)
5190 {
5191 u32 reset_mask;
5192
5193 reset_mask = cik_gpu_check_soft_reset(rdev);
5194
5195 if (reset_mask)
5196 r600_set_bios_scratch_engine_hung(rdev, true);
5197
5198 /* try soft reset */
5199 cik_gpu_soft_reset(rdev, reset_mask);
5200
5201 reset_mask = cik_gpu_check_soft_reset(rdev);
5202
5203 /* try pci config reset */
5204 if (reset_mask && radeon_hard_reset)
5205 cik_gpu_pci_config_reset(rdev);
5206
5207 reset_mask = cik_gpu_check_soft_reset(rdev);
5208
5209 if (!reset_mask)
5210 r600_set_bios_scratch_engine_hung(rdev, false);
5211
5212 return 0;
5213 }
5214
5215 /**
5216 * cik_gfx_is_lockup - check if the 3D engine is locked up
5217 *
5218 * @rdev: radeon_device pointer
5219 * @ring: radeon_ring structure holding ring information
5220 *
5221 * Check if the 3D engine is locked up (CIK).
5222 * Returns true if the engine is locked, false if not.
5223 */
5224 bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
5225 {
5226 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
5227
5228 if (!(reset_mask & (RADEON_RESET_GFX |
5229 RADEON_RESET_COMPUTE |
5230 RADEON_RESET_CP))) {
5231 radeon_ring_lockup_update(rdev, ring);
5232 return false;
5233 }
5234 return radeon_ring_test_lockup(rdev, ring);
5235 }
5236
5237 /* MC */
5238 /**
5239 * cik_mc_program - program the GPU memory controller
5240 *
5241 * @rdev: radeon_device pointer
5242 *
5243 * Set the location of vram, gart, and AGP in the GPU's
5244 * physical address space (CIK).
5245 */
5246 static void cik_mc_program(struct radeon_device *rdev)
5247 {
5248 struct evergreen_mc_save save;
5249 u32 tmp;
5250 int i, j;
5251
5252 /* Initialize HDP */
5253 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
5254 WREG32((0x2c14 + j), 0x00000000);
5255 WREG32((0x2c18 + j), 0x00000000);
5256 WREG32((0x2c1c + j), 0x00000000);
5257 WREG32((0x2c20 + j), 0x00000000);
5258 WREG32((0x2c24 + j), 0x00000000);
5259 }
5260 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
5261
5262 evergreen_mc_stop(rdev, &save);
5263 if (radeon_mc_wait_for_idle(rdev)) {
5264 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5265 }
5266 /* Lockout access through VGA aperture*/
5267 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
5268 /* Update configuration */
5269 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
5270 rdev->mc.vram_start >> 12);
5271 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
5272 rdev->mc.vram_end >> 12);
5273 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
5274 rdev->vram_scratch.gpu_addr >> 12);
5275 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
5276 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
5277 WREG32(MC_VM_FB_LOCATION, tmp);
5278 /* XXX double check these! */
5279 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
5280 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
5281 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
5282 WREG32(MC_VM_AGP_BASE, 0);
5283 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
5284 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
5285 if (radeon_mc_wait_for_idle(rdev)) {
5286 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5287 }
5288 evergreen_mc_resume(rdev, &save);
5289 /* we need to own VRAM, so turn off the VGA renderer here
5290 * to stop it overwriting our objects */
5291 rv515_vga_render_disable(rdev);
5292 }
5293
5294 /**
5295 * cik_mc_init - initialize the memory controller driver params
5296 *
5297 * @rdev: radeon_device pointer
5298 *
5299 * Look up the amount of vram, vram width, and decide how to place
5300 * vram and gart within the GPU's physical address space (CIK).
5301 * Returns 0 for success.
5302 */
5303 static int cik_mc_init(struct radeon_device *rdev)
5304 {
5305 u32 tmp;
5306 int chansize, numchan;
5307
5308 /* Get VRAM informations */
5309 rdev->mc.vram_is_ddr = true;
5310 tmp = RREG32(MC_ARB_RAMCFG);
5311 if (tmp & CHANSIZE_MASK) {
5312 chansize = 64;
5313 } else {
5314 chansize = 32;
5315 }
5316 tmp = RREG32(MC_SHARED_CHMAP);
5317 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
5318 case 0:
5319 default:
5320 numchan = 1;
5321 break;
5322 case 1:
5323 numchan = 2;
5324 break;
5325 case 2:
5326 numchan = 4;
5327 break;
5328 case 3:
5329 numchan = 8;
5330 break;
5331 case 4:
5332 numchan = 3;
5333 break;
5334 case 5:
5335 numchan = 6;
5336 break;
5337 case 6:
5338 numchan = 10;
5339 break;
5340 case 7:
5341 numchan = 12;
5342 break;
5343 case 8:
5344 numchan = 16;
5345 break;
5346 }
5347 rdev->mc.vram_width = numchan * chansize;
5348 /* Could aper size report 0 ? */
5349 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5350 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5351 /* size in MB on si */
5352 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5353 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5354 rdev->mc.visible_vram_size = rdev->mc.aper_size;
5355 si_vram_gtt_location(rdev, &rdev->mc);
5356 radeon_update_bandwidth_info(rdev);
5357
5358 return 0;
5359 }
5360
5361 /*
5362 * GART
5363 * VMID 0 is the physical GPU addresses as used by the kernel.
5364 * VMIDs 1-15 are used for userspace clients and are handled
5365 * by the radeon vm/hsa code.
5366 */
5367 /**
5368 * cik_pcie_gart_tlb_flush - gart tlb flush callback
5369 *
5370 * @rdev: radeon_device pointer
5371 *
5372 * Flush the TLB for the VMID 0 page table (CIK).
5373 */
5374 void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5375 {
5376 /* flush hdp cache */
5377 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5378
5379 /* bits 0-15 are the VM contexts0-15 */
5380 WREG32(VM_INVALIDATE_REQUEST, 0x1);
5381 }
5382
5383 /**
5384 * cik_pcie_gart_enable - gart enable
5385 *
5386 * @rdev: radeon_device pointer
5387 *
5388 * This sets up the TLBs, programs the page tables for VMID0,
5389 * sets up the hw for VMIDs 1-15 which are allocated on
5390 * demand, and sets up the global locations for the LDS, GDS,
5391 * and GPUVM for FSA64 clients (CIK).
5392 * Returns 0 for success, errors for failure.
5393 */
5394 static int cik_pcie_gart_enable(struct radeon_device *rdev)
5395 {
5396 int r, i;
5397
5398 if (rdev->gart.robj == NULL) {
5399 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5400 return -EINVAL;
5401 }
5402 r = radeon_gart_table_vram_pin(rdev);
5403 if (r)
5404 return r;
5405 radeon_gart_restore(rdev);
5406 /* Setup TLB control */
5407 WREG32(MC_VM_MX_L1_TLB_CNTL,
5408 (0xA << 7) |
5409 ENABLE_L1_TLB |
5410 ENABLE_L1_FRAGMENT_PROCESSING |
5411 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5412 ENABLE_ADVANCED_DRIVER_MODEL |
5413 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5414 /* Setup L2 cache */
5415 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5416 ENABLE_L2_FRAGMENT_PROCESSING |
5417 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5418 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5419 EFFECTIVE_L2_QUEUE_SIZE(7) |
5420 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5421 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5422 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5423 BANK_SELECT(4) |
5424 L2_CACHE_BIGK_FRAGMENT_SIZE(4));
5425 /* setup context0 */
5426 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5427 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5428 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5429 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5430 (u32)(rdev->dummy_page.addr >> 12));
5431 WREG32(VM_CONTEXT0_CNTL2, 0);
5432 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5433 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5434
5435 WREG32(0x15D4, 0);
5436 WREG32(0x15D8, 0);
5437 WREG32(0x15DC, 0);
5438
5439 /* empty context1-15 */
5440 /* FIXME start with 4G, once using 2 level pt switch to full
5441 * vm size space
5442 */
5443 /* set vm size, must be a multiple of 4 */
5444 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5445 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
5446 for (i = 1; i < 16; i++) {
5447 if (i < 8)
5448 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5449 rdev->gart.table_addr >> 12);
5450 else
5451 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5452 rdev->gart.table_addr >> 12);
5453 }
5454
5455 /* enable context1-15 */
5456 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5457 (u32)(rdev->dummy_page.addr >> 12));
5458 WREG32(VM_CONTEXT1_CNTL2, 4);
5459 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
5460 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
5461 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5462 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5463 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5464 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5465 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5466 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5467 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5468 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5469 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5470 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5471 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5472 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
5473
5474 if (rdev->family == CHIP_KAVERI) {
5475 u32 tmp = RREG32(CHUB_CONTROL);
5476 tmp &= ~BYPASS_VM;
5477 WREG32(CHUB_CONTROL, tmp);
5478 }
5479
5480 /* XXX SH_MEM regs */
5481 /* where to put LDS, scratch, GPUVM in FSA64 space */
5482 mutex_lock(&rdev->srbm_mutex);
5483 for (i = 0; i < 16; i++) {
5484 cik_srbm_select(rdev, 0, 0, 0, i);
5485 /* CP and shaders */
5486 WREG32(SH_MEM_CONFIG, 0);
5487 WREG32(SH_MEM_APE1_BASE, 1);
5488 WREG32(SH_MEM_APE1_LIMIT, 0);
5489 WREG32(SH_MEM_BASES, 0);
5490 /* SDMA GFX */
5491 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5492 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5493 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5494 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5495 /* XXX SDMA RLC - todo */
5496 }
5497 cik_srbm_select(rdev, 0, 0, 0, 0);
5498 mutex_unlock(&rdev->srbm_mutex);
5499
5500 cik_pcie_gart_tlb_flush(rdev);
5501 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5502 (unsigned)(rdev->mc.gtt_size >> 20),
5503 (unsigned long long)rdev->gart.table_addr);
5504 rdev->gart.ready = true;
5505 return 0;
5506 }
5507
5508 /**
5509 * cik_pcie_gart_disable - gart disable
5510 *
5511 * @rdev: radeon_device pointer
5512 *
5513 * This disables all VM page table (CIK).
5514 */
5515 static void cik_pcie_gart_disable(struct radeon_device *rdev)
5516 {
5517 /* Disable all tables */
5518 WREG32(VM_CONTEXT0_CNTL, 0);
5519 WREG32(VM_CONTEXT1_CNTL, 0);
5520 /* Setup TLB control */
5521 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5522 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5523 /* Setup L2 cache */
5524 WREG32(VM_L2_CNTL,
5525 ENABLE_L2_FRAGMENT_PROCESSING |
5526 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5527 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5528 EFFECTIVE_L2_QUEUE_SIZE(7) |
5529 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5530 WREG32(VM_L2_CNTL2, 0);
5531 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5532 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5533 radeon_gart_table_vram_unpin(rdev);
5534 }
5535
5536 /**
5537 * cik_pcie_gart_fini - vm fini callback
5538 *
5539 * @rdev: radeon_device pointer
5540 *
5541 * Tears down the driver GART/VM setup (CIK).
5542 */
5543 static void cik_pcie_gart_fini(struct radeon_device *rdev)
5544 {
5545 cik_pcie_gart_disable(rdev);
5546 radeon_gart_table_vram_free(rdev);
5547 radeon_gart_fini(rdev);
5548 }
5549
5550 /* vm parser */
5551 /**
5552 * cik_ib_parse - vm ib_parse callback
5553 *
5554 * @rdev: radeon_device pointer
5555 * @ib: indirect buffer pointer
5556 *
5557 * CIK uses hw IB checking so this is a nop (CIK).
5558 */
5559 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5560 {
5561 return 0;
5562 }
5563
5564 /*
5565 * vm
5566 * VMID 0 is the physical GPU addresses as used by the kernel.
5567 * VMIDs 1-15 are used for userspace clients and are handled
5568 * by the radeon vm/hsa code.
5569 */
5570 /**
5571 * cik_vm_init - cik vm init callback
5572 *
5573 * @rdev: radeon_device pointer
5574 *
5575 * Inits cik specific vm parameters (number of VMs, base of vram for
5576 * VMIDs 1-15) (CIK).
5577 * Returns 0 for success.
5578 */
5579 int cik_vm_init(struct radeon_device *rdev)
5580 {
5581 /* number of VMs */
5582 rdev->vm_manager.nvm = 16;
5583 /* base offset of vram pages */
5584 if (rdev->flags & RADEON_IS_IGP) {
5585 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5586 tmp <<= 22;
5587 rdev->vm_manager.vram_base_offset = tmp;
5588 } else
5589 rdev->vm_manager.vram_base_offset = 0;
5590
5591 return 0;
5592 }
5593
5594 /**
5595 * cik_vm_fini - cik vm fini callback
5596 *
5597 * @rdev: radeon_device pointer
5598 *
5599 * Tear down any asic specific VM setup (CIK).
5600 */
5601 void cik_vm_fini(struct radeon_device *rdev)
5602 {
5603 }
5604
5605 /**
5606 * cik_vm_decode_fault - print human readable fault info
5607 *
5608 * @rdev: radeon_device pointer
5609 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5610 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5611 *
5612 * Print human readable fault information (CIK).
5613 */
5614 static void cik_vm_decode_fault(struct radeon_device *rdev,
5615 u32 status, u32 addr, u32 mc_client)
5616 {
5617 u32 mc_id;
5618 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5619 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
5620 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5621 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
5622
5623 if (rdev->family == CHIP_HAWAII)
5624 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5625 else
5626 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5627
5628 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
5629 protections, vmid, addr,
5630 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
5631 block, mc_client, mc_id);
5632 }
5633
5634 /**
5635 * cik_vm_flush - cik vm flush using the CP
5636 *
5637 * @rdev: radeon_device pointer
5638 *
5639 * Update the page table base and flush the VM TLB
5640 * using the CP (CIK).
5641 */
5642 void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
5643 {
5644 struct radeon_ring *ring = &rdev->ring[ridx];
5645
5646 if (vm == NULL)
5647 return;
5648
5649 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5650 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5651 WRITE_DATA_DST_SEL(0)));
5652 if (vm->id < 8) {
5653 radeon_ring_write(ring,
5654 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
5655 } else {
5656 radeon_ring_write(ring,
5657 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
5658 }
5659 radeon_ring_write(ring, 0);
5660 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
5661
5662 /* update SH_MEM_* regs */
5663 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5664 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5665 WRITE_DATA_DST_SEL(0)));
5666 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5667 radeon_ring_write(ring, 0);
5668 radeon_ring_write(ring, VMID(vm->id));
5669
5670 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
5671 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5672 WRITE_DATA_DST_SEL(0)));
5673 radeon_ring_write(ring, SH_MEM_BASES >> 2);
5674 radeon_ring_write(ring, 0);
5675
5676 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
5677 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
5678 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
5679 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
5680
5681 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5682 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5683 WRITE_DATA_DST_SEL(0)));
5684 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5685 radeon_ring_write(ring, 0);
5686 radeon_ring_write(ring, VMID(0));
5687
5688 /* HDP flush */
5689 cik_hdp_flush_cp_ring_emit(rdev, ridx);
5690
5691 /* bits 0-15 are the VM contexts0-15 */
5692 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5693 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5694 WRITE_DATA_DST_SEL(0)));
5695 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5696 radeon_ring_write(ring, 0);
5697 radeon_ring_write(ring, 1 << vm->id);
5698
5699 /* compute doesn't have PFP */
5700 if (ridx == RADEON_RING_TYPE_GFX_INDEX) {
5701 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5702 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5703 radeon_ring_write(ring, 0x0);
5704 }
5705 }
5706
5707 /*
5708 * RLC
5709 * The RLC is a multi-purpose microengine that handles a
5710 * variety of functions, the most important of which is
5711 * the interrupt controller.
5712 */
5713 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
5714 bool enable)
5715 {
5716 u32 tmp = RREG32(CP_INT_CNTL_RING0);
5717
5718 if (enable)
5719 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5720 else
5721 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5722 WREG32(CP_INT_CNTL_RING0, tmp);
5723 }
5724
5725 static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
5726 {
5727 u32 tmp;
5728
5729 tmp = RREG32(RLC_LB_CNTL);
5730 if (enable)
5731 tmp |= LOAD_BALANCE_ENABLE;
5732 else
5733 tmp &= ~LOAD_BALANCE_ENABLE;
5734 WREG32(RLC_LB_CNTL, tmp);
5735 }
5736
5737 static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
5738 {
5739 u32 i, j, k;
5740 u32 mask;
5741
5742 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
5743 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
5744 cik_select_se_sh(rdev, i, j);
5745 for (k = 0; k < rdev->usec_timeout; k++) {
5746 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
5747 break;
5748 udelay(1);
5749 }
5750 }
5751 }
5752 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5753
5754 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
5755 for (k = 0; k < rdev->usec_timeout; k++) {
5756 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
5757 break;
5758 udelay(1);
5759 }
5760 }
5761
5762 static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
5763 {
5764 u32 tmp;
5765
5766 tmp = RREG32(RLC_CNTL);
5767 if (tmp != rlc)
5768 WREG32(RLC_CNTL, rlc);
5769 }
5770
5771 static u32 cik_halt_rlc(struct radeon_device *rdev)
5772 {
5773 u32 data, orig;
5774
5775 orig = data = RREG32(RLC_CNTL);
5776
5777 if (data & RLC_ENABLE) {
5778 u32 i;
5779
5780 data &= ~RLC_ENABLE;
5781 WREG32(RLC_CNTL, data);
5782
5783 for (i = 0; i < rdev->usec_timeout; i++) {
5784 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
5785 break;
5786 udelay(1);
5787 }
5788
5789 cik_wait_for_rlc_serdes(rdev);
5790 }
5791
5792 return orig;
5793 }
5794
5795 void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
5796 {
5797 u32 tmp, i, mask;
5798
5799 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
5800 WREG32(RLC_GPR_REG2, tmp);
5801
5802 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
5803 for (i = 0; i < rdev->usec_timeout; i++) {
5804 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
5805 break;
5806 udelay(1);
5807 }
5808
5809 for (i = 0; i < rdev->usec_timeout; i++) {
5810 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
5811 break;
5812 udelay(1);
5813 }
5814 }
5815
5816 void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
5817 {
5818 u32 tmp;
5819
5820 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
5821 WREG32(RLC_GPR_REG2, tmp);
5822 }
5823
5824 /**
5825 * cik_rlc_stop - stop the RLC ME
5826 *
5827 * @rdev: radeon_device pointer
5828 *
5829 * Halt the RLC ME (MicroEngine) (CIK).
5830 */
5831 static void cik_rlc_stop(struct radeon_device *rdev)
5832 {
5833 WREG32(RLC_CNTL, 0);
5834
5835 cik_enable_gui_idle_interrupt(rdev, false);
5836
5837 cik_wait_for_rlc_serdes(rdev);
5838 }
5839
5840 /**
5841 * cik_rlc_start - start the RLC ME
5842 *
5843 * @rdev: radeon_device pointer
5844 *
5845 * Unhalt the RLC ME (MicroEngine) (CIK).
5846 */
5847 static void cik_rlc_start(struct radeon_device *rdev)
5848 {
5849 WREG32(RLC_CNTL, RLC_ENABLE);
5850
5851 cik_enable_gui_idle_interrupt(rdev, true);
5852
5853 udelay(50);
5854 }
5855
5856 /**
5857 * cik_rlc_resume - setup the RLC hw
5858 *
5859 * @rdev: radeon_device pointer
5860 *
5861 * Initialize the RLC registers, load the ucode,
5862 * and start the RLC (CIK).
5863 * Returns 0 for success, -EINVAL if the ucode is not available.
5864 */
5865 static int cik_rlc_resume(struct radeon_device *rdev)
5866 {
5867 u32 i, size, tmp;
5868 const __be32 *fw_data;
5869
5870 if (!rdev->rlc_fw)
5871 return -EINVAL;
5872
5873 switch (rdev->family) {
5874 case CHIP_BONAIRE:
5875 case CHIP_HAWAII:
5876 default:
5877 size = BONAIRE_RLC_UCODE_SIZE;
5878 break;
5879 case CHIP_KAVERI:
5880 size = KV_RLC_UCODE_SIZE;
5881 break;
5882 case CHIP_KABINI:
5883 size = KB_RLC_UCODE_SIZE;
5884 break;
5885 case CHIP_MULLINS:
5886 size = ML_RLC_UCODE_SIZE;
5887 break;
5888 }
5889
5890 cik_rlc_stop(rdev);
5891
5892 /* disable CG */
5893 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
5894 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
5895
5896 si_rlc_reset(rdev);
5897
5898 cik_init_pg(rdev);
5899
5900 cik_init_cg(rdev);
5901
5902 WREG32(RLC_LB_CNTR_INIT, 0);
5903 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
5904
5905 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5906 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5907 WREG32(RLC_LB_PARAMS, 0x00600408);
5908 WREG32(RLC_LB_CNTL, 0x80000004);
5909
5910 WREG32(RLC_MC_CNTL, 0);
5911 WREG32(RLC_UCODE_CNTL, 0);
5912
5913 fw_data = (const __be32 *)rdev->rlc_fw->data;
5914 WREG32(RLC_GPM_UCODE_ADDR, 0);
5915 for (i = 0; i < size; i++)
5916 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
5917 WREG32(RLC_GPM_UCODE_ADDR, 0);
5918
5919 /* XXX - find out what chips support lbpw */
5920 cik_enable_lbpw(rdev, false);
5921
5922 if (rdev->family == CHIP_BONAIRE)
5923 WREG32(RLC_DRIVER_DMA_STATUS, 0);
5924
5925 cik_rlc_start(rdev);
5926
5927 return 0;
5928 }
5929
5930 static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
5931 {
5932 u32 data, orig, tmp, tmp2;
5933
5934 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5935
5936 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5937 cik_enable_gui_idle_interrupt(rdev, true);
5938
5939 tmp = cik_halt_rlc(rdev);
5940
5941 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5942 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5943 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5944 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
5945 WREG32(RLC_SERDES_WR_CTRL, tmp2);
5946
5947 cik_update_rlc(rdev, tmp);
5948
5949 data |= CGCG_EN | CGLS_EN;
5950 } else {
5951 cik_enable_gui_idle_interrupt(rdev, false);
5952
5953 RREG32(CB_CGTT_SCLK_CTRL);
5954 RREG32(CB_CGTT_SCLK_CTRL);
5955 RREG32(CB_CGTT_SCLK_CTRL);
5956 RREG32(CB_CGTT_SCLK_CTRL);
5957
5958 data &= ~(CGCG_EN | CGLS_EN);
5959 }
5960
5961 if (orig != data)
5962 WREG32(RLC_CGCG_CGLS_CTRL, data);
5963
5964 }
5965
5966 static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
5967 {
5968 u32 data, orig, tmp = 0;
5969
5970 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5971 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
5972 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5973 orig = data = RREG32(CP_MEM_SLP_CNTL);
5974 data |= CP_MEM_LS_EN;
5975 if (orig != data)
5976 WREG32(CP_MEM_SLP_CNTL, data);
5977 }
5978 }
5979
5980 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5981 data &= 0xfffffffd;
5982 if (orig != data)
5983 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5984
5985 tmp = cik_halt_rlc(rdev);
5986
5987 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5988 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5989 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5990 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
5991 WREG32(RLC_SERDES_WR_CTRL, data);
5992
5993 cik_update_rlc(rdev, tmp);
5994
5995 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
5996 orig = data = RREG32(CGTS_SM_CTRL_REG);
5997 data &= ~SM_MODE_MASK;
5998 data |= SM_MODE(0x2);
5999 data |= SM_MODE_ENABLE;
6000 data &= ~CGTS_OVERRIDE;
6001 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
6002 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
6003 data &= ~CGTS_LS_OVERRIDE;
6004 data &= ~ON_MONITOR_ADD_MASK;
6005 data |= ON_MONITOR_ADD_EN;
6006 data |= ON_MONITOR_ADD(0x96);
6007 if (orig != data)
6008 WREG32(CGTS_SM_CTRL_REG, data);
6009 }
6010 } else {
6011 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
6012 data |= 0x00000002;
6013 if (orig != data)
6014 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6015
6016 data = RREG32(RLC_MEM_SLP_CNTL);
6017 if (data & RLC_MEM_LS_EN) {
6018 data &= ~RLC_MEM_LS_EN;
6019 WREG32(RLC_MEM_SLP_CNTL, data);
6020 }
6021
6022 data = RREG32(CP_MEM_SLP_CNTL);
6023 if (data & CP_MEM_LS_EN) {
6024 data &= ~CP_MEM_LS_EN;
6025 WREG32(CP_MEM_SLP_CNTL, data);
6026 }
6027
6028 orig = data = RREG32(CGTS_SM_CTRL_REG);
6029 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
6030 if (orig != data)
6031 WREG32(CGTS_SM_CTRL_REG, data);
6032
6033 tmp = cik_halt_rlc(rdev);
6034
6035 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6036 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6037 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6038 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
6039 WREG32(RLC_SERDES_WR_CTRL, data);
6040
6041 cik_update_rlc(rdev, tmp);
6042 }
6043 }
6044
6045 static const u32 mc_cg_registers[] =
6046 {
6047 MC_HUB_MISC_HUB_CG,
6048 MC_HUB_MISC_SIP_CG,
6049 MC_HUB_MISC_VM_CG,
6050 MC_XPB_CLK_GAT,
6051 ATC_MISC_CG,
6052 MC_CITF_MISC_WR_CG,
6053 MC_CITF_MISC_RD_CG,
6054 MC_CITF_MISC_VM_CG,
6055 VM_L2_CG,
6056 };
6057
6058 static void cik_enable_mc_ls(struct radeon_device *rdev,
6059 bool enable)
6060 {
6061 int i;
6062 u32 orig, data;
6063
6064 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6065 orig = data = RREG32(mc_cg_registers[i]);
6066 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
6067 data |= MC_LS_ENABLE;
6068 else
6069 data &= ~MC_LS_ENABLE;
6070 if (data != orig)
6071 WREG32(mc_cg_registers[i], data);
6072 }
6073 }
6074
6075 static void cik_enable_mc_mgcg(struct radeon_device *rdev,
6076 bool enable)
6077 {
6078 int i;
6079 u32 orig, data;
6080
6081 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6082 orig = data = RREG32(mc_cg_registers[i]);
6083 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
6084 data |= MC_CG_ENABLE;
6085 else
6086 data &= ~MC_CG_ENABLE;
6087 if (data != orig)
6088 WREG32(mc_cg_registers[i], data);
6089 }
6090 }
6091
6092 static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
6093 bool enable)
6094 {
6095 u32 orig, data;
6096
6097 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
6098 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
6099 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
6100 } else {
6101 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
6102 data |= 0xff000000;
6103 if (data != orig)
6104 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
6105
6106 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
6107 data |= 0xff000000;
6108 if (data != orig)
6109 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
6110 }
6111 }
6112
6113 static void cik_enable_sdma_mgls(struct radeon_device *rdev,
6114 bool enable)
6115 {
6116 u32 orig, data;
6117
6118 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
6119 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6120 data |= 0x100;
6121 if (orig != data)
6122 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6123
6124 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6125 data |= 0x100;
6126 if (orig != data)
6127 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6128 } else {
6129 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6130 data &= ~0x100;
6131 if (orig != data)
6132 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6133
6134 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6135 data &= ~0x100;
6136 if (orig != data)
6137 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6138 }
6139 }
6140
6141 static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
6142 bool enable)
6143 {
6144 u32 orig, data;
6145
6146 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
6147 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6148 data = 0xfff;
6149 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6150
6151 orig = data = RREG32(UVD_CGC_CTRL);
6152 data |= DCM;
6153 if (orig != data)
6154 WREG32(UVD_CGC_CTRL, data);
6155 } else {
6156 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6157 data &= ~0xfff;
6158 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6159
6160 orig = data = RREG32(UVD_CGC_CTRL);
6161 data &= ~DCM;
6162 if (orig != data)
6163 WREG32(UVD_CGC_CTRL, data);
6164 }
6165 }
6166
6167 static void cik_enable_bif_mgls(struct radeon_device *rdev,
6168 bool enable)
6169 {
6170 u32 orig, data;
6171
6172 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
6173
6174 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
6175 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
6176 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
6177 else
6178 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
6179 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
6180
6181 if (orig != data)
6182 WREG32_PCIE_PORT(PCIE_CNTL2, data);
6183 }
6184
6185 static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
6186 bool enable)
6187 {
6188 u32 orig, data;
6189
6190 orig = data = RREG32(HDP_HOST_PATH_CNTL);
6191
6192 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
6193 data &= ~CLOCK_GATING_DIS;
6194 else
6195 data |= CLOCK_GATING_DIS;
6196
6197 if (orig != data)
6198 WREG32(HDP_HOST_PATH_CNTL, data);
6199 }
6200
6201 static void cik_enable_hdp_ls(struct radeon_device *rdev,
6202 bool enable)
6203 {
6204 u32 orig, data;
6205
6206 orig = data = RREG32(HDP_MEM_POWER_LS);
6207
6208 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
6209 data |= HDP_LS_ENABLE;
6210 else
6211 data &= ~HDP_LS_ENABLE;
6212
6213 if (orig != data)
6214 WREG32(HDP_MEM_POWER_LS, data);
6215 }
6216
6217 void cik_update_cg(struct radeon_device *rdev,
6218 u32 block, bool enable)
6219 {
6220
6221 if (block & RADEON_CG_BLOCK_GFX) {
6222 cik_enable_gui_idle_interrupt(rdev, false);
6223 /* order matters! */
6224 if (enable) {
6225 cik_enable_mgcg(rdev, true);
6226 cik_enable_cgcg(rdev, true);
6227 } else {
6228 cik_enable_cgcg(rdev, false);
6229 cik_enable_mgcg(rdev, false);
6230 }
6231 cik_enable_gui_idle_interrupt(rdev, true);
6232 }
6233
6234 if (block & RADEON_CG_BLOCK_MC) {
6235 if (!(rdev->flags & RADEON_IS_IGP)) {
6236 cik_enable_mc_mgcg(rdev, enable);
6237 cik_enable_mc_ls(rdev, enable);
6238 }
6239 }
6240
6241 if (block & RADEON_CG_BLOCK_SDMA) {
6242 cik_enable_sdma_mgcg(rdev, enable);
6243 cik_enable_sdma_mgls(rdev, enable);
6244 }
6245
6246 if (block & RADEON_CG_BLOCK_BIF) {
6247 cik_enable_bif_mgls(rdev, enable);
6248 }
6249
6250 if (block & RADEON_CG_BLOCK_UVD) {
6251 if (rdev->has_uvd)
6252 cik_enable_uvd_mgcg(rdev, enable);
6253 }
6254
6255 if (block & RADEON_CG_BLOCK_HDP) {
6256 cik_enable_hdp_mgcg(rdev, enable);
6257 cik_enable_hdp_ls(rdev, enable);
6258 }
6259
6260 if (block & RADEON_CG_BLOCK_VCE) {
6261 vce_v2_0_enable_mgcg(rdev, enable);
6262 }
6263 }
6264
6265 static void cik_init_cg(struct radeon_device *rdev)
6266 {
6267
6268 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
6269
6270 if (rdev->has_uvd)
6271 si_init_uvd_internal_cg(rdev);
6272
6273 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6274 RADEON_CG_BLOCK_SDMA |
6275 RADEON_CG_BLOCK_BIF |
6276 RADEON_CG_BLOCK_UVD |
6277 RADEON_CG_BLOCK_HDP), true);
6278 }
6279
6280 static void cik_fini_cg(struct radeon_device *rdev)
6281 {
6282 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6283 RADEON_CG_BLOCK_SDMA |
6284 RADEON_CG_BLOCK_BIF |
6285 RADEON_CG_BLOCK_UVD |
6286 RADEON_CG_BLOCK_HDP), false);
6287
6288 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
6289 }
6290
6291 static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
6292 bool enable)
6293 {
6294 u32 data, orig;
6295
6296 orig = data = RREG32(RLC_PG_CNTL);
6297 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6298 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6299 else
6300 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6301 if (orig != data)
6302 WREG32(RLC_PG_CNTL, data);
6303 }
6304
6305 static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
6306 bool enable)
6307 {
6308 u32 data, orig;
6309
6310 orig = data = RREG32(RLC_PG_CNTL);
6311 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6312 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6313 else
6314 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6315 if (orig != data)
6316 WREG32(RLC_PG_CNTL, data);
6317 }
6318
6319 static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
6320 {
6321 u32 data, orig;
6322
6323 orig = data = RREG32(RLC_PG_CNTL);
6324 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
6325 data &= ~DISABLE_CP_PG;
6326 else
6327 data |= DISABLE_CP_PG;
6328 if (orig != data)
6329 WREG32(RLC_PG_CNTL, data);
6330 }
6331
6332 static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
6333 {
6334 u32 data, orig;
6335
6336 orig = data = RREG32(RLC_PG_CNTL);
6337 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
6338 data &= ~DISABLE_GDS_PG;
6339 else
6340 data |= DISABLE_GDS_PG;
6341 if (orig != data)
6342 WREG32(RLC_PG_CNTL, data);
6343 }
6344
6345 #define CP_ME_TABLE_SIZE 96
6346 #define CP_ME_TABLE_OFFSET 2048
6347 #define CP_MEC_TABLE_OFFSET 4096
6348
6349 void cik_init_cp_pg_table(struct radeon_device *rdev)
6350 {
6351 const __be32 *fw_data;
6352 volatile u32 *dst_ptr;
6353 int me, i, max_me = 4;
6354 u32 bo_offset = 0;
6355 u32 table_offset;
6356
6357 if (rdev->family == CHIP_KAVERI)
6358 max_me = 5;
6359
6360 if (rdev->rlc.cp_table_ptr == NULL)
6361 return;
6362
6363 /* write the cp table buffer */
6364 dst_ptr = rdev->rlc.cp_table_ptr;
6365 for (me = 0; me < max_me; me++) {
6366 if (me == 0) {
6367 fw_data = (const __be32 *)rdev->ce_fw->data;
6368 table_offset = CP_ME_TABLE_OFFSET;
6369 } else if (me == 1) {
6370 fw_data = (const __be32 *)rdev->pfp_fw->data;
6371 table_offset = CP_ME_TABLE_OFFSET;
6372 } else if (me == 2) {
6373 fw_data = (const __be32 *)rdev->me_fw->data;
6374 table_offset = CP_ME_TABLE_OFFSET;
6375 } else {
6376 fw_data = (const __be32 *)rdev->mec_fw->data;
6377 table_offset = CP_MEC_TABLE_OFFSET;
6378 }
6379
6380 for (i = 0; i < CP_ME_TABLE_SIZE; i ++) {
6381 dst_ptr[bo_offset + i] = cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
6382 }
6383 bo_offset += CP_ME_TABLE_SIZE;
6384 }
6385 }
6386
6387 static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6388 bool enable)
6389 {
6390 u32 data, orig;
6391
6392 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
6393 orig = data = RREG32(RLC_PG_CNTL);
6394 data |= GFX_PG_ENABLE;
6395 if (orig != data)
6396 WREG32(RLC_PG_CNTL, data);
6397
6398 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6399 data |= AUTO_PG_EN;
6400 if (orig != data)
6401 WREG32(RLC_AUTO_PG_CTRL, data);
6402 } else {
6403 orig = data = RREG32(RLC_PG_CNTL);
6404 data &= ~GFX_PG_ENABLE;
6405 if (orig != data)
6406 WREG32(RLC_PG_CNTL, data);
6407
6408 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6409 data &= ~AUTO_PG_EN;
6410 if (orig != data)
6411 WREG32(RLC_AUTO_PG_CTRL, data);
6412
6413 data = RREG32(DB_RENDER_CONTROL);
6414 }
6415 }
6416
6417 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6418 {
6419 u32 mask = 0, tmp, tmp1;
6420 int i;
6421
6422 cik_select_se_sh(rdev, se, sh);
6423 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6424 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6425 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6426
6427 tmp &= 0xffff0000;
6428
6429 tmp |= tmp1;
6430 tmp >>= 16;
6431
6432 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6433 mask <<= 1;
6434 mask |= 1;
6435 }
6436
6437 return (~tmp) & mask;
6438 }
6439
6440 static void cik_init_ao_cu_mask(struct radeon_device *rdev)
6441 {
6442 u32 i, j, k, active_cu_number = 0;
6443 u32 mask, counter, cu_bitmap;
6444 u32 tmp = 0;
6445
6446 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6447 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6448 mask = 1;
6449 cu_bitmap = 0;
6450 counter = 0;
6451 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6452 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6453 if (counter < 2)
6454 cu_bitmap |= mask;
6455 counter ++;
6456 }
6457 mask <<= 1;
6458 }
6459
6460 active_cu_number += counter;
6461 tmp |= (cu_bitmap << (i * 16 + j * 8));
6462 }
6463 }
6464
6465 WREG32(RLC_PG_AO_CU_MASK, tmp);
6466
6467 tmp = RREG32(RLC_MAX_PG_CU);
6468 tmp &= ~MAX_PU_CU_MASK;
6469 tmp |= MAX_PU_CU(active_cu_number);
6470 WREG32(RLC_MAX_PG_CU, tmp);
6471 }
6472
6473 static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6474 bool enable)
6475 {
6476 u32 data, orig;
6477
6478 orig = data = RREG32(RLC_PG_CNTL);
6479 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
6480 data |= STATIC_PER_CU_PG_ENABLE;
6481 else
6482 data &= ~STATIC_PER_CU_PG_ENABLE;
6483 if (orig != data)
6484 WREG32(RLC_PG_CNTL, data);
6485 }
6486
6487 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6488 bool enable)
6489 {
6490 u32 data, orig;
6491
6492 orig = data = RREG32(RLC_PG_CNTL);
6493 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
6494 data |= DYN_PER_CU_PG_ENABLE;
6495 else
6496 data &= ~DYN_PER_CU_PG_ENABLE;
6497 if (orig != data)
6498 WREG32(RLC_PG_CNTL, data);
6499 }
6500
6501 #define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6502 #define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
6503
6504 static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6505 {
6506 u32 data, orig;
6507 u32 i;
6508
6509 if (rdev->rlc.cs_data) {
6510 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6511 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
6512 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
6513 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
6514 } else {
6515 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6516 for (i = 0; i < 3; i++)
6517 WREG32(RLC_GPM_SCRATCH_DATA, 0);
6518 }
6519 if (rdev->rlc.reg_list) {
6520 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6521 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6522 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
6523 }
6524
6525 orig = data = RREG32(RLC_PG_CNTL);
6526 data |= GFX_PG_SRC;
6527 if (orig != data)
6528 WREG32(RLC_PG_CNTL, data);
6529
6530 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6531 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
6532
6533 data = RREG32(CP_RB_WPTR_POLL_CNTL);
6534 data &= ~IDLE_POLL_COUNT_MASK;
6535 data |= IDLE_POLL_COUNT(0x60);
6536 WREG32(CP_RB_WPTR_POLL_CNTL, data);
6537
6538 data = 0x10101010;
6539 WREG32(RLC_PG_DELAY, data);
6540
6541 data = RREG32(RLC_PG_DELAY_2);
6542 data &= ~0xff;
6543 data |= 0x3;
6544 WREG32(RLC_PG_DELAY_2, data);
6545
6546 data = RREG32(RLC_AUTO_PG_CTRL);
6547 data &= ~GRBM_REG_SGIT_MASK;
6548 data |= GRBM_REG_SGIT(0x700);
6549 WREG32(RLC_AUTO_PG_CTRL, data);
6550
6551 }
6552
6553 static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
6554 {
6555 cik_enable_gfx_cgpg(rdev, enable);
6556 cik_enable_gfx_static_mgpg(rdev, enable);
6557 cik_enable_gfx_dynamic_mgpg(rdev, enable);
6558 }
6559
6560 u32 cik_get_csb_size(struct radeon_device *rdev)
6561 {
6562 u32 count = 0;
6563 const struct cs_section_def *sect = NULL;
6564 const struct cs_extent_def *ext = NULL;
6565
6566 if (rdev->rlc.cs_data == NULL)
6567 return 0;
6568
6569 /* begin clear state */
6570 count += 2;
6571 /* context control state */
6572 count += 3;
6573
6574 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6575 for (ext = sect->section; ext->extent != NULL; ++ext) {
6576 if (sect->id == SECT_CONTEXT)
6577 count += 2 + ext->reg_count;
6578 else
6579 return 0;
6580 }
6581 }
6582 /* pa_sc_raster_config/pa_sc_raster_config1 */
6583 count += 4;
6584 /* end clear state */
6585 count += 2;
6586 /* clear state */
6587 count += 2;
6588
6589 return count;
6590 }
6591
6592 void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
6593 {
6594 u32 count = 0, i;
6595 const struct cs_section_def *sect = NULL;
6596 const struct cs_extent_def *ext = NULL;
6597
6598 if (rdev->rlc.cs_data == NULL)
6599 return;
6600 if (buffer == NULL)
6601 return;
6602
6603 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6604 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
6605
6606 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6607 buffer[count++] = cpu_to_le32(0x80000000);
6608 buffer[count++] = cpu_to_le32(0x80000000);
6609
6610 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6611 for (ext = sect->section; ext->extent != NULL; ++ext) {
6612 if (sect->id == SECT_CONTEXT) {
6613 buffer[count++] =
6614 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
6615 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
6616 for (i = 0; i < ext->reg_count; i++)
6617 buffer[count++] = cpu_to_le32(ext->extent[i]);
6618 } else {
6619 return;
6620 }
6621 }
6622 }
6623
6624 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
6625 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
6626 switch (rdev->family) {
6627 case CHIP_BONAIRE:
6628 buffer[count++] = cpu_to_le32(0x16000012);
6629 buffer[count++] = cpu_to_le32(0x00000000);
6630 break;
6631 case CHIP_KAVERI:
6632 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6633 buffer[count++] = cpu_to_le32(0x00000000);
6634 break;
6635 case CHIP_KABINI:
6636 case CHIP_MULLINS:
6637 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6638 buffer[count++] = cpu_to_le32(0x00000000);
6639 break;
6640 case CHIP_HAWAII:
6641 buffer[count++] = cpu_to_le32(0x3a00161a);
6642 buffer[count++] = cpu_to_le32(0x0000002e);
6643 break;
6644 default:
6645 buffer[count++] = cpu_to_le32(0x00000000);
6646 buffer[count++] = cpu_to_le32(0x00000000);
6647 break;
6648 }
6649
6650 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6651 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
6652
6653 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
6654 buffer[count++] = cpu_to_le32(0);
6655 }
6656
6657 static void cik_init_pg(struct radeon_device *rdev)
6658 {
6659 if (rdev->pg_flags) {
6660 cik_enable_sck_slowdown_on_pu(rdev, true);
6661 cik_enable_sck_slowdown_on_pd(rdev, true);
6662 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
6663 cik_init_gfx_cgpg(rdev);
6664 cik_enable_cp_pg(rdev, true);
6665 cik_enable_gds_pg(rdev, true);
6666 }
6667 cik_init_ao_cu_mask(rdev);
6668 cik_update_gfx_pg(rdev, true);
6669 }
6670 }
6671
6672 static void cik_fini_pg(struct radeon_device *rdev)
6673 {
6674 if (rdev->pg_flags) {
6675 cik_update_gfx_pg(rdev, false);
6676 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
6677 cik_enable_cp_pg(rdev, false);
6678 cik_enable_gds_pg(rdev, false);
6679 }
6680 }
6681 }
6682
6683 /*
6684 * Interrupts
6685 * Starting with r6xx, interrupts are handled via a ring buffer.
6686 * Ring buffers are areas of GPU accessible memory that the GPU
6687 * writes interrupt vectors into and the host reads vectors out of.
6688 * There is a rptr (read pointer) that determines where the
6689 * host is currently reading, and a wptr (write pointer)
6690 * which determines where the GPU has written. When the
6691 * pointers are equal, the ring is idle. When the GPU
6692 * writes vectors to the ring buffer, it increments the
6693 * wptr. When there is an interrupt, the host then starts
6694 * fetching commands and processing them until the pointers are
6695 * equal again at which point it updates the rptr.
6696 */
6697
6698 /**
6699 * cik_enable_interrupts - Enable the interrupt ring buffer
6700 *
6701 * @rdev: radeon_device pointer
6702 *
6703 * Enable the interrupt ring buffer (CIK).
6704 */
6705 static void cik_enable_interrupts(struct radeon_device *rdev)
6706 {
6707 u32 ih_cntl = RREG32(IH_CNTL);
6708 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6709
6710 ih_cntl |= ENABLE_INTR;
6711 ih_rb_cntl |= IH_RB_ENABLE;
6712 WREG32(IH_CNTL, ih_cntl);
6713 WREG32(IH_RB_CNTL, ih_rb_cntl);
6714 rdev->ih.enabled = true;
6715 }
6716
6717 /**
6718 * cik_disable_interrupts - Disable the interrupt ring buffer
6719 *
6720 * @rdev: radeon_device pointer
6721 *
6722 * Disable the interrupt ring buffer (CIK).
6723 */
6724 static void cik_disable_interrupts(struct radeon_device *rdev)
6725 {
6726 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6727 u32 ih_cntl = RREG32(IH_CNTL);
6728
6729 ih_rb_cntl &= ~IH_RB_ENABLE;
6730 ih_cntl &= ~ENABLE_INTR;
6731 WREG32(IH_RB_CNTL, ih_rb_cntl);
6732 WREG32(IH_CNTL, ih_cntl);
6733 /* set rptr, wptr to 0 */
6734 WREG32(IH_RB_RPTR, 0);
6735 WREG32(IH_RB_WPTR, 0);
6736 rdev->ih.enabled = false;
6737 rdev->ih.rptr = 0;
6738 }
6739
6740 /**
6741 * cik_disable_interrupt_state - Disable all interrupt sources
6742 *
6743 * @rdev: radeon_device pointer
6744 *
6745 * Clear all interrupt enable bits used by the driver (CIK).
6746 */
6747 static void cik_disable_interrupt_state(struct radeon_device *rdev)
6748 {
6749 u32 tmp;
6750
6751 /* gfx ring */
6752 tmp = RREG32(CP_INT_CNTL_RING0) &
6753 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6754 WREG32(CP_INT_CNTL_RING0, tmp);
6755 /* sdma */
6756 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6757 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
6758 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6759 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
6760 /* compute queues */
6761 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
6762 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
6763 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
6764 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
6765 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
6766 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
6767 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
6768 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
6769 /* grbm */
6770 WREG32(GRBM_INT_CNTL, 0);
6771 /* vline/vblank, etc. */
6772 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6773 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6774 if (rdev->num_crtc >= 4) {
6775 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6776 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6777 }
6778 if (rdev->num_crtc >= 6) {
6779 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6780 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6781 }
6782 /* pflip */
6783 if (rdev->num_crtc >= 2) {
6784 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6785 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6786 }
6787 if (rdev->num_crtc >= 4) {
6788 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6789 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6790 }
6791 if (rdev->num_crtc >= 6) {
6792 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6793 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6794 }
6795
6796 /* dac hotplug */
6797 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
6798
6799 /* digital hotplug */
6800 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6801 WREG32(DC_HPD1_INT_CONTROL, tmp);
6802 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6803 WREG32(DC_HPD2_INT_CONTROL, tmp);
6804 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6805 WREG32(DC_HPD3_INT_CONTROL, tmp);
6806 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6807 WREG32(DC_HPD4_INT_CONTROL, tmp);
6808 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6809 WREG32(DC_HPD5_INT_CONTROL, tmp);
6810 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6811 WREG32(DC_HPD6_INT_CONTROL, tmp);
6812
6813 }
6814
6815 /**
6816 * cik_irq_init - init and enable the interrupt ring
6817 *
6818 * @rdev: radeon_device pointer
6819 *
6820 * Allocate a ring buffer for the interrupt controller,
6821 * enable the RLC, disable interrupts, enable the IH
6822 * ring buffer and enable it (CIK).
6823 * Called at device load and reume.
6824 * Returns 0 for success, errors for failure.
6825 */
6826 static int cik_irq_init(struct radeon_device *rdev)
6827 {
6828 int ret = 0;
6829 int rb_bufsz;
6830 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
6831
6832 /* allocate ring */
6833 ret = r600_ih_ring_alloc(rdev);
6834 if (ret)
6835 return ret;
6836
6837 /* disable irqs */
6838 cik_disable_interrupts(rdev);
6839
6840 /* init rlc */
6841 ret = cik_rlc_resume(rdev);
6842 if (ret) {
6843 r600_ih_ring_fini(rdev);
6844 return ret;
6845 }
6846
6847 /* setup interrupt control */
6848 /* XXX this should actually be a bus address, not an MC address. same on older asics */
6849 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
6850 interrupt_cntl = RREG32(INTERRUPT_CNTL);
6851 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
6852 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
6853 */
6854 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
6855 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
6856 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
6857 WREG32(INTERRUPT_CNTL, interrupt_cntl);
6858
6859 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
6860 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
6861
6862 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
6863 IH_WPTR_OVERFLOW_CLEAR |
6864 (rb_bufsz << 1));
6865
6866 if (rdev->wb.enabled)
6867 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
6868
6869 /* set the writeback address whether it's enabled or not */
6870 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
6871 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
6872
6873 WREG32(IH_RB_CNTL, ih_rb_cntl);
6874
6875 /* set rptr, wptr to 0 */
6876 WREG32(IH_RB_RPTR, 0);
6877 WREG32(IH_RB_WPTR, 0);
6878
6879 /* Default settings for IH_CNTL (disabled at first) */
6880 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6881 /* RPTR_REARM only works if msi's are enabled */
6882 if (rdev->msi_enabled)
6883 ih_cntl |= RPTR_REARM;
6884 WREG32(IH_CNTL, ih_cntl);
6885
6886 /* force the active interrupt state to all disabled */
6887 cik_disable_interrupt_state(rdev);
6888
6889 pci_set_master(rdev->pdev);
6890
6891 /* enable irqs */
6892 cik_enable_interrupts(rdev);
6893
6894 return ret;
6895 }
6896
6897 /**
6898 * cik_irq_set - enable/disable interrupt sources
6899 *
6900 * @rdev: radeon_device pointer
6901 *
6902 * Enable interrupt sources on the GPU (vblanks, hpd,
6903 * etc.) (CIK).
6904 * Returns 0 for success, errors for failure.
6905 */
6906 int cik_irq_set(struct radeon_device *rdev)
6907 {
6908 u32 cp_int_cntl;
6909 u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
6910 u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
6911 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
6912 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
6913 u32 grbm_int_cntl = 0;
6914 u32 dma_cntl, dma_cntl1;
6915 u32 thermal_int;
6916
6917 if (!rdev->irq.installed) {
6918 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6919 return -EINVAL;
6920 }
6921 /* don't enable anything if the ih is disabled */
6922 if (!rdev->ih.enabled) {
6923 cik_disable_interrupts(rdev);
6924 /* force the active interrupt state to all disabled */
6925 cik_disable_interrupt_state(rdev);
6926 return 0;
6927 }
6928
6929 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6930 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6931 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
6932
6933 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
6934 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
6935 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
6936 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
6937 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
6938 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
6939
6940 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6941 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6942
6943 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6944 cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6945 cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6946 cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6947 cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6948 cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6949 cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6950 cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6951
6952 if (rdev->flags & RADEON_IS_IGP)
6953 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &
6954 ~(THERM_INTH_MASK | THERM_INTL_MASK);
6955 else
6956 thermal_int = RREG32_SMC(CG_THERMAL_INT) &
6957 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
6958
6959 /* enable CP interrupts on all rings */
6960 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
6961 DRM_DEBUG("cik_irq_set: sw int gfx\n");
6962 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6963 }
6964 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
6965 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6966 DRM_DEBUG("si_irq_set: sw int cp1\n");
6967 if (ring->me == 1) {
6968 switch (ring->pipe) {
6969 case 0:
6970 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6971 break;
6972 case 1:
6973 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6974 break;
6975 case 2:
6976 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6977 break;
6978 case 3:
6979 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6980 break;
6981 default:
6982 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6983 break;
6984 }
6985 } else if (ring->me == 2) {
6986 switch (ring->pipe) {
6987 case 0:
6988 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6989 break;
6990 case 1:
6991 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6992 break;
6993 case 2:
6994 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6995 break;
6996 case 3:
6997 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6998 break;
6999 default:
7000 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7001 break;
7002 }
7003 } else {
7004 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
7005 }
7006 }
7007 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
7008 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7009 DRM_DEBUG("si_irq_set: sw int cp2\n");
7010 if (ring->me == 1) {
7011 switch (ring->pipe) {
7012 case 0:
7013 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7014 break;
7015 case 1:
7016 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
7017 break;
7018 case 2:
7019 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7020 break;
7021 case 3:
7022 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7023 break;
7024 default:
7025 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7026 break;
7027 }
7028 } else if (ring->me == 2) {
7029 switch (ring->pipe) {
7030 case 0:
7031 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
7032 break;
7033 case 1:
7034 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
7035 break;
7036 case 2:
7037 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7038 break;
7039 case 3:
7040 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7041 break;
7042 default:
7043 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7044 break;
7045 }
7046 } else {
7047 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
7048 }
7049 }
7050
7051 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
7052 DRM_DEBUG("cik_irq_set: sw int dma\n");
7053 dma_cntl |= TRAP_ENABLE;
7054 }
7055
7056 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
7057 DRM_DEBUG("cik_irq_set: sw int dma1\n");
7058 dma_cntl1 |= TRAP_ENABLE;
7059 }
7060
7061 if (rdev->irq.crtc_vblank_int[0] ||
7062 atomic_read(&rdev->irq.pflip[0])) {
7063 DRM_DEBUG("cik_irq_set: vblank 0\n");
7064 crtc1 |= VBLANK_INTERRUPT_MASK;
7065 }
7066 if (rdev->irq.crtc_vblank_int[1] ||
7067 atomic_read(&rdev->irq.pflip[1])) {
7068 DRM_DEBUG("cik_irq_set: vblank 1\n");
7069 crtc2 |= VBLANK_INTERRUPT_MASK;
7070 }
7071 if (rdev->irq.crtc_vblank_int[2] ||
7072 atomic_read(&rdev->irq.pflip[2])) {
7073 DRM_DEBUG("cik_irq_set: vblank 2\n");
7074 crtc3 |= VBLANK_INTERRUPT_MASK;
7075 }
7076 if (rdev->irq.crtc_vblank_int[3] ||
7077 atomic_read(&rdev->irq.pflip[3])) {
7078 DRM_DEBUG("cik_irq_set: vblank 3\n");
7079 crtc4 |= VBLANK_INTERRUPT_MASK;
7080 }
7081 if (rdev->irq.crtc_vblank_int[4] ||
7082 atomic_read(&rdev->irq.pflip[4])) {
7083 DRM_DEBUG("cik_irq_set: vblank 4\n");
7084 crtc5 |= VBLANK_INTERRUPT_MASK;
7085 }
7086 if (rdev->irq.crtc_vblank_int[5] ||
7087 atomic_read(&rdev->irq.pflip[5])) {
7088 DRM_DEBUG("cik_irq_set: vblank 5\n");
7089 crtc6 |= VBLANK_INTERRUPT_MASK;
7090 }
7091 if (rdev->irq.hpd[0]) {
7092 DRM_DEBUG("cik_irq_set: hpd 1\n");
7093 hpd1 |= DC_HPDx_INT_EN;
7094 }
7095 if (rdev->irq.hpd[1]) {
7096 DRM_DEBUG("cik_irq_set: hpd 2\n");
7097 hpd2 |= DC_HPDx_INT_EN;
7098 }
7099 if (rdev->irq.hpd[2]) {
7100 DRM_DEBUG("cik_irq_set: hpd 3\n");
7101 hpd3 |= DC_HPDx_INT_EN;
7102 }
7103 if (rdev->irq.hpd[3]) {
7104 DRM_DEBUG("cik_irq_set: hpd 4\n");
7105 hpd4 |= DC_HPDx_INT_EN;
7106 }
7107 if (rdev->irq.hpd[4]) {
7108 DRM_DEBUG("cik_irq_set: hpd 5\n");
7109 hpd5 |= DC_HPDx_INT_EN;
7110 }
7111 if (rdev->irq.hpd[5]) {
7112 DRM_DEBUG("cik_irq_set: hpd 6\n");
7113 hpd6 |= DC_HPDx_INT_EN;
7114 }
7115
7116 if (rdev->irq.dpm_thermal) {
7117 DRM_DEBUG("dpm thermal\n");
7118 if (rdev->flags & RADEON_IS_IGP)
7119 thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK;
7120 else
7121 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
7122 }
7123
7124 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
7125
7126 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
7127 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
7128
7129 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
7130 WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
7131 WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
7132 WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
7133 WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
7134 WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
7135 WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
7136 WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
7137
7138 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
7139
7140 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
7141 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
7142 if (rdev->num_crtc >= 4) {
7143 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
7144 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
7145 }
7146 if (rdev->num_crtc >= 6) {
7147 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
7148 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
7149 }
7150
7151 if (rdev->num_crtc >= 2) {
7152 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
7153 GRPH_PFLIP_INT_MASK);
7154 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
7155 GRPH_PFLIP_INT_MASK);
7156 }
7157 if (rdev->num_crtc >= 4) {
7158 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
7159 GRPH_PFLIP_INT_MASK);
7160 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
7161 GRPH_PFLIP_INT_MASK);
7162 }
7163 if (rdev->num_crtc >= 6) {
7164 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
7165 GRPH_PFLIP_INT_MASK);
7166 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
7167 GRPH_PFLIP_INT_MASK);
7168 }
7169
7170 WREG32(DC_HPD1_INT_CONTROL, hpd1);
7171 WREG32(DC_HPD2_INT_CONTROL, hpd2);
7172 WREG32(DC_HPD3_INT_CONTROL, hpd3);
7173 WREG32(DC_HPD4_INT_CONTROL, hpd4);
7174 WREG32(DC_HPD5_INT_CONTROL, hpd5);
7175 WREG32(DC_HPD6_INT_CONTROL, hpd6);
7176
7177 if (rdev->flags & RADEON_IS_IGP)
7178 WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int);
7179 else
7180 WREG32_SMC(CG_THERMAL_INT, thermal_int);
7181
7182 return 0;
7183 }
7184
7185 /**
7186 * cik_irq_ack - ack interrupt sources
7187 *
7188 * @rdev: radeon_device pointer
7189 *
7190 * Ack interrupt sources on the GPU (vblanks, hpd,
7191 * etc.) (CIK). Certain interrupts sources are sw
7192 * generated and do not require an explicit ack.
7193 */
7194 static inline void cik_irq_ack(struct radeon_device *rdev)
7195 {
7196 u32 tmp;
7197
7198 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
7199 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
7200 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
7201 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
7202 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
7203 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
7204 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
7205
7206 rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS +
7207 EVERGREEN_CRTC0_REGISTER_OFFSET);
7208 rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS +
7209 EVERGREEN_CRTC1_REGISTER_OFFSET);
7210 if (rdev->num_crtc >= 4) {
7211 rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS +
7212 EVERGREEN_CRTC2_REGISTER_OFFSET);
7213 rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS +
7214 EVERGREEN_CRTC3_REGISTER_OFFSET);
7215 }
7216 if (rdev->num_crtc >= 6) {
7217 rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS +
7218 EVERGREEN_CRTC4_REGISTER_OFFSET);
7219 rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS +
7220 EVERGREEN_CRTC5_REGISTER_OFFSET);
7221 }
7222
7223 if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
7224 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET,
7225 GRPH_PFLIP_INT_CLEAR);
7226 if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
7227 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET,
7228 GRPH_PFLIP_INT_CLEAR);
7229 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
7230 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
7231 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
7232 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
7233 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
7234 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
7235 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
7236 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
7237
7238 if (rdev->num_crtc >= 4) {
7239 if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
7240 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET,
7241 GRPH_PFLIP_INT_CLEAR);
7242 if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
7243 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET,
7244 GRPH_PFLIP_INT_CLEAR);
7245 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
7246 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
7247 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
7248 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
7249 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
7250 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
7251 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
7252 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
7253 }
7254
7255 if (rdev->num_crtc >= 6) {
7256 if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
7257 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET,
7258 GRPH_PFLIP_INT_CLEAR);
7259 if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
7260 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET,
7261 GRPH_PFLIP_INT_CLEAR);
7262 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
7263 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
7264 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
7265 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
7266 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
7267 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
7268 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
7269 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
7270 }
7271
7272 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7273 tmp = RREG32(DC_HPD1_INT_CONTROL);
7274 tmp |= DC_HPDx_INT_ACK;
7275 WREG32(DC_HPD1_INT_CONTROL, tmp);
7276 }
7277 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7278 tmp = RREG32(DC_HPD2_INT_CONTROL);
7279 tmp |= DC_HPDx_INT_ACK;
7280 WREG32(DC_HPD2_INT_CONTROL, tmp);
7281 }
7282 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7283 tmp = RREG32(DC_HPD3_INT_CONTROL);
7284 tmp |= DC_HPDx_INT_ACK;
7285 WREG32(DC_HPD3_INT_CONTROL, tmp);
7286 }
7287 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7288 tmp = RREG32(DC_HPD4_INT_CONTROL);
7289 tmp |= DC_HPDx_INT_ACK;
7290 WREG32(DC_HPD4_INT_CONTROL, tmp);
7291 }
7292 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7293 tmp = RREG32(DC_HPD5_INT_CONTROL);
7294 tmp |= DC_HPDx_INT_ACK;
7295 WREG32(DC_HPD5_INT_CONTROL, tmp);
7296 }
7297 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7298 tmp = RREG32(DC_HPD5_INT_CONTROL);
7299 tmp |= DC_HPDx_INT_ACK;
7300 WREG32(DC_HPD6_INT_CONTROL, tmp);
7301 }
7302 }
7303
7304 /**
7305 * cik_irq_disable - disable interrupts
7306 *
7307 * @rdev: radeon_device pointer
7308 *
7309 * Disable interrupts on the hw (CIK).
7310 */
7311 static void cik_irq_disable(struct radeon_device *rdev)
7312 {
7313 cik_disable_interrupts(rdev);
7314 /* Wait and acknowledge irq */
7315 mdelay(1);
7316 cik_irq_ack(rdev);
7317 cik_disable_interrupt_state(rdev);
7318 }
7319
7320 /**
7321 * cik_irq_disable - disable interrupts for suspend
7322 *
7323 * @rdev: radeon_device pointer
7324 *
7325 * Disable interrupts and stop the RLC (CIK).
7326 * Used for suspend.
7327 */
7328 static void cik_irq_suspend(struct radeon_device *rdev)
7329 {
7330 cik_irq_disable(rdev);
7331 cik_rlc_stop(rdev);
7332 }
7333
7334 /**
7335 * cik_irq_fini - tear down interrupt support
7336 *
7337 * @rdev: radeon_device pointer
7338 *
7339 * Disable interrupts on the hw and free the IH ring
7340 * buffer (CIK).
7341 * Used for driver unload.
7342 */
7343 static void cik_irq_fini(struct radeon_device *rdev)
7344 {
7345 cik_irq_suspend(rdev);
7346 r600_ih_ring_fini(rdev);
7347 }
7348
7349 /**
7350 * cik_get_ih_wptr - get the IH ring buffer wptr
7351 *
7352 * @rdev: radeon_device pointer
7353 *
7354 * Get the IH ring buffer wptr from either the register
7355 * or the writeback memory buffer (CIK). Also check for
7356 * ring buffer overflow and deal with it.
7357 * Used by cik_irq_process().
7358 * Returns the value of the wptr.
7359 */
7360 static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
7361 {
7362 u32 wptr, tmp;
7363
7364 if (rdev->wb.enabled)
7365 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
7366 else
7367 wptr = RREG32(IH_RB_WPTR);
7368
7369 if (wptr & RB_OVERFLOW) {
7370 /* When a ring buffer overflow happen start parsing interrupt
7371 * from the last not overwritten vector (wptr + 16). Hopefully
7372 * this should allow us to catchup.
7373 */
7374 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
7375 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
7376 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
7377 tmp = RREG32(IH_RB_CNTL);
7378 tmp |= IH_WPTR_OVERFLOW_CLEAR;
7379 WREG32(IH_RB_CNTL, tmp);
7380 wptr &= ~RB_OVERFLOW;
7381 }
7382 return (wptr & rdev->ih.ptr_mask);
7383 }
7384
7385 /* CIK IV Ring
7386 * Each IV ring entry is 128 bits:
7387 * [7:0] - interrupt source id
7388 * [31:8] - reserved
7389 * [59:32] - interrupt source data
7390 * [63:60] - reserved
7391 * [71:64] - RINGID
7392 * CP:
7393 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
7394 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7395 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7396 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7397 * PIPE_ID - ME0 0=3D
7398 * - ME1&2 compute dispatcher (4 pipes each)
7399 * SDMA:
7400 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
7401 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
7402 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
7403 * [79:72] - VMID
7404 * [95:80] - PASID
7405 * [127:96] - reserved
7406 */
7407 /**
7408 * cik_irq_process - interrupt handler
7409 *
7410 * @rdev: radeon_device pointer
7411 *
7412 * Interrupt hander (CIK). Walk the IH ring,
7413 * ack interrupts and schedule work to handle
7414 * interrupt events.
7415 * Returns irq process return code.
7416 */
7417 int cik_irq_process(struct radeon_device *rdev)
7418 {
7419 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7420 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7421 u32 wptr;
7422 u32 rptr;
7423 u32 src_id, src_data, ring_id;
7424 u8 me_id, pipe_id, queue_id;
7425 u32 ring_index;
7426 bool queue_hotplug = false;
7427 bool queue_reset = false;
7428 u32 addr, status, mc_client;
7429 bool queue_thermal = false;
7430
7431 if (!rdev->ih.enabled || rdev->shutdown)
7432 return IRQ_NONE;
7433
7434 wptr = cik_get_ih_wptr(rdev);
7435
7436 restart_ih:
7437 /* is somebody else already processing irqs? */
7438 if (atomic_xchg(&rdev->ih.lock, 1))
7439 return IRQ_NONE;
7440
7441 rptr = rdev->ih.rptr;
7442 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7443
7444 /* Order reading of wptr vs. reading of IH ring data */
7445 rmb();
7446
7447 /* display interrupts */
7448 cik_irq_ack(rdev);
7449
7450 while (rptr != wptr) {
7451 /* wptr/rptr are in bytes! */
7452 ring_index = rptr / 4;
7453 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7454 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7455 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
7456
7457 switch (src_id) {
7458 case 1: /* D1 vblank/vline */
7459 switch (src_data) {
7460 case 0: /* D1 vblank */
7461 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
7462 if (rdev->irq.crtc_vblank_int[0]) {
7463 drm_handle_vblank(rdev->ddev, 0);
7464 rdev->pm.vblank_sync = true;
7465 wake_up(&rdev->irq.vblank_queue);
7466 }
7467 if (atomic_read(&rdev->irq.pflip[0]))
7468 radeon_crtc_handle_vblank(rdev, 0);
7469 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7470 DRM_DEBUG("IH: D1 vblank\n");
7471 }
7472 break;
7473 case 1: /* D1 vline */
7474 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
7475 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7476 DRM_DEBUG("IH: D1 vline\n");
7477 }
7478 break;
7479 default:
7480 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7481 break;
7482 }
7483 break;
7484 case 2: /* D2 vblank/vline */
7485 switch (src_data) {
7486 case 0: /* D2 vblank */
7487 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
7488 if (rdev->irq.crtc_vblank_int[1]) {
7489 drm_handle_vblank(rdev->ddev, 1);
7490 rdev->pm.vblank_sync = true;
7491 wake_up(&rdev->irq.vblank_queue);
7492 }
7493 if (atomic_read(&rdev->irq.pflip[1]))
7494 radeon_crtc_handle_vblank(rdev, 1);
7495 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7496 DRM_DEBUG("IH: D2 vblank\n");
7497 }
7498 break;
7499 case 1: /* D2 vline */
7500 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
7501 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7502 DRM_DEBUG("IH: D2 vline\n");
7503 }
7504 break;
7505 default:
7506 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7507 break;
7508 }
7509 break;
7510 case 3: /* D3 vblank/vline */
7511 switch (src_data) {
7512 case 0: /* D3 vblank */
7513 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
7514 if (rdev->irq.crtc_vblank_int[2]) {
7515 drm_handle_vblank(rdev->ddev, 2);
7516 rdev->pm.vblank_sync = true;
7517 wake_up(&rdev->irq.vblank_queue);
7518 }
7519 if (atomic_read(&rdev->irq.pflip[2]))
7520 radeon_crtc_handle_vblank(rdev, 2);
7521 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7522 DRM_DEBUG("IH: D3 vblank\n");
7523 }
7524 break;
7525 case 1: /* D3 vline */
7526 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
7527 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7528 DRM_DEBUG("IH: D3 vline\n");
7529 }
7530 break;
7531 default:
7532 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7533 break;
7534 }
7535 break;
7536 case 4: /* D4 vblank/vline */
7537 switch (src_data) {
7538 case 0: /* D4 vblank */
7539 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
7540 if (rdev->irq.crtc_vblank_int[3]) {
7541 drm_handle_vblank(rdev->ddev, 3);
7542 rdev->pm.vblank_sync = true;
7543 wake_up(&rdev->irq.vblank_queue);
7544 }
7545 if (atomic_read(&rdev->irq.pflip[3]))
7546 radeon_crtc_handle_vblank(rdev, 3);
7547 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7548 DRM_DEBUG("IH: D4 vblank\n");
7549 }
7550 break;
7551 case 1: /* D4 vline */
7552 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
7553 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7554 DRM_DEBUG("IH: D4 vline\n");
7555 }
7556 break;
7557 default:
7558 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7559 break;
7560 }
7561 break;
7562 case 5: /* D5 vblank/vline */
7563 switch (src_data) {
7564 case 0: /* D5 vblank */
7565 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
7566 if (rdev->irq.crtc_vblank_int[4]) {
7567 drm_handle_vblank(rdev->ddev, 4);
7568 rdev->pm.vblank_sync = true;
7569 wake_up(&rdev->irq.vblank_queue);
7570 }
7571 if (atomic_read(&rdev->irq.pflip[4]))
7572 radeon_crtc_handle_vblank(rdev, 4);
7573 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7574 DRM_DEBUG("IH: D5 vblank\n");
7575 }
7576 break;
7577 case 1: /* D5 vline */
7578 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
7579 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7580 DRM_DEBUG("IH: D5 vline\n");
7581 }
7582 break;
7583 default:
7584 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7585 break;
7586 }
7587 break;
7588 case 6: /* D6 vblank/vline */
7589 switch (src_data) {
7590 case 0: /* D6 vblank */
7591 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
7592 if (rdev->irq.crtc_vblank_int[5]) {
7593 drm_handle_vblank(rdev->ddev, 5);
7594 rdev->pm.vblank_sync = true;
7595 wake_up(&rdev->irq.vblank_queue);
7596 }
7597 if (atomic_read(&rdev->irq.pflip[5]))
7598 radeon_crtc_handle_vblank(rdev, 5);
7599 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7600 DRM_DEBUG("IH: D6 vblank\n");
7601 }
7602 break;
7603 case 1: /* D6 vline */
7604 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
7605 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7606 DRM_DEBUG("IH: D6 vline\n");
7607 }
7608 break;
7609 default:
7610 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7611 break;
7612 }
7613 break;
7614 case 8: /* D1 page flip */
7615 case 10: /* D2 page flip */
7616 case 12: /* D3 page flip */
7617 case 14: /* D4 page flip */
7618 case 16: /* D5 page flip */
7619 case 18: /* D6 page flip */
7620 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
7621 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
7622 break;
7623 case 42: /* HPD hotplug */
7624 switch (src_data) {
7625 case 0:
7626 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7627 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
7628 queue_hotplug = true;
7629 DRM_DEBUG("IH: HPD1\n");
7630 }
7631 break;
7632 case 1:
7633 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7634 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
7635 queue_hotplug = true;
7636 DRM_DEBUG("IH: HPD2\n");
7637 }
7638 break;
7639 case 2:
7640 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7641 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
7642 queue_hotplug = true;
7643 DRM_DEBUG("IH: HPD3\n");
7644 }
7645 break;
7646 case 3:
7647 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7648 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
7649 queue_hotplug = true;
7650 DRM_DEBUG("IH: HPD4\n");
7651 }
7652 break;
7653 case 4:
7654 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7655 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
7656 queue_hotplug = true;
7657 DRM_DEBUG("IH: HPD5\n");
7658 }
7659 break;
7660 case 5:
7661 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7662 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
7663 queue_hotplug = true;
7664 DRM_DEBUG("IH: HPD6\n");
7665 }
7666 break;
7667 default:
7668 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7669 break;
7670 }
7671 break;
7672 case 124: /* UVD */
7673 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
7674 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
7675 break;
7676 case 146:
7677 case 147:
7678 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
7679 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
7680 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
7681 /* reset addr and status */
7682 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
7683 if (addr == 0x0 && status == 0x0)
7684 break;
7685 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
7686 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
7687 addr);
7688 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
7689 status);
7690 cik_vm_decode_fault(rdev, status, addr, mc_client);
7691 break;
7692 case 167: /* VCE */
7693 DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data);
7694 switch (src_data) {
7695 case 0:
7696 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX);
7697 break;
7698 case 1:
7699 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX);
7700 break;
7701 default:
7702 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
7703 break;
7704 }
7705 break;
7706 case 176: /* GFX RB CP_INT */
7707 case 177: /* GFX IB CP_INT */
7708 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7709 break;
7710 case 181: /* CP EOP event */
7711 DRM_DEBUG("IH: CP EOP\n");
7712 /* XXX check the bitfield order! */
7713 me_id = (ring_id & 0x60) >> 5;
7714 pipe_id = (ring_id & 0x18) >> 3;
7715 queue_id = (ring_id & 0x7) >> 0;
7716 switch (me_id) {
7717 case 0:
7718 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7719 break;
7720 case 1:
7721 case 2:
7722 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
7723 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7724 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
7725 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7726 break;
7727 }
7728 break;
7729 case 184: /* CP Privileged reg access */
7730 DRM_ERROR("Illegal register access in command stream\n");
7731 /* XXX check the bitfield order! */
7732 me_id = (ring_id & 0x60) >> 5;
7733 pipe_id = (ring_id & 0x18) >> 3;
7734 queue_id = (ring_id & 0x7) >> 0;
7735 switch (me_id) {
7736 case 0:
7737 /* This results in a full GPU reset, but all we need to do is soft
7738 * reset the CP for gfx
7739 */
7740 queue_reset = true;
7741 break;
7742 case 1:
7743 /* XXX compute */
7744 queue_reset = true;
7745 break;
7746 case 2:
7747 /* XXX compute */
7748 queue_reset = true;
7749 break;
7750 }
7751 break;
7752 case 185: /* CP Privileged inst */
7753 DRM_ERROR("Illegal instruction in command stream\n");
7754 /* XXX check the bitfield order! */
7755 me_id = (ring_id & 0x60) >> 5;
7756 pipe_id = (ring_id & 0x18) >> 3;
7757 queue_id = (ring_id & 0x7) >> 0;
7758 switch (me_id) {
7759 case 0:
7760 /* This results in a full GPU reset, but all we need to do is soft
7761 * reset the CP for gfx
7762 */
7763 queue_reset = true;
7764 break;
7765 case 1:
7766 /* XXX compute */
7767 queue_reset = true;
7768 break;
7769 case 2:
7770 /* XXX compute */
7771 queue_reset = true;
7772 break;
7773 }
7774 break;
7775 case 224: /* SDMA trap event */
7776 /* XXX check the bitfield order! */
7777 me_id = (ring_id & 0x3) >> 0;
7778 queue_id = (ring_id & 0xc) >> 2;
7779 DRM_DEBUG("IH: SDMA trap\n");
7780 switch (me_id) {
7781 case 0:
7782 switch (queue_id) {
7783 case 0:
7784 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
7785 break;
7786 case 1:
7787 /* XXX compute */
7788 break;
7789 case 2:
7790 /* XXX compute */
7791 break;
7792 }
7793 break;
7794 case 1:
7795 switch (queue_id) {
7796 case 0:
7797 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7798 break;
7799 case 1:
7800 /* XXX compute */
7801 break;
7802 case 2:
7803 /* XXX compute */
7804 break;
7805 }
7806 break;
7807 }
7808 break;
7809 case 230: /* thermal low to high */
7810 DRM_DEBUG("IH: thermal low to high\n");
7811 rdev->pm.dpm.thermal.high_to_low = false;
7812 queue_thermal = true;
7813 break;
7814 case 231: /* thermal high to low */
7815 DRM_DEBUG("IH: thermal high to low\n");
7816 rdev->pm.dpm.thermal.high_to_low = true;
7817 queue_thermal = true;
7818 break;
7819 case 233: /* GUI IDLE */
7820 DRM_DEBUG("IH: GUI idle\n");
7821 break;
7822 case 241: /* SDMA Privileged inst */
7823 case 247: /* SDMA Privileged inst */
7824 DRM_ERROR("Illegal instruction in SDMA command stream\n");
7825 /* XXX check the bitfield order! */
7826 me_id = (ring_id & 0x3) >> 0;
7827 queue_id = (ring_id & 0xc) >> 2;
7828 switch (me_id) {
7829 case 0:
7830 switch (queue_id) {
7831 case 0:
7832 queue_reset = true;
7833 break;
7834 case 1:
7835 /* XXX compute */
7836 queue_reset = true;
7837 break;
7838 case 2:
7839 /* XXX compute */
7840 queue_reset = true;
7841 break;
7842 }
7843 break;
7844 case 1:
7845 switch (queue_id) {
7846 case 0:
7847 queue_reset = true;
7848 break;
7849 case 1:
7850 /* XXX compute */
7851 queue_reset = true;
7852 break;
7853 case 2:
7854 /* XXX compute */
7855 queue_reset = true;
7856 break;
7857 }
7858 break;
7859 }
7860 break;
7861 default:
7862 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7863 break;
7864 }
7865
7866 /* wptr/rptr are in bytes! */
7867 rptr += 16;
7868 rptr &= rdev->ih.ptr_mask;
7869 }
7870 if (queue_hotplug)
7871 schedule_work(&rdev->hotplug_work);
7872 if (queue_reset)
7873 schedule_work(&rdev->reset_work);
7874 if (queue_thermal)
7875 schedule_work(&rdev->pm.dpm.thermal.work);
7876 rdev->ih.rptr = rptr;
7877 WREG32(IH_RB_RPTR, rdev->ih.rptr);
7878 atomic_set(&rdev->ih.lock, 0);
7879
7880 /* make sure wptr hasn't changed while processing */
7881 wptr = cik_get_ih_wptr(rdev);
7882 if (wptr != rptr)
7883 goto restart_ih;
7884
7885 return IRQ_HANDLED;
7886 }
7887
7888 /*
7889 * startup/shutdown callbacks
7890 */
7891 /**
7892 * cik_startup - program the asic to a functional state
7893 *
7894 * @rdev: radeon_device pointer
7895 *
7896 * Programs the asic to a functional state (CIK).
7897 * Called by cik_init() and cik_resume().
7898 * Returns 0 for success, error for failure.
7899 */
7900 static int cik_startup(struct radeon_device *rdev)
7901 {
7902 struct radeon_ring *ring;
7903 int r;
7904
7905 /* enable pcie gen2/3 link */
7906 cik_pcie_gen3_enable(rdev);
7907 /* enable aspm */
7908 cik_program_aspm(rdev);
7909
7910 /* scratch needs to be initialized before MC */
7911 r = r600_vram_scratch_init(rdev);
7912 if (r)
7913 return r;
7914
7915 cik_mc_program(rdev);
7916
7917 if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
7918 r = ci_mc_load_microcode(rdev);
7919 if (r) {
7920 DRM_ERROR("Failed to load MC firmware!\n");
7921 return r;
7922 }
7923 }
7924
7925 r = cik_pcie_gart_enable(rdev);
7926 if (r)
7927 return r;
7928 cik_gpu_init(rdev);
7929
7930 /* allocate rlc buffers */
7931 if (rdev->flags & RADEON_IS_IGP) {
7932 if (rdev->family == CHIP_KAVERI) {
7933 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
7934 rdev->rlc.reg_list_size =
7935 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
7936 } else {
7937 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
7938 rdev->rlc.reg_list_size =
7939 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
7940 }
7941 }
7942 rdev->rlc.cs_data = ci_cs_data;
7943 rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
7944 r = sumo_rlc_init(rdev);
7945 if (r) {
7946 DRM_ERROR("Failed to init rlc BOs!\n");
7947 return r;
7948 }
7949
7950 /* allocate wb buffer */
7951 r = radeon_wb_init(rdev);
7952 if (r)
7953 return r;
7954
7955 /* allocate mec buffers */
7956 r = cik_mec_init(rdev);
7957 if (r) {
7958 DRM_ERROR("Failed to init MEC BOs!\n");
7959 return r;
7960 }
7961
7962 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
7963 if (r) {
7964 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7965 return r;
7966 }
7967
7968 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7969 if (r) {
7970 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7971 return r;
7972 }
7973
7974 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7975 if (r) {
7976 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7977 return r;
7978 }
7979
7980 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
7981 if (r) {
7982 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7983 return r;
7984 }
7985
7986 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7987 if (r) {
7988 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7989 return r;
7990 }
7991
7992 r = radeon_uvd_resume(rdev);
7993 if (!r) {
7994 r = uvd_v4_2_resume(rdev);
7995 if (!r) {
7996 r = radeon_fence_driver_start_ring(rdev,
7997 R600_RING_TYPE_UVD_INDEX);
7998 if (r)
7999 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
8000 }
8001 }
8002 if (r)
8003 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
8004
8005 r = radeon_vce_resume(rdev);
8006 if (!r) {
8007 r = vce_v2_0_resume(rdev);
8008 if (!r)
8009 r = radeon_fence_driver_start_ring(rdev,
8010 TN_RING_TYPE_VCE1_INDEX);
8011 if (!r)
8012 r = radeon_fence_driver_start_ring(rdev,
8013 TN_RING_TYPE_VCE2_INDEX);
8014 }
8015 if (r) {
8016 dev_err(rdev->dev, "VCE init error (%d).\n", r);
8017 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
8018 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
8019 }
8020
8021 /* Enable IRQ */
8022 if (!rdev->irq.installed) {
8023 r = radeon_irq_kms_init(rdev);
8024 if (r)
8025 return r;
8026 }
8027
8028 r = cik_irq_init(rdev);
8029 if (r) {
8030 DRM_ERROR("radeon: IH init failed (%d).\n", r);
8031 radeon_irq_kms_fini(rdev);
8032 return r;
8033 }
8034 cik_irq_set(rdev);
8035
8036 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8037 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
8038 PACKET3(PACKET3_NOP, 0x3FFF));
8039 if (r)
8040 return r;
8041
8042 /* set up the compute queues */
8043 /* type-2 packets are deprecated on MEC, use type-3 instead */
8044 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8045 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
8046 PACKET3(PACKET3_NOP, 0x3FFF));
8047 if (r)
8048 return r;
8049 ring->me = 1; /* first MEC */
8050 ring->pipe = 0; /* first pipe */
8051 ring->queue = 0; /* first queue */
8052 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
8053
8054 /* type-2 packets are deprecated on MEC, use type-3 instead */
8055 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8056 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
8057 PACKET3(PACKET3_NOP, 0x3FFF));
8058 if (r)
8059 return r;
8060 /* dGPU only have 1 MEC */
8061 ring->me = 1; /* first MEC */
8062 ring->pipe = 0; /* first pipe */
8063 ring->queue = 1; /* second queue */
8064 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
8065
8066 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8067 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
8068 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
8069 if (r)
8070 return r;
8071
8072 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8073 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
8074 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
8075 if (r)
8076 return r;
8077
8078 r = cik_cp_resume(rdev);
8079 if (r)
8080 return r;
8081
8082 r = cik_sdma_resume(rdev);
8083 if (r)
8084 return r;
8085
8086 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8087 if (ring->ring_size) {
8088 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8089 RADEON_CP_PACKET2);
8090 if (!r)
8091 r = uvd_v1_0_init(rdev);
8092 if (r)
8093 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
8094 }
8095
8096 r = -ENOENT;
8097
8098 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8099 if (ring->ring_size)
8100 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8101 VCE_CMD_NO_OP);
8102
8103 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8104 if (ring->ring_size)
8105 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8106 VCE_CMD_NO_OP);
8107
8108 if (!r)
8109 r = vce_v1_0_init(rdev);
8110 else if (r != -ENOENT)
8111 DRM_ERROR("radeon: failed initializing VCE (%d).\n", r);
8112
8113 r = radeon_ib_pool_init(rdev);
8114 if (r) {
8115 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
8116 return r;
8117 }
8118
8119 r = radeon_vm_manager_init(rdev);
8120 if (r) {
8121 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
8122 return r;
8123 }
8124
8125 r = dce6_audio_init(rdev);
8126 if (r)
8127 return r;
8128
8129 return 0;
8130 }
8131
8132 /**
8133 * cik_resume - resume the asic to a functional state
8134 *
8135 * @rdev: radeon_device pointer
8136 *
8137 * Programs the asic to a functional state (CIK).
8138 * Called at resume.
8139 * Returns 0 for success, error for failure.
8140 */
8141 int cik_resume(struct radeon_device *rdev)
8142 {
8143 int r;
8144
8145 /* post card */
8146 atom_asic_init(rdev->mode_info.atom_context);
8147
8148 /* init golden registers */
8149 cik_init_golden_registers(rdev);
8150
8151 if (rdev->pm.pm_method == PM_METHOD_DPM)
8152 radeon_pm_resume(rdev);
8153
8154 rdev->accel_working = true;
8155 r = cik_startup(rdev);
8156 if (r) {
8157 DRM_ERROR("cik startup failed on resume\n");
8158 rdev->accel_working = false;
8159 return r;
8160 }
8161
8162 return r;
8163
8164 }
8165
8166 /**
8167 * cik_suspend - suspend the asic
8168 *
8169 * @rdev: radeon_device pointer
8170 *
8171 * Bring the chip into a state suitable for suspend (CIK).
8172 * Called at suspend.
8173 * Returns 0 for success.
8174 */
8175 int cik_suspend(struct radeon_device *rdev)
8176 {
8177 radeon_pm_suspend(rdev);
8178 dce6_audio_fini(rdev);
8179 radeon_vm_manager_fini(rdev);
8180 cik_cp_enable(rdev, false);
8181 cik_sdma_enable(rdev, false);
8182 uvd_v1_0_fini(rdev);
8183 radeon_uvd_suspend(rdev);
8184 radeon_vce_suspend(rdev);
8185 cik_fini_pg(rdev);
8186 cik_fini_cg(rdev);
8187 cik_irq_suspend(rdev);
8188 radeon_wb_disable(rdev);
8189 cik_pcie_gart_disable(rdev);
8190 return 0;
8191 }
8192
8193 /* Plan is to move initialization in that function and use
8194 * helper function so that radeon_device_init pretty much
8195 * do nothing more than calling asic specific function. This
8196 * should also allow to remove a bunch of callback function
8197 * like vram_info.
8198 */
8199 /**
8200 * cik_init - asic specific driver and hw init
8201 *
8202 * @rdev: radeon_device pointer
8203 *
8204 * Setup asic specific driver variables and program the hw
8205 * to a functional state (CIK).
8206 * Called at driver startup.
8207 * Returns 0 for success, errors for failure.
8208 */
8209 int cik_init(struct radeon_device *rdev)
8210 {
8211 struct radeon_ring *ring;
8212 int r;
8213
8214 /* Read BIOS */
8215 if (!radeon_get_bios(rdev)) {
8216 if (ASIC_IS_AVIVO(rdev))
8217 return -EINVAL;
8218 }
8219 /* Must be an ATOMBIOS */
8220 if (!rdev->is_atom_bios) {
8221 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
8222 return -EINVAL;
8223 }
8224 r = radeon_atombios_init(rdev);
8225 if (r)
8226 return r;
8227
8228 /* Post card if necessary */
8229 if (!radeon_card_posted(rdev)) {
8230 if (!rdev->bios) {
8231 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
8232 return -EINVAL;
8233 }
8234 DRM_INFO("GPU not posted. posting now...\n");
8235 atom_asic_init(rdev->mode_info.atom_context);
8236 }
8237 /* init golden registers */
8238 cik_init_golden_registers(rdev);
8239 /* Initialize scratch registers */
8240 cik_scratch_init(rdev);
8241 /* Initialize surface registers */
8242 radeon_surface_init(rdev);
8243 /* Initialize clocks */
8244 radeon_get_clock_info(rdev->ddev);
8245
8246 /* Fence driver */
8247 r = radeon_fence_driver_init(rdev);
8248 if (r)
8249 return r;
8250
8251 /* initialize memory controller */
8252 r = cik_mc_init(rdev);
8253 if (r)
8254 return r;
8255 /* Memory manager */
8256 r = radeon_bo_init(rdev);
8257 if (r)
8258 return r;
8259
8260 if (rdev->flags & RADEON_IS_IGP) {
8261 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8262 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
8263 r = cik_init_microcode(rdev);
8264 if (r) {
8265 DRM_ERROR("Failed to load firmware!\n");
8266 return r;
8267 }
8268 }
8269 } else {
8270 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8271 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
8272 !rdev->mc_fw) {
8273 r = cik_init_microcode(rdev);
8274 if (r) {
8275 DRM_ERROR("Failed to load firmware!\n");
8276 return r;
8277 }
8278 }
8279 }
8280
8281 /* Initialize power management */
8282 radeon_pm_init(rdev);
8283
8284 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8285 ring->ring_obj = NULL;
8286 r600_ring_init(rdev, ring, 1024 * 1024);
8287
8288 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8289 ring->ring_obj = NULL;
8290 r600_ring_init(rdev, ring, 1024 * 1024);
8291 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8292 if (r)
8293 return r;
8294
8295 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8296 ring->ring_obj = NULL;
8297 r600_ring_init(rdev, ring, 1024 * 1024);
8298 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8299 if (r)
8300 return r;
8301
8302 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8303 ring->ring_obj = NULL;
8304 r600_ring_init(rdev, ring, 256 * 1024);
8305
8306 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8307 ring->ring_obj = NULL;
8308 r600_ring_init(rdev, ring, 256 * 1024);
8309
8310 r = radeon_uvd_init(rdev);
8311 if (!r) {
8312 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8313 ring->ring_obj = NULL;
8314 r600_ring_init(rdev, ring, 4096);
8315 }
8316
8317 r = radeon_vce_init(rdev);
8318 if (!r) {
8319 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8320 ring->ring_obj = NULL;
8321 r600_ring_init(rdev, ring, 4096);
8322
8323 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8324 ring->ring_obj = NULL;
8325 r600_ring_init(rdev, ring, 4096);
8326 }
8327
8328 rdev->ih.ring_obj = NULL;
8329 r600_ih_ring_init(rdev, 64 * 1024);
8330
8331 r = r600_pcie_gart_init(rdev);
8332 if (r)
8333 return r;
8334
8335 rdev->accel_working = true;
8336 r = cik_startup(rdev);
8337 if (r) {
8338 dev_err(rdev->dev, "disabling GPU acceleration\n");
8339 cik_cp_fini(rdev);
8340 cik_sdma_fini(rdev);
8341 cik_irq_fini(rdev);
8342 sumo_rlc_fini(rdev);
8343 cik_mec_fini(rdev);
8344 radeon_wb_fini(rdev);
8345 radeon_ib_pool_fini(rdev);
8346 radeon_vm_manager_fini(rdev);
8347 radeon_irq_kms_fini(rdev);
8348 cik_pcie_gart_fini(rdev);
8349 rdev->accel_working = false;
8350 }
8351
8352 /* Don't start up if the MC ucode is missing.
8353 * The default clocks and voltages before the MC ucode
8354 * is loaded are not suffient for advanced operations.
8355 */
8356 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
8357 DRM_ERROR("radeon: MC ucode required for NI+.\n");
8358 return -EINVAL;
8359 }
8360
8361 return 0;
8362 }
8363
8364 /**
8365 * cik_fini - asic specific driver and hw fini
8366 *
8367 * @rdev: radeon_device pointer
8368 *
8369 * Tear down the asic specific driver variables and program the hw
8370 * to an idle state (CIK).
8371 * Called at driver unload.
8372 */
8373 void cik_fini(struct radeon_device *rdev)
8374 {
8375 radeon_pm_fini(rdev);
8376 cik_cp_fini(rdev);
8377 cik_sdma_fini(rdev);
8378 cik_fini_pg(rdev);
8379 cik_fini_cg(rdev);
8380 cik_irq_fini(rdev);
8381 sumo_rlc_fini(rdev);
8382 cik_mec_fini(rdev);
8383 radeon_wb_fini(rdev);
8384 radeon_vm_manager_fini(rdev);
8385 radeon_ib_pool_fini(rdev);
8386 radeon_irq_kms_fini(rdev);
8387 uvd_v1_0_fini(rdev);
8388 radeon_uvd_fini(rdev);
8389 radeon_vce_fini(rdev);
8390 cik_pcie_gart_fini(rdev);
8391 r600_vram_scratch_fini(rdev);
8392 radeon_gem_fini(rdev);
8393 radeon_fence_driver_fini(rdev);
8394 radeon_bo_fini(rdev);
8395 radeon_atombios_fini(rdev);
8396 kfree(rdev->bios);
8397 rdev->bios = NULL;
8398 }
8399
8400 void dce8_program_fmt(struct drm_encoder *encoder)
8401 {
8402 struct drm_device *dev = encoder->dev;
8403 struct radeon_device *rdev = dev->dev_private;
8404 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
8405 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
8406 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
8407 int bpc = 0;
8408 u32 tmp = 0;
8409 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
8410
8411 if (connector) {
8412 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
8413 bpc = radeon_get_monitor_bpc(connector);
8414 dither = radeon_connector->dither;
8415 }
8416
8417 /* LVDS/eDP FMT is set up by atom */
8418 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
8419 return;
8420
8421 /* not needed for analog */
8422 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
8423 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
8424 return;
8425
8426 if (bpc == 0)
8427 return;
8428
8429 switch (bpc) {
8430 case 6:
8431 if (dither == RADEON_FMT_DITHER_ENABLE)
8432 /* XXX sort out optimal dither settings */
8433 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8434 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
8435 else
8436 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
8437 break;
8438 case 8:
8439 if (dither == RADEON_FMT_DITHER_ENABLE)
8440 /* XXX sort out optimal dither settings */
8441 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8442 FMT_RGB_RANDOM_ENABLE |
8443 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
8444 else
8445 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
8446 break;
8447 case 10:
8448 if (dither == RADEON_FMT_DITHER_ENABLE)
8449 /* XXX sort out optimal dither settings */
8450 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8451 FMT_RGB_RANDOM_ENABLE |
8452 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
8453 else
8454 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
8455 break;
8456 default:
8457 /* not needed */
8458 break;
8459 }
8460
8461 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8462 }
8463
8464 /* display watermark setup */
8465 /**
8466 * dce8_line_buffer_adjust - Set up the line buffer
8467 *
8468 * @rdev: radeon_device pointer
8469 * @radeon_crtc: the selected display controller
8470 * @mode: the current display mode on the selected display
8471 * controller
8472 *
8473 * Setup up the line buffer allocation for
8474 * the selected display controller (CIK).
8475 * Returns the line buffer size in pixels.
8476 */
8477 static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8478 struct radeon_crtc *radeon_crtc,
8479 struct drm_display_mode *mode)
8480 {
8481 u32 tmp, buffer_alloc, i;
8482 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
8483 /*
8484 * Line Buffer Setup
8485 * There are 6 line buffers, one for each display controllers.
8486 * There are 3 partitions per LB. Select the number of partitions
8487 * to enable based on the display width. For display widths larger
8488 * than 4096, you need use to use 2 display controllers and combine
8489 * them using the stereo blender.
8490 */
8491 if (radeon_crtc->base.enabled && mode) {
8492 if (mode->crtc_hdisplay < 1920) {
8493 tmp = 1;
8494 buffer_alloc = 2;
8495 } else if (mode->crtc_hdisplay < 2560) {
8496 tmp = 2;
8497 buffer_alloc = 2;
8498 } else if (mode->crtc_hdisplay < 4096) {
8499 tmp = 0;
8500 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8501 } else {
8502 DRM_DEBUG_KMS("Mode too big for LB!\n");
8503 tmp = 0;
8504 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8505 }
8506 } else {
8507 tmp = 1;
8508 buffer_alloc = 0;
8509 }
8510
8511 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8512 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8513
8514 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8515 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8516 for (i = 0; i < rdev->usec_timeout; i++) {
8517 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8518 DMIF_BUFFERS_ALLOCATED_COMPLETED)
8519 break;
8520 udelay(1);
8521 }
8522
8523 if (radeon_crtc->base.enabled && mode) {
8524 switch (tmp) {
8525 case 0:
8526 default:
8527 return 4096 * 2;
8528 case 1:
8529 return 1920 * 2;
8530 case 2:
8531 return 2560 * 2;
8532 }
8533 }
8534
8535 /* controller not enabled, so no lb used */
8536 return 0;
8537 }
8538
8539 /**
8540 * cik_get_number_of_dram_channels - get the number of dram channels
8541 *
8542 * @rdev: radeon_device pointer
8543 *
8544 * Look up the number of video ram channels (CIK).
8545 * Used for display watermark bandwidth calculations
8546 * Returns the number of dram channels
8547 */
8548 static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8549 {
8550 u32 tmp = RREG32(MC_SHARED_CHMAP);
8551
8552 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8553 case 0:
8554 default:
8555 return 1;
8556 case 1:
8557 return 2;
8558 case 2:
8559 return 4;
8560 case 3:
8561 return 8;
8562 case 4:
8563 return 3;
8564 case 5:
8565 return 6;
8566 case 6:
8567 return 10;
8568 case 7:
8569 return 12;
8570 case 8:
8571 return 16;
8572 }
8573 }
8574
8575 struct dce8_wm_params {
8576 u32 dram_channels; /* number of dram channels */
8577 u32 yclk; /* bandwidth per dram data pin in kHz */
8578 u32 sclk; /* engine clock in kHz */
8579 u32 disp_clk; /* display clock in kHz */
8580 u32 src_width; /* viewport width */
8581 u32 active_time; /* active display time in ns */
8582 u32 blank_time; /* blank time in ns */
8583 bool interlaced; /* mode is interlaced */
8584 fixed20_12 vsc; /* vertical scale ratio */
8585 u32 num_heads; /* number of active crtcs */
8586 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8587 u32 lb_size; /* line buffer allocated to pipe */
8588 u32 vtaps; /* vertical scaler taps */
8589 };
8590
8591 /**
8592 * dce8_dram_bandwidth - get the dram bandwidth
8593 *
8594 * @wm: watermark calculation data
8595 *
8596 * Calculate the raw dram bandwidth (CIK).
8597 * Used for display watermark bandwidth calculations
8598 * Returns the dram bandwidth in MBytes/s
8599 */
8600 static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
8601 {
8602 /* Calculate raw DRAM Bandwidth */
8603 fixed20_12 dram_efficiency; /* 0.7 */
8604 fixed20_12 yclk, dram_channels, bandwidth;
8605 fixed20_12 a;
8606
8607 a.full = dfixed_const(1000);
8608 yclk.full = dfixed_const(wm->yclk);
8609 yclk.full = dfixed_div(yclk, a);
8610 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8611 a.full = dfixed_const(10);
8612 dram_efficiency.full = dfixed_const(7);
8613 dram_efficiency.full = dfixed_div(dram_efficiency, a);
8614 bandwidth.full = dfixed_mul(dram_channels, yclk);
8615 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
8616
8617 return dfixed_trunc(bandwidth);
8618 }
8619
8620 /**
8621 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
8622 *
8623 * @wm: watermark calculation data
8624 *
8625 * Calculate the dram bandwidth used for display (CIK).
8626 * Used for display watermark bandwidth calculations
8627 * Returns the dram bandwidth for display in MBytes/s
8628 */
8629 static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8630 {
8631 /* Calculate DRAM Bandwidth and the part allocated to display. */
8632 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
8633 fixed20_12 yclk, dram_channels, bandwidth;
8634 fixed20_12 a;
8635
8636 a.full = dfixed_const(1000);
8637 yclk.full = dfixed_const(wm->yclk);
8638 yclk.full = dfixed_div(yclk, a);
8639 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8640 a.full = dfixed_const(10);
8641 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
8642 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
8643 bandwidth.full = dfixed_mul(dram_channels, yclk);
8644 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
8645
8646 return dfixed_trunc(bandwidth);
8647 }
8648
8649 /**
8650 * dce8_data_return_bandwidth - get the data return bandwidth
8651 *
8652 * @wm: watermark calculation data
8653 *
8654 * Calculate the data return bandwidth used for display (CIK).
8655 * Used for display watermark bandwidth calculations
8656 * Returns the data return bandwidth in MBytes/s
8657 */
8658 static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
8659 {
8660 /* Calculate the display Data return Bandwidth */
8661 fixed20_12 return_efficiency; /* 0.8 */
8662 fixed20_12 sclk, bandwidth;
8663 fixed20_12 a;
8664
8665 a.full = dfixed_const(1000);
8666 sclk.full = dfixed_const(wm->sclk);
8667 sclk.full = dfixed_div(sclk, a);
8668 a.full = dfixed_const(10);
8669 return_efficiency.full = dfixed_const(8);
8670 return_efficiency.full = dfixed_div(return_efficiency, a);
8671 a.full = dfixed_const(32);
8672 bandwidth.full = dfixed_mul(a, sclk);
8673 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
8674
8675 return dfixed_trunc(bandwidth);
8676 }
8677
8678 /**
8679 * dce8_dmif_request_bandwidth - get the dmif bandwidth
8680 *
8681 * @wm: watermark calculation data
8682 *
8683 * Calculate the dmif bandwidth used for display (CIK).
8684 * Used for display watermark bandwidth calculations
8685 * Returns the dmif bandwidth in MBytes/s
8686 */
8687 static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
8688 {
8689 /* Calculate the DMIF Request Bandwidth */
8690 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
8691 fixed20_12 disp_clk, bandwidth;
8692 fixed20_12 a, b;
8693
8694 a.full = dfixed_const(1000);
8695 disp_clk.full = dfixed_const(wm->disp_clk);
8696 disp_clk.full = dfixed_div(disp_clk, a);
8697 a.full = dfixed_const(32);
8698 b.full = dfixed_mul(a, disp_clk);
8699
8700 a.full = dfixed_const(10);
8701 disp_clk_request_efficiency.full = dfixed_const(8);
8702 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
8703
8704 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
8705
8706 return dfixed_trunc(bandwidth);
8707 }
8708
8709 /**
8710 * dce8_available_bandwidth - get the min available bandwidth
8711 *
8712 * @wm: watermark calculation data
8713 *
8714 * Calculate the min available bandwidth used for display (CIK).
8715 * Used for display watermark bandwidth calculations
8716 * Returns the min available bandwidth in MBytes/s
8717 */
8718 static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
8719 {
8720 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
8721 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
8722 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
8723 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
8724
8725 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
8726 }
8727
8728 /**
8729 * dce8_average_bandwidth - get the average available bandwidth
8730 *
8731 * @wm: watermark calculation data
8732 *
8733 * Calculate the average available bandwidth used for display (CIK).
8734 * Used for display watermark bandwidth calculations
8735 * Returns the average available bandwidth in MBytes/s
8736 */
8737 static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
8738 {
8739 /* Calculate the display mode Average Bandwidth
8740 * DisplayMode should contain the source and destination dimensions,
8741 * timing, etc.
8742 */
8743 fixed20_12 bpp;
8744 fixed20_12 line_time;
8745 fixed20_12 src_width;
8746 fixed20_12 bandwidth;
8747 fixed20_12 a;
8748
8749 a.full = dfixed_const(1000);
8750 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
8751 line_time.full = dfixed_div(line_time, a);
8752 bpp.full = dfixed_const(wm->bytes_per_pixel);
8753 src_width.full = dfixed_const(wm->src_width);
8754 bandwidth.full = dfixed_mul(src_width, bpp);
8755 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
8756 bandwidth.full = dfixed_div(bandwidth, line_time);
8757
8758 return dfixed_trunc(bandwidth);
8759 }
8760
8761 /**
8762 * dce8_latency_watermark - get the latency watermark
8763 *
8764 * @wm: watermark calculation data
8765 *
8766 * Calculate the latency watermark (CIK).
8767 * Used for display watermark bandwidth calculations
8768 * Returns the latency watermark in ns
8769 */
8770 static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
8771 {
8772 /* First calculate the latency in ns */
8773 u32 mc_latency = 2000; /* 2000 ns. */
8774 u32 available_bandwidth = dce8_available_bandwidth(wm);
8775 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
8776 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
8777 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
8778 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
8779 (wm->num_heads * cursor_line_pair_return_time);
8780 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
8781 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
8782 u32 tmp, dmif_size = 12288;
8783 fixed20_12 a, b, c;
8784
8785 if (wm->num_heads == 0)
8786 return 0;
8787
8788 a.full = dfixed_const(2);
8789 b.full = dfixed_const(1);
8790 if ((wm->vsc.full > a.full) ||
8791 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
8792 (wm->vtaps >= 5) ||
8793 ((wm->vsc.full >= a.full) && wm->interlaced))
8794 max_src_lines_per_dst_line = 4;
8795 else
8796 max_src_lines_per_dst_line = 2;
8797
8798 a.full = dfixed_const(available_bandwidth);
8799 b.full = dfixed_const(wm->num_heads);
8800 a.full = dfixed_div(a, b);
8801
8802 b.full = dfixed_const(mc_latency + 512);
8803 c.full = dfixed_const(wm->disp_clk);
8804 b.full = dfixed_div(b, c);
8805
8806 c.full = dfixed_const(dmif_size);
8807 b.full = dfixed_div(c, b);
8808
8809 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
8810
8811 b.full = dfixed_const(1000);
8812 c.full = dfixed_const(wm->disp_clk);
8813 b.full = dfixed_div(c, b);
8814 c.full = dfixed_const(wm->bytes_per_pixel);
8815 b.full = dfixed_mul(b, c);
8816
8817 lb_fill_bw = min(tmp, dfixed_trunc(b));
8818
8819 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
8820 b.full = dfixed_const(1000);
8821 c.full = dfixed_const(lb_fill_bw);
8822 b.full = dfixed_div(c, b);
8823 a.full = dfixed_div(a, b);
8824 line_fill_time = dfixed_trunc(a);
8825
8826 if (line_fill_time < wm->active_time)
8827 return latency;
8828 else
8829 return latency + (line_fill_time - wm->active_time);
8830
8831 }
8832
8833 /**
8834 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
8835 * average and available dram bandwidth
8836 *
8837 * @wm: watermark calculation data
8838 *
8839 * Check if the display average bandwidth fits in the display
8840 * dram bandwidth (CIK).
8841 * Used for display watermark bandwidth calculations
8842 * Returns true if the display fits, false if not.
8843 */
8844 static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8845 {
8846 if (dce8_average_bandwidth(wm) <=
8847 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
8848 return true;
8849 else
8850 return false;
8851 }
8852
8853 /**
8854 * dce8_average_bandwidth_vs_available_bandwidth - check
8855 * average and available bandwidth
8856 *
8857 * @wm: watermark calculation data
8858 *
8859 * Check if the display average bandwidth fits in the display
8860 * available bandwidth (CIK).
8861 * Used for display watermark bandwidth calculations
8862 * Returns true if the display fits, false if not.
8863 */
8864 static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
8865 {
8866 if (dce8_average_bandwidth(wm) <=
8867 (dce8_available_bandwidth(wm) / wm->num_heads))
8868 return true;
8869 else
8870 return false;
8871 }
8872
8873 /**
8874 * dce8_check_latency_hiding - check latency hiding
8875 *
8876 * @wm: watermark calculation data
8877 *
8878 * Check latency hiding (CIK).
8879 * Used for display watermark bandwidth calculations
8880 * Returns true if the display fits, false if not.
8881 */
8882 static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
8883 {
8884 u32 lb_partitions = wm->lb_size / wm->src_width;
8885 u32 line_time = wm->active_time + wm->blank_time;
8886 u32 latency_tolerant_lines;
8887 u32 latency_hiding;
8888 fixed20_12 a;
8889
8890 a.full = dfixed_const(1);
8891 if (wm->vsc.full > a.full)
8892 latency_tolerant_lines = 1;
8893 else {
8894 if (lb_partitions <= (wm->vtaps + 1))
8895 latency_tolerant_lines = 1;
8896 else
8897 latency_tolerant_lines = 2;
8898 }
8899
8900 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
8901
8902 if (dce8_latency_watermark(wm) <= latency_hiding)
8903 return true;
8904 else
8905 return false;
8906 }
8907
8908 /**
8909 * dce8_program_watermarks - program display watermarks
8910 *
8911 * @rdev: radeon_device pointer
8912 * @radeon_crtc: the selected display controller
8913 * @lb_size: line buffer size
8914 * @num_heads: number of display controllers in use
8915 *
8916 * Calculate and program the display watermarks for the
8917 * selected display controller (CIK).
8918 */
8919 static void dce8_program_watermarks(struct radeon_device *rdev,
8920 struct radeon_crtc *radeon_crtc,
8921 u32 lb_size, u32 num_heads)
8922 {
8923 struct drm_display_mode *mode = &radeon_crtc->base.mode;
8924 struct dce8_wm_params wm_low, wm_high;
8925 u32 pixel_period;
8926 u32 line_time = 0;
8927 u32 latency_watermark_a = 0, latency_watermark_b = 0;
8928 u32 tmp, wm_mask;
8929
8930 if (radeon_crtc->base.enabled && num_heads && mode) {
8931 pixel_period = 1000000 / (u32)mode->clock;
8932 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
8933
8934 /* watermark for high clocks */
8935 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8936 rdev->pm.dpm_enabled) {
8937 wm_high.yclk =
8938 radeon_dpm_get_mclk(rdev, false) * 10;
8939 wm_high.sclk =
8940 radeon_dpm_get_sclk(rdev, false) * 10;
8941 } else {
8942 wm_high.yclk = rdev->pm.current_mclk * 10;
8943 wm_high.sclk = rdev->pm.current_sclk * 10;
8944 }
8945
8946 wm_high.disp_clk = mode->clock;
8947 wm_high.src_width = mode->crtc_hdisplay;
8948 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
8949 wm_high.blank_time = line_time - wm_high.active_time;
8950 wm_high.interlaced = false;
8951 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
8952 wm_high.interlaced = true;
8953 wm_high.vsc = radeon_crtc->vsc;
8954 wm_high.vtaps = 1;
8955 if (radeon_crtc->rmx_type != RMX_OFF)
8956 wm_high.vtaps = 2;
8957 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
8958 wm_high.lb_size = lb_size;
8959 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
8960 wm_high.num_heads = num_heads;
8961
8962 /* set for high clocks */
8963 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
8964
8965 /* possibly force display priority to high */
8966 /* should really do this at mode validation time... */
8967 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
8968 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
8969 !dce8_check_latency_hiding(&wm_high) ||
8970 (rdev->disp_priority == 2)) {
8971 DRM_DEBUG_KMS("force priority to high\n");
8972 }
8973
8974 /* watermark for low clocks */
8975 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8976 rdev->pm.dpm_enabled) {
8977 wm_low.yclk =
8978 radeon_dpm_get_mclk(rdev, true) * 10;
8979 wm_low.sclk =
8980 radeon_dpm_get_sclk(rdev, true) * 10;
8981 } else {
8982 wm_low.yclk = rdev->pm.current_mclk * 10;
8983 wm_low.sclk = rdev->pm.current_sclk * 10;
8984 }
8985
8986 wm_low.disp_clk = mode->clock;
8987 wm_low.src_width = mode->crtc_hdisplay;
8988 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
8989 wm_low.blank_time = line_time - wm_low.active_time;
8990 wm_low.interlaced = false;
8991 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
8992 wm_low.interlaced = true;
8993 wm_low.vsc = radeon_crtc->vsc;
8994 wm_low.vtaps = 1;
8995 if (radeon_crtc->rmx_type != RMX_OFF)
8996 wm_low.vtaps = 2;
8997 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
8998 wm_low.lb_size = lb_size;
8999 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
9000 wm_low.num_heads = num_heads;
9001
9002 /* set for low clocks */
9003 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
9004
9005 /* possibly force display priority to high */
9006 /* should really do this at mode validation time... */
9007 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
9008 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
9009 !dce8_check_latency_hiding(&wm_low) ||
9010 (rdev->disp_priority == 2)) {
9011 DRM_DEBUG_KMS("force priority to high\n");
9012 }
9013 }
9014
9015 /* select wm A */
9016 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9017 tmp = wm_mask;
9018 tmp &= ~LATENCY_WATERMARK_MASK(3);
9019 tmp |= LATENCY_WATERMARK_MASK(1);
9020 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9021 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9022 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
9023 LATENCY_HIGH_WATERMARK(line_time)));
9024 /* select wm B */
9025 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9026 tmp &= ~LATENCY_WATERMARK_MASK(3);
9027 tmp |= LATENCY_WATERMARK_MASK(2);
9028 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9029 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9030 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
9031 LATENCY_HIGH_WATERMARK(line_time)));
9032 /* restore original selection */
9033 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
9034
9035 /* save values for DPM */
9036 radeon_crtc->line_time = line_time;
9037 radeon_crtc->wm_high = latency_watermark_a;
9038 radeon_crtc->wm_low = latency_watermark_b;
9039 }
9040
9041 /**
9042 * dce8_bandwidth_update - program display watermarks
9043 *
9044 * @rdev: radeon_device pointer
9045 *
9046 * Calculate and program the display watermarks and line
9047 * buffer allocation (CIK).
9048 */
9049 void dce8_bandwidth_update(struct radeon_device *rdev)
9050 {
9051 struct drm_display_mode *mode = NULL;
9052 u32 num_heads = 0, lb_size;
9053 int i;
9054
9055 radeon_update_display_priority(rdev);
9056
9057 for (i = 0; i < rdev->num_crtc; i++) {
9058 if (rdev->mode_info.crtcs[i]->base.enabled)
9059 num_heads++;
9060 }
9061 for (i = 0; i < rdev->num_crtc; i++) {
9062 mode = &rdev->mode_info.crtcs[i]->base.mode;
9063 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
9064 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
9065 }
9066 }
9067
9068 /**
9069 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
9070 *
9071 * @rdev: radeon_device pointer
9072 *
9073 * Fetches a GPU clock counter snapshot (SI).
9074 * Returns the 64 bit clock counter snapshot.
9075 */
9076 uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
9077 {
9078 uint64_t clock;
9079
9080 mutex_lock(&rdev->gpu_clock_mutex);
9081 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
9082 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
9083 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
9084 mutex_unlock(&rdev->gpu_clock_mutex);
9085 return clock;
9086 }
9087
9088 static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
9089 u32 cntl_reg, u32 status_reg)
9090 {
9091 int r, i;
9092 struct atom_clock_dividers dividers;
9093 uint32_t tmp;
9094
9095 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9096 clock, false, &dividers);
9097 if (r)
9098 return r;
9099
9100 tmp = RREG32_SMC(cntl_reg);
9101 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
9102 tmp |= dividers.post_divider;
9103 WREG32_SMC(cntl_reg, tmp);
9104
9105 for (i = 0; i < 100; i++) {
9106 if (RREG32_SMC(status_reg) & DCLK_STATUS)
9107 break;
9108 mdelay(10);
9109 }
9110 if (i == 100)
9111 return -ETIMEDOUT;
9112
9113 return 0;
9114 }
9115
9116 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
9117 {
9118 int r = 0;
9119
9120 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
9121 if (r)
9122 return r;
9123
9124 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
9125 return r;
9126 }
9127
9128 int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
9129 {
9130 int r, i;
9131 struct atom_clock_dividers dividers;
9132 u32 tmp;
9133
9134 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9135 ecclk, false, &dividers);
9136 if (r)
9137 return r;
9138
9139 for (i = 0; i < 100; i++) {
9140 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9141 break;
9142 mdelay(10);
9143 }
9144 if (i == 100)
9145 return -ETIMEDOUT;
9146
9147 tmp = RREG32_SMC(CG_ECLK_CNTL);
9148 tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK);
9149 tmp |= dividers.post_divider;
9150 WREG32_SMC(CG_ECLK_CNTL, tmp);
9151
9152 for (i = 0; i < 100; i++) {
9153 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9154 break;
9155 mdelay(10);
9156 }
9157 if (i == 100)
9158 return -ETIMEDOUT;
9159
9160 return 0;
9161 }
9162
9163 static void cik_pcie_gen3_enable(struct radeon_device *rdev)
9164 {
9165 struct pci_dev *root = rdev->pdev->bus->self;
9166 int bridge_pos, gpu_pos;
9167 u32 speed_cntl, mask, current_data_rate;
9168 int ret, i;
9169 u16 tmp16;
9170
9171 if (radeon_pcie_gen2 == 0)
9172 return;
9173
9174 if (rdev->flags & RADEON_IS_IGP)
9175 return;
9176
9177 if (!(rdev->flags & RADEON_IS_PCIE))
9178 return;
9179
9180 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
9181 if (ret != 0)
9182 return;
9183
9184 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
9185 return;
9186
9187 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9188 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
9189 LC_CURRENT_DATA_RATE_SHIFT;
9190 if (mask & DRM_PCIE_SPEED_80) {
9191 if (current_data_rate == 2) {
9192 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
9193 return;
9194 }
9195 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
9196 } else if (mask & DRM_PCIE_SPEED_50) {
9197 if (current_data_rate == 1) {
9198 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
9199 return;
9200 }
9201 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
9202 }
9203
9204 bridge_pos = pci_pcie_cap(root);
9205 if (!bridge_pos)
9206 return;
9207
9208 gpu_pos = pci_pcie_cap(rdev->pdev);
9209 if (!gpu_pos)
9210 return;
9211
9212 if (mask & DRM_PCIE_SPEED_80) {
9213 /* re-try equalization if gen3 is not already enabled */
9214 if (current_data_rate != 2) {
9215 u16 bridge_cfg, gpu_cfg;
9216 u16 bridge_cfg2, gpu_cfg2;
9217 u32 max_lw, current_lw, tmp;
9218
9219 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9220 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9221
9222 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
9223 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9224
9225 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
9226 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9227
9228 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9229 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
9230 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
9231
9232 if (current_lw < max_lw) {
9233 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9234 if (tmp & LC_RENEGOTIATION_SUPPORT) {
9235 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
9236 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
9237 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
9238 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
9239 }
9240 }
9241
9242 for (i = 0; i < 10; i++) {
9243 /* check status */
9244 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
9245 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
9246 break;
9247
9248 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9249 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9250
9251 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
9252 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
9253
9254 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9255 tmp |= LC_SET_QUIESCE;
9256 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9257
9258 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9259 tmp |= LC_REDO_EQ;
9260 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9261
9262 mdelay(100);
9263
9264 /* linkctl */
9265 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
9266 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9267 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
9268 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9269
9270 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
9271 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9272 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
9273 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9274
9275 /* linkctl2 */
9276 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
9277 tmp16 &= ~((1 << 4) | (7 << 9));
9278 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
9279 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
9280
9281 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9282 tmp16 &= ~((1 << 4) | (7 << 9));
9283 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
9284 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9285
9286 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9287 tmp &= ~LC_SET_QUIESCE;
9288 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9289 }
9290 }
9291 }
9292
9293 /* set the link speed */
9294 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
9295 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
9296 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9297
9298 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9299 tmp16 &= ~0xf;
9300 if (mask & DRM_PCIE_SPEED_80)
9301 tmp16 |= 3; /* gen3 */
9302 else if (mask & DRM_PCIE_SPEED_50)
9303 tmp16 |= 2; /* gen2 */
9304 else
9305 tmp16 |= 1; /* gen1 */
9306 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9307
9308 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9309 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
9310 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9311
9312 for (i = 0; i < rdev->usec_timeout; i++) {
9313 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9314 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
9315 break;
9316 udelay(1);
9317 }
9318 }
9319
9320 static void cik_program_aspm(struct radeon_device *rdev)
9321 {
9322 u32 data, orig;
9323 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
9324 bool disable_clkreq = false;
9325
9326 if (radeon_aspm == 0)
9327 return;
9328
9329 /* XXX double check IGPs */
9330 if (rdev->flags & RADEON_IS_IGP)
9331 return;
9332
9333 if (!(rdev->flags & RADEON_IS_PCIE))
9334 return;
9335
9336 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9337 data &= ~LC_XMIT_N_FTS_MASK;
9338 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
9339 if (orig != data)
9340 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
9341
9342 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
9343 data |= LC_GO_TO_RECOVERY;
9344 if (orig != data)
9345 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
9346
9347 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
9348 data |= P_IGNORE_EDB_ERR;
9349 if (orig != data)
9350 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
9351
9352 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9353 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
9354 data |= LC_PMI_TO_L1_DIS;
9355 if (!disable_l0s)
9356 data |= LC_L0S_INACTIVITY(7);
9357
9358 if (!disable_l1) {
9359 data |= LC_L1_INACTIVITY(7);
9360 data &= ~LC_PMI_TO_L1_DIS;
9361 if (orig != data)
9362 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9363
9364 if (!disable_plloff_in_l1) {
9365 bool clk_req_support;
9366
9367 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
9368 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9369 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9370 if (orig != data)
9371 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
9372
9373 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
9374 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9375 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9376 if (orig != data)
9377 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
9378
9379 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
9380 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9381 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9382 if (orig != data)
9383 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
9384
9385 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
9386 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9387 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9388 if (orig != data)
9389 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
9390
9391 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9392 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
9393 data |= LC_DYN_LANES_PWR_STATE(3);
9394 if (orig != data)
9395 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
9396
9397 if (!disable_clkreq) {
9398 struct pci_dev *root = rdev->pdev->bus->self;
9399 u32 lnkcap;
9400
9401 clk_req_support = false;
9402 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
9403 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
9404 clk_req_support = true;
9405 } else {
9406 clk_req_support = false;
9407 }
9408
9409 if (clk_req_support) {
9410 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
9411 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
9412 if (orig != data)
9413 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
9414
9415 orig = data = RREG32_SMC(THM_CLK_CNTL);
9416 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
9417 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
9418 if (orig != data)
9419 WREG32_SMC(THM_CLK_CNTL, data);
9420
9421 orig = data = RREG32_SMC(MISC_CLK_CTRL);
9422 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
9423 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
9424 if (orig != data)
9425 WREG32_SMC(MISC_CLK_CTRL, data);
9426
9427 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
9428 data &= ~BCLK_AS_XCLK;
9429 if (orig != data)
9430 WREG32_SMC(CG_CLKPIN_CNTL, data);
9431
9432 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
9433 data &= ~FORCE_BIF_REFCLK_EN;
9434 if (orig != data)
9435 WREG32_SMC(CG_CLKPIN_CNTL_2, data);
9436
9437 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
9438 data &= ~MPLL_CLKOUT_SEL_MASK;
9439 data |= MPLL_CLKOUT_SEL(4);
9440 if (orig != data)
9441 WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
9442 }
9443 }
9444 } else {
9445 if (orig != data)
9446 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9447 }
9448
9449 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
9450 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
9451 if (orig != data)
9452 WREG32_PCIE_PORT(PCIE_CNTL2, data);
9453
9454 if (!disable_l0s) {
9455 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9456 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
9457 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9458 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
9459 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9460 data &= ~LC_L0S_INACTIVITY_MASK;
9461 if (orig != data)
9462 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9463 }
9464 }
9465 }
9466 }
Linux with added FPC-III support