?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
x86/xen: resume timer irqs early
[linux/fpc-iii.git] / drivers / gpu / drm / radeon / si.c
blob4d41a0dc179645beb2c83857a4765fe566e555f7
1 /*
2 * Copyright 2011 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
27 #include <drm/drmP.h>
28 #include "radeon.h"
29 #include "radeon_asic.h"
30 #include <drm/radeon_drm.h>
31 #include "sid.h"
32 #include "atom.h"
33 #include "si_blit_shaders.h"
34 #include "clearstate_si.h"
35 #include "radeon_ucode.h"
36
37
38 MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
39 MODULE_FIRMWARE("radeon/TAHITI_me.bin");
40 MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
41 MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
42 MODULE_FIRMWARE("radeon/TAHITI_mc2.bin");
43 MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
44 MODULE_FIRMWARE("radeon/TAHITI_smc.bin");
45 MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
46 MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
47 MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
48 MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
49 MODULE_FIRMWARE("radeon/PITCAIRN_mc2.bin");
50 MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
51 MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin");
52 MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
53 MODULE_FIRMWARE("radeon/VERDE_me.bin");
54 MODULE_FIRMWARE("radeon/VERDE_ce.bin");
55 MODULE_FIRMWARE("radeon/VERDE_mc.bin");
56 MODULE_FIRMWARE("radeon/VERDE_mc2.bin");
57 MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
58 MODULE_FIRMWARE("radeon/VERDE_smc.bin");
59 MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
60 MODULE_FIRMWARE("radeon/OLAND_me.bin");
61 MODULE_FIRMWARE("radeon/OLAND_ce.bin");
62 MODULE_FIRMWARE("radeon/OLAND_mc.bin");
63 MODULE_FIRMWARE("radeon/OLAND_mc2.bin");
64 MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
65 MODULE_FIRMWARE("radeon/OLAND_smc.bin");
66 MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
67 MODULE_FIRMWARE("radeon/HAINAN_me.bin");
68 MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
69 MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
70 MODULE_FIRMWARE("radeon/HAINAN_mc2.bin");
71 MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
72 MODULE_FIRMWARE("radeon/HAINAN_smc.bin");
73
74 static void si_pcie_gen3_enable(struct radeon_device *rdev);
75 static void si_program_aspm(struct radeon_device *rdev);
76 extern void sumo_rlc_fini(struct radeon_device *rdev);
77 extern int sumo_rlc_init(struct radeon_device *rdev);
78 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
79 extern void r600_ih_ring_fini(struct radeon_device *rdev);
80 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
81 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
82 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
83 extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev);
84 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
85 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
86 extern void si_dma_vm_set_page(struct radeon_device *rdev,
87 struct radeon_ib *ib,
88 uint64_t pe,
89 uint64_t addr, unsigned count,
90 uint32_t incr, uint32_t flags);
91 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
92 bool enable);
93 static void si_fini_pg(struct radeon_device *rdev);
94 static void si_fini_cg(struct radeon_device *rdev);
95 static void si_rlc_stop(struct radeon_device *rdev);
96
97 static const u32 verde_rlc_save_restore_register_list[] =
98 {
99 (0x8000 << 16) | (0x98f4 >> 2),
100 0x00000000,
101 (0x8040 << 16) | (0x98f4 >> 2),
102 0x00000000,
103 (0x8000 << 16) | (0xe80 >> 2),
104 0x00000000,
105 (0x8040 << 16) | (0xe80 >> 2),
106 0x00000000,
107 (0x8000 << 16) | (0x89bc >> 2),
108 0x00000000,
109 (0x8040 << 16) | (0x89bc >> 2),
110 0x00000000,
111 (0x8000 << 16) | (0x8c1c >> 2),
112 0x00000000,
113 (0x8040 << 16) | (0x8c1c >> 2),
114 0x00000000,
115 (0x9c00 << 16) | (0x98f0 >> 2),
116 0x00000000,
117 (0x9c00 << 16) | (0xe7c >> 2),
118 0x00000000,
119 (0x8000 << 16) | (0x9148 >> 2),
120 0x00000000,
121 (0x8040 << 16) | (0x9148 >> 2),
122 0x00000000,
123 (0x9c00 << 16) | (0x9150 >> 2),
124 0x00000000,
125 (0x9c00 << 16) | (0x897c >> 2),
126 0x00000000,
127 (0x9c00 << 16) | (0x8d8c >> 2),
128 0x00000000,
129 (0x9c00 << 16) | (0xac54 >> 2),
130 0X00000000,
131 0x3,
132 (0x9c00 << 16) | (0x98f8 >> 2),
133 0x00000000,
134 (0x9c00 << 16) | (0x9910 >> 2),
135 0x00000000,
136 (0x9c00 << 16) | (0x9914 >> 2),
137 0x00000000,
138 (0x9c00 << 16) | (0x9918 >> 2),
139 0x00000000,
140 (0x9c00 << 16) | (0x991c >> 2),
141 0x00000000,
142 (0x9c00 << 16) | (0x9920 >> 2),
143 0x00000000,
144 (0x9c00 << 16) | (0x9924 >> 2),
145 0x00000000,
146 (0x9c00 << 16) | (0x9928 >> 2),
147 0x00000000,
148 (0x9c00 << 16) | (0x992c >> 2),
149 0x00000000,
150 (0x9c00 << 16) | (0x9930 >> 2),
151 0x00000000,
152 (0x9c00 << 16) | (0x9934 >> 2),
153 0x00000000,
154 (0x9c00 << 16) | (0x9938 >> 2),
155 0x00000000,
156 (0x9c00 << 16) | (0x993c >> 2),
157 0x00000000,
158 (0x9c00 << 16) | (0x9940 >> 2),
159 0x00000000,
160 (0x9c00 << 16) | (0x9944 >> 2),
161 0x00000000,
162 (0x9c00 << 16) | (0x9948 >> 2),
163 0x00000000,
164 (0x9c00 << 16) | (0x994c >> 2),
165 0x00000000,
166 (0x9c00 << 16) | (0x9950 >> 2),
167 0x00000000,
168 (0x9c00 << 16) | (0x9954 >> 2),
169 0x00000000,
170 (0x9c00 << 16) | (0x9958 >> 2),
171 0x00000000,
172 (0x9c00 << 16) | (0x995c >> 2),
173 0x00000000,
174 (0x9c00 << 16) | (0x9960 >> 2),
175 0x00000000,
176 (0x9c00 << 16) | (0x9964 >> 2),
177 0x00000000,
178 (0x9c00 << 16) | (0x9968 >> 2),
179 0x00000000,
180 (0x9c00 << 16) | (0x996c >> 2),
181 0x00000000,
182 (0x9c00 << 16) | (0x9970 >> 2),
183 0x00000000,
184 (0x9c00 << 16) | (0x9974 >> 2),
185 0x00000000,
186 (0x9c00 << 16) | (0x9978 >> 2),
187 0x00000000,
188 (0x9c00 << 16) | (0x997c >> 2),
189 0x00000000,
190 (0x9c00 << 16) | (0x9980 >> 2),
191 0x00000000,
192 (0x9c00 << 16) | (0x9984 >> 2),
193 0x00000000,
194 (0x9c00 << 16) | (0x9988 >> 2),
195 0x00000000,
196 (0x9c00 << 16) | (0x998c >> 2),
197 0x00000000,
198 (0x9c00 << 16) | (0x8c00 >> 2),
199 0x00000000,
200 (0x9c00 << 16) | (0x8c14 >> 2),
201 0x00000000,
202 (0x9c00 << 16) | (0x8c04 >> 2),
203 0x00000000,
204 (0x9c00 << 16) | (0x8c08 >> 2),
205 0x00000000,
206 (0x8000 << 16) | (0x9b7c >> 2),
207 0x00000000,
208 (0x8040 << 16) | (0x9b7c >> 2),
209 0x00000000,
210 (0x8000 << 16) | (0xe84 >> 2),
211 0x00000000,
212 (0x8040 << 16) | (0xe84 >> 2),
213 0x00000000,
214 (0x8000 << 16) | (0x89c0 >> 2),
215 0x00000000,
216 (0x8040 << 16) | (0x89c0 >> 2),
217 0x00000000,
218 (0x8000 << 16) | (0x914c >> 2),
219 0x00000000,
220 (0x8040 << 16) | (0x914c >> 2),
221 0x00000000,
222 (0x8000 << 16) | (0x8c20 >> 2),
223 0x00000000,
224 (0x8040 << 16) | (0x8c20 >> 2),
225 0x00000000,
226 (0x8000 << 16) | (0x9354 >> 2),
227 0x00000000,
228 (0x8040 << 16) | (0x9354 >> 2),
229 0x00000000,
230 (0x9c00 << 16) | (0x9060 >> 2),
231 0x00000000,
232 (0x9c00 << 16) | (0x9364 >> 2),
233 0x00000000,
234 (0x9c00 << 16) | (0x9100 >> 2),
235 0x00000000,
236 (0x9c00 << 16) | (0x913c >> 2),
237 0x00000000,
238 (0x8000 << 16) | (0x90e0 >> 2),
239 0x00000000,
240 (0x8000 << 16) | (0x90e4 >> 2),
241 0x00000000,
242 (0x8000 << 16) | (0x90e8 >> 2),
243 0x00000000,
244 (0x8040 << 16) | (0x90e0 >> 2),
245 0x00000000,
246 (0x8040 << 16) | (0x90e4 >> 2),
247 0x00000000,
248 (0x8040 << 16) | (0x90e8 >> 2),
249 0x00000000,
250 (0x9c00 << 16) | (0x8bcc >> 2),
251 0x00000000,
252 (0x9c00 << 16) | (0x8b24 >> 2),
253 0x00000000,
254 (0x9c00 << 16) | (0x88c4 >> 2),
255 0x00000000,
256 (0x9c00 << 16) | (0x8e50 >> 2),
257 0x00000000,
258 (0x9c00 << 16) | (0x8c0c >> 2),
259 0x00000000,
260 (0x9c00 << 16) | (0x8e58 >> 2),
261 0x00000000,
262 (0x9c00 << 16) | (0x8e5c >> 2),
263 0x00000000,
264 (0x9c00 << 16) | (0x9508 >> 2),
265 0x00000000,
266 (0x9c00 << 16) | (0x950c >> 2),
267 0x00000000,
268 (0x9c00 << 16) | (0x9494 >> 2),
269 0x00000000,
270 (0x9c00 << 16) | (0xac0c >> 2),
271 0x00000000,
272 (0x9c00 << 16) | (0xac10 >> 2),
273 0x00000000,
274 (0x9c00 << 16) | (0xac14 >> 2),
275 0x00000000,
276 (0x9c00 << 16) | (0xae00 >> 2),
277 0x00000000,
278 (0x9c00 << 16) | (0xac08 >> 2),
279 0x00000000,
280 (0x9c00 << 16) | (0x88d4 >> 2),
281 0x00000000,
282 (0x9c00 << 16) | (0x88c8 >> 2),
283 0x00000000,
284 (0x9c00 << 16) | (0x88cc >> 2),
285 0x00000000,
286 (0x9c00 << 16) | (0x89b0 >> 2),
287 0x00000000,
288 (0x9c00 << 16) | (0x8b10 >> 2),
289 0x00000000,
290 (0x9c00 << 16) | (0x8a14 >> 2),
291 0x00000000,
292 (0x9c00 << 16) | (0x9830 >> 2),
293 0x00000000,
294 (0x9c00 << 16) | (0x9834 >> 2),
295 0x00000000,
296 (0x9c00 << 16) | (0x9838 >> 2),
297 0x00000000,
298 (0x9c00 << 16) | (0x9a10 >> 2),
299 0x00000000,
300 (0x8000 << 16) | (0x9870 >> 2),
301 0x00000000,
302 (0x8000 << 16) | (0x9874 >> 2),
303 0x00000000,
304 (0x8001 << 16) | (0x9870 >> 2),
305 0x00000000,
306 (0x8001 << 16) | (0x9874 >> 2),
307 0x00000000,
308 (0x8040 << 16) | (0x9870 >> 2),
309 0x00000000,
310 (0x8040 << 16) | (0x9874 >> 2),
311 0x00000000,
312 (0x8041 << 16) | (0x9870 >> 2),
313 0x00000000,
314 (0x8041 << 16) | (0x9874 >> 2),
315 0x00000000,
316 0x00000000
317 };
318
319 static const u32 tahiti_golden_rlc_registers[] =
320 {
321 0xc424, 0xffffffff, 0x00601005,
322 0xc47c, 0xffffffff, 0x10104040,
323 0xc488, 0xffffffff, 0x0100000a,
324 0xc314, 0xffffffff, 0x00000800,
325 0xc30c, 0xffffffff, 0x800000f4,
326 0xf4a8, 0xffffffff, 0x00000000
327 };
328
329 static const u32 tahiti_golden_registers[] =
330 {
331 0x9a10, 0x00010000, 0x00018208,
332 0x9830, 0xffffffff, 0x00000000,
333 0x9834, 0xf00fffff, 0x00000400,
334 0x9838, 0x0002021c, 0x00020200,
335 0xc78, 0x00000080, 0x00000000,
336 0xd030, 0x000300c0, 0x00800040,
337 0xd830, 0x000300c0, 0x00800040,
338 0x5bb0, 0x000000f0, 0x00000070,
339 0x5bc0, 0x00200000, 0x50100000,
340 0x7030, 0x31000311, 0x00000011,
341 0x277c, 0x00000003, 0x000007ff,
342 0x240c, 0x000007ff, 0x00000000,
343 0x8a14, 0xf000001f, 0x00000007,
344 0x8b24, 0xffffffff, 0x00ffffff,
345 0x8b10, 0x0000ff0f, 0x00000000,
346 0x28a4c, 0x07ffffff, 0x4e000000,
347 0x28350, 0x3f3f3fff, 0x2a00126a,
348 0x30, 0x000000ff, 0x0040,
349 0x34, 0x00000040, 0x00004040,
350 0x9100, 0x07ffffff, 0x03000000,
351 0x8e88, 0x01ff1f3f, 0x00000000,
352 0x8e84, 0x01ff1f3f, 0x00000000,
353 0x9060, 0x0000007f, 0x00000020,
354 0x9508, 0x00010000, 0x00010000,
355 0xac14, 0x00000200, 0x000002fb,
356 0xac10, 0xffffffff, 0x0000543b,
357 0xac0c, 0xffffffff, 0xa9210876,
358 0x88d0, 0xffffffff, 0x000fff40,
359 0x88d4, 0x0000001f, 0x00000010,
360 0x1410, 0x20000000, 0x20fffed8,
361 0x15c0, 0x000c0fc0, 0x000c0400
362 };
363
364 static const u32 tahiti_golden_registers2[] =
365 {
366 0xc64, 0x00000001, 0x00000001
367 };
368
369 static const u32 pitcairn_golden_rlc_registers[] =
370 {
371 0xc424, 0xffffffff, 0x00601004,
372 0xc47c, 0xffffffff, 0x10102020,
373 0xc488, 0xffffffff, 0x01000020,
374 0xc314, 0xffffffff, 0x00000800,
375 0xc30c, 0xffffffff, 0x800000a4
376 };
377
378 static const u32 pitcairn_golden_registers[] =
379 {
380 0x9a10, 0x00010000, 0x00018208,
381 0x9830, 0xffffffff, 0x00000000,
382 0x9834, 0xf00fffff, 0x00000400,
383 0x9838, 0x0002021c, 0x00020200,
384 0xc78, 0x00000080, 0x00000000,
385 0xd030, 0x000300c0, 0x00800040,
386 0xd830, 0x000300c0, 0x00800040,
387 0x5bb0, 0x000000f0, 0x00000070,
388 0x5bc0, 0x00200000, 0x50100000,
389 0x7030, 0x31000311, 0x00000011,
390 0x2ae4, 0x00073ffe, 0x000022a2,
391 0x240c, 0x000007ff, 0x00000000,
392 0x8a14, 0xf000001f, 0x00000007,
393 0x8b24, 0xffffffff, 0x00ffffff,
394 0x8b10, 0x0000ff0f, 0x00000000,
395 0x28a4c, 0x07ffffff, 0x4e000000,
396 0x28350, 0x3f3f3fff, 0x2a00126a,
397 0x30, 0x000000ff, 0x0040,
398 0x34, 0x00000040, 0x00004040,
399 0x9100, 0x07ffffff, 0x03000000,
400 0x9060, 0x0000007f, 0x00000020,
401 0x9508, 0x00010000, 0x00010000,
402 0xac14, 0x000003ff, 0x000000f7,
403 0xac10, 0xffffffff, 0x00000000,
404 0xac0c, 0xffffffff, 0x32761054,
405 0x88d4, 0x0000001f, 0x00000010,
406 0x15c0, 0x000c0fc0, 0x000c0400
407 };
408
409 static const u32 verde_golden_rlc_registers[] =
410 {
411 0xc424, 0xffffffff, 0x033f1005,
412 0xc47c, 0xffffffff, 0x10808020,
413 0xc488, 0xffffffff, 0x00800008,
414 0xc314, 0xffffffff, 0x00001000,
415 0xc30c, 0xffffffff, 0x80010014
416 };
417
418 static const u32 verde_golden_registers[] =
419 {
420 0x9a10, 0x00010000, 0x00018208,
421 0x9830, 0xffffffff, 0x00000000,
422 0x9834, 0xf00fffff, 0x00000400,
423 0x9838, 0x0002021c, 0x00020200,
424 0xc78, 0x00000080, 0x00000000,
425 0xd030, 0x000300c0, 0x00800040,
426 0xd030, 0x000300c0, 0x00800040,
427 0xd830, 0x000300c0, 0x00800040,
428 0xd830, 0x000300c0, 0x00800040,
429 0x5bb0, 0x000000f0, 0x00000070,
430 0x5bc0, 0x00200000, 0x50100000,
431 0x7030, 0x31000311, 0x00000011,
432 0x2ae4, 0x00073ffe, 0x000022a2,
433 0x2ae4, 0x00073ffe, 0x000022a2,
434 0x2ae4, 0x00073ffe, 0x000022a2,
435 0x240c, 0x000007ff, 0x00000000,
436 0x240c, 0x000007ff, 0x00000000,
437 0x240c, 0x000007ff, 0x00000000,
438 0x8a14, 0xf000001f, 0x00000007,
439 0x8a14, 0xf000001f, 0x00000007,
440 0x8a14, 0xf000001f, 0x00000007,
441 0x8b24, 0xffffffff, 0x00ffffff,
442 0x8b10, 0x0000ff0f, 0x00000000,
443 0x28a4c, 0x07ffffff, 0x4e000000,
444 0x28350, 0x3f3f3fff, 0x0000124a,
445 0x28350, 0x3f3f3fff, 0x0000124a,
446 0x28350, 0x3f3f3fff, 0x0000124a,
447 0x30, 0x000000ff, 0x0040,
448 0x34, 0x00000040, 0x00004040,
449 0x9100, 0x07ffffff, 0x03000000,
450 0x9100, 0x07ffffff, 0x03000000,
451 0x8e88, 0x01ff1f3f, 0x00000000,
452 0x8e88, 0x01ff1f3f, 0x00000000,
453 0x8e88, 0x01ff1f3f, 0x00000000,
454 0x8e84, 0x01ff1f3f, 0x00000000,
455 0x8e84, 0x01ff1f3f, 0x00000000,
456 0x8e84, 0x01ff1f3f, 0x00000000,
457 0x9060, 0x0000007f, 0x00000020,
458 0x9508, 0x00010000, 0x00010000,
459 0xac14, 0x000003ff, 0x00000003,
460 0xac14, 0x000003ff, 0x00000003,
461 0xac14, 0x000003ff, 0x00000003,
462 0xac10, 0xffffffff, 0x00000000,
463 0xac10, 0xffffffff, 0x00000000,
464 0xac10, 0xffffffff, 0x00000000,
465 0xac0c, 0xffffffff, 0x00001032,
466 0xac0c, 0xffffffff, 0x00001032,
467 0xac0c, 0xffffffff, 0x00001032,
468 0x88d4, 0x0000001f, 0x00000010,
469 0x88d4, 0x0000001f, 0x00000010,
470 0x88d4, 0x0000001f, 0x00000010,
471 0x15c0, 0x000c0fc0, 0x000c0400
472 };
473
474 static const u32 oland_golden_rlc_registers[] =
475 {
476 0xc424, 0xffffffff, 0x00601005,
477 0xc47c, 0xffffffff, 0x10104040,
478 0xc488, 0xffffffff, 0x0100000a,
479 0xc314, 0xffffffff, 0x00000800,
480 0xc30c, 0xffffffff, 0x800000f4
481 };
482
483 static const u32 oland_golden_registers[] =
484 {
485 0x9a10, 0x00010000, 0x00018208,
486 0x9830, 0xffffffff, 0x00000000,
487 0x9834, 0xf00fffff, 0x00000400,
488 0x9838, 0x0002021c, 0x00020200,
489 0xc78, 0x00000080, 0x00000000,
490 0xd030, 0x000300c0, 0x00800040,
491 0xd830, 0x000300c0, 0x00800040,
492 0x5bb0, 0x000000f0, 0x00000070,
493 0x5bc0, 0x00200000, 0x50100000,
494 0x7030, 0x31000311, 0x00000011,
495 0x2ae4, 0x00073ffe, 0x000022a2,
496 0x240c, 0x000007ff, 0x00000000,
497 0x8a14, 0xf000001f, 0x00000007,
498 0x8b24, 0xffffffff, 0x00ffffff,
499 0x8b10, 0x0000ff0f, 0x00000000,
500 0x28a4c, 0x07ffffff, 0x4e000000,
501 0x28350, 0x3f3f3fff, 0x00000082,
502 0x30, 0x000000ff, 0x0040,
503 0x34, 0x00000040, 0x00004040,
504 0x9100, 0x07ffffff, 0x03000000,
505 0x9060, 0x0000007f, 0x00000020,
506 0x9508, 0x00010000, 0x00010000,
507 0xac14, 0x000003ff, 0x000000f3,
508 0xac10, 0xffffffff, 0x00000000,
509 0xac0c, 0xffffffff, 0x00003210,
510 0x88d4, 0x0000001f, 0x00000010,
511 0x15c0, 0x000c0fc0, 0x000c0400
512 };
513
514 static const u32 hainan_golden_registers[] =
515 {
516 0x9a10, 0x00010000, 0x00018208,
517 0x9830, 0xffffffff, 0x00000000,
518 0x9834, 0xf00fffff, 0x00000400,
519 0x9838, 0x0002021c, 0x00020200,
520 0xd0c0, 0xff000fff, 0x00000100,
521 0xd030, 0x000300c0, 0x00800040,
522 0xd8c0, 0xff000fff, 0x00000100,
523 0xd830, 0x000300c0, 0x00800040,
524 0x2ae4, 0x00073ffe, 0x000022a2,
525 0x240c, 0x000007ff, 0x00000000,
526 0x8a14, 0xf000001f, 0x00000007,
527 0x8b24, 0xffffffff, 0x00ffffff,
528 0x8b10, 0x0000ff0f, 0x00000000,
529 0x28a4c, 0x07ffffff, 0x4e000000,
530 0x28350, 0x3f3f3fff, 0x00000000,
531 0x30, 0x000000ff, 0x0040,
532 0x34, 0x00000040, 0x00004040,
533 0x9100, 0x03e00000, 0x03600000,
534 0x9060, 0x0000007f, 0x00000020,
535 0x9508, 0x00010000, 0x00010000,
536 0xac14, 0x000003ff, 0x000000f1,
537 0xac10, 0xffffffff, 0x00000000,
538 0xac0c, 0xffffffff, 0x00003210,
539 0x88d4, 0x0000001f, 0x00000010,
540 0x15c0, 0x000c0fc0, 0x000c0400
541 };
542
543 static const u32 hainan_golden_registers2[] =
544 {
545 0x98f8, 0xffffffff, 0x02010001
546 };
547
548 static const u32 tahiti_mgcg_cgcg_init[] =
549 {
550 0xc400, 0xffffffff, 0xfffffffc,
551 0x802c, 0xffffffff, 0xe0000000,
552 0x9a60, 0xffffffff, 0x00000100,
553 0x92a4, 0xffffffff, 0x00000100,
554 0xc164, 0xffffffff, 0x00000100,
555 0x9774, 0xffffffff, 0x00000100,
556 0x8984, 0xffffffff, 0x06000100,
557 0x8a18, 0xffffffff, 0x00000100,
558 0x92a0, 0xffffffff, 0x00000100,
559 0xc380, 0xffffffff, 0x00000100,
560 0x8b28, 0xffffffff, 0x00000100,
561 0x9144, 0xffffffff, 0x00000100,
562 0x8d88, 0xffffffff, 0x00000100,
563 0x8d8c, 0xffffffff, 0x00000100,
564 0x9030, 0xffffffff, 0x00000100,
565 0x9034, 0xffffffff, 0x00000100,
566 0x9038, 0xffffffff, 0x00000100,
567 0x903c, 0xffffffff, 0x00000100,
568 0xad80, 0xffffffff, 0x00000100,
569 0xac54, 0xffffffff, 0x00000100,
570 0x897c, 0xffffffff, 0x06000100,
571 0x9868, 0xffffffff, 0x00000100,
572 0x9510, 0xffffffff, 0x00000100,
573 0xaf04, 0xffffffff, 0x00000100,
574 0xae04, 0xffffffff, 0x00000100,
575 0x949c, 0xffffffff, 0x00000100,
576 0x802c, 0xffffffff, 0xe0000000,
577 0x9160, 0xffffffff, 0x00010000,
578 0x9164, 0xffffffff, 0x00030002,
579 0x9168, 0xffffffff, 0x00040007,
580 0x916c, 0xffffffff, 0x00060005,
581 0x9170, 0xffffffff, 0x00090008,
582 0x9174, 0xffffffff, 0x00020001,
583 0x9178, 0xffffffff, 0x00040003,
584 0x917c, 0xffffffff, 0x00000007,
585 0x9180, 0xffffffff, 0x00060005,
586 0x9184, 0xffffffff, 0x00090008,
587 0x9188, 0xffffffff, 0x00030002,
588 0x918c, 0xffffffff, 0x00050004,
589 0x9190, 0xffffffff, 0x00000008,
590 0x9194, 0xffffffff, 0x00070006,
591 0x9198, 0xffffffff, 0x000a0009,
592 0x919c, 0xffffffff, 0x00040003,
593 0x91a0, 0xffffffff, 0x00060005,
594 0x91a4, 0xffffffff, 0x00000009,
595 0x91a8, 0xffffffff, 0x00080007,
596 0x91ac, 0xffffffff, 0x000b000a,
597 0x91b0, 0xffffffff, 0x00050004,
598 0x91b4, 0xffffffff, 0x00070006,
599 0x91b8, 0xffffffff, 0x0008000b,
600 0x91bc, 0xffffffff, 0x000a0009,
601 0x91c0, 0xffffffff, 0x000d000c,
602 0x91c4, 0xffffffff, 0x00060005,
603 0x91c8, 0xffffffff, 0x00080007,
604 0x91cc, 0xffffffff, 0x0000000b,
605 0x91d0, 0xffffffff, 0x000a0009,
606 0x91d4, 0xffffffff, 0x000d000c,
607 0x91d8, 0xffffffff, 0x00070006,
608 0x91dc, 0xffffffff, 0x00090008,
609 0x91e0, 0xffffffff, 0x0000000c,
610 0x91e4, 0xffffffff, 0x000b000a,
611 0x91e8, 0xffffffff, 0x000e000d,
612 0x91ec, 0xffffffff, 0x00080007,
613 0x91f0, 0xffffffff, 0x000a0009,
614 0x91f4, 0xffffffff, 0x0000000d,
615 0x91f8, 0xffffffff, 0x000c000b,
616 0x91fc, 0xffffffff, 0x000f000e,
617 0x9200, 0xffffffff, 0x00090008,
618 0x9204, 0xffffffff, 0x000b000a,
619 0x9208, 0xffffffff, 0x000c000f,
620 0x920c, 0xffffffff, 0x000e000d,
621 0x9210, 0xffffffff, 0x00110010,
622 0x9214, 0xffffffff, 0x000a0009,
623 0x9218, 0xffffffff, 0x000c000b,
624 0x921c, 0xffffffff, 0x0000000f,
625 0x9220, 0xffffffff, 0x000e000d,
626 0x9224, 0xffffffff, 0x00110010,
627 0x9228, 0xffffffff, 0x000b000a,
628 0x922c, 0xffffffff, 0x000d000c,
629 0x9230, 0xffffffff, 0x00000010,
630 0x9234, 0xffffffff, 0x000f000e,
631 0x9238, 0xffffffff, 0x00120011,
632 0x923c, 0xffffffff, 0x000c000b,
633 0x9240, 0xffffffff, 0x000e000d,
634 0x9244, 0xffffffff, 0x00000011,
635 0x9248, 0xffffffff, 0x0010000f,
636 0x924c, 0xffffffff, 0x00130012,
637 0x9250, 0xffffffff, 0x000d000c,
638 0x9254, 0xffffffff, 0x000f000e,
639 0x9258, 0xffffffff, 0x00100013,
640 0x925c, 0xffffffff, 0x00120011,
641 0x9260, 0xffffffff, 0x00150014,
642 0x9264, 0xffffffff, 0x000e000d,
643 0x9268, 0xffffffff, 0x0010000f,
644 0x926c, 0xffffffff, 0x00000013,
645 0x9270, 0xffffffff, 0x00120011,
646 0x9274, 0xffffffff, 0x00150014,
647 0x9278, 0xffffffff, 0x000f000e,
648 0x927c, 0xffffffff, 0x00110010,
649 0x9280, 0xffffffff, 0x00000014,
650 0x9284, 0xffffffff, 0x00130012,
651 0x9288, 0xffffffff, 0x00160015,
652 0x928c, 0xffffffff, 0x0010000f,
653 0x9290, 0xffffffff, 0x00120011,
654 0x9294, 0xffffffff, 0x00000015,
655 0x9298, 0xffffffff, 0x00140013,
656 0x929c, 0xffffffff, 0x00170016,
657 0x9150, 0xffffffff, 0x96940200,
658 0x8708, 0xffffffff, 0x00900100,
659 0xc478, 0xffffffff, 0x00000080,
660 0xc404, 0xffffffff, 0x0020003f,
661 0x30, 0xffffffff, 0x0000001c,
662 0x34, 0x000f0000, 0x000f0000,
663 0x160c, 0xffffffff, 0x00000100,
664 0x1024, 0xffffffff, 0x00000100,
665 0x102c, 0x00000101, 0x00000000,
666 0x20a8, 0xffffffff, 0x00000104,
667 0x264c, 0x000c0000, 0x000c0000,
668 0x2648, 0x000c0000, 0x000c0000,
669 0x55e4, 0xff000fff, 0x00000100,
670 0x55e8, 0x00000001, 0x00000001,
671 0x2f50, 0x00000001, 0x00000001,
672 0x30cc, 0xc0000fff, 0x00000104,
673 0xc1e4, 0x00000001, 0x00000001,
674 0xd0c0, 0xfffffff0, 0x00000100,
675 0xd8c0, 0xfffffff0, 0x00000100
676 };
677
678 static const u32 pitcairn_mgcg_cgcg_init[] =
679 {
680 0xc400, 0xffffffff, 0xfffffffc,
681 0x802c, 0xffffffff, 0xe0000000,
682 0x9a60, 0xffffffff, 0x00000100,
683 0x92a4, 0xffffffff, 0x00000100,
684 0xc164, 0xffffffff, 0x00000100,
685 0x9774, 0xffffffff, 0x00000100,
686 0x8984, 0xffffffff, 0x06000100,
687 0x8a18, 0xffffffff, 0x00000100,
688 0x92a0, 0xffffffff, 0x00000100,
689 0xc380, 0xffffffff, 0x00000100,
690 0x8b28, 0xffffffff, 0x00000100,
691 0x9144, 0xffffffff, 0x00000100,
692 0x8d88, 0xffffffff, 0x00000100,
693 0x8d8c, 0xffffffff, 0x00000100,
694 0x9030, 0xffffffff, 0x00000100,
695 0x9034, 0xffffffff, 0x00000100,
696 0x9038, 0xffffffff, 0x00000100,
697 0x903c, 0xffffffff, 0x00000100,
698 0xad80, 0xffffffff, 0x00000100,
699 0xac54, 0xffffffff, 0x00000100,
700 0x897c, 0xffffffff, 0x06000100,
701 0x9868, 0xffffffff, 0x00000100,
702 0x9510, 0xffffffff, 0x00000100,
703 0xaf04, 0xffffffff, 0x00000100,
704 0xae04, 0xffffffff, 0x00000100,
705 0x949c, 0xffffffff, 0x00000100,
706 0x802c, 0xffffffff, 0xe0000000,
707 0x9160, 0xffffffff, 0x00010000,
708 0x9164, 0xffffffff, 0x00030002,
709 0x9168, 0xffffffff, 0x00040007,
710 0x916c, 0xffffffff, 0x00060005,
711 0x9170, 0xffffffff, 0x00090008,
712 0x9174, 0xffffffff, 0x00020001,
713 0x9178, 0xffffffff, 0x00040003,
714 0x917c, 0xffffffff, 0x00000007,
715 0x9180, 0xffffffff, 0x00060005,
716 0x9184, 0xffffffff, 0x00090008,
717 0x9188, 0xffffffff, 0x00030002,
718 0x918c, 0xffffffff, 0x00050004,
719 0x9190, 0xffffffff, 0x00000008,
720 0x9194, 0xffffffff, 0x00070006,
721 0x9198, 0xffffffff, 0x000a0009,
722 0x919c, 0xffffffff, 0x00040003,
723 0x91a0, 0xffffffff, 0x00060005,
724 0x91a4, 0xffffffff, 0x00000009,
725 0x91a8, 0xffffffff, 0x00080007,
726 0x91ac, 0xffffffff, 0x000b000a,
727 0x91b0, 0xffffffff, 0x00050004,
728 0x91b4, 0xffffffff, 0x00070006,
729 0x91b8, 0xffffffff, 0x0008000b,
730 0x91bc, 0xffffffff, 0x000a0009,
731 0x91c0, 0xffffffff, 0x000d000c,
732 0x9200, 0xffffffff, 0x00090008,
733 0x9204, 0xffffffff, 0x000b000a,
734 0x9208, 0xffffffff, 0x000c000f,
735 0x920c, 0xffffffff, 0x000e000d,
736 0x9210, 0xffffffff, 0x00110010,
737 0x9214, 0xffffffff, 0x000a0009,
738 0x9218, 0xffffffff, 0x000c000b,
739 0x921c, 0xffffffff, 0x0000000f,
740 0x9220, 0xffffffff, 0x000e000d,
741 0x9224, 0xffffffff, 0x00110010,
742 0x9228, 0xffffffff, 0x000b000a,
743 0x922c, 0xffffffff, 0x000d000c,
744 0x9230, 0xffffffff, 0x00000010,
745 0x9234, 0xffffffff, 0x000f000e,
746 0x9238, 0xffffffff, 0x00120011,
747 0x923c, 0xffffffff, 0x000c000b,
748 0x9240, 0xffffffff, 0x000e000d,
749 0x9244, 0xffffffff, 0x00000011,
750 0x9248, 0xffffffff, 0x0010000f,
751 0x924c, 0xffffffff, 0x00130012,
752 0x9250, 0xffffffff, 0x000d000c,
753 0x9254, 0xffffffff, 0x000f000e,
754 0x9258, 0xffffffff, 0x00100013,
755 0x925c, 0xffffffff, 0x00120011,
756 0x9260, 0xffffffff, 0x00150014,
757 0x9150, 0xffffffff, 0x96940200,
758 0x8708, 0xffffffff, 0x00900100,
759 0xc478, 0xffffffff, 0x00000080,
760 0xc404, 0xffffffff, 0x0020003f,
761 0x30, 0xffffffff, 0x0000001c,
762 0x34, 0x000f0000, 0x000f0000,
763 0x160c, 0xffffffff, 0x00000100,
764 0x1024, 0xffffffff, 0x00000100,
765 0x102c, 0x00000101, 0x00000000,
766 0x20a8, 0xffffffff, 0x00000104,
767 0x55e4, 0xff000fff, 0x00000100,
768 0x55e8, 0x00000001, 0x00000001,
769 0x2f50, 0x00000001, 0x00000001,
770 0x30cc, 0xc0000fff, 0x00000104,
771 0xc1e4, 0x00000001, 0x00000001,
772 0xd0c0, 0xfffffff0, 0x00000100,
773 0xd8c0, 0xfffffff0, 0x00000100
774 };
775
776 static const u32 verde_mgcg_cgcg_init[] =
777 {
778 0xc400, 0xffffffff, 0xfffffffc,
779 0x802c, 0xffffffff, 0xe0000000,
780 0x9a60, 0xffffffff, 0x00000100,
781 0x92a4, 0xffffffff, 0x00000100,
782 0xc164, 0xffffffff, 0x00000100,
783 0x9774, 0xffffffff, 0x00000100,
784 0x8984, 0xffffffff, 0x06000100,
785 0x8a18, 0xffffffff, 0x00000100,
786 0x92a0, 0xffffffff, 0x00000100,
787 0xc380, 0xffffffff, 0x00000100,
788 0x8b28, 0xffffffff, 0x00000100,
789 0x9144, 0xffffffff, 0x00000100,
790 0x8d88, 0xffffffff, 0x00000100,
791 0x8d8c, 0xffffffff, 0x00000100,
792 0x9030, 0xffffffff, 0x00000100,
793 0x9034, 0xffffffff, 0x00000100,
794 0x9038, 0xffffffff, 0x00000100,
795 0x903c, 0xffffffff, 0x00000100,
796 0xad80, 0xffffffff, 0x00000100,
797 0xac54, 0xffffffff, 0x00000100,
798 0x897c, 0xffffffff, 0x06000100,
799 0x9868, 0xffffffff, 0x00000100,
800 0x9510, 0xffffffff, 0x00000100,
801 0xaf04, 0xffffffff, 0x00000100,
802 0xae04, 0xffffffff, 0x00000100,
803 0x949c, 0xffffffff, 0x00000100,
804 0x802c, 0xffffffff, 0xe0000000,
805 0x9160, 0xffffffff, 0x00010000,
806 0x9164, 0xffffffff, 0x00030002,
807 0x9168, 0xffffffff, 0x00040007,
808 0x916c, 0xffffffff, 0x00060005,
809 0x9170, 0xffffffff, 0x00090008,
810 0x9174, 0xffffffff, 0x00020001,
811 0x9178, 0xffffffff, 0x00040003,
812 0x917c, 0xffffffff, 0x00000007,
813 0x9180, 0xffffffff, 0x00060005,
814 0x9184, 0xffffffff, 0x00090008,
815 0x9188, 0xffffffff, 0x00030002,
816 0x918c, 0xffffffff, 0x00050004,
817 0x9190, 0xffffffff, 0x00000008,
818 0x9194, 0xffffffff, 0x00070006,
819 0x9198, 0xffffffff, 0x000a0009,
820 0x919c, 0xffffffff, 0x00040003,
821 0x91a0, 0xffffffff, 0x00060005,
822 0x91a4, 0xffffffff, 0x00000009,
823 0x91a8, 0xffffffff, 0x00080007,
824 0x91ac, 0xffffffff, 0x000b000a,
825 0x91b0, 0xffffffff, 0x00050004,
826 0x91b4, 0xffffffff, 0x00070006,
827 0x91b8, 0xffffffff, 0x0008000b,
828 0x91bc, 0xffffffff, 0x000a0009,
829 0x91c0, 0xffffffff, 0x000d000c,
830 0x9200, 0xffffffff, 0x00090008,
831 0x9204, 0xffffffff, 0x000b000a,
832 0x9208, 0xffffffff, 0x000c000f,
833 0x920c, 0xffffffff, 0x000e000d,
834 0x9210, 0xffffffff, 0x00110010,
835 0x9214, 0xffffffff, 0x000a0009,
836 0x9218, 0xffffffff, 0x000c000b,
837 0x921c, 0xffffffff, 0x0000000f,
838 0x9220, 0xffffffff, 0x000e000d,
839 0x9224, 0xffffffff, 0x00110010,
840 0x9228, 0xffffffff, 0x000b000a,
841 0x922c, 0xffffffff, 0x000d000c,
842 0x9230, 0xffffffff, 0x00000010,
843 0x9234, 0xffffffff, 0x000f000e,
844 0x9238, 0xffffffff, 0x00120011,
845 0x923c, 0xffffffff, 0x000c000b,
846 0x9240, 0xffffffff, 0x000e000d,
847 0x9244, 0xffffffff, 0x00000011,
848 0x9248, 0xffffffff, 0x0010000f,
849 0x924c, 0xffffffff, 0x00130012,
850 0x9250, 0xffffffff, 0x000d000c,
851 0x9254, 0xffffffff, 0x000f000e,
852 0x9258, 0xffffffff, 0x00100013,
853 0x925c, 0xffffffff, 0x00120011,
854 0x9260, 0xffffffff, 0x00150014,
855 0x9150, 0xffffffff, 0x96940200,
856 0x8708, 0xffffffff, 0x00900100,
857 0xc478, 0xffffffff, 0x00000080,
858 0xc404, 0xffffffff, 0x0020003f,
859 0x30, 0xffffffff, 0x0000001c,
860 0x34, 0x000f0000, 0x000f0000,
861 0x160c, 0xffffffff, 0x00000100,
862 0x1024, 0xffffffff, 0x00000100,
863 0x102c, 0x00000101, 0x00000000,
864 0x20a8, 0xffffffff, 0x00000104,
865 0x264c, 0x000c0000, 0x000c0000,
866 0x2648, 0x000c0000, 0x000c0000,
867 0x55e4, 0xff000fff, 0x00000100,
868 0x55e8, 0x00000001, 0x00000001,
869 0x2f50, 0x00000001, 0x00000001,
870 0x30cc, 0xc0000fff, 0x00000104,
871 0xc1e4, 0x00000001, 0x00000001,
872 0xd0c0, 0xfffffff0, 0x00000100,
873 0xd8c0, 0xfffffff0, 0x00000100
874 };
875
876 static const u32 oland_mgcg_cgcg_init[] =
877 {
878 0xc400, 0xffffffff, 0xfffffffc,
879 0x802c, 0xffffffff, 0xe0000000,
880 0x9a60, 0xffffffff, 0x00000100,
881 0x92a4, 0xffffffff, 0x00000100,
882 0xc164, 0xffffffff, 0x00000100,
883 0x9774, 0xffffffff, 0x00000100,
884 0x8984, 0xffffffff, 0x06000100,
885 0x8a18, 0xffffffff, 0x00000100,
886 0x92a0, 0xffffffff, 0x00000100,
887 0xc380, 0xffffffff, 0x00000100,
888 0x8b28, 0xffffffff, 0x00000100,
889 0x9144, 0xffffffff, 0x00000100,
890 0x8d88, 0xffffffff, 0x00000100,
891 0x8d8c, 0xffffffff, 0x00000100,
892 0x9030, 0xffffffff, 0x00000100,
893 0x9034, 0xffffffff, 0x00000100,
894 0x9038, 0xffffffff, 0x00000100,
895 0x903c, 0xffffffff, 0x00000100,
896 0xad80, 0xffffffff, 0x00000100,
897 0xac54, 0xffffffff, 0x00000100,
898 0x897c, 0xffffffff, 0x06000100,
899 0x9868, 0xffffffff, 0x00000100,
900 0x9510, 0xffffffff, 0x00000100,
901 0xaf04, 0xffffffff, 0x00000100,
902 0xae04, 0xffffffff, 0x00000100,
903 0x949c, 0xffffffff, 0x00000100,
904 0x802c, 0xffffffff, 0xe0000000,
905 0x9160, 0xffffffff, 0x00010000,
906 0x9164, 0xffffffff, 0x00030002,
907 0x9168, 0xffffffff, 0x00040007,
908 0x916c, 0xffffffff, 0x00060005,
909 0x9170, 0xffffffff, 0x00090008,
910 0x9174, 0xffffffff, 0x00020001,
911 0x9178, 0xffffffff, 0x00040003,
912 0x917c, 0xffffffff, 0x00000007,
913 0x9180, 0xffffffff, 0x00060005,
914 0x9184, 0xffffffff, 0x00090008,
915 0x9188, 0xffffffff, 0x00030002,
916 0x918c, 0xffffffff, 0x00050004,
917 0x9190, 0xffffffff, 0x00000008,
918 0x9194, 0xffffffff, 0x00070006,
919 0x9198, 0xffffffff, 0x000a0009,
920 0x919c, 0xffffffff, 0x00040003,
921 0x91a0, 0xffffffff, 0x00060005,
922 0x91a4, 0xffffffff, 0x00000009,
923 0x91a8, 0xffffffff, 0x00080007,
924 0x91ac, 0xffffffff, 0x000b000a,
925 0x91b0, 0xffffffff, 0x00050004,
926 0x91b4, 0xffffffff, 0x00070006,
927 0x91b8, 0xffffffff, 0x0008000b,
928 0x91bc, 0xffffffff, 0x000a0009,
929 0x91c0, 0xffffffff, 0x000d000c,
930 0x91c4, 0xffffffff, 0x00060005,
931 0x91c8, 0xffffffff, 0x00080007,
932 0x91cc, 0xffffffff, 0x0000000b,
933 0x91d0, 0xffffffff, 0x000a0009,
934 0x91d4, 0xffffffff, 0x000d000c,
935 0x9150, 0xffffffff, 0x96940200,
936 0x8708, 0xffffffff, 0x00900100,
937 0xc478, 0xffffffff, 0x00000080,
938 0xc404, 0xffffffff, 0x0020003f,
939 0x30, 0xffffffff, 0x0000001c,
940 0x34, 0x000f0000, 0x000f0000,
941 0x160c, 0xffffffff, 0x00000100,
942 0x1024, 0xffffffff, 0x00000100,
943 0x102c, 0x00000101, 0x00000000,
944 0x20a8, 0xffffffff, 0x00000104,
945 0x264c, 0x000c0000, 0x000c0000,
946 0x2648, 0x000c0000, 0x000c0000,
947 0x55e4, 0xff000fff, 0x00000100,
948 0x55e8, 0x00000001, 0x00000001,
949 0x2f50, 0x00000001, 0x00000001,
950 0x30cc, 0xc0000fff, 0x00000104,
951 0xc1e4, 0x00000001, 0x00000001,
952 0xd0c0, 0xfffffff0, 0x00000100,
953 0xd8c0, 0xfffffff0, 0x00000100
954 };
955
956 static const u32 hainan_mgcg_cgcg_init[] =
957 {
958 0xc400, 0xffffffff, 0xfffffffc,
959 0x802c, 0xffffffff, 0xe0000000,
960 0x9a60, 0xffffffff, 0x00000100,
961 0x92a4, 0xffffffff, 0x00000100,
962 0xc164, 0xffffffff, 0x00000100,
963 0x9774, 0xffffffff, 0x00000100,
964 0x8984, 0xffffffff, 0x06000100,
965 0x8a18, 0xffffffff, 0x00000100,
966 0x92a0, 0xffffffff, 0x00000100,
967 0xc380, 0xffffffff, 0x00000100,
968 0x8b28, 0xffffffff, 0x00000100,
969 0x9144, 0xffffffff, 0x00000100,
970 0x8d88, 0xffffffff, 0x00000100,
971 0x8d8c, 0xffffffff, 0x00000100,
972 0x9030, 0xffffffff, 0x00000100,
973 0x9034, 0xffffffff, 0x00000100,
974 0x9038, 0xffffffff, 0x00000100,
975 0x903c, 0xffffffff, 0x00000100,
976 0xad80, 0xffffffff, 0x00000100,
977 0xac54, 0xffffffff, 0x00000100,
978 0x897c, 0xffffffff, 0x06000100,
979 0x9868, 0xffffffff, 0x00000100,
980 0x9510, 0xffffffff, 0x00000100,
981 0xaf04, 0xffffffff, 0x00000100,
982 0xae04, 0xffffffff, 0x00000100,
983 0x949c, 0xffffffff, 0x00000100,
984 0x802c, 0xffffffff, 0xe0000000,
985 0x9160, 0xffffffff, 0x00010000,
986 0x9164, 0xffffffff, 0x00030002,
987 0x9168, 0xffffffff, 0x00040007,
988 0x916c, 0xffffffff, 0x00060005,
989 0x9170, 0xffffffff, 0x00090008,
990 0x9174, 0xffffffff, 0x00020001,
991 0x9178, 0xffffffff, 0x00040003,
992 0x917c, 0xffffffff, 0x00000007,
993 0x9180, 0xffffffff, 0x00060005,
994 0x9184, 0xffffffff, 0x00090008,
995 0x9188, 0xffffffff, 0x00030002,
996 0x918c, 0xffffffff, 0x00050004,
997 0x9190, 0xffffffff, 0x00000008,
998 0x9194, 0xffffffff, 0x00070006,
999 0x9198, 0xffffffff, 0x000a0009,
1000 0x919c, 0xffffffff, 0x00040003,
1001 0x91a0, 0xffffffff, 0x00060005,
1002 0x91a4, 0xffffffff, 0x00000009,
1003 0x91a8, 0xffffffff, 0x00080007,
1004 0x91ac, 0xffffffff, 0x000b000a,
1005 0x91b0, 0xffffffff, 0x00050004,
1006 0x91b4, 0xffffffff, 0x00070006,
1007 0x91b8, 0xffffffff, 0x0008000b,
1008 0x91bc, 0xffffffff, 0x000a0009,
1009 0x91c0, 0xffffffff, 0x000d000c,
1010 0x91c4, 0xffffffff, 0x00060005,
1011 0x91c8, 0xffffffff, 0x00080007,
1012 0x91cc, 0xffffffff, 0x0000000b,
1013 0x91d0, 0xffffffff, 0x000a0009,
1014 0x91d4, 0xffffffff, 0x000d000c,
1015 0x9150, 0xffffffff, 0x96940200,
1016 0x8708, 0xffffffff, 0x00900100,
1017 0xc478, 0xffffffff, 0x00000080,
1018 0xc404, 0xffffffff, 0x0020003f,
1019 0x30, 0xffffffff, 0x0000001c,
1020 0x34, 0x000f0000, 0x000f0000,
1021 0x160c, 0xffffffff, 0x00000100,
1022 0x1024, 0xffffffff, 0x00000100,
1023 0x20a8, 0xffffffff, 0x00000104,
1024 0x264c, 0x000c0000, 0x000c0000,
1025 0x2648, 0x000c0000, 0x000c0000,
1026 0x2f50, 0x00000001, 0x00000001,
1027 0x30cc, 0xc0000fff, 0x00000104,
1028 0xc1e4, 0x00000001, 0x00000001,
1029 0xd0c0, 0xfffffff0, 0x00000100,
1030 0xd8c0, 0xfffffff0, 0x00000100
1031 };
1032
1033 static u32 verde_pg_init[] =
1034 {
1035 0x353c, 0xffffffff, 0x40000,
1036 0x3538, 0xffffffff, 0x200010ff,
1037 0x353c, 0xffffffff, 0x0,
1038 0x353c, 0xffffffff, 0x0,
1039 0x353c, 0xffffffff, 0x0,
1040 0x353c, 0xffffffff, 0x0,
1041 0x353c, 0xffffffff, 0x0,
1042 0x353c, 0xffffffff, 0x7007,
1043 0x3538, 0xffffffff, 0x300010ff,
1044 0x353c, 0xffffffff, 0x0,
1045 0x353c, 0xffffffff, 0x0,
1046 0x353c, 0xffffffff, 0x0,
1047 0x353c, 0xffffffff, 0x0,
1048 0x353c, 0xffffffff, 0x0,
1049 0x353c, 0xffffffff, 0x400000,
1050 0x3538, 0xffffffff, 0x100010ff,
1051 0x353c, 0xffffffff, 0x0,
1052 0x353c, 0xffffffff, 0x0,
1053 0x353c, 0xffffffff, 0x0,
1054 0x353c, 0xffffffff, 0x0,
1055 0x353c, 0xffffffff, 0x0,
1056 0x353c, 0xffffffff, 0x120200,
1057 0x3538, 0xffffffff, 0x500010ff,
1058 0x353c, 0xffffffff, 0x0,
1059 0x353c, 0xffffffff, 0x0,
1060 0x353c, 0xffffffff, 0x0,
1061 0x353c, 0xffffffff, 0x0,
1062 0x353c, 0xffffffff, 0x0,
1063 0x353c, 0xffffffff, 0x1e1e16,
1064 0x3538, 0xffffffff, 0x600010ff,
1065 0x353c, 0xffffffff, 0x0,
1066 0x353c, 0xffffffff, 0x0,
1067 0x353c, 0xffffffff, 0x0,
1068 0x353c, 0xffffffff, 0x0,
1069 0x353c, 0xffffffff, 0x0,
1070 0x353c, 0xffffffff, 0x171f1e,
1071 0x3538, 0xffffffff, 0x700010ff,
1072 0x353c, 0xffffffff, 0x0,
1073 0x353c, 0xffffffff, 0x0,
1074 0x353c, 0xffffffff, 0x0,
1075 0x353c, 0xffffffff, 0x0,
1076 0x353c, 0xffffffff, 0x0,
1077 0x353c, 0xffffffff, 0x0,
1078 0x3538, 0xffffffff, 0x9ff,
1079 0x3500, 0xffffffff, 0x0,
1080 0x3504, 0xffffffff, 0x10000800,
1081 0x3504, 0xffffffff, 0xf,
1082 0x3504, 0xffffffff, 0xf,
1083 0x3500, 0xffffffff, 0x4,
1084 0x3504, 0xffffffff, 0x1000051e,
1085 0x3504, 0xffffffff, 0xffff,
1086 0x3504, 0xffffffff, 0xffff,
1087 0x3500, 0xffffffff, 0x8,
1088 0x3504, 0xffffffff, 0x80500,
1089 0x3500, 0xffffffff, 0x12,
1090 0x3504, 0xffffffff, 0x9050c,
1091 0x3500, 0xffffffff, 0x1d,
1092 0x3504, 0xffffffff, 0xb052c,
1093 0x3500, 0xffffffff, 0x2a,
1094 0x3504, 0xffffffff, 0x1053e,
1095 0x3500, 0xffffffff, 0x2d,
1096 0x3504, 0xffffffff, 0x10546,
1097 0x3500, 0xffffffff, 0x30,
1098 0x3504, 0xffffffff, 0xa054e,
1099 0x3500, 0xffffffff, 0x3c,
1100 0x3504, 0xffffffff, 0x1055f,
1101 0x3500, 0xffffffff, 0x3f,
1102 0x3504, 0xffffffff, 0x10567,
1103 0x3500, 0xffffffff, 0x42,
1104 0x3504, 0xffffffff, 0x1056f,
1105 0x3500, 0xffffffff, 0x45,
1106 0x3504, 0xffffffff, 0x10572,
1107 0x3500, 0xffffffff, 0x48,
1108 0x3504, 0xffffffff, 0x20575,
1109 0x3500, 0xffffffff, 0x4c,
1110 0x3504, 0xffffffff, 0x190801,
1111 0x3500, 0xffffffff, 0x67,
1112 0x3504, 0xffffffff, 0x1082a,
1113 0x3500, 0xffffffff, 0x6a,
1114 0x3504, 0xffffffff, 0x1b082d,
1115 0x3500, 0xffffffff, 0x87,
1116 0x3504, 0xffffffff, 0x310851,
1117 0x3500, 0xffffffff, 0xba,
1118 0x3504, 0xffffffff, 0x891,
1119 0x3500, 0xffffffff, 0xbc,
1120 0x3504, 0xffffffff, 0x893,
1121 0x3500, 0xffffffff, 0xbe,
1122 0x3504, 0xffffffff, 0x20895,
1123 0x3500, 0xffffffff, 0xc2,
1124 0x3504, 0xffffffff, 0x20899,
1125 0x3500, 0xffffffff, 0xc6,
1126 0x3504, 0xffffffff, 0x2089d,
1127 0x3500, 0xffffffff, 0xca,
1128 0x3504, 0xffffffff, 0x8a1,
1129 0x3500, 0xffffffff, 0xcc,
1130 0x3504, 0xffffffff, 0x8a3,
1131 0x3500, 0xffffffff, 0xce,
1132 0x3504, 0xffffffff, 0x308a5,
1133 0x3500, 0xffffffff, 0xd3,
1134 0x3504, 0xffffffff, 0x6d08cd,
1135 0x3500, 0xffffffff, 0x142,
1136 0x3504, 0xffffffff, 0x2000095a,
1137 0x3504, 0xffffffff, 0x1,
1138 0x3500, 0xffffffff, 0x144,
1139 0x3504, 0xffffffff, 0x301f095b,
1140 0x3500, 0xffffffff, 0x165,
1141 0x3504, 0xffffffff, 0xc094d,
1142 0x3500, 0xffffffff, 0x173,
1143 0x3504, 0xffffffff, 0xf096d,
1144 0x3500, 0xffffffff, 0x184,
1145 0x3504, 0xffffffff, 0x15097f,
1146 0x3500, 0xffffffff, 0x19b,
1147 0x3504, 0xffffffff, 0xc0998,
1148 0x3500, 0xffffffff, 0x1a9,
1149 0x3504, 0xffffffff, 0x409a7,
1150 0x3500, 0xffffffff, 0x1af,
1151 0x3504, 0xffffffff, 0xcdc,
1152 0x3500, 0xffffffff, 0x1b1,
1153 0x3504, 0xffffffff, 0x800,
1154 0x3508, 0xffffffff, 0x6c9b2000,
1155 0x3510, 0xfc00, 0x2000,
1156 0x3544, 0xffffffff, 0xfc0,
1157 0x28d4, 0x00000100, 0x100
1158 };
1159
1160 static void si_init_golden_registers(struct radeon_device *rdev)
1161 {
1162 switch (rdev->family) {
1163 case CHIP_TAHITI:
1164 radeon_program_register_sequence(rdev,
1165 tahiti_golden_registers,
1166 (const u32)ARRAY_SIZE(tahiti_golden_registers));
1167 radeon_program_register_sequence(rdev,
1168 tahiti_golden_rlc_registers,
1169 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
1170 radeon_program_register_sequence(rdev,
1171 tahiti_mgcg_cgcg_init,
1172 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1173 radeon_program_register_sequence(rdev,
1174 tahiti_golden_registers2,
1175 (const u32)ARRAY_SIZE(tahiti_golden_registers2));
1176 break;
1177 case CHIP_PITCAIRN:
1178 radeon_program_register_sequence(rdev,
1179 pitcairn_golden_registers,
1180 (const u32)ARRAY_SIZE(pitcairn_golden_registers));
1181 radeon_program_register_sequence(rdev,
1182 pitcairn_golden_rlc_registers,
1183 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
1184 radeon_program_register_sequence(rdev,
1185 pitcairn_mgcg_cgcg_init,
1186 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1187 break;
1188 case CHIP_VERDE:
1189 radeon_program_register_sequence(rdev,
1190 verde_golden_registers,
1191 (const u32)ARRAY_SIZE(verde_golden_registers));
1192 radeon_program_register_sequence(rdev,
1193 verde_golden_rlc_registers,
1194 (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
1195 radeon_program_register_sequence(rdev,
1196 verde_mgcg_cgcg_init,
1197 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
1198 radeon_program_register_sequence(rdev,
1199 verde_pg_init,
1200 (const u32)ARRAY_SIZE(verde_pg_init));
1201 break;
1202 case CHIP_OLAND:
1203 radeon_program_register_sequence(rdev,
1204 oland_golden_registers,
1205 (const u32)ARRAY_SIZE(oland_golden_registers));
1206 radeon_program_register_sequence(rdev,
1207 oland_golden_rlc_registers,
1208 (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
1209 radeon_program_register_sequence(rdev,
1210 oland_mgcg_cgcg_init,
1211 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
1212 break;
1213 case CHIP_HAINAN:
1214 radeon_program_register_sequence(rdev,
1215 hainan_golden_registers,
1216 (const u32)ARRAY_SIZE(hainan_golden_registers));
1217 radeon_program_register_sequence(rdev,
1218 hainan_golden_registers2,
1219 (const u32)ARRAY_SIZE(hainan_golden_registers2));
1220 radeon_program_register_sequence(rdev,
1221 hainan_mgcg_cgcg_init,
1222 (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
1223 break;
1224 default:
1225 break;
1226 }
1227 }
1228
1229 #define PCIE_BUS_CLK 10000
1230 #define TCLK (PCIE_BUS_CLK / 10)
1231
1232 /**
1233 * si_get_xclk - get the xclk
1234 *
1235 * @rdev: radeon_device pointer
1236 *
1237 * Returns the reference clock used by the gfx engine
1238 * (SI).
1239 */
1240 u32 si_get_xclk(struct radeon_device *rdev)
1241 {
1242 u32 reference_clock = rdev->clock.spll.reference_freq;
1243 u32 tmp;
1244
1245 tmp = RREG32(CG_CLKPIN_CNTL_2);
1246 if (tmp & MUX_TCLK_TO_XCLK)
1247 return TCLK;
1248
1249 tmp = RREG32(CG_CLKPIN_CNTL);
1250 if (tmp & XTALIN_DIVIDE)
1251 return reference_clock / 4;
1252
1253 return reference_clock;
1254 }
1255
1256 /* get temperature in millidegrees */
1257 int si_get_temp(struct radeon_device *rdev)
1258 {
1259 u32 temp;
1260 int actual_temp = 0;
1261
1262 temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
1263 CTF_TEMP_SHIFT;
1264
1265 if (temp & 0x200)
1266 actual_temp = 255;
1267 else
1268 actual_temp = temp & 0x1ff;
1269
1270 actual_temp = (actual_temp * 1000);
1271
1272 return actual_temp;
1273 }
1274
1275 #define TAHITI_IO_MC_REGS_SIZE 36
1276
1277 static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1278 {0x0000006f, 0x03044000},
1279 {0x00000070, 0x0480c018},
1280 {0x00000071, 0x00000040},
1281 {0x00000072, 0x01000000},
1282 {0x00000074, 0x000000ff},
1283 {0x00000075, 0x00143400},
1284 {0x00000076, 0x08ec0800},
1285 {0x00000077, 0x040000cc},
1286 {0x00000079, 0x00000000},
1287 {0x0000007a, 0x21000409},
1288 {0x0000007c, 0x00000000},
1289 {0x0000007d, 0xe8000000},
1290 {0x0000007e, 0x044408a8},
1291 {0x0000007f, 0x00000003},
1292 {0x00000080, 0x00000000},
1293 {0x00000081, 0x01000000},
1294 {0x00000082, 0x02000000},
1295 {0x00000083, 0x00000000},
1296 {0x00000084, 0xe3f3e4f4},
1297 {0x00000085, 0x00052024},
1298 {0x00000087, 0x00000000},
1299 {0x00000088, 0x66036603},
1300 {0x00000089, 0x01000000},
1301 {0x0000008b, 0x1c0a0000},
1302 {0x0000008c, 0xff010000},
1303 {0x0000008e, 0xffffefff},
1304 {0x0000008f, 0xfff3efff},
1305 {0x00000090, 0xfff3efbf},
1306 {0x00000094, 0x00101101},
1307 {0x00000095, 0x00000fff},
1308 {0x00000096, 0x00116fff},
1309 {0x00000097, 0x60010000},
1310 {0x00000098, 0x10010000},
1311 {0x00000099, 0x00006000},
1312 {0x0000009a, 0x00001000},
1313 {0x0000009f, 0x00a77400}
1314 };
1315
1316 static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1317 {0x0000006f, 0x03044000},
1318 {0x00000070, 0x0480c018},
1319 {0x00000071, 0x00000040},
1320 {0x00000072, 0x01000000},
1321 {0x00000074, 0x000000ff},
1322 {0x00000075, 0x00143400},
1323 {0x00000076, 0x08ec0800},
1324 {0x00000077, 0x040000cc},
1325 {0x00000079, 0x00000000},
1326 {0x0000007a, 0x21000409},
1327 {0x0000007c, 0x00000000},
1328 {0x0000007d, 0xe8000000},
1329 {0x0000007e, 0x044408a8},
1330 {0x0000007f, 0x00000003},
1331 {0x00000080, 0x00000000},
1332 {0x00000081, 0x01000000},
1333 {0x00000082, 0x02000000},
1334 {0x00000083, 0x00000000},
1335 {0x00000084, 0xe3f3e4f4},
1336 {0x00000085, 0x00052024},
1337 {0x00000087, 0x00000000},
1338 {0x00000088, 0x66036603},
1339 {0x00000089, 0x01000000},
1340 {0x0000008b, 0x1c0a0000},
1341 {0x0000008c, 0xff010000},
1342 {0x0000008e, 0xffffefff},
1343 {0x0000008f, 0xfff3efff},
1344 {0x00000090, 0xfff3efbf},
1345 {0x00000094, 0x00101101},
1346 {0x00000095, 0x00000fff},
1347 {0x00000096, 0x00116fff},
1348 {0x00000097, 0x60010000},
1349 {0x00000098, 0x10010000},
1350 {0x00000099, 0x00006000},
1351 {0x0000009a, 0x00001000},
1352 {0x0000009f, 0x00a47400}
1353 };
1354
1355 static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1356 {0x0000006f, 0x03044000},
1357 {0x00000070, 0x0480c018},
1358 {0x00000071, 0x00000040},
1359 {0x00000072, 0x01000000},
1360 {0x00000074, 0x000000ff},
1361 {0x00000075, 0x00143400},
1362 {0x00000076, 0x08ec0800},
1363 {0x00000077, 0x040000cc},
1364 {0x00000079, 0x00000000},
1365 {0x0000007a, 0x21000409},
1366 {0x0000007c, 0x00000000},
1367 {0x0000007d, 0xe8000000},
1368 {0x0000007e, 0x044408a8},
1369 {0x0000007f, 0x00000003},
1370 {0x00000080, 0x00000000},
1371 {0x00000081, 0x01000000},
1372 {0x00000082, 0x02000000},
1373 {0x00000083, 0x00000000},
1374 {0x00000084, 0xe3f3e4f4},
1375 {0x00000085, 0x00052024},
1376 {0x00000087, 0x00000000},
1377 {0x00000088, 0x66036603},
1378 {0x00000089, 0x01000000},
1379 {0x0000008b, 0x1c0a0000},
1380 {0x0000008c, 0xff010000},
1381 {0x0000008e, 0xffffefff},
1382 {0x0000008f, 0xfff3efff},
1383 {0x00000090, 0xfff3efbf},
1384 {0x00000094, 0x00101101},
1385 {0x00000095, 0x00000fff},
1386 {0x00000096, 0x00116fff},
1387 {0x00000097, 0x60010000},
1388 {0x00000098, 0x10010000},
1389 {0x00000099, 0x00006000},
1390 {0x0000009a, 0x00001000},
1391 {0x0000009f, 0x00a37400}
1392 };
1393
1394 static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1395 {0x0000006f, 0x03044000},
1396 {0x00000070, 0x0480c018},
1397 {0x00000071, 0x00000040},
1398 {0x00000072, 0x01000000},
1399 {0x00000074, 0x000000ff},
1400 {0x00000075, 0x00143400},
1401 {0x00000076, 0x08ec0800},
1402 {0x00000077, 0x040000cc},
1403 {0x00000079, 0x00000000},
1404 {0x0000007a, 0x21000409},
1405 {0x0000007c, 0x00000000},
1406 {0x0000007d, 0xe8000000},
1407 {0x0000007e, 0x044408a8},
1408 {0x0000007f, 0x00000003},
1409 {0x00000080, 0x00000000},
1410 {0x00000081, 0x01000000},
1411 {0x00000082, 0x02000000},
1412 {0x00000083, 0x00000000},
1413 {0x00000084, 0xe3f3e4f4},
1414 {0x00000085, 0x00052024},
1415 {0x00000087, 0x00000000},
1416 {0x00000088, 0x66036603},
1417 {0x00000089, 0x01000000},
1418 {0x0000008b, 0x1c0a0000},
1419 {0x0000008c, 0xff010000},
1420 {0x0000008e, 0xffffefff},
1421 {0x0000008f, 0xfff3efff},
1422 {0x00000090, 0xfff3efbf},
1423 {0x00000094, 0x00101101},
1424 {0x00000095, 0x00000fff},
1425 {0x00000096, 0x00116fff},
1426 {0x00000097, 0x60010000},
1427 {0x00000098, 0x10010000},
1428 {0x00000099, 0x00006000},
1429 {0x0000009a, 0x00001000},
1430 {0x0000009f, 0x00a17730}
1431 };
1432
1433 static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1434 {0x0000006f, 0x03044000},
1435 {0x00000070, 0x0480c018},
1436 {0x00000071, 0x00000040},
1437 {0x00000072, 0x01000000},
1438 {0x00000074, 0x000000ff},
1439 {0x00000075, 0x00143400},
1440 {0x00000076, 0x08ec0800},
1441 {0x00000077, 0x040000cc},
1442 {0x00000079, 0x00000000},
1443 {0x0000007a, 0x21000409},
1444 {0x0000007c, 0x00000000},
1445 {0x0000007d, 0xe8000000},
1446 {0x0000007e, 0x044408a8},
1447 {0x0000007f, 0x00000003},
1448 {0x00000080, 0x00000000},
1449 {0x00000081, 0x01000000},
1450 {0x00000082, 0x02000000},
1451 {0x00000083, 0x00000000},
1452 {0x00000084, 0xe3f3e4f4},
1453 {0x00000085, 0x00052024},
1454 {0x00000087, 0x00000000},
1455 {0x00000088, 0x66036603},
1456 {0x00000089, 0x01000000},
1457 {0x0000008b, 0x1c0a0000},
1458 {0x0000008c, 0xff010000},
1459 {0x0000008e, 0xffffefff},
1460 {0x0000008f, 0xfff3efff},
1461 {0x00000090, 0xfff3efbf},
1462 {0x00000094, 0x00101101},
1463 {0x00000095, 0x00000fff},
1464 {0x00000096, 0x00116fff},
1465 {0x00000097, 0x60010000},
1466 {0x00000098, 0x10010000},
1467 {0x00000099, 0x00006000},
1468 {0x0000009a, 0x00001000},
1469 {0x0000009f, 0x00a07730}
1470 };
1471
1472 /* ucode loading */
1473 static int si_mc_load_microcode(struct radeon_device *rdev)
1474 {
1475 const __be32 *fw_data;
1476 u32 running, blackout = 0;
1477 u32 *io_mc_regs;
1478 int i, regs_size, ucode_size;
1479
1480 if (!rdev->mc_fw)
1481 return -EINVAL;
1482
1483 ucode_size = rdev->mc_fw->size / 4;
1484
1485 switch (rdev->family) {
1486 case CHIP_TAHITI:
1487 io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1488 regs_size = TAHITI_IO_MC_REGS_SIZE;
1489 break;
1490 case CHIP_PITCAIRN:
1491 io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1492 regs_size = TAHITI_IO_MC_REGS_SIZE;
1493 break;
1494 case CHIP_VERDE:
1495 default:
1496 io_mc_regs = (u32 *)&verde_io_mc_regs;
1497 regs_size = TAHITI_IO_MC_REGS_SIZE;
1498 break;
1499 case CHIP_OLAND:
1500 io_mc_regs = (u32 *)&oland_io_mc_regs;
1501 regs_size = TAHITI_IO_MC_REGS_SIZE;
1502 break;
1503 case CHIP_HAINAN:
1504 io_mc_regs = (u32 *)&hainan_io_mc_regs;
1505 regs_size = TAHITI_IO_MC_REGS_SIZE;
1506 break;
1507 }
1508
1509 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1510
1511 if (running == 0) {
1512 if (running) {
1513 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1514 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1515 }
1516
1517 /* reset the engine and set to writable */
1518 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1519 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1520
1521 /* load mc io regs */
1522 for (i = 0; i < regs_size; i++) {
1523 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1524 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1525 }
1526 /* load the MC ucode */
1527 fw_data = (const __be32 *)rdev->mc_fw->data;
1528 for (i = 0; i < ucode_size; i++)
1529 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1530
1531 /* put the engine back into the active state */
1532 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1533 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1534 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1535
1536 /* wait for training to complete */
1537 for (i = 0; i < rdev->usec_timeout; i++) {
1538 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1539 break;
1540 udelay(1);
1541 }
1542 for (i = 0; i < rdev->usec_timeout; i++) {
1543 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1544 break;
1545 udelay(1);
1546 }
1547
1548 if (running)
1549 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1550 }
1551
1552 return 0;
1553 }
1554
1555 static int si_init_microcode(struct radeon_device *rdev)
1556 {
1557 const char *chip_name;
1558 const char *rlc_chip_name;
1559 size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
1560 size_t smc_req_size, mc2_req_size;
1561 char fw_name[30];
1562 int err;
1563
1564 DRM_DEBUG("\n");
1565
1566 switch (rdev->family) {
1567 case CHIP_TAHITI:
1568 chip_name = "TAHITI";
1569 rlc_chip_name = "TAHITI";
1570 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1571 me_req_size = SI_PM4_UCODE_SIZE * 4;
1572 ce_req_size = SI_CE_UCODE_SIZE * 4;
1573 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1574 mc_req_size = SI_MC_UCODE_SIZE * 4;
1575 mc2_req_size = TAHITI_MC_UCODE_SIZE * 4;
1576 smc_req_size = ALIGN(TAHITI_SMC_UCODE_SIZE, 4);
1577 break;
1578 case CHIP_PITCAIRN:
1579 chip_name = "PITCAIRN";
1580 rlc_chip_name = "PITCAIRN";
1581 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1582 me_req_size = SI_PM4_UCODE_SIZE * 4;
1583 ce_req_size = SI_CE_UCODE_SIZE * 4;
1584 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1585 mc_req_size = SI_MC_UCODE_SIZE * 4;
1586 mc2_req_size = PITCAIRN_MC_UCODE_SIZE * 4;
1587 smc_req_size = ALIGN(PITCAIRN_SMC_UCODE_SIZE, 4);
1588 break;
1589 case CHIP_VERDE:
1590 chip_name = "VERDE";
1591 rlc_chip_name = "VERDE";
1592 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1593 me_req_size = SI_PM4_UCODE_SIZE * 4;
1594 ce_req_size = SI_CE_UCODE_SIZE * 4;
1595 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1596 mc_req_size = SI_MC_UCODE_SIZE * 4;
1597 mc2_req_size = VERDE_MC_UCODE_SIZE * 4;
1598 smc_req_size = ALIGN(VERDE_SMC_UCODE_SIZE, 4);
1599 break;
1600 case CHIP_OLAND:
1601 chip_name = "OLAND";
1602 rlc_chip_name = "OLAND";
1603 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1604 me_req_size = SI_PM4_UCODE_SIZE * 4;
1605 ce_req_size = SI_CE_UCODE_SIZE * 4;
1606 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1607 mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
1608 smc_req_size = ALIGN(OLAND_SMC_UCODE_SIZE, 4);
1609 break;
1610 case CHIP_HAINAN:
1611 chip_name = "HAINAN";
1612 rlc_chip_name = "HAINAN";
1613 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1614 me_req_size = SI_PM4_UCODE_SIZE * 4;
1615 ce_req_size = SI_CE_UCODE_SIZE * 4;
1616 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1617 mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
1618 smc_req_size = ALIGN(HAINAN_SMC_UCODE_SIZE, 4);
1619 break;
1620 default: BUG();
1621 }
1622
1623 DRM_INFO("Loading %s Microcode\n", chip_name);
1624
1625 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1626 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1627 if (err)
1628 goto out;
1629 if (rdev->pfp_fw->size != pfp_req_size) {
1630 printk(KERN_ERR
1631 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1632 rdev->pfp_fw->size, fw_name);
1633 err = -EINVAL;
1634 goto out;
1635 }
1636
1637 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1638 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1639 if (err)
1640 goto out;
1641 if (rdev->me_fw->size != me_req_size) {
1642 printk(KERN_ERR
1643 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1644 rdev->me_fw->size, fw_name);
1645 err = -EINVAL;
1646 }
1647
1648 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1649 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1650 if (err)
1651 goto out;
1652 if (rdev->ce_fw->size != ce_req_size) {
1653 printk(KERN_ERR
1654 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1655 rdev->ce_fw->size, fw_name);
1656 err = -EINVAL;
1657 }
1658
1659 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
1660 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1661 if (err)
1662 goto out;
1663 if (rdev->rlc_fw->size != rlc_req_size) {
1664 printk(KERN_ERR
1665 "si_rlc: Bogus length %zu in firmware \"%s\"\n",
1666 rdev->rlc_fw->size, fw_name);
1667 err = -EINVAL;
1668 }
1669
1670 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
1671 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1672 if (err) {
1673 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1674 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1675 if (err)
1676 goto out;
1677 }
1678 if ((rdev->mc_fw->size != mc_req_size) &&
1679 (rdev->mc_fw->size != mc2_req_size)) {
1680 printk(KERN_ERR
1681 "si_mc: Bogus length %zu in firmware \"%s\"\n",
1682 rdev->mc_fw->size, fw_name);
1683 err = -EINVAL;
1684 }
1685 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
1686
1687 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1688 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1689 if (err) {
1690 printk(KERN_ERR
1691 "smc: error loading firmware \"%s\"\n",
1692 fw_name);
1693 release_firmware(rdev->smc_fw);
1694 rdev->smc_fw = NULL;
1695 err = 0;
1696 } else if (rdev->smc_fw->size != smc_req_size) {
1697 printk(KERN_ERR
1698 "si_smc: Bogus length %zu in firmware \"%s\"\n",
1699 rdev->smc_fw->size, fw_name);
1700 err = -EINVAL;
1701 }
1702
1703 out:
1704 if (err) {
1705 if (err != -EINVAL)
1706 printk(KERN_ERR
1707 "si_cp: Failed to load firmware \"%s\"\n",
1708 fw_name);
1709 release_firmware(rdev->pfp_fw);
1710 rdev->pfp_fw = NULL;
1711 release_firmware(rdev->me_fw);
1712 rdev->me_fw = NULL;
1713 release_firmware(rdev->ce_fw);
1714 rdev->ce_fw = NULL;
1715 release_firmware(rdev->rlc_fw);
1716 rdev->rlc_fw = NULL;
1717 release_firmware(rdev->mc_fw);
1718 rdev->mc_fw = NULL;
1719 release_firmware(rdev->smc_fw);
1720 rdev->smc_fw = NULL;
1721 }
1722 return err;
1723 }
1724
1725 /* watermark setup */
1726 static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1727 struct radeon_crtc *radeon_crtc,
1728 struct drm_display_mode *mode,
1729 struct drm_display_mode *other_mode)
1730 {
1731 u32 tmp, buffer_alloc, i;
1732 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1733 /*
1734 * Line Buffer Setup
1735 * There are 3 line buffers, each one shared by 2 display controllers.
1736 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1737 * the display controllers. The paritioning is done via one of four
1738 * preset allocations specified in bits 21:20:
1739 * 0 - half lb
1740 * 2 - whole lb, other crtc must be disabled
1741 */
1742 /* this can get tricky if we have two large displays on a paired group
1743 * of crtcs. Ideally for multiple large displays we'd assign them to
1744 * non-linked crtcs for maximum line buffer allocation.
1745 */
1746 if (radeon_crtc->base.enabled && mode) {
1747 if (other_mode) {
1748 tmp = 0; /* 1/2 */
1749 buffer_alloc = 1;
1750 } else {
1751 tmp = 2; /* whole */
1752 buffer_alloc = 2;
1753 }
1754 } else {
1755 tmp = 0;
1756 buffer_alloc = 0;
1757 }
1758
1759 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
1760 DC_LB_MEMORY_CONFIG(tmp));
1761
1762 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1763 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1764 for (i = 0; i < rdev->usec_timeout; i++) {
1765 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1766 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1767 break;
1768 udelay(1);
1769 }
1770
1771 if (radeon_crtc->base.enabled && mode) {
1772 switch (tmp) {
1773 case 0:
1774 default:
1775 return 4096 * 2;
1776 case 2:
1777 return 8192 * 2;
1778 }
1779 }
1780
1781 /* controller not enabled, so no lb used */
1782 return 0;
1783 }
1784
1785 static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
1786 {
1787 u32 tmp = RREG32(MC_SHARED_CHMAP);
1788
1789 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1790 case 0:
1791 default:
1792 return 1;
1793 case 1:
1794 return 2;
1795 case 2:
1796 return 4;
1797 case 3:
1798 return 8;
1799 case 4:
1800 return 3;
1801 case 5:
1802 return 6;
1803 case 6:
1804 return 10;
1805 case 7:
1806 return 12;
1807 case 8:
1808 return 16;
1809 }
1810 }
1811
1812 struct dce6_wm_params {
1813 u32 dram_channels; /* number of dram channels */
1814 u32 yclk; /* bandwidth per dram data pin in kHz */
1815 u32 sclk; /* engine clock in kHz */
1816 u32 disp_clk; /* display clock in kHz */
1817 u32 src_width; /* viewport width */
1818 u32 active_time; /* active display time in ns */
1819 u32 blank_time; /* blank time in ns */
1820 bool interlaced; /* mode is interlaced */
1821 fixed20_12 vsc; /* vertical scale ratio */
1822 u32 num_heads; /* number of active crtcs */
1823 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1824 u32 lb_size; /* line buffer allocated to pipe */
1825 u32 vtaps; /* vertical scaler taps */
1826 };
1827
1828 static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
1829 {
1830 /* Calculate raw DRAM Bandwidth */
1831 fixed20_12 dram_efficiency; /* 0.7 */
1832 fixed20_12 yclk, dram_channels, bandwidth;
1833 fixed20_12 a;
1834
1835 a.full = dfixed_const(1000);
1836 yclk.full = dfixed_const(wm->yclk);
1837 yclk.full = dfixed_div(yclk, a);
1838 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1839 a.full = dfixed_const(10);
1840 dram_efficiency.full = dfixed_const(7);
1841 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1842 bandwidth.full = dfixed_mul(dram_channels, yclk);
1843 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1844
1845 return dfixed_trunc(bandwidth);
1846 }
1847
1848 static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
1849 {
1850 /* Calculate DRAM Bandwidth and the part allocated to display. */
1851 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1852 fixed20_12 yclk, dram_channels, bandwidth;
1853 fixed20_12 a;
1854
1855 a.full = dfixed_const(1000);
1856 yclk.full = dfixed_const(wm->yclk);
1857 yclk.full = dfixed_div(yclk, a);
1858 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1859 a.full = dfixed_const(10);
1860 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1861 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1862 bandwidth.full = dfixed_mul(dram_channels, yclk);
1863 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1864
1865 return dfixed_trunc(bandwidth);
1866 }
1867
1868 static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
1869 {
1870 /* Calculate the display Data return Bandwidth */
1871 fixed20_12 return_efficiency; /* 0.8 */
1872 fixed20_12 sclk, bandwidth;
1873 fixed20_12 a;
1874
1875 a.full = dfixed_const(1000);
1876 sclk.full = dfixed_const(wm->sclk);
1877 sclk.full = dfixed_div(sclk, a);
1878 a.full = dfixed_const(10);
1879 return_efficiency.full = dfixed_const(8);
1880 return_efficiency.full = dfixed_div(return_efficiency, a);
1881 a.full = dfixed_const(32);
1882 bandwidth.full = dfixed_mul(a, sclk);
1883 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1884
1885 return dfixed_trunc(bandwidth);
1886 }
1887
1888 static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
1889 {
1890 return 32;
1891 }
1892
1893 static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
1894 {
1895 /* Calculate the DMIF Request Bandwidth */
1896 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1897 fixed20_12 disp_clk, sclk, bandwidth;
1898 fixed20_12 a, b1, b2;
1899 u32 min_bandwidth;
1900
1901 a.full = dfixed_const(1000);
1902 disp_clk.full = dfixed_const(wm->disp_clk);
1903 disp_clk.full = dfixed_div(disp_clk, a);
1904 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
1905 b1.full = dfixed_mul(a, disp_clk);
1906
1907 a.full = dfixed_const(1000);
1908 sclk.full = dfixed_const(wm->sclk);
1909 sclk.full = dfixed_div(sclk, a);
1910 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
1911 b2.full = dfixed_mul(a, sclk);
1912
1913 a.full = dfixed_const(10);
1914 disp_clk_request_efficiency.full = dfixed_const(8);
1915 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
1916
1917 min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
1918
1919 a.full = dfixed_const(min_bandwidth);
1920 bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
1921
1922 return dfixed_trunc(bandwidth);
1923 }
1924
1925 static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
1926 {
1927 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1928 u32 dram_bandwidth = dce6_dram_bandwidth(wm);
1929 u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
1930 u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
1931
1932 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
1933 }
1934
1935 static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
1936 {
1937 /* Calculate the display mode Average Bandwidth
1938 * DisplayMode should contain the source and destination dimensions,
1939 * timing, etc.
1940 */
1941 fixed20_12 bpp;
1942 fixed20_12 line_time;
1943 fixed20_12 src_width;
1944 fixed20_12 bandwidth;
1945 fixed20_12 a;
1946
1947 a.full = dfixed_const(1000);
1948 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
1949 line_time.full = dfixed_div(line_time, a);
1950 bpp.full = dfixed_const(wm->bytes_per_pixel);
1951 src_width.full = dfixed_const(wm->src_width);
1952 bandwidth.full = dfixed_mul(src_width, bpp);
1953 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
1954 bandwidth.full = dfixed_div(bandwidth, line_time);
1955
1956 return dfixed_trunc(bandwidth);
1957 }
1958
1959 static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
1960 {
1961 /* First calcualte the latency in ns */
1962 u32 mc_latency = 2000; /* 2000 ns. */
1963 u32 available_bandwidth = dce6_available_bandwidth(wm);
1964 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
1965 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
1966 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
1967 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
1968 (wm->num_heads * cursor_line_pair_return_time);
1969 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
1970 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
1971 u32 tmp, dmif_size = 12288;
1972 fixed20_12 a, b, c;
1973
1974 if (wm->num_heads == 0)
1975 return 0;
1976
1977 a.full = dfixed_const(2);
1978 b.full = dfixed_const(1);
1979 if ((wm->vsc.full > a.full) ||
1980 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
1981 (wm->vtaps >= 5) ||
1982 ((wm->vsc.full >= a.full) && wm->interlaced))
1983 max_src_lines_per_dst_line = 4;
1984 else
1985 max_src_lines_per_dst_line = 2;
1986
1987 a.full = dfixed_const(available_bandwidth);
1988 b.full = dfixed_const(wm->num_heads);
1989 a.full = dfixed_div(a, b);
1990
1991 b.full = dfixed_const(mc_latency + 512);
1992 c.full = dfixed_const(wm->disp_clk);
1993 b.full = dfixed_div(b, c);
1994
1995 c.full = dfixed_const(dmif_size);
1996 b.full = dfixed_div(c, b);
1997
1998 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
1999
2000 b.full = dfixed_const(1000);
2001 c.full = dfixed_const(wm->disp_clk);
2002 b.full = dfixed_div(c, b);
2003 c.full = dfixed_const(wm->bytes_per_pixel);
2004 b.full = dfixed_mul(b, c);
2005
2006 lb_fill_bw = min(tmp, dfixed_trunc(b));
2007
2008 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2009 b.full = dfixed_const(1000);
2010 c.full = dfixed_const(lb_fill_bw);
2011 b.full = dfixed_div(c, b);
2012 a.full = dfixed_div(a, b);
2013 line_fill_time = dfixed_trunc(a);
2014
2015 if (line_fill_time < wm->active_time)
2016 return latency;
2017 else
2018 return latency + (line_fill_time - wm->active_time);
2019
2020 }
2021
2022 static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2023 {
2024 if (dce6_average_bandwidth(wm) <=
2025 (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
2026 return true;
2027 else
2028 return false;
2029 };
2030
2031 static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
2032 {
2033 if (dce6_average_bandwidth(wm) <=
2034 (dce6_available_bandwidth(wm) / wm->num_heads))
2035 return true;
2036 else
2037 return false;
2038 };
2039
2040 static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
2041 {
2042 u32 lb_partitions = wm->lb_size / wm->src_width;
2043 u32 line_time = wm->active_time + wm->blank_time;
2044 u32 latency_tolerant_lines;
2045 u32 latency_hiding;
2046 fixed20_12 a;
2047
2048 a.full = dfixed_const(1);
2049 if (wm->vsc.full > a.full)
2050 latency_tolerant_lines = 1;
2051 else {
2052 if (lb_partitions <= (wm->vtaps + 1))
2053 latency_tolerant_lines = 1;
2054 else
2055 latency_tolerant_lines = 2;
2056 }
2057
2058 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2059
2060 if (dce6_latency_watermark(wm) <= latency_hiding)
2061 return true;
2062 else
2063 return false;
2064 }
2065
2066 static void dce6_program_watermarks(struct radeon_device *rdev,
2067 struct radeon_crtc *radeon_crtc,
2068 u32 lb_size, u32 num_heads)
2069 {
2070 struct drm_display_mode *mode = &radeon_crtc->base.mode;
2071 struct dce6_wm_params wm_low, wm_high;
2072 u32 dram_channels;
2073 u32 pixel_period;
2074 u32 line_time = 0;
2075 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2076 u32 priority_a_mark = 0, priority_b_mark = 0;
2077 u32 priority_a_cnt = PRIORITY_OFF;
2078 u32 priority_b_cnt = PRIORITY_OFF;
2079 u32 tmp, arb_control3;
2080 fixed20_12 a, b, c;
2081
2082 if (radeon_crtc->base.enabled && num_heads && mode) {
2083 pixel_period = 1000000 / (u32)mode->clock;
2084 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2085 priority_a_cnt = 0;
2086 priority_b_cnt = 0;
2087
2088 if (rdev->family == CHIP_ARUBA)
2089 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2090 else
2091 dram_channels = si_get_number_of_dram_channels(rdev);
2092
2093 /* watermark for high clocks */
2094 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2095 wm_high.yclk =
2096 radeon_dpm_get_mclk(rdev, false) * 10;
2097 wm_high.sclk =
2098 radeon_dpm_get_sclk(rdev, false) * 10;
2099 } else {
2100 wm_high.yclk = rdev->pm.current_mclk * 10;
2101 wm_high.sclk = rdev->pm.current_sclk * 10;
2102 }
2103
2104 wm_high.disp_clk = mode->clock;
2105 wm_high.src_width = mode->crtc_hdisplay;
2106 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2107 wm_high.blank_time = line_time - wm_high.active_time;
2108 wm_high.interlaced = false;
2109 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2110 wm_high.interlaced = true;
2111 wm_high.vsc = radeon_crtc->vsc;
2112 wm_high.vtaps = 1;
2113 if (radeon_crtc->rmx_type != RMX_OFF)
2114 wm_high.vtaps = 2;
2115 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2116 wm_high.lb_size = lb_size;
2117 wm_high.dram_channels = dram_channels;
2118 wm_high.num_heads = num_heads;
2119
2120 /* watermark for low clocks */
2121 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2122 wm_low.yclk =
2123 radeon_dpm_get_mclk(rdev, true) * 10;
2124 wm_low.sclk =
2125 radeon_dpm_get_sclk(rdev, true) * 10;
2126 } else {
2127 wm_low.yclk = rdev->pm.current_mclk * 10;
2128 wm_low.sclk = rdev->pm.current_sclk * 10;
2129 }
2130
2131 wm_low.disp_clk = mode->clock;
2132 wm_low.src_width = mode->crtc_hdisplay;
2133 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2134 wm_low.blank_time = line_time - wm_low.active_time;
2135 wm_low.interlaced = false;
2136 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2137 wm_low.interlaced = true;
2138 wm_low.vsc = radeon_crtc->vsc;
2139 wm_low.vtaps = 1;
2140 if (radeon_crtc->rmx_type != RMX_OFF)
2141 wm_low.vtaps = 2;
2142 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2143 wm_low.lb_size = lb_size;
2144 wm_low.dram_channels = dram_channels;
2145 wm_low.num_heads = num_heads;
2146
2147 /* set for high clocks */
2148 latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
2149 /* set for low clocks */
2150 latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
2151
2152 /* possibly force display priority to high */
2153 /* should really do this at mode validation time... */
2154 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2155 !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2156 !dce6_check_latency_hiding(&wm_high) ||
2157 (rdev->disp_priority == 2)) {
2158 DRM_DEBUG_KMS("force priority to high\n");
2159 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2160 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2161 }
2162 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2163 !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2164 !dce6_check_latency_hiding(&wm_low) ||
2165 (rdev->disp_priority == 2)) {
2166 DRM_DEBUG_KMS("force priority to high\n");
2167 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2168 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2169 }
2170
2171 a.full = dfixed_const(1000);
2172 b.full = dfixed_const(mode->clock);
2173 b.full = dfixed_div(b, a);
2174 c.full = dfixed_const(latency_watermark_a);
2175 c.full = dfixed_mul(c, b);
2176 c.full = dfixed_mul(c, radeon_crtc->hsc);
2177 c.full = dfixed_div(c, a);
2178 a.full = dfixed_const(16);
2179 c.full = dfixed_div(c, a);
2180 priority_a_mark = dfixed_trunc(c);
2181 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2182
2183 a.full = dfixed_const(1000);
2184 b.full = dfixed_const(mode->clock);
2185 b.full = dfixed_div(b, a);
2186 c.full = dfixed_const(latency_watermark_b);
2187 c.full = dfixed_mul(c, b);
2188 c.full = dfixed_mul(c, radeon_crtc->hsc);
2189 c.full = dfixed_div(c, a);
2190 a.full = dfixed_const(16);
2191 c.full = dfixed_div(c, a);
2192 priority_b_mark = dfixed_trunc(c);
2193 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2194 }
2195
2196 /* select wm A */
2197 arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2198 tmp = arb_control3;
2199 tmp &= ~LATENCY_WATERMARK_MASK(3);
2200 tmp |= LATENCY_WATERMARK_MASK(1);
2201 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2202 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2203 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2204 LATENCY_HIGH_WATERMARK(line_time)));
2205 /* select wm B */
2206 tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2207 tmp &= ~LATENCY_WATERMARK_MASK(3);
2208 tmp |= LATENCY_WATERMARK_MASK(2);
2209 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2210 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2211 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2212 LATENCY_HIGH_WATERMARK(line_time)));
2213 /* restore original selection */
2214 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
2215
2216 /* write the priority marks */
2217 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2218 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2219
2220 /* save values for DPM */
2221 radeon_crtc->line_time = line_time;
2222 radeon_crtc->wm_high = latency_watermark_a;
2223 radeon_crtc->wm_low = latency_watermark_b;
2224 }
2225
2226 void dce6_bandwidth_update(struct radeon_device *rdev)
2227 {
2228 struct drm_display_mode *mode0 = NULL;
2229 struct drm_display_mode *mode1 = NULL;
2230 u32 num_heads = 0, lb_size;
2231 int i;
2232
2233 radeon_update_display_priority(rdev);
2234
2235 for (i = 0; i < rdev->num_crtc; i++) {
2236 if (rdev->mode_info.crtcs[i]->base.enabled)
2237 num_heads++;
2238 }
2239 for (i = 0; i < rdev->num_crtc; i += 2) {
2240 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2241 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2242 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2243 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2244 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2245 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2246 }
2247 }
2248
2249 /*
2250 * Core functions
2251 */
2252 static void si_tiling_mode_table_init(struct radeon_device *rdev)
2253 {
2254 const u32 num_tile_mode_states = 32;
2255 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2256
2257 switch (rdev->config.si.mem_row_size_in_kb) {
2258 case 1:
2259 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2260 break;
2261 case 2:
2262 default:
2263 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2264 break;
2265 case 4:
2266 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2267 break;
2268 }
2269
2270 if ((rdev->family == CHIP_TAHITI) ||
2271 (rdev->family == CHIP_PITCAIRN)) {
2272 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2273 switch (reg_offset) {
2274 case 0: /* non-AA compressed depth or any compressed stencil */
2275 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2276 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2277 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2278 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2279 NUM_BANKS(ADDR_SURF_16_BANK) |
2280 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2281 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2282 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2283 break;
2284 case 1: /* 2xAA/4xAA compressed depth only */
2285 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2286 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2287 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2288 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2289 NUM_BANKS(ADDR_SURF_16_BANK) |
2290 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2291 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2292 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2293 break;
2294 case 2: /* 8xAA compressed depth only */
2295 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2296 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2297 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2298 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2299 NUM_BANKS(ADDR_SURF_16_BANK) |
2300 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2301 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2302 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2303 break;
2304 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2305 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2306 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2307 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2308 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2309 NUM_BANKS(ADDR_SURF_16_BANK) |
2310 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2311 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2312 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2313 break;
2314 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2315 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2316 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2317 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2318 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2319 NUM_BANKS(ADDR_SURF_16_BANK) |
2320 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2321 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2322 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2323 break;
2324 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2325 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2326 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2327 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2328 TILE_SPLIT(split_equal_to_row_size) |
2329 NUM_BANKS(ADDR_SURF_16_BANK) |
2330 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2331 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2332 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2333 break;
2334 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2335 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2336 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2337 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2338 TILE_SPLIT(split_equal_to_row_size) |
2339 NUM_BANKS(ADDR_SURF_16_BANK) |
2340 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2341 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2342 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2343 break;
2344 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2345 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2346 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2347 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2348 TILE_SPLIT(split_equal_to_row_size) |
2349 NUM_BANKS(ADDR_SURF_16_BANK) |
2350 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2351 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2352 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2353 break;
2354 case 8: /* 1D and 1D Array Surfaces */
2355 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2356 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2357 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2358 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2359 NUM_BANKS(ADDR_SURF_16_BANK) |
2360 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2361 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2362 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2363 break;
2364 case 9: /* Displayable maps. */
2365 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2366 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2367 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2368 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2369 NUM_BANKS(ADDR_SURF_16_BANK) |
2370 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2371 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2372 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2373 break;
2374 case 10: /* Display 8bpp. */
2375 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2376 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2377 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2378 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2379 NUM_BANKS(ADDR_SURF_16_BANK) |
2380 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2381 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2382 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2383 break;
2384 case 11: /* Display 16bpp. */
2385 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2386 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2387 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2388 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2389 NUM_BANKS(ADDR_SURF_16_BANK) |
2390 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2391 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2392 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2393 break;
2394 case 12: /* Display 32bpp. */
2395 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2396 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2397 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2398 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2399 NUM_BANKS(ADDR_SURF_16_BANK) |
2400 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2401 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2402 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2403 break;
2404 case 13: /* Thin. */
2405 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2406 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2407 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2408 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2409 NUM_BANKS(ADDR_SURF_16_BANK) |
2410 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2411 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2412 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2413 break;
2414 case 14: /* Thin 8 bpp. */
2415 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2416 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2417 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2418 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2419 NUM_BANKS(ADDR_SURF_16_BANK) |
2420 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2421 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2422 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2423 break;
2424 case 15: /* Thin 16 bpp. */
2425 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2426 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2427 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2428 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2429 NUM_BANKS(ADDR_SURF_16_BANK) |
2430 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2431 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2432 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2433 break;
2434 case 16: /* Thin 32 bpp. */
2435 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2436 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2437 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2438 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2439 NUM_BANKS(ADDR_SURF_16_BANK) |
2440 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2441 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2442 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2443 break;
2444 case 17: /* Thin 64 bpp. */
2445 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2446 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2447 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2448 TILE_SPLIT(split_equal_to_row_size) |
2449 NUM_BANKS(ADDR_SURF_16_BANK) |
2450 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2451 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2452 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2453 break;
2454 case 21: /* 8 bpp PRT. */
2455 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2456 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2457 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2458 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2459 NUM_BANKS(ADDR_SURF_16_BANK) |
2460 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2461 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2462 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2463 break;
2464 case 22: /* 16 bpp PRT */
2465 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2466 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2467 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2468 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2469 NUM_BANKS(ADDR_SURF_16_BANK) |
2470 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2471 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2472 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2473 break;
2474 case 23: /* 32 bpp PRT */
2475 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2476 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2477 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2478 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2479 NUM_BANKS(ADDR_SURF_16_BANK) |
2480 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2481 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2482 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2483 break;
2484 case 24: /* 64 bpp PRT */
2485 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2486 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2487 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2488 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2489 NUM_BANKS(ADDR_SURF_16_BANK) |
2490 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2491 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2492 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2493 break;
2494 case 25: /* 128 bpp PRT */
2495 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2496 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2497 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2498 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2499 NUM_BANKS(ADDR_SURF_8_BANK) |
2500 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2501 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2502 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2503 break;
2504 default:
2505 gb_tile_moden = 0;
2506 break;
2507 }
2508 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
2509 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2510 }
2511 } else if ((rdev->family == CHIP_VERDE) ||
2512 (rdev->family == CHIP_OLAND) ||
2513 (rdev->family == CHIP_HAINAN)) {
2514 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2515 switch (reg_offset) {
2516 case 0: /* non-AA compressed depth or any compressed stencil */
2517 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2518 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2519 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2520 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2521 NUM_BANKS(ADDR_SURF_16_BANK) |
2522 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2523 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2524 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2525 break;
2526 case 1: /* 2xAA/4xAA compressed depth only */
2527 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2528 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2529 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2530 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2531 NUM_BANKS(ADDR_SURF_16_BANK) |
2532 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2533 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2534 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2535 break;
2536 case 2: /* 8xAA compressed depth only */
2537 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2538 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2539 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2540 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2541 NUM_BANKS(ADDR_SURF_16_BANK) |
2542 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2543 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2544 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2545 break;
2546 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2547 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2548 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2549 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2550 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2551 NUM_BANKS(ADDR_SURF_16_BANK) |
2552 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2553 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2554 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2555 break;
2556 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2557 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2558 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2559 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2560 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2561 NUM_BANKS(ADDR_SURF_16_BANK) |
2562 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2563 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2564 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2565 break;
2566 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2567 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2568 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2569 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2570 TILE_SPLIT(split_equal_to_row_size) |
2571 NUM_BANKS(ADDR_SURF_16_BANK) |
2572 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2573 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2574 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2575 break;
2576 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2577 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2578 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2579 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2580 TILE_SPLIT(split_equal_to_row_size) |
2581 NUM_BANKS(ADDR_SURF_16_BANK) |
2582 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2583 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2584 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2585 break;
2586 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2587 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2588 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2589 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2590 TILE_SPLIT(split_equal_to_row_size) |
2591 NUM_BANKS(ADDR_SURF_16_BANK) |
2592 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2593 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2594 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2595 break;
2596 case 8: /* 1D and 1D Array Surfaces */
2597 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2598 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2599 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2600 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2601 NUM_BANKS(ADDR_SURF_16_BANK) |
2602 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2603 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2604 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2605 break;
2606 case 9: /* Displayable maps. */
2607 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2608 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2609 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2610 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2611 NUM_BANKS(ADDR_SURF_16_BANK) |
2612 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2613 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2614 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2615 break;
2616 case 10: /* Display 8bpp. */
2617 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2618 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2619 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2620 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2621 NUM_BANKS(ADDR_SURF_16_BANK) |
2622 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2623 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2624 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2625 break;
2626 case 11: /* Display 16bpp. */
2627 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2628 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2629 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2630 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2631 NUM_BANKS(ADDR_SURF_16_BANK) |
2632 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2633 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2634 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2635 break;
2636 case 12: /* Display 32bpp. */
2637 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2638 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2639 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2640 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2641 NUM_BANKS(ADDR_SURF_16_BANK) |
2642 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2643 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2644 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2645 break;
2646 case 13: /* Thin. */
2647 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2648 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2649 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2650 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2651 NUM_BANKS(ADDR_SURF_16_BANK) |
2652 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2653 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2654 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2655 break;
2656 case 14: /* Thin 8 bpp. */
2657 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2658 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2659 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2660 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2661 NUM_BANKS(ADDR_SURF_16_BANK) |
2662 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2663 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2664 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2665 break;
2666 case 15: /* Thin 16 bpp. */
2667 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2668 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2669 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2670 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2671 NUM_BANKS(ADDR_SURF_16_BANK) |
2672 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2673 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2674 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2675 break;
2676 case 16: /* Thin 32 bpp. */
2677 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2678 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2679 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2680 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2681 NUM_BANKS(ADDR_SURF_16_BANK) |
2682 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2683 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2684 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2685 break;
2686 case 17: /* Thin 64 bpp. */
2687 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2688 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2689 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2690 TILE_SPLIT(split_equal_to_row_size) |
2691 NUM_BANKS(ADDR_SURF_16_BANK) |
2692 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2693 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2694 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2695 break;
2696 case 21: /* 8 bpp PRT. */
2697 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2698 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2699 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2700 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2701 NUM_BANKS(ADDR_SURF_16_BANK) |
2702 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2703 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2704 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2705 break;
2706 case 22: /* 16 bpp PRT */
2707 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2708 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2709 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2710 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2711 NUM_BANKS(ADDR_SURF_16_BANK) |
2712 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2713 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2714 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2715 break;
2716 case 23: /* 32 bpp PRT */
2717 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2718 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2719 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2720 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2721 NUM_BANKS(ADDR_SURF_16_BANK) |
2722 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2723 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2724 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2725 break;
2726 case 24: /* 64 bpp PRT */
2727 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2728 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2729 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2730 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2731 NUM_BANKS(ADDR_SURF_16_BANK) |
2732 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2733 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2734 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2735 break;
2736 case 25: /* 128 bpp PRT */
2737 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2738 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2739 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2740 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2741 NUM_BANKS(ADDR_SURF_8_BANK) |
2742 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2743 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2744 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2745 break;
2746 default:
2747 gb_tile_moden = 0;
2748 break;
2749 }
2750 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
2751 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2752 }
2753 } else
2754 DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2755 }
2756
2757 static void si_select_se_sh(struct radeon_device *rdev,
2758 u32 se_num, u32 sh_num)
2759 {
2760 u32 data = INSTANCE_BROADCAST_WRITES;
2761
2762 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
2763 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
2764 else if (se_num == 0xffffffff)
2765 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2766 else if (sh_num == 0xffffffff)
2767 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2768 else
2769 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2770 WREG32(GRBM_GFX_INDEX, data);
2771 }
2772
2773 static u32 si_create_bitmask(u32 bit_width)
2774 {
2775 u32 i, mask = 0;
2776
2777 for (i = 0; i < bit_width; i++) {
2778 mask <<= 1;
2779 mask |= 1;
2780 }
2781 return mask;
2782 }
2783
2784 static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2785 {
2786 u32 data, mask;
2787
2788 data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2789 if (data & 1)
2790 data &= INACTIVE_CUS_MASK;
2791 else
2792 data = 0;
2793 data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2794
2795 data >>= INACTIVE_CUS_SHIFT;
2796
2797 mask = si_create_bitmask(cu_per_sh);
2798
2799 return ~data & mask;
2800 }
2801
2802 static void si_setup_spi(struct radeon_device *rdev,
2803 u32 se_num, u32 sh_per_se,
2804 u32 cu_per_sh)
2805 {
2806 int i, j, k;
2807 u32 data, mask, active_cu;
2808
2809 for (i = 0; i < se_num; i++) {
2810 for (j = 0; j < sh_per_se; j++) {
2811 si_select_se_sh(rdev, i, j);
2812 data = RREG32(SPI_STATIC_THREAD_MGMT_3);
2813 active_cu = si_get_cu_enabled(rdev, cu_per_sh);
2814
2815 mask = 1;
2816 for (k = 0; k < 16; k++) {
2817 mask <<= k;
2818 if (active_cu & mask) {
2819 data &= ~mask;
2820 WREG32(SPI_STATIC_THREAD_MGMT_3, data);
2821 break;
2822 }
2823 }
2824 }
2825 }
2826 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2827 }
2828
2829 static u32 si_get_rb_disabled(struct radeon_device *rdev,
2830 u32 max_rb_num_per_se,
2831 u32 sh_per_se)
2832 {
2833 u32 data, mask;
2834
2835 data = RREG32(CC_RB_BACKEND_DISABLE);
2836 if (data & 1)
2837 data &= BACKEND_DISABLE_MASK;
2838 else
2839 data = 0;
2840 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
2841
2842 data >>= BACKEND_DISABLE_SHIFT;
2843
2844 mask = si_create_bitmask(max_rb_num_per_se / sh_per_se);
2845
2846 return data & mask;
2847 }
2848
2849 static void si_setup_rb(struct radeon_device *rdev,
2850 u32 se_num, u32 sh_per_se,
2851 u32 max_rb_num_per_se)
2852 {
2853 int i, j;
2854 u32 data, mask;
2855 u32 disabled_rbs = 0;
2856 u32 enabled_rbs = 0;
2857
2858 for (i = 0; i < se_num; i++) {
2859 for (j = 0; j < sh_per_se; j++) {
2860 si_select_se_sh(rdev, i, j);
2861 data = si_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
2862 disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
2863 }
2864 }
2865 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2866
2867 mask = 1;
2868 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
2869 if (!(disabled_rbs & mask))
2870 enabled_rbs |= mask;
2871 mask <<= 1;
2872 }
2873
2874 rdev->config.si.backend_enable_mask = enabled_rbs;
2875
2876 for (i = 0; i < se_num; i++) {
2877 si_select_se_sh(rdev, i, 0xffffffff);
2878 data = 0;
2879 for (j = 0; j < sh_per_se; j++) {
2880 switch (enabled_rbs & 3) {
2881 case 1:
2882 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
2883 break;
2884 case 2:
2885 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
2886 break;
2887 case 3:
2888 default:
2889 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
2890 break;
2891 }
2892 enabled_rbs >>= 2;
2893 }
2894 WREG32(PA_SC_RASTER_CONFIG, data);
2895 }
2896 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2897 }
2898
2899 static void si_gpu_init(struct radeon_device *rdev)
2900 {
2901 u32 gb_addr_config = 0;
2902 u32 mc_shared_chmap, mc_arb_ramcfg;
2903 u32 sx_debug_1;
2904 u32 hdp_host_path_cntl;
2905 u32 tmp;
2906 int i, j;
2907
2908 switch (rdev->family) {
2909 case CHIP_TAHITI:
2910 rdev->config.si.max_shader_engines = 2;
2911 rdev->config.si.max_tile_pipes = 12;
2912 rdev->config.si.max_cu_per_sh = 8;
2913 rdev->config.si.max_sh_per_se = 2;
2914 rdev->config.si.max_backends_per_se = 4;
2915 rdev->config.si.max_texture_channel_caches = 12;
2916 rdev->config.si.max_gprs = 256;
2917 rdev->config.si.max_gs_threads = 32;
2918 rdev->config.si.max_hw_contexts = 8;
2919
2920 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2921 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2922 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2923 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2924 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
2925 break;
2926 case CHIP_PITCAIRN:
2927 rdev->config.si.max_shader_engines = 2;
2928 rdev->config.si.max_tile_pipes = 8;
2929 rdev->config.si.max_cu_per_sh = 5;
2930 rdev->config.si.max_sh_per_se = 2;
2931 rdev->config.si.max_backends_per_se = 4;
2932 rdev->config.si.max_texture_channel_caches = 8;
2933 rdev->config.si.max_gprs = 256;
2934 rdev->config.si.max_gs_threads = 32;
2935 rdev->config.si.max_hw_contexts = 8;
2936
2937 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2938 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2939 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2940 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2941 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
2942 break;
2943 case CHIP_VERDE:
2944 default:
2945 rdev->config.si.max_shader_engines = 1;
2946 rdev->config.si.max_tile_pipes = 4;
2947 rdev->config.si.max_cu_per_sh = 5;
2948 rdev->config.si.max_sh_per_se = 2;
2949 rdev->config.si.max_backends_per_se = 4;
2950 rdev->config.si.max_texture_channel_caches = 4;
2951 rdev->config.si.max_gprs = 256;
2952 rdev->config.si.max_gs_threads = 32;
2953 rdev->config.si.max_hw_contexts = 8;
2954
2955 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2956 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2957 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2958 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2959 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
2960 break;
2961 case CHIP_OLAND:
2962 rdev->config.si.max_shader_engines = 1;
2963 rdev->config.si.max_tile_pipes = 4;
2964 rdev->config.si.max_cu_per_sh = 6;
2965 rdev->config.si.max_sh_per_se = 1;
2966 rdev->config.si.max_backends_per_se = 2;
2967 rdev->config.si.max_texture_channel_caches = 4;
2968 rdev->config.si.max_gprs = 256;
2969 rdev->config.si.max_gs_threads = 16;
2970 rdev->config.si.max_hw_contexts = 8;
2971
2972 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2973 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2974 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2975 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2976 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
2977 break;
2978 case CHIP_HAINAN:
2979 rdev->config.si.max_shader_engines = 1;
2980 rdev->config.si.max_tile_pipes = 4;
2981 rdev->config.si.max_cu_per_sh = 5;
2982 rdev->config.si.max_sh_per_se = 1;
2983 rdev->config.si.max_backends_per_se = 1;
2984 rdev->config.si.max_texture_channel_caches = 2;
2985 rdev->config.si.max_gprs = 256;
2986 rdev->config.si.max_gs_threads = 16;
2987 rdev->config.si.max_hw_contexts = 8;
2988
2989 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2990 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2991 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2992 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2993 gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
2994 break;
2995 }
2996
2997 /* Initialize HDP */
2998 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2999 WREG32((0x2c14 + j), 0x00000000);
3000 WREG32((0x2c18 + j), 0x00000000);
3001 WREG32((0x2c1c + j), 0x00000000);
3002 WREG32((0x2c20 + j), 0x00000000);
3003 WREG32((0x2c24 + j), 0x00000000);
3004 }
3005
3006 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3007
3008 evergreen_fix_pci_max_read_req_size(rdev);
3009
3010 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3011
3012 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3013 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3014
3015 rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
3016 rdev->config.si.mem_max_burst_length_bytes = 256;
3017 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3018 rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3019 if (rdev->config.si.mem_row_size_in_kb > 4)
3020 rdev->config.si.mem_row_size_in_kb = 4;
3021 /* XXX use MC settings? */
3022 rdev->config.si.shader_engine_tile_size = 32;
3023 rdev->config.si.num_gpus = 1;
3024 rdev->config.si.multi_gpu_tile_size = 64;
3025
3026 /* fix up row size */
3027 gb_addr_config &= ~ROW_SIZE_MASK;
3028 switch (rdev->config.si.mem_row_size_in_kb) {
3029 case 1:
3030 default:
3031 gb_addr_config |= ROW_SIZE(0);
3032 break;
3033 case 2:
3034 gb_addr_config |= ROW_SIZE(1);
3035 break;
3036 case 4:
3037 gb_addr_config |= ROW_SIZE(2);
3038 break;
3039 }
3040
3041 /* setup tiling info dword. gb_addr_config is not adequate since it does
3042 * not have bank info, so create a custom tiling dword.
3043 * bits 3:0 num_pipes
3044 * bits 7:4 num_banks
3045 * bits 11:8 group_size
3046 * bits 15:12 row_size
3047 */
3048 rdev->config.si.tile_config = 0;
3049 switch (rdev->config.si.num_tile_pipes) {
3050 case 1:
3051 rdev->config.si.tile_config |= (0 << 0);
3052 break;
3053 case 2:
3054 rdev->config.si.tile_config |= (1 << 0);
3055 break;
3056 case 4:
3057 rdev->config.si.tile_config |= (2 << 0);
3058 break;
3059 case 8:
3060 default:
3061 /* XXX what about 12? */
3062 rdev->config.si.tile_config |= (3 << 0);
3063 break;
3064 }
3065 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3066 case 0: /* four banks */
3067 rdev->config.si.tile_config |= 0 << 4;
3068 break;
3069 case 1: /* eight banks */
3070 rdev->config.si.tile_config |= 1 << 4;
3071 break;
3072 case 2: /* sixteen banks */
3073 default:
3074 rdev->config.si.tile_config |= 2 << 4;
3075 break;
3076 }
3077 rdev->config.si.tile_config |=
3078 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3079 rdev->config.si.tile_config |=
3080 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3081
3082 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3083 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3084 WREG32(DMIF_ADDR_CALC, gb_addr_config);
3085 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3086 WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
3087 WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
3088 if (rdev->has_uvd) {
3089 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3090 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3091 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3092 }
3093
3094 si_tiling_mode_table_init(rdev);
3095
3096 si_setup_rb(rdev, rdev->config.si.max_shader_engines,
3097 rdev->config.si.max_sh_per_se,
3098 rdev->config.si.max_backends_per_se);
3099
3100 si_setup_spi(rdev, rdev->config.si.max_shader_engines,
3101 rdev->config.si.max_sh_per_se,
3102 rdev->config.si.max_cu_per_sh);
3103
3104
3105 /* set HW defaults for 3D engine */
3106 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3107 ROQ_IB2_START(0x2b)));
3108 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3109
3110 sx_debug_1 = RREG32(SX_DEBUG_1);
3111 WREG32(SX_DEBUG_1, sx_debug_1);
3112
3113 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3114
3115 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
3116 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
3117 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
3118 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
3119
3120 WREG32(VGT_NUM_INSTANCES, 1);
3121
3122 WREG32(CP_PERFMON_CNTL, 0);
3123
3124 WREG32(SQ_CONFIG, 0);
3125
3126 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3127 FORCE_EOV_MAX_REZ_CNT(255)));
3128
3129 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3130 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3131
3132 WREG32(VGT_GS_VERTEX_REUSE, 16);
3133 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3134
3135 WREG32(CB_PERFCOUNTER0_SELECT0, 0);
3136 WREG32(CB_PERFCOUNTER0_SELECT1, 0);
3137 WREG32(CB_PERFCOUNTER1_SELECT0, 0);
3138 WREG32(CB_PERFCOUNTER1_SELECT1, 0);
3139 WREG32(CB_PERFCOUNTER2_SELECT0, 0);
3140 WREG32(CB_PERFCOUNTER2_SELECT1, 0);
3141 WREG32(CB_PERFCOUNTER3_SELECT0, 0);
3142 WREG32(CB_PERFCOUNTER3_SELECT1, 0);
3143
3144 tmp = RREG32(HDP_MISC_CNTL);
3145 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3146 WREG32(HDP_MISC_CNTL, tmp);
3147
3148 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3149 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3150
3151 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3152
3153 udelay(50);
3154 }
3155
3156 /*
3157 * GPU scratch registers helpers function.
3158 */
3159 static void si_scratch_init(struct radeon_device *rdev)
3160 {
3161 int i;
3162
3163 rdev->scratch.num_reg = 7;
3164 rdev->scratch.reg_base = SCRATCH_REG0;
3165 for (i = 0; i < rdev->scratch.num_reg; i++) {
3166 rdev->scratch.free[i] = true;
3167 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3168 }
3169 }
3170
3171 void si_fence_ring_emit(struct radeon_device *rdev,
3172 struct radeon_fence *fence)
3173 {
3174 struct radeon_ring *ring = &rdev->ring[fence->ring];
3175 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3176
3177 /* flush read cache over gart */
3178 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3179 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3180 radeon_ring_write(ring, 0);
3181 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3182 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3183 PACKET3_TC_ACTION_ENA |
3184 PACKET3_SH_KCACHE_ACTION_ENA |
3185 PACKET3_SH_ICACHE_ACTION_ENA);
3186 radeon_ring_write(ring, 0xFFFFFFFF);
3187 radeon_ring_write(ring, 0);
3188 radeon_ring_write(ring, 10); /* poll interval */
3189 /* EVENT_WRITE_EOP - flush caches, send int */
3190 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3191 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
3192 radeon_ring_write(ring, addr & 0xffffffff);
3193 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
3194 radeon_ring_write(ring, fence->seq);
3195 radeon_ring_write(ring, 0);
3196 }
3197
3198 /*
3199 * IB stuff
3200 */
3201 void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3202 {
3203 struct radeon_ring *ring = &rdev->ring[ib->ring];
3204 u32 header;
3205
3206 if (ib->is_const_ib) {
3207 /* set switch buffer packet before const IB */
3208 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3209 radeon_ring_write(ring, 0);
3210
3211 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3212 } else {
3213 u32 next_rptr;
3214 if (ring->rptr_save_reg) {
3215 next_rptr = ring->wptr + 3 + 4 + 8;
3216 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3217 radeon_ring_write(ring, ((ring->rptr_save_reg -
3218 PACKET3_SET_CONFIG_REG_START) >> 2));
3219 radeon_ring_write(ring, next_rptr);
3220 } else if (rdev->wb.enabled) {
3221 next_rptr = ring->wptr + 5 + 4 + 8;
3222 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3223 radeon_ring_write(ring, (1 << 8));
3224 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3225 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3226 radeon_ring_write(ring, next_rptr);
3227 }
3228
3229 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3230 }
3231
3232 radeon_ring_write(ring, header);
3233 radeon_ring_write(ring,
3234 #ifdef __BIG_ENDIAN
3235 (2 << 0) |
3236 #endif
3237 (ib->gpu_addr & 0xFFFFFFFC));
3238 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3239 radeon_ring_write(ring, ib->length_dw |
3240 (ib->vm ? (ib->vm->id << 24) : 0));
3241
3242 if (!ib->is_const_ib) {
3243 /* flush read cache over gart for this vmid */
3244 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3245 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3246 radeon_ring_write(ring, ib->vm ? ib->vm->id : 0);
3247 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3248 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3249 PACKET3_TC_ACTION_ENA |
3250 PACKET3_SH_KCACHE_ACTION_ENA |
3251 PACKET3_SH_ICACHE_ACTION_ENA);
3252 radeon_ring_write(ring, 0xFFFFFFFF);
3253 radeon_ring_write(ring, 0);
3254 radeon_ring_write(ring, 10); /* poll interval */
3255 }
3256 }
3257
3258 /*
3259 * CP.
3260 */
3261 static void si_cp_enable(struct radeon_device *rdev, bool enable)
3262 {
3263 if (enable)
3264 WREG32(CP_ME_CNTL, 0);
3265 else {
3266 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3267 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3268 WREG32(SCRATCH_UMSK, 0);
3269 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3270 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3271 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3272 }
3273 udelay(50);
3274 }
3275
3276 static int si_cp_load_microcode(struct radeon_device *rdev)
3277 {
3278 const __be32 *fw_data;
3279 int i;
3280
3281 if (!rdev->me_fw || !rdev->pfp_fw)
3282 return -EINVAL;
3283
3284 si_cp_enable(rdev, false);
3285
3286 /* PFP */
3287 fw_data = (const __be32 *)rdev->pfp_fw->data;
3288 WREG32(CP_PFP_UCODE_ADDR, 0);
3289 for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
3290 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3291 WREG32(CP_PFP_UCODE_ADDR, 0);
3292
3293 /* CE */
3294 fw_data = (const __be32 *)rdev->ce_fw->data;
3295 WREG32(CP_CE_UCODE_ADDR, 0);
3296 for (i = 0; i < SI_CE_UCODE_SIZE; i++)
3297 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3298 WREG32(CP_CE_UCODE_ADDR, 0);
3299
3300 /* ME */
3301 fw_data = (const __be32 *)rdev->me_fw->data;
3302 WREG32(CP_ME_RAM_WADDR, 0);
3303 for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
3304 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3305 WREG32(CP_ME_RAM_WADDR, 0);
3306
3307 WREG32(CP_PFP_UCODE_ADDR, 0);
3308 WREG32(CP_CE_UCODE_ADDR, 0);
3309 WREG32(CP_ME_RAM_WADDR, 0);
3310 WREG32(CP_ME_RAM_RADDR, 0);
3311 return 0;
3312 }
3313
3314 static int si_cp_start(struct radeon_device *rdev)
3315 {
3316 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3317 int r, i;
3318
3319 r = radeon_ring_lock(rdev, ring, 7 + 4);
3320 if (r) {
3321 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3322 return r;
3323 }
3324 /* init the CP */
3325 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3326 radeon_ring_write(ring, 0x1);
3327 radeon_ring_write(ring, 0x0);
3328 radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
3329 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3330 radeon_ring_write(ring, 0);
3331 radeon_ring_write(ring, 0);
3332
3333 /* init the CE partitions */
3334 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3335 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3336 radeon_ring_write(ring, 0xc000);
3337 radeon_ring_write(ring, 0xe000);
3338 radeon_ring_unlock_commit(rdev, ring);
3339
3340 si_cp_enable(rdev, true);
3341
3342 r = radeon_ring_lock(rdev, ring, si_default_size + 10);
3343 if (r) {
3344 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3345 return r;
3346 }
3347
3348 /* setup clear context state */
3349 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3350 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3351
3352 for (i = 0; i < si_default_size; i++)
3353 radeon_ring_write(ring, si_default_state[i]);
3354
3355 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3356 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3357
3358 /* set clear context state */
3359 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3360 radeon_ring_write(ring, 0);
3361
3362 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3363 radeon_ring_write(ring, 0x00000316);
3364 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3365 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3366
3367 radeon_ring_unlock_commit(rdev, ring);
3368
3369 for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
3370 ring = &rdev->ring[i];
3371 r = radeon_ring_lock(rdev, ring, 2);
3372
3373 /* clear the compute context state */
3374 radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
3375 radeon_ring_write(ring, 0);
3376
3377 radeon_ring_unlock_commit(rdev, ring);
3378 }
3379
3380 return 0;
3381 }
3382
3383 static void si_cp_fini(struct radeon_device *rdev)
3384 {
3385 struct radeon_ring *ring;
3386 si_cp_enable(rdev, false);
3387
3388 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3389 radeon_ring_fini(rdev, ring);
3390 radeon_scratch_free(rdev, ring->rptr_save_reg);
3391
3392 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3393 radeon_ring_fini(rdev, ring);
3394 radeon_scratch_free(rdev, ring->rptr_save_reg);
3395
3396 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3397 radeon_ring_fini(rdev, ring);
3398 radeon_scratch_free(rdev, ring->rptr_save_reg);
3399 }
3400
3401 static int si_cp_resume(struct radeon_device *rdev)
3402 {
3403 struct radeon_ring *ring;
3404 u32 tmp;
3405 u32 rb_bufsz;
3406 int r;
3407
3408 si_enable_gui_idle_interrupt(rdev, false);
3409
3410 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3411 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3412
3413 /* Set the write pointer delay */
3414 WREG32(CP_RB_WPTR_DELAY, 0);
3415
3416 WREG32(CP_DEBUG, 0);
3417 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3418
3419 /* ring 0 - compute and gfx */
3420 /* Set ring buffer size */
3421 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3422 rb_bufsz = order_base_2(ring->ring_size / 8);
3423 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3424 #ifdef __BIG_ENDIAN
3425 tmp |= BUF_SWAP_32BIT;
3426 #endif
3427 WREG32(CP_RB0_CNTL, tmp);
3428
3429 /* Initialize the ring buffer's read and write pointers */
3430 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3431 ring->wptr = 0;
3432 WREG32(CP_RB0_WPTR, ring->wptr);
3433
3434 /* set the wb address whether it's enabled or not */
3435 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3436 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3437
3438 if (rdev->wb.enabled)
3439 WREG32(SCRATCH_UMSK, 0xff);
3440 else {
3441 tmp |= RB_NO_UPDATE;
3442 WREG32(SCRATCH_UMSK, 0);
3443 }
3444
3445 mdelay(1);
3446 WREG32(CP_RB0_CNTL, tmp);
3447
3448 WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
3449
3450 ring->rptr = RREG32(CP_RB0_RPTR);
3451
3452 /* ring1 - compute only */
3453 /* Set ring buffer size */
3454 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3455 rb_bufsz = order_base_2(ring->ring_size / 8);
3456 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3457 #ifdef __BIG_ENDIAN
3458 tmp |= BUF_SWAP_32BIT;
3459 #endif
3460 WREG32(CP_RB1_CNTL, tmp);
3461
3462 /* Initialize the ring buffer's read and write pointers */
3463 WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
3464 ring->wptr = 0;
3465 WREG32(CP_RB1_WPTR, ring->wptr);
3466
3467 /* set the wb address whether it's enabled or not */
3468 WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
3469 WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
3470
3471 mdelay(1);
3472 WREG32(CP_RB1_CNTL, tmp);
3473
3474 WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
3475
3476 ring->rptr = RREG32(CP_RB1_RPTR);
3477
3478 /* ring2 - compute only */
3479 /* Set ring buffer size */
3480 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3481 rb_bufsz = order_base_2(ring->ring_size / 8);
3482 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3483 #ifdef __BIG_ENDIAN
3484 tmp |= BUF_SWAP_32BIT;
3485 #endif
3486 WREG32(CP_RB2_CNTL, tmp);
3487
3488 /* Initialize the ring buffer's read and write pointers */
3489 WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
3490 ring->wptr = 0;
3491 WREG32(CP_RB2_WPTR, ring->wptr);
3492
3493 /* set the wb address whether it's enabled or not */
3494 WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
3495 WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
3496
3497 mdelay(1);
3498 WREG32(CP_RB2_CNTL, tmp);
3499
3500 WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
3501
3502 ring->rptr = RREG32(CP_RB2_RPTR);
3503
3504 /* start the rings */
3505 si_cp_start(rdev);
3506 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3507 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
3508 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
3509 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3510 if (r) {
3511 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3512 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3513 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3514 return r;
3515 }
3516 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3517 if (r) {
3518 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3519 }
3520 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3521 if (r) {
3522 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3523 }
3524
3525 si_enable_gui_idle_interrupt(rdev, true);
3526
3527 return 0;
3528 }
3529
3530 u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
3531 {
3532 u32 reset_mask = 0;
3533 u32 tmp;
3534
3535 /* GRBM_STATUS */
3536 tmp = RREG32(GRBM_STATUS);
3537 if (tmp & (PA_BUSY | SC_BUSY |
3538 BCI_BUSY | SX_BUSY |
3539 TA_BUSY | VGT_BUSY |
3540 DB_BUSY | CB_BUSY |
3541 GDS_BUSY | SPI_BUSY |
3542 IA_BUSY | IA_BUSY_NO_DMA))
3543 reset_mask |= RADEON_RESET_GFX;
3544
3545 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3546 CP_BUSY | CP_COHERENCY_BUSY))
3547 reset_mask |= RADEON_RESET_CP;
3548
3549 if (tmp & GRBM_EE_BUSY)
3550 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3551
3552 /* GRBM_STATUS2 */
3553 tmp = RREG32(GRBM_STATUS2);
3554 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3555 reset_mask |= RADEON_RESET_RLC;
3556
3557 /* DMA_STATUS_REG 0 */
3558 tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3559 if (!(tmp & DMA_IDLE))
3560 reset_mask |= RADEON_RESET_DMA;
3561
3562 /* DMA_STATUS_REG 1 */
3563 tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3564 if (!(tmp & DMA_IDLE))
3565 reset_mask |= RADEON_RESET_DMA1;
3566
3567 /* SRBM_STATUS2 */
3568 tmp = RREG32(SRBM_STATUS2);
3569 if (tmp & DMA_BUSY)
3570 reset_mask |= RADEON_RESET_DMA;
3571
3572 if (tmp & DMA1_BUSY)
3573 reset_mask |= RADEON_RESET_DMA1;
3574
3575 /* SRBM_STATUS */
3576 tmp = RREG32(SRBM_STATUS);
3577
3578 if (tmp & IH_BUSY)
3579 reset_mask |= RADEON_RESET_IH;
3580
3581 if (tmp & SEM_BUSY)
3582 reset_mask |= RADEON_RESET_SEM;
3583
3584 if (tmp & GRBM_RQ_PENDING)
3585 reset_mask |= RADEON_RESET_GRBM;
3586
3587 if (tmp & VMC_BUSY)
3588 reset_mask |= RADEON_RESET_VMC;
3589
3590 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3591 MCC_BUSY | MCD_BUSY))
3592 reset_mask |= RADEON_RESET_MC;
3593
3594 if (evergreen_is_display_hung(rdev))
3595 reset_mask |= RADEON_RESET_DISPLAY;
3596
3597 /* VM_L2_STATUS */
3598 tmp = RREG32(VM_L2_STATUS);
3599 if (tmp & L2_BUSY)
3600 reset_mask |= RADEON_RESET_VMC;
3601
3602 /* Skip MC reset as it's mostly likely not hung, just busy */
3603 if (reset_mask & RADEON_RESET_MC) {
3604 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3605 reset_mask &= ~RADEON_RESET_MC;
3606 }
3607
3608 return reset_mask;
3609 }
3610
3611 static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3612 {
3613 struct evergreen_mc_save save;
3614 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3615 u32 tmp;
3616
3617 if (reset_mask == 0)
3618 return;
3619
3620 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3621
3622 evergreen_print_gpu_status_regs(rdev);
3623 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3624 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3625 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3626 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3627
3628 /* disable PG/CG */
3629 si_fini_pg(rdev);
3630 si_fini_cg(rdev);
3631
3632 /* stop the rlc */
3633 si_rlc_stop(rdev);
3634
3635 /* Disable CP parsing/prefetching */
3636 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3637
3638 if (reset_mask & RADEON_RESET_DMA) {
3639 /* dma0 */
3640 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3641 tmp &= ~DMA_RB_ENABLE;
3642 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
3643 }
3644 if (reset_mask & RADEON_RESET_DMA1) {
3645 /* dma1 */
3646 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3647 tmp &= ~DMA_RB_ENABLE;
3648 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3649 }
3650
3651 udelay(50);
3652
3653 evergreen_mc_stop(rdev, &save);
3654 if (evergreen_mc_wait_for_idle(rdev)) {
3655 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3656 }
3657
3658 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3659 grbm_soft_reset = SOFT_RESET_CB |
3660 SOFT_RESET_DB |
3661 SOFT_RESET_GDS |
3662 SOFT_RESET_PA |
3663 SOFT_RESET_SC |
3664 SOFT_RESET_BCI |
3665 SOFT_RESET_SPI |
3666 SOFT_RESET_SX |
3667 SOFT_RESET_TC |
3668 SOFT_RESET_TA |
3669 SOFT_RESET_VGT |
3670 SOFT_RESET_IA;
3671 }
3672
3673 if (reset_mask & RADEON_RESET_CP) {
3674 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3675
3676 srbm_soft_reset |= SOFT_RESET_GRBM;
3677 }
3678
3679 if (reset_mask & RADEON_RESET_DMA)
3680 srbm_soft_reset |= SOFT_RESET_DMA;
3681
3682 if (reset_mask & RADEON_RESET_DMA1)
3683 srbm_soft_reset |= SOFT_RESET_DMA1;
3684
3685 if (reset_mask & RADEON_RESET_DISPLAY)
3686 srbm_soft_reset |= SOFT_RESET_DC;
3687
3688 if (reset_mask & RADEON_RESET_RLC)
3689 grbm_soft_reset |= SOFT_RESET_RLC;
3690
3691 if (reset_mask & RADEON_RESET_SEM)
3692 srbm_soft_reset |= SOFT_RESET_SEM;
3693
3694 if (reset_mask & RADEON_RESET_IH)
3695 srbm_soft_reset |= SOFT_RESET_IH;
3696
3697 if (reset_mask & RADEON_RESET_GRBM)
3698 srbm_soft_reset |= SOFT_RESET_GRBM;
3699
3700 if (reset_mask & RADEON_RESET_VMC)
3701 srbm_soft_reset |= SOFT_RESET_VMC;
3702
3703 if (reset_mask & RADEON_RESET_MC)
3704 srbm_soft_reset |= SOFT_RESET_MC;
3705
3706 if (grbm_soft_reset) {
3707 tmp = RREG32(GRBM_SOFT_RESET);
3708 tmp |= grbm_soft_reset;
3709 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3710 WREG32(GRBM_SOFT_RESET, tmp);
3711 tmp = RREG32(GRBM_SOFT_RESET);
3712
3713 udelay(50);
3714
3715 tmp &= ~grbm_soft_reset;
3716 WREG32(GRBM_SOFT_RESET, tmp);
3717 tmp = RREG32(GRBM_SOFT_RESET);
3718 }
3719
3720 if (srbm_soft_reset) {
3721 tmp = RREG32(SRBM_SOFT_RESET);
3722 tmp |= srbm_soft_reset;
3723 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3724 WREG32(SRBM_SOFT_RESET, tmp);
3725 tmp = RREG32(SRBM_SOFT_RESET);
3726
3727 udelay(50);
3728
3729 tmp &= ~srbm_soft_reset;
3730 WREG32(SRBM_SOFT_RESET, tmp);
3731 tmp = RREG32(SRBM_SOFT_RESET);
3732 }
3733
3734 /* Wait a little for things to settle down */
3735 udelay(50);
3736
3737 evergreen_mc_resume(rdev, &save);
3738 udelay(50);
3739
3740 evergreen_print_gpu_status_regs(rdev);
3741 }
3742
3743 int si_asic_reset(struct radeon_device *rdev)
3744 {
3745 u32 reset_mask;
3746
3747 reset_mask = si_gpu_check_soft_reset(rdev);
3748
3749 if (reset_mask)
3750 r600_set_bios_scratch_engine_hung(rdev, true);
3751
3752 si_gpu_soft_reset(rdev, reset_mask);
3753
3754 reset_mask = si_gpu_check_soft_reset(rdev);
3755
3756 if (!reset_mask)
3757 r600_set_bios_scratch_engine_hung(rdev, false);
3758
3759 return 0;
3760 }
3761
3762 /**
3763 * si_gfx_is_lockup - Check if the GFX engine is locked up
3764 *
3765 * @rdev: radeon_device pointer
3766 * @ring: radeon_ring structure holding ring information
3767 *
3768 * Check if the GFX engine is locked up.
3769 * Returns true if the engine appears to be locked up, false if not.
3770 */
3771 bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3772 {
3773 u32 reset_mask = si_gpu_check_soft_reset(rdev);
3774
3775 if (!(reset_mask & (RADEON_RESET_GFX |
3776 RADEON_RESET_COMPUTE |
3777 RADEON_RESET_CP))) {
3778 radeon_ring_lockup_update(ring);
3779 return false;
3780 }
3781 /* force CP activities */
3782 radeon_ring_force_activity(rdev, ring);
3783 return radeon_ring_test_lockup(rdev, ring);
3784 }
3785
3786 /* MC */
3787 static void si_mc_program(struct radeon_device *rdev)
3788 {
3789 struct evergreen_mc_save save;
3790 u32 tmp;
3791 int i, j;
3792
3793 /* Initialize HDP */
3794 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3795 WREG32((0x2c14 + j), 0x00000000);
3796 WREG32((0x2c18 + j), 0x00000000);
3797 WREG32((0x2c1c + j), 0x00000000);
3798 WREG32((0x2c20 + j), 0x00000000);
3799 WREG32((0x2c24 + j), 0x00000000);
3800 }
3801 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
3802
3803 evergreen_mc_stop(rdev, &save);
3804 if (radeon_mc_wait_for_idle(rdev)) {
3805 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3806 }
3807 if (!ASIC_IS_NODCE(rdev))
3808 /* Lockout access through VGA aperture*/
3809 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
3810 /* Update configuration */
3811 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
3812 rdev->mc.vram_start >> 12);
3813 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
3814 rdev->mc.vram_end >> 12);
3815 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
3816 rdev->vram_scratch.gpu_addr >> 12);
3817 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
3818 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
3819 WREG32(MC_VM_FB_LOCATION, tmp);
3820 /* XXX double check these! */
3821 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
3822 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
3823 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
3824 WREG32(MC_VM_AGP_BASE, 0);
3825 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
3826 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
3827 if (radeon_mc_wait_for_idle(rdev)) {
3828 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3829 }
3830 evergreen_mc_resume(rdev, &save);
3831 if (!ASIC_IS_NODCE(rdev)) {
3832 /* we need to own VRAM, so turn off the VGA renderer here
3833 * to stop it overwriting our objects */
3834 rv515_vga_render_disable(rdev);
3835 }
3836 }
3837
3838 void si_vram_gtt_location(struct radeon_device *rdev,
3839 struct radeon_mc *mc)
3840 {
3841 if (mc->mc_vram_size > 0xFFC0000000ULL) {
3842 /* leave room for at least 1024M GTT */
3843 dev_warn(rdev->dev, "limiting VRAM\n");
3844 mc->real_vram_size = 0xFFC0000000ULL;
3845 mc->mc_vram_size = 0xFFC0000000ULL;
3846 }
3847 radeon_vram_location(rdev, &rdev->mc, 0);
3848 rdev->mc.gtt_base_align = 0;
3849 radeon_gtt_location(rdev, mc);
3850 }
3851
3852 static int si_mc_init(struct radeon_device *rdev)
3853 {
3854 u32 tmp;
3855 int chansize, numchan;
3856
3857 /* Get VRAM informations */
3858 rdev->mc.vram_is_ddr = true;
3859 tmp = RREG32(MC_ARB_RAMCFG);
3860 if (tmp & CHANSIZE_OVERRIDE) {
3861 chansize = 16;
3862 } else if (tmp & CHANSIZE_MASK) {
3863 chansize = 64;
3864 } else {
3865 chansize = 32;
3866 }
3867 tmp = RREG32(MC_SHARED_CHMAP);
3868 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3869 case 0:
3870 default:
3871 numchan = 1;
3872 break;
3873 case 1:
3874 numchan = 2;
3875 break;
3876 case 2:
3877 numchan = 4;
3878 break;
3879 case 3:
3880 numchan = 8;
3881 break;
3882 case 4:
3883 numchan = 3;
3884 break;
3885 case 5:
3886 numchan = 6;
3887 break;
3888 case 6:
3889 numchan = 10;
3890 break;
3891 case 7:
3892 numchan = 12;
3893 break;
3894 case 8:
3895 numchan = 16;
3896 break;
3897 }
3898 rdev->mc.vram_width = numchan * chansize;
3899 /* Could aper size report 0 ? */
3900 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3901 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3902 /* size in MB on si */
3903 tmp = RREG32(CONFIG_MEMSIZE);
3904 /* some boards may have garbage in the upper 16 bits */
3905 if (tmp & 0xffff0000) {
3906 DRM_INFO("Probable bad vram size: 0x%08x\n", tmp);
3907 if (tmp & 0xffff)
3908 tmp &= 0xffff;
3909 }
3910 rdev->mc.mc_vram_size = tmp * 1024ULL * 1024ULL;
3911 rdev->mc.real_vram_size = rdev->mc.mc_vram_size;
3912 rdev->mc.visible_vram_size = rdev->mc.aper_size;
3913 si_vram_gtt_location(rdev, &rdev->mc);
3914 radeon_update_bandwidth_info(rdev);
3915
3916 return 0;
3917 }
3918
3919 /*
3920 * GART
3921 */
3922 void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
3923 {
3924 /* flush hdp cache */
3925 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
3926
3927 /* bits 0-15 are the VM contexts0-15 */
3928 WREG32(VM_INVALIDATE_REQUEST, 1);
3929 }
3930
3931 static int si_pcie_gart_enable(struct radeon_device *rdev)
3932 {
3933 int r, i;
3934
3935 if (rdev->gart.robj == NULL) {
3936 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
3937 return -EINVAL;
3938 }
3939 r = radeon_gart_table_vram_pin(rdev);
3940 if (r)
3941 return r;
3942 radeon_gart_restore(rdev);
3943 /* Setup TLB control */
3944 WREG32(MC_VM_MX_L1_TLB_CNTL,
3945 (0xA << 7) |
3946 ENABLE_L1_TLB |
3947 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
3948 ENABLE_ADVANCED_DRIVER_MODEL |
3949 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
3950 /* Setup L2 cache */
3951 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
3952 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
3953 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
3954 EFFECTIVE_L2_QUEUE_SIZE(7) |
3955 CONTEXT1_IDENTITY_ACCESS_MODE(1));
3956 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
3957 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
3958 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
3959 /* setup context0 */
3960 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
3961 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
3962 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
3963 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
3964 (u32)(rdev->dummy_page.addr >> 12));
3965 WREG32(VM_CONTEXT0_CNTL2, 0);
3966 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
3967 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
3968
3969 WREG32(0x15D4, 0);
3970 WREG32(0x15D8, 0);
3971 WREG32(0x15DC, 0);
3972
3973 /* empty context1-15 */
3974 /* set vm size, must be a multiple of 4 */
3975 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
3976 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
3977 /* Assign the pt base to something valid for now; the pts used for
3978 * the VMs are determined by the application and setup and assigned
3979 * on the fly in the vm part of radeon_gart.c
3980 */
3981 for (i = 1; i < 16; i++) {
3982 if (i < 8)
3983 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
3984 rdev->gart.table_addr >> 12);
3985 else
3986 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
3987 rdev->gart.table_addr >> 12);
3988 }
3989
3990 /* enable context1-15 */
3991 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
3992 (u32)(rdev->dummy_page.addr >> 12));
3993 WREG32(VM_CONTEXT1_CNTL2, 4);
3994 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
3995 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3996 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
3997 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3998 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
3999 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
4000 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
4001 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
4002 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
4003 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
4004 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
4005 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4006 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
4007
4008 si_pcie_gart_tlb_flush(rdev);
4009 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
4010 (unsigned)(rdev->mc.gtt_size >> 20),
4011 (unsigned long long)rdev->gart.table_addr);
4012 rdev->gart.ready = true;
4013 return 0;
4014 }
4015
4016 static void si_pcie_gart_disable(struct radeon_device *rdev)
4017 {
4018 /* Disable all tables */
4019 WREG32(VM_CONTEXT0_CNTL, 0);
4020 WREG32(VM_CONTEXT1_CNTL, 0);
4021 /* Setup TLB control */
4022 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4023 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4024 /* Setup L2 cache */
4025 WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4026 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4027 EFFECTIVE_L2_QUEUE_SIZE(7) |
4028 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4029 WREG32(VM_L2_CNTL2, 0);
4030 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4031 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
4032 radeon_gart_table_vram_unpin(rdev);
4033 }
4034
4035 static void si_pcie_gart_fini(struct radeon_device *rdev)
4036 {
4037 si_pcie_gart_disable(rdev);
4038 radeon_gart_table_vram_free(rdev);
4039 radeon_gart_fini(rdev);
4040 }
4041
4042 /* vm parser */
4043 static bool si_vm_reg_valid(u32 reg)
4044 {
4045 /* context regs are fine */
4046 if (reg >= 0x28000)
4047 return true;
4048
4049 /* check config regs */
4050 switch (reg) {
4051 case GRBM_GFX_INDEX:
4052 case CP_STRMOUT_CNTL:
4053 case VGT_VTX_VECT_EJECT_REG:
4054 case VGT_CACHE_INVALIDATION:
4055 case VGT_ESGS_RING_SIZE:
4056 case VGT_GSVS_RING_SIZE:
4057 case VGT_GS_VERTEX_REUSE:
4058 case VGT_PRIMITIVE_TYPE:
4059 case VGT_INDEX_TYPE:
4060 case VGT_NUM_INDICES:
4061 case VGT_NUM_INSTANCES:
4062 case VGT_TF_RING_SIZE:
4063 case VGT_HS_OFFCHIP_PARAM:
4064 case VGT_TF_MEMORY_BASE:
4065 case PA_CL_ENHANCE:
4066 case PA_SU_LINE_STIPPLE_VALUE:
4067 case PA_SC_LINE_STIPPLE_STATE:
4068 case PA_SC_ENHANCE:
4069 case SQC_CACHES:
4070 case SPI_STATIC_THREAD_MGMT_1:
4071 case SPI_STATIC_THREAD_MGMT_2:
4072 case SPI_STATIC_THREAD_MGMT_3:
4073 case SPI_PS_MAX_WAVE_ID:
4074 case SPI_CONFIG_CNTL:
4075 case SPI_CONFIG_CNTL_1:
4076 case TA_CNTL_AUX:
4077 return true;
4078 default:
4079 DRM_ERROR("Invalid register 0x%x in CS\n", reg);
4080 return false;
4081 }
4082 }
4083
4084 static int si_vm_packet3_ce_check(struct radeon_device *rdev,
4085 u32 *ib, struct radeon_cs_packet *pkt)
4086 {
4087 switch (pkt->opcode) {
4088 case PACKET3_NOP:
4089 case PACKET3_SET_BASE:
4090 case PACKET3_SET_CE_DE_COUNTERS:
4091 case PACKET3_LOAD_CONST_RAM:
4092 case PACKET3_WRITE_CONST_RAM:
4093 case PACKET3_WRITE_CONST_RAM_OFFSET:
4094 case PACKET3_DUMP_CONST_RAM:
4095 case PACKET3_INCREMENT_CE_COUNTER:
4096 case PACKET3_WAIT_ON_DE_COUNTER:
4097 case PACKET3_CE_WRITE:
4098 break;
4099 default:
4100 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
4101 return -EINVAL;
4102 }
4103 return 0;
4104 }
4105
4106 static int si_vm_packet3_cp_dma_check(u32 *ib, u32 idx)
4107 {
4108 u32 start_reg, reg, i;
4109 u32 command = ib[idx + 4];
4110 u32 info = ib[idx + 1];
4111 u32 idx_value = ib[idx];
4112 if (command & PACKET3_CP_DMA_CMD_SAS) {
4113 /* src address space is register */
4114 if (((info & 0x60000000) >> 29) == 0) {
4115 start_reg = idx_value << 2;
4116 if (command & PACKET3_CP_DMA_CMD_SAIC) {
4117 reg = start_reg;
4118 if (!si_vm_reg_valid(reg)) {
4119 DRM_ERROR("CP DMA Bad SRC register\n");
4120 return -EINVAL;
4121 }
4122 } else {
4123 for (i = 0; i < (command & 0x1fffff); i++) {
4124 reg = start_reg + (4 * i);
4125 if (!si_vm_reg_valid(reg)) {
4126 DRM_ERROR("CP DMA Bad SRC register\n");
4127 return -EINVAL;
4128 }
4129 }
4130 }
4131 }
4132 }
4133 if (command & PACKET3_CP_DMA_CMD_DAS) {
4134 /* dst address space is register */
4135 if (((info & 0x00300000) >> 20) == 0) {
4136 start_reg = ib[idx + 2];
4137 if (command & PACKET3_CP_DMA_CMD_DAIC) {
4138 reg = start_reg;
4139 if (!si_vm_reg_valid(reg)) {
4140 DRM_ERROR("CP DMA Bad DST register\n");
4141 return -EINVAL;
4142 }
4143 } else {
4144 for (i = 0; i < (command & 0x1fffff); i++) {
4145 reg = start_reg + (4 * i);
4146 if (!si_vm_reg_valid(reg)) {
4147 DRM_ERROR("CP DMA Bad DST register\n");
4148 return -EINVAL;
4149 }
4150 }
4151 }
4152 }
4153 }
4154 return 0;
4155 }
4156
4157 static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
4158 u32 *ib, struct radeon_cs_packet *pkt)
4159 {
4160 int r;
4161 u32 idx = pkt->idx + 1;
4162 u32 idx_value = ib[idx];
4163 u32 start_reg, end_reg, reg, i;
4164
4165 switch (pkt->opcode) {
4166 case PACKET3_NOP:
4167 case PACKET3_SET_BASE:
4168 case PACKET3_CLEAR_STATE:
4169 case PACKET3_INDEX_BUFFER_SIZE:
4170 case PACKET3_DISPATCH_DIRECT:
4171 case PACKET3_DISPATCH_INDIRECT:
4172 case PACKET3_ALLOC_GDS:
4173 case PACKET3_WRITE_GDS_RAM:
4174 case PACKET3_ATOMIC_GDS:
4175 case PACKET3_ATOMIC:
4176 case PACKET3_OCCLUSION_QUERY:
4177 case PACKET3_SET_PREDICATION:
4178 case PACKET3_COND_EXEC:
4179 case PACKET3_PRED_EXEC:
4180 case PACKET3_DRAW_INDIRECT:
4181 case PACKET3_DRAW_INDEX_INDIRECT:
4182 case PACKET3_INDEX_BASE:
4183 case PACKET3_DRAW_INDEX_2:
4184 case PACKET3_CONTEXT_CONTROL:
4185 case PACKET3_INDEX_TYPE:
4186 case PACKET3_DRAW_INDIRECT_MULTI:
4187 case PACKET3_DRAW_INDEX_AUTO:
4188 case PACKET3_DRAW_INDEX_IMMD:
4189 case PACKET3_NUM_INSTANCES:
4190 case PACKET3_DRAW_INDEX_MULTI_AUTO:
4191 case PACKET3_STRMOUT_BUFFER_UPDATE:
4192 case PACKET3_DRAW_INDEX_OFFSET_2:
4193 case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
4194 case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
4195 case PACKET3_MPEG_INDEX:
4196 case PACKET3_WAIT_REG_MEM:
4197 case PACKET3_MEM_WRITE:
4198 case PACKET3_PFP_SYNC_ME:
4199 case PACKET3_SURFACE_SYNC:
4200 case PACKET3_EVENT_WRITE:
4201 case PACKET3_EVENT_WRITE_EOP:
4202 case PACKET3_EVENT_WRITE_EOS:
4203 case PACKET3_SET_CONTEXT_REG:
4204 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4205 case PACKET3_SET_SH_REG:
4206 case PACKET3_SET_SH_REG_OFFSET:
4207 case PACKET3_INCREMENT_DE_COUNTER:
4208 case PACKET3_WAIT_ON_CE_COUNTER:
4209 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4210 case PACKET3_ME_WRITE:
4211 break;
4212 case PACKET3_COPY_DATA:
4213 if ((idx_value & 0xf00) == 0) {
4214 reg = ib[idx + 3] * 4;
4215 if (!si_vm_reg_valid(reg))
4216 return -EINVAL;
4217 }
4218 break;
4219 case PACKET3_WRITE_DATA:
4220 if ((idx_value & 0xf00) == 0) {
4221 start_reg = ib[idx + 1] * 4;
4222 if (idx_value & 0x10000) {
4223 if (!si_vm_reg_valid(start_reg))
4224 return -EINVAL;
4225 } else {
4226 for (i = 0; i < (pkt->count - 2); i++) {
4227 reg = start_reg + (4 * i);
4228 if (!si_vm_reg_valid(reg))
4229 return -EINVAL;
4230 }
4231 }
4232 }
4233 break;
4234 case PACKET3_COND_WRITE:
4235 if (idx_value & 0x100) {
4236 reg = ib[idx + 5] * 4;
4237 if (!si_vm_reg_valid(reg))
4238 return -EINVAL;
4239 }
4240 break;
4241 case PACKET3_COPY_DW:
4242 if (idx_value & 0x2) {
4243 reg = ib[idx + 3] * 4;
4244 if (!si_vm_reg_valid(reg))
4245 return -EINVAL;
4246 }
4247 break;
4248 case PACKET3_SET_CONFIG_REG:
4249 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
4250 end_reg = 4 * pkt->count + start_reg - 4;
4251 if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
4252 (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
4253 (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
4254 DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
4255 return -EINVAL;
4256 }
4257 for (i = 0; i < pkt->count; i++) {
4258 reg = start_reg + (4 * i);
4259 if (!si_vm_reg_valid(reg))
4260 return -EINVAL;
4261 }
4262 break;
4263 case PACKET3_CP_DMA:
4264 r = si_vm_packet3_cp_dma_check(ib, idx);
4265 if (r)
4266 return r;
4267 break;
4268 default:
4269 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
4270 return -EINVAL;
4271 }
4272 return 0;
4273 }
4274
4275 static int si_vm_packet3_compute_check(struct radeon_device *rdev,
4276 u32 *ib, struct radeon_cs_packet *pkt)
4277 {
4278 int r;
4279 u32 idx = pkt->idx + 1;
4280 u32 idx_value = ib[idx];
4281 u32 start_reg, reg, i;
4282
4283 switch (pkt->opcode) {
4284 case PACKET3_NOP:
4285 case PACKET3_SET_BASE:
4286 case PACKET3_CLEAR_STATE:
4287 case PACKET3_DISPATCH_DIRECT:
4288 case PACKET3_DISPATCH_INDIRECT:
4289 case PACKET3_ALLOC_GDS:
4290 case PACKET3_WRITE_GDS_RAM:
4291 case PACKET3_ATOMIC_GDS:
4292 case PACKET3_ATOMIC:
4293 case PACKET3_OCCLUSION_QUERY:
4294 case PACKET3_SET_PREDICATION:
4295 case PACKET3_COND_EXEC:
4296 case PACKET3_PRED_EXEC:
4297 case PACKET3_CONTEXT_CONTROL:
4298 case PACKET3_STRMOUT_BUFFER_UPDATE:
4299 case PACKET3_WAIT_REG_MEM:
4300 case PACKET3_MEM_WRITE:
4301 case PACKET3_PFP_SYNC_ME:
4302 case PACKET3_SURFACE_SYNC:
4303 case PACKET3_EVENT_WRITE:
4304 case PACKET3_EVENT_WRITE_EOP:
4305 case PACKET3_EVENT_WRITE_EOS:
4306 case PACKET3_SET_CONTEXT_REG:
4307 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4308 case PACKET3_SET_SH_REG:
4309 case PACKET3_SET_SH_REG_OFFSET:
4310 case PACKET3_INCREMENT_DE_COUNTER:
4311 case PACKET3_WAIT_ON_CE_COUNTER:
4312 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4313 case PACKET3_ME_WRITE:
4314 break;
4315 case PACKET3_COPY_DATA:
4316 if ((idx_value & 0xf00) == 0) {
4317 reg = ib[idx + 3] * 4;
4318 if (!si_vm_reg_valid(reg))
4319 return -EINVAL;
4320 }
4321 break;
4322 case PACKET3_WRITE_DATA:
4323 if ((idx_value & 0xf00) == 0) {
4324 start_reg = ib[idx + 1] * 4;
4325 if (idx_value & 0x10000) {
4326 if (!si_vm_reg_valid(start_reg))
4327 return -EINVAL;
4328 } else {
4329 for (i = 0; i < (pkt->count - 2); i++) {
4330 reg = start_reg + (4 * i);
4331 if (!si_vm_reg_valid(reg))
4332 return -EINVAL;
4333 }
4334 }
4335 }
4336 break;
4337 case PACKET3_COND_WRITE:
4338 if (idx_value & 0x100) {
4339 reg = ib[idx + 5] * 4;
4340 if (!si_vm_reg_valid(reg))
4341 return -EINVAL;
4342 }
4343 break;
4344 case PACKET3_COPY_DW:
4345 if (idx_value & 0x2) {
4346 reg = ib[idx + 3] * 4;
4347 if (!si_vm_reg_valid(reg))
4348 return -EINVAL;
4349 }
4350 break;
4351 case PACKET3_CP_DMA:
4352 r = si_vm_packet3_cp_dma_check(ib, idx);
4353 if (r)
4354 return r;
4355 break;
4356 default:
4357 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
4358 return -EINVAL;
4359 }
4360 return 0;
4361 }
4362
4363 int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4364 {
4365 int ret = 0;
4366 u32 idx = 0;
4367 struct radeon_cs_packet pkt;
4368
4369 do {
4370 pkt.idx = idx;
4371 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
4372 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
4373 pkt.one_reg_wr = 0;
4374 switch (pkt.type) {
4375 case RADEON_PACKET_TYPE0:
4376 dev_err(rdev->dev, "Packet0 not allowed!\n");
4377 ret = -EINVAL;
4378 break;
4379 case RADEON_PACKET_TYPE2:
4380 idx += 1;
4381 break;
4382 case RADEON_PACKET_TYPE3:
4383 pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
4384 if (ib->is_const_ib)
4385 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
4386 else {
4387 switch (ib->ring) {
4388 case RADEON_RING_TYPE_GFX_INDEX:
4389 ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
4390 break;
4391 case CAYMAN_RING_TYPE_CP1_INDEX:
4392 case CAYMAN_RING_TYPE_CP2_INDEX:
4393 ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
4394 break;
4395 default:
4396 dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
4397 ret = -EINVAL;
4398 break;
4399 }
4400 }
4401 idx += pkt.count + 2;
4402 break;
4403 default:
4404 dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
4405 ret = -EINVAL;
4406 break;
4407 }
4408 if (ret)
4409 break;
4410 } while (idx < ib->length_dw);
4411
4412 return ret;
4413 }
4414
4415 /*
4416 * vm
4417 */
4418 int si_vm_init(struct radeon_device *rdev)
4419 {
4420 /* number of VMs */
4421 rdev->vm_manager.nvm = 16;
4422 /* base offset of vram pages */
4423 rdev->vm_manager.vram_base_offset = 0;
4424
4425 return 0;
4426 }
4427
4428 void si_vm_fini(struct radeon_device *rdev)
4429 {
4430 }
4431
4432 /**
4433 * si_vm_decode_fault - print human readable fault info
4434 *
4435 * @rdev: radeon_device pointer
4436 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
4437 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
4438 *
4439 * Print human readable fault information (SI).
4440 */
4441 static void si_vm_decode_fault(struct radeon_device *rdev,
4442 u32 status, u32 addr)
4443 {
4444 u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
4445 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
4446 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
4447 char *block;
4448
4449 if (rdev->family == CHIP_TAHITI) {
4450 switch (mc_id) {
4451 case 160:
4452 case 144:
4453 case 96:
4454 case 80:
4455 case 224:
4456 case 208:
4457 case 32:
4458 case 16:
4459 block = "CB";
4460 break;
4461 case 161:
4462 case 145:
4463 case 97:
4464 case 81:
4465 case 225:
4466 case 209:
4467 case 33:
4468 case 17:
4469 block = "CB_FMASK";
4470 break;
4471 case 162:
4472 case 146:
4473 case 98:
4474 case 82:
4475 case 226:
4476 case 210:
4477 case 34:
4478 case 18:
4479 block = "CB_CMASK";
4480 break;
4481 case 163:
4482 case 147:
4483 case 99:
4484 case 83:
4485 case 227:
4486 case 211:
4487 case 35:
4488 case 19:
4489 block = "CB_IMMED";
4490 break;
4491 case 164:
4492 case 148:
4493 case 100:
4494 case 84:
4495 case 228:
4496 case 212:
4497 case 36:
4498 case 20:
4499 block = "DB";
4500 break;
4501 case 165:
4502 case 149:
4503 case 101:
4504 case 85:
4505 case 229:
4506 case 213:
4507 case 37:
4508 case 21:
4509 block = "DB_HTILE";
4510 break;
4511 case 167:
4512 case 151:
4513 case 103:
4514 case 87:
4515 case 231:
4516 case 215:
4517 case 39:
4518 case 23:
4519 block = "DB_STEN";
4520 break;
4521 case 72:
4522 case 68:
4523 case 64:
4524 case 8:
4525 case 4:
4526 case 0:
4527 case 136:
4528 case 132:
4529 case 128:
4530 case 200:
4531 case 196:
4532 case 192:
4533 block = "TC";
4534 break;
4535 case 112:
4536 case 48:
4537 block = "CP";
4538 break;
4539 case 49:
4540 case 177:
4541 case 50:
4542 case 178:
4543 block = "SH";
4544 break;
4545 case 53:
4546 case 190:
4547 block = "VGT";
4548 break;
4549 case 117:
4550 block = "IH";
4551 break;
4552 case 51:
4553 case 115:
4554 block = "RLC";
4555 break;
4556 case 119:
4557 case 183:
4558 block = "DMA0";
4559 break;
4560 case 61:
4561 block = "DMA1";
4562 break;
4563 case 248:
4564 case 120:
4565 block = "HDP";
4566 break;
4567 default:
4568 block = "unknown";
4569 break;
4570 }
4571 } else {
4572 switch (mc_id) {
4573 case 32:
4574 case 16:
4575 case 96:
4576 case 80:
4577 case 160:
4578 case 144:
4579 case 224:
4580 case 208:
4581 block = "CB";
4582 break;
4583 case 33:
4584 case 17:
4585 case 97:
4586 case 81:
4587 case 161:
4588 case 145:
4589 case 225:
4590 case 209:
4591 block = "CB_FMASK";
4592 break;
4593 case 34:
4594 case 18:
4595 case 98:
4596 case 82:
4597 case 162:
4598 case 146:
4599 case 226:
4600 case 210:
4601 block = "CB_CMASK";
4602 break;
4603 case 35:
4604 case 19:
4605 case 99:
4606 case 83:
4607 case 163:
4608 case 147:
4609 case 227:
4610 case 211:
4611 block = "CB_IMMED";
4612 break;
4613 case 36:
4614 case 20:
4615 case 100:
4616 case 84:
4617 case 164:
4618 case 148:
4619 case 228:
4620 case 212:
4621 block = "DB";
4622 break;
4623 case 37:
4624 case 21:
4625 case 101:
4626 case 85:
4627 case 165:
4628 case 149:
4629 case 229:
4630 case 213:
4631 block = "DB_HTILE";
4632 break;
4633 case 39:
4634 case 23:
4635 case 103:
4636 case 87:
4637 case 167:
4638 case 151:
4639 case 231:
4640 case 215:
4641 block = "DB_STEN";
4642 break;
4643 case 72:
4644 case 68:
4645 case 8:
4646 case 4:
4647 case 136:
4648 case 132:
4649 case 200:
4650 case 196:
4651 block = "TC";
4652 break;
4653 case 112:
4654 case 48:
4655 block = "CP";
4656 break;
4657 case 49:
4658 case 177:
4659 case 50:
4660 case 178:
4661 block = "SH";
4662 break;
4663 case 53:
4664 block = "VGT";
4665 break;
4666 case 117:
4667 block = "IH";
4668 break;
4669 case 51:
4670 case 115:
4671 block = "RLC";
4672 break;
4673 case 119:
4674 case 183:
4675 block = "DMA0";
4676 break;
4677 case 61:
4678 block = "DMA1";
4679 break;
4680 case 248:
4681 case 120:
4682 block = "HDP";
4683 break;
4684 default:
4685 block = "unknown";
4686 break;
4687 }
4688 }
4689
4690 printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
4691 protections, vmid, addr,
4692 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
4693 block, mc_id);
4694 }
4695
4696 /**
4697 * si_vm_set_page - update the page tables using the CP
4698 *
4699 * @rdev: radeon_device pointer
4700 * @ib: indirect buffer to fill with commands
4701 * @pe: addr of the page entry
4702 * @addr: dst addr to write into pe
4703 * @count: number of page entries to update
4704 * @incr: increase next addr by incr bytes
4705 * @flags: access flags
4706 *
4707 * Update the page tables using the CP (SI).
4708 */
4709 void si_vm_set_page(struct radeon_device *rdev,
4710 struct radeon_ib *ib,
4711 uint64_t pe,
4712 uint64_t addr, unsigned count,
4713 uint32_t incr, uint32_t flags)
4714 {
4715 uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
4716 uint64_t value;
4717 unsigned ndw;
4718
4719 if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) {
4720 while (count) {
4721 ndw = 2 + count * 2;
4722 if (ndw > 0x3FFE)
4723 ndw = 0x3FFE;
4724
4725 ib->ptr[ib->length_dw++] = PACKET3(PACKET3_WRITE_DATA, ndw);
4726 ib->ptr[ib->length_dw++] = (WRITE_DATA_ENGINE_SEL(0) |
4727 WRITE_DATA_DST_SEL(1));
4728 ib->ptr[ib->length_dw++] = pe;
4729 ib->ptr[ib->length_dw++] = upper_32_bits(pe);
4730 for (; ndw > 2; ndw -= 2, --count, pe += 8) {
4731 if (flags & RADEON_VM_PAGE_SYSTEM) {
4732 value = radeon_vm_map_gart(rdev, addr);
4733 value &= 0xFFFFFFFFFFFFF000ULL;
4734 } else if (flags & RADEON_VM_PAGE_VALID) {
4735 value = addr;
4736 } else {
4737 value = 0;
4738 }
4739 addr += incr;
4740 value |= r600_flags;
4741 ib->ptr[ib->length_dw++] = value;
4742 ib->ptr[ib->length_dw++] = upper_32_bits(value);
4743 }
4744 }
4745 } else {
4746 /* DMA */
4747 si_dma_vm_set_page(rdev, ib, pe, addr, count, incr, flags);
4748 }
4749 }
4750
4751 void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
4752 {
4753 struct radeon_ring *ring = &rdev->ring[ridx];
4754
4755 if (vm == NULL)
4756 return;
4757
4758 /* write new base address */
4759 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4760 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4761 WRITE_DATA_DST_SEL(0)));
4762
4763 if (vm->id < 8) {
4764 radeon_ring_write(ring,
4765 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
4766 } else {
4767 radeon_ring_write(ring,
4768 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
4769 }
4770 radeon_ring_write(ring, 0);
4771 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
4772
4773 /* flush hdp cache */
4774 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4775 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4776 WRITE_DATA_DST_SEL(0)));
4777 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
4778 radeon_ring_write(ring, 0);
4779 radeon_ring_write(ring, 0x1);
4780
4781 /* bits 0-15 are the VM contexts0-15 */
4782 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4783 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4784 WRITE_DATA_DST_SEL(0)));
4785 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
4786 radeon_ring_write(ring, 0);
4787 radeon_ring_write(ring, 1 << vm->id);
4788
4789 /* sync PFP to ME, otherwise we might get invalid PFP reads */
4790 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
4791 radeon_ring_write(ring, 0x0);
4792 }
4793
4794 /*
4795 * Power and clock gating
4796 */
4797 static void si_wait_for_rlc_serdes(struct radeon_device *rdev)
4798 {
4799 int i;
4800
4801 for (i = 0; i < rdev->usec_timeout; i++) {
4802 if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0)
4803 break;
4804 udelay(1);
4805 }
4806
4807 for (i = 0; i < rdev->usec_timeout; i++) {
4808 if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0)
4809 break;
4810 udelay(1);
4811 }
4812 }
4813
4814 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
4815 bool enable)
4816 {
4817 u32 tmp = RREG32(CP_INT_CNTL_RING0);
4818 u32 mask;
4819 int i;
4820
4821 if (enable)
4822 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4823 else
4824 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4825 WREG32(CP_INT_CNTL_RING0, tmp);
4826
4827 if (!enable) {
4828 /* read a gfx register */
4829 tmp = RREG32(DB_DEPTH_INFO);
4830
4831 mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS;
4832 for (i = 0; i < rdev->usec_timeout; i++) {
4833 if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS))
4834 break;
4835 udelay(1);
4836 }
4837 }
4838 }
4839
4840 static void si_set_uvd_dcm(struct radeon_device *rdev,
4841 bool sw_mode)
4842 {
4843 u32 tmp, tmp2;
4844
4845 tmp = RREG32(UVD_CGC_CTRL);
4846 tmp &= ~(CLK_OD_MASK | CG_DT_MASK);
4847 tmp |= DCM | CG_DT(1) | CLK_OD(4);
4848
4849 if (sw_mode) {
4850 tmp &= ~0x7ffff800;
4851 tmp2 = DYN_OR_EN | DYN_RR_EN | G_DIV_ID(7);
4852 } else {
4853 tmp |= 0x7ffff800;
4854 tmp2 = 0;
4855 }
4856
4857 WREG32(UVD_CGC_CTRL, tmp);
4858 WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2);
4859 }
4860
4861 void si_init_uvd_internal_cg(struct radeon_device *rdev)
4862 {
4863 bool hw_mode = true;
4864
4865 if (hw_mode) {
4866 si_set_uvd_dcm(rdev, false);
4867 } else {
4868 u32 tmp = RREG32(UVD_CGC_CTRL);
4869 tmp &= ~DCM;
4870 WREG32(UVD_CGC_CTRL, tmp);
4871 }
4872 }
4873
4874 static u32 si_halt_rlc(struct radeon_device *rdev)
4875 {
4876 u32 data, orig;
4877
4878 orig = data = RREG32(RLC_CNTL);
4879
4880 if (data & RLC_ENABLE) {
4881 data &= ~RLC_ENABLE;
4882 WREG32(RLC_CNTL, data);
4883
4884 si_wait_for_rlc_serdes(rdev);
4885 }
4886
4887 return orig;
4888 }
4889
4890 static void si_update_rlc(struct radeon_device *rdev, u32 rlc)
4891 {
4892 u32 tmp;
4893
4894 tmp = RREG32(RLC_CNTL);
4895 if (tmp != rlc)
4896 WREG32(RLC_CNTL, rlc);
4897 }
4898
4899 static void si_enable_dma_pg(struct radeon_device *rdev, bool enable)
4900 {
4901 u32 data, orig;
4902
4903 orig = data = RREG32(DMA_PG);
4904 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA))
4905 data |= PG_CNTL_ENABLE;
4906 else
4907 data &= ~PG_CNTL_ENABLE;
4908 if (orig != data)
4909 WREG32(DMA_PG, data);
4910 }
4911
4912 static void si_init_dma_pg(struct radeon_device *rdev)
4913 {
4914 u32 tmp;
4915
4916 WREG32(DMA_PGFSM_WRITE, 0x00002000);
4917 WREG32(DMA_PGFSM_CONFIG, 0x100010ff);
4918
4919 for (tmp = 0; tmp < 5; tmp++)
4920 WREG32(DMA_PGFSM_WRITE, 0);
4921 }
4922
4923 static void si_enable_gfx_cgpg(struct radeon_device *rdev,
4924 bool enable)
4925 {
4926 u32 tmp;
4927
4928 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
4929 tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10);
4930 WREG32(RLC_TTOP_D, tmp);
4931
4932 tmp = RREG32(RLC_PG_CNTL);
4933 tmp |= GFX_PG_ENABLE;
4934 WREG32(RLC_PG_CNTL, tmp);
4935
4936 tmp = RREG32(RLC_AUTO_PG_CTRL);
4937 tmp |= AUTO_PG_EN;
4938 WREG32(RLC_AUTO_PG_CTRL, tmp);
4939 } else {
4940 tmp = RREG32(RLC_AUTO_PG_CTRL);
4941 tmp &= ~AUTO_PG_EN;
4942 WREG32(RLC_AUTO_PG_CTRL, tmp);
4943
4944 tmp = RREG32(DB_RENDER_CONTROL);
4945 }
4946 }
4947
4948 static void si_init_gfx_cgpg(struct radeon_device *rdev)
4949 {
4950 u32 tmp;
4951
4952 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4953
4954 tmp = RREG32(RLC_PG_CNTL);
4955 tmp |= GFX_PG_SRC;
4956 WREG32(RLC_PG_CNTL, tmp);
4957
4958 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4959
4960 tmp = RREG32(RLC_AUTO_PG_CTRL);
4961
4962 tmp &= ~GRBM_REG_SGIT_MASK;
4963 tmp |= GRBM_REG_SGIT(0x700);
4964 tmp &= ~PG_AFTER_GRBM_REG_ST_MASK;
4965 WREG32(RLC_AUTO_PG_CTRL, tmp);
4966 }
4967
4968 static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
4969 {
4970 u32 mask = 0, tmp, tmp1;
4971 int i;
4972
4973 si_select_se_sh(rdev, se, sh);
4974 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
4975 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
4976 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
4977
4978 tmp &= 0xffff0000;
4979
4980 tmp |= tmp1;
4981 tmp >>= 16;
4982
4983 for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) {
4984 mask <<= 1;
4985 mask |= 1;
4986 }
4987
4988 return (~tmp) & mask;
4989 }
4990
4991 static void si_init_ao_cu_mask(struct radeon_device *rdev)
4992 {
4993 u32 i, j, k, active_cu_number = 0;
4994 u32 mask, counter, cu_bitmap;
4995 u32 tmp = 0;
4996
4997 for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
4998 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
4999 mask = 1;
5000 cu_bitmap = 0;
5001 counter = 0;
5002 for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) {
5003 if (si_get_cu_active_bitmap(rdev, i, j) & mask) {
5004 if (counter < 2)
5005 cu_bitmap |= mask;
5006 counter++;
5007 }
5008 mask <<= 1;
5009 }
5010
5011 active_cu_number += counter;
5012 tmp |= (cu_bitmap << (i * 16 + j * 8));
5013 }
5014 }
5015
5016 WREG32(RLC_PG_AO_CU_MASK, tmp);
5017
5018 tmp = RREG32(RLC_MAX_PG_CU);
5019 tmp &= ~MAX_PU_CU_MASK;
5020 tmp |= MAX_PU_CU(active_cu_number);
5021 WREG32(RLC_MAX_PG_CU, tmp);
5022 }
5023
5024 static void si_enable_cgcg(struct radeon_device *rdev,
5025 bool enable)
5026 {
5027 u32 data, orig, tmp;
5028
5029 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5030
5031 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5032 si_enable_gui_idle_interrupt(rdev, true);
5033
5034 WREG32(RLC_GCPM_GENERAL_3, 0x00000080);
5035
5036 tmp = si_halt_rlc(rdev);
5037
5038 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5039 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5040 WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff);
5041
5042 si_wait_for_rlc_serdes(rdev);
5043
5044 si_update_rlc(rdev, tmp);
5045
5046 WREG32(RLC_SERDES_WR_CTRL, 0x007000ff);
5047
5048 data |= CGCG_EN | CGLS_EN;
5049 } else {
5050 si_enable_gui_idle_interrupt(rdev, false);
5051
5052 RREG32(CB_CGTT_SCLK_CTRL);
5053 RREG32(CB_CGTT_SCLK_CTRL);
5054 RREG32(CB_CGTT_SCLK_CTRL);
5055 RREG32(CB_CGTT_SCLK_CTRL);
5056
5057 data &= ~(CGCG_EN | CGLS_EN);
5058 }
5059
5060 if (orig != data)
5061 WREG32(RLC_CGCG_CGLS_CTRL, data);
5062 }
5063
5064 static void si_enable_mgcg(struct radeon_device *rdev,
5065 bool enable)
5066 {
5067 u32 data, orig, tmp = 0;
5068
5069 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5070 orig = data = RREG32(CGTS_SM_CTRL_REG);
5071 data = 0x96940200;
5072 if (orig != data)
5073 WREG32(CGTS_SM_CTRL_REG, data);
5074
5075 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5076 orig = data = RREG32(CP_MEM_SLP_CNTL);
5077 data |= CP_MEM_LS_EN;
5078 if (orig != data)
5079 WREG32(CP_MEM_SLP_CNTL, data);
5080 }
5081
5082 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5083 data &= 0xffffffc0;
5084 if (orig != data)
5085 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5086
5087 tmp = si_halt_rlc(rdev);
5088
5089 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5090 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5091 WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff);
5092
5093 si_update_rlc(rdev, tmp);
5094 } else {
5095 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5096 data |= 0x00000003;
5097 if (orig != data)
5098 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5099
5100 data = RREG32(CP_MEM_SLP_CNTL);
5101 if (data & CP_MEM_LS_EN) {
5102 data &= ~CP_MEM_LS_EN;
5103 WREG32(CP_MEM_SLP_CNTL, data);
5104 }
5105 orig = data = RREG32(CGTS_SM_CTRL_REG);
5106 data |= LS_OVERRIDE | OVERRIDE;
5107 if (orig != data)
5108 WREG32(CGTS_SM_CTRL_REG, data);
5109
5110 tmp = si_halt_rlc(rdev);
5111
5112 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5113 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5114 WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff);
5115
5116 si_update_rlc(rdev, tmp);
5117 }
5118 }
5119
5120 static void si_enable_uvd_mgcg(struct radeon_device *rdev,
5121 bool enable)
5122 {
5123 u32 orig, data, tmp;
5124
5125 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
5126 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5127 tmp |= 0x3fff;
5128 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5129
5130 orig = data = RREG32(UVD_CGC_CTRL);
5131 data |= DCM;
5132 if (orig != data)
5133 WREG32(UVD_CGC_CTRL, data);
5134
5135 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0);
5136 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0);
5137 } else {
5138 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5139 tmp &= ~0x3fff;
5140 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5141
5142 orig = data = RREG32(UVD_CGC_CTRL);
5143 data &= ~DCM;
5144 if (orig != data)
5145 WREG32(UVD_CGC_CTRL, data);
5146
5147 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff);
5148 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff);
5149 }
5150 }
5151
5152 static const u32 mc_cg_registers[] =
5153 {
5154 MC_HUB_MISC_HUB_CG,
5155 MC_HUB_MISC_SIP_CG,
5156 MC_HUB_MISC_VM_CG,
5157 MC_XPB_CLK_GAT,
5158 ATC_MISC_CG,
5159 MC_CITF_MISC_WR_CG,
5160 MC_CITF_MISC_RD_CG,
5161 MC_CITF_MISC_VM_CG,
5162 VM_L2_CG,
5163 };
5164
5165 static void si_enable_mc_ls(struct radeon_device *rdev,
5166 bool enable)
5167 {
5168 int i;
5169 u32 orig, data;
5170
5171 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5172 orig = data = RREG32(mc_cg_registers[i]);
5173 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
5174 data |= MC_LS_ENABLE;
5175 else
5176 data &= ~MC_LS_ENABLE;
5177 if (data != orig)
5178 WREG32(mc_cg_registers[i], data);
5179 }
5180 }
5181
5182 static void si_enable_mc_mgcg(struct radeon_device *rdev,
5183 bool enable)
5184 {
5185 int i;
5186 u32 orig, data;
5187
5188 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5189 orig = data = RREG32(mc_cg_registers[i]);
5190 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
5191 data |= MC_CG_ENABLE;
5192 else
5193 data &= ~MC_CG_ENABLE;
5194 if (data != orig)
5195 WREG32(mc_cg_registers[i], data);
5196 }
5197 }
5198
5199 static void si_enable_dma_mgcg(struct radeon_device *rdev,
5200 bool enable)
5201 {
5202 u32 orig, data, offset;
5203 int i;
5204
5205 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
5206 for (i = 0; i < 2; i++) {
5207 if (i == 0)
5208 offset = DMA0_REGISTER_OFFSET;
5209 else
5210 offset = DMA1_REGISTER_OFFSET;
5211 orig = data = RREG32(DMA_POWER_CNTL + offset);
5212 data &= ~MEM_POWER_OVERRIDE;
5213 if (data != orig)
5214 WREG32(DMA_POWER_CNTL + offset, data);
5215 WREG32(DMA_CLK_CTRL + offset, 0x00000100);
5216 }
5217 } else {
5218 for (i = 0; i < 2; i++) {
5219 if (i == 0)
5220 offset = DMA0_REGISTER_OFFSET;
5221 else
5222 offset = DMA1_REGISTER_OFFSET;
5223 orig = data = RREG32(DMA_POWER_CNTL + offset);
5224 data |= MEM_POWER_OVERRIDE;
5225 if (data != orig)
5226 WREG32(DMA_POWER_CNTL + offset, data);
5227
5228 orig = data = RREG32(DMA_CLK_CTRL + offset);
5229 data = 0xff000000;
5230 if (data != orig)
5231 WREG32(DMA_CLK_CTRL + offset, data);
5232 }
5233 }
5234 }
5235
5236 static void si_enable_bif_mgls(struct radeon_device *rdev,
5237 bool enable)
5238 {
5239 u32 orig, data;
5240
5241 orig = data = RREG32_PCIE(PCIE_CNTL2);
5242
5243 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5244 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5245 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5246 else
5247 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5248 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
5249
5250 if (orig != data)
5251 WREG32_PCIE(PCIE_CNTL2, data);
5252 }
5253
5254 static void si_enable_hdp_mgcg(struct radeon_device *rdev,
5255 bool enable)
5256 {
5257 u32 orig, data;
5258
5259 orig = data = RREG32(HDP_HOST_PATH_CNTL);
5260
5261 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
5262 data &= ~CLOCK_GATING_DIS;
5263 else
5264 data |= CLOCK_GATING_DIS;
5265
5266 if (orig != data)
5267 WREG32(HDP_HOST_PATH_CNTL, data);
5268 }
5269
5270 static void si_enable_hdp_ls(struct radeon_device *rdev,
5271 bool enable)
5272 {
5273 u32 orig, data;
5274
5275 orig = data = RREG32(HDP_MEM_POWER_LS);
5276
5277 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
5278 data |= HDP_LS_ENABLE;
5279 else
5280 data &= ~HDP_LS_ENABLE;
5281
5282 if (orig != data)
5283 WREG32(HDP_MEM_POWER_LS, data);
5284 }
5285
5286 void si_update_cg(struct radeon_device *rdev,
5287 u32 block, bool enable)
5288 {
5289 if (block & RADEON_CG_BLOCK_GFX) {
5290 si_enable_gui_idle_interrupt(rdev, false);
5291 /* order matters! */
5292 if (enable) {
5293 si_enable_mgcg(rdev, true);
5294 si_enable_cgcg(rdev, true);
5295 } else {
5296 si_enable_cgcg(rdev, false);
5297 si_enable_mgcg(rdev, false);
5298 }
5299 si_enable_gui_idle_interrupt(rdev, true);
5300 }
5301
5302 if (block & RADEON_CG_BLOCK_MC) {
5303 si_enable_mc_mgcg(rdev, enable);
5304 si_enable_mc_ls(rdev, enable);
5305 }
5306
5307 if (block & RADEON_CG_BLOCK_SDMA) {
5308 si_enable_dma_mgcg(rdev, enable);
5309 }
5310
5311 if (block & RADEON_CG_BLOCK_BIF) {
5312 si_enable_bif_mgls(rdev, enable);
5313 }
5314
5315 if (block & RADEON_CG_BLOCK_UVD) {
5316 if (rdev->has_uvd) {
5317 si_enable_uvd_mgcg(rdev, enable);
5318 }
5319 }
5320
5321 if (block & RADEON_CG_BLOCK_HDP) {
5322 si_enable_hdp_mgcg(rdev, enable);
5323 si_enable_hdp_ls(rdev, enable);
5324 }
5325 }
5326
5327 static void si_init_cg(struct radeon_device *rdev)
5328 {
5329 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5330 RADEON_CG_BLOCK_MC |
5331 RADEON_CG_BLOCK_SDMA |
5332 RADEON_CG_BLOCK_BIF |
5333 RADEON_CG_BLOCK_HDP), true);
5334 if (rdev->has_uvd) {
5335 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, true);
5336 si_init_uvd_internal_cg(rdev);
5337 }
5338 }
5339
5340 static void si_fini_cg(struct radeon_device *rdev)
5341 {
5342 if (rdev->has_uvd) {
5343 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, false);
5344 }
5345 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5346 RADEON_CG_BLOCK_MC |
5347 RADEON_CG_BLOCK_SDMA |
5348 RADEON_CG_BLOCK_BIF |
5349 RADEON_CG_BLOCK_HDP), false);
5350 }
5351
5352 u32 si_get_csb_size(struct radeon_device *rdev)
5353 {
5354 u32 count = 0;
5355 const struct cs_section_def *sect = NULL;
5356 const struct cs_extent_def *ext = NULL;
5357
5358 if (rdev->rlc.cs_data == NULL)
5359 return 0;
5360
5361 /* begin clear state */
5362 count += 2;
5363 /* context control state */
5364 count += 3;
5365
5366 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5367 for (ext = sect->section; ext->extent != NULL; ++ext) {
5368 if (sect->id == SECT_CONTEXT)
5369 count += 2 + ext->reg_count;
5370 else
5371 return 0;
5372 }
5373 }
5374 /* pa_sc_raster_config */
5375 count += 3;
5376 /* end clear state */
5377 count += 2;
5378 /* clear state */
5379 count += 2;
5380
5381 return count;
5382 }
5383
5384 void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
5385 {
5386 u32 count = 0, i;
5387 const struct cs_section_def *sect = NULL;
5388 const struct cs_extent_def *ext = NULL;
5389
5390 if (rdev->rlc.cs_data == NULL)
5391 return;
5392 if (buffer == NULL)
5393 return;
5394
5395 buffer[count++] = PACKET3(PACKET3_PREAMBLE_CNTL, 0);
5396 buffer[count++] = PACKET3_PREAMBLE_BEGIN_CLEAR_STATE;
5397
5398 buffer[count++] = PACKET3(PACKET3_CONTEXT_CONTROL, 1);
5399 buffer[count++] = 0x80000000;
5400 buffer[count++] = 0x80000000;
5401
5402 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5403 for (ext = sect->section; ext->extent != NULL; ++ext) {
5404 if (sect->id == SECT_CONTEXT) {
5405 buffer[count++] = PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count);
5406 buffer[count++] = ext->reg_index - 0xa000;
5407 for (i = 0; i < ext->reg_count; i++)
5408 buffer[count++] = ext->extent[i];
5409 } else {
5410 return;
5411 }
5412 }
5413 }
5414
5415 buffer[count++] = PACKET3(PACKET3_SET_CONTEXT_REG, 1);
5416 buffer[count++] = PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START;
5417 switch (rdev->family) {
5418 case CHIP_TAHITI:
5419 case CHIP_PITCAIRN:
5420 buffer[count++] = 0x2a00126a;
5421 break;
5422 case CHIP_VERDE:
5423 buffer[count++] = 0x0000124a;
5424 break;
5425 case CHIP_OLAND:
5426 buffer[count++] = 0x00000082;
5427 break;
5428 case CHIP_HAINAN:
5429 buffer[count++] = 0x00000000;
5430 break;
5431 default:
5432 buffer[count++] = 0x00000000;
5433 break;
5434 }
5435
5436 buffer[count++] = PACKET3(PACKET3_PREAMBLE_CNTL, 0);
5437 buffer[count++] = PACKET3_PREAMBLE_END_CLEAR_STATE;
5438
5439 buffer[count++] = PACKET3(PACKET3_CLEAR_STATE, 0);
5440 buffer[count++] = 0;
5441 }
5442
5443 static void si_init_pg(struct radeon_device *rdev)
5444 {
5445 if (rdev->pg_flags) {
5446 if (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA) {
5447 si_init_dma_pg(rdev);
5448 }
5449 si_init_ao_cu_mask(rdev);
5450 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
5451 si_init_gfx_cgpg(rdev);
5452 }
5453 si_enable_dma_pg(rdev, true);
5454 si_enable_gfx_cgpg(rdev, true);
5455 } else {
5456 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5457 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5458 }
5459 }
5460
5461 static void si_fini_pg(struct radeon_device *rdev)
5462 {
5463 if (rdev->pg_flags) {
5464 si_enable_dma_pg(rdev, false);
5465 si_enable_gfx_cgpg(rdev, false);
5466 }
5467 }
5468
5469 /*
5470 * RLC
5471 */
5472 void si_rlc_reset(struct radeon_device *rdev)
5473 {
5474 u32 tmp = RREG32(GRBM_SOFT_RESET);
5475
5476 tmp |= SOFT_RESET_RLC;
5477 WREG32(GRBM_SOFT_RESET, tmp);
5478 udelay(50);
5479 tmp &= ~SOFT_RESET_RLC;
5480 WREG32(GRBM_SOFT_RESET, tmp);
5481 udelay(50);
5482 }
5483
5484 static void si_rlc_stop(struct radeon_device *rdev)
5485 {
5486 WREG32(RLC_CNTL, 0);
5487
5488 si_enable_gui_idle_interrupt(rdev, false);
5489
5490 si_wait_for_rlc_serdes(rdev);
5491 }
5492
5493 static void si_rlc_start(struct radeon_device *rdev)
5494 {
5495 WREG32(RLC_CNTL, RLC_ENABLE);
5496
5497 si_enable_gui_idle_interrupt(rdev, true);
5498
5499 udelay(50);
5500 }
5501
5502 static bool si_lbpw_supported(struct radeon_device *rdev)
5503 {
5504 u32 tmp;
5505
5506 /* Enable LBPW only for DDR3 */
5507 tmp = RREG32(MC_SEQ_MISC0);
5508 if ((tmp & 0xF0000000) == 0xB0000000)
5509 return true;
5510 return false;
5511 }
5512
5513 static void si_enable_lbpw(struct radeon_device *rdev, bool enable)
5514 {
5515 u32 tmp;
5516
5517 tmp = RREG32(RLC_LB_CNTL);
5518 if (enable)
5519 tmp |= LOAD_BALANCE_ENABLE;
5520 else
5521 tmp &= ~LOAD_BALANCE_ENABLE;
5522 WREG32(RLC_LB_CNTL, tmp);
5523
5524 if (!enable) {
5525 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5526 WREG32(SPI_LB_CU_MASK, 0x00ff);
5527 }
5528 }
5529
5530 static int si_rlc_resume(struct radeon_device *rdev)
5531 {
5532 u32 i;
5533 const __be32 *fw_data;
5534
5535 if (!rdev->rlc_fw)
5536 return -EINVAL;
5537
5538 si_rlc_stop(rdev);
5539
5540 si_rlc_reset(rdev);
5541
5542 si_init_pg(rdev);
5543
5544 si_init_cg(rdev);
5545
5546 WREG32(RLC_RL_BASE, 0);
5547 WREG32(RLC_RL_SIZE, 0);
5548 WREG32(RLC_LB_CNTL, 0);
5549 WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
5550 WREG32(RLC_LB_CNTR_INIT, 0);
5551 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5552
5553 WREG32(RLC_MC_CNTL, 0);
5554 WREG32(RLC_UCODE_CNTL, 0);
5555
5556 fw_data = (const __be32 *)rdev->rlc_fw->data;
5557 for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
5558 WREG32(RLC_UCODE_ADDR, i);
5559 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
5560 }
5561 WREG32(RLC_UCODE_ADDR, 0);
5562
5563 si_enable_lbpw(rdev, si_lbpw_supported(rdev));
5564
5565 si_rlc_start(rdev);
5566
5567 return 0;
5568 }
5569
5570 static void si_enable_interrupts(struct radeon_device *rdev)
5571 {
5572 u32 ih_cntl = RREG32(IH_CNTL);
5573 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5574
5575 ih_cntl |= ENABLE_INTR;
5576 ih_rb_cntl |= IH_RB_ENABLE;
5577 WREG32(IH_CNTL, ih_cntl);
5578 WREG32(IH_RB_CNTL, ih_rb_cntl);
5579 rdev->ih.enabled = true;
5580 }
5581
5582 static void si_disable_interrupts(struct radeon_device *rdev)
5583 {
5584 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5585 u32 ih_cntl = RREG32(IH_CNTL);
5586
5587 ih_rb_cntl &= ~IH_RB_ENABLE;
5588 ih_cntl &= ~ENABLE_INTR;
5589 WREG32(IH_RB_CNTL, ih_rb_cntl);
5590 WREG32(IH_CNTL, ih_cntl);
5591 /* set rptr, wptr to 0 */
5592 WREG32(IH_RB_RPTR, 0);
5593 WREG32(IH_RB_WPTR, 0);
5594 rdev->ih.enabled = false;
5595 rdev->ih.rptr = 0;
5596 }
5597
5598 static void si_disable_interrupt_state(struct radeon_device *rdev)
5599 {
5600 u32 tmp;
5601
5602 tmp = RREG32(CP_INT_CNTL_RING0) &
5603 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5604 WREG32(CP_INT_CNTL_RING0, tmp);
5605 WREG32(CP_INT_CNTL_RING1, 0);
5606 WREG32(CP_INT_CNTL_RING2, 0);
5607 tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5608 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
5609 tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5610 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
5611 WREG32(GRBM_INT_CNTL, 0);
5612 if (rdev->num_crtc >= 2) {
5613 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5614 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5615 }
5616 if (rdev->num_crtc >= 4) {
5617 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5618 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5619 }
5620 if (rdev->num_crtc >= 6) {
5621 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5622 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5623 }
5624
5625 if (rdev->num_crtc >= 2) {
5626 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5627 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5628 }
5629 if (rdev->num_crtc >= 4) {
5630 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5631 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5632 }
5633 if (rdev->num_crtc >= 6) {
5634 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5635 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5636 }
5637
5638 if (!ASIC_IS_NODCE(rdev)) {
5639 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
5640
5641 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5642 WREG32(DC_HPD1_INT_CONTROL, tmp);
5643 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5644 WREG32(DC_HPD2_INT_CONTROL, tmp);
5645 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5646 WREG32(DC_HPD3_INT_CONTROL, tmp);
5647 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5648 WREG32(DC_HPD4_INT_CONTROL, tmp);
5649 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5650 WREG32(DC_HPD5_INT_CONTROL, tmp);
5651 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5652 WREG32(DC_HPD6_INT_CONTROL, tmp);
5653 }
5654 }
5655
5656 static int si_irq_init(struct radeon_device *rdev)
5657 {
5658 int ret = 0;
5659 int rb_bufsz;
5660 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
5661
5662 /* allocate ring */
5663 ret = r600_ih_ring_alloc(rdev);
5664 if (ret)
5665 return ret;
5666
5667 /* disable irqs */
5668 si_disable_interrupts(rdev);
5669
5670 /* init rlc */
5671 ret = si_rlc_resume(rdev);
5672 if (ret) {
5673 r600_ih_ring_fini(rdev);
5674 return ret;
5675 }
5676
5677 /* setup interrupt control */
5678 /* set dummy read address to ring address */
5679 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
5680 interrupt_cntl = RREG32(INTERRUPT_CNTL);
5681 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
5682 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
5683 */
5684 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
5685 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
5686 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
5687 WREG32(INTERRUPT_CNTL, interrupt_cntl);
5688
5689 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
5690 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
5691
5692 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
5693 IH_WPTR_OVERFLOW_CLEAR |
5694 (rb_bufsz << 1));
5695
5696 if (rdev->wb.enabled)
5697 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
5698
5699 /* set the writeback address whether it's enabled or not */
5700 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
5701 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
5702
5703 WREG32(IH_RB_CNTL, ih_rb_cntl);
5704
5705 /* set rptr, wptr to 0 */
5706 WREG32(IH_RB_RPTR, 0);
5707 WREG32(IH_RB_WPTR, 0);
5708
5709 /* Default settings for IH_CNTL (disabled at first) */
5710 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
5711 /* RPTR_REARM only works if msi's are enabled */
5712 if (rdev->msi_enabled)
5713 ih_cntl |= RPTR_REARM;
5714 WREG32(IH_CNTL, ih_cntl);
5715
5716 /* force the active interrupt state to all disabled */
5717 si_disable_interrupt_state(rdev);
5718
5719 pci_set_master(rdev->pdev);
5720
5721 /* enable irqs */
5722 si_enable_interrupts(rdev);
5723
5724 return ret;
5725 }
5726
5727 int si_irq_set(struct radeon_device *rdev)
5728 {
5729 u32 cp_int_cntl;
5730 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
5731 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
5732 u32 hpd1 = 0, hpd2 = 0, hpd3 = 0, hpd4 = 0, hpd5 = 0, hpd6 = 0;
5733 u32 grbm_int_cntl = 0;
5734 u32 dma_cntl, dma_cntl1;
5735 u32 thermal_int = 0;
5736
5737 if (!rdev->irq.installed) {
5738 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
5739 return -EINVAL;
5740 }
5741 /* don't enable anything if the ih is disabled */
5742 if (!rdev->ih.enabled) {
5743 si_disable_interrupts(rdev);
5744 /* force the active interrupt state to all disabled */
5745 si_disable_interrupt_state(rdev);
5746 return 0;
5747 }
5748
5749 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
5750 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5751
5752 if (!ASIC_IS_NODCE(rdev)) {
5753 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
5754 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
5755 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
5756 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
5757 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
5758 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
5759 }
5760
5761 dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5762 dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5763
5764 thermal_int = RREG32(CG_THERMAL_INT) &
5765 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
5766
5767 /* enable CP interrupts on all rings */
5768 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
5769 DRM_DEBUG("si_irq_set: sw int gfx\n");
5770 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
5771 }
5772 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
5773 DRM_DEBUG("si_irq_set: sw int cp1\n");
5774 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
5775 }
5776 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
5777 DRM_DEBUG("si_irq_set: sw int cp2\n");
5778 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
5779 }
5780 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
5781 DRM_DEBUG("si_irq_set: sw int dma\n");
5782 dma_cntl |= TRAP_ENABLE;
5783 }
5784
5785 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
5786 DRM_DEBUG("si_irq_set: sw int dma1\n");
5787 dma_cntl1 |= TRAP_ENABLE;
5788 }
5789 if (rdev->irq.crtc_vblank_int[0] ||
5790 atomic_read(&rdev->irq.pflip[0])) {
5791 DRM_DEBUG("si_irq_set: vblank 0\n");
5792 crtc1 |= VBLANK_INT_MASK;
5793 }
5794 if (rdev->irq.crtc_vblank_int[1] ||
5795 atomic_read(&rdev->irq.pflip[1])) {
5796 DRM_DEBUG("si_irq_set: vblank 1\n");
5797 crtc2 |= VBLANK_INT_MASK;
5798 }
5799 if (rdev->irq.crtc_vblank_int[2] ||
5800 atomic_read(&rdev->irq.pflip[2])) {
5801 DRM_DEBUG("si_irq_set: vblank 2\n");
5802 crtc3 |= VBLANK_INT_MASK;
5803 }
5804 if (rdev->irq.crtc_vblank_int[3] ||
5805 atomic_read(&rdev->irq.pflip[3])) {
5806 DRM_DEBUG("si_irq_set: vblank 3\n");
5807 crtc4 |= VBLANK_INT_MASK;
5808 }
5809 if (rdev->irq.crtc_vblank_int[4] ||
5810 atomic_read(&rdev->irq.pflip[4])) {
5811 DRM_DEBUG("si_irq_set: vblank 4\n");
5812 crtc5 |= VBLANK_INT_MASK;
5813 }
5814 if (rdev->irq.crtc_vblank_int[5] ||
5815 atomic_read(&rdev->irq.pflip[5])) {
5816 DRM_DEBUG("si_irq_set: vblank 5\n");
5817 crtc6 |= VBLANK_INT_MASK;
5818 }
5819 if (rdev->irq.hpd[0]) {
5820 DRM_DEBUG("si_irq_set: hpd 1\n");
5821 hpd1 |= DC_HPDx_INT_EN;
5822 }
5823 if (rdev->irq.hpd[1]) {
5824 DRM_DEBUG("si_irq_set: hpd 2\n");
5825 hpd2 |= DC_HPDx_INT_EN;
5826 }
5827 if (rdev->irq.hpd[2]) {
5828 DRM_DEBUG("si_irq_set: hpd 3\n");
5829 hpd3 |= DC_HPDx_INT_EN;
5830 }
5831 if (rdev->irq.hpd[3]) {
5832 DRM_DEBUG("si_irq_set: hpd 4\n");
5833 hpd4 |= DC_HPDx_INT_EN;
5834 }
5835 if (rdev->irq.hpd[4]) {
5836 DRM_DEBUG("si_irq_set: hpd 5\n");
5837 hpd5 |= DC_HPDx_INT_EN;
5838 }
5839 if (rdev->irq.hpd[5]) {
5840 DRM_DEBUG("si_irq_set: hpd 6\n");
5841 hpd6 |= DC_HPDx_INT_EN;
5842 }
5843
5844 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
5845 WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
5846 WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
5847
5848 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
5849 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
5850
5851 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
5852
5853 if (rdev->irq.dpm_thermal) {
5854 DRM_DEBUG("dpm thermal\n");
5855 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
5856 }
5857
5858 if (rdev->num_crtc >= 2) {
5859 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
5860 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
5861 }
5862 if (rdev->num_crtc >= 4) {
5863 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
5864 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
5865 }
5866 if (rdev->num_crtc >= 6) {
5867 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
5868 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
5869 }
5870
5871 if (rdev->num_crtc >= 2) {
5872 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
5873 GRPH_PFLIP_INT_MASK);
5874 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
5875 GRPH_PFLIP_INT_MASK);
5876 }
5877 if (rdev->num_crtc >= 4) {
5878 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
5879 GRPH_PFLIP_INT_MASK);
5880 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
5881 GRPH_PFLIP_INT_MASK);
5882 }
5883 if (rdev->num_crtc >= 6) {
5884 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
5885 GRPH_PFLIP_INT_MASK);
5886 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
5887 GRPH_PFLIP_INT_MASK);
5888 }
5889
5890 if (!ASIC_IS_NODCE(rdev)) {
5891 WREG32(DC_HPD1_INT_CONTROL, hpd1);
5892 WREG32(DC_HPD2_INT_CONTROL, hpd2);
5893 WREG32(DC_HPD3_INT_CONTROL, hpd3);
5894 WREG32(DC_HPD4_INT_CONTROL, hpd4);
5895 WREG32(DC_HPD5_INT_CONTROL, hpd5);
5896 WREG32(DC_HPD6_INT_CONTROL, hpd6);
5897 }
5898
5899 WREG32(CG_THERMAL_INT, thermal_int);
5900
5901 return 0;
5902 }
5903
5904 static inline void si_irq_ack(struct radeon_device *rdev)
5905 {
5906 u32 tmp;
5907
5908 if (ASIC_IS_NODCE(rdev))
5909 return;
5910
5911 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
5912 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
5913 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
5914 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
5915 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
5916 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
5917 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
5918 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
5919 if (rdev->num_crtc >= 4) {
5920 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
5921 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
5922 }
5923 if (rdev->num_crtc >= 6) {
5924 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
5925 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
5926 }
5927
5928 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
5929 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5930 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
5931 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5932 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
5933 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
5934 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
5935 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
5936 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
5937 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
5938 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
5939 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
5940
5941 if (rdev->num_crtc >= 4) {
5942 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
5943 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5944 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
5945 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5946 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
5947 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
5948 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
5949 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
5950 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
5951 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
5952 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
5953 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
5954 }
5955
5956 if (rdev->num_crtc >= 6) {
5957 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
5958 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5959 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
5960 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5961 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
5962 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
5963 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
5964 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
5965 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
5966 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
5967 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
5968 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
5969 }
5970
5971 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
5972 tmp = RREG32(DC_HPD1_INT_CONTROL);
5973 tmp |= DC_HPDx_INT_ACK;
5974 WREG32(DC_HPD1_INT_CONTROL, tmp);
5975 }
5976 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
5977 tmp = RREG32(DC_HPD2_INT_CONTROL);
5978 tmp |= DC_HPDx_INT_ACK;
5979 WREG32(DC_HPD2_INT_CONTROL, tmp);
5980 }
5981 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
5982 tmp = RREG32(DC_HPD3_INT_CONTROL);
5983 tmp |= DC_HPDx_INT_ACK;
5984 WREG32(DC_HPD3_INT_CONTROL, tmp);
5985 }
5986 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
5987 tmp = RREG32(DC_HPD4_INT_CONTROL);
5988 tmp |= DC_HPDx_INT_ACK;
5989 WREG32(DC_HPD4_INT_CONTROL, tmp);
5990 }
5991 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
5992 tmp = RREG32(DC_HPD5_INT_CONTROL);
5993 tmp |= DC_HPDx_INT_ACK;
5994 WREG32(DC_HPD5_INT_CONTROL, tmp);
5995 }
5996 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
5997 tmp = RREG32(DC_HPD5_INT_CONTROL);
5998 tmp |= DC_HPDx_INT_ACK;
5999 WREG32(DC_HPD6_INT_CONTROL, tmp);
6000 }
6001 }
6002
6003 static void si_irq_disable(struct radeon_device *rdev)
6004 {
6005 si_disable_interrupts(rdev);
6006 /* Wait and acknowledge irq */
6007 mdelay(1);
6008 si_irq_ack(rdev);
6009 si_disable_interrupt_state(rdev);
6010 }
6011
6012 static void si_irq_suspend(struct radeon_device *rdev)
6013 {
6014 si_irq_disable(rdev);
6015 si_rlc_stop(rdev);
6016 }
6017
6018 static void si_irq_fini(struct radeon_device *rdev)
6019 {
6020 si_irq_suspend(rdev);
6021 r600_ih_ring_fini(rdev);
6022 }
6023
6024 static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
6025 {
6026 u32 wptr, tmp;
6027
6028 if (rdev->wb.enabled)
6029 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
6030 else
6031 wptr = RREG32(IH_RB_WPTR);
6032
6033 if (wptr & RB_OVERFLOW) {
6034 /* When a ring buffer overflow happen start parsing interrupt
6035 * from the last not overwritten vector (wptr + 16). Hopefully
6036 * this should allow us to catchup.
6037 */
6038 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
6039 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
6040 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6041 tmp = RREG32(IH_RB_CNTL);
6042 tmp |= IH_WPTR_OVERFLOW_CLEAR;
6043 WREG32(IH_RB_CNTL, tmp);
6044 wptr &= ~RB_OVERFLOW;
6045 }
6046 return (wptr & rdev->ih.ptr_mask);
6047 }
6048
6049 /* SI IV Ring
6050 * Each IV ring entry is 128 bits:
6051 * [7:0] - interrupt source id
6052 * [31:8] - reserved
6053 * [59:32] - interrupt source data
6054 * [63:60] - reserved
6055 * [71:64] - RINGID
6056 * [79:72] - VMID
6057 * [127:80] - reserved
6058 */
6059 int si_irq_process(struct radeon_device *rdev)
6060 {
6061 u32 wptr;
6062 u32 rptr;
6063 u32 src_id, src_data, ring_id;
6064 u32 ring_index;
6065 bool queue_hotplug = false;
6066 bool queue_thermal = false;
6067 u32 status, addr;
6068
6069 if (!rdev->ih.enabled || rdev->shutdown)
6070 return IRQ_NONE;
6071
6072 wptr = si_get_ih_wptr(rdev);
6073
6074 restart_ih:
6075 /* is somebody else already processing irqs? */
6076 if (atomic_xchg(&rdev->ih.lock, 1))
6077 return IRQ_NONE;
6078
6079 rptr = rdev->ih.rptr;
6080 DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
6081
6082 /* Order reading of wptr vs. reading of IH ring data */
6083 rmb();
6084
6085 /* display interrupts */
6086 si_irq_ack(rdev);
6087
6088 while (rptr != wptr) {
6089 /* wptr/rptr are in bytes! */
6090 ring_index = rptr / 4;
6091 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
6092 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
6093 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
6094
6095 switch (src_id) {
6096 case 1: /* D1 vblank/vline */
6097 switch (src_data) {
6098 case 0: /* D1 vblank */
6099 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
6100 if (rdev->irq.crtc_vblank_int[0]) {
6101 drm_handle_vblank(rdev->ddev, 0);
6102 rdev->pm.vblank_sync = true;
6103 wake_up(&rdev->irq.vblank_queue);
6104 }
6105 if (atomic_read(&rdev->irq.pflip[0]))
6106 radeon_crtc_handle_flip(rdev, 0);
6107 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
6108 DRM_DEBUG("IH: D1 vblank\n");
6109 }
6110 break;
6111 case 1: /* D1 vline */
6112 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
6113 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
6114 DRM_DEBUG("IH: D1 vline\n");
6115 }
6116 break;
6117 default:
6118 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6119 break;
6120 }
6121 break;
6122 case 2: /* D2 vblank/vline */
6123 switch (src_data) {
6124 case 0: /* D2 vblank */
6125 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
6126 if (rdev->irq.crtc_vblank_int[1]) {
6127 drm_handle_vblank(rdev->ddev, 1);
6128 rdev->pm.vblank_sync = true;
6129 wake_up(&rdev->irq.vblank_queue);
6130 }
6131 if (atomic_read(&rdev->irq.pflip[1]))
6132 radeon_crtc_handle_flip(rdev, 1);
6133 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
6134 DRM_DEBUG("IH: D2 vblank\n");
6135 }
6136 break;
6137 case 1: /* D2 vline */
6138 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
6139 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
6140 DRM_DEBUG("IH: D2 vline\n");
6141 }
6142 break;
6143 default:
6144 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6145 break;
6146 }
6147 break;
6148 case 3: /* D3 vblank/vline */
6149 switch (src_data) {
6150 case 0: /* D3 vblank */
6151 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
6152 if (rdev->irq.crtc_vblank_int[2]) {
6153 drm_handle_vblank(rdev->ddev, 2);
6154 rdev->pm.vblank_sync = true;
6155 wake_up(&rdev->irq.vblank_queue);
6156 }
6157 if (atomic_read(&rdev->irq.pflip[2]))
6158 radeon_crtc_handle_flip(rdev, 2);
6159 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
6160 DRM_DEBUG("IH: D3 vblank\n");
6161 }
6162 break;
6163 case 1: /* D3 vline */
6164 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
6165 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
6166 DRM_DEBUG("IH: D3 vline\n");
6167 }
6168 break;
6169 default:
6170 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6171 break;
6172 }
6173 break;
6174 case 4: /* D4 vblank/vline */
6175 switch (src_data) {
6176 case 0: /* D4 vblank */
6177 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
6178 if (rdev->irq.crtc_vblank_int[3]) {
6179 drm_handle_vblank(rdev->ddev, 3);
6180 rdev->pm.vblank_sync = true;
6181 wake_up(&rdev->irq.vblank_queue);
6182 }
6183 if (atomic_read(&rdev->irq.pflip[3]))
6184 radeon_crtc_handle_flip(rdev, 3);
6185 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
6186 DRM_DEBUG("IH: D4 vblank\n");
6187 }
6188 break;
6189 case 1: /* D4 vline */
6190 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
6191 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
6192 DRM_DEBUG("IH: D4 vline\n");
6193 }
6194 break;
6195 default:
6196 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6197 break;
6198 }
6199 break;
6200 case 5: /* D5 vblank/vline */
6201 switch (src_data) {
6202 case 0: /* D5 vblank */
6203 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
6204 if (rdev->irq.crtc_vblank_int[4]) {
6205 drm_handle_vblank(rdev->ddev, 4);
6206 rdev->pm.vblank_sync = true;
6207 wake_up(&rdev->irq.vblank_queue);
6208 }
6209 if (atomic_read(&rdev->irq.pflip[4]))
6210 radeon_crtc_handle_flip(rdev, 4);
6211 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
6212 DRM_DEBUG("IH: D5 vblank\n");
6213 }
6214 break;
6215 case 1: /* D5 vline */
6216 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
6217 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
6218 DRM_DEBUG("IH: D5 vline\n");
6219 }
6220 break;
6221 default:
6222 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6223 break;
6224 }
6225 break;
6226 case 6: /* D6 vblank/vline */
6227 switch (src_data) {
6228 case 0: /* D6 vblank */
6229 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
6230 if (rdev->irq.crtc_vblank_int[5]) {
6231 drm_handle_vblank(rdev->ddev, 5);
6232 rdev->pm.vblank_sync = true;
6233 wake_up(&rdev->irq.vblank_queue);
6234 }
6235 if (atomic_read(&rdev->irq.pflip[5]))
6236 radeon_crtc_handle_flip(rdev, 5);
6237 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
6238 DRM_DEBUG("IH: D6 vblank\n");
6239 }
6240 break;
6241 case 1: /* D6 vline */
6242 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
6243 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
6244 DRM_DEBUG("IH: D6 vline\n");
6245 }
6246 break;
6247 default:
6248 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6249 break;
6250 }
6251 break;
6252 case 8: /* D1 page flip */
6253 case 10: /* D2 page flip */
6254 case 12: /* D3 page flip */
6255 case 14: /* D4 page flip */
6256 case 16: /* D5 page flip */
6257 case 18: /* D6 page flip */
6258 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
6259 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
6260 break;
6261 case 42: /* HPD hotplug */
6262 switch (src_data) {
6263 case 0:
6264 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
6265 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
6266 queue_hotplug = true;
6267 DRM_DEBUG("IH: HPD1\n");
6268 }
6269 break;
6270 case 1:
6271 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
6272 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
6273 queue_hotplug = true;
6274 DRM_DEBUG("IH: HPD2\n");
6275 }
6276 break;
6277 case 2:
6278 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6279 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
6280 queue_hotplug = true;
6281 DRM_DEBUG("IH: HPD3\n");
6282 }
6283 break;
6284 case 3:
6285 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6286 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
6287 queue_hotplug = true;
6288 DRM_DEBUG("IH: HPD4\n");
6289 }
6290 break;
6291 case 4:
6292 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6293 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
6294 queue_hotplug = true;
6295 DRM_DEBUG("IH: HPD5\n");
6296 }
6297 break;
6298 case 5:
6299 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6300 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
6301 queue_hotplug = true;
6302 DRM_DEBUG("IH: HPD6\n");
6303 }
6304 break;
6305 default:
6306 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6307 break;
6308 }
6309 break;
6310 case 124: /* UVD */
6311 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
6312 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
6313 break;
6314 case 146:
6315 case 147:
6316 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
6317 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
6318 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
6319 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
6320 addr);
6321 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
6322 status);
6323 si_vm_decode_fault(rdev, status, addr);
6324 /* reset addr and status */
6325 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
6326 break;
6327 case 176: /* RINGID0 CP_INT */
6328 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6329 break;
6330 case 177: /* RINGID1 CP_INT */
6331 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6332 break;
6333 case 178: /* RINGID2 CP_INT */
6334 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6335 break;
6336 case 181: /* CP EOP event */
6337 DRM_DEBUG("IH: CP EOP\n");
6338 switch (ring_id) {
6339 case 0:
6340 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6341 break;
6342 case 1:
6343 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6344 break;
6345 case 2:
6346 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6347 break;
6348 }
6349 break;
6350 case 224: /* DMA trap event */
6351 DRM_DEBUG("IH: DMA trap\n");
6352 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
6353 break;
6354 case 230: /* thermal low to high */
6355 DRM_DEBUG("IH: thermal low to high\n");
6356 rdev->pm.dpm.thermal.high_to_low = false;
6357 queue_thermal = true;
6358 break;
6359 case 231: /* thermal high to low */
6360 DRM_DEBUG("IH: thermal high to low\n");
6361 rdev->pm.dpm.thermal.high_to_low = true;
6362 queue_thermal = true;
6363 break;
6364 case 233: /* GUI IDLE */
6365 DRM_DEBUG("IH: GUI idle\n");
6366 break;
6367 case 244: /* DMA trap event */
6368 DRM_DEBUG("IH: DMA1 trap\n");
6369 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6370 break;
6371 default:
6372 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6373 break;
6374 }
6375
6376 /* wptr/rptr are in bytes! */
6377 rptr += 16;
6378 rptr &= rdev->ih.ptr_mask;
6379 }
6380 if (queue_hotplug)
6381 schedule_work(&rdev->hotplug_work);
6382 if (queue_thermal && rdev->pm.dpm_enabled)
6383 schedule_work(&rdev->pm.dpm.thermal.work);
6384 rdev->ih.rptr = rptr;
6385 WREG32(IH_RB_RPTR, rdev->ih.rptr);
6386 atomic_set(&rdev->ih.lock, 0);
6387
6388 /* make sure wptr hasn't changed while processing */
6389 wptr = si_get_ih_wptr(rdev);
6390 if (wptr != rptr)
6391 goto restart_ih;
6392
6393 return IRQ_HANDLED;
6394 }
6395
6396 /*
6397 * startup/shutdown callbacks
6398 */
6399 static int si_startup(struct radeon_device *rdev)
6400 {
6401 struct radeon_ring *ring;
6402 int r;
6403
6404 /* enable pcie gen2/3 link */
6405 si_pcie_gen3_enable(rdev);
6406 /* enable aspm */
6407 si_program_aspm(rdev);
6408
6409 /* scratch needs to be initialized before MC */
6410 r = r600_vram_scratch_init(rdev);
6411 if (r)
6412 return r;
6413
6414 si_mc_program(rdev);
6415
6416 r = si_mc_load_microcode(rdev);
6417 if (r) {
6418 DRM_ERROR("Failed to load MC firmware!\n");
6419 return r;
6420 }
6421
6422 r = si_pcie_gart_enable(rdev);
6423 if (r)
6424 return r;
6425 si_gpu_init(rdev);
6426
6427 /* allocate rlc buffers */
6428 if (rdev->family == CHIP_VERDE) {
6429 rdev->rlc.reg_list = verde_rlc_save_restore_register_list;
6430 rdev->rlc.reg_list_size =
6431 (u32)ARRAY_SIZE(verde_rlc_save_restore_register_list);
6432 }
6433 rdev->rlc.cs_data = si_cs_data;
6434 r = sumo_rlc_init(rdev);
6435 if (r) {
6436 DRM_ERROR("Failed to init rlc BOs!\n");
6437 return r;
6438 }
6439
6440 /* allocate wb buffer */
6441 r = radeon_wb_init(rdev);
6442 if (r)
6443 return r;
6444
6445 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
6446 if (r) {
6447 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6448 return r;
6449 }
6450
6451 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6452 if (r) {
6453 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6454 return r;
6455 }
6456
6457 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6458 if (r) {
6459 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6460 return r;
6461 }
6462
6463 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
6464 if (r) {
6465 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6466 return r;
6467 }
6468
6469 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6470 if (r) {
6471 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6472 return r;
6473 }
6474
6475 if (rdev->has_uvd) {
6476 r = uvd_v2_2_resume(rdev);
6477 if (!r) {
6478 r = radeon_fence_driver_start_ring(rdev,
6479 R600_RING_TYPE_UVD_INDEX);
6480 if (r)
6481 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
6482 }
6483 if (r)
6484 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
6485 }
6486
6487 /* Enable IRQ */
6488 if (!rdev->irq.installed) {
6489 r = radeon_irq_kms_init(rdev);
6490 if (r)
6491 return r;
6492 }
6493
6494 r = si_irq_init(rdev);
6495 if (r) {
6496 DRM_ERROR("radeon: IH init failed (%d).\n", r);
6497 radeon_irq_kms_fini(rdev);
6498 return r;
6499 }
6500 si_irq_set(rdev);
6501
6502 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6503 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
6504 CP_RB0_RPTR, CP_RB0_WPTR,
6505 RADEON_CP_PACKET2);
6506 if (r)
6507 return r;
6508
6509 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6510 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
6511 CP_RB1_RPTR, CP_RB1_WPTR,
6512 RADEON_CP_PACKET2);
6513 if (r)
6514 return r;
6515
6516 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6517 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
6518 CP_RB2_RPTR, CP_RB2_WPTR,
6519 RADEON_CP_PACKET2);
6520 if (r)
6521 return r;
6522
6523 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6524 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
6525 DMA_RB_RPTR + DMA0_REGISTER_OFFSET,
6526 DMA_RB_WPTR + DMA0_REGISTER_OFFSET,
6527 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
6528 if (r)
6529 return r;
6530
6531 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6532 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
6533 DMA_RB_RPTR + DMA1_REGISTER_OFFSET,
6534 DMA_RB_WPTR + DMA1_REGISTER_OFFSET,
6535 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
6536 if (r)
6537 return r;
6538
6539 r = si_cp_load_microcode(rdev);
6540 if (r)
6541 return r;
6542 r = si_cp_resume(rdev);
6543 if (r)
6544 return r;
6545
6546 r = cayman_dma_resume(rdev);
6547 if (r)
6548 return r;
6549
6550 if (rdev->has_uvd) {
6551 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6552 if (ring->ring_size) {
6553 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
6554 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
6555 RADEON_CP_PACKET2);
6556 if (!r)
6557 r = uvd_v1_0_init(rdev);
6558 if (r)
6559 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
6560 }
6561 }
6562
6563 r = radeon_ib_pool_init(rdev);
6564 if (r) {
6565 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
6566 return r;
6567 }
6568
6569 r = radeon_vm_manager_init(rdev);
6570 if (r) {
6571 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
6572 return r;
6573 }
6574
6575 r = dce6_audio_init(rdev);
6576 if (r)
6577 return r;
6578
6579 return 0;
6580 }
6581
6582 int si_resume(struct radeon_device *rdev)
6583 {
6584 int r;
6585
6586 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
6587 * posting will perform necessary task to bring back GPU into good
6588 * shape.
6589 */
6590 /* post card */
6591 atom_asic_init(rdev->mode_info.atom_context);
6592
6593 /* init golden registers */
6594 si_init_golden_registers(rdev);
6595
6596 rdev->accel_working = true;
6597 r = si_startup(rdev);
6598 if (r) {
6599 DRM_ERROR("si startup failed on resume\n");
6600 rdev->accel_working = false;
6601 return r;
6602 }
6603
6604 return r;
6605
6606 }
6607
6608 int si_suspend(struct radeon_device *rdev)
6609 {
6610 dce6_audio_fini(rdev);
6611 radeon_vm_manager_fini(rdev);
6612 si_cp_enable(rdev, false);
6613 cayman_dma_stop(rdev);
6614 if (rdev->has_uvd) {
6615 uvd_v1_0_fini(rdev);
6616 radeon_uvd_suspend(rdev);
6617 }
6618 si_fini_pg(rdev);
6619 si_fini_cg(rdev);
6620 si_irq_suspend(rdev);
6621 radeon_wb_disable(rdev);
6622 si_pcie_gart_disable(rdev);
6623 return 0;
6624 }
6625
6626 /* Plan is to move initialization in that function and use
6627 * helper function so that radeon_device_init pretty much
6628 * do nothing more than calling asic specific function. This
6629 * should also allow to remove a bunch of callback function
6630 * like vram_info.
6631 */
6632 int si_init(struct radeon_device *rdev)
6633 {
6634 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6635 int r;
6636
6637 /* Read BIOS */
6638 if (!radeon_get_bios(rdev)) {
6639 if (ASIC_IS_AVIVO(rdev))
6640 return -EINVAL;
6641 }
6642 /* Must be an ATOMBIOS */
6643 if (!rdev->is_atom_bios) {
6644 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
6645 return -EINVAL;
6646 }
6647 r = radeon_atombios_init(rdev);
6648 if (r)
6649 return r;
6650
6651 /* Post card if necessary */
6652 if (!radeon_card_posted(rdev)) {
6653 if (!rdev->bios) {
6654 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
6655 return -EINVAL;
6656 }
6657 DRM_INFO("GPU not posted. posting now...\n");
6658 atom_asic_init(rdev->mode_info.atom_context);
6659 }
6660 /* init golden registers */
6661 si_init_golden_registers(rdev);
6662 /* Initialize scratch registers */
6663 si_scratch_init(rdev);
6664 /* Initialize surface registers */
6665 radeon_surface_init(rdev);
6666 /* Initialize clocks */
6667 radeon_get_clock_info(rdev->ddev);
6668
6669 /* Fence driver */
6670 r = radeon_fence_driver_init(rdev);
6671 if (r)
6672 return r;
6673
6674 /* initialize memory controller */
6675 r = si_mc_init(rdev);
6676 if (r)
6677 return r;
6678 /* Memory manager */
6679 r = radeon_bo_init(rdev);
6680 if (r)
6681 return r;
6682
6683 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
6684 !rdev->rlc_fw || !rdev->mc_fw) {
6685 r = si_init_microcode(rdev);
6686 if (r) {
6687 DRM_ERROR("Failed to load firmware!\n");
6688 return r;
6689 }
6690 }
6691
6692 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6693 ring->ring_obj = NULL;
6694 r600_ring_init(rdev, ring, 1024 * 1024);
6695
6696 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6697 ring->ring_obj = NULL;
6698 r600_ring_init(rdev, ring, 1024 * 1024);
6699
6700 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6701 ring->ring_obj = NULL;
6702 r600_ring_init(rdev, ring, 1024 * 1024);
6703
6704 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6705 ring->ring_obj = NULL;
6706 r600_ring_init(rdev, ring, 64 * 1024);
6707
6708 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6709 ring->ring_obj = NULL;
6710 r600_ring_init(rdev, ring, 64 * 1024);
6711
6712 if (rdev->has_uvd) {
6713 r = radeon_uvd_init(rdev);
6714 if (!r) {
6715 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6716 ring->ring_obj = NULL;
6717 r600_ring_init(rdev, ring, 4096);
6718 }
6719 }
6720
6721 rdev->ih.ring_obj = NULL;
6722 r600_ih_ring_init(rdev, 64 * 1024);
6723
6724 r = r600_pcie_gart_init(rdev);
6725 if (r)
6726 return r;
6727
6728 rdev->accel_working = true;
6729 r = si_startup(rdev);
6730 if (r) {
6731 dev_err(rdev->dev, "disabling GPU acceleration\n");
6732 si_cp_fini(rdev);
6733 cayman_dma_fini(rdev);
6734 si_irq_fini(rdev);
6735 sumo_rlc_fini(rdev);
6736 radeon_wb_fini(rdev);
6737 radeon_ib_pool_fini(rdev);
6738 radeon_vm_manager_fini(rdev);
6739 radeon_irq_kms_fini(rdev);
6740 si_pcie_gart_fini(rdev);
6741 rdev->accel_working = false;
6742 }
6743
6744 /* Don't start up if the MC ucode is missing.
6745 * The default clocks and voltages before the MC ucode
6746 * is loaded are not suffient for advanced operations.
6747 */
6748 if (!rdev->mc_fw) {
6749 DRM_ERROR("radeon: MC ucode required for NI+.\n");
6750 return -EINVAL;
6751 }
6752
6753 return 0;
6754 }
6755
6756 void si_fini(struct radeon_device *rdev)
6757 {
6758 si_cp_fini(rdev);
6759 cayman_dma_fini(rdev);
6760 si_fini_pg(rdev);
6761 si_fini_cg(rdev);
6762 si_irq_fini(rdev);
6763 sumo_rlc_fini(rdev);
6764 radeon_wb_fini(rdev);
6765 radeon_vm_manager_fini(rdev);
6766 radeon_ib_pool_fini(rdev);
6767 radeon_irq_kms_fini(rdev);
6768 if (rdev->has_uvd) {
6769 uvd_v1_0_fini(rdev);
6770 radeon_uvd_fini(rdev);
6771 }
6772 si_pcie_gart_fini(rdev);
6773 r600_vram_scratch_fini(rdev);
6774 radeon_gem_fini(rdev);
6775 radeon_fence_driver_fini(rdev);
6776 radeon_bo_fini(rdev);
6777 radeon_atombios_fini(rdev);
6778 kfree(rdev->bios);
6779 rdev->bios = NULL;
6780 }
6781
6782 /**
6783 * si_get_gpu_clock_counter - return GPU clock counter snapshot
6784 *
6785 * @rdev: radeon_device pointer
6786 *
6787 * Fetches a GPU clock counter snapshot (SI).
6788 * Returns the 64 bit clock counter snapshot.
6789 */
6790 uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
6791 {
6792 uint64_t clock;
6793
6794 mutex_lock(&rdev->gpu_clock_mutex);
6795 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
6796 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
6797 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
6798 mutex_unlock(&rdev->gpu_clock_mutex);
6799 return clock;
6800 }
6801
6802 int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
6803 {
6804 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
6805 int r;
6806
6807 /* bypass vclk and dclk with bclk */
6808 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6809 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
6810 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6811
6812 /* put PLL in bypass mode */
6813 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
6814
6815 if (!vclk || !dclk) {
6816 /* keep the Bypass mode, put PLL to sleep */
6817 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
6818 return 0;
6819 }
6820
6821 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
6822 16384, 0x03FFFFFF, 0, 128, 5,
6823 &fb_div, &vclk_div, &dclk_div);
6824 if (r)
6825 return r;
6826
6827 /* set RESET_ANTI_MUX to 0 */
6828 WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
6829
6830 /* set VCO_MODE to 1 */
6831 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
6832
6833 /* toggle UPLL_SLEEP to 1 then back to 0 */
6834 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
6835 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
6836
6837 /* deassert UPLL_RESET */
6838 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
6839
6840 mdelay(1);
6841
6842 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
6843 if (r)
6844 return r;
6845
6846 /* assert UPLL_RESET again */
6847 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
6848
6849 /* disable spread spectrum. */
6850 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
6851
6852 /* set feedback divider */
6853 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
6854
6855 /* set ref divider to 0 */
6856 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
6857
6858 if (fb_div < 307200)
6859 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
6860 else
6861 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
6862
6863 /* set PDIV_A and PDIV_B */
6864 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6865 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
6866 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
6867
6868 /* give the PLL some time to settle */
6869 mdelay(15);
6870
6871 /* deassert PLL_RESET */
6872 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
6873
6874 mdelay(15);
6875
6876 /* switch from bypass mode to normal mode */
6877 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
6878
6879 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
6880 if (r)
6881 return r;
6882
6883 /* switch VCLK and DCLK selection */
6884 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6885 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
6886 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6887
6888 mdelay(100);
6889
6890 return 0;
6891 }
6892
6893 static void si_pcie_gen3_enable(struct radeon_device *rdev)
6894 {
6895 struct pci_dev *root = rdev->pdev->bus->self;
6896 int bridge_pos, gpu_pos;
6897 u32 speed_cntl, mask, current_data_rate;
6898 int ret, i;
6899 u16 tmp16;
6900
6901 if (radeon_pcie_gen2 == 0)
6902 return;
6903
6904 if (rdev->flags & RADEON_IS_IGP)
6905 return;
6906
6907 if (!(rdev->flags & RADEON_IS_PCIE))
6908 return;
6909
6910 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
6911 if (ret != 0)
6912 return;
6913
6914 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
6915 return;
6916
6917 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
6918 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
6919 LC_CURRENT_DATA_RATE_SHIFT;
6920 if (mask & DRM_PCIE_SPEED_80) {
6921 if (current_data_rate == 2) {
6922 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
6923 return;
6924 }
6925 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
6926 } else if (mask & DRM_PCIE_SPEED_50) {
6927 if (current_data_rate == 1) {
6928 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
6929 return;
6930 }
6931 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
6932 }
6933
6934 bridge_pos = pci_pcie_cap(root);
6935 if (!bridge_pos)
6936 return;
6937
6938 gpu_pos = pci_pcie_cap(rdev->pdev);
6939 if (!gpu_pos)
6940 return;
6941
6942 if (mask & DRM_PCIE_SPEED_80) {
6943 /* re-try equalization if gen3 is not already enabled */
6944 if (current_data_rate != 2) {
6945 u16 bridge_cfg, gpu_cfg;
6946 u16 bridge_cfg2, gpu_cfg2;
6947 u32 max_lw, current_lw, tmp;
6948
6949 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
6950 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
6951
6952 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
6953 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
6954
6955 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
6956 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
6957
6958 tmp = RREG32_PCIE(PCIE_LC_STATUS1);
6959 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
6960 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
6961
6962 if (current_lw < max_lw) {
6963 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
6964 if (tmp & LC_RENEGOTIATION_SUPPORT) {
6965 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
6966 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
6967 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
6968 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
6969 }
6970 }
6971
6972 for (i = 0; i < 10; i++) {
6973 /* check status */
6974 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
6975 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
6976 break;
6977
6978 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
6979 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
6980
6981 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
6982 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
6983
6984 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6985 tmp |= LC_SET_QUIESCE;
6986 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6987
6988 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6989 tmp |= LC_REDO_EQ;
6990 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6991
6992 mdelay(100);
6993
6994 /* linkctl */
6995 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
6996 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
6997 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
6998 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
6999
7000 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
7001 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
7002 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
7003 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
7004
7005 /* linkctl2 */
7006 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
7007 tmp16 &= ~((1 << 4) | (7 << 9));
7008 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
7009 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
7010
7011 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
7012 tmp16 &= ~((1 << 4) | (7 << 9));
7013 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
7014 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
7015
7016 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7017 tmp &= ~LC_SET_QUIESCE;
7018 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7019 }
7020 }
7021 }
7022
7023 /* set the link speed */
7024 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
7025 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
7026 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7027
7028 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
7029 tmp16 &= ~0xf;
7030 if (mask & DRM_PCIE_SPEED_80)
7031 tmp16 |= 3; /* gen3 */
7032 else if (mask & DRM_PCIE_SPEED_50)
7033 tmp16 |= 2; /* gen2 */
7034 else
7035 tmp16 |= 1; /* gen1 */
7036 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
7037
7038 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7039 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
7040 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7041
7042 for (i = 0; i < rdev->usec_timeout; i++) {
7043 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7044 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
7045 break;
7046 udelay(1);
7047 }
7048 }
7049
7050 static void si_program_aspm(struct radeon_device *rdev)
7051 {
7052 u32 data, orig;
7053 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
7054 bool disable_clkreq = false;
7055
7056 if (radeon_aspm == 0)
7057 return;
7058
7059 if (!(rdev->flags & RADEON_IS_PCIE))
7060 return;
7061
7062 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7063 data &= ~LC_XMIT_N_FTS_MASK;
7064 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
7065 if (orig != data)
7066 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
7067
7068 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
7069 data |= LC_GO_TO_RECOVERY;
7070 if (orig != data)
7071 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
7072
7073 orig = data = RREG32_PCIE(PCIE_P_CNTL);
7074 data |= P_IGNORE_EDB_ERR;
7075 if (orig != data)
7076 WREG32_PCIE(PCIE_P_CNTL, data);
7077
7078 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7079 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
7080 data |= LC_PMI_TO_L1_DIS;
7081 if (!disable_l0s)
7082 data |= LC_L0S_INACTIVITY(7);
7083
7084 if (!disable_l1) {
7085 data |= LC_L1_INACTIVITY(7);
7086 data &= ~LC_PMI_TO_L1_DIS;
7087 if (orig != data)
7088 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7089
7090 if (!disable_plloff_in_l1) {
7091 bool clk_req_support;
7092
7093 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7094 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7095 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7096 if (orig != data)
7097 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7098
7099 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7100 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7101 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7102 if (orig != data)
7103 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7104
7105 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7106 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7107 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7108 if (orig != data)
7109 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7110
7111 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7112 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7113 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7114 if (orig != data)
7115 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7116
7117 if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) {
7118 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7119 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7120 if (orig != data)
7121 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7122
7123 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7124 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7125 if (orig != data)
7126 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7127
7128 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2);
7129 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7130 if (orig != data)
7131 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data);
7132
7133 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3);
7134 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7135 if (orig != data)
7136 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data);
7137
7138 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7139 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7140 if (orig != data)
7141 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7142
7143 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7144 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7145 if (orig != data)
7146 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7147
7148 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2);
7149 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7150 if (orig != data)
7151 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data);
7152
7153 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3);
7154 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7155 if (orig != data)
7156 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data);
7157 }
7158 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7159 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
7160 data |= LC_DYN_LANES_PWR_STATE(3);
7161 if (orig != data)
7162 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
7163
7164 orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL);
7165 data &= ~LS2_EXIT_TIME_MASK;
7166 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7167 data |= LS2_EXIT_TIME(5);
7168 if (orig != data)
7169 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
7170
7171 orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL);
7172 data &= ~LS2_EXIT_TIME_MASK;
7173 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7174 data |= LS2_EXIT_TIME(5);
7175 if (orig != data)
7176 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
7177
7178 if (!disable_clkreq) {
7179 struct pci_dev *root = rdev->pdev->bus->self;
7180 u32 lnkcap;
7181
7182 clk_req_support = false;
7183 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
7184 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
7185 clk_req_support = true;
7186 } else {
7187 clk_req_support = false;
7188 }
7189
7190 if (clk_req_support) {
7191 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
7192 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
7193 if (orig != data)
7194 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
7195
7196 orig = data = RREG32(THM_CLK_CNTL);
7197 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
7198 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
7199 if (orig != data)
7200 WREG32(THM_CLK_CNTL, data);
7201
7202 orig = data = RREG32(MISC_CLK_CNTL);
7203 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
7204 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
7205 if (orig != data)
7206 WREG32(MISC_CLK_CNTL, data);
7207
7208 orig = data = RREG32(CG_CLKPIN_CNTL);
7209 data &= ~BCLK_AS_XCLK;
7210 if (orig != data)
7211 WREG32(CG_CLKPIN_CNTL, data);
7212
7213 orig = data = RREG32(CG_CLKPIN_CNTL_2);
7214 data &= ~FORCE_BIF_REFCLK_EN;
7215 if (orig != data)
7216 WREG32(CG_CLKPIN_CNTL_2, data);
7217
7218 orig = data = RREG32(MPLL_BYPASSCLK_SEL);
7219 data &= ~MPLL_CLKOUT_SEL_MASK;
7220 data |= MPLL_CLKOUT_SEL(4);
7221 if (orig != data)
7222 WREG32(MPLL_BYPASSCLK_SEL, data);
7223
7224 orig = data = RREG32(SPLL_CNTL_MODE);
7225 data &= ~SPLL_REFCLK_SEL_MASK;
7226 if (orig != data)
7227 WREG32(SPLL_CNTL_MODE, data);
7228 }
7229 }
7230 } else {
7231 if (orig != data)
7232 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7233 }
7234
7235 orig = data = RREG32_PCIE(PCIE_CNTL2);
7236 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
7237 if (orig != data)
7238 WREG32_PCIE(PCIE_CNTL2, data);
7239
7240 if (!disable_l0s) {
7241 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7242 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
7243 data = RREG32_PCIE(PCIE_LC_STATUS1);
7244 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
7245 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7246 data &= ~LC_L0S_INACTIVITY_MASK;
7247 if (orig != data)
7248 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7249 }
7250 }
7251 }
7252 }
Linux with added FPC-III support