?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
x86/xen: resume timer irqs early
[linux/fpc-iii.git] / drivers / gpu / drm / radeon / evergreen.c
blob7ca58fc7a1c6f5d2866e9d49600f794685dcee5a
1 /*
2 * Copyright 2010 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24 #include <linux/firmware.h>
25 #include <linux/platform_device.h>
26 #include <linux/slab.h>
27 #include <drm/drmP.h>
28 #include "radeon.h"
29 #include "radeon_asic.h"
30 #include <drm/radeon_drm.h>
31 #include "evergreend.h"
32 #include "atom.h"
33 #include "avivod.h"
34 #include "evergreen_reg.h"
35 #include "evergreen_blit_shaders.h"
36 #include "radeon_ucode.h"
37
38 static const u32 crtc_offsets[6] =
39 {
40 EVERGREEN_CRTC0_REGISTER_OFFSET,
41 EVERGREEN_CRTC1_REGISTER_OFFSET,
42 EVERGREEN_CRTC2_REGISTER_OFFSET,
43 EVERGREEN_CRTC3_REGISTER_OFFSET,
44 EVERGREEN_CRTC4_REGISTER_OFFSET,
45 EVERGREEN_CRTC5_REGISTER_OFFSET
46 };
47
48 #include "clearstate_evergreen.h"
49
50 static const u32 sumo_rlc_save_restore_register_list[] =
51 {
52 0x98fc,
53 0x9830,
54 0x9834,
55 0x9838,
56 0x9870,
57 0x9874,
58 0x8a14,
59 0x8b24,
60 0x8bcc,
61 0x8b10,
62 0x8d00,
63 0x8d04,
64 0x8c00,
65 0x8c04,
66 0x8c08,
67 0x8c0c,
68 0x8d8c,
69 0x8c20,
70 0x8c24,
71 0x8c28,
72 0x8c18,
73 0x8c1c,
74 0x8cf0,
75 0x8e2c,
76 0x8e38,
77 0x8c30,
78 0x9508,
79 0x9688,
80 0x9608,
81 0x960c,
82 0x9610,
83 0x9614,
84 0x88c4,
85 0x88d4,
86 0xa008,
87 0x900c,
88 0x9100,
89 0x913c,
90 0x98f8,
91 0x98f4,
92 0x9b7c,
93 0x3f8c,
94 0x8950,
95 0x8954,
96 0x8a18,
97 0x8b28,
98 0x9144,
99 0x9148,
100 0x914c,
101 0x3f90,
102 0x3f94,
103 0x915c,
104 0x9160,
105 0x9178,
106 0x917c,
107 0x9180,
108 0x918c,
109 0x9190,
110 0x9194,
111 0x9198,
112 0x919c,
113 0x91a8,
114 0x91ac,
115 0x91b0,
116 0x91b4,
117 0x91b8,
118 0x91c4,
119 0x91c8,
120 0x91cc,
121 0x91d0,
122 0x91d4,
123 0x91e0,
124 0x91e4,
125 0x91ec,
126 0x91f0,
127 0x91f4,
128 0x9200,
129 0x9204,
130 0x929c,
131 0x9150,
132 0x802c,
133 };
134
135 static void evergreen_gpu_init(struct radeon_device *rdev);
136 void evergreen_fini(struct radeon_device *rdev);
137 void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
138 void evergreen_program_aspm(struct radeon_device *rdev);
139 extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
140 int ring, u32 cp_int_cntl);
141 extern void cayman_vm_decode_fault(struct radeon_device *rdev,
142 u32 status, u32 addr);
143 void cik_init_cp_pg_table(struct radeon_device *rdev);
144
145 extern u32 si_get_csb_size(struct radeon_device *rdev);
146 extern void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
147 extern u32 cik_get_csb_size(struct radeon_device *rdev);
148 extern void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
149
150 static const u32 evergreen_golden_registers[] =
151 {
152 0x3f90, 0xffff0000, 0xff000000,
153 0x9148, 0xffff0000, 0xff000000,
154 0x3f94, 0xffff0000, 0xff000000,
155 0x914c, 0xffff0000, 0xff000000,
156 0x9b7c, 0xffffffff, 0x00000000,
157 0x8a14, 0xffffffff, 0x00000007,
158 0x8b10, 0xffffffff, 0x00000000,
159 0x960c, 0xffffffff, 0x54763210,
160 0x88c4, 0xffffffff, 0x000000c2,
161 0x88d4, 0xffffffff, 0x00000010,
162 0x8974, 0xffffffff, 0x00000000,
163 0xc78, 0x00000080, 0x00000080,
164 0x5eb4, 0xffffffff, 0x00000002,
165 0x5e78, 0xffffffff, 0x001000f0,
166 0x6104, 0x01000300, 0x00000000,
167 0x5bc0, 0x00300000, 0x00000000,
168 0x7030, 0xffffffff, 0x00000011,
169 0x7c30, 0xffffffff, 0x00000011,
170 0x10830, 0xffffffff, 0x00000011,
171 0x11430, 0xffffffff, 0x00000011,
172 0x12030, 0xffffffff, 0x00000011,
173 0x12c30, 0xffffffff, 0x00000011,
174 0xd02c, 0xffffffff, 0x08421000,
175 0x240c, 0xffffffff, 0x00000380,
176 0x8b24, 0xffffffff, 0x00ff0fff,
177 0x28a4c, 0x06000000, 0x06000000,
178 0x10c, 0x00000001, 0x00000001,
179 0x8d00, 0xffffffff, 0x100e4848,
180 0x8d04, 0xffffffff, 0x00164745,
181 0x8c00, 0xffffffff, 0xe4000003,
182 0x8c04, 0xffffffff, 0x40600060,
183 0x8c08, 0xffffffff, 0x001c001c,
184 0x8cf0, 0xffffffff, 0x08e00620,
185 0x8c20, 0xffffffff, 0x00800080,
186 0x8c24, 0xffffffff, 0x00800080,
187 0x8c18, 0xffffffff, 0x20202078,
188 0x8c1c, 0xffffffff, 0x00001010,
189 0x28350, 0xffffffff, 0x00000000,
190 0xa008, 0xffffffff, 0x00010000,
191 0x5c4, 0xffffffff, 0x00000001,
192 0x9508, 0xffffffff, 0x00000002,
193 0x913c, 0x0000000f, 0x0000000a
194 };
195
196 static const u32 evergreen_golden_registers2[] =
197 {
198 0x2f4c, 0xffffffff, 0x00000000,
199 0x54f4, 0xffffffff, 0x00000000,
200 0x54f0, 0xffffffff, 0x00000000,
201 0x5498, 0xffffffff, 0x00000000,
202 0x549c, 0xffffffff, 0x00000000,
203 0x5494, 0xffffffff, 0x00000000,
204 0x53cc, 0xffffffff, 0x00000000,
205 0x53c8, 0xffffffff, 0x00000000,
206 0x53c4, 0xffffffff, 0x00000000,
207 0x53c0, 0xffffffff, 0x00000000,
208 0x53bc, 0xffffffff, 0x00000000,
209 0x53b8, 0xffffffff, 0x00000000,
210 0x53b4, 0xffffffff, 0x00000000,
211 0x53b0, 0xffffffff, 0x00000000
212 };
213
214 static const u32 cypress_mgcg_init[] =
215 {
216 0x802c, 0xffffffff, 0xc0000000,
217 0x5448, 0xffffffff, 0x00000100,
218 0x55e4, 0xffffffff, 0x00000100,
219 0x160c, 0xffffffff, 0x00000100,
220 0x5644, 0xffffffff, 0x00000100,
221 0xc164, 0xffffffff, 0x00000100,
222 0x8a18, 0xffffffff, 0x00000100,
223 0x897c, 0xffffffff, 0x06000100,
224 0x8b28, 0xffffffff, 0x00000100,
225 0x9144, 0xffffffff, 0x00000100,
226 0x9a60, 0xffffffff, 0x00000100,
227 0x9868, 0xffffffff, 0x00000100,
228 0x8d58, 0xffffffff, 0x00000100,
229 0x9510, 0xffffffff, 0x00000100,
230 0x949c, 0xffffffff, 0x00000100,
231 0x9654, 0xffffffff, 0x00000100,
232 0x9030, 0xffffffff, 0x00000100,
233 0x9034, 0xffffffff, 0x00000100,
234 0x9038, 0xffffffff, 0x00000100,
235 0x903c, 0xffffffff, 0x00000100,
236 0x9040, 0xffffffff, 0x00000100,
237 0xa200, 0xffffffff, 0x00000100,
238 0xa204, 0xffffffff, 0x00000100,
239 0xa208, 0xffffffff, 0x00000100,
240 0xa20c, 0xffffffff, 0x00000100,
241 0x971c, 0xffffffff, 0x00000100,
242 0x977c, 0xffffffff, 0x00000100,
243 0x3f80, 0xffffffff, 0x00000100,
244 0xa210, 0xffffffff, 0x00000100,
245 0xa214, 0xffffffff, 0x00000100,
246 0x4d8, 0xffffffff, 0x00000100,
247 0x9784, 0xffffffff, 0x00000100,
248 0x9698, 0xffffffff, 0x00000100,
249 0x4d4, 0xffffffff, 0x00000200,
250 0x30cc, 0xffffffff, 0x00000100,
251 0xd0c0, 0xffffffff, 0xff000100,
252 0x802c, 0xffffffff, 0x40000000,
253 0x915c, 0xffffffff, 0x00010000,
254 0x9160, 0xffffffff, 0x00030002,
255 0x9178, 0xffffffff, 0x00070000,
256 0x917c, 0xffffffff, 0x00030002,
257 0x9180, 0xffffffff, 0x00050004,
258 0x918c, 0xffffffff, 0x00010006,
259 0x9190, 0xffffffff, 0x00090008,
260 0x9194, 0xffffffff, 0x00070000,
261 0x9198, 0xffffffff, 0x00030002,
262 0x919c, 0xffffffff, 0x00050004,
263 0x91a8, 0xffffffff, 0x00010006,
264 0x91ac, 0xffffffff, 0x00090008,
265 0x91b0, 0xffffffff, 0x00070000,
266 0x91b4, 0xffffffff, 0x00030002,
267 0x91b8, 0xffffffff, 0x00050004,
268 0x91c4, 0xffffffff, 0x00010006,
269 0x91c8, 0xffffffff, 0x00090008,
270 0x91cc, 0xffffffff, 0x00070000,
271 0x91d0, 0xffffffff, 0x00030002,
272 0x91d4, 0xffffffff, 0x00050004,
273 0x91e0, 0xffffffff, 0x00010006,
274 0x91e4, 0xffffffff, 0x00090008,
275 0x91e8, 0xffffffff, 0x00000000,
276 0x91ec, 0xffffffff, 0x00070000,
277 0x91f0, 0xffffffff, 0x00030002,
278 0x91f4, 0xffffffff, 0x00050004,
279 0x9200, 0xffffffff, 0x00010006,
280 0x9204, 0xffffffff, 0x00090008,
281 0x9208, 0xffffffff, 0x00070000,
282 0x920c, 0xffffffff, 0x00030002,
283 0x9210, 0xffffffff, 0x00050004,
284 0x921c, 0xffffffff, 0x00010006,
285 0x9220, 0xffffffff, 0x00090008,
286 0x9224, 0xffffffff, 0x00070000,
287 0x9228, 0xffffffff, 0x00030002,
288 0x922c, 0xffffffff, 0x00050004,
289 0x9238, 0xffffffff, 0x00010006,
290 0x923c, 0xffffffff, 0x00090008,
291 0x9240, 0xffffffff, 0x00070000,
292 0x9244, 0xffffffff, 0x00030002,
293 0x9248, 0xffffffff, 0x00050004,
294 0x9254, 0xffffffff, 0x00010006,
295 0x9258, 0xffffffff, 0x00090008,
296 0x925c, 0xffffffff, 0x00070000,
297 0x9260, 0xffffffff, 0x00030002,
298 0x9264, 0xffffffff, 0x00050004,
299 0x9270, 0xffffffff, 0x00010006,
300 0x9274, 0xffffffff, 0x00090008,
301 0x9278, 0xffffffff, 0x00070000,
302 0x927c, 0xffffffff, 0x00030002,
303 0x9280, 0xffffffff, 0x00050004,
304 0x928c, 0xffffffff, 0x00010006,
305 0x9290, 0xffffffff, 0x00090008,
306 0x9294, 0xffffffff, 0x00000000,
307 0x929c, 0xffffffff, 0x00000001,
308 0x802c, 0xffffffff, 0x40010000,
309 0x915c, 0xffffffff, 0x00010000,
310 0x9160, 0xffffffff, 0x00030002,
311 0x9178, 0xffffffff, 0x00070000,
312 0x917c, 0xffffffff, 0x00030002,
313 0x9180, 0xffffffff, 0x00050004,
314 0x918c, 0xffffffff, 0x00010006,
315 0x9190, 0xffffffff, 0x00090008,
316 0x9194, 0xffffffff, 0x00070000,
317 0x9198, 0xffffffff, 0x00030002,
318 0x919c, 0xffffffff, 0x00050004,
319 0x91a8, 0xffffffff, 0x00010006,
320 0x91ac, 0xffffffff, 0x00090008,
321 0x91b0, 0xffffffff, 0x00070000,
322 0x91b4, 0xffffffff, 0x00030002,
323 0x91b8, 0xffffffff, 0x00050004,
324 0x91c4, 0xffffffff, 0x00010006,
325 0x91c8, 0xffffffff, 0x00090008,
326 0x91cc, 0xffffffff, 0x00070000,
327 0x91d0, 0xffffffff, 0x00030002,
328 0x91d4, 0xffffffff, 0x00050004,
329 0x91e0, 0xffffffff, 0x00010006,
330 0x91e4, 0xffffffff, 0x00090008,
331 0x91e8, 0xffffffff, 0x00000000,
332 0x91ec, 0xffffffff, 0x00070000,
333 0x91f0, 0xffffffff, 0x00030002,
334 0x91f4, 0xffffffff, 0x00050004,
335 0x9200, 0xffffffff, 0x00010006,
336 0x9204, 0xffffffff, 0x00090008,
337 0x9208, 0xffffffff, 0x00070000,
338 0x920c, 0xffffffff, 0x00030002,
339 0x9210, 0xffffffff, 0x00050004,
340 0x921c, 0xffffffff, 0x00010006,
341 0x9220, 0xffffffff, 0x00090008,
342 0x9224, 0xffffffff, 0x00070000,
343 0x9228, 0xffffffff, 0x00030002,
344 0x922c, 0xffffffff, 0x00050004,
345 0x9238, 0xffffffff, 0x00010006,
346 0x923c, 0xffffffff, 0x00090008,
347 0x9240, 0xffffffff, 0x00070000,
348 0x9244, 0xffffffff, 0x00030002,
349 0x9248, 0xffffffff, 0x00050004,
350 0x9254, 0xffffffff, 0x00010006,
351 0x9258, 0xffffffff, 0x00090008,
352 0x925c, 0xffffffff, 0x00070000,
353 0x9260, 0xffffffff, 0x00030002,
354 0x9264, 0xffffffff, 0x00050004,
355 0x9270, 0xffffffff, 0x00010006,
356 0x9274, 0xffffffff, 0x00090008,
357 0x9278, 0xffffffff, 0x00070000,
358 0x927c, 0xffffffff, 0x00030002,
359 0x9280, 0xffffffff, 0x00050004,
360 0x928c, 0xffffffff, 0x00010006,
361 0x9290, 0xffffffff, 0x00090008,
362 0x9294, 0xffffffff, 0x00000000,
363 0x929c, 0xffffffff, 0x00000001,
364 0x802c, 0xffffffff, 0xc0000000
365 };
366
367 static const u32 redwood_mgcg_init[] =
368 {
369 0x802c, 0xffffffff, 0xc0000000,
370 0x5448, 0xffffffff, 0x00000100,
371 0x55e4, 0xffffffff, 0x00000100,
372 0x160c, 0xffffffff, 0x00000100,
373 0x5644, 0xffffffff, 0x00000100,
374 0xc164, 0xffffffff, 0x00000100,
375 0x8a18, 0xffffffff, 0x00000100,
376 0x897c, 0xffffffff, 0x06000100,
377 0x8b28, 0xffffffff, 0x00000100,
378 0x9144, 0xffffffff, 0x00000100,
379 0x9a60, 0xffffffff, 0x00000100,
380 0x9868, 0xffffffff, 0x00000100,
381 0x8d58, 0xffffffff, 0x00000100,
382 0x9510, 0xffffffff, 0x00000100,
383 0x949c, 0xffffffff, 0x00000100,
384 0x9654, 0xffffffff, 0x00000100,
385 0x9030, 0xffffffff, 0x00000100,
386 0x9034, 0xffffffff, 0x00000100,
387 0x9038, 0xffffffff, 0x00000100,
388 0x903c, 0xffffffff, 0x00000100,
389 0x9040, 0xffffffff, 0x00000100,
390 0xa200, 0xffffffff, 0x00000100,
391 0xa204, 0xffffffff, 0x00000100,
392 0xa208, 0xffffffff, 0x00000100,
393 0xa20c, 0xffffffff, 0x00000100,
394 0x971c, 0xffffffff, 0x00000100,
395 0x977c, 0xffffffff, 0x00000100,
396 0x3f80, 0xffffffff, 0x00000100,
397 0xa210, 0xffffffff, 0x00000100,
398 0xa214, 0xffffffff, 0x00000100,
399 0x4d8, 0xffffffff, 0x00000100,
400 0x9784, 0xffffffff, 0x00000100,
401 0x9698, 0xffffffff, 0x00000100,
402 0x4d4, 0xffffffff, 0x00000200,
403 0x30cc, 0xffffffff, 0x00000100,
404 0xd0c0, 0xffffffff, 0xff000100,
405 0x802c, 0xffffffff, 0x40000000,
406 0x915c, 0xffffffff, 0x00010000,
407 0x9160, 0xffffffff, 0x00030002,
408 0x9178, 0xffffffff, 0x00070000,
409 0x917c, 0xffffffff, 0x00030002,
410 0x9180, 0xffffffff, 0x00050004,
411 0x918c, 0xffffffff, 0x00010006,
412 0x9190, 0xffffffff, 0x00090008,
413 0x9194, 0xffffffff, 0x00070000,
414 0x9198, 0xffffffff, 0x00030002,
415 0x919c, 0xffffffff, 0x00050004,
416 0x91a8, 0xffffffff, 0x00010006,
417 0x91ac, 0xffffffff, 0x00090008,
418 0x91b0, 0xffffffff, 0x00070000,
419 0x91b4, 0xffffffff, 0x00030002,
420 0x91b8, 0xffffffff, 0x00050004,
421 0x91c4, 0xffffffff, 0x00010006,
422 0x91c8, 0xffffffff, 0x00090008,
423 0x91cc, 0xffffffff, 0x00070000,
424 0x91d0, 0xffffffff, 0x00030002,
425 0x91d4, 0xffffffff, 0x00050004,
426 0x91e0, 0xffffffff, 0x00010006,
427 0x91e4, 0xffffffff, 0x00090008,
428 0x91e8, 0xffffffff, 0x00000000,
429 0x91ec, 0xffffffff, 0x00070000,
430 0x91f0, 0xffffffff, 0x00030002,
431 0x91f4, 0xffffffff, 0x00050004,
432 0x9200, 0xffffffff, 0x00010006,
433 0x9204, 0xffffffff, 0x00090008,
434 0x9294, 0xffffffff, 0x00000000,
435 0x929c, 0xffffffff, 0x00000001,
436 0x802c, 0xffffffff, 0xc0000000
437 };
438
439 static const u32 cedar_golden_registers[] =
440 {
441 0x3f90, 0xffff0000, 0xff000000,
442 0x9148, 0xffff0000, 0xff000000,
443 0x3f94, 0xffff0000, 0xff000000,
444 0x914c, 0xffff0000, 0xff000000,
445 0x9b7c, 0xffffffff, 0x00000000,
446 0x8a14, 0xffffffff, 0x00000007,
447 0x8b10, 0xffffffff, 0x00000000,
448 0x960c, 0xffffffff, 0x54763210,
449 0x88c4, 0xffffffff, 0x000000c2,
450 0x88d4, 0xffffffff, 0x00000000,
451 0x8974, 0xffffffff, 0x00000000,
452 0xc78, 0x00000080, 0x00000080,
453 0x5eb4, 0xffffffff, 0x00000002,
454 0x5e78, 0xffffffff, 0x001000f0,
455 0x6104, 0x01000300, 0x00000000,
456 0x5bc0, 0x00300000, 0x00000000,
457 0x7030, 0xffffffff, 0x00000011,
458 0x7c30, 0xffffffff, 0x00000011,
459 0x10830, 0xffffffff, 0x00000011,
460 0x11430, 0xffffffff, 0x00000011,
461 0xd02c, 0xffffffff, 0x08421000,
462 0x240c, 0xffffffff, 0x00000380,
463 0x8b24, 0xffffffff, 0x00ff0fff,
464 0x28a4c, 0x06000000, 0x06000000,
465 0x10c, 0x00000001, 0x00000001,
466 0x8d00, 0xffffffff, 0x100e4848,
467 0x8d04, 0xffffffff, 0x00164745,
468 0x8c00, 0xffffffff, 0xe4000003,
469 0x8c04, 0xffffffff, 0x40600060,
470 0x8c08, 0xffffffff, 0x001c001c,
471 0x8cf0, 0xffffffff, 0x08e00410,
472 0x8c20, 0xffffffff, 0x00800080,
473 0x8c24, 0xffffffff, 0x00800080,
474 0x8c18, 0xffffffff, 0x20202078,
475 0x8c1c, 0xffffffff, 0x00001010,
476 0x28350, 0xffffffff, 0x00000000,
477 0xa008, 0xffffffff, 0x00010000,
478 0x5c4, 0xffffffff, 0x00000001,
479 0x9508, 0xffffffff, 0x00000002
480 };
481
482 static const u32 cedar_mgcg_init[] =
483 {
484 0x802c, 0xffffffff, 0xc0000000,
485 0x5448, 0xffffffff, 0x00000100,
486 0x55e4, 0xffffffff, 0x00000100,
487 0x160c, 0xffffffff, 0x00000100,
488 0x5644, 0xffffffff, 0x00000100,
489 0xc164, 0xffffffff, 0x00000100,
490 0x8a18, 0xffffffff, 0x00000100,
491 0x897c, 0xffffffff, 0x06000100,
492 0x8b28, 0xffffffff, 0x00000100,
493 0x9144, 0xffffffff, 0x00000100,
494 0x9a60, 0xffffffff, 0x00000100,
495 0x9868, 0xffffffff, 0x00000100,
496 0x8d58, 0xffffffff, 0x00000100,
497 0x9510, 0xffffffff, 0x00000100,
498 0x949c, 0xffffffff, 0x00000100,
499 0x9654, 0xffffffff, 0x00000100,
500 0x9030, 0xffffffff, 0x00000100,
501 0x9034, 0xffffffff, 0x00000100,
502 0x9038, 0xffffffff, 0x00000100,
503 0x903c, 0xffffffff, 0x00000100,
504 0x9040, 0xffffffff, 0x00000100,
505 0xa200, 0xffffffff, 0x00000100,
506 0xa204, 0xffffffff, 0x00000100,
507 0xa208, 0xffffffff, 0x00000100,
508 0xa20c, 0xffffffff, 0x00000100,
509 0x971c, 0xffffffff, 0x00000100,
510 0x977c, 0xffffffff, 0x00000100,
511 0x3f80, 0xffffffff, 0x00000100,
512 0xa210, 0xffffffff, 0x00000100,
513 0xa214, 0xffffffff, 0x00000100,
514 0x4d8, 0xffffffff, 0x00000100,
515 0x9784, 0xffffffff, 0x00000100,
516 0x9698, 0xffffffff, 0x00000100,
517 0x4d4, 0xffffffff, 0x00000200,
518 0x30cc, 0xffffffff, 0x00000100,
519 0xd0c0, 0xffffffff, 0xff000100,
520 0x802c, 0xffffffff, 0x40000000,
521 0x915c, 0xffffffff, 0x00010000,
522 0x9178, 0xffffffff, 0x00050000,
523 0x917c, 0xffffffff, 0x00030002,
524 0x918c, 0xffffffff, 0x00010004,
525 0x9190, 0xffffffff, 0x00070006,
526 0x9194, 0xffffffff, 0x00050000,
527 0x9198, 0xffffffff, 0x00030002,
528 0x91a8, 0xffffffff, 0x00010004,
529 0x91ac, 0xffffffff, 0x00070006,
530 0x91e8, 0xffffffff, 0x00000000,
531 0x9294, 0xffffffff, 0x00000000,
532 0x929c, 0xffffffff, 0x00000001,
533 0x802c, 0xffffffff, 0xc0000000
534 };
535
536 static const u32 juniper_mgcg_init[] =
537 {
538 0x802c, 0xffffffff, 0xc0000000,
539 0x5448, 0xffffffff, 0x00000100,
540 0x55e4, 0xffffffff, 0x00000100,
541 0x160c, 0xffffffff, 0x00000100,
542 0x5644, 0xffffffff, 0x00000100,
543 0xc164, 0xffffffff, 0x00000100,
544 0x8a18, 0xffffffff, 0x00000100,
545 0x897c, 0xffffffff, 0x06000100,
546 0x8b28, 0xffffffff, 0x00000100,
547 0x9144, 0xffffffff, 0x00000100,
548 0x9a60, 0xffffffff, 0x00000100,
549 0x9868, 0xffffffff, 0x00000100,
550 0x8d58, 0xffffffff, 0x00000100,
551 0x9510, 0xffffffff, 0x00000100,
552 0x949c, 0xffffffff, 0x00000100,
553 0x9654, 0xffffffff, 0x00000100,
554 0x9030, 0xffffffff, 0x00000100,
555 0x9034, 0xffffffff, 0x00000100,
556 0x9038, 0xffffffff, 0x00000100,
557 0x903c, 0xffffffff, 0x00000100,
558 0x9040, 0xffffffff, 0x00000100,
559 0xa200, 0xffffffff, 0x00000100,
560 0xa204, 0xffffffff, 0x00000100,
561 0xa208, 0xffffffff, 0x00000100,
562 0xa20c, 0xffffffff, 0x00000100,
563 0x971c, 0xffffffff, 0x00000100,
564 0xd0c0, 0xffffffff, 0xff000100,
565 0x802c, 0xffffffff, 0x40000000,
566 0x915c, 0xffffffff, 0x00010000,
567 0x9160, 0xffffffff, 0x00030002,
568 0x9178, 0xffffffff, 0x00070000,
569 0x917c, 0xffffffff, 0x00030002,
570 0x9180, 0xffffffff, 0x00050004,
571 0x918c, 0xffffffff, 0x00010006,
572 0x9190, 0xffffffff, 0x00090008,
573 0x9194, 0xffffffff, 0x00070000,
574 0x9198, 0xffffffff, 0x00030002,
575 0x919c, 0xffffffff, 0x00050004,
576 0x91a8, 0xffffffff, 0x00010006,
577 0x91ac, 0xffffffff, 0x00090008,
578 0x91b0, 0xffffffff, 0x00070000,
579 0x91b4, 0xffffffff, 0x00030002,
580 0x91b8, 0xffffffff, 0x00050004,
581 0x91c4, 0xffffffff, 0x00010006,
582 0x91c8, 0xffffffff, 0x00090008,
583 0x91cc, 0xffffffff, 0x00070000,
584 0x91d0, 0xffffffff, 0x00030002,
585 0x91d4, 0xffffffff, 0x00050004,
586 0x91e0, 0xffffffff, 0x00010006,
587 0x91e4, 0xffffffff, 0x00090008,
588 0x91e8, 0xffffffff, 0x00000000,
589 0x91ec, 0xffffffff, 0x00070000,
590 0x91f0, 0xffffffff, 0x00030002,
591 0x91f4, 0xffffffff, 0x00050004,
592 0x9200, 0xffffffff, 0x00010006,
593 0x9204, 0xffffffff, 0x00090008,
594 0x9208, 0xffffffff, 0x00070000,
595 0x920c, 0xffffffff, 0x00030002,
596 0x9210, 0xffffffff, 0x00050004,
597 0x921c, 0xffffffff, 0x00010006,
598 0x9220, 0xffffffff, 0x00090008,
599 0x9224, 0xffffffff, 0x00070000,
600 0x9228, 0xffffffff, 0x00030002,
601 0x922c, 0xffffffff, 0x00050004,
602 0x9238, 0xffffffff, 0x00010006,
603 0x923c, 0xffffffff, 0x00090008,
604 0x9240, 0xffffffff, 0x00070000,
605 0x9244, 0xffffffff, 0x00030002,
606 0x9248, 0xffffffff, 0x00050004,
607 0x9254, 0xffffffff, 0x00010006,
608 0x9258, 0xffffffff, 0x00090008,
609 0x925c, 0xffffffff, 0x00070000,
610 0x9260, 0xffffffff, 0x00030002,
611 0x9264, 0xffffffff, 0x00050004,
612 0x9270, 0xffffffff, 0x00010006,
613 0x9274, 0xffffffff, 0x00090008,
614 0x9278, 0xffffffff, 0x00070000,
615 0x927c, 0xffffffff, 0x00030002,
616 0x9280, 0xffffffff, 0x00050004,
617 0x928c, 0xffffffff, 0x00010006,
618 0x9290, 0xffffffff, 0x00090008,
619 0x9294, 0xffffffff, 0x00000000,
620 0x929c, 0xffffffff, 0x00000001,
621 0x802c, 0xffffffff, 0xc0000000,
622 0x977c, 0xffffffff, 0x00000100,
623 0x3f80, 0xffffffff, 0x00000100,
624 0xa210, 0xffffffff, 0x00000100,
625 0xa214, 0xffffffff, 0x00000100,
626 0x4d8, 0xffffffff, 0x00000100,
627 0x9784, 0xffffffff, 0x00000100,
628 0x9698, 0xffffffff, 0x00000100,
629 0x4d4, 0xffffffff, 0x00000200,
630 0x30cc, 0xffffffff, 0x00000100,
631 0x802c, 0xffffffff, 0xc0000000
632 };
633
634 static const u32 supersumo_golden_registers[] =
635 {
636 0x5eb4, 0xffffffff, 0x00000002,
637 0x5c4, 0xffffffff, 0x00000001,
638 0x7030, 0xffffffff, 0x00000011,
639 0x7c30, 0xffffffff, 0x00000011,
640 0x6104, 0x01000300, 0x00000000,
641 0x5bc0, 0x00300000, 0x00000000,
642 0x8c04, 0xffffffff, 0x40600060,
643 0x8c08, 0xffffffff, 0x001c001c,
644 0x8c20, 0xffffffff, 0x00800080,
645 0x8c24, 0xffffffff, 0x00800080,
646 0x8c18, 0xffffffff, 0x20202078,
647 0x8c1c, 0xffffffff, 0x00001010,
648 0x918c, 0xffffffff, 0x00010006,
649 0x91a8, 0xffffffff, 0x00010006,
650 0x91c4, 0xffffffff, 0x00010006,
651 0x91e0, 0xffffffff, 0x00010006,
652 0x9200, 0xffffffff, 0x00010006,
653 0x9150, 0xffffffff, 0x6e944040,
654 0x917c, 0xffffffff, 0x00030002,
655 0x9180, 0xffffffff, 0x00050004,
656 0x9198, 0xffffffff, 0x00030002,
657 0x919c, 0xffffffff, 0x00050004,
658 0x91b4, 0xffffffff, 0x00030002,
659 0x91b8, 0xffffffff, 0x00050004,
660 0x91d0, 0xffffffff, 0x00030002,
661 0x91d4, 0xffffffff, 0x00050004,
662 0x91f0, 0xffffffff, 0x00030002,
663 0x91f4, 0xffffffff, 0x00050004,
664 0x915c, 0xffffffff, 0x00010000,
665 0x9160, 0xffffffff, 0x00030002,
666 0x3f90, 0xffff0000, 0xff000000,
667 0x9178, 0xffffffff, 0x00070000,
668 0x9194, 0xffffffff, 0x00070000,
669 0x91b0, 0xffffffff, 0x00070000,
670 0x91cc, 0xffffffff, 0x00070000,
671 0x91ec, 0xffffffff, 0x00070000,
672 0x9148, 0xffff0000, 0xff000000,
673 0x9190, 0xffffffff, 0x00090008,
674 0x91ac, 0xffffffff, 0x00090008,
675 0x91c8, 0xffffffff, 0x00090008,
676 0x91e4, 0xffffffff, 0x00090008,
677 0x9204, 0xffffffff, 0x00090008,
678 0x3f94, 0xffff0000, 0xff000000,
679 0x914c, 0xffff0000, 0xff000000,
680 0x929c, 0xffffffff, 0x00000001,
681 0x8a18, 0xffffffff, 0x00000100,
682 0x8b28, 0xffffffff, 0x00000100,
683 0x9144, 0xffffffff, 0x00000100,
684 0x5644, 0xffffffff, 0x00000100,
685 0x9b7c, 0xffffffff, 0x00000000,
686 0x8030, 0xffffffff, 0x0000100a,
687 0x8a14, 0xffffffff, 0x00000007,
688 0x8b24, 0xffffffff, 0x00ff0fff,
689 0x8b10, 0xffffffff, 0x00000000,
690 0x28a4c, 0x06000000, 0x06000000,
691 0x4d8, 0xffffffff, 0x00000100,
692 0x913c, 0xffff000f, 0x0100000a,
693 0x960c, 0xffffffff, 0x54763210,
694 0x88c4, 0xffffffff, 0x000000c2,
695 0x88d4, 0xffffffff, 0x00000010,
696 0x8974, 0xffffffff, 0x00000000,
697 0xc78, 0x00000080, 0x00000080,
698 0x5e78, 0xffffffff, 0x001000f0,
699 0xd02c, 0xffffffff, 0x08421000,
700 0xa008, 0xffffffff, 0x00010000,
701 0x8d00, 0xffffffff, 0x100e4848,
702 0x8d04, 0xffffffff, 0x00164745,
703 0x8c00, 0xffffffff, 0xe4000003,
704 0x8cf0, 0x1fffffff, 0x08e00620,
705 0x28350, 0xffffffff, 0x00000000,
706 0x9508, 0xffffffff, 0x00000002
707 };
708
709 static const u32 sumo_golden_registers[] =
710 {
711 0x900c, 0x00ffffff, 0x0017071f,
712 0x8c18, 0xffffffff, 0x10101060,
713 0x8c1c, 0xffffffff, 0x00001010,
714 0x8c30, 0x0000000f, 0x00000005,
715 0x9688, 0x0000000f, 0x00000007
716 };
717
718 static const u32 wrestler_golden_registers[] =
719 {
720 0x5eb4, 0xffffffff, 0x00000002,
721 0x5c4, 0xffffffff, 0x00000001,
722 0x7030, 0xffffffff, 0x00000011,
723 0x7c30, 0xffffffff, 0x00000011,
724 0x6104, 0x01000300, 0x00000000,
725 0x5bc0, 0x00300000, 0x00000000,
726 0x918c, 0xffffffff, 0x00010006,
727 0x91a8, 0xffffffff, 0x00010006,
728 0x9150, 0xffffffff, 0x6e944040,
729 0x917c, 0xffffffff, 0x00030002,
730 0x9198, 0xffffffff, 0x00030002,
731 0x915c, 0xffffffff, 0x00010000,
732 0x3f90, 0xffff0000, 0xff000000,
733 0x9178, 0xffffffff, 0x00070000,
734 0x9194, 0xffffffff, 0x00070000,
735 0x9148, 0xffff0000, 0xff000000,
736 0x9190, 0xffffffff, 0x00090008,
737 0x91ac, 0xffffffff, 0x00090008,
738 0x3f94, 0xffff0000, 0xff000000,
739 0x914c, 0xffff0000, 0xff000000,
740 0x929c, 0xffffffff, 0x00000001,
741 0x8a18, 0xffffffff, 0x00000100,
742 0x8b28, 0xffffffff, 0x00000100,
743 0x9144, 0xffffffff, 0x00000100,
744 0x9b7c, 0xffffffff, 0x00000000,
745 0x8030, 0xffffffff, 0x0000100a,
746 0x8a14, 0xffffffff, 0x00000001,
747 0x8b24, 0xffffffff, 0x00ff0fff,
748 0x8b10, 0xffffffff, 0x00000000,
749 0x28a4c, 0x06000000, 0x06000000,
750 0x4d8, 0xffffffff, 0x00000100,
751 0x913c, 0xffff000f, 0x0100000a,
752 0x960c, 0xffffffff, 0x54763210,
753 0x88c4, 0xffffffff, 0x000000c2,
754 0x88d4, 0xffffffff, 0x00000010,
755 0x8974, 0xffffffff, 0x00000000,
756 0xc78, 0x00000080, 0x00000080,
757 0x5e78, 0xffffffff, 0x001000f0,
758 0xd02c, 0xffffffff, 0x08421000,
759 0xa008, 0xffffffff, 0x00010000,
760 0x8d00, 0xffffffff, 0x100e4848,
761 0x8d04, 0xffffffff, 0x00164745,
762 0x8c00, 0xffffffff, 0xe4000003,
763 0x8cf0, 0x1fffffff, 0x08e00410,
764 0x28350, 0xffffffff, 0x00000000,
765 0x9508, 0xffffffff, 0x00000002,
766 0x900c, 0xffffffff, 0x0017071f,
767 0x8c18, 0xffffffff, 0x10101060,
768 0x8c1c, 0xffffffff, 0x00001010
769 };
770
771 static const u32 barts_golden_registers[] =
772 {
773 0x5eb4, 0xffffffff, 0x00000002,
774 0x5e78, 0x8f311ff1, 0x001000f0,
775 0x3f90, 0xffff0000, 0xff000000,
776 0x9148, 0xffff0000, 0xff000000,
777 0x3f94, 0xffff0000, 0xff000000,
778 0x914c, 0xffff0000, 0xff000000,
779 0xc78, 0x00000080, 0x00000080,
780 0xbd4, 0x70073777, 0x00010001,
781 0xd02c, 0xbfffff1f, 0x08421000,
782 0xd0b8, 0x03773777, 0x02011003,
783 0x5bc0, 0x00200000, 0x50100000,
784 0x98f8, 0x33773777, 0x02011003,
785 0x98fc, 0xffffffff, 0x76543210,
786 0x7030, 0x31000311, 0x00000011,
787 0x2f48, 0x00000007, 0x02011003,
788 0x6b28, 0x00000010, 0x00000012,
789 0x7728, 0x00000010, 0x00000012,
790 0x10328, 0x00000010, 0x00000012,
791 0x10f28, 0x00000010, 0x00000012,
792 0x11b28, 0x00000010, 0x00000012,
793 0x12728, 0x00000010, 0x00000012,
794 0x240c, 0x000007ff, 0x00000380,
795 0x8a14, 0xf000001f, 0x00000007,
796 0x8b24, 0x3fff3fff, 0x00ff0fff,
797 0x8b10, 0x0000ff0f, 0x00000000,
798 0x28a4c, 0x07ffffff, 0x06000000,
799 0x10c, 0x00000001, 0x00010003,
800 0xa02c, 0xffffffff, 0x0000009b,
801 0x913c, 0x0000000f, 0x0100000a,
802 0x8d00, 0xffff7f7f, 0x100e4848,
803 0x8d04, 0x00ffffff, 0x00164745,
804 0x8c00, 0xfffc0003, 0xe4000003,
805 0x8c04, 0xf8ff00ff, 0x40600060,
806 0x8c08, 0x00ff00ff, 0x001c001c,
807 0x8cf0, 0x1fff1fff, 0x08e00620,
808 0x8c20, 0x0fff0fff, 0x00800080,
809 0x8c24, 0x0fff0fff, 0x00800080,
810 0x8c18, 0xffffffff, 0x20202078,
811 0x8c1c, 0x0000ffff, 0x00001010,
812 0x28350, 0x00000f01, 0x00000000,
813 0x9508, 0x3700001f, 0x00000002,
814 0x960c, 0xffffffff, 0x54763210,
815 0x88c4, 0x001f3ae3, 0x000000c2,
816 0x88d4, 0x0000001f, 0x00000010,
817 0x8974, 0xffffffff, 0x00000000
818 };
819
820 static const u32 turks_golden_registers[] =
821 {
822 0x5eb4, 0xffffffff, 0x00000002,
823 0x5e78, 0x8f311ff1, 0x001000f0,
824 0x8c8, 0x00003000, 0x00001070,
825 0x8cc, 0x000fffff, 0x00040035,
826 0x3f90, 0xffff0000, 0xfff00000,
827 0x9148, 0xffff0000, 0xfff00000,
828 0x3f94, 0xffff0000, 0xfff00000,
829 0x914c, 0xffff0000, 0xfff00000,
830 0xc78, 0x00000080, 0x00000080,
831 0xbd4, 0x00073007, 0x00010002,
832 0xd02c, 0xbfffff1f, 0x08421000,
833 0xd0b8, 0x03773777, 0x02010002,
834 0x5bc0, 0x00200000, 0x50100000,
835 0x98f8, 0x33773777, 0x00010002,
836 0x98fc, 0xffffffff, 0x33221100,
837 0x7030, 0x31000311, 0x00000011,
838 0x2f48, 0x33773777, 0x00010002,
839 0x6b28, 0x00000010, 0x00000012,
840 0x7728, 0x00000010, 0x00000012,
841 0x10328, 0x00000010, 0x00000012,
842 0x10f28, 0x00000010, 0x00000012,
843 0x11b28, 0x00000010, 0x00000012,
844 0x12728, 0x00000010, 0x00000012,
845 0x240c, 0x000007ff, 0x00000380,
846 0x8a14, 0xf000001f, 0x00000007,
847 0x8b24, 0x3fff3fff, 0x00ff0fff,
848 0x8b10, 0x0000ff0f, 0x00000000,
849 0x28a4c, 0x07ffffff, 0x06000000,
850 0x10c, 0x00000001, 0x00010003,
851 0xa02c, 0xffffffff, 0x0000009b,
852 0x913c, 0x0000000f, 0x0100000a,
853 0x8d00, 0xffff7f7f, 0x100e4848,
854 0x8d04, 0x00ffffff, 0x00164745,
855 0x8c00, 0xfffc0003, 0xe4000003,
856 0x8c04, 0xf8ff00ff, 0x40600060,
857 0x8c08, 0x00ff00ff, 0x001c001c,
858 0x8cf0, 0x1fff1fff, 0x08e00410,
859 0x8c20, 0x0fff0fff, 0x00800080,
860 0x8c24, 0x0fff0fff, 0x00800080,
861 0x8c18, 0xffffffff, 0x20202078,
862 0x8c1c, 0x0000ffff, 0x00001010,
863 0x28350, 0x00000f01, 0x00000000,
864 0x9508, 0x3700001f, 0x00000002,
865 0x960c, 0xffffffff, 0x54763210,
866 0x88c4, 0x001f3ae3, 0x000000c2,
867 0x88d4, 0x0000001f, 0x00000010,
868 0x8974, 0xffffffff, 0x00000000
869 };
870
871 static const u32 caicos_golden_registers[] =
872 {
873 0x5eb4, 0xffffffff, 0x00000002,
874 0x5e78, 0x8f311ff1, 0x001000f0,
875 0x8c8, 0x00003420, 0x00001450,
876 0x8cc, 0x000fffff, 0x00040035,
877 0x3f90, 0xffff0000, 0xfffc0000,
878 0x9148, 0xffff0000, 0xfffc0000,
879 0x3f94, 0xffff0000, 0xfffc0000,
880 0x914c, 0xffff0000, 0xfffc0000,
881 0xc78, 0x00000080, 0x00000080,
882 0xbd4, 0x00073007, 0x00010001,
883 0xd02c, 0xbfffff1f, 0x08421000,
884 0xd0b8, 0x03773777, 0x02010001,
885 0x5bc0, 0x00200000, 0x50100000,
886 0x98f8, 0x33773777, 0x02010001,
887 0x98fc, 0xffffffff, 0x33221100,
888 0x7030, 0x31000311, 0x00000011,
889 0x2f48, 0x33773777, 0x02010001,
890 0x6b28, 0x00000010, 0x00000012,
891 0x7728, 0x00000010, 0x00000012,
892 0x10328, 0x00000010, 0x00000012,
893 0x10f28, 0x00000010, 0x00000012,
894 0x11b28, 0x00000010, 0x00000012,
895 0x12728, 0x00000010, 0x00000012,
896 0x240c, 0x000007ff, 0x00000380,
897 0x8a14, 0xf000001f, 0x00000001,
898 0x8b24, 0x3fff3fff, 0x00ff0fff,
899 0x8b10, 0x0000ff0f, 0x00000000,
900 0x28a4c, 0x07ffffff, 0x06000000,
901 0x10c, 0x00000001, 0x00010003,
902 0xa02c, 0xffffffff, 0x0000009b,
903 0x913c, 0x0000000f, 0x0100000a,
904 0x8d00, 0xffff7f7f, 0x100e4848,
905 0x8d04, 0x00ffffff, 0x00164745,
906 0x8c00, 0xfffc0003, 0xe4000003,
907 0x8c04, 0xf8ff00ff, 0x40600060,
908 0x8c08, 0x00ff00ff, 0x001c001c,
909 0x8cf0, 0x1fff1fff, 0x08e00410,
910 0x8c20, 0x0fff0fff, 0x00800080,
911 0x8c24, 0x0fff0fff, 0x00800080,
912 0x8c18, 0xffffffff, 0x20202078,
913 0x8c1c, 0x0000ffff, 0x00001010,
914 0x28350, 0x00000f01, 0x00000000,
915 0x9508, 0x3700001f, 0x00000002,
916 0x960c, 0xffffffff, 0x54763210,
917 0x88c4, 0x001f3ae3, 0x000000c2,
918 0x88d4, 0x0000001f, 0x00000010,
919 0x8974, 0xffffffff, 0x00000000
920 };
921
922 static void evergreen_init_golden_registers(struct radeon_device *rdev)
923 {
924 switch (rdev->family) {
925 case CHIP_CYPRESS:
926 case CHIP_HEMLOCK:
927 radeon_program_register_sequence(rdev,
928 evergreen_golden_registers,
929 (const u32)ARRAY_SIZE(evergreen_golden_registers));
930 radeon_program_register_sequence(rdev,
931 evergreen_golden_registers2,
932 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
933 radeon_program_register_sequence(rdev,
934 cypress_mgcg_init,
935 (const u32)ARRAY_SIZE(cypress_mgcg_init));
936 break;
937 case CHIP_JUNIPER:
938 radeon_program_register_sequence(rdev,
939 evergreen_golden_registers,
940 (const u32)ARRAY_SIZE(evergreen_golden_registers));
941 radeon_program_register_sequence(rdev,
942 evergreen_golden_registers2,
943 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
944 radeon_program_register_sequence(rdev,
945 juniper_mgcg_init,
946 (const u32)ARRAY_SIZE(juniper_mgcg_init));
947 break;
948 case CHIP_REDWOOD:
949 radeon_program_register_sequence(rdev,
950 evergreen_golden_registers,
951 (const u32)ARRAY_SIZE(evergreen_golden_registers));
952 radeon_program_register_sequence(rdev,
953 evergreen_golden_registers2,
954 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
955 radeon_program_register_sequence(rdev,
956 redwood_mgcg_init,
957 (const u32)ARRAY_SIZE(redwood_mgcg_init));
958 break;
959 case CHIP_CEDAR:
960 radeon_program_register_sequence(rdev,
961 cedar_golden_registers,
962 (const u32)ARRAY_SIZE(cedar_golden_registers));
963 radeon_program_register_sequence(rdev,
964 evergreen_golden_registers2,
965 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
966 radeon_program_register_sequence(rdev,
967 cedar_mgcg_init,
968 (const u32)ARRAY_SIZE(cedar_mgcg_init));
969 break;
970 case CHIP_PALM:
971 radeon_program_register_sequence(rdev,
972 wrestler_golden_registers,
973 (const u32)ARRAY_SIZE(wrestler_golden_registers));
974 break;
975 case CHIP_SUMO:
976 radeon_program_register_sequence(rdev,
977 supersumo_golden_registers,
978 (const u32)ARRAY_SIZE(supersumo_golden_registers));
979 break;
980 case CHIP_SUMO2:
981 radeon_program_register_sequence(rdev,
982 supersumo_golden_registers,
983 (const u32)ARRAY_SIZE(supersumo_golden_registers));
984 radeon_program_register_sequence(rdev,
985 sumo_golden_registers,
986 (const u32)ARRAY_SIZE(sumo_golden_registers));
987 break;
988 case CHIP_BARTS:
989 radeon_program_register_sequence(rdev,
990 barts_golden_registers,
991 (const u32)ARRAY_SIZE(barts_golden_registers));
992 break;
993 case CHIP_TURKS:
994 radeon_program_register_sequence(rdev,
995 turks_golden_registers,
996 (const u32)ARRAY_SIZE(turks_golden_registers));
997 break;
998 case CHIP_CAICOS:
999 radeon_program_register_sequence(rdev,
1000 caicos_golden_registers,
1001 (const u32)ARRAY_SIZE(caicos_golden_registers));
1002 break;
1003 default:
1004 break;
1005 }
1006 }
1007
1008 void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1009 unsigned *bankh, unsigned *mtaspect,
1010 unsigned *tile_split)
1011 {
1012 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1013 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1014 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1015 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1016 switch (*bankw) {
1017 default:
1018 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1019 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1020 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1021 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1022 }
1023 switch (*bankh) {
1024 default:
1025 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1026 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1027 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1028 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1029 }
1030 switch (*mtaspect) {
1031 default:
1032 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1033 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1034 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1035 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1036 }
1037 }
1038
1039 static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1040 u32 cntl_reg, u32 status_reg)
1041 {
1042 int r, i;
1043 struct atom_clock_dividers dividers;
1044
1045 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1046 clock, false, &dividers);
1047 if (r)
1048 return r;
1049
1050 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1051
1052 for (i = 0; i < 100; i++) {
1053 if (RREG32(status_reg) & DCLK_STATUS)
1054 break;
1055 mdelay(10);
1056 }
1057 if (i == 100)
1058 return -ETIMEDOUT;
1059
1060 return 0;
1061 }
1062
1063 int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1064 {
1065 int r = 0;
1066 u32 cg_scratch = RREG32(CG_SCRATCH1);
1067
1068 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1069 if (r)
1070 goto done;
1071 cg_scratch &= 0xffff0000;
1072 cg_scratch |= vclk / 100; /* Mhz */
1073
1074 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1075 if (r)
1076 goto done;
1077 cg_scratch &= 0x0000ffff;
1078 cg_scratch |= (dclk / 100) << 16; /* Mhz */
1079
1080 done:
1081 WREG32(CG_SCRATCH1, cg_scratch);
1082
1083 return r;
1084 }
1085
1086 int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1087 {
1088 /* start off with something large */
1089 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1090 int r;
1091
1092 /* bypass vclk and dclk with bclk */
1093 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1094 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1095 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1096
1097 /* put PLL in bypass mode */
1098 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1099
1100 if (!vclk || !dclk) {
1101 /* keep the Bypass mode, put PLL to sleep */
1102 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1103 return 0;
1104 }
1105
1106 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1107 16384, 0x03FFFFFF, 0, 128, 5,
1108 &fb_div, &vclk_div, &dclk_div);
1109 if (r)
1110 return r;
1111
1112 /* set VCO_MODE to 1 */
1113 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1114
1115 /* toggle UPLL_SLEEP to 1 then back to 0 */
1116 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1117 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1118
1119 /* deassert UPLL_RESET */
1120 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1121
1122 mdelay(1);
1123
1124 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1125 if (r)
1126 return r;
1127
1128 /* assert UPLL_RESET again */
1129 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1130
1131 /* disable spread spectrum. */
1132 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1133
1134 /* set feedback divider */
1135 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1136
1137 /* set ref divider to 0 */
1138 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1139
1140 if (fb_div < 307200)
1141 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1142 else
1143 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1144
1145 /* set PDIV_A and PDIV_B */
1146 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1147 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1148 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1149
1150 /* give the PLL some time to settle */
1151 mdelay(15);
1152
1153 /* deassert PLL_RESET */
1154 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1155
1156 mdelay(15);
1157
1158 /* switch from bypass mode to normal mode */
1159 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1160
1161 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1162 if (r)
1163 return r;
1164
1165 /* switch VCLK and DCLK selection */
1166 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1167 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1168 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1169
1170 mdelay(100);
1171
1172 return 0;
1173 }
1174
1175 void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1176 {
1177 u16 ctl, v;
1178 int err;
1179
1180 err = pcie_capability_read_word(rdev->pdev, PCI_EXP_DEVCTL, &ctl);
1181 if (err)
1182 return;
1183
1184 v = (ctl & PCI_EXP_DEVCTL_READRQ) >> 12;
1185
1186 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1187 * to avoid hangs or perfomance issues
1188 */
1189 if ((v == 0) || (v == 6) || (v == 7)) {
1190 ctl &= ~PCI_EXP_DEVCTL_READRQ;
1191 ctl |= (2 << 12);
1192 pcie_capability_write_word(rdev->pdev, PCI_EXP_DEVCTL, ctl);
1193 }
1194 }
1195
1196 static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1197 {
1198 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1199 return true;
1200 else
1201 return false;
1202 }
1203
1204 static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1205 {
1206 u32 pos1, pos2;
1207
1208 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1209 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1210
1211 if (pos1 != pos2)
1212 return true;
1213 else
1214 return false;
1215 }
1216
1217 /**
1218 * dce4_wait_for_vblank - vblank wait asic callback.
1219 *
1220 * @rdev: radeon_device pointer
1221 * @crtc: crtc to wait for vblank on
1222 *
1223 * Wait for vblank on the requested crtc (evergreen+).
1224 */
1225 void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1226 {
1227 unsigned i = 0;
1228
1229 if (crtc >= rdev->num_crtc)
1230 return;
1231
1232 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1233 return;
1234
1235 /* depending on when we hit vblank, we may be close to active; if so,
1236 * wait for another frame.
1237 */
1238 while (dce4_is_in_vblank(rdev, crtc)) {
1239 if (i++ % 100 == 0) {
1240 if (!dce4_is_counter_moving(rdev, crtc))
1241 break;
1242 }
1243 }
1244
1245 while (!dce4_is_in_vblank(rdev, crtc)) {
1246 if (i++ % 100 == 0) {
1247 if (!dce4_is_counter_moving(rdev, crtc))
1248 break;
1249 }
1250 }
1251 }
1252
1253 /**
1254 * radeon_irq_kms_pflip_irq_get - pre-pageflip callback.
1255 *
1256 * @rdev: radeon_device pointer
1257 * @crtc: crtc to prepare for pageflip on
1258 *
1259 * Pre-pageflip callback (evergreen+).
1260 * Enables the pageflip irq (vblank irq).
1261 */
1262 void evergreen_pre_page_flip(struct radeon_device *rdev, int crtc)
1263 {
1264 /* enable the pflip int */
1265 radeon_irq_kms_pflip_irq_get(rdev, crtc);
1266 }
1267
1268 /**
1269 * evergreen_post_page_flip - pos-pageflip callback.
1270 *
1271 * @rdev: radeon_device pointer
1272 * @crtc: crtc to cleanup pageflip on
1273 *
1274 * Post-pageflip callback (evergreen+).
1275 * Disables the pageflip irq (vblank irq).
1276 */
1277 void evergreen_post_page_flip(struct radeon_device *rdev, int crtc)
1278 {
1279 /* disable the pflip int */
1280 radeon_irq_kms_pflip_irq_put(rdev, crtc);
1281 }
1282
1283 /**
1284 * evergreen_page_flip - pageflip callback.
1285 *
1286 * @rdev: radeon_device pointer
1287 * @crtc_id: crtc to cleanup pageflip on
1288 * @crtc_base: new address of the crtc (GPU MC address)
1289 *
1290 * Does the actual pageflip (evergreen+).
1291 * During vblank we take the crtc lock and wait for the update_pending
1292 * bit to go high, when it does, we release the lock, and allow the
1293 * double buffered update to take place.
1294 * Returns the current update pending status.
1295 */
1296 u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
1297 {
1298 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1299 u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
1300 int i;
1301
1302 /* Lock the graphics update lock */
1303 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
1304 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1305
1306 /* update the scanout addresses */
1307 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1308 upper_32_bits(crtc_base));
1309 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1310 (u32)crtc_base);
1311
1312 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1313 upper_32_bits(crtc_base));
1314 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1315 (u32)crtc_base);
1316
1317 /* Wait for update_pending to go high. */
1318 for (i = 0; i < rdev->usec_timeout; i++) {
1319 if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING)
1320 break;
1321 udelay(1);
1322 }
1323 DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
1324
1325 /* Unlock the lock, so double-buffering can take place inside vblank */
1326 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
1327 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1328
1329 /* Return current update_pending status: */
1330 return RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING;
1331 }
1332
1333 /* get temperature in millidegrees */
1334 int evergreen_get_temp(struct radeon_device *rdev)
1335 {
1336 u32 temp, toffset;
1337 int actual_temp = 0;
1338
1339 if (rdev->family == CHIP_JUNIPER) {
1340 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1341 TOFFSET_SHIFT;
1342 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1343 TS0_ADC_DOUT_SHIFT;
1344
1345 if (toffset & 0x100)
1346 actual_temp = temp / 2 - (0x200 - toffset);
1347 else
1348 actual_temp = temp / 2 + toffset;
1349
1350 actual_temp = actual_temp * 1000;
1351
1352 } else {
1353 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1354 ASIC_T_SHIFT;
1355
1356 if (temp & 0x400)
1357 actual_temp = -256;
1358 else if (temp & 0x200)
1359 actual_temp = 255;
1360 else if (temp & 0x100) {
1361 actual_temp = temp & 0x1ff;
1362 actual_temp |= ~0x1ff;
1363 } else
1364 actual_temp = temp & 0xff;
1365
1366 actual_temp = (actual_temp * 1000) / 2;
1367 }
1368
1369 return actual_temp;
1370 }
1371
1372 int sumo_get_temp(struct radeon_device *rdev)
1373 {
1374 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
1375 int actual_temp = temp - 49;
1376
1377 return actual_temp * 1000;
1378 }
1379
1380 /**
1381 * sumo_pm_init_profile - Initialize power profiles callback.
1382 *
1383 * @rdev: radeon_device pointer
1384 *
1385 * Initialize the power states used in profile mode
1386 * (sumo, trinity, SI).
1387 * Used for profile mode only.
1388 */
1389 void sumo_pm_init_profile(struct radeon_device *rdev)
1390 {
1391 int idx;
1392
1393 /* default */
1394 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1395 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1396 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1397 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1398
1399 /* low,mid sh/mh */
1400 if (rdev->flags & RADEON_IS_MOBILITY)
1401 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1402 else
1403 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1404
1405 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1406 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1407 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1408 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1409
1410 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1411 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1412 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1413 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1414
1415 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1416 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1417 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1418 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1419
1420 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1421 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1422 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1423 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1424
1425 /* high sh/mh */
1426 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1427 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1428 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1429 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1430 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1431 rdev->pm.power_state[idx].num_clock_modes - 1;
1432
1433 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1434 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1435 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1436 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1437 rdev->pm.power_state[idx].num_clock_modes - 1;
1438 }
1439
1440 /**
1441 * btc_pm_init_profile - Initialize power profiles callback.
1442 *
1443 * @rdev: radeon_device pointer
1444 *
1445 * Initialize the power states used in profile mode
1446 * (BTC, cayman).
1447 * Used for profile mode only.
1448 */
1449 void btc_pm_init_profile(struct radeon_device *rdev)
1450 {
1451 int idx;
1452
1453 /* default */
1454 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1455 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1456 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1457 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1458 /* starting with BTC, there is one state that is used for both
1459 * MH and SH. Difference is that we always use the high clock index for
1460 * mclk.
1461 */
1462 if (rdev->flags & RADEON_IS_MOBILITY)
1463 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1464 else
1465 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1466 /* low sh */
1467 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1468 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1469 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1470 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1471 /* mid sh */
1472 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1473 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1474 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1475 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1476 /* high sh */
1477 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1478 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1479 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1480 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1481 /* low mh */
1482 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1483 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1484 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1485 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1486 /* mid mh */
1487 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1488 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1489 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1490 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1491 /* high mh */
1492 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1493 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1494 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1495 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1496 }
1497
1498 /**
1499 * evergreen_pm_misc - set additional pm hw parameters callback.
1500 *
1501 * @rdev: radeon_device pointer
1502 *
1503 * Set non-clock parameters associated with a power state
1504 * (voltage, etc.) (evergreen+).
1505 */
1506 void evergreen_pm_misc(struct radeon_device *rdev)
1507 {
1508 int req_ps_idx = rdev->pm.requested_power_state_index;
1509 int req_cm_idx = rdev->pm.requested_clock_mode_index;
1510 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1511 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
1512
1513 if (voltage->type == VOLTAGE_SW) {
1514 /* 0xff0x are flags rather then an actual voltage */
1515 if ((voltage->voltage & 0xff00) == 0xff00)
1516 return;
1517 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
1518 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
1519 rdev->pm.current_vddc = voltage->voltage;
1520 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1521 }
1522
1523 /* starting with BTC, there is one state that is used for both
1524 * MH and SH. Difference is that we always use the high clock index for
1525 * mclk and vddci.
1526 */
1527 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1528 (rdev->family >= CHIP_BARTS) &&
1529 rdev->pm.active_crtc_count &&
1530 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1531 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1532 voltage = &rdev->pm.power_state[req_ps_idx].
1533 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1534
1535 /* 0xff0x are flags rather then an actual voltage */
1536 if ((voltage->vddci & 0xff00) == 0xff00)
1537 return;
1538 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1539 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1540 rdev->pm.current_vddci = voltage->vddci;
1541 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
1542 }
1543 }
1544 }
1545
1546 /**
1547 * evergreen_pm_prepare - pre-power state change callback.
1548 *
1549 * @rdev: radeon_device pointer
1550 *
1551 * Prepare for a power state change (evergreen+).
1552 */
1553 void evergreen_pm_prepare(struct radeon_device *rdev)
1554 {
1555 struct drm_device *ddev = rdev->ddev;
1556 struct drm_crtc *crtc;
1557 struct radeon_crtc *radeon_crtc;
1558 u32 tmp;
1559
1560 /* disable any active CRTCs */
1561 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1562 radeon_crtc = to_radeon_crtc(crtc);
1563 if (radeon_crtc->enabled) {
1564 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1565 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1566 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1567 }
1568 }
1569 }
1570
1571 /**
1572 * evergreen_pm_finish - post-power state change callback.
1573 *
1574 * @rdev: radeon_device pointer
1575 *
1576 * Clean up after a power state change (evergreen+).
1577 */
1578 void evergreen_pm_finish(struct radeon_device *rdev)
1579 {
1580 struct drm_device *ddev = rdev->ddev;
1581 struct drm_crtc *crtc;
1582 struct radeon_crtc *radeon_crtc;
1583 u32 tmp;
1584
1585 /* enable any active CRTCs */
1586 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1587 radeon_crtc = to_radeon_crtc(crtc);
1588 if (radeon_crtc->enabled) {
1589 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1590 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1591 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1592 }
1593 }
1594 }
1595
1596 /**
1597 * evergreen_hpd_sense - hpd sense callback.
1598 *
1599 * @rdev: radeon_device pointer
1600 * @hpd: hpd (hotplug detect) pin
1601 *
1602 * Checks if a digital monitor is connected (evergreen+).
1603 * Returns true if connected, false if not connected.
1604 */
1605 bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1606 {
1607 bool connected = false;
1608
1609 switch (hpd) {
1610 case RADEON_HPD_1:
1611 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
1612 connected = true;
1613 break;
1614 case RADEON_HPD_2:
1615 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
1616 connected = true;
1617 break;
1618 case RADEON_HPD_3:
1619 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
1620 connected = true;
1621 break;
1622 case RADEON_HPD_4:
1623 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
1624 connected = true;
1625 break;
1626 case RADEON_HPD_5:
1627 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
1628 connected = true;
1629 break;
1630 case RADEON_HPD_6:
1631 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
1632 connected = true;
1633 break;
1634 default:
1635 break;
1636 }
1637
1638 return connected;
1639 }
1640
1641 /**
1642 * evergreen_hpd_set_polarity - hpd set polarity callback.
1643 *
1644 * @rdev: radeon_device pointer
1645 * @hpd: hpd (hotplug detect) pin
1646 *
1647 * Set the polarity of the hpd pin (evergreen+).
1648 */
1649 void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1650 enum radeon_hpd_id hpd)
1651 {
1652 u32 tmp;
1653 bool connected = evergreen_hpd_sense(rdev, hpd);
1654
1655 switch (hpd) {
1656 case RADEON_HPD_1:
1657 tmp = RREG32(DC_HPD1_INT_CONTROL);
1658 if (connected)
1659 tmp &= ~DC_HPDx_INT_POLARITY;
1660 else
1661 tmp |= DC_HPDx_INT_POLARITY;
1662 WREG32(DC_HPD1_INT_CONTROL, tmp);
1663 break;
1664 case RADEON_HPD_2:
1665 tmp = RREG32(DC_HPD2_INT_CONTROL);
1666 if (connected)
1667 tmp &= ~DC_HPDx_INT_POLARITY;
1668 else
1669 tmp |= DC_HPDx_INT_POLARITY;
1670 WREG32(DC_HPD2_INT_CONTROL, tmp);
1671 break;
1672 case RADEON_HPD_3:
1673 tmp = RREG32(DC_HPD3_INT_CONTROL);
1674 if (connected)
1675 tmp &= ~DC_HPDx_INT_POLARITY;
1676 else
1677 tmp |= DC_HPDx_INT_POLARITY;
1678 WREG32(DC_HPD3_INT_CONTROL, tmp);
1679 break;
1680 case RADEON_HPD_4:
1681 tmp = RREG32(DC_HPD4_INT_CONTROL);
1682 if (connected)
1683 tmp &= ~DC_HPDx_INT_POLARITY;
1684 else
1685 tmp |= DC_HPDx_INT_POLARITY;
1686 WREG32(DC_HPD4_INT_CONTROL, tmp);
1687 break;
1688 case RADEON_HPD_5:
1689 tmp = RREG32(DC_HPD5_INT_CONTROL);
1690 if (connected)
1691 tmp &= ~DC_HPDx_INT_POLARITY;
1692 else
1693 tmp |= DC_HPDx_INT_POLARITY;
1694 WREG32(DC_HPD5_INT_CONTROL, tmp);
1695 break;
1696 case RADEON_HPD_6:
1697 tmp = RREG32(DC_HPD6_INT_CONTROL);
1698 if (connected)
1699 tmp &= ~DC_HPDx_INT_POLARITY;
1700 else
1701 tmp |= DC_HPDx_INT_POLARITY;
1702 WREG32(DC_HPD6_INT_CONTROL, tmp);
1703 break;
1704 default:
1705 break;
1706 }
1707 }
1708
1709 /**
1710 * evergreen_hpd_init - hpd setup callback.
1711 *
1712 * @rdev: radeon_device pointer
1713 *
1714 * Setup the hpd pins used by the card (evergreen+).
1715 * Enable the pin, set the polarity, and enable the hpd interrupts.
1716 */
1717 void evergreen_hpd_init(struct radeon_device *rdev)
1718 {
1719 struct drm_device *dev = rdev->ddev;
1720 struct drm_connector *connector;
1721 unsigned enabled = 0;
1722 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1723 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
1724
1725 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1726 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1727
1728 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1729 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1730 /* don't try to enable hpd on eDP or LVDS avoid breaking the
1731 * aux dp channel on imac and help (but not completely fix)
1732 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1733 * also avoid interrupt storms during dpms.
1734 */
1735 continue;
1736 }
1737 switch (radeon_connector->hpd.hpd) {
1738 case RADEON_HPD_1:
1739 WREG32(DC_HPD1_CONTROL, tmp);
1740 break;
1741 case RADEON_HPD_2:
1742 WREG32(DC_HPD2_CONTROL, tmp);
1743 break;
1744 case RADEON_HPD_3:
1745 WREG32(DC_HPD3_CONTROL, tmp);
1746 break;
1747 case RADEON_HPD_4:
1748 WREG32(DC_HPD4_CONTROL, tmp);
1749 break;
1750 case RADEON_HPD_5:
1751 WREG32(DC_HPD5_CONTROL, tmp);
1752 break;
1753 case RADEON_HPD_6:
1754 WREG32(DC_HPD6_CONTROL, tmp);
1755 break;
1756 default:
1757 break;
1758 }
1759 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
1760 enabled |= 1 << radeon_connector->hpd.hpd;
1761 }
1762 radeon_irq_kms_enable_hpd(rdev, enabled);
1763 }
1764
1765 /**
1766 * evergreen_hpd_fini - hpd tear down callback.
1767 *
1768 * @rdev: radeon_device pointer
1769 *
1770 * Tear down the hpd pins used by the card (evergreen+).
1771 * Disable the hpd interrupts.
1772 */
1773 void evergreen_hpd_fini(struct radeon_device *rdev)
1774 {
1775 struct drm_device *dev = rdev->ddev;
1776 struct drm_connector *connector;
1777 unsigned disabled = 0;
1778
1779 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1780 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1781 switch (radeon_connector->hpd.hpd) {
1782 case RADEON_HPD_1:
1783 WREG32(DC_HPD1_CONTROL, 0);
1784 break;
1785 case RADEON_HPD_2:
1786 WREG32(DC_HPD2_CONTROL, 0);
1787 break;
1788 case RADEON_HPD_3:
1789 WREG32(DC_HPD3_CONTROL, 0);
1790 break;
1791 case RADEON_HPD_4:
1792 WREG32(DC_HPD4_CONTROL, 0);
1793 break;
1794 case RADEON_HPD_5:
1795 WREG32(DC_HPD5_CONTROL, 0);
1796 break;
1797 case RADEON_HPD_6:
1798 WREG32(DC_HPD6_CONTROL, 0);
1799 break;
1800 default:
1801 break;
1802 }
1803 disabled |= 1 << radeon_connector->hpd.hpd;
1804 }
1805 radeon_irq_kms_disable_hpd(rdev, disabled);
1806 }
1807
1808 /* watermark setup */
1809
1810 static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1811 struct radeon_crtc *radeon_crtc,
1812 struct drm_display_mode *mode,
1813 struct drm_display_mode *other_mode)
1814 {
1815 u32 tmp, buffer_alloc, i;
1816 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1817 /*
1818 * Line Buffer Setup
1819 * There are 3 line buffers, each one shared by 2 display controllers.
1820 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1821 * the display controllers. The paritioning is done via one of four
1822 * preset allocations specified in bits 2:0:
1823 * first display controller
1824 * 0 - first half of lb (3840 * 2)
1825 * 1 - first 3/4 of lb (5760 * 2)
1826 * 2 - whole lb (7680 * 2), other crtc must be disabled
1827 * 3 - first 1/4 of lb (1920 * 2)
1828 * second display controller
1829 * 4 - second half of lb (3840 * 2)
1830 * 5 - second 3/4 of lb (5760 * 2)
1831 * 6 - whole lb (7680 * 2), other crtc must be disabled
1832 * 7 - last 1/4 of lb (1920 * 2)
1833 */
1834 /* this can get tricky if we have two large displays on a paired group
1835 * of crtcs. Ideally for multiple large displays we'd assign them to
1836 * non-linked crtcs for maximum line buffer allocation.
1837 */
1838 if (radeon_crtc->base.enabled && mode) {
1839 if (other_mode) {
1840 tmp = 0; /* 1/2 */
1841 buffer_alloc = 1;
1842 } else {
1843 tmp = 2; /* whole */
1844 buffer_alloc = 2;
1845 }
1846 } else {
1847 tmp = 0;
1848 buffer_alloc = 0;
1849 }
1850
1851 /* second controller of the pair uses second half of the lb */
1852 if (radeon_crtc->crtc_id % 2)
1853 tmp += 4;
1854 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1855
1856 if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1857 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1858 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1859 for (i = 0; i < rdev->usec_timeout; i++) {
1860 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1861 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1862 break;
1863 udelay(1);
1864 }
1865 }
1866
1867 if (radeon_crtc->base.enabled && mode) {
1868 switch (tmp) {
1869 case 0:
1870 case 4:
1871 default:
1872 if (ASIC_IS_DCE5(rdev))
1873 return 4096 * 2;
1874 else
1875 return 3840 * 2;
1876 case 1:
1877 case 5:
1878 if (ASIC_IS_DCE5(rdev))
1879 return 6144 * 2;
1880 else
1881 return 5760 * 2;
1882 case 2:
1883 case 6:
1884 if (ASIC_IS_DCE5(rdev))
1885 return 8192 * 2;
1886 else
1887 return 7680 * 2;
1888 case 3:
1889 case 7:
1890 if (ASIC_IS_DCE5(rdev))
1891 return 2048 * 2;
1892 else
1893 return 1920 * 2;
1894 }
1895 }
1896
1897 /* controller not enabled, so no lb used */
1898 return 0;
1899 }
1900
1901 u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
1902 {
1903 u32 tmp = RREG32(MC_SHARED_CHMAP);
1904
1905 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1906 case 0:
1907 default:
1908 return 1;
1909 case 1:
1910 return 2;
1911 case 2:
1912 return 4;
1913 case 3:
1914 return 8;
1915 }
1916 }
1917
1918 struct evergreen_wm_params {
1919 u32 dram_channels; /* number of dram channels */
1920 u32 yclk; /* bandwidth per dram data pin in kHz */
1921 u32 sclk; /* engine clock in kHz */
1922 u32 disp_clk; /* display clock in kHz */
1923 u32 src_width; /* viewport width */
1924 u32 active_time; /* active display time in ns */
1925 u32 blank_time; /* blank time in ns */
1926 bool interlaced; /* mode is interlaced */
1927 fixed20_12 vsc; /* vertical scale ratio */
1928 u32 num_heads; /* number of active crtcs */
1929 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1930 u32 lb_size; /* line buffer allocated to pipe */
1931 u32 vtaps; /* vertical scaler taps */
1932 };
1933
1934 static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1935 {
1936 /* Calculate DRAM Bandwidth and the part allocated to display. */
1937 fixed20_12 dram_efficiency; /* 0.7 */
1938 fixed20_12 yclk, dram_channels, bandwidth;
1939 fixed20_12 a;
1940
1941 a.full = dfixed_const(1000);
1942 yclk.full = dfixed_const(wm->yclk);
1943 yclk.full = dfixed_div(yclk, a);
1944 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1945 a.full = dfixed_const(10);
1946 dram_efficiency.full = dfixed_const(7);
1947 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1948 bandwidth.full = dfixed_mul(dram_channels, yclk);
1949 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1950
1951 return dfixed_trunc(bandwidth);
1952 }
1953
1954 static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1955 {
1956 /* Calculate DRAM Bandwidth and the part allocated to display. */
1957 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1958 fixed20_12 yclk, dram_channels, bandwidth;
1959 fixed20_12 a;
1960
1961 a.full = dfixed_const(1000);
1962 yclk.full = dfixed_const(wm->yclk);
1963 yclk.full = dfixed_div(yclk, a);
1964 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1965 a.full = dfixed_const(10);
1966 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1967 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1968 bandwidth.full = dfixed_mul(dram_channels, yclk);
1969 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1970
1971 return dfixed_trunc(bandwidth);
1972 }
1973
1974 static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
1975 {
1976 /* Calculate the display Data return Bandwidth */
1977 fixed20_12 return_efficiency; /* 0.8 */
1978 fixed20_12 sclk, bandwidth;
1979 fixed20_12 a;
1980
1981 a.full = dfixed_const(1000);
1982 sclk.full = dfixed_const(wm->sclk);
1983 sclk.full = dfixed_div(sclk, a);
1984 a.full = dfixed_const(10);
1985 return_efficiency.full = dfixed_const(8);
1986 return_efficiency.full = dfixed_div(return_efficiency, a);
1987 a.full = dfixed_const(32);
1988 bandwidth.full = dfixed_mul(a, sclk);
1989 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1990
1991 return dfixed_trunc(bandwidth);
1992 }
1993
1994 static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
1995 {
1996 /* Calculate the DMIF Request Bandwidth */
1997 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1998 fixed20_12 disp_clk, bandwidth;
1999 fixed20_12 a;
2000
2001 a.full = dfixed_const(1000);
2002 disp_clk.full = dfixed_const(wm->disp_clk);
2003 disp_clk.full = dfixed_div(disp_clk, a);
2004 a.full = dfixed_const(10);
2005 disp_clk_request_efficiency.full = dfixed_const(8);
2006 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2007 a.full = dfixed_const(32);
2008 bandwidth.full = dfixed_mul(a, disp_clk);
2009 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2010
2011 return dfixed_trunc(bandwidth);
2012 }
2013
2014 static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2015 {
2016 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2017 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2018 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2019 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2020
2021 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2022 }
2023
2024 static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2025 {
2026 /* Calculate the display mode Average Bandwidth
2027 * DisplayMode should contain the source and destination dimensions,
2028 * timing, etc.
2029 */
2030 fixed20_12 bpp;
2031 fixed20_12 line_time;
2032 fixed20_12 src_width;
2033 fixed20_12 bandwidth;
2034 fixed20_12 a;
2035
2036 a.full = dfixed_const(1000);
2037 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2038 line_time.full = dfixed_div(line_time, a);
2039 bpp.full = dfixed_const(wm->bytes_per_pixel);
2040 src_width.full = dfixed_const(wm->src_width);
2041 bandwidth.full = dfixed_mul(src_width, bpp);
2042 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2043 bandwidth.full = dfixed_div(bandwidth, line_time);
2044
2045 return dfixed_trunc(bandwidth);
2046 }
2047
2048 static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2049 {
2050 /* First calcualte the latency in ns */
2051 u32 mc_latency = 2000; /* 2000 ns. */
2052 u32 available_bandwidth = evergreen_available_bandwidth(wm);
2053 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2054 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2055 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2056 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2057 (wm->num_heads * cursor_line_pair_return_time);
2058 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2059 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2060 fixed20_12 a, b, c;
2061
2062 if (wm->num_heads == 0)
2063 return 0;
2064
2065 a.full = dfixed_const(2);
2066 b.full = dfixed_const(1);
2067 if ((wm->vsc.full > a.full) ||
2068 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2069 (wm->vtaps >= 5) ||
2070 ((wm->vsc.full >= a.full) && wm->interlaced))
2071 max_src_lines_per_dst_line = 4;
2072 else
2073 max_src_lines_per_dst_line = 2;
2074
2075 a.full = dfixed_const(available_bandwidth);
2076 b.full = dfixed_const(wm->num_heads);
2077 a.full = dfixed_div(a, b);
2078
2079 b.full = dfixed_const(1000);
2080 c.full = dfixed_const(wm->disp_clk);
2081 b.full = dfixed_div(c, b);
2082 c.full = dfixed_const(wm->bytes_per_pixel);
2083 b.full = dfixed_mul(b, c);
2084
2085 lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
2086
2087 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2088 b.full = dfixed_const(1000);
2089 c.full = dfixed_const(lb_fill_bw);
2090 b.full = dfixed_div(c, b);
2091 a.full = dfixed_div(a, b);
2092 line_fill_time = dfixed_trunc(a);
2093
2094 if (line_fill_time < wm->active_time)
2095 return latency;
2096 else
2097 return latency + (line_fill_time - wm->active_time);
2098
2099 }
2100
2101 static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2102 {
2103 if (evergreen_average_bandwidth(wm) <=
2104 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2105 return true;
2106 else
2107 return false;
2108 };
2109
2110 static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2111 {
2112 if (evergreen_average_bandwidth(wm) <=
2113 (evergreen_available_bandwidth(wm) / wm->num_heads))
2114 return true;
2115 else
2116 return false;
2117 };
2118
2119 static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2120 {
2121 u32 lb_partitions = wm->lb_size / wm->src_width;
2122 u32 line_time = wm->active_time + wm->blank_time;
2123 u32 latency_tolerant_lines;
2124 u32 latency_hiding;
2125 fixed20_12 a;
2126
2127 a.full = dfixed_const(1);
2128 if (wm->vsc.full > a.full)
2129 latency_tolerant_lines = 1;
2130 else {
2131 if (lb_partitions <= (wm->vtaps + 1))
2132 latency_tolerant_lines = 1;
2133 else
2134 latency_tolerant_lines = 2;
2135 }
2136
2137 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2138
2139 if (evergreen_latency_watermark(wm) <= latency_hiding)
2140 return true;
2141 else
2142 return false;
2143 }
2144
2145 static void evergreen_program_watermarks(struct radeon_device *rdev,
2146 struct radeon_crtc *radeon_crtc,
2147 u32 lb_size, u32 num_heads)
2148 {
2149 struct drm_display_mode *mode = &radeon_crtc->base.mode;
2150 struct evergreen_wm_params wm_low, wm_high;
2151 u32 dram_channels;
2152 u32 pixel_period;
2153 u32 line_time = 0;
2154 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2155 u32 priority_a_mark = 0, priority_b_mark = 0;
2156 u32 priority_a_cnt = PRIORITY_OFF;
2157 u32 priority_b_cnt = PRIORITY_OFF;
2158 u32 pipe_offset = radeon_crtc->crtc_id * 16;
2159 u32 tmp, arb_control3;
2160 fixed20_12 a, b, c;
2161
2162 if (radeon_crtc->base.enabled && num_heads && mode) {
2163 pixel_period = 1000000 / (u32)mode->clock;
2164 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2165 priority_a_cnt = 0;
2166 priority_b_cnt = 0;
2167 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2168
2169 /* watermark for high clocks */
2170 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2171 wm_high.yclk =
2172 radeon_dpm_get_mclk(rdev, false) * 10;
2173 wm_high.sclk =
2174 radeon_dpm_get_sclk(rdev, false) * 10;
2175 } else {
2176 wm_high.yclk = rdev->pm.current_mclk * 10;
2177 wm_high.sclk = rdev->pm.current_sclk * 10;
2178 }
2179
2180 wm_high.disp_clk = mode->clock;
2181 wm_high.src_width = mode->crtc_hdisplay;
2182 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2183 wm_high.blank_time = line_time - wm_high.active_time;
2184 wm_high.interlaced = false;
2185 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2186 wm_high.interlaced = true;
2187 wm_high.vsc = radeon_crtc->vsc;
2188 wm_high.vtaps = 1;
2189 if (radeon_crtc->rmx_type != RMX_OFF)
2190 wm_high.vtaps = 2;
2191 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2192 wm_high.lb_size = lb_size;
2193 wm_high.dram_channels = dram_channels;
2194 wm_high.num_heads = num_heads;
2195
2196 /* watermark for low clocks */
2197 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2198 wm_low.yclk =
2199 radeon_dpm_get_mclk(rdev, true) * 10;
2200 wm_low.sclk =
2201 radeon_dpm_get_sclk(rdev, true) * 10;
2202 } else {
2203 wm_low.yclk = rdev->pm.current_mclk * 10;
2204 wm_low.sclk = rdev->pm.current_sclk * 10;
2205 }
2206
2207 wm_low.disp_clk = mode->clock;
2208 wm_low.src_width = mode->crtc_hdisplay;
2209 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2210 wm_low.blank_time = line_time - wm_low.active_time;
2211 wm_low.interlaced = false;
2212 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2213 wm_low.interlaced = true;
2214 wm_low.vsc = radeon_crtc->vsc;
2215 wm_low.vtaps = 1;
2216 if (radeon_crtc->rmx_type != RMX_OFF)
2217 wm_low.vtaps = 2;
2218 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2219 wm_low.lb_size = lb_size;
2220 wm_low.dram_channels = dram_channels;
2221 wm_low.num_heads = num_heads;
2222
2223 /* set for high clocks */
2224 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
2225 /* set for low clocks */
2226 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
2227
2228 /* possibly force display priority to high */
2229 /* should really do this at mode validation time... */
2230 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2231 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2232 !evergreen_check_latency_hiding(&wm_high) ||
2233 (rdev->disp_priority == 2)) {
2234 DRM_DEBUG_KMS("force priority a to high\n");
2235 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2236 }
2237 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2238 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2239 !evergreen_check_latency_hiding(&wm_low) ||
2240 (rdev->disp_priority == 2)) {
2241 DRM_DEBUG_KMS("force priority b to high\n");
2242 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2243 }
2244
2245 a.full = dfixed_const(1000);
2246 b.full = dfixed_const(mode->clock);
2247 b.full = dfixed_div(b, a);
2248 c.full = dfixed_const(latency_watermark_a);
2249 c.full = dfixed_mul(c, b);
2250 c.full = dfixed_mul(c, radeon_crtc->hsc);
2251 c.full = dfixed_div(c, a);
2252 a.full = dfixed_const(16);
2253 c.full = dfixed_div(c, a);
2254 priority_a_mark = dfixed_trunc(c);
2255 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2256
2257 a.full = dfixed_const(1000);
2258 b.full = dfixed_const(mode->clock);
2259 b.full = dfixed_div(b, a);
2260 c.full = dfixed_const(latency_watermark_b);
2261 c.full = dfixed_mul(c, b);
2262 c.full = dfixed_mul(c, radeon_crtc->hsc);
2263 c.full = dfixed_div(c, a);
2264 a.full = dfixed_const(16);
2265 c.full = dfixed_div(c, a);
2266 priority_b_mark = dfixed_trunc(c);
2267 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2268 }
2269
2270 /* select wm A */
2271 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2272 tmp = arb_control3;
2273 tmp &= ~LATENCY_WATERMARK_MASK(3);
2274 tmp |= LATENCY_WATERMARK_MASK(1);
2275 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2276 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2277 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2278 LATENCY_HIGH_WATERMARK(line_time)));
2279 /* select wm B */
2280 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2281 tmp &= ~LATENCY_WATERMARK_MASK(3);
2282 tmp |= LATENCY_WATERMARK_MASK(2);
2283 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2284 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2285 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2286 LATENCY_HIGH_WATERMARK(line_time)));
2287 /* restore original selection */
2288 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2289
2290 /* write the priority marks */
2291 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2292 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2293
2294 /* save values for DPM */
2295 radeon_crtc->line_time = line_time;
2296 radeon_crtc->wm_high = latency_watermark_a;
2297 radeon_crtc->wm_low = latency_watermark_b;
2298 }
2299
2300 /**
2301 * evergreen_bandwidth_update - update display watermarks callback.
2302 *
2303 * @rdev: radeon_device pointer
2304 *
2305 * Update the display watermarks based on the requested mode(s)
2306 * (evergreen+).
2307 */
2308 void evergreen_bandwidth_update(struct radeon_device *rdev)
2309 {
2310 struct drm_display_mode *mode0 = NULL;
2311 struct drm_display_mode *mode1 = NULL;
2312 u32 num_heads = 0, lb_size;
2313 int i;
2314
2315 radeon_update_display_priority(rdev);
2316
2317 for (i = 0; i < rdev->num_crtc; i++) {
2318 if (rdev->mode_info.crtcs[i]->base.enabled)
2319 num_heads++;
2320 }
2321 for (i = 0; i < rdev->num_crtc; i += 2) {
2322 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2323 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2324 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2325 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2326 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2327 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2328 }
2329 }
2330
2331 /**
2332 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2333 *
2334 * @rdev: radeon_device pointer
2335 *
2336 * Wait for the MC (memory controller) to be idle.
2337 * (evergreen+).
2338 * Returns 0 if the MC is idle, -1 if not.
2339 */
2340 int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
2341 {
2342 unsigned i;
2343 u32 tmp;
2344
2345 for (i = 0; i < rdev->usec_timeout; i++) {
2346 /* read MC_STATUS */
2347 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2348 if (!tmp)
2349 return 0;
2350 udelay(1);
2351 }
2352 return -1;
2353 }
2354
2355 /*
2356 * GART
2357 */
2358 void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2359 {
2360 unsigned i;
2361 u32 tmp;
2362
2363 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2364
2365 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2366 for (i = 0; i < rdev->usec_timeout; i++) {
2367 /* read MC_STATUS */
2368 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2369 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2370 if (tmp == 2) {
2371 printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
2372 return;
2373 }
2374 if (tmp) {
2375 return;
2376 }
2377 udelay(1);
2378 }
2379 }
2380
2381 static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
2382 {
2383 u32 tmp;
2384 int r;
2385
2386 if (rdev->gart.robj == NULL) {
2387 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2388 return -EINVAL;
2389 }
2390 r = radeon_gart_table_vram_pin(rdev);
2391 if (r)
2392 return r;
2393 radeon_gart_restore(rdev);
2394 /* Setup L2 cache */
2395 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2396 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2397 EFFECTIVE_L2_QUEUE_SIZE(7));
2398 WREG32(VM_L2_CNTL2, 0);
2399 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2400 /* Setup TLB control */
2401 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2402 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2403 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2404 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2405 if (rdev->flags & RADEON_IS_IGP) {
2406 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2407 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2408 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2409 } else {
2410 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2411 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2412 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2413 if ((rdev->family == CHIP_JUNIPER) ||
2414 (rdev->family == CHIP_CYPRESS) ||
2415 (rdev->family == CHIP_HEMLOCK) ||
2416 (rdev->family == CHIP_BARTS))
2417 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
2418 }
2419 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2420 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2421 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2422 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2423 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2424 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2425 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2426 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2427 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2428 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2429 (u32)(rdev->dummy_page.addr >> 12));
2430 WREG32(VM_CONTEXT1_CNTL, 0);
2431
2432 evergreen_pcie_gart_tlb_flush(rdev);
2433 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2434 (unsigned)(rdev->mc.gtt_size >> 20),
2435 (unsigned long long)rdev->gart.table_addr);
2436 rdev->gart.ready = true;
2437 return 0;
2438 }
2439
2440 static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
2441 {
2442 u32 tmp;
2443
2444 /* Disable all tables */
2445 WREG32(VM_CONTEXT0_CNTL, 0);
2446 WREG32(VM_CONTEXT1_CNTL, 0);
2447
2448 /* Setup L2 cache */
2449 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2450 EFFECTIVE_L2_QUEUE_SIZE(7));
2451 WREG32(VM_L2_CNTL2, 0);
2452 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2453 /* Setup TLB control */
2454 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2455 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2456 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2457 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2458 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2459 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2460 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2461 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2462 radeon_gart_table_vram_unpin(rdev);
2463 }
2464
2465 static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
2466 {
2467 evergreen_pcie_gart_disable(rdev);
2468 radeon_gart_table_vram_free(rdev);
2469 radeon_gart_fini(rdev);
2470 }
2471
2472
2473 static void evergreen_agp_enable(struct radeon_device *rdev)
2474 {
2475 u32 tmp;
2476
2477 /* Setup L2 cache */
2478 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2479 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2480 EFFECTIVE_L2_QUEUE_SIZE(7));
2481 WREG32(VM_L2_CNTL2, 0);
2482 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2483 /* Setup TLB control */
2484 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2485 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2486 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2487 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2488 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2489 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2490 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2491 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2492 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2493 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2494 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2495 WREG32(VM_CONTEXT0_CNTL, 0);
2496 WREG32(VM_CONTEXT1_CNTL, 0);
2497 }
2498
2499 void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
2500 {
2501 u32 crtc_enabled, tmp, frame_count, blackout;
2502 int i, j;
2503
2504 if (!ASIC_IS_NODCE(rdev)) {
2505 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2506 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
2507
2508 /* disable VGA render */
2509 WREG32(VGA_RENDER_CONTROL, 0);
2510 }
2511 /* blank the display controllers */
2512 for (i = 0; i < rdev->num_crtc; i++) {
2513 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2514 if (crtc_enabled) {
2515 save->crtc_enabled[i] = true;
2516 if (ASIC_IS_DCE6(rdev)) {
2517 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2518 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2519 radeon_wait_for_vblank(rdev, i);
2520 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2521 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2522 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2523 }
2524 } else {
2525 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2526 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2527 radeon_wait_for_vblank(rdev, i);
2528 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2529 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2530 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2531 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2532 }
2533 }
2534 /* wait for the next frame */
2535 frame_count = radeon_get_vblank_counter(rdev, i);
2536 for (j = 0; j < rdev->usec_timeout; j++) {
2537 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2538 break;
2539 udelay(1);
2540 }
2541
2542 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2543 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2544 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2545 tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2546 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2547 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2548 save->crtc_enabled[i] = false;
2549 /* ***** */
2550 } else {
2551 save->crtc_enabled[i] = false;
2552 }
2553 }
2554
2555 radeon_mc_wait_for_idle(rdev);
2556
2557 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2558 if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2559 /* Block CPU access */
2560 WREG32(BIF_FB_EN, 0);
2561 /* blackout the MC */
2562 blackout &= ~BLACKOUT_MODE_MASK;
2563 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
2564 }
2565 /* wait for the MC to settle */
2566 udelay(100);
2567
2568 /* lock double buffered regs */
2569 for (i = 0; i < rdev->num_crtc; i++) {
2570 if (save->crtc_enabled[i]) {
2571 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2572 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2573 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2574 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2575 }
2576 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2577 if (!(tmp & 1)) {
2578 tmp |= 1;
2579 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2580 }
2581 }
2582 }
2583 }
2584
2585 void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
2586 {
2587 u32 tmp, frame_count;
2588 int i, j;
2589
2590 /* update crtc base addresses */
2591 for (i = 0; i < rdev->num_crtc; i++) {
2592 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2593 upper_32_bits(rdev->mc.vram_start));
2594 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2595 upper_32_bits(rdev->mc.vram_start));
2596 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
2597 (u32)rdev->mc.vram_start);
2598 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
2599 (u32)rdev->mc.vram_start);
2600 }
2601
2602 if (!ASIC_IS_NODCE(rdev)) {
2603 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2604 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2605 }
2606
2607 /* unlock regs and wait for update */
2608 for (i = 0; i < rdev->num_crtc; i++) {
2609 if (save->crtc_enabled[i]) {
2610 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2611 if ((tmp & 0x3) != 0) {
2612 tmp &= ~0x3;
2613 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2614 }
2615 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2616 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2617 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2618 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2619 }
2620 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2621 if (tmp & 1) {
2622 tmp &= ~1;
2623 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2624 }
2625 for (j = 0; j < rdev->usec_timeout; j++) {
2626 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2627 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2628 break;
2629 udelay(1);
2630 }
2631 }
2632 }
2633
2634 /* unblackout the MC */
2635 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2636 tmp &= ~BLACKOUT_MODE_MASK;
2637 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2638 /* allow CPU access */
2639 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2640
2641 for (i = 0; i < rdev->num_crtc; i++) {
2642 if (save->crtc_enabled[i]) {
2643 if (ASIC_IS_DCE6(rdev)) {
2644 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2645 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2646 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2647 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2648 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2649 } else {
2650 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2651 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2652 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2653 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2654 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2655 }
2656 /* wait for the next frame */
2657 frame_count = radeon_get_vblank_counter(rdev, i);
2658 for (j = 0; j < rdev->usec_timeout; j++) {
2659 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2660 break;
2661 udelay(1);
2662 }
2663 }
2664 }
2665 if (!ASIC_IS_NODCE(rdev)) {
2666 /* Unlock vga access */
2667 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2668 mdelay(1);
2669 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2670 }
2671 }
2672
2673 void evergreen_mc_program(struct radeon_device *rdev)
2674 {
2675 struct evergreen_mc_save save;
2676 u32 tmp;
2677 int i, j;
2678
2679 /* Initialize HDP */
2680 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2681 WREG32((0x2c14 + j), 0x00000000);
2682 WREG32((0x2c18 + j), 0x00000000);
2683 WREG32((0x2c1c + j), 0x00000000);
2684 WREG32((0x2c20 + j), 0x00000000);
2685 WREG32((0x2c24 + j), 0x00000000);
2686 }
2687 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2688
2689 evergreen_mc_stop(rdev, &save);
2690 if (evergreen_mc_wait_for_idle(rdev)) {
2691 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2692 }
2693 /* Lockout access through VGA aperture*/
2694 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2695 /* Update configuration */
2696 if (rdev->flags & RADEON_IS_AGP) {
2697 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2698 /* VRAM before AGP */
2699 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2700 rdev->mc.vram_start >> 12);
2701 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2702 rdev->mc.gtt_end >> 12);
2703 } else {
2704 /* VRAM after AGP */
2705 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2706 rdev->mc.gtt_start >> 12);
2707 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2708 rdev->mc.vram_end >> 12);
2709 }
2710 } else {
2711 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2712 rdev->mc.vram_start >> 12);
2713 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2714 rdev->mc.vram_end >> 12);
2715 }
2716 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
2717 /* llano/ontario only */
2718 if ((rdev->family == CHIP_PALM) ||
2719 (rdev->family == CHIP_SUMO) ||
2720 (rdev->family == CHIP_SUMO2)) {
2721 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2722 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2723 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2724 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2725 }
2726 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2727 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2728 WREG32(MC_VM_FB_LOCATION, tmp);
2729 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2730 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
2731 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
2732 if (rdev->flags & RADEON_IS_AGP) {
2733 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2734 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2735 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2736 } else {
2737 WREG32(MC_VM_AGP_BASE, 0);
2738 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2739 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2740 }
2741 if (evergreen_mc_wait_for_idle(rdev)) {
2742 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2743 }
2744 evergreen_mc_resume(rdev, &save);
2745 /* we need to own VRAM, so turn off the VGA renderer here
2746 * to stop it overwriting our objects */
2747 rv515_vga_render_disable(rdev);
2748 }
2749
2750 /*
2751 * CP.
2752 */
2753 void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2754 {
2755 struct radeon_ring *ring = &rdev->ring[ib->ring];
2756 u32 next_rptr;
2757
2758 /* set to DX10/11 mode */
2759 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2760 radeon_ring_write(ring, 1);
2761
2762 if (ring->rptr_save_reg) {
2763 next_rptr = ring->wptr + 3 + 4;
2764 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2765 radeon_ring_write(ring, ((ring->rptr_save_reg -
2766 PACKET3_SET_CONFIG_REG_START) >> 2));
2767 radeon_ring_write(ring, next_rptr);
2768 } else if (rdev->wb.enabled) {
2769 next_rptr = ring->wptr + 5 + 4;
2770 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2771 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2772 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2773 radeon_ring_write(ring, next_rptr);
2774 radeon_ring_write(ring, 0);
2775 }
2776
2777 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2778 radeon_ring_write(ring,
2779 #ifdef __BIG_ENDIAN
2780 (2 << 0) |
2781 #endif
2782 (ib->gpu_addr & 0xFFFFFFFC));
2783 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2784 radeon_ring_write(ring, ib->length_dw);
2785 }
2786
2787
2788 static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2789 {
2790 const __be32 *fw_data;
2791 int i;
2792
2793 if (!rdev->me_fw || !rdev->pfp_fw)
2794 return -EINVAL;
2795
2796 r700_cp_stop(rdev);
2797 WREG32(CP_RB_CNTL,
2798 #ifdef __BIG_ENDIAN
2799 BUF_SWAP_32BIT |
2800 #endif
2801 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2802
2803 fw_data = (const __be32 *)rdev->pfp_fw->data;
2804 WREG32(CP_PFP_UCODE_ADDR, 0);
2805 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2806 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2807 WREG32(CP_PFP_UCODE_ADDR, 0);
2808
2809 fw_data = (const __be32 *)rdev->me_fw->data;
2810 WREG32(CP_ME_RAM_WADDR, 0);
2811 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2812 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2813
2814 WREG32(CP_PFP_UCODE_ADDR, 0);
2815 WREG32(CP_ME_RAM_WADDR, 0);
2816 WREG32(CP_ME_RAM_RADDR, 0);
2817 return 0;
2818 }
2819
2820 static int evergreen_cp_start(struct radeon_device *rdev)
2821 {
2822 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2823 int r, i;
2824 uint32_t cp_me;
2825
2826 r = radeon_ring_lock(rdev, ring, 7);
2827 if (r) {
2828 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2829 return r;
2830 }
2831 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2832 radeon_ring_write(ring, 0x1);
2833 radeon_ring_write(ring, 0x0);
2834 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
2835 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2836 radeon_ring_write(ring, 0);
2837 radeon_ring_write(ring, 0);
2838 radeon_ring_unlock_commit(rdev, ring);
2839
2840 cp_me = 0xff;
2841 WREG32(CP_ME_CNTL, cp_me);
2842
2843 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
2844 if (r) {
2845 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2846 return r;
2847 }
2848
2849 /* setup clear context state */
2850 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2851 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2852
2853 for (i = 0; i < evergreen_default_size; i++)
2854 radeon_ring_write(ring, evergreen_default_state[i]);
2855
2856 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2857 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2858
2859 /* set clear context state */
2860 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2861 radeon_ring_write(ring, 0);
2862
2863 /* SQ_VTX_BASE_VTX_LOC */
2864 radeon_ring_write(ring, 0xc0026f00);
2865 radeon_ring_write(ring, 0x00000000);
2866 radeon_ring_write(ring, 0x00000000);
2867 radeon_ring_write(ring, 0x00000000);
2868
2869 /* Clear consts */
2870 radeon_ring_write(ring, 0xc0036f00);
2871 radeon_ring_write(ring, 0x00000bc4);
2872 radeon_ring_write(ring, 0xffffffff);
2873 radeon_ring_write(ring, 0xffffffff);
2874 radeon_ring_write(ring, 0xffffffff);
2875
2876 radeon_ring_write(ring, 0xc0026900);
2877 radeon_ring_write(ring, 0x00000316);
2878 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
2879 radeon_ring_write(ring, 0x00000010); /* */
2880
2881 radeon_ring_unlock_commit(rdev, ring);
2882
2883 return 0;
2884 }
2885
2886 static int evergreen_cp_resume(struct radeon_device *rdev)
2887 {
2888 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2889 u32 tmp;
2890 u32 rb_bufsz;
2891 int r;
2892
2893 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
2894 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
2895 SOFT_RESET_PA |
2896 SOFT_RESET_SH |
2897 SOFT_RESET_VGT |
2898 SOFT_RESET_SPI |
2899 SOFT_RESET_SX));
2900 RREG32(GRBM_SOFT_RESET);
2901 mdelay(15);
2902 WREG32(GRBM_SOFT_RESET, 0);
2903 RREG32(GRBM_SOFT_RESET);
2904
2905 /* Set ring buffer size */
2906 rb_bufsz = order_base_2(ring->ring_size / 8);
2907 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2908 #ifdef __BIG_ENDIAN
2909 tmp |= BUF_SWAP_32BIT;
2910 #endif
2911 WREG32(CP_RB_CNTL, tmp);
2912 WREG32(CP_SEM_WAIT_TIMER, 0x0);
2913 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
2914
2915 /* Set the write pointer delay */
2916 WREG32(CP_RB_WPTR_DELAY, 0);
2917
2918 /* Initialize the ring buffer's read and write pointers */
2919 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
2920 WREG32(CP_RB_RPTR_WR, 0);
2921 ring->wptr = 0;
2922 WREG32(CP_RB_WPTR, ring->wptr);
2923
2924 /* set the wb address whether it's enabled or not */
2925 WREG32(CP_RB_RPTR_ADDR,
2926 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
2927 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2928 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2929
2930 if (rdev->wb.enabled)
2931 WREG32(SCRATCH_UMSK, 0xff);
2932 else {
2933 tmp |= RB_NO_UPDATE;
2934 WREG32(SCRATCH_UMSK, 0);
2935 }
2936
2937 mdelay(1);
2938 WREG32(CP_RB_CNTL, tmp);
2939
2940 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
2941 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
2942
2943 ring->rptr = RREG32(CP_RB_RPTR);
2944
2945 evergreen_cp_start(rdev);
2946 ring->ready = true;
2947 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
2948 if (r) {
2949 ring->ready = false;
2950 return r;
2951 }
2952 return 0;
2953 }
2954
2955 /*
2956 * Core functions
2957 */
2958 static void evergreen_gpu_init(struct radeon_device *rdev)
2959 {
2960 u32 gb_addr_config;
2961 u32 mc_shared_chmap, mc_arb_ramcfg;
2962 u32 sx_debug_1;
2963 u32 smx_dc_ctl0;
2964 u32 sq_config;
2965 u32 sq_lds_resource_mgmt;
2966 u32 sq_gpr_resource_mgmt_1;
2967 u32 sq_gpr_resource_mgmt_2;
2968 u32 sq_gpr_resource_mgmt_3;
2969 u32 sq_thread_resource_mgmt;
2970 u32 sq_thread_resource_mgmt_2;
2971 u32 sq_stack_resource_mgmt_1;
2972 u32 sq_stack_resource_mgmt_2;
2973 u32 sq_stack_resource_mgmt_3;
2974 u32 vgt_cache_invalidation;
2975 u32 hdp_host_path_cntl, tmp;
2976 u32 disabled_rb_mask;
2977 int i, j, num_shader_engines, ps_thread_count;
2978
2979 switch (rdev->family) {
2980 case CHIP_CYPRESS:
2981 case CHIP_HEMLOCK:
2982 rdev->config.evergreen.num_ses = 2;
2983 rdev->config.evergreen.max_pipes = 4;
2984 rdev->config.evergreen.max_tile_pipes = 8;
2985 rdev->config.evergreen.max_simds = 10;
2986 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
2987 rdev->config.evergreen.max_gprs = 256;
2988 rdev->config.evergreen.max_threads = 248;
2989 rdev->config.evergreen.max_gs_threads = 32;
2990 rdev->config.evergreen.max_stack_entries = 512;
2991 rdev->config.evergreen.sx_num_of_sets = 4;
2992 rdev->config.evergreen.sx_max_export_size = 256;
2993 rdev->config.evergreen.sx_max_export_pos_size = 64;
2994 rdev->config.evergreen.sx_max_export_smx_size = 192;
2995 rdev->config.evergreen.max_hw_contexts = 8;
2996 rdev->config.evergreen.sq_num_cf_insts = 2;
2997
2998 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
2999 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3000 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3001 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
3002 break;
3003 case CHIP_JUNIPER:
3004 rdev->config.evergreen.num_ses = 1;
3005 rdev->config.evergreen.max_pipes = 4;
3006 rdev->config.evergreen.max_tile_pipes = 4;
3007 rdev->config.evergreen.max_simds = 10;
3008 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3009 rdev->config.evergreen.max_gprs = 256;
3010 rdev->config.evergreen.max_threads = 248;
3011 rdev->config.evergreen.max_gs_threads = 32;
3012 rdev->config.evergreen.max_stack_entries = 512;
3013 rdev->config.evergreen.sx_num_of_sets = 4;
3014 rdev->config.evergreen.sx_max_export_size = 256;
3015 rdev->config.evergreen.sx_max_export_pos_size = 64;
3016 rdev->config.evergreen.sx_max_export_smx_size = 192;
3017 rdev->config.evergreen.max_hw_contexts = 8;
3018 rdev->config.evergreen.sq_num_cf_insts = 2;
3019
3020 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3021 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3022 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3023 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
3024 break;
3025 case CHIP_REDWOOD:
3026 rdev->config.evergreen.num_ses = 1;
3027 rdev->config.evergreen.max_pipes = 4;
3028 rdev->config.evergreen.max_tile_pipes = 4;
3029 rdev->config.evergreen.max_simds = 5;
3030 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3031 rdev->config.evergreen.max_gprs = 256;
3032 rdev->config.evergreen.max_threads = 248;
3033 rdev->config.evergreen.max_gs_threads = 32;
3034 rdev->config.evergreen.max_stack_entries = 256;
3035 rdev->config.evergreen.sx_num_of_sets = 4;
3036 rdev->config.evergreen.sx_max_export_size = 256;
3037 rdev->config.evergreen.sx_max_export_pos_size = 64;
3038 rdev->config.evergreen.sx_max_export_smx_size = 192;
3039 rdev->config.evergreen.max_hw_contexts = 8;
3040 rdev->config.evergreen.sq_num_cf_insts = 2;
3041
3042 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3043 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3044 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3045 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
3046 break;
3047 case CHIP_CEDAR:
3048 default:
3049 rdev->config.evergreen.num_ses = 1;
3050 rdev->config.evergreen.max_pipes = 2;
3051 rdev->config.evergreen.max_tile_pipes = 2;
3052 rdev->config.evergreen.max_simds = 2;
3053 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3054 rdev->config.evergreen.max_gprs = 256;
3055 rdev->config.evergreen.max_threads = 192;
3056 rdev->config.evergreen.max_gs_threads = 16;
3057 rdev->config.evergreen.max_stack_entries = 256;
3058 rdev->config.evergreen.sx_num_of_sets = 4;
3059 rdev->config.evergreen.sx_max_export_size = 128;
3060 rdev->config.evergreen.sx_max_export_pos_size = 32;
3061 rdev->config.evergreen.sx_max_export_smx_size = 96;
3062 rdev->config.evergreen.max_hw_contexts = 4;
3063 rdev->config.evergreen.sq_num_cf_insts = 1;
3064
3065 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3066 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3067 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3068 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3069 break;
3070 case CHIP_PALM:
3071 rdev->config.evergreen.num_ses = 1;
3072 rdev->config.evergreen.max_pipes = 2;
3073 rdev->config.evergreen.max_tile_pipes = 2;
3074 rdev->config.evergreen.max_simds = 2;
3075 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3076 rdev->config.evergreen.max_gprs = 256;
3077 rdev->config.evergreen.max_threads = 192;
3078 rdev->config.evergreen.max_gs_threads = 16;
3079 rdev->config.evergreen.max_stack_entries = 256;
3080 rdev->config.evergreen.sx_num_of_sets = 4;
3081 rdev->config.evergreen.sx_max_export_size = 128;
3082 rdev->config.evergreen.sx_max_export_pos_size = 32;
3083 rdev->config.evergreen.sx_max_export_smx_size = 96;
3084 rdev->config.evergreen.max_hw_contexts = 4;
3085 rdev->config.evergreen.sq_num_cf_insts = 1;
3086
3087 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3088 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3089 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3090 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3091 break;
3092 case CHIP_SUMO:
3093 rdev->config.evergreen.num_ses = 1;
3094 rdev->config.evergreen.max_pipes = 4;
3095 rdev->config.evergreen.max_tile_pipes = 4;
3096 if (rdev->pdev->device == 0x9648)
3097 rdev->config.evergreen.max_simds = 3;
3098 else if ((rdev->pdev->device == 0x9647) ||
3099 (rdev->pdev->device == 0x964a))
3100 rdev->config.evergreen.max_simds = 4;
3101 else
3102 rdev->config.evergreen.max_simds = 5;
3103 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3104 rdev->config.evergreen.max_gprs = 256;
3105 rdev->config.evergreen.max_threads = 248;
3106 rdev->config.evergreen.max_gs_threads = 32;
3107 rdev->config.evergreen.max_stack_entries = 256;
3108 rdev->config.evergreen.sx_num_of_sets = 4;
3109 rdev->config.evergreen.sx_max_export_size = 256;
3110 rdev->config.evergreen.sx_max_export_pos_size = 64;
3111 rdev->config.evergreen.sx_max_export_smx_size = 192;
3112 rdev->config.evergreen.max_hw_contexts = 8;
3113 rdev->config.evergreen.sq_num_cf_insts = 2;
3114
3115 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3116 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3117 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3118 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
3119 break;
3120 case CHIP_SUMO2:
3121 rdev->config.evergreen.num_ses = 1;
3122 rdev->config.evergreen.max_pipes = 4;
3123 rdev->config.evergreen.max_tile_pipes = 4;
3124 rdev->config.evergreen.max_simds = 2;
3125 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3126 rdev->config.evergreen.max_gprs = 256;
3127 rdev->config.evergreen.max_threads = 248;
3128 rdev->config.evergreen.max_gs_threads = 32;
3129 rdev->config.evergreen.max_stack_entries = 512;
3130 rdev->config.evergreen.sx_num_of_sets = 4;
3131 rdev->config.evergreen.sx_max_export_size = 256;
3132 rdev->config.evergreen.sx_max_export_pos_size = 64;
3133 rdev->config.evergreen.sx_max_export_smx_size = 192;
3134 rdev->config.evergreen.max_hw_contexts = 4;
3135 rdev->config.evergreen.sq_num_cf_insts = 2;
3136
3137 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3138 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3139 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3140 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
3141 break;
3142 case CHIP_BARTS:
3143 rdev->config.evergreen.num_ses = 2;
3144 rdev->config.evergreen.max_pipes = 4;
3145 rdev->config.evergreen.max_tile_pipes = 8;
3146 rdev->config.evergreen.max_simds = 7;
3147 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3148 rdev->config.evergreen.max_gprs = 256;
3149 rdev->config.evergreen.max_threads = 248;
3150 rdev->config.evergreen.max_gs_threads = 32;
3151 rdev->config.evergreen.max_stack_entries = 512;
3152 rdev->config.evergreen.sx_num_of_sets = 4;
3153 rdev->config.evergreen.sx_max_export_size = 256;
3154 rdev->config.evergreen.sx_max_export_pos_size = 64;
3155 rdev->config.evergreen.sx_max_export_smx_size = 192;
3156 rdev->config.evergreen.max_hw_contexts = 8;
3157 rdev->config.evergreen.sq_num_cf_insts = 2;
3158
3159 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3160 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3161 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3162 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
3163 break;
3164 case CHIP_TURKS:
3165 rdev->config.evergreen.num_ses = 1;
3166 rdev->config.evergreen.max_pipes = 4;
3167 rdev->config.evergreen.max_tile_pipes = 4;
3168 rdev->config.evergreen.max_simds = 6;
3169 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3170 rdev->config.evergreen.max_gprs = 256;
3171 rdev->config.evergreen.max_threads = 248;
3172 rdev->config.evergreen.max_gs_threads = 32;
3173 rdev->config.evergreen.max_stack_entries = 256;
3174 rdev->config.evergreen.sx_num_of_sets = 4;
3175 rdev->config.evergreen.sx_max_export_size = 256;
3176 rdev->config.evergreen.sx_max_export_pos_size = 64;
3177 rdev->config.evergreen.sx_max_export_smx_size = 192;
3178 rdev->config.evergreen.max_hw_contexts = 8;
3179 rdev->config.evergreen.sq_num_cf_insts = 2;
3180
3181 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3182 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3183 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3184 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
3185 break;
3186 case CHIP_CAICOS:
3187 rdev->config.evergreen.num_ses = 1;
3188 rdev->config.evergreen.max_pipes = 2;
3189 rdev->config.evergreen.max_tile_pipes = 2;
3190 rdev->config.evergreen.max_simds = 2;
3191 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3192 rdev->config.evergreen.max_gprs = 256;
3193 rdev->config.evergreen.max_threads = 192;
3194 rdev->config.evergreen.max_gs_threads = 16;
3195 rdev->config.evergreen.max_stack_entries = 256;
3196 rdev->config.evergreen.sx_num_of_sets = 4;
3197 rdev->config.evergreen.sx_max_export_size = 128;
3198 rdev->config.evergreen.sx_max_export_pos_size = 32;
3199 rdev->config.evergreen.sx_max_export_smx_size = 96;
3200 rdev->config.evergreen.max_hw_contexts = 4;
3201 rdev->config.evergreen.sq_num_cf_insts = 1;
3202
3203 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3204 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3205 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3206 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
3207 break;
3208 }
3209
3210 /* Initialize HDP */
3211 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3212 WREG32((0x2c14 + j), 0x00000000);
3213 WREG32((0x2c18 + j), 0x00000000);
3214 WREG32((0x2c1c + j), 0x00000000);
3215 WREG32((0x2c20 + j), 0x00000000);
3216 WREG32((0x2c24 + j), 0x00000000);
3217 }
3218
3219 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3220
3221 evergreen_fix_pci_max_read_req_size(rdev);
3222
3223 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3224 if ((rdev->family == CHIP_PALM) ||
3225 (rdev->family == CHIP_SUMO) ||
3226 (rdev->family == CHIP_SUMO2))
3227 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3228 else
3229 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3230
3231 /* setup tiling info dword. gb_addr_config is not adequate since it does
3232 * not have bank info, so create a custom tiling dword.
3233 * bits 3:0 num_pipes
3234 * bits 7:4 num_banks
3235 * bits 11:8 group_size
3236 * bits 15:12 row_size
3237 */
3238 rdev->config.evergreen.tile_config = 0;
3239 switch (rdev->config.evergreen.max_tile_pipes) {
3240 case 1:
3241 default:
3242 rdev->config.evergreen.tile_config |= (0 << 0);
3243 break;
3244 case 2:
3245 rdev->config.evergreen.tile_config |= (1 << 0);
3246 break;
3247 case 4:
3248 rdev->config.evergreen.tile_config |= (2 << 0);
3249 break;
3250 case 8:
3251 rdev->config.evergreen.tile_config |= (3 << 0);
3252 break;
3253 }
3254 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
3255 if (rdev->flags & RADEON_IS_IGP)
3256 rdev->config.evergreen.tile_config |= 1 << 4;
3257 else {
3258 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3259 case 0: /* four banks */
3260 rdev->config.evergreen.tile_config |= 0 << 4;
3261 break;
3262 case 1: /* eight banks */
3263 rdev->config.evergreen.tile_config |= 1 << 4;
3264 break;
3265 case 2: /* sixteen banks */
3266 default:
3267 rdev->config.evergreen.tile_config |= 2 << 4;
3268 break;
3269 }
3270 }
3271 rdev->config.evergreen.tile_config |= 0 << 8;
3272 rdev->config.evergreen.tile_config |=
3273 ((gb_addr_config & 0x30000000) >> 28) << 12;
3274
3275 num_shader_engines = (gb_addr_config & NUM_SHADER_ENGINES(3) >> 12) + 1;
3276
3277 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3278 u32 efuse_straps_4;
3279 u32 efuse_straps_3;
3280
3281 efuse_straps_4 = RREG32_RCU(0x204);
3282 efuse_straps_3 = RREG32_RCU(0x203);
3283 tmp = (((efuse_straps_4 & 0xf) << 4) |
3284 ((efuse_straps_3 & 0xf0000000) >> 28));
3285 } else {
3286 tmp = 0;
3287 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3288 u32 rb_disable_bitmap;
3289
3290 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3291 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3292 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3293 tmp <<= 4;
3294 tmp |= rb_disable_bitmap;
3295 }
3296 }
3297 /* enabled rb are just the one not disabled :) */
3298 disabled_rb_mask = tmp;
3299 tmp = 0;
3300 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3301 tmp |= (1 << i);
3302 /* if all the backends are disabled, fix it up here */
3303 if ((disabled_rb_mask & tmp) == tmp) {
3304 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3305 disabled_rb_mask &= ~(1 << i);
3306 }
3307
3308 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3309 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3310
3311 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3312 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3313 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3314 WREG32(DMA_TILING_CONFIG, gb_addr_config);
3315 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3316 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3317 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3318
3319 if ((rdev->config.evergreen.max_backends == 1) &&
3320 (rdev->flags & RADEON_IS_IGP)) {
3321 if ((disabled_rb_mask & 3) == 1) {
3322 /* RB0 disabled, RB1 enabled */
3323 tmp = 0x11111111;
3324 } else {
3325 /* RB1 disabled, RB0 enabled */
3326 tmp = 0x00000000;
3327 }
3328 } else {
3329 tmp = gb_addr_config & NUM_PIPES_MASK;
3330 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3331 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3332 }
3333 WREG32(GB_BACKEND_MAP, tmp);
3334
3335 WREG32(CGTS_SYS_TCC_DISABLE, 0);
3336 WREG32(CGTS_TCC_DISABLE, 0);
3337 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3338 WREG32(CGTS_USER_TCC_DISABLE, 0);
3339
3340 /* set HW defaults for 3D engine */
3341 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3342 ROQ_IB2_START(0x2b)));
3343
3344 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3345
3346 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3347 SYNC_GRADIENT |
3348 SYNC_WALKER |
3349 SYNC_ALIGNER));
3350
3351 sx_debug_1 = RREG32(SX_DEBUG_1);
3352 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3353 WREG32(SX_DEBUG_1, sx_debug_1);
3354
3355
3356 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3357 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3358 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3359 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3360
3361 if (rdev->family <= CHIP_SUMO2)
3362 WREG32(SMX_SAR_CTL0, 0x00010000);
3363
3364 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3365 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3366 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3367
3368 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3369 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3370 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3371
3372 WREG32(VGT_NUM_INSTANCES, 1);
3373 WREG32(SPI_CONFIG_CNTL, 0);
3374 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3375 WREG32(CP_PERFMON_CNTL, 0);
3376
3377 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3378 FETCH_FIFO_HIWATER(0x4) |
3379 DONE_FIFO_HIWATER(0xe0) |
3380 ALU_UPDATE_FIFO_HIWATER(0x8)));
3381
3382 sq_config = RREG32(SQ_CONFIG);
3383 sq_config &= ~(PS_PRIO(3) |
3384 VS_PRIO(3) |
3385 GS_PRIO(3) |
3386 ES_PRIO(3));
3387 sq_config |= (VC_ENABLE |
3388 EXPORT_SRC_C |
3389 PS_PRIO(0) |
3390 VS_PRIO(1) |
3391 GS_PRIO(2) |
3392 ES_PRIO(3));
3393
3394 switch (rdev->family) {
3395 case CHIP_CEDAR:
3396 case CHIP_PALM:
3397 case CHIP_SUMO:
3398 case CHIP_SUMO2:
3399 case CHIP_CAICOS:
3400 /* no vertex cache */
3401 sq_config &= ~VC_ENABLE;
3402 break;
3403 default:
3404 break;
3405 }
3406
3407 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3408
3409 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3410 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3411 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3412 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3413 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3414 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3415 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3416
3417 switch (rdev->family) {
3418 case CHIP_CEDAR:
3419 case CHIP_PALM:
3420 case CHIP_SUMO:
3421 case CHIP_SUMO2:
3422 ps_thread_count = 96;
3423 break;
3424 default:
3425 ps_thread_count = 128;
3426 break;
3427 }
3428
3429 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
3430 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3431 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3432 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3433 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3434 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3435
3436 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3437 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3438 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3439 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3440 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3441 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3442
3443 WREG32(SQ_CONFIG, sq_config);
3444 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3445 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3446 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3447 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3448 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3449 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3450 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3451 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3452 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3453 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3454
3455 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3456 FORCE_EOV_MAX_REZ_CNT(255)));
3457
3458 switch (rdev->family) {
3459 case CHIP_CEDAR:
3460 case CHIP_PALM:
3461 case CHIP_SUMO:
3462 case CHIP_SUMO2:
3463 case CHIP_CAICOS:
3464 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
3465 break;
3466 default:
3467 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
3468 break;
3469 }
3470 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3471 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3472
3473 WREG32(VGT_GS_VERTEX_REUSE, 16);
3474 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
3475 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3476
3477 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3478 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3479
3480 WREG32(CB_PERF_CTR0_SEL_0, 0);
3481 WREG32(CB_PERF_CTR0_SEL_1, 0);
3482 WREG32(CB_PERF_CTR1_SEL_0, 0);
3483 WREG32(CB_PERF_CTR1_SEL_1, 0);
3484 WREG32(CB_PERF_CTR2_SEL_0, 0);
3485 WREG32(CB_PERF_CTR2_SEL_1, 0);
3486 WREG32(CB_PERF_CTR3_SEL_0, 0);
3487 WREG32(CB_PERF_CTR3_SEL_1, 0);
3488
3489 /* clear render buffer base addresses */
3490 WREG32(CB_COLOR0_BASE, 0);
3491 WREG32(CB_COLOR1_BASE, 0);
3492 WREG32(CB_COLOR2_BASE, 0);
3493 WREG32(CB_COLOR3_BASE, 0);
3494 WREG32(CB_COLOR4_BASE, 0);
3495 WREG32(CB_COLOR5_BASE, 0);
3496 WREG32(CB_COLOR6_BASE, 0);
3497 WREG32(CB_COLOR7_BASE, 0);
3498 WREG32(CB_COLOR8_BASE, 0);
3499 WREG32(CB_COLOR9_BASE, 0);
3500 WREG32(CB_COLOR10_BASE, 0);
3501 WREG32(CB_COLOR11_BASE, 0);
3502
3503 /* set the shader const cache sizes to 0 */
3504 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3505 WREG32(i, 0);
3506 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3507 WREG32(i, 0);
3508
3509 tmp = RREG32(HDP_MISC_CNTL);
3510 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3511 WREG32(HDP_MISC_CNTL, tmp);
3512
3513 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3514 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3515
3516 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3517
3518 udelay(50);
3519
3520 }
3521
3522 int evergreen_mc_init(struct radeon_device *rdev)
3523 {
3524 u32 tmp;
3525 int chansize, numchan;
3526
3527 /* Get VRAM informations */
3528 rdev->mc.vram_is_ddr = true;
3529 if ((rdev->family == CHIP_PALM) ||
3530 (rdev->family == CHIP_SUMO) ||
3531 (rdev->family == CHIP_SUMO2))
3532 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3533 else
3534 tmp = RREG32(MC_ARB_RAMCFG);
3535 if (tmp & CHANSIZE_OVERRIDE) {
3536 chansize = 16;
3537 } else if (tmp & CHANSIZE_MASK) {
3538 chansize = 64;
3539 } else {
3540 chansize = 32;
3541 }
3542 tmp = RREG32(MC_SHARED_CHMAP);
3543 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3544 case 0:
3545 default:
3546 numchan = 1;
3547 break;
3548 case 1:
3549 numchan = 2;
3550 break;
3551 case 2:
3552 numchan = 4;
3553 break;
3554 case 3:
3555 numchan = 8;
3556 break;
3557 }
3558 rdev->mc.vram_width = numchan * chansize;
3559 /* Could aper size report 0 ? */
3560 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3561 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3562 /* Setup GPU memory space */
3563 if ((rdev->family == CHIP_PALM) ||
3564 (rdev->family == CHIP_SUMO) ||
3565 (rdev->family == CHIP_SUMO2)) {
3566 /* size in bytes on fusion */
3567 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3568 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3569 } else {
3570 /* size in MB on evergreen/cayman/tn */
3571 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3572 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3573 }
3574 rdev->mc.visible_vram_size = rdev->mc.aper_size;
3575 r700_vram_gtt_location(rdev, &rdev->mc);
3576 radeon_update_bandwidth_info(rdev);
3577
3578 return 0;
3579 }
3580
3581 void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
3582 {
3583 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
3584 RREG32(GRBM_STATUS));
3585 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
3586 RREG32(GRBM_STATUS_SE0));
3587 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
3588 RREG32(GRBM_STATUS_SE1));
3589 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
3590 RREG32(SRBM_STATUS));
3591 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
3592 RREG32(SRBM_STATUS2));
3593 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3594 RREG32(CP_STALLED_STAT1));
3595 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3596 RREG32(CP_STALLED_STAT2));
3597 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
3598 RREG32(CP_BUSY_STAT));
3599 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
3600 RREG32(CP_STAT));
3601 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
3602 RREG32(DMA_STATUS_REG));
3603 if (rdev->family >= CHIP_CAYMAN) {
3604 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
3605 RREG32(DMA_STATUS_REG + 0x800));
3606 }
3607 }
3608
3609 bool evergreen_is_display_hung(struct radeon_device *rdev)
3610 {
3611 u32 crtc_hung = 0;
3612 u32 crtc_status[6];
3613 u32 i, j, tmp;
3614
3615 for (i = 0; i < rdev->num_crtc; i++) {
3616 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3617 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3618 crtc_hung |= (1 << i);
3619 }
3620 }
3621
3622 for (j = 0; j < 10; j++) {
3623 for (i = 0; i < rdev->num_crtc; i++) {
3624 if (crtc_hung & (1 << i)) {
3625 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3626 if (tmp != crtc_status[i])
3627 crtc_hung &= ~(1 << i);
3628 }
3629 }
3630 if (crtc_hung == 0)
3631 return false;
3632 udelay(100);
3633 }
3634
3635 return true;
3636 }
3637
3638 u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3639 {
3640 u32 reset_mask = 0;
3641 u32 tmp;
3642
3643 /* GRBM_STATUS */
3644 tmp = RREG32(GRBM_STATUS);
3645 if (tmp & (PA_BUSY | SC_BUSY |
3646 SH_BUSY | SX_BUSY |
3647 TA_BUSY | VGT_BUSY |
3648 DB_BUSY | CB_BUSY |
3649 SPI_BUSY | VGT_BUSY_NO_DMA))
3650 reset_mask |= RADEON_RESET_GFX;
3651
3652 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3653 CP_BUSY | CP_COHERENCY_BUSY))
3654 reset_mask |= RADEON_RESET_CP;
3655
3656 if (tmp & GRBM_EE_BUSY)
3657 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3658
3659 /* DMA_STATUS_REG */
3660 tmp = RREG32(DMA_STATUS_REG);
3661 if (!(tmp & DMA_IDLE))
3662 reset_mask |= RADEON_RESET_DMA;
3663
3664 /* SRBM_STATUS2 */
3665 tmp = RREG32(SRBM_STATUS2);
3666 if (tmp & DMA_BUSY)
3667 reset_mask |= RADEON_RESET_DMA;
3668
3669 /* SRBM_STATUS */
3670 tmp = RREG32(SRBM_STATUS);
3671 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3672 reset_mask |= RADEON_RESET_RLC;
3673
3674 if (tmp & IH_BUSY)
3675 reset_mask |= RADEON_RESET_IH;
3676
3677 if (tmp & SEM_BUSY)
3678 reset_mask |= RADEON_RESET_SEM;
3679
3680 if (tmp & GRBM_RQ_PENDING)
3681 reset_mask |= RADEON_RESET_GRBM;
3682
3683 if (tmp & VMC_BUSY)
3684 reset_mask |= RADEON_RESET_VMC;
3685
3686 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3687 MCC_BUSY | MCD_BUSY))
3688 reset_mask |= RADEON_RESET_MC;
3689
3690 if (evergreen_is_display_hung(rdev))
3691 reset_mask |= RADEON_RESET_DISPLAY;
3692
3693 /* VM_L2_STATUS */
3694 tmp = RREG32(VM_L2_STATUS);
3695 if (tmp & L2_BUSY)
3696 reset_mask |= RADEON_RESET_VMC;
3697
3698 /* Skip MC reset as it's mostly likely not hung, just busy */
3699 if (reset_mask & RADEON_RESET_MC) {
3700 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3701 reset_mask &= ~RADEON_RESET_MC;
3702 }
3703
3704 return reset_mask;
3705 }
3706
3707 static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3708 {
3709 struct evergreen_mc_save save;
3710 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3711 u32 tmp;
3712
3713 if (reset_mask == 0)
3714 return;
3715
3716 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3717
3718 evergreen_print_gpu_status_regs(rdev);
3719
3720 /* Disable CP parsing/prefetching */
3721 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3722
3723 if (reset_mask & RADEON_RESET_DMA) {
3724 /* Disable DMA */
3725 tmp = RREG32(DMA_RB_CNTL);
3726 tmp &= ~DMA_RB_ENABLE;
3727 WREG32(DMA_RB_CNTL, tmp);
3728 }
3729
3730 udelay(50);
3731
3732 evergreen_mc_stop(rdev, &save);
3733 if (evergreen_mc_wait_for_idle(rdev)) {
3734 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3735 }
3736
3737 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3738 grbm_soft_reset |= SOFT_RESET_DB |
3739 SOFT_RESET_CB |
3740 SOFT_RESET_PA |
3741 SOFT_RESET_SC |
3742 SOFT_RESET_SPI |
3743 SOFT_RESET_SX |
3744 SOFT_RESET_SH |
3745 SOFT_RESET_TC |
3746 SOFT_RESET_TA |
3747 SOFT_RESET_VC |
3748 SOFT_RESET_VGT;
3749 }
3750
3751 if (reset_mask & RADEON_RESET_CP) {
3752 grbm_soft_reset |= SOFT_RESET_CP |
3753 SOFT_RESET_VGT;
3754
3755 srbm_soft_reset |= SOFT_RESET_GRBM;
3756 }
3757
3758 if (reset_mask & RADEON_RESET_DMA)
3759 srbm_soft_reset |= SOFT_RESET_DMA;
3760
3761 if (reset_mask & RADEON_RESET_DISPLAY)
3762 srbm_soft_reset |= SOFT_RESET_DC;
3763
3764 if (reset_mask & RADEON_RESET_RLC)
3765 srbm_soft_reset |= SOFT_RESET_RLC;
3766
3767 if (reset_mask & RADEON_RESET_SEM)
3768 srbm_soft_reset |= SOFT_RESET_SEM;
3769
3770 if (reset_mask & RADEON_RESET_IH)
3771 srbm_soft_reset |= SOFT_RESET_IH;
3772
3773 if (reset_mask & RADEON_RESET_GRBM)
3774 srbm_soft_reset |= SOFT_RESET_GRBM;
3775
3776 if (reset_mask & RADEON_RESET_VMC)
3777 srbm_soft_reset |= SOFT_RESET_VMC;
3778
3779 if (!(rdev->flags & RADEON_IS_IGP)) {
3780 if (reset_mask & RADEON_RESET_MC)
3781 srbm_soft_reset |= SOFT_RESET_MC;
3782 }
3783
3784 if (grbm_soft_reset) {
3785 tmp = RREG32(GRBM_SOFT_RESET);
3786 tmp |= grbm_soft_reset;
3787 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3788 WREG32(GRBM_SOFT_RESET, tmp);
3789 tmp = RREG32(GRBM_SOFT_RESET);
3790
3791 udelay(50);
3792
3793 tmp &= ~grbm_soft_reset;
3794 WREG32(GRBM_SOFT_RESET, tmp);
3795 tmp = RREG32(GRBM_SOFT_RESET);
3796 }
3797
3798 if (srbm_soft_reset) {
3799 tmp = RREG32(SRBM_SOFT_RESET);
3800 tmp |= srbm_soft_reset;
3801 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3802 WREG32(SRBM_SOFT_RESET, tmp);
3803 tmp = RREG32(SRBM_SOFT_RESET);
3804
3805 udelay(50);
3806
3807 tmp &= ~srbm_soft_reset;
3808 WREG32(SRBM_SOFT_RESET, tmp);
3809 tmp = RREG32(SRBM_SOFT_RESET);
3810 }
3811
3812 /* Wait a little for things to settle down */
3813 udelay(50);
3814
3815 evergreen_mc_resume(rdev, &save);
3816 udelay(50);
3817
3818 evergreen_print_gpu_status_regs(rdev);
3819 }
3820
3821 int evergreen_asic_reset(struct radeon_device *rdev)
3822 {
3823 u32 reset_mask;
3824
3825 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3826
3827 if (reset_mask)
3828 r600_set_bios_scratch_engine_hung(rdev, true);
3829
3830 evergreen_gpu_soft_reset(rdev, reset_mask);
3831
3832 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3833
3834 if (!reset_mask)
3835 r600_set_bios_scratch_engine_hung(rdev, false);
3836
3837 return 0;
3838 }
3839
3840 /**
3841 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
3842 *
3843 * @rdev: radeon_device pointer
3844 * @ring: radeon_ring structure holding ring information
3845 *
3846 * Check if the GFX engine is locked up.
3847 * Returns true if the engine appears to be locked up, false if not.
3848 */
3849 bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3850 {
3851 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3852
3853 if (!(reset_mask & (RADEON_RESET_GFX |
3854 RADEON_RESET_COMPUTE |
3855 RADEON_RESET_CP))) {
3856 radeon_ring_lockup_update(ring);
3857 return false;
3858 }
3859 /* force CP activities */
3860 radeon_ring_force_activity(rdev, ring);
3861 return radeon_ring_test_lockup(rdev, ring);
3862 }
3863
3864 /*
3865 * RLC
3866 */
3867 #define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000
3868 #define RLC_CLEAR_STATE_END_MARKER 0x00000001
3869
3870 void sumo_rlc_fini(struct radeon_device *rdev)
3871 {
3872 int r;
3873
3874 /* save restore block */
3875 if (rdev->rlc.save_restore_obj) {
3876 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3877 if (unlikely(r != 0))
3878 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
3879 radeon_bo_unpin(rdev->rlc.save_restore_obj);
3880 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3881
3882 radeon_bo_unref(&rdev->rlc.save_restore_obj);
3883 rdev->rlc.save_restore_obj = NULL;
3884 }
3885
3886 /* clear state block */
3887 if (rdev->rlc.clear_state_obj) {
3888 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
3889 if (unlikely(r != 0))
3890 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
3891 radeon_bo_unpin(rdev->rlc.clear_state_obj);
3892 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
3893
3894 radeon_bo_unref(&rdev->rlc.clear_state_obj);
3895 rdev->rlc.clear_state_obj = NULL;
3896 }
3897
3898 /* clear state block */
3899 if (rdev->rlc.cp_table_obj) {
3900 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
3901 if (unlikely(r != 0))
3902 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
3903 radeon_bo_unpin(rdev->rlc.cp_table_obj);
3904 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
3905
3906 radeon_bo_unref(&rdev->rlc.cp_table_obj);
3907 rdev->rlc.cp_table_obj = NULL;
3908 }
3909 }
3910
3911 #define CP_ME_TABLE_SIZE 96
3912
3913 int sumo_rlc_init(struct radeon_device *rdev)
3914 {
3915 const u32 *src_ptr;
3916 volatile u32 *dst_ptr;
3917 u32 dws, data, i, j, k, reg_num;
3918 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
3919 u64 reg_list_mc_addr;
3920 const struct cs_section_def *cs_data;
3921 int r;
3922
3923 src_ptr = rdev->rlc.reg_list;
3924 dws = rdev->rlc.reg_list_size;
3925 if (rdev->family >= CHIP_BONAIRE) {
3926 dws += (5 * 16) + 48 + 48 + 64;
3927 }
3928 cs_data = rdev->rlc.cs_data;
3929
3930 if (src_ptr) {
3931 /* save restore block */
3932 if (rdev->rlc.save_restore_obj == NULL) {
3933 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
3934 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.save_restore_obj);
3935 if (r) {
3936 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
3937 return r;
3938 }
3939 }
3940
3941 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3942 if (unlikely(r != 0)) {
3943 sumo_rlc_fini(rdev);
3944 return r;
3945 }
3946 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
3947 &rdev->rlc.save_restore_gpu_addr);
3948 if (r) {
3949 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3950 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
3951 sumo_rlc_fini(rdev);
3952 return r;
3953 }
3954
3955 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
3956 if (r) {
3957 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
3958 sumo_rlc_fini(rdev);
3959 return r;
3960 }
3961 /* write the sr buffer */
3962 dst_ptr = rdev->rlc.sr_ptr;
3963 if (rdev->family >= CHIP_TAHITI) {
3964 /* SI */
3965 for (i = 0; i < rdev->rlc.reg_list_size; i++)
3966 dst_ptr[i] = src_ptr[i];
3967 } else {
3968 /* ON/LN/TN */
3969 /* format:
3970 * dw0: (reg2 << 16) | reg1
3971 * dw1: reg1 save space
3972 * dw2: reg2 save space
3973 */
3974 for (i = 0; i < dws; i++) {
3975 data = src_ptr[i] >> 2;
3976 i++;
3977 if (i < dws)
3978 data |= (src_ptr[i] >> 2) << 16;
3979 j = (((i - 1) * 3) / 2);
3980 dst_ptr[j] = data;
3981 }
3982 j = ((i * 3) / 2);
3983 dst_ptr[j] = RLC_SAVE_RESTORE_LIST_END_MARKER;
3984 }
3985 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
3986 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3987 }
3988
3989 if (cs_data) {
3990 /* clear state block */
3991 if (rdev->family >= CHIP_BONAIRE) {
3992 rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
3993 } else if (rdev->family >= CHIP_TAHITI) {
3994 rdev->rlc.clear_state_size = si_get_csb_size(rdev);
3995 dws = rdev->rlc.clear_state_size + (256 / 4);
3996 } else {
3997 reg_list_num = 0;
3998 dws = 0;
3999 for (i = 0; cs_data[i].section != NULL; i++) {
4000 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4001 reg_list_num++;
4002 dws += cs_data[i].section[j].reg_count;
4003 }
4004 }
4005 reg_list_blk_index = (3 * reg_list_num + 2);
4006 dws += reg_list_blk_index;
4007 rdev->rlc.clear_state_size = dws;
4008 }
4009
4010 if (rdev->rlc.clear_state_obj == NULL) {
4011 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4012 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.clear_state_obj);
4013 if (r) {
4014 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4015 sumo_rlc_fini(rdev);
4016 return r;
4017 }
4018 }
4019 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4020 if (unlikely(r != 0)) {
4021 sumo_rlc_fini(rdev);
4022 return r;
4023 }
4024 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4025 &rdev->rlc.clear_state_gpu_addr);
4026 if (r) {
4027 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4028 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4029 sumo_rlc_fini(rdev);
4030 return r;
4031 }
4032
4033 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4034 if (r) {
4035 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4036 sumo_rlc_fini(rdev);
4037 return r;
4038 }
4039 /* set up the cs buffer */
4040 dst_ptr = rdev->rlc.cs_ptr;
4041 if (rdev->family >= CHIP_BONAIRE) {
4042 cik_get_csb_buffer(rdev, dst_ptr);
4043 } else if (rdev->family >= CHIP_TAHITI) {
4044 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
4045 dst_ptr[0] = upper_32_bits(reg_list_mc_addr);
4046 dst_ptr[1] = lower_32_bits(reg_list_mc_addr);
4047 dst_ptr[2] = rdev->rlc.clear_state_size;
4048 si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4049 } else {
4050 reg_list_hdr_blk_index = 0;
4051 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4052 data = upper_32_bits(reg_list_mc_addr);
4053 dst_ptr[reg_list_hdr_blk_index] = data;
4054 reg_list_hdr_blk_index++;
4055 for (i = 0; cs_data[i].section != NULL; i++) {
4056 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4057 reg_num = cs_data[i].section[j].reg_count;
4058 data = reg_list_mc_addr & 0xffffffff;
4059 dst_ptr[reg_list_hdr_blk_index] = data;
4060 reg_list_hdr_blk_index++;
4061
4062 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4063 dst_ptr[reg_list_hdr_blk_index] = data;
4064 reg_list_hdr_blk_index++;
4065
4066 data = 0x08000000 | (reg_num * 4);
4067 dst_ptr[reg_list_hdr_blk_index] = data;
4068 reg_list_hdr_blk_index++;
4069
4070 for (k = 0; k < reg_num; k++) {
4071 data = cs_data[i].section[j].extent[k];
4072 dst_ptr[reg_list_blk_index + k] = data;
4073 }
4074 reg_list_mc_addr += reg_num * 4;
4075 reg_list_blk_index += reg_num;
4076 }
4077 }
4078 dst_ptr[reg_list_hdr_blk_index] = RLC_CLEAR_STATE_END_MARKER;
4079 }
4080 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4081 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4082 }
4083
4084 if (rdev->rlc.cp_table_size) {
4085 if (rdev->rlc.cp_table_obj == NULL) {
4086 r = radeon_bo_create(rdev, rdev->rlc.cp_table_size, PAGE_SIZE, true,
4087 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.cp_table_obj);
4088 if (r) {
4089 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4090 sumo_rlc_fini(rdev);
4091 return r;
4092 }
4093 }
4094
4095 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4096 if (unlikely(r != 0)) {
4097 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4098 sumo_rlc_fini(rdev);
4099 return r;
4100 }
4101 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4102 &rdev->rlc.cp_table_gpu_addr);
4103 if (r) {
4104 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4105 dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4106 sumo_rlc_fini(rdev);
4107 return r;
4108 }
4109 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4110 if (r) {
4111 dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4112 sumo_rlc_fini(rdev);
4113 return r;
4114 }
4115
4116 cik_init_cp_pg_table(rdev);
4117
4118 radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4119 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4120
4121 }
4122
4123 return 0;
4124 }
4125
4126 static void evergreen_rlc_start(struct radeon_device *rdev)
4127 {
4128 u32 mask = RLC_ENABLE;
4129
4130 if (rdev->flags & RADEON_IS_IGP) {
4131 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
4132 }
4133
4134 WREG32(RLC_CNTL, mask);
4135 }
4136
4137 int evergreen_rlc_resume(struct radeon_device *rdev)
4138 {
4139 u32 i;
4140 const __be32 *fw_data;
4141
4142 if (!rdev->rlc_fw)
4143 return -EINVAL;
4144
4145 r600_rlc_stop(rdev);
4146
4147 WREG32(RLC_HB_CNTL, 0);
4148
4149 if (rdev->flags & RADEON_IS_IGP) {
4150 if (rdev->family == CHIP_ARUBA) {
4151 u32 always_on_bitmap =
4152 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4153 /* find out the number of active simds */
4154 u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4155 tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4156 tmp = hweight32(~tmp);
4157 if (tmp == rdev->config.cayman.max_simds_per_se) {
4158 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4159 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4160 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4161 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4162 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4163 }
4164 } else {
4165 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4166 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4167 }
4168 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4169 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4170 } else {
4171 WREG32(RLC_HB_BASE, 0);
4172 WREG32(RLC_HB_RPTR, 0);
4173 WREG32(RLC_HB_WPTR, 0);
4174 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4175 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4176 }
4177 WREG32(RLC_MC_CNTL, 0);
4178 WREG32(RLC_UCODE_CNTL, 0);
4179
4180 fw_data = (const __be32 *)rdev->rlc_fw->data;
4181 if (rdev->family >= CHIP_ARUBA) {
4182 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4183 WREG32(RLC_UCODE_ADDR, i);
4184 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4185 }
4186 } else if (rdev->family >= CHIP_CAYMAN) {
4187 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4188 WREG32(RLC_UCODE_ADDR, i);
4189 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4190 }
4191 } else {
4192 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4193 WREG32(RLC_UCODE_ADDR, i);
4194 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4195 }
4196 }
4197 WREG32(RLC_UCODE_ADDR, 0);
4198
4199 evergreen_rlc_start(rdev);
4200
4201 return 0;
4202 }
4203
4204 /* Interrupts */
4205
4206 u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4207 {
4208 if (crtc >= rdev->num_crtc)
4209 return 0;
4210 else
4211 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
4212 }
4213
4214 void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4215 {
4216 u32 tmp;
4217
4218 if (rdev->family >= CHIP_CAYMAN) {
4219 cayman_cp_int_cntl_setup(rdev, 0,
4220 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4221 cayman_cp_int_cntl_setup(rdev, 1, 0);
4222 cayman_cp_int_cntl_setup(rdev, 2, 0);
4223 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4224 WREG32(CAYMAN_DMA1_CNTL, tmp);
4225 } else
4226 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4227 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4228 WREG32(DMA_CNTL, tmp);
4229 WREG32(GRBM_INT_CNTL, 0);
4230 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4231 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4232 if (rdev->num_crtc >= 4) {
4233 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4234 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4235 }
4236 if (rdev->num_crtc >= 6) {
4237 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4238 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4239 }
4240
4241 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4242 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4243 if (rdev->num_crtc >= 4) {
4244 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4245 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4246 }
4247 if (rdev->num_crtc >= 6) {
4248 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4249 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4250 }
4251
4252 /* only one DAC on DCE5 */
4253 if (!ASIC_IS_DCE5(rdev))
4254 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4255 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4256
4257 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4258 WREG32(DC_HPD1_INT_CONTROL, tmp);
4259 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4260 WREG32(DC_HPD2_INT_CONTROL, tmp);
4261 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4262 WREG32(DC_HPD3_INT_CONTROL, tmp);
4263 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4264 WREG32(DC_HPD4_INT_CONTROL, tmp);
4265 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4266 WREG32(DC_HPD5_INT_CONTROL, tmp);
4267 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4268 WREG32(DC_HPD6_INT_CONTROL, tmp);
4269
4270 }
4271
4272 int evergreen_irq_set(struct radeon_device *rdev)
4273 {
4274 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4275 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4276 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
4277 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
4278 u32 grbm_int_cntl = 0;
4279 u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
4280 u32 dma_cntl, dma_cntl1 = 0;
4281 u32 thermal_int = 0;
4282
4283 if (!rdev->irq.installed) {
4284 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4285 return -EINVAL;
4286 }
4287 /* don't enable anything if the ih is disabled */
4288 if (!rdev->ih.enabled) {
4289 r600_disable_interrupts(rdev);
4290 /* force the active interrupt state to all disabled */
4291 evergreen_disable_interrupt_state(rdev);
4292 return 0;
4293 }
4294
4295 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
4296 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
4297 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
4298 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
4299 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
4300 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
4301 if (rdev->family == CHIP_ARUBA)
4302 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4303 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4304 else
4305 thermal_int = RREG32(CG_THERMAL_INT) &
4306 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4307
4308 afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4309 afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4310 afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4311 afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4312 afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4313 afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4314
4315 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4316
4317 if (rdev->family >= CHIP_CAYMAN) {
4318 /* enable CP interrupts on all rings */
4319 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4320 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4321 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4322 }
4323 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
4324 DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4325 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4326 }
4327 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
4328 DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4329 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4330 }
4331 } else {
4332 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4333 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4334 cp_int_cntl |= RB_INT_ENABLE;
4335 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4336 }
4337 }
4338
4339 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4340 DRM_DEBUG("r600_irq_set: sw int dma\n");
4341 dma_cntl |= TRAP_ENABLE;
4342 }
4343
4344 if (rdev->family >= CHIP_CAYMAN) {
4345 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4346 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4347 DRM_DEBUG("r600_irq_set: sw int dma1\n");
4348 dma_cntl1 |= TRAP_ENABLE;
4349 }
4350 }
4351
4352 if (rdev->irq.dpm_thermal) {
4353 DRM_DEBUG("dpm thermal\n");
4354 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4355 }
4356
4357 if (rdev->irq.crtc_vblank_int[0] ||
4358 atomic_read(&rdev->irq.pflip[0])) {
4359 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
4360 crtc1 |= VBLANK_INT_MASK;
4361 }
4362 if (rdev->irq.crtc_vblank_int[1] ||
4363 atomic_read(&rdev->irq.pflip[1])) {
4364 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
4365 crtc2 |= VBLANK_INT_MASK;
4366 }
4367 if (rdev->irq.crtc_vblank_int[2] ||
4368 atomic_read(&rdev->irq.pflip[2])) {
4369 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
4370 crtc3 |= VBLANK_INT_MASK;
4371 }
4372 if (rdev->irq.crtc_vblank_int[3] ||
4373 atomic_read(&rdev->irq.pflip[3])) {
4374 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
4375 crtc4 |= VBLANK_INT_MASK;
4376 }
4377 if (rdev->irq.crtc_vblank_int[4] ||
4378 atomic_read(&rdev->irq.pflip[4])) {
4379 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
4380 crtc5 |= VBLANK_INT_MASK;
4381 }
4382 if (rdev->irq.crtc_vblank_int[5] ||
4383 atomic_read(&rdev->irq.pflip[5])) {
4384 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
4385 crtc6 |= VBLANK_INT_MASK;
4386 }
4387 if (rdev->irq.hpd[0]) {
4388 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
4389 hpd1 |= DC_HPDx_INT_EN;
4390 }
4391 if (rdev->irq.hpd[1]) {
4392 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
4393 hpd2 |= DC_HPDx_INT_EN;
4394 }
4395 if (rdev->irq.hpd[2]) {
4396 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
4397 hpd3 |= DC_HPDx_INT_EN;
4398 }
4399 if (rdev->irq.hpd[3]) {
4400 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
4401 hpd4 |= DC_HPDx_INT_EN;
4402 }
4403 if (rdev->irq.hpd[4]) {
4404 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
4405 hpd5 |= DC_HPDx_INT_EN;
4406 }
4407 if (rdev->irq.hpd[5]) {
4408 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
4409 hpd6 |= DC_HPDx_INT_EN;
4410 }
4411 if (rdev->irq.afmt[0]) {
4412 DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
4413 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4414 }
4415 if (rdev->irq.afmt[1]) {
4416 DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
4417 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4418 }
4419 if (rdev->irq.afmt[2]) {
4420 DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
4421 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4422 }
4423 if (rdev->irq.afmt[3]) {
4424 DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
4425 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4426 }
4427 if (rdev->irq.afmt[4]) {
4428 DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
4429 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4430 }
4431 if (rdev->irq.afmt[5]) {
4432 DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
4433 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4434 }
4435
4436 if (rdev->family >= CHIP_CAYMAN) {
4437 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4438 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4439 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4440 } else
4441 WREG32(CP_INT_CNTL, cp_int_cntl);
4442
4443 WREG32(DMA_CNTL, dma_cntl);
4444
4445 if (rdev->family >= CHIP_CAYMAN)
4446 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4447
4448 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4449
4450 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4451 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
4452 if (rdev->num_crtc >= 4) {
4453 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4454 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
4455 }
4456 if (rdev->num_crtc >= 6) {
4457 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4458 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4459 }
4460
4461 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
4462 GRPH_PFLIP_INT_MASK);
4463 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
4464 GRPH_PFLIP_INT_MASK);
4465 if (rdev->num_crtc >= 4) {
4466 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
4467 GRPH_PFLIP_INT_MASK);
4468 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
4469 GRPH_PFLIP_INT_MASK);
4470 }
4471 if (rdev->num_crtc >= 6) {
4472 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
4473 GRPH_PFLIP_INT_MASK);
4474 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
4475 GRPH_PFLIP_INT_MASK);
4476 }
4477
4478 WREG32(DC_HPD1_INT_CONTROL, hpd1);
4479 WREG32(DC_HPD2_INT_CONTROL, hpd2);
4480 WREG32(DC_HPD3_INT_CONTROL, hpd3);
4481 WREG32(DC_HPD4_INT_CONTROL, hpd4);
4482 WREG32(DC_HPD5_INT_CONTROL, hpd5);
4483 WREG32(DC_HPD6_INT_CONTROL, hpd6);
4484 if (rdev->family == CHIP_ARUBA)
4485 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4486 else
4487 WREG32(CG_THERMAL_INT, thermal_int);
4488
4489 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
4490 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
4491 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
4492 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
4493 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
4494 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
4495
4496 return 0;
4497 }
4498
4499 static void evergreen_irq_ack(struct radeon_device *rdev)
4500 {
4501 u32 tmp;
4502
4503 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4504 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4505 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4506 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4507 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4508 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4509 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4510 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4511 if (rdev->num_crtc >= 4) {
4512 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4513 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4514 }
4515 if (rdev->num_crtc >= 6) {
4516 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4517 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4518 }
4519
4520 rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4521 rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4522 rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4523 rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4524 rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4525 rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4526
4527 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4528 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4529 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4530 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4531 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
4532 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
4533 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
4534 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
4535 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
4536 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
4537 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
4538 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4539
4540 if (rdev->num_crtc >= 4) {
4541 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4542 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4543 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4544 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4545 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4546 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4547 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4548 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4549 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4550 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4551 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4552 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4553 }
4554
4555 if (rdev->num_crtc >= 6) {
4556 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4557 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4558 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4559 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4560 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4561 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4562 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4563 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4564 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4565 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4566 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4567 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4568 }
4569
4570 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4571 tmp = RREG32(DC_HPD1_INT_CONTROL);
4572 tmp |= DC_HPDx_INT_ACK;
4573 WREG32(DC_HPD1_INT_CONTROL, tmp);
4574 }
4575 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4576 tmp = RREG32(DC_HPD2_INT_CONTROL);
4577 tmp |= DC_HPDx_INT_ACK;
4578 WREG32(DC_HPD2_INT_CONTROL, tmp);
4579 }
4580 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4581 tmp = RREG32(DC_HPD3_INT_CONTROL);
4582 tmp |= DC_HPDx_INT_ACK;
4583 WREG32(DC_HPD3_INT_CONTROL, tmp);
4584 }
4585 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4586 tmp = RREG32(DC_HPD4_INT_CONTROL);
4587 tmp |= DC_HPDx_INT_ACK;
4588 WREG32(DC_HPD4_INT_CONTROL, tmp);
4589 }
4590 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4591 tmp = RREG32(DC_HPD5_INT_CONTROL);
4592 tmp |= DC_HPDx_INT_ACK;
4593 WREG32(DC_HPD5_INT_CONTROL, tmp);
4594 }
4595 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4596 tmp = RREG32(DC_HPD5_INT_CONTROL);
4597 tmp |= DC_HPDx_INT_ACK;
4598 WREG32(DC_HPD6_INT_CONTROL, tmp);
4599 }
4600 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4601 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
4602 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4603 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
4604 }
4605 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4606 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
4607 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4608 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
4609 }
4610 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4611 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
4612 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4613 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
4614 }
4615 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4616 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
4617 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4618 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
4619 }
4620 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4621 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
4622 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4623 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
4624 }
4625 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4626 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
4627 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4628 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
4629 }
4630 }
4631
4632 static void evergreen_irq_disable(struct radeon_device *rdev)
4633 {
4634 r600_disable_interrupts(rdev);
4635 /* Wait and acknowledge irq */
4636 mdelay(1);
4637 evergreen_irq_ack(rdev);
4638 evergreen_disable_interrupt_state(rdev);
4639 }
4640
4641 void evergreen_irq_suspend(struct radeon_device *rdev)
4642 {
4643 evergreen_irq_disable(rdev);
4644 r600_rlc_stop(rdev);
4645 }
4646
4647 static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
4648 {
4649 u32 wptr, tmp;
4650
4651 if (rdev->wb.enabled)
4652 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4653 else
4654 wptr = RREG32(IH_RB_WPTR);
4655
4656 if (wptr & RB_OVERFLOW) {
4657 /* When a ring buffer overflow happen start parsing interrupt
4658 * from the last not overwritten vector (wptr + 16). Hopefully
4659 * this should allow us to catchup.
4660 */
4661 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
4662 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
4663 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4664 tmp = RREG32(IH_RB_CNTL);
4665 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4666 WREG32(IH_RB_CNTL, tmp);
4667 wptr &= ~RB_OVERFLOW;
4668 }
4669 return (wptr & rdev->ih.ptr_mask);
4670 }
4671
4672 int evergreen_irq_process(struct radeon_device *rdev)
4673 {
4674 u32 wptr;
4675 u32 rptr;
4676 u32 src_id, src_data;
4677 u32 ring_index;
4678 bool queue_hotplug = false;
4679 bool queue_hdmi = false;
4680 bool queue_thermal = false;
4681 u32 status, addr;
4682
4683 if (!rdev->ih.enabled || rdev->shutdown)
4684 return IRQ_NONE;
4685
4686 wptr = evergreen_get_ih_wptr(rdev);
4687
4688 restart_ih:
4689 /* is somebody else already processing irqs? */
4690 if (atomic_xchg(&rdev->ih.lock, 1))
4691 return IRQ_NONE;
4692
4693 rptr = rdev->ih.rptr;
4694 DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4695
4696 /* Order reading of wptr vs. reading of IH ring data */
4697 rmb();
4698
4699 /* display interrupts */
4700 evergreen_irq_ack(rdev);
4701
4702 while (rptr != wptr) {
4703 /* wptr/rptr are in bytes! */
4704 ring_index = rptr / 4;
4705 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4706 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4707
4708 switch (src_id) {
4709 case 1: /* D1 vblank/vline */
4710 switch (src_data) {
4711 case 0: /* D1 vblank */
4712 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
4713 if (rdev->irq.crtc_vblank_int[0]) {
4714 drm_handle_vblank(rdev->ddev, 0);
4715 rdev->pm.vblank_sync = true;
4716 wake_up(&rdev->irq.vblank_queue);
4717 }
4718 if (atomic_read(&rdev->irq.pflip[0]))
4719 radeon_crtc_handle_flip(rdev, 0);
4720 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
4721 DRM_DEBUG("IH: D1 vblank\n");
4722 }
4723 break;
4724 case 1: /* D1 vline */
4725 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
4726 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
4727 DRM_DEBUG("IH: D1 vline\n");
4728 }
4729 break;
4730 default:
4731 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4732 break;
4733 }
4734 break;
4735 case 2: /* D2 vblank/vline */
4736 switch (src_data) {
4737 case 0: /* D2 vblank */
4738 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
4739 if (rdev->irq.crtc_vblank_int[1]) {
4740 drm_handle_vblank(rdev->ddev, 1);
4741 rdev->pm.vblank_sync = true;
4742 wake_up(&rdev->irq.vblank_queue);
4743 }
4744 if (atomic_read(&rdev->irq.pflip[1]))
4745 radeon_crtc_handle_flip(rdev, 1);
4746 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
4747 DRM_DEBUG("IH: D2 vblank\n");
4748 }
4749 break;
4750 case 1: /* D2 vline */
4751 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
4752 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
4753 DRM_DEBUG("IH: D2 vline\n");
4754 }
4755 break;
4756 default:
4757 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4758 break;
4759 }
4760 break;
4761 case 3: /* D3 vblank/vline */
4762 switch (src_data) {
4763 case 0: /* D3 vblank */
4764 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
4765 if (rdev->irq.crtc_vblank_int[2]) {
4766 drm_handle_vblank(rdev->ddev, 2);
4767 rdev->pm.vblank_sync = true;
4768 wake_up(&rdev->irq.vblank_queue);
4769 }
4770 if (atomic_read(&rdev->irq.pflip[2]))
4771 radeon_crtc_handle_flip(rdev, 2);
4772 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
4773 DRM_DEBUG("IH: D3 vblank\n");
4774 }
4775 break;
4776 case 1: /* D3 vline */
4777 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
4778 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
4779 DRM_DEBUG("IH: D3 vline\n");
4780 }
4781 break;
4782 default:
4783 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4784 break;
4785 }
4786 break;
4787 case 4: /* D4 vblank/vline */
4788 switch (src_data) {
4789 case 0: /* D4 vblank */
4790 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
4791 if (rdev->irq.crtc_vblank_int[3]) {
4792 drm_handle_vblank(rdev->ddev, 3);
4793 rdev->pm.vblank_sync = true;
4794 wake_up(&rdev->irq.vblank_queue);
4795 }
4796 if (atomic_read(&rdev->irq.pflip[3]))
4797 radeon_crtc_handle_flip(rdev, 3);
4798 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
4799 DRM_DEBUG("IH: D4 vblank\n");
4800 }
4801 break;
4802 case 1: /* D4 vline */
4803 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
4804 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
4805 DRM_DEBUG("IH: D4 vline\n");
4806 }
4807 break;
4808 default:
4809 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4810 break;
4811 }
4812 break;
4813 case 5: /* D5 vblank/vline */
4814 switch (src_data) {
4815 case 0: /* D5 vblank */
4816 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
4817 if (rdev->irq.crtc_vblank_int[4]) {
4818 drm_handle_vblank(rdev->ddev, 4);
4819 rdev->pm.vblank_sync = true;
4820 wake_up(&rdev->irq.vblank_queue);
4821 }
4822 if (atomic_read(&rdev->irq.pflip[4]))
4823 radeon_crtc_handle_flip(rdev, 4);
4824 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
4825 DRM_DEBUG("IH: D5 vblank\n");
4826 }
4827 break;
4828 case 1: /* D5 vline */
4829 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
4830 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
4831 DRM_DEBUG("IH: D5 vline\n");
4832 }
4833 break;
4834 default:
4835 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4836 break;
4837 }
4838 break;
4839 case 6: /* D6 vblank/vline */
4840 switch (src_data) {
4841 case 0: /* D6 vblank */
4842 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
4843 if (rdev->irq.crtc_vblank_int[5]) {
4844 drm_handle_vblank(rdev->ddev, 5);
4845 rdev->pm.vblank_sync = true;
4846 wake_up(&rdev->irq.vblank_queue);
4847 }
4848 if (atomic_read(&rdev->irq.pflip[5]))
4849 radeon_crtc_handle_flip(rdev, 5);
4850 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
4851 DRM_DEBUG("IH: D6 vblank\n");
4852 }
4853 break;
4854 case 1: /* D6 vline */
4855 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
4856 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
4857 DRM_DEBUG("IH: D6 vline\n");
4858 }
4859 break;
4860 default:
4861 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4862 break;
4863 }
4864 break;
4865 case 8: /* D1 page flip */
4866 case 10: /* D2 page flip */
4867 case 12: /* D3 page flip */
4868 case 14: /* D4 page flip */
4869 case 16: /* D5 page flip */
4870 case 18: /* D6 page flip */
4871 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
4872 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
4873 break;
4874 case 42: /* HPD hotplug */
4875 switch (src_data) {
4876 case 0:
4877 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4878 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
4879 queue_hotplug = true;
4880 DRM_DEBUG("IH: HPD1\n");
4881 }
4882 break;
4883 case 1:
4884 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4885 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
4886 queue_hotplug = true;
4887 DRM_DEBUG("IH: HPD2\n");
4888 }
4889 break;
4890 case 2:
4891 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4892 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
4893 queue_hotplug = true;
4894 DRM_DEBUG("IH: HPD3\n");
4895 }
4896 break;
4897 case 3:
4898 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4899 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
4900 queue_hotplug = true;
4901 DRM_DEBUG("IH: HPD4\n");
4902 }
4903 break;
4904 case 4:
4905 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4906 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
4907 queue_hotplug = true;
4908 DRM_DEBUG("IH: HPD5\n");
4909 }
4910 break;
4911 case 5:
4912 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4913 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
4914 queue_hotplug = true;
4915 DRM_DEBUG("IH: HPD6\n");
4916 }
4917 break;
4918 default:
4919 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4920 break;
4921 }
4922 break;
4923 case 44: /* hdmi */
4924 switch (src_data) {
4925 case 0:
4926 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4927 rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
4928 queue_hdmi = true;
4929 DRM_DEBUG("IH: HDMI0\n");
4930 }
4931 break;
4932 case 1:
4933 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4934 rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
4935 queue_hdmi = true;
4936 DRM_DEBUG("IH: HDMI1\n");
4937 }
4938 break;
4939 case 2:
4940 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4941 rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
4942 queue_hdmi = true;
4943 DRM_DEBUG("IH: HDMI2\n");
4944 }
4945 break;
4946 case 3:
4947 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4948 rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
4949 queue_hdmi = true;
4950 DRM_DEBUG("IH: HDMI3\n");
4951 }
4952 break;
4953 case 4:
4954 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4955 rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
4956 queue_hdmi = true;
4957 DRM_DEBUG("IH: HDMI4\n");
4958 }
4959 break;
4960 case 5:
4961 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4962 rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
4963 queue_hdmi = true;
4964 DRM_DEBUG("IH: HDMI5\n");
4965 }
4966 break;
4967 default:
4968 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
4969 break;
4970 }
4971 case 124: /* UVD */
4972 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
4973 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
4974 break;
4975 case 146:
4976 case 147:
4977 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
4978 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
4979 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
4980 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4981 addr);
4982 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4983 status);
4984 cayman_vm_decode_fault(rdev, status, addr);
4985 /* reset addr and status */
4986 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
4987 break;
4988 case 176: /* CP_INT in ring buffer */
4989 case 177: /* CP_INT in IB1 */
4990 case 178: /* CP_INT in IB2 */
4991 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
4992 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4993 break;
4994 case 181: /* CP EOP event */
4995 DRM_DEBUG("IH: CP EOP\n");
4996 if (rdev->family >= CHIP_CAYMAN) {
4997 switch (src_data) {
4998 case 0:
4999 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5000 break;
5001 case 1:
5002 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5003 break;
5004 case 2:
5005 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5006 break;
5007 }
5008 } else
5009 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5010 break;
5011 case 224: /* DMA trap event */
5012 DRM_DEBUG("IH: DMA trap\n");
5013 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
5014 break;
5015 case 230: /* thermal low to high */
5016 DRM_DEBUG("IH: thermal low to high\n");
5017 rdev->pm.dpm.thermal.high_to_low = false;
5018 queue_thermal = true;
5019 break;
5020 case 231: /* thermal high to low */
5021 DRM_DEBUG("IH: thermal high to low\n");
5022 rdev->pm.dpm.thermal.high_to_low = true;
5023 queue_thermal = true;
5024 break;
5025 case 233: /* GUI IDLE */
5026 DRM_DEBUG("IH: GUI idle\n");
5027 break;
5028 case 244: /* DMA trap event */
5029 if (rdev->family >= CHIP_CAYMAN) {
5030 DRM_DEBUG("IH: DMA1 trap\n");
5031 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
5032 }
5033 break;
5034 default:
5035 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5036 break;
5037 }
5038
5039 /* wptr/rptr are in bytes! */
5040 rptr += 16;
5041 rptr &= rdev->ih.ptr_mask;
5042 }
5043 if (queue_hotplug)
5044 schedule_work(&rdev->hotplug_work);
5045 if (queue_hdmi)
5046 schedule_work(&rdev->audio_work);
5047 if (queue_thermal && rdev->pm.dpm_enabled)
5048 schedule_work(&rdev->pm.dpm.thermal.work);
5049 rdev->ih.rptr = rptr;
5050 WREG32(IH_RB_RPTR, rdev->ih.rptr);
5051 atomic_set(&rdev->ih.lock, 0);
5052
5053 /* make sure wptr hasn't changed while processing */
5054 wptr = evergreen_get_ih_wptr(rdev);
5055 if (wptr != rptr)
5056 goto restart_ih;
5057
5058 return IRQ_HANDLED;
5059 }
5060
5061 static int evergreen_startup(struct radeon_device *rdev)
5062 {
5063 struct radeon_ring *ring;
5064 int r;
5065
5066 /* enable pcie gen2 link */
5067 evergreen_pcie_gen2_enable(rdev);
5068 /* enable aspm */
5069 evergreen_program_aspm(rdev);
5070
5071 /* scratch needs to be initialized before MC */
5072 r = r600_vram_scratch_init(rdev);
5073 if (r)
5074 return r;
5075
5076 evergreen_mc_program(rdev);
5077
5078 if (ASIC_IS_DCE5(rdev)) {
5079 r = ni_mc_load_microcode(rdev);
5080 if (r) {
5081 DRM_ERROR("Failed to load MC firmware!\n");
5082 return r;
5083 }
5084 }
5085
5086 if (rdev->flags & RADEON_IS_AGP) {
5087 evergreen_agp_enable(rdev);
5088 } else {
5089 r = evergreen_pcie_gart_enable(rdev);
5090 if (r)
5091 return r;
5092 }
5093 evergreen_gpu_init(rdev);
5094
5095 /* allocate rlc buffers */
5096 if (rdev->flags & RADEON_IS_IGP) {
5097 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5098 rdev->rlc.reg_list_size =
5099 (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
5100 rdev->rlc.cs_data = evergreen_cs_data;
5101 r = sumo_rlc_init(rdev);
5102 if (r) {
5103 DRM_ERROR("Failed to init rlc BOs!\n");
5104 return r;
5105 }
5106 }
5107
5108 /* allocate wb buffer */
5109 r = radeon_wb_init(rdev);
5110 if (r)
5111 return r;
5112
5113 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5114 if (r) {
5115 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5116 return r;
5117 }
5118
5119 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5120 if (r) {
5121 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5122 return r;
5123 }
5124
5125 r = uvd_v2_2_resume(rdev);
5126 if (!r) {
5127 r = radeon_fence_driver_start_ring(rdev,
5128 R600_RING_TYPE_UVD_INDEX);
5129 if (r)
5130 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
5131 }
5132
5133 if (r)
5134 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
5135
5136 /* Enable IRQ */
5137 if (!rdev->irq.installed) {
5138 r = radeon_irq_kms_init(rdev);
5139 if (r)
5140 return r;
5141 }
5142
5143 r = r600_irq_init(rdev);
5144 if (r) {
5145 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5146 radeon_irq_kms_fini(rdev);
5147 return r;
5148 }
5149 evergreen_irq_set(rdev);
5150
5151 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5152 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5153 R600_CP_RB_RPTR, R600_CP_RB_WPTR,
5154 RADEON_CP_PACKET2);
5155 if (r)
5156 return r;
5157
5158 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5159 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5160 DMA_RB_RPTR, DMA_RB_WPTR,
5161 DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5162 if (r)
5163 return r;
5164
5165 r = evergreen_cp_load_microcode(rdev);
5166 if (r)
5167 return r;
5168 r = evergreen_cp_resume(rdev);
5169 if (r)
5170 return r;
5171 r = r600_dma_resume(rdev);
5172 if (r)
5173 return r;
5174
5175 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5176 if (ring->ring_size) {
5177 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
5178 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
5179 RADEON_CP_PACKET2);
5180 if (!r)
5181 r = uvd_v1_0_init(rdev);
5182
5183 if (r)
5184 DRM_ERROR("radeon: error initializing UVD (%d).\n", r);
5185 }
5186
5187 r = radeon_ib_pool_init(rdev);
5188 if (r) {
5189 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
5190 return r;
5191 }
5192
5193 r = r600_audio_init(rdev);
5194 if (r) {
5195 DRM_ERROR("radeon: audio init failed\n");
5196 return r;
5197 }
5198
5199 return 0;
5200 }
5201
5202 int evergreen_resume(struct radeon_device *rdev)
5203 {
5204 int r;
5205
5206 /* reset the asic, the gfx blocks are often in a bad state
5207 * after the driver is unloaded or after a resume
5208 */
5209 if (radeon_asic_reset(rdev))
5210 dev_warn(rdev->dev, "GPU reset failed !\n");
5211 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5212 * posting will perform necessary task to bring back GPU into good
5213 * shape.
5214 */
5215 /* post card */
5216 atom_asic_init(rdev->mode_info.atom_context);
5217
5218 /* init golden registers */
5219 evergreen_init_golden_registers(rdev);
5220
5221 rdev->accel_working = true;
5222 r = evergreen_startup(rdev);
5223 if (r) {
5224 DRM_ERROR("evergreen startup failed on resume\n");
5225 rdev->accel_working = false;
5226 return r;
5227 }
5228
5229 return r;
5230
5231 }
5232
5233 int evergreen_suspend(struct radeon_device *rdev)
5234 {
5235 r600_audio_fini(rdev);
5236 uvd_v1_0_fini(rdev);
5237 radeon_uvd_suspend(rdev);
5238 r700_cp_stop(rdev);
5239 r600_dma_stop(rdev);
5240 evergreen_irq_suspend(rdev);
5241 radeon_wb_disable(rdev);
5242 evergreen_pcie_gart_disable(rdev);
5243
5244 return 0;
5245 }
5246
5247 /* Plan is to move initialization in that function and use
5248 * helper function so that radeon_device_init pretty much
5249 * do nothing more than calling asic specific function. This
5250 * should also allow to remove a bunch of callback function
5251 * like vram_info.
5252 */
5253 int evergreen_init(struct radeon_device *rdev)
5254 {
5255 int r;
5256
5257 /* Read BIOS */
5258 if (!radeon_get_bios(rdev)) {
5259 if (ASIC_IS_AVIVO(rdev))
5260 return -EINVAL;
5261 }
5262 /* Must be an ATOMBIOS */
5263 if (!rdev->is_atom_bios) {
5264 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
5265 return -EINVAL;
5266 }
5267 r = radeon_atombios_init(rdev);
5268 if (r)
5269 return r;
5270 /* reset the asic, the gfx blocks are often in a bad state
5271 * after the driver is unloaded or after a resume
5272 */
5273 if (radeon_asic_reset(rdev))
5274 dev_warn(rdev->dev, "GPU reset failed !\n");
5275 /* Post card if necessary */
5276 if (!radeon_card_posted(rdev)) {
5277 if (!rdev->bios) {
5278 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5279 return -EINVAL;
5280 }
5281 DRM_INFO("GPU not posted. posting now...\n");
5282 atom_asic_init(rdev->mode_info.atom_context);
5283 }
5284 /* init golden registers */
5285 evergreen_init_golden_registers(rdev);
5286 /* Initialize scratch registers */
5287 r600_scratch_init(rdev);
5288 /* Initialize surface registers */
5289 radeon_surface_init(rdev);
5290 /* Initialize clocks */
5291 radeon_get_clock_info(rdev->ddev);
5292 /* Fence driver */
5293 r = radeon_fence_driver_init(rdev);
5294 if (r)
5295 return r;
5296 /* initialize AGP */
5297 if (rdev->flags & RADEON_IS_AGP) {
5298 r = radeon_agp_init(rdev);
5299 if (r)
5300 radeon_agp_disable(rdev);
5301 }
5302 /* initialize memory controller */
5303 r = evergreen_mc_init(rdev);
5304 if (r)
5305 return r;
5306 /* Memory manager */
5307 r = radeon_bo_init(rdev);
5308 if (r)
5309 return r;
5310
5311 if (ASIC_IS_DCE5(rdev)) {
5312 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5313 r = ni_init_microcode(rdev);
5314 if (r) {
5315 DRM_ERROR("Failed to load firmware!\n");
5316 return r;
5317 }
5318 }
5319 } else {
5320 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5321 r = r600_init_microcode(rdev);
5322 if (r) {
5323 DRM_ERROR("Failed to load firmware!\n");
5324 return r;
5325 }
5326 }
5327 }
5328
5329 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5330 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
5331
5332 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5333 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5334
5335 r = radeon_uvd_init(rdev);
5336 if (!r) {
5337 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
5338 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
5339 4096);
5340 }
5341
5342 rdev->ih.ring_obj = NULL;
5343 r600_ih_ring_init(rdev, 64 * 1024);
5344
5345 r = r600_pcie_gart_init(rdev);
5346 if (r)
5347 return r;
5348
5349 rdev->accel_working = true;
5350 r = evergreen_startup(rdev);
5351 if (r) {
5352 dev_err(rdev->dev, "disabling GPU acceleration\n");
5353 r700_cp_fini(rdev);
5354 r600_dma_fini(rdev);
5355 r600_irq_fini(rdev);
5356 if (rdev->flags & RADEON_IS_IGP)
5357 sumo_rlc_fini(rdev);
5358 radeon_wb_fini(rdev);
5359 radeon_ib_pool_fini(rdev);
5360 radeon_irq_kms_fini(rdev);
5361 evergreen_pcie_gart_fini(rdev);
5362 rdev->accel_working = false;
5363 }
5364
5365 /* Don't start up if the MC ucode is missing on BTC parts.
5366 * The default clocks and voltages before the MC ucode
5367 * is loaded are not suffient for advanced operations.
5368 */
5369 if (ASIC_IS_DCE5(rdev)) {
5370 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5371 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5372 return -EINVAL;
5373 }
5374 }
5375
5376 return 0;
5377 }
5378
5379 void evergreen_fini(struct radeon_device *rdev)
5380 {
5381 r600_audio_fini(rdev);
5382 r700_cp_fini(rdev);
5383 r600_dma_fini(rdev);
5384 r600_irq_fini(rdev);
5385 if (rdev->flags & RADEON_IS_IGP)
5386 sumo_rlc_fini(rdev);
5387 radeon_wb_fini(rdev);
5388 radeon_ib_pool_fini(rdev);
5389 radeon_irq_kms_fini(rdev);
5390 uvd_v1_0_fini(rdev);
5391 radeon_uvd_fini(rdev);
5392 evergreen_pcie_gart_fini(rdev);
5393 r600_vram_scratch_fini(rdev);
5394 radeon_gem_fini(rdev);
5395 radeon_fence_driver_fini(rdev);
5396 radeon_agp_fini(rdev);
5397 radeon_bo_fini(rdev);
5398 radeon_atombios_fini(rdev);
5399 kfree(rdev->bios);
5400 rdev->bios = NULL;
5401 }
5402
5403 void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
5404 {
5405 u32 link_width_cntl, speed_cntl;
5406
5407 if (radeon_pcie_gen2 == 0)
5408 return;
5409
5410 if (rdev->flags & RADEON_IS_IGP)
5411 return;
5412
5413 if (!(rdev->flags & RADEON_IS_PCIE))
5414 return;
5415
5416 /* x2 cards have a special sequence */
5417 if (ASIC_IS_X2(rdev))
5418 return;
5419
5420 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5421 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
5422 return;
5423
5424 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5425 if (speed_cntl & LC_CURRENT_DATA_RATE) {
5426 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5427 return;
5428 }
5429
5430 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5431
5432 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5433 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5434
5435 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5436 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5437 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5438
5439 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5440 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
5441 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5442
5443 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5444 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
5445 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5446
5447 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5448 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
5449 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5450
5451 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5452 speed_cntl |= LC_GEN2_EN_STRAP;
5453 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5454
5455 } else {
5456 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5457 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5458 if (1)
5459 link_width_cntl |= LC_UPCONFIGURE_DIS;
5460 else
5461 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5462 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5463 }
5464 }
5465
5466 void evergreen_program_aspm(struct radeon_device *rdev)
5467 {
5468 u32 data, orig;
5469 u32 pcie_lc_cntl, pcie_lc_cntl_old;
5470 bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5471 /* fusion_platform = true
5472 * if the system is a fusion system
5473 * (APU or DGPU in a fusion system).
5474 * todo: check if the system is a fusion platform.
5475 */
5476 bool fusion_platform = false;
5477
5478 if (radeon_aspm == 0)
5479 return;
5480
5481 if (!(rdev->flags & RADEON_IS_PCIE))
5482 return;
5483
5484 switch (rdev->family) {
5485 case CHIP_CYPRESS:
5486 case CHIP_HEMLOCK:
5487 case CHIP_JUNIPER:
5488 case CHIP_REDWOOD:
5489 case CHIP_CEDAR:
5490 case CHIP_SUMO:
5491 case CHIP_SUMO2:
5492 case CHIP_PALM:
5493 case CHIP_ARUBA:
5494 disable_l0s = true;
5495 break;
5496 default:
5497 disable_l0s = false;
5498 break;
5499 }
5500
5501 if (rdev->flags & RADEON_IS_IGP)
5502 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5503
5504 data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5505 if (fusion_platform)
5506 data &= ~MULTI_PIF;
5507 else
5508 data |= MULTI_PIF;
5509 if (data != orig)
5510 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5511
5512 data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5513 if (fusion_platform)
5514 data &= ~MULTI_PIF;
5515 else
5516 data |= MULTI_PIF;
5517 if (data != orig)
5518 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5519
5520 pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5521 pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5522 if (!disable_l0s) {
5523 if (rdev->family >= CHIP_BARTS)
5524 pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5525 else
5526 pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5527 }
5528
5529 if (!disable_l1) {
5530 if (rdev->family >= CHIP_BARTS)
5531 pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5532 else
5533 pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5534
5535 if (!disable_plloff_in_l1) {
5536 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5537 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5538 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5539 if (data != orig)
5540 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5541
5542 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5543 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5544 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5545 if (data != orig)
5546 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5547
5548 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5549 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5550 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5551 if (data != orig)
5552 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5553
5554 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5555 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5556 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5557 if (data != orig)
5558 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5559
5560 if (rdev->family >= CHIP_BARTS) {
5561 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5562 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5563 data |= PLL_RAMP_UP_TIME_0(4);
5564 if (data != orig)
5565 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5566
5567 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5568 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5569 data |= PLL_RAMP_UP_TIME_1(4);
5570 if (data != orig)
5571 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5572
5573 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5574 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5575 data |= PLL_RAMP_UP_TIME_0(4);
5576 if (data != orig)
5577 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5578
5579 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5580 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5581 data |= PLL_RAMP_UP_TIME_1(4);
5582 if (data != orig)
5583 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5584 }
5585
5586 data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5587 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5588 data |= LC_DYN_LANES_PWR_STATE(3);
5589 if (data != orig)
5590 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5591
5592 if (rdev->family >= CHIP_BARTS) {
5593 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5594 data &= ~LS2_EXIT_TIME_MASK;
5595 data |= LS2_EXIT_TIME(1);
5596 if (data != orig)
5597 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5598
5599 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5600 data &= ~LS2_EXIT_TIME_MASK;
5601 data |= LS2_EXIT_TIME(1);
5602 if (data != orig)
5603 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5604 }
5605 }
5606 }
5607
5608 /* evergreen parts only */
5609 if (rdev->family < CHIP_BARTS)
5610 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5611
5612 if (pcie_lc_cntl != pcie_lc_cntl_old)
5613 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5614 }
Linux with added FPC-III support