?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Linux 4.19.133
[linux/fpc-iii.git] / drivers / gpu / drm / msm / adreno / a5xx_gpu.c
blobba6f3c14495c0ceb605276e81edce5ddb055a1c3
1 /* Copyright (c) 2016-2017 The Linux Foundation. All rights reserved.
2 *
3 * This program is free software; you can redistribute it and/or modify
4 * it under the terms of the GNU General Public License version 2 and
5 * only version 2 as published by the Free Software Foundation.
6 *
7 * This program is distributed in the hope that it will be useful,
8 * but WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 * GNU General Public License for more details.
11 *
12 */
13
14 #include <linux/kernel.h>
15 #include <linux/types.h>
16 #include <linux/cpumask.h>
17 #include <linux/qcom_scm.h>
18 #include <linux/dma-mapping.h>
19 #include <linux/of_address.h>
20 #include <linux/soc/qcom/mdt_loader.h>
21 #include <linux/pm_opp.h>
22 #include <linux/nvmem-consumer.h>
23 #include <linux/iopoll.h>
24 #include <linux/slab.h>
25 #include "msm_gem.h"
26 #include "msm_mmu.h"
27 #include "a5xx_gpu.h"
28
29 extern bool hang_debug;
30 static void a5xx_dump(struct msm_gpu *gpu);
31
32 #define GPU_PAS_ID 13
33
34 static int zap_shader_load_mdt(struct msm_gpu *gpu, const char *fwname)
35 {
36 struct device *dev = &gpu->pdev->dev;
37 const struct firmware *fw;
38 struct device_node *np, *mem_np;
39 struct resource r;
40 phys_addr_t mem_phys;
41 ssize_t mem_size;
42 void *mem_region = NULL;
43 int ret;
44
45 if (!IS_ENABLED(CONFIG_ARCH_QCOM))
46 return -EINVAL;
47
48 np = of_get_child_by_name(dev->of_node, "zap-shader");
49 if (!np)
50 return -ENODEV;
51
52 mem_np = of_parse_phandle(np, "memory-region", 0);
53 of_node_put(np);
54 if (!mem_np)
55 return -EINVAL;
56
57 ret = of_address_to_resource(mem_np, 0, &r);
58 of_node_put(mem_np);
59 if (ret)
60 return ret;
61
62 mem_phys = r.start;
63 mem_size = resource_size(&r);
64
65 /* Request the MDT file for the firmware */
66 fw = adreno_request_fw(to_adreno_gpu(gpu), fwname);
67 if (IS_ERR(fw)) {
68 DRM_DEV_ERROR(dev, "Unable to load %s\n", fwname);
69 return PTR_ERR(fw);
70 }
71
72 /* Figure out how much memory we need */
73 mem_size = qcom_mdt_get_size(fw);
74 if (mem_size < 0) {
75 ret = mem_size;
76 goto out;
77 }
78
79 /* Allocate memory for the firmware image */
80 mem_region = memremap(mem_phys, mem_size, MEMREMAP_WC);
81 if (!mem_region) {
82 ret = -ENOMEM;
83 goto out;
84 }
85
86 /*
87 * Load the rest of the MDT
88 *
89 * Note that we could be dealing with two different paths, since
90 * with upstream linux-firmware it would be in a qcom/ subdir..
91 * adreno_request_fw() handles this, but qcom_mdt_load() does
92 * not. But since we've already gotten thru adreno_request_fw()
93 * we know which of the two cases it is:
94 */
95 if (to_adreno_gpu(gpu)->fwloc == FW_LOCATION_LEGACY) {
96 ret = qcom_mdt_load(dev, fw, fwname, GPU_PAS_ID,
97 mem_region, mem_phys, mem_size, NULL);
98 } else {
99 char *newname;
100
101 newname = kasprintf(GFP_KERNEL, "qcom/%s", fwname);
102
103 ret = qcom_mdt_load(dev, fw, newname, GPU_PAS_ID,
104 mem_region, mem_phys, mem_size, NULL);
105 kfree(newname);
106 }
107 if (ret)
108 goto out;
109
110 /* Send the image to the secure world */
111 ret = qcom_scm_pas_auth_and_reset(GPU_PAS_ID);
112 if (ret)
113 DRM_DEV_ERROR(dev, "Unable to authorize the image\n");
114
115 out:
116 if (mem_region)
117 memunmap(mem_region);
118
119 release_firmware(fw);
120
121 return ret;
122 }
123
124 static void a5xx_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
125 {
126 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
127 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
128 uint32_t wptr;
129 unsigned long flags;
130
131 spin_lock_irqsave(&ring->lock, flags);
132
133 /* Copy the shadow to the actual register */
134 ring->cur = ring->next;
135
136 /* Make sure to wrap wptr if we need to */
137 wptr = get_wptr(ring);
138
139 spin_unlock_irqrestore(&ring->lock, flags);
140
141 /* Make sure everything is posted before making a decision */
142 mb();
143
144 /* Update HW if this is the current ring and we are not in preempt */
145 if (a5xx_gpu->cur_ring == ring && !a5xx_in_preempt(a5xx_gpu))
146 gpu_write(gpu, REG_A5XX_CP_RB_WPTR, wptr);
147 }
148
149 static void a5xx_submit_in_rb(struct msm_gpu *gpu, struct msm_gem_submit *submit,
150 struct msm_file_private *ctx)
151 {
152 struct msm_drm_private *priv = gpu->dev->dev_private;
153 struct msm_ringbuffer *ring = submit->ring;
154 struct msm_gem_object *obj;
155 uint32_t *ptr, dwords;
156 unsigned int i;
157
158 for (i = 0; i < submit->nr_cmds; i++) {
159 switch (submit->cmd[i].type) {
160 case MSM_SUBMIT_CMD_IB_TARGET_BUF:
161 break;
162 case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
163 if (priv->lastctx == ctx)
164 break;
165 case MSM_SUBMIT_CMD_BUF:
166 /* copy commands into RB: */
167 obj = submit->bos[submit->cmd[i].idx].obj;
168 dwords = submit->cmd[i].size;
169
170 ptr = msm_gem_get_vaddr(&obj->base);
171
172 /* _get_vaddr() shouldn't fail at this point,
173 * since we've already mapped it once in
174 * submit_reloc()
175 */
176 if (WARN_ON(!ptr))
177 return;
178
179 for (i = 0; i < dwords; i++) {
180 /* normally the OUT_PKTn() would wait
181 * for space for the packet. But since
182 * we just OUT_RING() the whole thing,
183 * need to call adreno_wait_ring()
184 * ourself:
185 */
186 adreno_wait_ring(ring, 1);
187 OUT_RING(ring, ptr[i]);
188 }
189
190 msm_gem_put_vaddr(&obj->base);
191
192 break;
193 }
194 }
195
196 a5xx_flush(gpu, ring);
197 a5xx_preempt_trigger(gpu);
198
199 /* we might not necessarily have a cmd from userspace to
200 * trigger an event to know that submit has completed, so
201 * do this manually:
202 */
203 a5xx_idle(gpu, ring);
204 ring->memptrs->fence = submit->seqno;
205 msm_gpu_retire(gpu);
206 }
207
208 static void a5xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit,
209 struct msm_file_private *ctx)
210 {
211 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
212 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
213 struct msm_drm_private *priv = gpu->dev->dev_private;
214 struct msm_ringbuffer *ring = submit->ring;
215 unsigned int i, ibs = 0;
216
217 if (IS_ENABLED(CONFIG_DRM_MSM_GPU_SUDO) && submit->in_rb) {
218 priv->lastctx = NULL;
219 a5xx_submit_in_rb(gpu, submit, ctx);
220 return;
221 }
222
223 OUT_PKT7(ring, CP_PREEMPT_ENABLE_GLOBAL, 1);
224 OUT_RING(ring, 0x02);
225
226 /* Turn off protected mode to write to special registers */
227 OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1);
228 OUT_RING(ring, 0);
229
230 /* Set the save preemption record for the ring/command */
231 OUT_PKT4(ring, REG_A5XX_CP_CONTEXT_SWITCH_SAVE_ADDR_LO, 2);
232 OUT_RING(ring, lower_32_bits(a5xx_gpu->preempt_iova[submit->ring->id]));
233 OUT_RING(ring, upper_32_bits(a5xx_gpu->preempt_iova[submit->ring->id]));
234
235 /* Turn back on protected mode */
236 OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1);
237 OUT_RING(ring, 1);
238
239 /* Enable local preemption for finegrain preemption */
240 OUT_PKT7(ring, CP_PREEMPT_ENABLE_GLOBAL, 1);
241 OUT_RING(ring, 0x02);
242
243 /* Allow CP_CONTEXT_SWITCH_YIELD packets in the IB2 */
244 OUT_PKT7(ring, CP_YIELD_ENABLE, 1);
245 OUT_RING(ring, 0x02);
246
247 /* Submit the commands */
248 for (i = 0; i < submit->nr_cmds; i++) {
249 switch (submit->cmd[i].type) {
250 case MSM_SUBMIT_CMD_IB_TARGET_BUF:
251 break;
252 case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
253 if (priv->lastctx == ctx)
254 break;
255 case MSM_SUBMIT_CMD_BUF:
256 OUT_PKT7(ring, CP_INDIRECT_BUFFER_PFE, 3);
257 OUT_RING(ring, lower_32_bits(submit->cmd[i].iova));
258 OUT_RING(ring, upper_32_bits(submit->cmd[i].iova));
259 OUT_RING(ring, submit->cmd[i].size);
260 ibs++;
261 break;
262 }
263 }
264
265 /*
266 * Write the render mode to NULL (0) to indicate to the CP that the IBs
267 * are done rendering - otherwise a lucky preemption would start
268 * replaying from the last checkpoint
269 */
270 OUT_PKT7(ring, CP_SET_RENDER_MODE, 5);
271 OUT_RING(ring, 0);
272 OUT_RING(ring, 0);
273 OUT_RING(ring, 0);
274 OUT_RING(ring, 0);
275 OUT_RING(ring, 0);
276
277 /* Turn off IB level preemptions */
278 OUT_PKT7(ring, CP_YIELD_ENABLE, 1);
279 OUT_RING(ring, 0x01);
280
281 /* Write the fence to the scratch register */
282 OUT_PKT4(ring, REG_A5XX_CP_SCRATCH_REG(2), 1);
283 OUT_RING(ring, submit->seqno);
284
285 /*
286 * Execute a CACHE_FLUSH_TS event. This will ensure that the
287 * timestamp is written to the memory and then triggers the interrupt
288 */
289 OUT_PKT7(ring, CP_EVENT_WRITE, 4);
290 OUT_RING(ring, CACHE_FLUSH_TS | (1 << 31));
291 OUT_RING(ring, lower_32_bits(rbmemptr(ring, fence)));
292 OUT_RING(ring, upper_32_bits(rbmemptr(ring, fence)));
293 OUT_RING(ring, submit->seqno);
294
295 /* Yield the floor on command completion */
296 OUT_PKT7(ring, CP_CONTEXT_SWITCH_YIELD, 4);
297 /*
298 * If dword[2:1] are non zero, they specify an address for the CP to
299 * write the value of dword[3] to on preemption complete. Write 0 to
300 * skip the write
301 */
302 OUT_RING(ring, 0x00);
303 OUT_RING(ring, 0x00);
304 /* Data value - not used if the address above is 0 */
305 OUT_RING(ring, 0x01);
306 /* Set bit 0 to trigger an interrupt on preempt complete */
307 OUT_RING(ring, 0x01);
308
309 a5xx_flush(gpu, ring);
310
311 /* Check to see if we need to start preemption */
312 a5xx_preempt_trigger(gpu);
313 }
314
315 static const struct {
316 u32 offset;
317 u32 value;
318 } a5xx_hwcg[] = {
319 {REG_A5XX_RBBM_CLOCK_CNTL_SP0, 0x02222222},
320 {REG_A5XX_RBBM_CLOCK_CNTL_SP1, 0x02222222},
321 {REG_A5XX_RBBM_CLOCK_CNTL_SP2, 0x02222222},
322 {REG_A5XX_RBBM_CLOCK_CNTL_SP3, 0x02222222},
323 {REG_A5XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220},
324 {REG_A5XX_RBBM_CLOCK_CNTL2_SP1, 0x02222220},
325 {REG_A5XX_RBBM_CLOCK_CNTL2_SP2, 0x02222220},
326 {REG_A5XX_RBBM_CLOCK_CNTL2_SP3, 0x02222220},
327 {REG_A5XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF},
328 {REG_A5XX_RBBM_CLOCK_HYST_SP1, 0x0000F3CF},
329 {REG_A5XX_RBBM_CLOCK_HYST_SP2, 0x0000F3CF},
330 {REG_A5XX_RBBM_CLOCK_HYST_SP3, 0x0000F3CF},
331 {REG_A5XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
332 {REG_A5XX_RBBM_CLOCK_DELAY_SP1, 0x00000080},
333 {REG_A5XX_RBBM_CLOCK_DELAY_SP2, 0x00000080},
334 {REG_A5XX_RBBM_CLOCK_DELAY_SP3, 0x00000080},
335 {REG_A5XX_RBBM_CLOCK_CNTL_TP0, 0x22222222},
336 {REG_A5XX_RBBM_CLOCK_CNTL_TP1, 0x22222222},
337 {REG_A5XX_RBBM_CLOCK_CNTL_TP2, 0x22222222},
338 {REG_A5XX_RBBM_CLOCK_CNTL_TP3, 0x22222222},
339 {REG_A5XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
340 {REG_A5XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222},
341 {REG_A5XX_RBBM_CLOCK_CNTL2_TP2, 0x22222222},
342 {REG_A5XX_RBBM_CLOCK_CNTL2_TP3, 0x22222222},
343 {REG_A5XX_RBBM_CLOCK_CNTL3_TP0, 0x00002222},
344 {REG_A5XX_RBBM_CLOCK_CNTL3_TP1, 0x00002222},
345 {REG_A5XX_RBBM_CLOCK_CNTL3_TP2, 0x00002222},
346 {REG_A5XX_RBBM_CLOCK_CNTL3_TP3, 0x00002222},
347 {REG_A5XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
348 {REG_A5XX_RBBM_CLOCK_HYST_TP1, 0x77777777},
349 {REG_A5XX_RBBM_CLOCK_HYST_TP2, 0x77777777},
350 {REG_A5XX_RBBM_CLOCK_HYST_TP3, 0x77777777},
351 {REG_A5XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
352 {REG_A5XX_RBBM_CLOCK_HYST2_TP1, 0x77777777},
353 {REG_A5XX_RBBM_CLOCK_HYST2_TP2, 0x77777777},
354 {REG_A5XX_RBBM_CLOCK_HYST2_TP3, 0x77777777},
355 {REG_A5XX_RBBM_CLOCK_HYST3_TP0, 0x00007777},
356 {REG_A5XX_RBBM_CLOCK_HYST3_TP1, 0x00007777},
357 {REG_A5XX_RBBM_CLOCK_HYST3_TP2, 0x00007777},
358 {REG_A5XX_RBBM_CLOCK_HYST3_TP3, 0x00007777},
359 {REG_A5XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
360 {REG_A5XX_RBBM_CLOCK_DELAY_TP1, 0x11111111},
361 {REG_A5XX_RBBM_CLOCK_DELAY_TP2, 0x11111111},
362 {REG_A5XX_RBBM_CLOCK_DELAY_TP3, 0x11111111},
363 {REG_A5XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
364 {REG_A5XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111},
365 {REG_A5XX_RBBM_CLOCK_DELAY2_TP2, 0x11111111},
366 {REG_A5XX_RBBM_CLOCK_DELAY2_TP3, 0x11111111},
367 {REG_A5XX_RBBM_CLOCK_DELAY3_TP0, 0x00001111},
368 {REG_A5XX_RBBM_CLOCK_DELAY3_TP1, 0x00001111},
369 {REG_A5XX_RBBM_CLOCK_DELAY3_TP2, 0x00001111},
370 {REG_A5XX_RBBM_CLOCK_DELAY3_TP3, 0x00001111},
371 {REG_A5XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
372 {REG_A5XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222},
373 {REG_A5XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222},
374 {REG_A5XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222},
375 {REG_A5XX_RBBM_CLOCK_HYST_UCHE, 0x00444444},
376 {REG_A5XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
377 {REG_A5XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
378 {REG_A5XX_RBBM_CLOCK_CNTL_RB1, 0x22222222},
379 {REG_A5XX_RBBM_CLOCK_CNTL_RB2, 0x22222222},
380 {REG_A5XX_RBBM_CLOCK_CNTL_RB3, 0x22222222},
381 {REG_A5XX_RBBM_CLOCK_CNTL2_RB0, 0x00222222},
382 {REG_A5XX_RBBM_CLOCK_CNTL2_RB1, 0x00222222},
383 {REG_A5XX_RBBM_CLOCK_CNTL2_RB2, 0x00222222},
384 {REG_A5XX_RBBM_CLOCK_CNTL2_RB3, 0x00222222},
385 {REG_A5XX_RBBM_CLOCK_CNTL_CCU0, 0x00022220},
386 {REG_A5XX_RBBM_CLOCK_CNTL_CCU1, 0x00022220},
387 {REG_A5XX_RBBM_CLOCK_CNTL_CCU2, 0x00022220},
388 {REG_A5XX_RBBM_CLOCK_CNTL_CCU3, 0x00022220},
389 {REG_A5XX_RBBM_CLOCK_CNTL_RAC, 0x05522222},
390 {REG_A5XX_RBBM_CLOCK_CNTL2_RAC, 0x00505555},
391 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU0, 0x04040404},
392 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU1, 0x04040404},
393 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU2, 0x04040404},
394 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU3, 0x04040404},
395 {REG_A5XX_RBBM_CLOCK_HYST_RAC, 0x07444044},
396 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0, 0x00000002},
397 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_1, 0x00000002},
398 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_2, 0x00000002},
399 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_3, 0x00000002},
400 {REG_A5XX_RBBM_CLOCK_DELAY_RAC, 0x00010011},
401 {REG_A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
402 {REG_A5XX_RBBM_CLOCK_MODE_GPC, 0x02222222},
403 {REG_A5XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
404 {REG_A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
405 {REG_A5XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
406 {REG_A5XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
407 {REG_A5XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
408 {REG_A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
409 {REG_A5XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
410 {REG_A5XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}
411 };
412
413 void a5xx_set_hwcg(struct msm_gpu *gpu, bool state)
414 {
415 unsigned int i;
416
417 for (i = 0; i < ARRAY_SIZE(a5xx_hwcg); i++)
418 gpu_write(gpu, a5xx_hwcg[i].offset,
419 state ? a5xx_hwcg[i].value : 0);
420
421 gpu_write(gpu, REG_A5XX_RBBM_CLOCK_CNTL, state ? 0xAAA8AA00 : 0);
422 gpu_write(gpu, REG_A5XX_RBBM_ISDB_CNT, state ? 0x182 : 0x180);
423 }
424
425 static int a5xx_me_init(struct msm_gpu *gpu)
426 {
427 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
428 struct msm_ringbuffer *ring = gpu->rb[0];
429
430 OUT_PKT7(ring, CP_ME_INIT, 8);
431
432 OUT_RING(ring, 0x0000002F);
433
434 /* Enable multiple hardware contexts */
435 OUT_RING(ring, 0x00000003);
436
437 /* Enable error detection */
438 OUT_RING(ring, 0x20000000);
439
440 /* Don't enable header dump */
441 OUT_RING(ring, 0x00000000);
442 OUT_RING(ring, 0x00000000);
443
444 /* Specify workarounds for various microcode issues */
445 if (adreno_is_a530(adreno_gpu)) {
446 /* Workaround for token end syncs
447 * Force a WFI after every direct-render 3D mode draw and every
448 * 2D mode 3 draw
449 */
450 OUT_RING(ring, 0x0000000B);
451 } else {
452 /* No workarounds enabled */
453 OUT_RING(ring, 0x00000000);
454 }
455
456 OUT_RING(ring, 0x00000000);
457 OUT_RING(ring, 0x00000000);
458
459 gpu->funcs->flush(gpu, ring);
460 return a5xx_idle(gpu, ring) ? 0 : -EINVAL;
461 }
462
463 static int a5xx_preempt_start(struct msm_gpu *gpu)
464 {
465 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
466 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
467 struct msm_ringbuffer *ring = gpu->rb[0];
468
469 if (gpu->nr_rings == 1)
470 return 0;
471
472 /* Turn off protected mode to write to special registers */
473 OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1);
474 OUT_RING(ring, 0);
475
476 /* Set the save preemption record for the ring/command */
477 OUT_PKT4(ring, REG_A5XX_CP_CONTEXT_SWITCH_SAVE_ADDR_LO, 2);
478 OUT_RING(ring, lower_32_bits(a5xx_gpu->preempt_iova[ring->id]));
479 OUT_RING(ring, upper_32_bits(a5xx_gpu->preempt_iova[ring->id]));
480
481 /* Turn back on protected mode */
482 OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1);
483 OUT_RING(ring, 1);
484
485 OUT_PKT7(ring, CP_PREEMPT_ENABLE_GLOBAL, 1);
486 OUT_RING(ring, 0x00);
487
488 OUT_PKT7(ring, CP_PREEMPT_ENABLE_LOCAL, 1);
489 OUT_RING(ring, 0x01);
490
491 OUT_PKT7(ring, CP_YIELD_ENABLE, 1);
492 OUT_RING(ring, 0x01);
493
494 /* Yield the floor on command completion */
495 OUT_PKT7(ring, CP_CONTEXT_SWITCH_YIELD, 4);
496 OUT_RING(ring, 0x00);
497 OUT_RING(ring, 0x00);
498 OUT_RING(ring, 0x01);
499 OUT_RING(ring, 0x01);
500
501 gpu->funcs->flush(gpu, ring);
502
503 return a5xx_idle(gpu, ring) ? 0 : -EINVAL;
504 }
505
506 static int a5xx_ucode_init(struct msm_gpu *gpu)
507 {
508 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
509 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
510 int ret;
511
512 if (!a5xx_gpu->pm4_bo) {
513 a5xx_gpu->pm4_bo = adreno_fw_create_bo(gpu,
514 adreno_gpu->fw[ADRENO_FW_PM4], &a5xx_gpu->pm4_iova);
515
516 if (IS_ERR(a5xx_gpu->pm4_bo)) {
517 ret = PTR_ERR(a5xx_gpu->pm4_bo);
518 a5xx_gpu->pm4_bo = NULL;
519 dev_err(gpu->dev->dev, "could not allocate PM4: %d\n",
520 ret);
521 return ret;
522 }
523 }
524
525 if (!a5xx_gpu->pfp_bo) {
526 a5xx_gpu->pfp_bo = adreno_fw_create_bo(gpu,
527 adreno_gpu->fw[ADRENO_FW_PFP], &a5xx_gpu->pfp_iova);
528
529 if (IS_ERR(a5xx_gpu->pfp_bo)) {
530 ret = PTR_ERR(a5xx_gpu->pfp_bo);
531 a5xx_gpu->pfp_bo = NULL;
532 dev_err(gpu->dev->dev, "could not allocate PFP: %d\n",
533 ret);
534 return ret;
535 }
536 }
537
538 gpu_write64(gpu, REG_A5XX_CP_ME_INSTR_BASE_LO,
539 REG_A5XX_CP_ME_INSTR_BASE_HI, a5xx_gpu->pm4_iova);
540
541 gpu_write64(gpu, REG_A5XX_CP_PFP_INSTR_BASE_LO,
542 REG_A5XX_CP_PFP_INSTR_BASE_HI, a5xx_gpu->pfp_iova);
543
544 return 0;
545 }
546
547 #define SCM_GPU_ZAP_SHADER_RESUME 0
548
549 static int a5xx_zap_shader_resume(struct msm_gpu *gpu)
550 {
551 int ret;
552
553 ret = qcom_scm_set_remote_state(SCM_GPU_ZAP_SHADER_RESUME, GPU_PAS_ID);
554 if (ret)
555 DRM_ERROR("%s: zap-shader resume failed: %d\n",
556 gpu->name, ret);
557
558 return ret;
559 }
560
561 static int a5xx_zap_shader_init(struct msm_gpu *gpu)
562 {
563 static bool loaded;
564 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
565 struct platform_device *pdev = gpu->pdev;
566 int ret;
567
568 /*
569 * If the zap shader is already loaded into memory we just need to kick
570 * the remote processor to reinitialize it
571 */
572 if (loaded)
573 return a5xx_zap_shader_resume(gpu);
574
575 /* We need SCM to be able to load the firmware */
576 if (!qcom_scm_is_available()) {
577 DRM_DEV_ERROR(&pdev->dev, "SCM is not available\n");
578 return -EPROBE_DEFER;
579 }
580
581 /* Each GPU has a target specific zap shader firmware name to use */
582 if (!adreno_gpu->info->zapfw) {
583 DRM_DEV_ERROR(&pdev->dev,
584 "Zap shader firmware file not specified for this target\n");
585 return -ENODEV;
586 }
587
588 ret = zap_shader_load_mdt(gpu, adreno_gpu->info->zapfw);
589
590 loaded = !ret;
591
592 return ret;
593 }
594
595 #define A5XX_INT_MASK (A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR | \
596 A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT | \
597 A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT | \
598 A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT | \
599 A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT | \
600 A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW | \
601 A5XX_RBBM_INT_0_MASK_CP_HW_ERROR | \
602 A5XX_RBBM_INT_0_MASK_MISC_HANG_DETECT | \
603 A5XX_RBBM_INT_0_MASK_CP_SW | \
604 A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS | \
605 A5XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS | \
606 A5XX_RBBM_INT_0_MASK_GPMU_VOLTAGE_DROOP)
607
608 static int a5xx_hw_init(struct msm_gpu *gpu)
609 {
610 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
611 int ret;
612
613 gpu_write(gpu, REG_A5XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x00000003);
614
615 /* Make all blocks contribute to the GPU BUSY perf counter */
616 gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_GPU_BUSY_MASKED, 0xFFFFFFFF);
617
618 /* Enable RBBM error reporting bits */
619 gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL0, 0x00000001);
620
621 if (adreno_gpu->info->quirks & ADRENO_QUIRK_FAULT_DETECT_MASK) {
622 /*
623 * Mask out the activity signals from RB1-3 to avoid false
624 * positives
625 */
626
627 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL11,
628 0xF0000000);
629 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL12,
630 0xFFFFFFFF);
631 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL13,
632 0xFFFFFFFF);
633 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL14,
634 0xFFFFFFFF);
635 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL15,
636 0xFFFFFFFF);
637 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL16,
638 0xFFFFFFFF);
639 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL17,
640 0xFFFFFFFF);
641 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL18,
642 0xFFFFFFFF);
643 }
644
645 /* Enable fault detection */
646 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_INT_CNTL,
647 (1 << 30) | 0xFFFF);
648
649 /* Turn on performance counters */
650 gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_CNTL, 0x01);
651
652 /* Select CP0 to always count cycles */
653 gpu_write(gpu, REG_A5XX_CP_PERFCTR_CP_SEL_0, PERF_CP_ALWAYS_COUNT);
654
655 /* Select RBBM0 to countable 6 to get the busy status for devfreq */
656 gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_RBBM_SEL_0, 6);
657
658 /* Increase VFD cache access so LRZ and other data gets evicted less */
659 gpu_write(gpu, REG_A5XX_UCHE_CACHE_WAYS, 0x02);
660
661 /* Disable L2 bypass in the UCHE */
662 gpu_write(gpu, REG_A5XX_UCHE_TRAP_BASE_LO, 0xFFFF0000);
663 gpu_write(gpu, REG_A5XX_UCHE_TRAP_BASE_HI, 0x0001FFFF);
664 gpu_write(gpu, REG_A5XX_UCHE_WRITE_THRU_BASE_LO, 0xFFFF0000);
665 gpu_write(gpu, REG_A5XX_UCHE_WRITE_THRU_BASE_HI, 0x0001FFFF);
666
667 /* Set the GMEM VA range (0 to gpu->gmem) */
668 gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MIN_LO, 0x00100000);
669 gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MIN_HI, 0x00000000);
670 gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MAX_LO,
671 0x00100000 + adreno_gpu->gmem - 1);
672 gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MAX_HI, 0x00000000);
673
674 gpu_write(gpu, REG_A5XX_CP_MEQ_THRESHOLDS, 0x40);
675 gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x40);
676 gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_2, 0x80000060);
677 gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_1, 0x40201B16);
678
679 gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL, (0x400 << 11 | 0x300 << 22));
680
681 if (adreno_gpu->info->quirks & ADRENO_QUIRK_TWO_PASS_USE_WFI)
682 gpu_rmw(gpu, REG_A5XX_PC_DBG_ECO_CNTL, 0, (1 << 8));
683
684 gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL, 0xc0200100);
685
686 /* Enable USE_RETENTION_FLOPS */
687 gpu_write(gpu, REG_A5XX_CP_CHICKEN_DBG, 0x02000000);
688
689 /* Enable ME/PFP split notification */
690 gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL1, 0xA6FFFFFF);
691
692 /* Enable HWCG */
693 a5xx_set_hwcg(gpu, true);
694
695 gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL2, 0x0000003F);
696
697 /* Set the highest bank bit */
698 gpu_write(gpu, REG_A5XX_TPL1_MODE_CNTL, 2 << 7);
699 gpu_write(gpu, REG_A5XX_RB_MODE_CNTL, 2 << 1);
700
701 /* Protect registers from the CP */
702 gpu_write(gpu, REG_A5XX_CP_PROTECT_CNTL, 0x00000007);
703
704 /* RBBM */
705 gpu_write(gpu, REG_A5XX_CP_PROTECT(0), ADRENO_PROTECT_RW(0x04, 4));
706 gpu_write(gpu, REG_A5XX_CP_PROTECT(1), ADRENO_PROTECT_RW(0x08, 8));
707 gpu_write(gpu, REG_A5XX_CP_PROTECT(2), ADRENO_PROTECT_RW(0x10, 16));
708 gpu_write(gpu, REG_A5XX_CP_PROTECT(3), ADRENO_PROTECT_RW(0x20, 32));
709 gpu_write(gpu, REG_A5XX_CP_PROTECT(4), ADRENO_PROTECT_RW(0x40, 64));
710 gpu_write(gpu, REG_A5XX_CP_PROTECT(5), ADRENO_PROTECT_RW(0x80, 64));
711
712 /* Content protect */
713 gpu_write(gpu, REG_A5XX_CP_PROTECT(6),
714 ADRENO_PROTECT_RW(REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO,
715 16));
716 gpu_write(gpu, REG_A5XX_CP_PROTECT(7),
717 ADRENO_PROTECT_RW(REG_A5XX_RBBM_SECVID_TRUST_CNTL, 2));
718
719 /* CP */
720 gpu_write(gpu, REG_A5XX_CP_PROTECT(8), ADRENO_PROTECT_RW(0x800, 64));
721 gpu_write(gpu, REG_A5XX_CP_PROTECT(9), ADRENO_PROTECT_RW(0x840, 8));
722 gpu_write(gpu, REG_A5XX_CP_PROTECT(10), ADRENO_PROTECT_RW(0x880, 32));
723 gpu_write(gpu, REG_A5XX_CP_PROTECT(11), ADRENO_PROTECT_RW(0xAA0, 1));
724
725 /* RB */
726 gpu_write(gpu, REG_A5XX_CP_PROTECT(12), ADRENO_PROTECT_RW(0xCC0, 1));
727 gpu_write(gpu, REG_A5XX_CP_PROTECT(13), ADRENO_PROTECT_RW(0xCF0, 2));
728
729 /* VPC */
730 gpu_write(gpu, REG_A5XX_CP_PROTECT(14), ADRENO_PROTECT_RW(0xE68, 8));
731 gpu_write(gpu, REG_A5XX_CP_PROTECT(15), ADRENO_PROTECT_RW(0xE70, 4));
732
733 /* UCHE */
734 gpu_write(gpu, REG_A5XX_CP_PROTECT(16), ADRENO_PROTECT_RW(0xE80, 16));
735
736 if (adreno_is_a530(adreno_gpu))
737 gpu_write(gpu, REG_A5XX_CP_PROTECT(17),
738 ADRENO_PROTECT_RW(0x10000, 0x8000));
739
740 gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_CNTL, 0);
741 /*
742 * Disable the trusted memory range - we don't actually supported secure
743 * memory rendering at this point in time and we don't want to block off
744 * part of the virtual memory space.
745 */
746 gpu_write64(gpu, REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO,
747 REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_HI, 0x00000000);
748 gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_TRUSTED_SIZE, 0x00000000);
749
750 ret = adreno_hw_init(gpu);
751 if (ret)
752 return ret;
753
754 a5xx_preempt_hw_init(gpu);
755
756 a5xx_gpmu_ucode_init(gpu);
757
758 ret = a5xx_ucode_init(gpu);
759 if (ret)
760 return ret;
761
762 /* Disable the interrupts through the initial bringup stage */
763 gpu_write(gpu, REG_A5XX_RBBM_INT_0_MASK, A5XX_INT_MASK);
764
765 /* Clear ME_HALT to start the micro engine */
766 gpu_write(gpu, REG_A5XX_CP_PFP_ME_CNTL, 0);
767 ret = a5xx_me_init(gpu);
768 if (ret)
769 return ret;
770
771 ret = a5xx_power_init(gpu);
772 if (ret)
773 return ret;
774
775 /*
776 * Send a pipeline event stat to get misbehaving counters to start
777 * ticking correctly
778 */
779 if (adreno_is_a530(adreno_gpu)) {
780 OUT_PKT7(gpu->rb[0], CP_EVENT_WRITE, 1);
781 OUT_RING(gpu->rb[0], 0x0F);
782
783 gpu->funcs->flush(gpu, gpu->rb[0]);
784 if (!a5xx_idle(gpu, gpu->rb[0]))
785 return -EINVAL;
786 }
787
788 /*
789 * Try to load a zap shader into the secure world. If successful
790 * we can use the CP to switch out of secure mode. If not then we
791 * have no resource but to try to switch ourselves out manually. If we
792 * guessed wrong then access to the RBBM_SECVID_TRUST_CNTL register will
793 * be blocked and a permissions violation will soon follow.
794 */
795 ret = a5xx_zap_shader_init(gpu);
796 if (!ret) {
797 OUT_PKT7(gpu->rb[0], CP_SET_SECURE_MODE, 1);
798 OUT_RING(gpu->rb[0], 0x00000000);
799
800 gpu->funcs->flush(gpu, gpu->rb[0]);
801 if (!a5xx_idle(gpu, gpu->rb[0]))
802 return -EINVAL;
803 } else {
804 /* Print a warning so if we die, we know why */
805 dev_warn_once(gpu->dev->dev,
806 "Zap shader not enabled - using SECVID_TRUST_CNTL instead\n");
807 gpu_write(gpu, REG_A5XX_RBBM_SECVID_TRUST_CNTL, 0x0);
808 }
809
810 /* Last step - yield the ringbuffer */
811 a5xx_preempt_start(gpu);
812
813 return 0;
814 }
815
816 static void a5xx_recover(struct msm_gpu *gpu)
817 {
818 int i;
819
820 adreno_dump_info(gpu);
821
822 for (i = 0; i < 8; i++) {
823 printk("CP_SCRATCH_REG%d: %u\n", i,
824 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(i)));
825 }
826
827 if (hang_debug)
828 a5xx_dump(gpu);
829
830 gpu_write(gpu, REG_A5XX_RBBM_SW_RESET_CMD, 1);
831 gpu_read(gpu, REG_A5XX_RBBM_SW_RESET_CMD);
832 gpu_write(gpu, REG_A5XX_RBBM_SW_RESET_CMD, 0);
833 adreno_recover(gpu);
834 }
835
836 static void a5xx_destroy(struct msm_gpu *gpu)
837 {
838 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
839 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
840
841 DBG("%s", gpu->name);
842
843 a5xx_preempt_fini(gpu);
844
845 if (a5xx_gpu->pm4_bo) {
846 if (a5xx_gpu->pm4_iova)
847 msm_gem_put_iova(a5xx_gpu->pm4_bo, gpu->aspace);
848 drm_gem_object_put_unlocked(a5xx_gpu->pm4_bo);
849 }
850
851 if (a5xx_gpu->pfp_bo) {
852 if (a5xx_gpu->pfp_iova)
853 msm_gem_put_iova(a5xx_gpu->pfp_bo, gpu->aspace);
854 drm_gem_object_put_unlocked(a5xx_gpu->pfp_bo);
855 }
856
857 if (a5xx_gpu->gpmu_bo) {
858 if (a5xx_gpu->gpmu_iova)
859 msm_gem_put_iova(a5xx_gpu->gpmu_bo, gpu->aspace);
860 drm_gem_object_put_unlocked(a5xx_gpu->gpmu_bo);
861 }
862
863 adreno_gpu_cleanup(adreno_gpu);
864 kfree(a5xx_gpu);
865 }
866
867 static inline bool _a5xx_check_idle(struct msm_gpu *gpu)
868 {
869 if (gpu_read(gpu, REG_A5XX_RBBM_STATUS) & ~A5XX_RBBM_STATUS_HI_BUSY)
870 return false;
871
872 /*
873 * Nearly every abnormality ends up pausing the GPU and triggering a
874 * fault so we can safely just watch for this one interrupt to fire
875 */
876 return !(gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS) &
877 A5XX_RBBM_INT_0_MASK_MISC_HANG_DETECT);
878 }
879
880 bool a5xx_idle(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
881 {
882 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
883 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
884
885 if (ring != a5xx_gpu->cur_ring) {
886 WARN(1, "Tried to idle a non-current ringbuffer\n");
887 return false;
888 }
889
890 /* wait for CP to drain ringbuffer: */
891 if (!adreno_idle(gpu, ring))
892 return false;
893
894 if (spin_until(_a5xx_check_idle(gpu))) {
895 DRM_ERROR("%s: %ps: timeout waiting for GPU to idle: status %8.8X irq %8.8X rptr/wptr %d/%d\n",
896 gpu->name, __builtin_return_address(0),
897 gpu_read(gpu, REG_A5XX_RBBM_STATUS),
898 gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS),
899 gpu_read(gpu, REG_A5XX_CP_RB_RPTR),
900 gpu_read(gpu, REG_A5XX_CP_RB_WPTR));
901 return false;
902 }
903
904 return true;
905 }
906
907 static int a5xx_fault_handler(void *arg, unsigned long iova, int flags)
908 {
909 struct msm_gpu *gpu = arg;
910 pr_warn_ratelimited("*** gpu fault: iova=%08lx, flags=%d (%u,%u,%u,%u)\n",
911 iova, flags,
912 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(4)),
913 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(5)),
914 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(6)),
915 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(7)));
916
917 return -EFAULT;
918 }
919
920 static void a5xx_cp_err_irq(struct msm_gpu *gpu)
921 {
922 u32 status = gpu_read(gpu, REG_A5XX_CP_INTERRUPT_STATUS);
923
924 if (status & A5XX_CP_INT_CP_OPCODE_ERROR) {
925 u32 val;
926
927 gpu_write(gpu, REG_A5XX_CP_PFP_STAT_ADDR, 0);
928
929 /*
930 * REG_A5XX_CP_PFP_STAT_DATA is indexed, and we want index 1 so
931 * read it twice
932 */
933
934 gpu_read(gpu, REG_A5XX_CP_PFP_STAT_DATA);
935 val = gpu_read(gpu, REG_A5XX_CP_PFP_STAT_DATA);
936
937 dev_err_ratelimited(gpu->dev->dev, "CP | opcode error | possible opcode=0x%8.8X\n",
938 val);
939 }
940
941 if (status & A5XX_CP_INT_CP_HW_FAULT_ERROR)
942 dev_err_ratelimited(gpu->dev->dev, "CP | HW fault | status=0x%8.8X\n",
943 gpu_read(gpu, REG_A5XX_CP_HW_FAULT));
944
945 if (status & A5XX_CP_INT_CP_DMA_ERROR)
946 dev_err_ratelimited(gpu->dev->dev, "CP | DMA error\n");
947
948 if (status & A5XX_CP_INT_CP_REGISTER_PROTECTION_ERROR) {
949 u32 val = gpu_read(gpu, REG_A5XX_CP_PROTECT_STATUS);
950
951 dev_err_ratelimited(gpu->dev->dev,
952 "CP | protected mode error | %s | addr=0x%8.8X | status=0x%8.8X\n",
953 val & (1 << 24) ? "WRITE" : "READ",
954 (val & 0xFFFFF) >> 2, val);
955 }
956
957 if (status & A5XX_CP_INT_CP_AHB_ERROR) {
958 u32 status = gpu_read(gpu, REG_A5XX_CP_AHB_FAULT);
959 const char *access[16] = { "reserved", "reserved",
960 "timestamp lo", "timestamp hi", "pfp read", "pfp write",
961 "", "", "me read", "me write", "", "", "crashdump read",
962 "crashdump write" };
963
964 dev_err_ratelimited(gpu->dev->dev,
965 "CP | AHB error | addr=%X access=%s error=%d | status=0x%8.8X\n",
966 status & 0xFFFFF, access[(status >> 24) & 0xF],
967 (status & (1 << 31)), status);
968 }
969 }
970
971 static void a5xx_rbbm_err_irq(struct msm_gpu *gpu, u32 status)
972 {
973 if (status & A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR) {
974 u32 val = gpu_read(gpu, REG_A5XX_RBBM_AHB_ERROR_STATUS);
975
976 dev_err_ratelimited(gpu->dev->dev,
977 "RBBM | AHB bus error | %s | addr=0x%X | ports=0x%X:0x%X\n",
978 val & (1 << 28) ? "WRITE" : "READ",
979 (val & 0xFFFFF) >> 2, (val >> 20) & 0x3,
980 (val >> 24) & 0xF);
981
982 /* Clear the error */
983 gpu_write(gpu, REG_A5XX_RBBM_AHB_CMD, (1 << 4));
984
985 /* Clear the interrupt */
986 gpu_write(gpu, REG_A5XX_RBBM_INT_CLEAR_CMD,
987 A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR);
988 }
989
990 if (status & A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT)
991 dev_err_ratelimited(gpu->dev->dev, "RBBM | AHB transfer timeout\n");
992
993 if (status & A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT)
994 dev_err_ratelimited(gpu->dev->dev, "RBBM | ME master split | status=0x%X\n",
995 gpu_read(gpu, REG_A5XX_RBBM_AHB_ME_SPLIT_STATUS));
996
997 if (status & A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT)
998 dev_err_ratelimited(gpu->dev->dev, "RBBM | PFP master split | status=0x%X\n",
999 gpu_read(gpu, REG_A5XX_RBBM_AHB_PFP_SPLIT_STATUS));
1000
1001 if (status & A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT)
1002 dev_err_ratelimited(gpu->dev->dev, "RBBM | ETS master split | status=0x%X\n",
1003 gpu_read(gpu, REG_A5XX_RBBM_AHB_ETS_SPLIT_STATUS));
1004
1005 if (status & A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW)
1006 dev_err_ratelimited(gpu->dev->dev, "RBBM | ATB ASYNC overflow\n");
1007
1008 if (status & A5XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW)
1009 dev_err_ratelimited(gpu->dev->dev, "RBBM | ATB bus overflow\n");
1010 }
1011
1012 static void a5xx_uche_err_irq(struct msm_gpu *gpu)
1013 {
1014 uint64_t addr = (uint64_t) gpu_read(gpu, REG_A5XX_UCHE_TRAP_LOG_HI);
1015
1016 addr |= gpu_read(gpu, REG_A5XX_UCHE_TRAP_LOG_LO);
1017
1018 dev_err_ratelimited(gpu->dev->dev, "UCHE | Out of bounds access | addr=0x%llX\n",
1019 addr);
1020 }
1021
1022 static void a5xx_gpmu_err_irq(struct msm_gpu *gpu)
1023 {
1024 dev_err_ratelimited(gpu->dev->dev, "GPMU | voltage droop\n");
1025 }
1026
1027 static void a5xx_fault_detect_irq(struct msm_gpu *gpu)
1028 {
1029 struct drm_device *dev = gpu->dev;
1030 struct msm_drm_private *priv = dev->dev_private;
1031 struct msm_ringbuffer *ring = gpu->funcs->active_ring(gpu);
1032
1033 dev_err(dev->dev, "gpu fault ring %d fence %x status %8.8X rb %4.4x/%4.4x ib1 %16.16llX/%4.4x ib2 %16.16llX/%4.4x\n",
1034 ring ? ring->id : -1, ring ? ring->seqno : 0,
1035 gpu_read(gpu, REG_A5XX_RBBM_STATUS),
1036 gpu_read(gpu, REG_A5XX_CP_RB_RPTR),
1037 gpu_read(gpu, REG_A5XX_CP_RB_WPTR),
1038 gpu_read64(gpu, REG_A5XX_CP_IB1_BASE, REG_A5XX_CP_IB1_BASE_HI),
1039 gpu_read(gpu, REG_A5XX_CP_IB1_BUFSZ),
1040 gpu_read64(gpu, REG_A5XX_CP_IB2_BASE, REG_A5XX_CP_IB2_BASE_HI),
1041 gpu_read(gpu, REG_A5XX_CP_IB2_BUFSZ));
1042
1043 /* Turn off the hangcheck timer to keep it from bothering us */
1044 del_timer(&gpu->hangcheck_timer);
1045
1046 queue_work(priv->wq, &gpu->recover_work);
1047 }
1048
1049 #define RBBM_ERROR_MASK \
1050 (A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR | \
1051 A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT | \
1052 A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT | \
1053 A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT | \
1054 A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT | \
1055 A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW)
1056
1057 static irqreturn_t a5xx_irq(struct msm_gpu *gpu)
1058 {
1059 u32 status = gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS);
1060
1061 /*
1062 * Clear all the interrupts except RBBM_AHB_ERROR - if we clear it
1063 * before the source is cleared the interrupt will storm.
1064 */
1065 gpu_write(gpu, REG_A5XX_RBBM_INT_CLEAR_CMD,
1066 status & ~A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR);
1067
1068 /* Pass status to a5xx_rbbm_err_irq because we've already cleared it */
1069 if (status & RBBM_ERROR_MASK)
1070 a5xx_rbbm_err_irq(gpu, status);
1071
1072 if (status & A5XX_RBBM_INT_0_MASK_CP_HW_ERROR)
1073 a5xx_cp_err_irq(gpu);
1074
1075 if (status & A5XX_RBBM_INT_0_MASK_MISC_HANG_DETECT)
1076 a5xx_fault_detect_irq(gpu);
1077
1078 if (status & A5XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS)
1079 a5xx_uche_err_irq(gpu);
1080
1081 if (status & A5XX_RBBM_INT_0_MASK_GPMU_VOLTAGE_DROOP)
1082 a5xx_gpmu_err_irq(gpu);
1083
1084 if (status & A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS) {
1085 a5xx_preempt_trigger(gpu);
1086 msm_gpu_retire(gpu);
1087 }
1088
1089 if (status & A5XX_RBBM_INT_0_MASK_CP_SW)
1090 a5xx_preempt_irq(gpu);
1091
1092 return IRQ_HANDLED;
1093 }
1094
1095 static const u32 a5xx_register_offsets[REG_ADRENO_REGISTER_MAX] = {
1096 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_BASE, REG_A5XX_CP_RB_BASE),
1097 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_BASE_HI, REG_A5XX_CP_RB_BASE_HI),
1098 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR_ADDR, REG_A5XX_CP_RB_RPTR_ADDR),
1099 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR_ADDR_HI,
1100 REG_A5XX_CP_RB_RPTR_ADDR_HI),
1101 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR, REG_A5XX_CP_RB_RPTR),
1102 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_WPTR, REG_A5XX_CP_RB_WPTR),
1103 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_CNTL, REG_A5XX_CP_RB_CNTL),
1104 };
1105
1106 static const u32 a5xx_registers[] = {
1107 0x0000, 0x0002, 0x0004, 0x0020, 0x0022, 0x0026, 0x0029, 0x002B,
1108 0x002E, 0x0035, 0x0038, 0x0042, 0x0044, 0x0044, 0x0047, 0x0095,
1109 0x0097, 0x00BB, 0x03A0, 0x0464, 0x0469, 0x046F, 0x04D2, 0x04D3,
1110 0x04E0, 0x0533, 0x0540, 0x0555, 0x0800, 0x081A, 0x081F, 0x0841,
1111 0x0860, 0x0860, 0x0880, 0x08A0, 0x0B00, 0x0B12, 0x0B15, 0x0B28,
1112 0x0B78, 0x0B7F, 0x0BB0, 0x0BBD, 0x0BC0, 0x0BC6, 0x0BD0, 0x0C53,
1113 0x0C60, 0x0C61, 0x0C80, 0x0C82, 0x0C84, 0x0C85, 0x0C90, 0x0C98,
1114 0x0CA0, 0x0CA0, 0x0CB0, 0x0CB2, 0x2180, 0x2185, 0x2580, 0x2585,
1115 0x0CC1, 0x0CC1, 0x0CC4, 0x0CC7, 0x0CCC, 0x0CCC, 0x0CD0, 0x0CD8,
1116 0x0CE0, 0x0CE5, 0x0CE8, 0x0CE8, 0x0CEC, 0x0CF1, 0x0CFB, 0x0D0E,
1117 0x2100, 0x211E, 0x2140, 0x2145, 0x2500, 0x251E, 0x2540, 0x2545,
1118 0x0D10, 0x0D17, 0x0D20, 0x0D23, 0x0D30, 0x0D30, 0x20C0, 0x20C0,
1119 0x24C0, 0x24C0, 0x0E40, 0x0E43, 0x0E4A, 0x0E4A, 0x0E50, 0x0E57,
1120 0x0E60, 0x0E7C, 0x0E80, 0x0E8E, 0x0E90, 0x0E96, 0x0EA0, 0x0EA8,
1121 0x0EB0, 0x0EB2, 0xE140, 0xE147, 0xE150, 0xE187, 0xE1A0, 0xE1A9,
1122 0xE1B0, 0xE1B6, 0xE1C0, 0xE1C7, 0xE1D0, 0xE1D1, 0xE200, 0xE201,
1123 0xE210, 0xE21C, 0xE240, 0xE268, 0xE000, 0xE006, 0xE010, 0xE09A,
1124 0xE0A0, 0xE0A4, 0xE0AA, 0xE0EB, 0xE100, 0xE105, 0xE380, 0xE38F,
1125 0xE3B0, 0xE3B0, 0xE400, 0xE405, 0xE408, 0xE4E9, 0xE4F0, 0xE4F0,
1126 0xE280, 0xE280, 0xE282, 0xE2A3, 0xE2A5, 0xE2C2, 0xE940, 0xE947,
1127 0xE950, 0xE987, 0xE9A0, 0xE9A9, 0xE9B0, 0xE9B6, 0xE9C0, 0xE9C7,
1128 0xE9D0, 0xE9D1, 0xEA00, 0xEA01, 0xEA10, 0xEA1C, 0xEA40, 0xEA68,
1129 0xE800, 0xE806, 0xE810, 0xE89A, 0xE8A0, 0xE8A4, 0xE8AA, 0xE8EB,
1130 0xE900, 0xE905, 0xEB80, 0xEB8F, 0xEBB0, 0xEBB0, 0xEC00, 0xEC05,
1131 0xEC08, 0xECE9, 0xECF0, 0xECF0, 0xEA80, 0xEA80, 0xEA82, 0xEAA3,
1132 0xEAA5, 0xEAC2, 0xA800, 0xA800, 0xA820, 0xA828, 0xA840, 0xA87D,
1133 0XA880, 0xA88D, 0xA890, 0xA8A3, 0xA8D0, 0xA8D8, 0xA8E0, 0xA8F5,
1134 0xAC60, 0xAC60, ~0,
1135 };
1136
1137 static void a5xx_dump(struct msm_gpu *gpu)
1138 {
1139 dev_info(gpu->dev->dev, "status: %08x\n",
1140 gpu_read(gpu, REG_A5XX_RBBM_STATUS));
1141 adreno_dump(gpu);
1142 }
1143
1144 static int a5xx_pm_resume(struct msm_gpu *gpu)
1145 {
1146 int ret;
1147
1148 /* Turn on the core power */
1149 ret = msm_gpu_pm_resume(gpu);
1150 if (ret)
1151 return ret;
1152
1153 /* Turn the RBCCU domain first to limit the chances of voltage droop */
1154 gpu_write(gpu, REG_A5XX_GPMU_RBCCU_POWER_CNTL, 0x778000);
1155
1156 /* Wait 3 usecs before polling */
1157 udelay(3);
1158
1159 ret = spin_usecs(gpu, 20, REG_A5XX_GPMU_RBCCU_PWR_CLK_STATUS,
1160 (1 << 20), (1 << 20));
1161 if (ret) {
1162 DRM_ERROR("%s: timeout waiting for RBCCU GDSC enable: %X\n",
1163 gpu->name,
1164 gpu_read(gpu, REG_A5XX_GPMU_RBCCU_PWR_CLK_STATUS));
1165 return ret;
1166 }
1167
1168 /* Turn on the SP domain */
1169 gpu_write(gpu, REG_A5XX_GPMU_SP_POWER_CNTL, 0x778000);
1170 ret = spin_usecs(gpu, 20, REG_A5XX_GPMU_SP_PWR_CLK_STATUS,
1171 (1 << 20), (1 << 20));
1172 if (ret)
1173 DRM_ERROR("%s: timeout waiting for SP GDSC enable\n",
1174 gpu->name);
1175
1176 return ret;
1177 }
1178
1179 static int a5xx_pm_suspend(struct msm_gpu *gpu)
1180 {
1181 /* Clear the VBIF pipe before shutting down */
1182 gpu_write(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL0, 0xF);
1183 spin_until((gpu_read(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL1) & 0xF) == 0xF);
1184
1185 gpu_write(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL0, 0);
1186
1187 /*
1188 * Reset the VBIF before power collapse to avoid issue with FIFO
1189 * entries
1190 */
1191 gpu_write(gpu, REG_A5XX_RBBM_BLOCK_SW_RESET_CMD, 0x003C0000);
1192 gpu_write(gpu, REG_A5XX_RBBM_BLOCK_SW_RESET_CMD, 0x00000000);
1193
1194 return msm_gpu_pm_suspend(gpu);
1195 }
1196
1197 static int a5xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value)
1198 {
1199 *value = gpu_read64(gpu, REG_A5XX_RBBM_PERFCTR_CP_0_LO,
1200 REG_A5XX_RBBM_PERFCTR_CP_0_HI);
1201
1202 return 0;
1203 }
1204
1205 struct a5xx_crashdumper {
1206 void *ptr;
1207 struct drm_gem_object *bo;
1208 u64 iova;
1209 };
1210
1211 struct a5xx_gpu_state {
1212 struct msm_gpu_state base;
1213 u32 *hlsqregs;
1214 };
1215
1216 #define gpu_poll_timeout(gpu, addr, val, cond, interval, timeout) \
1217 readl_poll_timeout((gpu)->mmio + ((addr) << 2), val, cond, \
1218 interval, timeout)
1219
1220 static int a5xx_crashdumper_init(struct msm_gpu *gpu,
1221 struct a5xx_crashdumper *dumper)
1222 {
1223 dumper->ptr = msm_gem_kernel_new_locked(gpu->dev,
1224 SZ_1M, MSM_BO_UNCACHED, gpu->aspace,
1225 &dumper->bo, &dumper->iova);
1226
1227 if (IS_ERR(dumper->ptr))
1228 return PTR_ERR(dumper->ptr);
1229
1230 return 0;
1231 }
1232
1233 static void a5xx_crashdumper_free(struct msm_gpu *gpu,
1234 struct a5xx_crashdumper *dumper)
1235 {
1236 msm_gem_put_iova(dumper->bo, gpu->aspace);
1237 msm_gem_put_vaddr(dumper->bo);
1238
1239 drm_gem_object_unreference(dumper->bo);
1240 }
1241
1242 static int a5xx_crashdumper_run(struct msm_gpu *gpu,
1243 struct a5xx_crashdumper *dumper)
1244 {
1245 u32 val;
1246
1247 if (IS_ERR_OR_NULL(dumper->ptr))
1248 return -EINVAL;
1249
1250 gpu_write64(gpu, REG_A5XX_CP_CRASH_SCRIPT_BASE_LO,
1251 REG_A5XX_CP_CRASH_SCRIPT_BASE_HI, dumper->iova);
1252
1253 gpu_write(gpu, REG_A5XX_CP_CRASH_DUMP_CNTL, 1);
1254
1255 return gpu_poll_timeout(gpu, REG_A5XX_CP_CRASH_DUMP_CNTL, val,
1256 val & 0x04, 100, 10000);
1257 }
1258
1259 /*
1260 * These are a list of the registers that need to be read through the HLSQ
1261 * aperture through the crashdumper. These are not nominally accessible from
1262 * the CPU on a secure platform.
1263 */
1264 static const struct {
1265 u32 type;
1266 u32 regoffset;
1267 u32 count;
1268 } a5xx_hlsq_aperture_regs[] = {
1269 { 0x35, 0xe00, 0x32 }, /* HSLQ non-context */
1270 { 0x31, 0x2080, 0x1 }, /* HLSQ 2D context 0 */
1271 { 0x33, 0x2480, 0x1 }, /* HLSQ 2D context 1 */
1272 { 0x32, 0xe780, 0x62 }, /* HLSQ 3D context 0 */
1273 { 0x34, 0xef80, 0x62 }, /* HLSQ 3D context 1 */
1274 { 0x3f, 0x0ec0, 0x40 }, /* SP non-context */
1275 { 0x3d, 0x2040, 0x1 }, /* SP 2D context 0 */
1276 { 0x3b, 0x2440, 0x1 }, /* SP 2D context 1 */
1277 { 0x3e, 0xe580, 0x170 }, /* SP 3D context 0 */
1278 { 0x3c, 0xed80, 0x170 }, /* SP 3D context 1 */
1279 { 0x3a, 0x0f00, 0x1c }, /* TP non-context */
1280 { 0x38, 0x2000, 0xa }, /* TP 2D context 0 */
1281 { 0x36, 0x2400, 0xa }, /* TP 2D context 1 */
1282 { 0x39, 0xe700, 0x80 }, /* TP 3D context 0 */
1283 { 0x37, 0xef00, 0x80 }, /* TP 3D context 1 */
1284 };
1285
1286 static void a5xx_gpu_state_get_hlsq_regs(struct msm_gpu *gpu,
1287 struct a5xx_gpu_state *a5xx_state)
1288 {
1289 struct a5xx_crashdumper dumper = { 0 };
1290 u32 offset, count = 0;
1291 u64 *ptr;
1292 int i;
1293
1294 if (a5xx_crashdumper_init(gpu, &dumper))
1295 return;
1296
1297 /* The script will be written at offset 0 */
1298 ptr = dumper.ptr;
1299
1300 /* Start writing the data at offset 256k */
1301 offset = dumper.iova + (256 * SZ_1K);
1302
1303 /* Count how many additional registers to get from the HLSQ aperture */
1304 for (i = 0; i < ARRAY_SIZE(a5xx_hlsq_aperture_regs); i++)
1305 count += a5xx_hlsq_aperture_regs[i].count;
1306
1307 a5xx_state->hlsqregs = kcalloc(count, sizeof(u32), GFP_KERNEL);
1308 if (!a5xx_state->hlsqregs)
1309 return;
1310
1311 /* Build the crashdump script */
1312 for (i = 0; i < ARRAY_SIZE(a5xx_hlsq_aperture_regs); i++) {
1313 u32 type = a5xx_hlsq_aperture_regs[i].type;
1314 u32 c = a5xx_hlsq_aperture_regs[i].count;
1315
1316 /* Write the register to select the desired bank */
1317 *ptr++ = ((u64) type << 8);
1318 *ptr++ = (((u64) REG_A5XX_HLSQ_DBG_READ_SEL) << 44) |
1319 (1 << 21) | 1;
1320
1321 *ptr++ = offset;
1322 *ptr++ = (((u64) REG_A5XX_HLSQ_DBG_AHB_READ_APERTURE) << 44)
1323 | c;
1324
1325 offset += c * sizeof(u32);
1326 }
1327
1328 /* Write two zeros to close off the script */
1329 *ptr++ = 0;
1330 *ptr++ = 0;
1331
1332 if (a5xx_crashdumper_run(gpu, &dumper)) {
1333 kfree(a5xx_state->hlsqregs);
1334 a5xx_crashdumper_free(gpu, &dumper);
1335 return;
1336 }
1337
1338 /* Copy the data from the crashdumper to the state */
1339 memcpy(a5xx_state->hlsqregs, dumper.ptr + (256 * SZ_1K),
1340 count * sizeof(u32));
1341
1342 a5xx_crashdumper_free(gpu, &dumper);
1343 }
1344
1345 static struct msm_gpu_state *a5xx_gpu_state_get(struct msm_gpu *gpu)
1346 {
1347 struct a5xx_gpu_state *a5xx_state = kzalloc(sizeof(*a5xx_state),
1348 GFP_KERNEL);
1349
1350 if (!a5xx_state)
1351 return ERR_PTR(-ENOMEM);
1352
1353 /* Temporarily disable hardware clock gating before reading the hw */
1354 a5xx_set_hwcg(gpu, false);
1355
1356 /* First get the generic state from the adreno core */
1357 adreno_gpu_state_get(gpu, &(a5xx_state->base));
1358
1359 a5xx_state->base.rbbm_status = gpu_read(gpu, REG_A5XX_RBBM_STATUS);
1360
1361 /* Get the HLSQ regs with the help of the crashdumper */
1362 a5xx_gpu_state_get_hlsq_regs(gpu, a5xx_state);
1363
1364 a5xx_set_hwcg(gpu, true);
1365
1366 return &a5xx_state->base;
1367 }
1368
1369 static void a5xx_gpu_state_destroy(struct kref *kref)
1370 {
1371 struct msm_gpu_state *state = container_of(kref,
1372 struct msm_gpu_state, ref);
1373 struct a5xx_gpu_state *a5xx_state = container_of(state,
1374 struct a5xx_gpu_state, base);
1375
1376 kfree(a5xx_state->hlsqregs);
1377
1378 adreno_gpu_state_destroy(state);
1379 kfree(a5xx_state);
1380 }
1381
1382 int a5xx_gpu_state_put(struct msm_gpu_state *state)
1383 {
1384 if (IS_ERR_OR_NULL(state))
1385 return 1;
1386
1387 return kref_put(&state->ref, a5xx_gpu_state_destroy);
1388 }
1389
1390
1391 #if defined(CONFIG_DEBUG_FS) || defined(CONFIG_DEV_COREDUMP)
1392 void a5xx_show(struct msm_gpu *gpu, struct msm_gpu_state *state,
1393 struct drm_printer *p)
1394 {
1395 int i, j;
1396 u32 pos = 0;
1397 struct a5xx_gpu_state *a5xx_state = container_of(state,
1398 struct a5xx_gpu_state, base);
1399
1400 if (IS_ERR_OR_NULL(state))
1401 return;
1402
1403 adreno_show(gpu, state, p);
1404
1405 /* Dump the additional a5xx HLSQ registers */
1406 if (!a5xx_state->hlsqregs)
1407 return;
1408
1409 drm_printf(p, "registers-hlsq:\n");
1410
1411 for (i = 0; i < ARRAY_SIZE(a5xx_hlsq_aperture_regs); i++) {
1412 u32 o = a5xx_hlsq_aperture_regs[i].regoffset;
1413 u32 c = a5xx_hlsq_aperture_regs[i].count;
1414
1415 for (j = 0; j < c; j++, pos++, o++) {
1416 /*
1417 * To keep the crashdump simple we pull the entire range
1418 * for each register type but not all of the registers
1419 * in the range are valid. Fortunately invalid registers
1420 * stick out like a sore thumb with a value of
1421 * 0xdeadbeef
1422 */
1423 if (a5xx_state->hlsqregs[pos] == 0xdeadbeef)
1424 continue;
1425
1426 drm_printf(p, " - { offset: 0x%04x, value: 0x%08x }\n",
1427 o << 2, a5xx_state->hlsqregs[pos]);
1428 }
1429 }
1430 }
1431 #endif
1432
1433 static struct msm_ringbuffer *a5xx_active_ring(struct msm_gpu *gpu)
1434 {
1435 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1436 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
1437
1438 return a5xx_gpu->cur_ring;
1439 }
1440
1441 static int a5xx_gpu_busy(struct msm_gpu *gpu, uint64_t *value)
1442 {
1443 *value = gpu_read64(gpu, REG_A5XX_RBBM_PERFCTR_RBBM_0_LO,
1444 REG_A5XX_RBBM_PERFCTR_RBBM_0_HI);
1445
1446 return 0;
1447 }
1448
1449 static const struct adreno_gpu_funcs funcs = {
1450 .base = {
1451 .get_param = adreno_get_param,
1452 .hw_init = a5xx_hw_init,
1453 .pm_suspend = a5xx_pm_suspend,
1454 .pm_resume = a5xx_pm_resume,
1455 .recover = a5xx_recover,
1456 .submit = a5xx_submit,
1457 .flush = a5xx_flush,
1458 .active_ring = a5xx_active_ring,
1459 .irq = a5xx_irq,
1460 .destroy = a5xx_destroy,
1461 #if defined(CONFIG_DEBUG_FS) || defined(CONFIG_DEV_COREDUMP)
1462 .show = a5xx_show,
1463 #endif
1464 #if defined(CONFIG_DEBUG_FS)
1465 .debugfs_init = a5xx_debugfs_init,
1466 #endif
1467 .gpu_busy = a5xx_gpu_busy,
1468 .gpu_state_get = a5xx_gpu_state_get,
1469 .gpu_state_put = a5xx_gpu_state_put,
1470 },
1471 .get_timestamp = a5xx_get_timestamp,
1472 };
1473
1474 static void check_speed_bin(struct device *dev)
1475 {
1476 struct nvmem_cell *cell;
1477 u32 bin, val;
1478
1479 cell = nvmem_cell_get(dev, "speed_bin");
1480
1481 /* If a nvmem cell isn't defined, nothing to do */
1482 if (IS_ERR(cell))
1483 return;
1484
1485 bin = *((u32 *) nvmem_cell_read(cell, NULL));
1486 nvmem_cell_put(cell);
1487
1488 val = (1 << bin);
1489
1490 dev_pm_opp_set_supported_hw(dev, &val, 1);
1491 }
1492
1493 struct msm_gpu *a5xx_gpu_init(struct drm_device *dev)
1494 {
1495 struct msm_drm_private *priv = dev->dev_private;
1496 struct platform_device *pdev = priv->gpu_pdev;
1497 struct a5xx_gpu *a5xx_gpu = NULL;
1498 struct adreno_gpu *adreno_gpu;
1499 struct msm_gpu *gpu;
1500 int ret;
1501
1502 if (!pdev) {
1503 dev_err(dev->dev, "No A5XX device is defined\n");
1504 return ERR_PTR(-ENXIO);
1505 }
1506
1507 a5xx_gpu = kzalloc(sizeof(*a5xx_gpu), GFP_KERNEL);
1508 if (!a5xx_gpu)
1509 return ERR_PTR(-ENOMEM);
1510
1511 adreno_gpu = &a5xx_gpu->base;
1512 gpu = &adreno_gpu->base;
1513
1514 adreno_gpu->registers = a5xx_registers;
1515 adreno_gpu->reg_offsets = a5xx_register_offsets;
1516
1517 a5xx_gpu->lm_leakage = 0x4E001A;
1518
1519 check_speed_bin(&pdev->dev);
1520
1521 ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs, 4);
1522 if (ret) {
1523 a5xx_destroy(&(a5xx_gpu->base.base));
1524 return ERR_PTR(ret);
1525 }
1526
1527 if (gpu->aspace)
1528 msm_mmu_set_fault_handler(gpu->aspace->mmu, gpu, a5xx_fault_handler);
1529
1530 /* Set up the preemption specific bits and pieces for each ringbuffer */
1531 a5xx_preempt_init(gpu);
1532
1533 return gpu;
1534 }
Linux with added FPC-III support