2 * Copyright (C) 2007 Ben Skeggs.
6 * Permission is hereby granted, free of charge, to any person obtaining
7 * a copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sublicense, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial
16 * portions of the Software.
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
21 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
22 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
23 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
24 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
30 #include "nouveau_drv.h"
32 struct nv50_instmem_priv
{
33 uint32_t save1700
[5]; /* 0x1700->0x1710 */
35 struct nouveau_gpuobj_ref
*pramin_pt
;
36 struct nouveau_gpuobj_ref
*pramin_bar
;
37 struct nouveau_gpuobj_ref
*fb_bar
;
42 #define NV50_INSTMEM_PAGE_SHIFT 12
43 #define NV50_INSTMEM_PAGE_SIZE (1 << NV50_INSTMEM_PAGE_SHIFT)
44 #define NV50_INSTMEM_PT_SIZE(a) (((a) >> 12) << 3)
46 /*NOTE: - Assumes 0x1700 already covers the correct MiB of PRAMIN
48 #define BAR0_WI32(g, o, v) do { \
50 if ((g)->im_backing) { \
51 offset = (g)->im_backing_start; \
53 offset = chan->ramin->gpuobj->im_backing_start; \
54 offset += (g)->im_pramin->start; \
57 nv_wr32(dev, NV_RAMIN + (offset & 0xfffff), (v)); \
61 nv50_instmem_init(struct drm_device
*dev
)
63 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
64 struct nouveau_channel
*chan
;
65 uint32_t c_offset
, c_size
, c_ramfc
, c_vmpd
, c_base
, pt_size
;
66 uint32_t save_nv001700
;
68 struct nv50_instmem_priv
*priv
;
71 priv
= kzalloc(sizeof(*priv
), GFP_KERNEL
);
74 dev_priv
->engine
.instmem
.priv
= priv
;
76 /* Save state, will restore at takedown. */
77 for (i
= 0x1700; i
<= 0x1710; i
+= 4)
78 priv
->save1700
[(i
-0x1700)/4] = nv_rd32(dev
, i
);
80 /* Reserve the last MiB of VRAM, we should probably try to avoid
81 * setting up the below tables over the top of the VBIOS image at
84 dev_priv
->ramin_rsvd_vram
= 1 << 20;
85 c_offset
= dev_priv
->vram_size
- dev_priv
->ramin_rsvd_vram
;
87 c_vmpd
= ((dev_priv
->chipset
& 0xf0) == 0x50) ? 0x1400 : 0x200;
88 c_ramfc
= ((dev_priv
->chipset
& 0xf0) == 0x50) ? 0x0 : 0x20;
89 c_base
= c_vmpd
+ 0x4000;
90 pt_size
= NV50_INSTMEM_PT_SIZE(dev_priv
->ramin_size
);
92 NV_DEBUG(dev
, " Rsvd VRAM base: 0x%08x\n", c_offset
);
93 NV_DEBUG(dev
, " VBIOS image: 0x%08x\n",
94 (nv_rd32(dev
, 0x619f04) & ~0xff) << 8);
95 NV_DEBUG(dev
, " Aperture size: %d MiB\n", dev_priv
->ramin_size
>> 20);
96 NV_DEBUG(dev
, " PT size: %d KiB\n", pt_size
>> 10);
98 /* Determine VM layout, we need to do this first to make sure
99 * we allocate enough memory for all the page tables.
101 dev_priv
->vm_gart_base
= roundup(NV50_VM_BLOCK
, NV50_VM_BLOCK
);
102 dev_priv
->vm_gart_size
= NV50_VM_BLOCK
;
104 dev_priv
->vm_vram_base
= dev_priv
->vm_gart_base
+ dev_priv
->vm_gart_size
;
105 dev_priv
->vm_vram_size
= dev_priv
->vram_size
;
106 if (dev_priv
->vm_vram_size
> NV50_VM_MAX_VRAM
)
107 dev_priv
->vm_vram_size
= NV50_VM_MAX_VRAM
;
108 dev_priv
->vm_vram_size
= roundup(dev_priv
->vm_vram_size
, NV50_VM_BLOCK
);
109 dev_priv
->vm_vram_pt_nr
= dev_priv
->vm_vram_size
/ NV50_VM_BLOCK
;
111 dev_priv
->vm_end
= dev_priv
->vm_vram_base
+ dev_priv
->vm_vram_size
;
113 NV_DEBUG(dev
, "NV50VM: GART 0x%016llx-0x%016llx\n",
114 dev_priv
->vm_gart_base
,
115 dev_priv
->vm_gart_base
+ dev_priv
->vm_gart_size
- 1);
116 NV_DEBUG(dev
, "NV50VM: VRAM 0x%016llx-0x%016llx\n",
117 dev_priv
->vm_vram_base
,
118 dev_priv
->vm_vram_base
+ dev_priv
->vm_vram_size
- 1);
120 c_size
+= dev_priv
->vm_vram_pt_nr
* (NV50_VM_BLOCK
/ 65536 * 8);
122 /* Map BAR0 PRAMIN aperture over the memory we want to use */
123 save_nv001700
= nv_rd32(dev
, NV50_PUNK_BAR0_PRAMIN
);
124 nv_wr32(dev
, NV50_PUNK_BAR0_PRAMIN
, (c_offset
>> 16));
126 /* Create a fake channel, and use it as our "dummy" channels 0/127.
127 * The main reason for creating a channel is so we can use the gpuobj
128 * code. However, it's probably worth noting that NVIDIA also setup
129 * their channels 0/127 with the same values they configure here.
130 * So, there may be some other reason for doing this.
132 * Have to create the entire channel manually, as the real channel
133 * creation code assumes we have PRAMIN access, and we don't until
136 chan
= kzalloc(sizeof(*chan
), GFP_KERNEL
);
141 chan
->file_priv
= (struct drm_file
*)-2;
142 dev_priv
->fifos
[0] = dev_priv
->fifos
[127] = chan
;
144 /* Channel's PRAMIN object + heap */
145 ret
= nouveau_gpuobj_new_fake(dev
, 0, c_offset
, c_size
, 0,
150 if (nouveau_mem_init_heap(&chan
->ramin_heap
, c_base
, c_size
- c_base
))
153 /* RAMFC + zero channel's PRAMIN up to start of VM pagedir */
154 ret
= nouveau_gpuobj_new_fake(dev
, c_ramfc
, c_offset
+ c_ramfc
,
155 0x4000, 0, NULL
, &chan
->ramfc
);
159 for (i
= 0; i
< c_vmpd
; i
+= 4)
160 BAR0_WI32(chan
->ramin
->gpuobj
, i
, 0);
162 /* VM page directory */
163 ret
= nouveau_gpuobj_new_fake(dev
, c_vmpd
, c_offset
+ c_vmpd
,
164 0x4000, 0, &chan
->vm_pd
, NULL
);
167 for (i
= 0; i
< 0x4000; i
+= 8) {
168 BAR0_WI32(chan
->vm_pd
, i
+ 0x00, 0x00000000);
169 BAR0_WI32(chan
->vm_pd
, i
+ 0x04, 0x00000000);
172 /* PRAMIN page table, cheat and map into VM at 0x0000000000.
173 * We map the entire fake channel into the start of the PRAMIN BAR
175 ret
= nouveau_gpuobj_new_ref(dev
, chan
, NULL
, 0, pt_size
, 0x1000,
176 0, &priv
->pramin_pt
);
181 if (dev_priv
->vram_sys_base
) {
182 v
+= dev_priv
->vram_sys_base
;
187 while (v
< dev_priv
->vram_sys_base
+ c_offset
+ c_size
) {
188 BAR0_WI32(priv
->pramin_pt
->gpuobj
, i
+ 0, lower_32_bits(v
));
189 BAR0_WI32(priv
->pramin_pt
->gpuobj
, i
+ 4, upper_32_bits(v
));
194 while (i
< pt_size
) {
195 BAR0_WI32(priv
->pramin_pt
->gpuobj
, i
+ 0, 0x00000000);
196 BAR0_WI32(priv
->pramin_pt
->gpuobj
, i
+ 4, 0x00000000);
200 BAR0_WI32(chan
->vm_pd
, 0x00, priv
->pramin_pt
->instance
| 0x63);
201 BAR0_WI32(chan
->vm_pd
, 0x04, 0x00000000);
203 /* VRAM page table(s), mapped into VM at +1GiB */
204 for (i
= 0; i
< dev_priv
->vm_vram_pt_nr
; i
++) {
205 ret
= nouveau_gpuobj_new_ref(dev
, chan
, NULL
, 0,
206 NV50_VM_BLOCK
/65536*8, 0, 0,
207 &chan
->vm_vram_pt
[i
]);
209 NV_ERROR(dev
, "Error creating VRAM page tables: %d\n",
211 dev_priv
->vm_vram_pt_nr
= i
;
214 dev_priv
->vm_vram_pt
[i
] = chan
->vm_vram_pt
[i
]->gpuobj
;
216 for (v
= 0; v
< dev_priv
->vm_vram_pt
[i
]->im_pramin
->size
;
218 BAR0_WI32(dev_priv
->vm_vram_pt
[i
], v
, 0);
220 BAR0_WI32(chan
->vm_pd
, 0x10 + (i
*8),
221 chan
->vm_vram_pt
[i
]->instance
| 0x61);
222 BAR0_WI32(chan
->vm_pd
, 0x14 + (i
*8), 0);
225 /* DMA object for PRAMIN BAR */
226 ret
= nouveau_gpuobj_new_ref(dev
, chan
, chan
, 0, 6*4, 16, 0,
230 BAR0_WI32(priv
->pramin_bar
->gpuobj
, 0x00, 0x7fc00000);
231 BAR0_WI32(priv
->pramin_bar
->gpuobj
, 0x04, dev_priv
->ramin_size
- 1);
232 BAR0_WI32(priv
->pramin_bar
->gpuobj
, 0x08, 0x00000000);
233 BAR0_WI32(priv
->pramin_bar
->gpuobj
, 0x0c, 0x00000000);
234 BAR0_WI32(priv
->pramin_bar
->gpuobj
, 0x10, 0x00000000);
235 BAR0_WI32(priv
->pramin_bar
->gpuobj
, 0x14, 0x00000000);
237 /* DMA object for FB BAR */
238 ret
= nouveau_gpuobj_new_ref(dev
, chan
, chan
, 0, 6*4, 16, 0,
242 BAR0_WI32(priv
->fb_bar
->gpuobj
, 0x00, 0x7fc00000);
243 BAR0_WI32(priv
->fb_bar
->gpuobj
, 0x04, 0x40000000 +
244 drm_get_resource_len(dev
, 1) - 1);
245 BAR0_WI32(priv
->fb_bar
->gpuobj
, 0x08, 0x40000000);
246 BAR0_WI32(priv
->fb_bar
->gpuobj
, 0x0c, 0x00000000);
247 BAR0_WI32(priv
->fb_bar
->gpuobj
, 0x10, 0x00000000);
248 BAR0_WI32(priv
->fb_bar
->gpuobj
, 0x14, 0x00000000);
250 /* Poke the relevant regs, and pray it works :) */
251 nv_wr32(dev
, NV50_PUNK_BAR_CFG_BASE
, (chan
->ramin
->instance
>> 12));
252 nv_wr32(dev
, NV50_PUNK_UNK1710
, 0);
253 nv_wr32(dev
, NV50_PUNK_BAR_CFG_BASE
, (chan
->ramin
->instance
>> 12) |
254 NV50_PUNK_BAR_CFG_BASE_VALID
);
255 nv_wr32(dev
, NV50_PUNK_BAR1_CTXDMA
, (priv
->fb_bar
->instance
>> 4) |
256 NV50_PUNK_BAR1_CTXDMA_VALID
);
257 nv_wr32(dev
, NV50_PUNK_BAR3_CTXDMA
, (priv
->pramin_bar
->instance
>> 4) |
258 NV50_PUNK_BAR3_CTXDMA_VALID
);
260 for (i
= 0; i
< 8; i
++)
261 nv_wr32(dev
, 0x1900 + (i
*4), 0);
263 /* Assume that praying isn't enough, check that we can re-read the
264 * entire fake channel back from the PRAMIN BAR */
265 dev_priv
->engine
.instmem
.prepare_access(dev
, false);
266 for (i
= 0; i
< c_size
; i
+= 4) {
267 if (nv_rd32(dev
, NV_RAMIN
+ i
) != nv_ri32(dev
, i
)) {
268 NV_ERROR(dev
, "Error reading back PRAMIN at 0x%08x\n",
270 dev_priv
->engine
.instmem
.finish_access(dev
);
274 dev_priv
->engine
.instmem
.finish_access(dev
);
276 nv_wr32(dev
, NV50_PUNK_BAR0_PRAMIN
, save_nv001700
);
278 /* Global PRAMIN heap */
279 if (nouveau_mem_init_heap(&dev_priv
->ramin_heap
,
280 c_size
, dev_priv
->ramin_size
- c_size
)) {
281 dev_priv
->ramin_heap
= NULL
;
282 NV_ERROR(dev
, "Failed to init RAMIN heap\n");
285 /*XXX: incorrect, but needed to make hash func "work" */
286 dev_priv
->ramht_offset
= 0x10000;
287 dev_priv
->ramht_bits
= 9;
288 dev_priv
->ramht_size
= (1 << dev_priv
->ramht_bits
);
293 nv50_instmem_takedown(struct drm_device
*dev
)
295 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
296 struct nv50_instmem_priv
*priv
= dev_priv
->engine
.instmem
.priv
;
297 struct nouveau_channel
*chan
= dev_priv
->fifos
[0];
305 /* Restore state from before init */
306 for (i
= 0x1700; i
<= 0x1710; i
+= 4)
307 nv_wr32(dev
, i
, priv
->save1700
[(i
- 0x1700) / 4]);
309 nouveau_gpuobj_ref_del(dev
, &priv
->fb_bar
);
310 nouveau_gpuobj_ref_del(dev
, &priv
->pramin_bar
);
311 nouveau_gpuobj_ref_del(dev
, &priv
->pramin_pt
);
313 /* Destroy dummy channel */
315 for (i
= 0; i
< dev_priv
->vm_vram_pt_nr
; i
++) {
316 nouveau_gpuobj_ref_del(dev
, &chan
->vm_vram_pt
[i
]);
317 dev_priv
->vm_vram_pt
[i
] = NULL
;
319 dev_priv
->vm_vram_pt_nr
= 0;
321 nouveau_gpuobj_del(dev
, &chan
->vm_pd
);
322 nouveau_gpuobj_ref_del(dev
, &chan
->ramfc
);
323 nouveau_gpuobj_ref_del(dev
, &chan
->ramin
);
324 nouveau_mem_takedown(&chan
->ramin_heap
);
326 dev_priv
->fifos
[0] = dev_priv
->fifos
[127] = NULL
;
330 dev_priv
->engine
.instmem
.priv
= NULL
;
335 nv50_instmem_suspend(struct drm_device
*dev
)
337 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
338 struct nouveau_channel
*chan
= dev_priv
->fifos
[0];
339 struct nouveau_gpuobj
*ramin
= chan
->ramin
->gpuobj
;
342 ramin
->im_backing_suspend
= vmalloc(ramin
->im_pramin
->size
);
343 if (!ramin
->im_backing_suspend
)
346 for (i
= 0; i
< ramin
->im_pramin
->size
; i
+= 4)
347 ramin
->im_backing_suspend
[i
/4] = nv_ri32(dev
, i
);
352 nv50_instmem_resume(struct drm_device
*dev
)
354 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
355 struct nv50_instmem_priv
*priv
= dev_priv
->engine
.instmem
.priv
;
356 struct nouveau_channel
*chan
= dev_priv
->fifos
[0];
357 struct nouveau_gpuobj
*ramin
= chan
->ramin
->gpuobj
;
360 nv_wr32(dev
, NV50_PUNK_BAR0_PRAMIN
, (ramin
->im_backing_start
>> 16));
361 for (i
= 0; i
< ramin
->im_pramin
->size
; i
+= 4)
362 BAR0_WI32(ramin
, i
, ramin
->im_backing_suspend
[i
/4]);
363 vfree(ramin
->im_backing_suspend
);
364 ramin
->im_backing_suspend
= NULL
;
366 /* Poke the relevant regs, and pray it works :) */
367 nv_wr32(dev
, NV50_PUNK_BAR_CFG_BASE
, (chan
->ramin
->instance
>> 12));
368 nv_wr32(dev
, NV50_PUNK_UNK1710
, 0);
369 nv_wr32(dev
, NV50_PUNK_BAR_CFG_BASE
, (chan
->ramin
->instance
>> 12) |
370 NV50_PUNK_BAR_CFG_BASE_VALID
);
371 nv_wr32(dev
, NV50_PUNK_BAR1_CTXDMA
, (priv
->fb_bar
->instance
>> 4) |
372 NV50_PUNK_BAR1_CTXDMA_VALID
);
373 nv_wr32(dev
, NV50_PUNK_BAR3_CTXDMA
, (priv
->pramin_bar
->instance
>> 4) |
374 NV50_PUNK_BAR3_CTXDMA_VALID
);
376 for (i
= 0; i
< 8; i
++)
377 nv_wr32(dev
, 0x1900 + (i
*4), 0);
381 nv50_instmem_populate(struct drm_device
*dev
, struct nouveau_gpuobj
*gpuobj
,
386 if (gpuobj
->im_backing
)
389 *sz
= ALIGN(*sz
, NV50_INSTMEM_PAGE_SIZE
);
393 ret
= nouveau_bo_new(dev
, NULL
, *sz
, 0, TTM_PL_FLAG_VRAM
, 0, 0x0000,
394 true, false, &gpuobj
->im_backing
);
396 NV_ERROR(dev
, "error getting PRAMIN backing pages: %d\n", ret
);
400 ret
= nouveau_bo_pin(gpuobj
->im_backing
, TTM_PL_FLAG_VRAM
);
402 NV_ERROR(dev
, "error pinning PRAMIN backing VRAM: %d\n", ret
);
403 nouveau_bo_ref(NULL
, &gpuobj
->im_backing
);
407 gpuobj
->im_backing_start
= gpuobj
->im_backing
->bo
.mem
.mm_node
->start
;
408 gpuobj
->im_backing_start
<<= PAGE_SHIFT
;
414 nv50_instmem_clear(struct drm_device
*dev
, struct nouveau_gpuobj
*gpuobj
)
416 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
418 if (gpuobj
&& gpuobj
->im_backing
) {
419 if (gpuobj
->im_bound
)
420 dev_priv
->engine
.instmem
.unbind(dev
, gpuobj
);
421 nouveau_bo_unpin(gpuobj
->im_backing
);
422 nouveau_bo_ref(NULL
, &gpuobj
->im_backing
);
423 gpuobj
->im_backing
= NULL
;
428 nv50_instmem_bind(struct drm_device
*dev
, struct nouveau_gpuobj
*gpuobj
)
430 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
431 struct nv50_instmem_priv
*priv
= dev_priv
->engine
.instmem
.priv
;
432 struct nouveau_gpuobj
*pramin_pt
= priv
->pramin_pt
->gpuobj
;
433 uint32_t pte
, pte_end
;
436 if (!gpuobj
->im_backing
|| !gpuobj
->im_pramin
|| gpuobj
->im_bound
)
439 NV_DEBUG(dev
, "st=0x%0llx sz=0x%0llx\n",
440 gpuobj
->im_pramin
->start
, gpuobj
->im_pramin
->size
);
442 pte
= (gpuobj
->im_pramin
->start
>> 12) << 1;
443 pte_end
= ((gpuobj
->im_pramin
->size
>> 12) << 1) + pte
;
444 vram
= gpuobj
->im_backing_start
;
446 NV_DEBUG(dev
, "pramin=0x%llx, pte=%d, pte_end=%d\n",
447 gpuobj
->im_pramin
->start
, pte
, pte_end
);
448 NV_DEBUG(dev
, "first vram page: 0x%08x\n", gpuobj
->im_backing_start
);
451 if (dev_priv
->vram_sys_base
) {
452 vram
+= dev_priv
->vram_sys_base
;
456 dev_priv
->engine
.instmem
.prepare_access(dev
, true);
457 while (pte
< pte_end
) {
458 nv_wo32(dev
, pramin_pt
, pte
++, lower_32_bits(vram
));
459 nv_wo32(dev
, pramin_pt
, pte
++, upper_32_bits(vram
));
460 vram
+= NV50_INSTMEM_PAGE_SIZE
;
462 dev_priv
->engine
.instmem
.finish_access(dev
);
464 nv_wr32(dev
, 0x100c80, 0x00040001);
465 if (!nv_wait(0x100c80, 0x00000001, 0x00000000)) {
466 NV_ERROR(dev
, "timeout: (0x100c80 & 1) == 0 (1)\n");
467 NV_ERROR(dev
, "0x100c80 = 0x%08x\n", nv_rd32(dev
, 0x100c80));
471 nv_wr32(dev
, 0x100c80, 0x00060001);
472 if (!nv_wait(0x100c80, 0x00000001, 0x00000000)) {
473 NV_ERROR(dev
, "timeout: (0x100c80 & 1) == 0 (2)\n");
474 NV_ERROR(dev
, "0x100c80 = 0x%08x\n", nv_rd32(dev
, 0x100c80));
478 gpuobj
->im_bound
= 1;
483 nv50_instmem_unbind(struct drm_device
*dev
, struct nouveau_gpuobj
*gpuobj
)
485 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
486 struct nv50_instmem_priv
*priv
= dev_priv
->engine
.instmem
.priv
;
487 uint32_t pte
, pte_end
;
489 if (gpuobj
->im_bound
== 0)
492 pte
= (gpuobj
->im_pramin
->start
>> 12) << 1;
493 pte_end
= ((gpuobj
->im_pramin
->size
>> 12) << 1) + pte
;
495 dev_priv
->engine
.instmem
.prepare_access(dev
, true);
496 while (pte
< pte_end
) {
497 nv_wo32(dev
, priv
->pramin_pt
->gpuobj
, pte
++, 0x00000000);
498 nv_wo32(dev
, priv
->pramin_pt
->gpuobj
, pte
++, 0x00000000);
500 dev_priv
->engine
.instmem
.finish_access(dev
);
502 gpuobj
->im_bound
= 0;
507 nv50_instmem_prepare_access(struct drm_device
*dev
, bool write
)
509 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
510 struct nv50_instmem_priv
*priv
= dev_priv
->engine
.instmem
.priv
;
512 priv
->last_access_wr
= write
;
516 nv50_instmem_finish_access(struct drm_device
*dev
)
518 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
519 struct nv50_instmem_priv
*priv
= dev_priv
->engine
.instmem
.priv
;
521 if (priv
->last_access_wr
) {
522 nv_wr32(dev
, 0x070000, 0x00000001);
523 if (!nv_wait(0x070000, 0x00000001, 0x00000000))
524 NV_ERROR(dev
, "PRAMIN flush timeout\n");