2 * Copyright (C) The Weather Channel, Inc. 2002. All Rights Reserved.
3 * Copyright 2005 Stephane Marchesin
5 * The Weather Channel (TM) funded Tungsten Graphics to develop the
6 * initial release of the Radeon 8500 driver under the XFree86 license.
7 * This notice must be preserved.
9 * Permission is hereby granted, free of charge, to any person obtaining a
10 * copy of this software and associated documentation files (the "Software"),
11 * to deal in the Software without restriction, including without limitation
12 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
13 * and/or sell copies of the Software, and to permit persons to whom the
14 * Software is furnished to do so, subject to the following conditions:
16 * The above copyright notice and this permission notice (including the next
17 * paragraph) shall be included in all copies or substantial portions of the
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 * THE AUTHORS AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
24 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
25 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
26 * DEALINGS IN THE SOFTWARE.
29 * Keith Whitwell <keith@tungstengraphics.com>
35 #include "drm_sarea.h"
36 #include "nouveau_drv.h"
38 static struct mem_block
*
39 split_block(struct mem_block
*p
, uint64_t start
, uint64_t size
,
40 struct drm_file
*file_priv
)
42 /* Maybe cut off the start of an existing block */
43 if (start
> p
->start
) {
44 struct mem_block
*newblock
=
45 kmalloc(sizeof(*newblock
), GFP_KERNEL
);
48 newblock
->start
= start
;
49 newblock
->size
= p
->size
- (start
- p
->start
);
50 newblock
->file_priv
= NULL
;
51 newblock
->next
= p
->next
;
53 p
->next
->prev
= newblock
;
55 p
->size
-= newblock
->size
;
59 /* Maybe cut off the end of an existing block */
61 struct mem_block
*newblock
=
62 kmalloc(sizeof(*newblock
), GFP_KERNEL
);
65 newblock
->start
= start
+ size
;
66 newblock
->size
= p
->size
- size
;
67 newblock
->file_priv
= NULL
;
68 newblock
->next
= p
->next
;
70 p
->next
->prev
= newblock
;
76 /* Our block is in the middle */
77 p
->file_priv
= file_priv
;
82 nouveau_mem_alloc_block(struct mem_block
*heap
, uint64_t size
,
83 int align2
, struct drm_file
*file_priv
, int tail
)
86 uint64_t mask
= (1 << align2
) - 1;
92 list_for_each_prev(p
, heap
) {
93 uint64_t start
= ((p
->start
+ p
->size
) - size
) & ~mask
;
95 if (p
->file_priv
== NULL
&& start
>= p
->start
&&
96 start
+ size
<= p
->start
+ p
->size
)
97 return split_block(p
, start
, size
, file_priv
);
100 list_for_each(p
, heap
) {
101 uint64_t start
= (p
->start
+ mask
) & ~mask
;
103 if (p
->file_priv
== NULL
&&
104 start
+ size
<= p
->start
+ p
->size
)
105 return split_block(p
, start
, size
, file_priv
);
112 void nouveau_mem_free_block(struct mem_block
*p
)
116 /* Assumes a single contiguous range. Needs a special file_priv in
117 * 'heap' to stop it being subsumed.
119 if (p
->next
->file_priv
== NULL
) {
120 struct mem_block
*q
= p
->next
;
127 if (p
->prev
->file_priv
== NULL
) {
128 struct mem_block
*q
= p
->prev
;
136 /* Initialize. How to check for an uninitialized heap?
138 int nouveau_mem_init_heap(struct mem_block
**heap
, uint64_t start
,
141 struct mem_block
*blocks
= kmalloc(sizeof(*blocks
), GFP_KERNEL
);
146 *heap
= kmalloc(sizeof(**heap
), GFP_KERNEL
);
152 blocks
->start
= start
;
154 blocks
->file_priv
= NULL
;
155 blocks
->next
= blocks
->prev
= *heap
;
157 memset(*heap
, 0, sizeof(**heap
));
158 (*heap
)->file_priv
= (struct drm_file
*) -1;
159 (*heap
)->next
= (*heap
)->prev
= blocks
;
164 * Free all blocks associated with the releasing file_priv
166 void nouveau_mem_release(struct drm_file
*file_priv
, struct mem_block
*heap
)
170 if (!heap
|| !heap
->next
)
173 list_for_each(p
, heap
) {
174 if (p
->file_priv
== file_priv
)
178 /* Assumes a single contiguous range. Needs a special file_priv in
179 * 'heap' to stop it being subsumed.
181 list_for_each(p
, heap
) {
182 while ((p
->file_priv
== NULL
) &&
183 (p
->next
->file_priv
== NULL
) &&
185 struct mem_block
*q
= p
->next
;
195 * NV10-NV40 tiling helpers
199 nv10_mem_set_region_tiling(struct drm_device
*dev
, int i
, uint32_t addr
,
200 uint32_t size
, uint32_t pitch
)
202 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
203 struct nouveau_fifo_engine
*pfifo
= &dev_priv
->engine
.fifo
;
204 struct nouveau_fb_engine
*pfb
= &dev_priv
->engine
.fb
;
205 struct nouveau_pgraph_engine
*pgraph
= &dev_priv
->engine
.graph
;
206 struct nouveau_tile_reg
*tile
= &dev_priv
->tile
.reg
[i
];
210 tile
->used
= !!pitch
;
211 nouveau_fence_unref((void **)&tile
->fence
);
213 if (!pfifo
->cache_flush(dev
))
216 pfifo
->reassign(dev
, false);
217 pfifo
->cache_flush(dev
);
218 pfifo
->cache_pull(dev
, false);
220 nouveau_wait_for_idle(dev
);
222 pgraph
->set_region_tiling(dev
, i
, addr
, size
, pitch
);
223 pfb
->set_region_tiling(dev
, i
, addr
, size
, pitch
);
225 pfifo
->cache_pull(dev
, true);
226 pfifo
->reassign(dev
, true);
229 struct nouveau_tile_reg
*
230 nv10_mem_set_tiling(struct drm_device
*dev
, uint32_t addr
, uint32_t size
,
233 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
234 struct nouveau_fb_engine
*pfb
= &dev_priv
->engine
.fb
;
235 struct nouveau_tile_reg
*tile
= dev_priv
->tile
.reg
, *found
= NULL
;
238 spin_lock(&dev_priv
->tile
.lock
);
240 for (i
= 0; i
< pfb
->num_tiles
; i
++) {
242 /* Tile region in use. */
246 !nouveau_fence_signalled(tile
[i
].fence
, NULL
))
247 /* Pending tile region. */
250 if (max(tile
[i
].addr
, addr
) <
251 min(tile
[i
].addr
+ tile
[i
].size
, addr
+ size
))
252 /* Kill an intersecting tile region. */
253 nv10_mem_set_region_tiling(dev
, i
, 0, 0, 0);
255 if (pitch
&& !found
) {
256 /* Free tile region. */
257 nv10_mem_set_region_tiling(dev
, i
, addr
, size
, pitch
);
262 spin_unlock(&dev_priv
->tile
.lock
);
268 nv10_mem_expire_tiling(struct drm_device
*dev
, struct nouveau_tile_reg
*tile
,
269 struct nouveau_fence
*fence
)
272 /* Mark it as pending. */
274 nouveau_fence_ref(fence
);
284 nv50_mem_vm_bind_linear(struct drm_device
*dev
, uint64_t virt
, uint32_t size
,
285 uint32_t flags
, uint64_t phys
)
287 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
288 struct nouveau_gpuobj
*pgt
;
292 virt
= ((virt
- dev_priv
->vm_vram_base
) >> 16) << 1;
293 size
= (size
>> 16) << 1;
295 phys
|= ((uint64_t)flags
<< 32);
297 if (dev_priv
->vram_sys_base
) {
298 phys
+= dev_priv
->vram_sys_base
;
302 dev_priv
->engine
.instmem
.prepare_access(dev
, true);
304 unsigned offset_h
= upper_32_bits(phys
);
305 unsigned offset_l
= lower_32_bits(phys
);
308 for (i
= 7; i
>= 0; i
--) {
309 block
= 1 << (i
+ 1);
310 if (size
>= block
&& !(virt
& (block
- 1)))
313 offset_l
|= (i
<< 7);
319 pgt
= dev_priv
->vm_vram_pt
[virt
>> 14];
325 block
-= (end
- pte
);
329 nv_wo32(dev
, pgt
, pte
++, offset_l
);
330 nv_wo32(dev
, pgt
, pte
++, offset_h
);
334 dev_priv
->engine
.instmem
.finish_access(dev
);
336 nv_wr32(dev
, 0x100c80, 0x00050001);
337 if (!nv_wait(0x100c80, 0x00000001, 0x00000000)) {
338 NV_ERROR(dev
, "timeout: (0x100c80 & 1) == 0 (2)\n");
339 NV_ERROR(dev
, "0x100c80 = 0x%08x\n", nv_rd32(dev
, 0x100c80));
343 nv_wr32(dev
, 0x100c80, 0x00000001);
344 if (!nv_wait(0x100c80, 0x00000001, 0x00000000)) {
345 NV_ERROR(dev
, "timeout: (0x100c80 & 1) == 0 (2)\n");
346 NV_ERROR(dev
, "0x100c80 = 0x%08x\n", nv_rd32(dev
, 0x100c80));
350 nv_wr32(dev
, 0x100c80, 0x00040001);
351 if (!nv_wait(0x100c80, 0x00000001, 0x00000000)) {
352 NV_ERROR(dev
, "timeout: (0x100c80 & 1) == 0 (2)\n");
353 NV_ERROR(dev
, "0x100c80 = 0x%08x\n", nv_rd32(dev
, 0x100c80));
357 nv_wr32(dev
, 0x100c80, 0x00060001);
358 if (!nv_wait(0x100c80, 0x00000001, 0x00000000)) {
359 NV_ERROR(dev
, "timeout: (0x100c80 & 1) == 0 (2)\n");
360 NV_ERROR(dev
, "0x100c80 = 0x%08x\n", nv_rd32(dev
, 0x100c80));
368 nv50_mem_vm_unbind(struct drm_device
*dev
, uint64_t virt
, uint32_t size
)
370 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
371 struct nouveau_gpuobj
*pgt
;
372 unsigned pages
, pte
, end
;
374 virt
-= dev_priv
->vm_vram_base
;
375 pages
= (size
>> 16) << 1;
377 dev_priv
->engine
.instmem
.prepare_access(dev
, true);
379 pgt
= dev_priv
->vm_vram_pt
[virt
>> 29];
380 pte
= (virt
& 0x1ffe0000ULL
) >> 15;
385 pages
-= (end
- pte
);
386 virt
+= (end
- pte
) << 15;
389 nv_wo32(dev
, pgt
, pte
++, 0);
391 dev_priv
->engine
.instmem
.finish_access(dev
);
393 nv_wr32(dev
, 0x100c80, 0x00050001);
394 if (!nv_wait(0x100c80, 0x00000001, 0x00000000)) {
395 NV_ERROR(dev
, "timeout: (0x100c80 & 1) == 0 (2)\n");
396 NV_ERROR(dev
, "0x100c80 = 0x%08x\n", nv_rd32(dev
, 0x100c80));
400 nv_wr32(dev
, 0x100c80, 0x00000001);
401 if (!nv_wait(0x100c80, 0x00000001, 0x00000000)) {
402 NV_ERROR(dev
, "timeout: (0x100c80 & 1) == 0 (2)\n");
403 NV_ERROR(dev
, "0x100c80 = 0x%08x\n", nv_rd32(dev
, 0x100c80));
407 nv_wr32(dev
, 0x100c80, 0x00040001);
408 if (!nv_wait(0x100c80, 0x00000001, 0x00000000)) {
409 NV_ERROR(dev
, "timeout: (0x100c80 & 1) == 0 (2)\n");
410 NV_ERROR(dev
, "0x100c80 = 0x%08x\n", nv_rd32(dev
, 0x100c80));
414 nv_wr32(dev
, 0x100c80, 0x00060001);
415 if (!nv_wait(0x100c80, 0x00000001, 0x00000000)) {
416 NV_ERROR(dev
, "timeout: (0x100c80 & 1) == 0 (2)\n");
417 NV_ERROR(dev
, "0x100c80 = 0x%08x\n", nv_rd32(dev
, 0x100c80));
424 void nouveau_mem_takedown(struct mem_block
**heap
)
431 for (p
= (*heap
)->next
; p
!= *heap
;) {
432 struct mem_block
*q
= p
;
441 void nouveau_mem_close(struct drm_device
*dev
)
443 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
445 nouveau_bo_unpin(dev_priv
->vga_ram
);
446 nouveau_bo_ref(NULL
, &dev_priv
->vga_ram
);
448 ttm_bo_device_release(&dev_priv
->ttm
.bdev
);
450 nouveau_ttm_global_release(dev_priv
);
452 if (drm_core_has_AGP(dev
) && dev
->agp
&&
453 drm_core_check_feature(dev
, DRIVER_MODESET
)) {
454 struct drm_agp_mem
*entry
, *tempe
;
456 /* Remove AGP resources, but leave dev->agp
457 intact until drv_cleanup is called. */
458 list_for_each_entry_safe(entry
, tempe
, &dev
->agp
->memory
, head
) {
460 drm_unbind_agp(entry
->memory
);
461 drm_free_agp(entry
->memory
, entry
->pages
);
464 INIT_LIST_HEAD(&dev
->agp
->memory
);
466 if (dev
->agp
->acquired
)
467 drm_agp_release(dev
);
469 dev
->agp
->acquired
= 0;
470 dev
->agp
->enabled
= 0;
473 if (dev_priv
->fb_mtrr
) {
474 drm_mtrr_del(dev_priv
->fb_mtrr
, drm_get_resource_start(dev
, 1),
475 drm_get_resource_len(dev
, 1), DRM_MTRR_WC
);
476 dev_priv
->fb_mtrr
= 0;
481 nouveau_mem_detect_nv04(struct drm_device
*dev
)
483 uint32_t boot0
= nv_rd32(dev
, NV03_BOOT_0
);
485 if (boot0
& 0x00000100)
486 return (((boot0
>> 12) & 0xf) * 2 + 2) * 1024 * 1024;
488 switch (boot0
& NV03_BOOT_0_RAM_AMOUNT
) {
489 case NV04_BOOT_0_RAM_AMOUNT_32MB
:
490 return 32 * 1024 * 1024;
491 case NV04_BOOT_0_RAM_AMOUNT_16MB
:
492 return 16 * 1024 * 1024;
493 case NV04_BOOT_0_RAM_AMOUNT_8MB
:
494 return 8 * 1024 * 1024;
495 case NV04_BOOT_0_RAM_AMOUNT_4MB
:
496 return 4 * 1024 * 1024;
503 nouveau_mem_detect_nforce(struct drm_device
*dev
)
505 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
506 struct pci_dev
*bridge
;
509 bridge
= pci_get_bus_and_slot(0, PCI_DEVFN(0, 1));
511 NV_ERROR(dev
, "no bridge device\n");
515 if (dev_priv
->flags
& NV_NFORCE
) {
516 pci_read_config_dword(bridge
, 0x7C, &mem
);
517 return (uint64_t)(((mem
>> 6) & 31) + 1)*1024*1024;
519 if (dev_priv
->flags
& NV_NFORCE2
) {
520 pci_read_config_dword(bridge
, 0x84, &mem
);
521 return (uint64_t)(((mem
>> 4) & 127) + 1)*1024*1024;
524 NV_ERROR(dev
, "impossible!\n");
528 /* returns the amount of FB ram in bytes */
530 nouveau_mem_detect(struct drm_device
*dev
)
532 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
534 if (dev_priv
->card_type
== NV_04
) {
535 dev_priv
->vram_size
= nouveau_mem_detect_nv04(dev
);
537 if (dev_priv
->flags
& (NV_NFORCE
| NV_NFORCE2
)) {
538 dev_priv
->vram_size
= nouveau_mem_detect_nforce(dev
);
540 dev_priv
->vram_size
= nv_rd32(dev
, NV04_FIFO_DATA
);
541 dev_priv
->vram_size
&= NV10_FIFO_DATA_RAM_AMOUNT_MB_MASK
;
542 if (dev_priv
->chipset
== 0xaa || dev_priv
->chipset
== 0xac)
543 dev_priv
->vram_sys_base
= nv_rd32(dev
, 0x100e10) << 12;
546 NV_INFO(dev
, "Detected %dMiB VRAM\n", (int)(dev_priv
->vram_size
>> 20));
547 if (dev_priv
->vram_sys_base
) {
548 NV_INFO(dev
, "Stolen system memory at: 0x%010llx\n",
549 dev_priv
->vram_sys_base
);
552 if (dev_priv
->vram_size
)
558 static void nouveau_mem_reset_agp(struct drm_device
*dev
)
560 uint32_t saved_pci_nv_1
, saved_pci_nv_19
, pmc_enable
;
562 saved_pci_nv_1
= nv_rd32(dev
, NV04_PBUS_PCI_NV_1
);
563 saved_pci_nv_19
= nv_rd32(dev
, NV04_PBUS_PCI_NV_19
);
565 /* clear busmaster bit */
566 nv_wr32(dev
, NV04_PBUS_PCI_NV_1
, saved_pci_nv_1
& ~0x4);
567 /* clear SBA and AGP bits */
568 nv_wr32(dev
, NV04_PBUS_PCI_NV_19
, saved_pci_nv_19
& 0xfffff0ff);
570 /* power cycle pgraph, if enabled */
571 pmc_enable
= nv_rd32(dev
, NV03_PMC_ENABLE
);
572 if (pmc_enable
& NV_PMC_ENABLE_PGRAPH
) {
573 nv_wr32(dev
, NV03_PMC_ENABLE
,
574 pmc_enable
& ~NV_PMC_ENABLE_PGRAPH
);
575 nv_wr32(dev
, NV03_PMC_ENABLE
, nv_rd32(dev
, NV03_PMC_ENABLE
) |
576 NV_PMC_ENABLE_PGRAPH
);
579 /* and restore (gives effect of resetting AGP) */
580 nv_wr32(dev
, NV04_PBUS_PCI_NV_19
, saved_pci_nv_19
);
581 nv_wr32(dev
, NV04_PBUS_PCI_NV_1
, saved_pci_nv_1
);
586 nouveau_mem_init_agp(struct drm_device
*dev
)
589 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
590 struct drm_agp_info info
;
591 struct drm_agp_mode mode
;
597 nouveau_mem_reset_agp(dev
);
599 if (!dev
->agp
->acquired
) {
600 ret
= drm_agp_acquire(dev
);
602 NV_ERROR(dev
, "Unable to acquire AGP: %d\n", ret
);
607 ret
= drm_agp_info(dev
, &info
);
609 NV_ERROR(dev
, "Unable to get AGP info: %d\n", ret
);
613 /* see agp.h for the AGPSTAT_* modes available */
614 mode
.mode
= info
.mode
;
615 ret
= drm_agp_enable(dev
, mode
);
617 NV_ERROR(dev
, "Unable to enable AGP: %d\n", ret
);
621 dev_priv
->gart_info
.type
= NOUVEAU_GART_AGP
;
622 dev_priv
->gart_info
.aper_base
= info
.aperture_base
;
623 dev_priv
->gart_info
.aper_size
= info
.aperture_size
;
629 nouveau_mem_init(struct drm_device
*dev
)
631 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
632 struct ttm_bo_device
*bdev
= &dev_priv
->ttm
.bdev
;
633 int ret
, dma_bits
= 32;
635 dev_priv
->fb_phys
= drm_get_resource_start(dev
, 1);
636 dev_priv
->gart_info
.type
= NOUVEAU_GART_NONE
;
638 if (dev_priv
->card_type
>= NV_50
&&
639 pci_dma_supported(dev
->pdev
, DMA_BIT_MASK(40)))
642 ret
= pci_set_dma_mask(dev
->pdev
, DMA_BIT_MASK(dma_bits
));
644 NV_ERROR(dev
, "Error setting DMA mask: %d\n", ret
);
648 ret
= nouveau_ttm_global_init(dev_priv
);
652 ret
= ttm_bo_device_init(&dev_priv
->ttm
.bdev
,
653 dev_priv
->ttm
.bo_global_ref
.ref
.object
,
654 &nouveau_bo_driver
, DRM_FILE_PAGE_OFFSET
,
655 dma_bits
<= 32 ? true : false);
657 NV_ERROR(dev
, "Error initialising bo driver: %d\n", ret
);
661 INIT_LIST_HEAD(&dev_priv
->ttm
.bo_list
);
662 spin_lock_init(&dev_priv
->ttm
.bo_list_lock
);
663 spin_lock_init(&dev_priv
->tile
.lock
);
665 dev_priv
->fb_available_size
= dev_priv
->vram_size
;
666 dev_priv
->fb_mappable_pages
= dev_priv
->fb_available_size
;
667 if (dev_priv
->fb_mappable_pages
> drm_get_resource_len(dev
, 1))
668 dev_priv
->fb_mappable_pages
= drm_get_resource_len(dev
, 1);
669 dev_priv
->fb_mappable_pages
>>= PAGE_SHIFT
;
671 /* remove reserved space at end of vram from available amount */
672 dev_priv
->fb_available_size
-= dev_priv
->ramin_rsvd_vram
;
673 dev_priv
->fb_aper_free
= dev_priv
->fb_available_size
;
676 ret
= ttm_bo_init_mm(bdev
, TTM_PL_VRAM
,
677 dev_priv
->fb_available_size
>> PAGE_SHIFT
);
679 NV_ERROR(dev
, "Failed VRAM mm init: %d\n", ret
);
683 ret
= nouveau_bo_new(dev
, NULL
, 256*1024, 0, TTM_PL_FLAG_VRAM
,
684 0, 0, true, true, &dev_priv
->vga_ram
);
686 ret
= nouveau_bo_pin(dev_priv
->vga_ram
, TTM_PL_FLAG_VRAM
);
688 NV_WARN(dev
, "failed to reserve VGA memory\n");
689 nouveau_bo_ref(NULL
, &dev_priv
->vga_ram
);
693 #if !defined(__powerpc__) && !defined(__ia64__)
694 if (drm_device_is_agp(dev
) && dev
->agp
) {
695 ret
= nouveau_mem_init_agp(dev
);
697 NV_ERROR(dev
, "Error initialising AGP: %d\n", ret
);
701 if (dev_priv
->gart_info
.type
== NOUVEAU_GART_NONE
) {
702 ret
= nouveau_sgdma_init(dev
);
704 NV_ERROR(dev
, "Error initialising PCI(E): %d\n", ret
);
709 NV_INFO(dev
, "%d MiB GART (aperture)\n",
710 (int)(dev_priv
->gart_info
.aper_size
>> 20));
711 dev_priv
->gart_info
.aper_free
= dev_priv
->gart_info
.aper_size
;
713 ret
= ttm_bo_init_mm(bdev
, TTM_PL_TT
,
714 dev_priv
->gart_info
.aper_size
>> PAGE_SHIFT
);
716 NV_ERROR(dev
, "Failed TT mm init: %d\n", ret
);
720 dev_priv
->fb_mtrr
= drm_mtrr_add(drm_get_resource_start(dev
, 1),
721 drm_get_resource_len(dev
, 1),