2 * Copyright 2009 Jerome Glisse.
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sub license, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19 * USE OR OTHER DEALINGS IN THE SOFTWARE.
21 * The above copyright notice and this permission notice (including the
22 * next paragraph) shall be included in all copies or substantial portions
28 * Jerome Glisse <glisse@freedesktop.org>
29 * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
32 #include <linux/list.h>
34 #include "radeon_drm.h"
37 struct radeon_object
{
38 struct ttm_buffer_object tobj
;
39 struct list_head list
;
40 struct radeon_device
*rdev
;
41 struct drm_gem_object
*gobj
;
42 struct ttm_bo_kmap_obj kmap
;
47 uint32_t tiling_flags
;
52 int radeon_ttm_init(struct radeon_device
*rdev
);
53 void radeon_ttm_fini(struct radeon_device
*rdev
);
56 * To exclude mutual BO access we rely on bo_reserve exclusion, as all
57 * function are calling it.
60 static int radeon_object_reserve(struct radeon_object
*robj
, bool interruptible
)
62 return ttm_bo_reserve(&robj
->tobj
, interruptible
, false, false, 0);
65 static void radeon_object_unreserve(struct radeon_object
*robj
)
67 ttm_bo_unreserve(&robj
->tobj
);
70 static void radeon_ttm_object_object_destroy(struct ttm_buffer_object
*tobj
)
72 struct radeon_object
*robj
;
74 robj
= container_of(tobj
, struct radeon_object
, tobj
);
75 list_del_init(&robj
->list
);
76 radeon_object_clear_surface_reg(robj
);
80 static inline void radeon_object_gpu_addr(struct radeon_object
*robj
)
82 /* Default gpu address */
83 robj
->gpu_addr
= 0xFFFFFFFFFFFFFFFFULL
;
84 if (robj
->tobj
.mem
.mm_node
== NULL
) {
87 robj
->gpu_addr
= ((u64
)robj
->tobj
.mem
.mm_node
->start
) << PAGE_SHIFT
;
88 switch (robj
->tobj
.mem
.mem_type
) {
90 robj
->gpu_addr
+= (u64
)robj
->rdev
->mc
.vram_location
;
93 robj
->gpu_addr
+= (u64
)robj
->rdev
->mc
.gtt_location
;
96 DRM_ERROR("Unknown placement %d\n", robj
->tobj
.mem
.mem_type
);
97 robj
->gpu_addr
= 0xFFFFFFFFFFFFFFFFULL
;
102 static inline uint32_t radeon_object_flags_from_domain(uint32_t domain
)
105 if (domain
& RADEON_GEM_DOMAIN_VRAM
) {
106 flags
|= TTM_PL_FLAG_VRAM
| TTM_PL_FLAG_WC
| TTM_PL_FLAG_UNCACHED
;
108 if (domain
& RADEON_GEM_DOMAIN_GTT
) {
109 flags
|= TTM_PL_FLAG_TT
| TTM_PL_MASK_CACHING
;
111 if (domain
& RADEON_GEM_DOMAIN_CPU
) {
112 flags
|= TTM_PL_FLAG_SYSTEM
| TTM_PL_MASK_CACHING
;
115 flags
|= TTM_PL_FLAG_SYSTEM
| TTM_PL_MASK_CACHING
;
120 int radeon_object_create(struct radeon_device
*rdev
,
121 struct drm_gem_object
*gobj
,
126 struct radeon_object
**robj_ptr
)
128 struct radeon_object
*robj
;
129 enum ttm_bo_type type
;
133 if (unlikely(rdev
->mman
.bdev
.dev_mapping
== NULL
)) {
134 rdev
->mman
.bdev
.dev_mapping
= rdev
->ddev
->dev_mapping
;
137 type
= ttm_bo_type_kernel
;
139 type
= ttm_bo_type_device
;
142 robj
= kzalloc(sizeof(struct radeon_object
), GFP_KERNEL
);
148 robj
->surface_reg
= -1;
149 INIT_LIST_HEAD(&robj
->list
);
151 flags
= radeon_object_flags_from_domain(domain
);
152 r
= ttm_buffer_object_init(&rdev
->mman
.bdev
, &robj
->tobj
, size
, type
, flags
,
153 0, 0, false, NULL
, size
,
154 &radeon_ttm_object_object_destroy
);
155 if (unlikely(r
!= 0)) {
156 /* ttm call radeon_ttm_object_object_destroy if error happen */
157 DRM_ERROR("Failed to allocate TTM object (%ld, 0x%08X, %u)\n",
163 list_add_tail(&robj
->list
, &rdev
->gem
.objects
);
168 int radeon_object_kmap(struct radeon_object
*robj
, void **ptr
)
172 spin_lock(&robj
->tobj
.lock
);
177 spin_unlock(&robj
->tobj
.lock
);
180 spin_unlock(&robj
->tobj
.lock
);
181 r
= ttm_bo_kmap(&robj
->tobj
, 0, robj
->tobj
.num_pages
, &robj
->kmap
);
185 spin_lock(&robj
->tobj
.lock
);
186 robj
->kptr
= ttm_kmap_obj_virtual(&robj
->kmap
, &robj
->is_iomem
);
187 spin_unlock(&robj
->tobj
.lock
);
194 void radeon_object_kunmap(struct radeon_object
*robj
)
196 spin_lock(&robj
->tobj
.lock
);
197 if (robj
->kptr
== NULL
) {
198 spin_unlock(&robj
->tobj
.lock
);
202 spin_unlock(&robj
->tobj
.lock
);
203 ttm_bo_kunmap(&robj
->kmap
);
206 void radeon_object_unref(struct radeon_object
**robj
)
208 struct ttm_buffer_object
*tobj
;
210 if ((*robj
) == NULL
) {
213 tobj
= &((*robj
)->tobj
);
220 int radeon_object_mmap(struct radeon_object
*robj
, uint64_t *offset
)
222 *offset
= robj
->tobj
.addr_space_offset
;
226 int radeon_object_pin(struct radeon_object
*robj
, uint32_t domain
,
233 flags
= radeon_object_flags_from_domain(domain
);
234 spin_lock(&robj
->tobj
.lock
);
235 if (robj
->pin_count
) {
237 if (gpu_addr
!= NULL
) {
238 *gpu_addr
= robj
->gpu_addr
;
240 spin_unlock(&robj
->tobj
.lock
);
243 spin_unlock(&robj
->tobj
.lock
);
244 r
= radeon_object_reserve(robj
, false);
245 if (unlikely(r
!= 0)) {
246 DRM_ERROR("radeon: failed to reserve object for pinning it.\n");
249 tmp
= robj
->tobj
.mem
.placement
;
250 ttm_flag_masked(&tmp
, flags
, TTM_PL_MASK_MEM
);
251 robj
->tobj
.proposed_placement
= tmp
| TTM_PL_FLAG_NO_EVICT
| TTM_PL_MASK_CACHING
;
252 r
= ttm_buffer_object_validate(&robj
->tobj
,
253 robj
->tobj
.proposed_placement
,
255 radeon_object_gpu_addr(robj
);
256 if (gpu_addr
!= NULL
) {
257 *gpu_addr
= robj
->gpu_addr
;
260 if (unlikely(r
!= 0)) {
261 DRM_ERROR("radeon: failed to pin object.\n");
263 radeon_object_unreserve(robj
);
267 void radeon_object_unpin(struct radeon_object
*robj
)
272 spin_lock(&robj
->tobj
.lock
);
273 if (!robj
->pin_count
) {
274 spin_unlock(&robj
->tobj
.lock
);
275 printk(KERN_WARNING
"Unpin not necessary for %p !\n", robj
);
279 if (robj
->pin_count
) {
280 spin_unlock(&robj
->tobj
.lock
);
283 spin_unlock(&robj
->tobj
.lock
);
284 r
= radeon_object_reserve(robj
, false);
285 if (unlikely(r
!= 0)) {
286 DRM_ERROR("radeon: failed to reserve object for unpinning it.\n");
289 flags
= robj
->tobj
.mem
.placement
;
290 robj
->tobj
.proposed_placement
= flags
& ~TTM_PL_FLAG_NO_EVICT
;
291 r
= ttm_buffer_object_validate(&robj
->tobj
,
292 robj
->tobj
.proposed_placement
,
294 if (unlikely(r
!= 0)) {
295 DRM_ERROR("radeon: failed to unpin buffer.\n");
297 radeon_object_unreserve(robj
);
300 int radeon_object_wait(struct radeon_object
*robj
)
304 /* FIXME: should use block reservation instead */
305 r
= radeon_object_reserve(robj
, true);
306 if (unlikely(r
!= 0)) {
307 DRM_ERROR("radeon: failed to reserve object for waiting.\n");
310 spin_lock(&robj
->tobj
.lock
);
311 if (robj
->tobj
.sync_obj
) {
312 r
= ttm_bo_wait(&robj
->tobj
, true, true, false);
314 spin_unlock(&robj
->tobj
.lock
);
315 radeon_object_unreserve(robj
);
319 int radeon_object_evict_vram(struct radeon_device
*rdev
)
321 if (rdev
->flags
& RADEON_IS_IGP
) {
322 /* Useless to evict on IGP chips */
325 return ttm_bo_evict_mm(&rdev
->mman
.bdev
, TTM_PL_VRAM
);
328 void radeon_object_force_delete(struct radeon_device
*rdev
)
330 struct radeon_object
*robj
, *n
;
331 struct drm_gem_object
*gobj
;
333 if (list_empty(&rdev
->gem
.objects
)) {
336 DRM_ERROR("Userspace still has active objects !\n");
337 list_for_each_entry_safe(robj
, n
, &rdev
->gem
.objects
, list
) {
338 mutex_lock(&rdev
->ddev
->struct_mutex
);
340 DRM_ERROR("Force free for (%p,%p,%lu,%lu)\n",
341 gobj
, robj
, (unsigned long)gobj
->size
,
342 *((unsigned long *)&gobj
->refcount
));
343 list_del_init(&robj
->list
);
344 radeon_object_unref(&robj
);
345 gobj
->driver_private
= NULL
;
346 drm_gem_object_unreference(gobj
);
347 mutex_unlock(&rdev
->ddev
->struct_mutex
);
351 int radeon_object_init(struct radeon_device
*rdev
)
353 return radeon_ttm_init(rdev
);
356 void radeon_object_fini(struct radeon_device
*rdev
)
358 radeon_ttm_fini(rdev
);
361 void radeon_object_list_add_object(struct radeon_object_list
*lobj
,
362 struct list_head
*head
)
365 list_add(&lobj
->list
, head
);
367 list_add_tail(&lobj
->list
, head
);
371 int radeon_object_list_reserve(struct list_head
*head
)
373 struct radeon_object_list
*lobj
;
377 list_for_each(i
, head
) {
378 lobj
= list_entry(i
, struct radeon_object_list
, list
);
379 if (!lobj
->robj
->pin_count
) {
380 r
= radeon_object_reserve(lobj
->robj
, true);
381 if (unlikely(r
!= 0)) {
382 DRM_ERROR("radeon: failed to reserve object.\n");
391 void radeon_object_list_unreserve(struct list_head
*head
)
393 struct radeon_object_list
*lobj
;
396 list_for_each(i
, head
) {
397 lobj
= list_entry(i
, struct radeon_object_list
, list
);
398 if (!lobj
->robj
->pin_count
) {
399 radeon_object_unreserve(lobj
->robj
);
405 int radeon_object_list_validate(struct list_head
*head
, void *fence
)
407 struct radeon_object_list
*lobj
;
408 struct radeon_object
*robj
;
409 struct radeon_fence
*old_fence
= NULL
;
413 r
= radeon_object_list_reserve(head
);
414 if (unlikely(r
!= 0)) {
415 radeon_object_list_unreserve(head
);
418 list_for_each(i
, head
) {
419 lobj
= list_entry(i
, struct radeon_object_list
, list
);
421 if (!robj
->pin_count
) {
423 robj
->tobj
.proposed_placement
=
424 radeon_object_flags_from_domain(lobj
->wdomain
);
426 robj
->tobj
.proposed_placement
=
427 radeon_object_flags_from_domain(lobj
->rdomain
);
429 r
= ttm_buffer_object_validate(&robj
->tobj
,
430 robj
->tobj
.proposed_placement
,
433 DRM_ERROR("radeon: failed to validate.\n");
436 radeon_object_gpu_addr(robj
);
438 lobj
->gpu_offset
= robj
->gpu_addr
;
439 lobj
->tiling_flags
= robj
->tiling_flags
;
441 old_fence
= (struct radeon_fence
*)robj
->tobj
.sync_obj
;
442 robj
->tobj
.sync_obj
= radeon_fence_ref(fence
);
443 robj
->tobj
.sync_obj_arg
= NULL
;
446 radeon_fence_unref(&old_fence
);
452 void radeon_object_list_unvalidate(struct list_head
*head
)
454 struct radeon_object_list
*lobj
;
455 struct radeon_fence
*old_fence
= NULL
;
458 list_for_each(i
, head
) {
459 lobj
= list_entry(i
, struct radeon_object_list
, list
);
460 old_fence
= (struct radeon_fence
*)lobj
->robj
->tobj
.sync_obj
;
461 lobj
->robj
->tobj
.sync_obj
= NULL
;
463 radeon_fence_unref(&old_fence
);
466 radeon_object_list_unreserve(head
);
469 void radeon_object_list_clean(struct list_head
*head
)
471 radeon_object_list_unreserve(head
);
474 int radeon_object_fbdev_mmap(struct radeon_object
*robj
,
475 struct vm_area_struct
*vma
)
477 return ttm_fbdev_mmap(vma
, &robj
->tobj
);
480 unsigned long radeon_object_size(struct radeon_object
*robj
)
482 return robj
->tobj
.num_pages
<< PAGE_SHIFT
;
485 int radeon_object_get_surface_reg(struct radeon_object
*robj
)
487 struct radeon_device
*rdev
= robj
->rdev
;
488 struct radeon_surface_reg
*reg
;
489 struct radeon_object
*old_object
;
493 if (!robj
->tiling_flags
)
496 if (robj
->surface_reg
>= 0) {
497 reg
= &rdev
->surface_regs
[robj
->surface_reg
];
498 i
= robj
->surface_reg
;
503 for (i
= 0; i
< RADEON_GEM_MAX_SURFACES
; i
++) {
505 reg
= &rdev
->surface_regs
[i
];
509 old_object
= reg
->robj
;
510 if (old_object
->pin_count
== 0)
514 /* if we are all out */
515 if (i
== RADEON_GEM_MAX_SURFACES
) {
518 /* find someone with a surface reg and nuke their BO */
519 reg
= &rdev
->surface_regs
[steal
];
520 old_object
= reg
->robj
;
521 /* blow away the mapping */
522 DRM_DEBUG("stealing surface reg %d from %p\n", steal
, old_object
);
523 ttm_bo_unmap_virtual(&old_object
->tobj
);
524 old_object
->surface_reg
= -1;
528 robj
->surface_reg
= i
;
532 radeon_set_surface_reg(rdev
, i
, robj
->tiling_flags
, robj
->pitch
,
533 robj
->tobj
.mem
.mm_node
->start
<< PAGE_SHIFT
,
534 robj
->tobj
.num_pages
<< PAGE_SHIFT
);
538 void radeon_object_clear_surface_reg(struct radeon_object
*robj
)
540 struct radeon_device
*rdev
= robj
->rdev
;
541 struct radeon_surface_reg
*reg
;
543 if (robj
->surface_reg
== -1)
546 reg
= &rdev
->surface_regs
[robj
->surface_reg
];
547 radeon_clear_surface_reg(rdev
, robj
->surface_reg
);
550 robj
->surface_reg
= -1;
553 void radeon_object_set_tiling_flags(struct radeon_object
*robj
,
554 uint32_t tiling_flags
, uint32_t pitch
)
556 robj
->tiling_flags
= tiling_flags
;
560 void radeon_object_get_tiling_flags(struct radeon_object
*robj
,
561 uint32_t *tiling_flags
,
565 *tiling_flags
= robj
->tiling_flags
;
567 *pitch
= robj
->pitch
;
570 int radeon_object_check_tiling(struct radeon_object
*robj
, bool has_moved
,
573 if (!(robj
->tiling_flags
& RADEON_TILING_SURFACE
))
577 radeon_object_clear_surface_reg(robj
);
581 if (robj
->tobj
.mem
.mem_type
!= TTM_PL_VRAM
) {
585 if (robj
->surface_reg
>= 0)
586 radeon_object_clear_surface_reg(robj
);
590 if ((robj
->surface_reg
>= 0) && !has_moved
)
593 return radeon_object_get_surface_reg(robj
);
596 void radeon_bo_move_notify(struct ttm_buffer_object
*bo
,
597 struct ttm_mem_reg
*mem
)
599 struct radeon_object
*robj
= container_of(bo
, struct radeon_object
, tobj
);
600 radeon_object_check_tiling(robj
, 0, 1);
603 void radeon_bo_fault_reserve_notify(struct ttm_buffer_object
*bo
)
605 struct radeon_object
*robj
= container_of(bo
, struct radeon_object
, tobj
);
606 radeon_object_check_tiling(robj
, 0, 0);