2 * Copyright © 2015 Broadcom
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
10 * DOC: VC4 GEM BO management support
12 * The VC4 GPU architecture (both scanout and rendering) has direct
13 * access to system memory with no MMU in between. To support it, we
14 * use the GEM CMA helper functions to allocate contiguous ranges of
15 * physical memory for our BOs.
17 * Since the CMA allocator is very slow, we keep a cache of recently
18 * freed BOs around so that the kernel's allocation of objects for 3D
19 * rendering can return quickly.
22 #include <linux/dma-buf.h>
25 #include "uapi/drm/vc4_drm.h"
27 static const char * const bo_type_names
[] = {
38 static bool is_user_label(int label
)
40 return label
>= VC4_BO_TYPE_COUNT
;
43 static void vc4_bo_stats_dump(struct vc4_dev
*vc4
)
47 for (i
= 0; i
< vc4
->num_labels
; i
++) {
48 if (!vc4
->bo_labels
[i
].num_allocated
)
51 DRM_INFO("%30s: %6dkb BOs (%d)\n",
52 vc4
->bo_labels
[i
].name
,
53 vc4
->bo_labels
[i
].size_allocated
/ 1024,
54 vc4
->bo_labels
[i
].num_allocated
);
57 mutex_lock(&vc4
->purgeable
.lock
);
58 if (vc4
->purgeable
.num
)
59 DRM_INFO("%30s: %6zdkb BOs (%d)\n", "userspace BO cache",
60 vc4
->purgeable
.size
/ 1024, vc4
->purgeable
.num
);
62 if (vc4
->purgeable
.purged_num
)
63 DRM_INFO("%30s: %6zdkb BOs (%d)\n", "total purged BO",
64 vc4
->purgeable
.purged_size
/ 1024,
65 vc4
->purgeable
.purged_num
);
66 mutex_unlock(&vc4
->purgeable
.lock
);
69 #ifdef CONFIG_DEBUG_FS
70 int vc4_bo_stats_debugfs(struct seq_file
*m
, void *unused
)
72 struct drm_info_node
*node
= (struct drm_info_node
*)m
->private;
73 struct drm_device
*dev
= node
->minor
->dev
;
74 struct vc4_dev
*vc4
= to_vc4_dev(dev
);
77 mutex_lock(&vc4
->bo_lock
);
78 for (i
= 0; i
< vc4
->num_labels
; i
++) {
79 if (!vc4
->bo_labels
[i
].num_allocated
)
82 seq_printf(m
, "%30s: %6dkb BOs (%d)\n",
83 vc4
->bo_labels
[i
].name
,
84 vc4
->bo_labels
[i
].size_allocated
/ 1024,
85 vc4
->bo_labels
[i
].num_allocated
);
87 mutex_unlock(&vc4
->bo_lock
);
89 mutex_lock(&vc4
->purgeable
.lock
);
90 if (vc4
->purgeable
.num
)
91 seq_printf(m
, "%30s: %6zdkb BOs (%d)\n", "userspace BO cache",
92 vc4
->purgeable
.size
/ 1024, vc4
->purgeable
.num
);
94 if (vc4
->purgeable
.purged_num
)
95 seq_printf(m
, "%30s: %6zdkb BOs (%d)\n", "total purged BO",
96 vc4
->purgeable
.purged_size
/ 1024,
97 vc4
->purgeable
.purged_num
);
98 mutex_unlock(&vc4
->purgeable
.lock
);
104 /* Takes ownership of *name and returns the appropriate slot for it in
105 * the bo_labels[] array, extending it as necessary.
107 * This is inefficient and could use a hash table instead of walking
108 * an array and strcmp()ing. However, the assumption is that user
109 * labeling will be infrequent (scanout buffers and other long-lived
110 * objects, or debug driver builds), so we can live with it for now.
112 static int vc4_get_user_label(struct vc4_dev
*vc4
, const char *name
)
117 for (i
= 0; i
< vc4
->num_labels
; i
++) {
118 if (!vc4
->bo_labels
[i
].name
) {
120 } else if (strcmp(vc4
->bo_labels
[i
].name
, name
) == 0) {
126 if (free_slot
!= -1) {
127 WARN_ON(vc4
->bo_labels
[free_slot
].num_allocated
!= 0);
128 vc4
->bo_labels
[free_slot
].name
= name
;
131 u32 new_label_count
= vc4
->num_labels
+ 1;
132 struct vc4_label
*new_labels
=
133 krealloc(vc4
->bo_labels
,
134 new_label_count
* sizeof(*new_labels
),
142 free_slot
= vc4
->num_labels
;
143 vc4
->bo_labels
= new_labels
;
144 vc4
->num_labels
= new_label_count
;
146 vc4
->bo_labels
[free_slot
].name
= name
;
147 vc4
->bo_labels
[free_slot
].num_allocated
= 0;
148 vc4
->bo_labels
[free_slot
].size_allocated
= 0;
154 static void vc4_bo_set_label(struct drm_gem_object
*gem_obj
, int label
)
156 struct vc4_bo
*bo
= to_vc4_bo(gem_obj
);
157 struct vc4_dev
*vc4
= to_vc4_dev(gem_obj
->dev
);
159 lockdep_assert_held(&vc4
->bo_lock
);
162 vc4
->bo_labels
[label
].num_allocated
++;
163 vc4
->bo_labels
[label
].size_allocated
+= gem_obj
->size
;
166 vc4
->bo_labels
[bo
->label
].num_allocated
--;
167 vc4
->bo_labels
[bo
->label
].size_allocated
-= gem_obj
->size
;
169 if (vc4
->bo_labels
[bo
->label
].num_allocated
== 0 &&
170 is_user_label(bo
->label
)) {
171 /* Free user BO label slots on last unreference.
172 * Slots are just where we track the stats for a given
173 * name, and once a name is unused we can reuse that
176 kfree(vc4
->bo_labels
[bo
->label
].name
);
177 vc4
->bo_labels
[bo
->label
].name
= NULL
;
183 static uint32_t bo_page_index(size_t size
)
185 return (size
/ PAGE_SIZE
) - 1;
188 static void vc4_bo_destroy(struct vc4_bo
*bo
)
190 struct drm_gem_object
*obj
= &bo
->base
.base
;
191 struct vc4_dev
*vc4
= to_vc4_dev(obj
->dev
);
193 lockdep_assert_held(&vc4
->bo_lock
);
195 vc4_bo_set_label(obj
, -1);
197 if (bo
->validated_shader
) {
198 kfree(bo
->validated_shader
->uniform_addr_offsets
);
199 kfree(bo
->validated_shader
->texture_samples
);
200 kfree(bo
->validated_shader
);
201 bo
->validated_shader
= NULL
;
204 reservation_object_fini(&bo
->_resv
);
206 drm_gem_cma_free_object(obj
);
209 static void vc4_bo_remove_from_cache(struct vc4_bo
*bo
)
211 struct vc4_dev
*vc4
= to_vc4_dev(bo
->base
.base
.dev
);
213 lockdep_assert_held(&vc4
->bo_lock
);
214 list_del(&bo
->unref_head
);
215 list_del(&bo
->size_head
);
218 static struct list_head
*vc4_get_cache_list_for_size(struct drm_device
*dev
,
221 struct vc4_dev
*vc4
= to_vc4_dev(dev
);
222 uint32_t page_index
= bo_page_index(size
);
224 if (vc4
->bo_cache
.size_list_size
<= page_index
) {
225 uint32_t new_size
= max(vc4
->bo_cache
.size_list_size
* 2,
227 struct list_head
*new_list
;
230 new_list
= kmalloc_array(new_size
, sizeof(struct list_head
),
235 /* Rebase the old cached BO lists to their new list
238 for (i
= 0; i
< vc4
->bo_cache
.size_list_size
; i
++) {
239 struct list_head
*old_list
=
240 &vc4
->bo_cache
.size_list
[i
];
242 if (list_empty(old_list
))
243 INIT_LIST_HEAD(&new_list
[i
]);
245 list_replace(old_list
, &new_list
[i
]);
247 /* And initialize the brand new BO list heads. */
248 for (i
= vc4
->bo_cache
.size_list_size
; i
< new_size
; i
++)
249 INIT_LIST_HEAD(&new_list
[i
]);
251 kfree(vc4
->bo_cache
.size_list
);
252 vc4
->bo_cache
.size_list
= new_list
;
253 vc4
->bo_cache
.size_list_size
= new_size
;
256 return &vc4
->bo_cache
.size_list
[page_index
];
259 static void vc4_bo_cache_purge(struct drm_device
*dev
)
261 struct vc4_dev
*vc4
= to_vc4_dev(dev
);
263 mutex_lock(&vc4
->bo_lock
);
264 while (!list_empty(&vc4
->bo_cache
.time_list
)) {
265 struct vc4_bo
*bo
= list_last_entry(&vc4
->bo_cache
.time_list
,
266 struct vc4_bo
, unref_head
);
267 vc4_bo_remove_from_cache(bo
);
270 mutex_unlock(&vc4
->bo_lock
);
273 void vc4_bo_add_to_purgeable_pool(struct vc4_bo
*bo
)
275 struct vc4_dev
*vc4
= to_vc4_dev(bo
->base
.base
.dev
);
277 mutex_lock(&vc4
->purgeable
.lock
);
278 list_add_tail(&bo
->size_head
, &vc4
->purgeable
.list
);
279 vc4
->purgeable
.num
++;
280 vc4
->purgeable
.size
+= bo
->base
.base
.size
;
281 mutex_unlock(&vc4
->purgeable
.lock
);
284 static void vc4_bo_remove_from_purgeable_pool_locked(struct vc4_bo
*bo
)
286 struct vc4_dev
*vc4
= to_vc4_dev(bo
->base
.base
.dev
);
288 /* list_del_init() is used here because the caller might release
289 * the purgeable lock in order to acquire the madv one and update the
291 * During this short period of time a user might decide to mark
292 * the BO as unpurgeable, and if bo->madv is set to
293 * VC4_MADV_DONTNEED it will try to remove the BO from the
294 * purgeable list which will fail if the ->next/prev fields
295 * are set to LIST_POISON1/LIST_POISON2 (which is what
297 * Re-initializing the list element guarantees that list_del()
298 * will work correctly even if it's a NOP.
300 list_del_init(&bo
->size_head
);
301 vc4
->purgeable
.num
--;
302 vc4
->purgeable
.size
-= bo
->base
.base
.size
;
305 void vc4_bo_remove_from_purgeable_pool(struct vc4_bo
*bo
)
307 struct vc4_dev
*vc4
= to_vc4_dev(bo
->base
.base
.dev
);
309 mutex_lock(&vc4
->purgeable
.lock
);
310 vc4_bo_remove_from_purgeable_pool_locked(bo
);
311 mutex_unlock(&vc4
->purgeable
.lock
);
314 static void vc4_bo_purge(struct drm_gem_object
*obj
)
316 struct vc4_bo
*bo
= to_vc4_bo(obj
);
317 struct drm_device
*dev
= obj
->dev
;
319 WARN_ON(!mutex_is_locked(&bo
->madv_lock
));
320 WARN_ON(bo
->madv
!= VC4_MADV_DONTNEED
);
322 drm_vma_node_unmap(&obj
->vma_node
, dev
->anon_inode
->i_mapping
);
324 dma_free_wc(dev
->dev
, obj
->size
, bo
->base
.vaddr
, bo
->base
.paddr
);
325 bo
->base
.vaddr
= NULL
;
326 bo
->madv
= __VC4_MADV_PURGED
;
329 static void vc4_bo_userspace_cache_purge(struct drm_device
*dev
)
331 struct vc4_dev
*vc4
= to_vc4_dev(dev
);
333 mutex_lock(&vc4
->purgeable
.lock
);
334 while (!list_empty(&vc4
->purgeable
.list
)) {
335 struct vc4_bo
*bo
= list_first_entry(&vc4
->purgeable
.list
,
336 struct vc4_bo
, size_head
);
337 struct drm_gem_object
*obj
= &bo
->base
.base
;
338 size_t purged_size
= 0;
340 vc4_bo_remove_from_purgeable_pool_locked(bo
);
342 /* Release the purgeable lock while we're purging the BO so
343 * that other people can continue inserting things in the
344 * purgeable pool without having to wait for all BOs to be
347 mutex_unlock(&vc4
->purgeable
.lock
);
348 mutex_lock(&bo
->madv_lock
);
350 /* Since we released the purgeable pool lock before acquiring
351 * the BO madv one, the user may have marked the BO as WILLNEED
352 * and re-used it in the meantime.
353 * Before purging the BO we need to make sure
354 * - it is still marked as DONTNEED
355 * - it has not been re-inserted in the purgeable list
356 * - it is not used by HW blocks
357 * If one of these conditions is not met, just skip the entry.
359 if (bo
->madv
== VC4_MADV_DONTNEED
&&
360 list_empty(&bo
->size_head
) &&
361 !refcount_read(&bo
->usecnt
)) {
362 purged_size
= bo
->base
.base
.size
;
365 mutex_unlock(&bo
->madv_lock
);
366 mutex_lock(&vc4
->purgeable
.lock
);
369 vc4
->purgeable
.purged_size
+= purged_size
;
370 vc4
->purgeable
.purged_num
++;
373 mutex_unlock(&vc4
->purgeable
.lock
);
376 static struct vc4_bo
*vc4_bo_get_from_cache(struct drm_device
*dev
,
378 enum vc4_kernel_bo_type type
)
380 struct vc4_dev
*vc4
= to_vc4_dev(dev
);
381 uint32_t page_index
= bo_page_index(size
);
382 struct vc4_bo
*bo
= NULL
;
384 size
= roundup(size
, PAGE_SIZE
);
386 mutex_lock(&vc4
->bo_lock
);
387 if (page_index
>= vc4
->bo_cache
.size_list_size
)
390 if (list_empty(&vc4
->bo_cache
.size_list
[page_index
]))
393 bo
= list_first_entry(&vc4
->bo_cache
.size_list
[page_index
],
394 struct vc4_bo
, size_head
);
395 vc4_bo_remove_from_cache(bo
);
396 kref_init(&bo
->base
.base
.refcount
);
400 vc4_bo_set_label(&bo
->base
.base
, type
);
401 mutex_unlock(&vc4
->bo_lock
);
406 * vc4_gem_create_object - Implementation of driver->gem_create_object.
408 * @size: Size in bytes of the memory the object will reference
410 * This lets the CMA helpers allocate object structs for us, and keep
411 * our BO stats correct.
413 struct drm_gem_object
*vc4_create_object(struct drm_device
*dev
, size_t size
)
415 struct vc4_dev
*vc4
= to_vc4_dev(dev
);
418 bo
= kzalloc(sizeof(*bo
), GFP_KERNEL
);
420 return ERR_PTR(-ENOMEM
);
422 bo
->madv
= VC4_MADV_WILLNEED
;
423 refcount_set(&bo
->usecnt
, 0);
424 mutex_init(&bo
->madv_lock
);
425 mutex_lock(&vc4
->bo_lock
);
426 bo
->label
= VC4_BO_TYPE_KERNEL
;
427 vc4
->bo_labels
[VC4_BO_TYPE_KERNEL
].num_allocated
++;
428 vc4
->bo_labels
[VC4_BO_TYPE_KERNEL
].size_allocated
+= size
;
429 mutex_unlock(&vc4
->bo_lock
);
430 bo
->resv
= &bo
->_resv
;
431 reservation_object_init(bo
->resv
);
433 return &bo
->base
.base
;
436 struct vc4_bo
*vc4_bo_create(struct drm_device
*dev
, size_t unaligned_size
,
437 bool allow_unzeroed
, enum vc4_kernel_bo_type type
)
439 size_t size
= roundup(unaligned_size
, PAGE_SIZE
);
440 struct vc4_dev
*vc4
= to_vc4_dev(dev
);
441 struct drm_gem_cma_object
*cma_obj
;
445 return ERR_PTR(-EINVAL
);
447 /* First, try to get a vc4_bo from the kernel BO cache. */
448 bo
= vc4_bo_get_from_cache(dev
, size
, type
);
451 memset(bo
->base
.vaddr
, 0, bo
->base
.base
.size
);
455 cma_obj
= drm_gem_cma_create(dev
, size
);
456 if (IS_ERR(cma_obj
)) {
458 * If we've run out of CMA memory, kill the cache of
459 * CMA allocations we've got laying around and try again.
461 vc4_bo_cache_purge(dev
);
462 cma_obj
= drm_gem_cma_create(dev
, size
);
465 if (IS_ERR(cma_obj
)) {
467 * Still not enough CMA memory, purge the userspace BO
469 * This is sub-optimal since we purge the whole userspace
470 * BO cache which forces user that want to re-use the BO to
471 * restore its initial content.
472 * Ideally, we should purge entries one by one and retry
473 * after each to see if CMA allocation succeeds. Or even
474 * better, try to find an entry with at least the same
477 vc4_bo_userspace_cache_purge(dev
);
478 cma_obj
= drm_gem_cma_create(dev
, size
);
481 if (IS_ERR(cma_obj
)) {
482 DRM_ERROR("Failed to allocate from CMA:\n");
483 vc4_bo_stats_dump(vc4
);
484 return ERR_PTR(-ENOMEM
);
486 bo
= to_vc4_bo(&cma_obj
->base
);
488 /* By default, BOs do not support the MADV ioctl. This will be enabled
489 * only on BOs that are exposed to userspace (V3D, V3D_SHADER and DUMB
492 bo
->madv
= __VC4_MADV_NOTSUPP
;
494 mutex_lock(&vc4
->bo_lock
);
495 vc4_bo_set_label(&cma_obj
->base
, type
);
496 mutex_unlock(&vc4
->bo_lock
);
501 int vc4_dumb_create(struct drm_file
*file_priv
,
502 struct drm_device
*dev
,
503 struct drm_mode_create_dumb
*args
)
505 int min_pitch
= DIV_ROUND_UP(args
->width
* args
->bpp
, 8);
506 struct vc4_bo
*bo
= NULL
;
509 if (args
->pitch
< min_pitch
)
510 args
->pitch
= min_pitch
;
512 if (args
->size
< args
->pitch
* args
->height
)
513 args
->size
= args
->pitch
* args
->height
;
515 bo
= vc4_bo_create(dev
, args
->size
, false, VC4_BO_TYPE_DUMB
);
519 bo
->madv
= VC4_MADV_WILLNEED
;
521 ret
= drm_gem_handle_create(file_priv
, &bo
->base
.base
, &args
->handle
);
522 drm_gem_object_put_unlocked(&bo
->base
.base
);
527 static void vc4_bo_cache_free_old(struct drm_device
*dev
)
529 struct vc4_dev
*vc4
= to_vc4_dev(dev
);
530 unsigned long expire_time
= jiffies
- msecs_to_jiffies(1000);
532 lockdep_assert_held(&vc4
->bo_lock
);
534 while (!list_empty(&vc4
->bo_cache
.time_list
)) {
535 struct vc4_bo
*bo
= list_last_entry(&vc4
->bo_cache
.time_list
,
536 struct vc4_bo
, unref_head
);
537 if (time_before(expire_time
, bo
->free_time
)) {
538 mod_timer(&vc4
->bo_cache
.time_timer
,
539 round_jiffies_up(jiffies
+
540 msecs_to_jiffies(1000)));
544 vc4_bo_remove_from_cache(bo
);
549 /* Called on the last userspace/kernel unreference of the BO. Returns
550 * it to the BO cache if possible, otherwise frees it.
552 void vc4_free_object(struct drm_gem_object
*gem_bo
)
554 struct drm_device
*dev
= gem_bo
->dev
;
555 struct vc4_dev
*vc4
= to_vc4_dev(dev
);
556 struct vc4_bo
*bo
= to_vc4_bo(gem_bo
);
557 struct list_head
*cache_list
;
559 /* Remove the BO from the purgeable list. */
560 mutex_lock(&bo
->madv_lock
);
561 if (bo
->madv
== VC4_MADV_DONTNEED
&& !refcount_read(&bo
->usecnt
))
562 vc4_bo_remove_from_purgeable_pool(bo
);
563 mutex_unlock(&bo
->madv_lock
);
565 mutex_lock(&vc4
->bo_lock
);
566 /* If the object references someone else's memory, we can't cache it.
568 if (gem_bo
->import_attach
) {
573 /* Don't cache if it was publicly named. */
579 /* If this object was partially constructed but CMA allocation
580 * had failed, just free it. Can also happen when the BO has been
583 if (!bo
->base
.vaddr
) {
588 cache_list
= vc4_get_cache_list_for_size(dev
, gem_bo
->size
);
594 if (bo
->validated_shader
) {
595 kfree(bo
->validated_shader
->uniform_addr_offsets
);
596 kfree(bo
->validated_shader
->texture_samples
);
597 kfree(bo
->validated_shader
);
598 bo
->validated_shader
= NULL
;
601 /* Reset madv and usecnt before adding the BO to the cache. */
602 bo
->madv
= __VC4_MADV_NOTSUPP
;
603 refcount_set(&bo
->usecnt
, 0);
605 bo
->t_format
= false;
606 bo
->free_time
= jiffies
;
607 list_add(&bo
->size_head
, cache_list
);
608 list_add(&bo
->unref_head
, &vc4
->bo_cache
.time_list
);
610 vc4_bo_set_label(&bo
->base
.base
, VC4_BO_TYPE_KERNEL_CACHE
);
612 vc4_bo_cache_free_old(dev
);
615 mutex_unlock(&vc4
->bo_lock
);
618 static void vc4_bo_cache_time_work(struct work_struct
*work
)
620 struct vc4_dev
*vc4
=
621 container_of(work
, struct vc4_dev
, bo_cache
.time_work
);
622 struct drm_device
*dev
= vc4
->dev
;
624 mutex_lock(&vc4
->bo_lock
);
625 vc4_bo_cache_free_old(dev
);
626 mutex_unlock(&vc4
->bo_lock
);
629 int vc4_bo_inc_usecnt(struct vc4_bo
*bo
)
633 /* Fast path: if the BO is already retained by someone, no need to
634 * check the madv status.
636 if (refcount_inc_not_zero(&bo
->usecnt
))
639 mutex_lock(&bo
->madv_lock
);
641 case VC4_MADV_WILLNEED
:
642 if (!refcount_inc_not_zero(&bo
->usecnt
))
643 refcount_set(&bo
->usecnt
, 1);
646 case VC4_MADV_DONTNEED
:
647 /* We shouldn't use a BO marked as purgeable if at least
648 * someone else retained its content by incrementing usecnt.
649 * Luckily the BO hasn't been purged yet, but something wrong
650 * is happening here. Just throw an error instead of
651 * authorizing this use case.
653 case __VC4_MADV_PURGED
:
654 /* We can't use a purged BO. */
656 /* Invalid madv value. */
660 mutex_unlock(&bo
->madv_lock
);
665 void vc4_bo_dec_usecnt(struct vc4_bo
*bo
)
667 /* Fast path: if the BO is still retained by someone, no need to test
670 if (refcount_dec_not_one(&bo
->usecnt
))
673 mutex_lock(&bo
->madv_lock
);
674 if (refcount_dec_and_test(&bo
->usecnt
) &&
675 bo
->madv
== VC4_MADV_DONTNEED
)
676 vc4_bo_add_to_purgeable_pool(bo
);
677 mutex_unlock(&bo
->madv_lock
);
680 static void vc4_bo_cache_time_timer(struct timer_list
*t
)
682 struct vc4_dev
*vc4
= from_timer(vc4
, t
, bo_cache
.time_timer
);
684 schedule_work(&vc4
->bo_cache
.time_work
);
687 struct reservation_object
*vc4_prime_res_obj(struct drm_gem_object
*obj
)
689 struct vc4_bo
*bo
= to_vc4_bo(obj
);
695 vc4_prime_export(struct drm_device
*dev
, struct drm_gem_object
*obj
, int flags
)
697 struct vc4_bo
*bo
= to_vc4_bo(obj
);
698 struct dma_buf
*dmabuf
;
701 if (bo
->validated_shader
) {
702 DRM_DEBUG("Attempting to export shader BO\n");
703 return ERR_PTR(-EINVAL
);
706 /* Note: as soon as the BO is exported it becomes unpurgeable, because
707 * noone ever decrements the usecnt even if the reference held by the
708 * exported BO is released. This shouldn't be a problem since we don't
709 * expect exported BOs to be marked as purgeable.
711 ret
= vc4_bo_inc_usecnt(bo
);
713 DRM_ERROR("Failed to increment BO usecnt\n");
717 dmabuf
= drm_gem_prime_export(dev
, obj
, flags
);
719 vc4_bo_dec_usecnt(bo
);
724 vm_fault_t
vc4_fault(struct vm_fault
*vmf
)
726 struct vm_area_struct
*vma
= vmf
->vma
;
727 struct drm_gem_object
*obj
= vma
->vm_private_data
;
728 struct vc4_bo
*bo
= to_vc4_bo(obj
);
730 /* The only reason we would end up here is when user-space accesses
731 * BO's memory after it's been purged.
733 mutex_lock(&bo
->madv_lock
);
734 WARN_ON(bo
->madv
!= __VC4_MADV_PURGED
);
735 mutex_unlock(&bo
->madv_lock
);
737 return VM_FAULT_SIGBUS
;
740 int vc4_mmap(struct file
*filp
, struct vm_area_struct
*vma
)
742 struct drm_gem_object
*gem_obj
;
743 unsigned long vm_pgoff
;
747 ret
= drm_gem_mmap(filp
, vma
);
751 gem_obj
= vma
->vm_private_data
;
752 bo
= to_vc4_bo(gem_obj
);
754 if (bo
->validated_shader
&& (vma
->vm_flags
& VM_WRITE
)) {
755 DRM_DEBUG("mmaping of shader BOs for writing not allowed.\n");
759 if (bo
->madv
!= VC4_MADV_WILLNEED
) {
760 DRM_DEBUG("mmaping of %s BO not allowed\n",
761 bo
->madv
== VC4_MADV_DONTNEED
?
762 "purgeable" : "purged");
767 * Clear the VM_PFNMAP flag that was set by drm_gem_mmap(), and set the
768 * vm_pgoff (used as a fake buffer offset by DRM) to 0 as we want to map
771 vma
->vm_flags
&= ~VM_PFNMAP
;
773 /* This ->vm_pgoff dance is needed to make all parties happy:
774 * - dma_mmap_wc() uses ->vm_pgoff as an offset within the allocated
775 * mem-region, hence the need to set it to zero (the value set by
776 * the DRM core is a virtual offset encoding the GEM object-id)
777 * - the mmap() core logic needs ->vm_pgoff to be restored to its
778 * initial value before returning from this function because it
779 * encodes the offset of this GEM in the dev->anon_inode pseudo-file
780 * and this information will be used when we invalidate userspace
781 * mappings with drm_vma_node_unmap() (called from vc4_gem_purge()).
783 vm_pgoff
= vma
->vm_pgoff
;
785 ret
= dma_mmap_wc(bo
->base
.base
.dev
->dev
, vma
, bo
->base
.vaddr
,
786 bo
->base
.paddr
, vma
->vm_end
- vma
->vm_start
);
787 vma
->vm_pgoff
= vm_pgoff
;
790 drm_gem_vm_close(vma
);
795 int vc4_prime_mmap(struct drm_gem_object
*obj
, struct vm_area_struct
*vma
)
797 struct vc4_bo
*bo
= to_vc4_bo(obj
);
799 if (bo
->validated_shader
&& (vma
->vm_flags
& VM_WRITE
)) {
800 DRM_DEBUG("mmaping of shader BOs for writing not allowed.\n");
804 return drm_gem_cma_prime_mmap(obj
, vma
);
807 void *vc4_prime_vmap(struct drm_gem_object
*obj
)
809 struct vc4_bo
*bo
= to_vc4_bo(obj
);
811 if (bo
->validated_shader
) {
812 DRM_DEBUG("mmaping of shader BOs not allowed.\n");
813 return ERR_PTR(-EINVAL
);
816 return drm_gem_cma_prime_vmap(obj
);
819 struct drm_gem_object
*
820 vc4_prime_import_sg_table(struct drm_device
*dev
,
821 struct dma_buf_attachment
*attach
,
822 struct sg_table
*sgt
)
824 struct drm_gem_object
*obj
;
827 obj
= drm_gem_cma_prime_import_sg_table(dev
, attach
, sgt
);
832 bo
->resv
= attach
->dmabuf
->resv
;
837 int vc4_create_bo_ioctl(struct drm_device
*dev
, void *data
,
838 struct drm_file
*file_priv
)
840 struct drm_vc4_create_bo
*args
= data
;
841 struct vc4_bo
*bo
= NULL
;
845 * We can't allocate from the BO cache, because the BOs don't
846 * get zeroed, and that might leak data between users.
848 bo
= vc4_bo_create(dev
, args
->size
, false, VC4_BO_TYPE_V3D
);
852 bo
->madv
= VC4_MADV_WILLNEED
;
854 ret
= drm_gem_handle_create(file_priv
, &bo
->base
.base
, &args
->handle
);
855 drm_gem_object_put_unlocked(&bo
->base
.base
);
860 int vc4_mmap_bo_ioctl(struct drm_device
*dev
, void *data
,
861 struct drm_file
*file_priv
)
863 struct drm_vc4_mmap_bo
*args
= data
;
864 struct drm_gem_object
*gem_obj
;
866 gem_obj
= drm_gem_object_lookup(file_priv
, args
->handle
);
868 DRM_DEBUG("Failed to look up GEM BO %d\n", args
->handle
);
872 /* The mmap offset was set up at BO allocation time. */
873 args
->offset
= drm_vma_node_offset_addr(&gem_obj
->vma_node
);
875 drm_gem_object_put_unlocked(gem_obj
);
880 vc4_create_shader_bo_ioctl(struct drm_device
*dev
, void *data
,
881 struct drm_file
*file_priv
)
883 struct drm_vc4_create_shader_bo
*args
= data
;
884 struct vc4_bo
*bo
= NULL
;
890 if (args
->size
% sizeof(u64
) != 0)
893 if (args
->flags
!= 0) {
894 DRM_INFO("Unknown flags set: 0x%08x\n", args
->flags
);
898 if (args
->pad
!= 0) {
899 DRM_INFO("Pad set: 0x%08x\n", args
->pad
);
903 bo
= vc4_bo_create(dev
, args
->size
, true, VC4_BO_TYPE_V3D_SHADER
);
907 bo
->madv
= VC4_MADV_WILLNEED
;
909 if (copy_from_user(bo
->base
.vaddr
,
910 (void __user
*)(uintptr_t)args
->data
,
915 /* Clear the rest of the memory from allocating from the BO
918 memset(bo
->base
.vaddr
+ args
->size
, 0,
919 bo
->base
.base
.size
- args
->size
);
921 bo
->validated_shader
= vc4_validate_shader(&bo
->base
);
922 if (!bo
->validated_shader
) {
927 /* We have to create the handle after validation, to avoid
928 * races for users to do doing things like mmap the shader BO.
930 ret
= drm_gem_handle_create(file_priv
, &bo
->base
.base
, &args
->handle
);
933 drm_gem_object_put_unlocked(&bo
->base
.base
);
939 * vc4_set_tiling_ioctl() - Sets the tiling modifier for a BO.
941 * @data: ioctl argument
942 * @file_priv: DRM file for this fd
944 * The tiling state of the BO decides the default modifier of an fb if
945 * no specific modifier was set by userspace, and the return value of
946 * vc4_get_tiling_ioctl() (so that userspace can treat a BO it
947 * received from dmabuf as the same tiling format as the producer
950 int vc4_set_tiling_ioctl(struct drm_device
*dev
, void *data
,
951 struct drm_file
*file_priv
)
953 struct drm_vc4_set_tiling
*args
= data
;
954 struct drm_gem_object
*gem_obj
;
958 if (args
->flags
!= 0)
961 switch (args
->modifier
) {
962 case DRM_FORMAT_MOD_NONE
:
965 case DRM_FORMAT_MOD_BROADCOM_VC4_T_TILED
:
972 gem_obj
= drm_gem_object_lookup(file_priv
, args
->handle
);
974 DRM_DEBUG("Failed to look up GEM BO %d\n", args
->handle
);
977 bo
= to_vc4_bo(gem_obj
);
978 bo
->t_format
= t_format
;
980 drm_gem_object_put_unlocked(gem_obj
);
986 * vc4_get_tiling_ioctl() - Gets the tiling modifier for a BO.
988 * @data: ioctl argument
989 * @file_priv: DRM file for this fd
991 * Returns the tiling modifier for a BO as set by vc4_set_tiling_ioctl().
993 int vc4_get_tiling_ioctl(struct drm_device
*dev
, void *data
,
994 struct drm_file
*file_priv
)
996 struct drm_vc4_get_tiling
*args
= data
;
997 struct drm_gem_object
*gem_obj
;
1000 if (args
->flags
!= 0 || args
->modifier
!= 0)
1003 gem_obj
= drm_gem_object_lookup(file_priv
, args
->handle
);
1005 DRM_DEBUG("Failed to look up GEM BO %d\n", args
->handle
);
1008 bo
= to_vc4_bo(gem_obj
);
1011 args
->modifier
= DRM_FORMAT_MOD_BROADCOM_VC4_T_TILED
;
1013 args
->modifier
= DRM_FORMAT_MOD_NONE
;
1015 drm_gem_object_put_unlocked(gem_obj
);
1020 int vc4_bo_cache_init(struct drm_device
*dev
)
1022 struct vc4_dev
*vc4
= to_vc4_dev(dev
);
1025 /* Create the initial set of BO labels that the kernel will
1026 * use. This lets us avoid a bunch of string reallocation in
1027 * the kernel's draw and BO allocation paths.
1029 vc4
->bo_labels
= kcalloc(VC4_BO_TYPE_COUNT
, sizeof(*vc4
->bo_labels
),
1031 if (!vc4
->bo_labels
)
1033 vc4
->num_labels
= VC4_BO_TYPE_COUNT
;
1035 BUILD_BUG_ON(ARRAY_SIZE(bo_type_names
) != VC4_BO_TYPE_COUNT
);
1036 for (i
= 0; i
< VC4_BO_TYPE_COUNT
; i
++)
1037 vc4
->bo_labels
[i
].name
= bo_type_names
[i
];
1039 mutex_init(&vc4
->bo_lock
);
1041 INIT_LIST_HEAD(&vc4
->bo_cache
.time_list
);
1043 INIT_WORK(&vc4
->bo_cache
.time_work
, vc4_bo_cache_time_work
);
1044 timer_setup(&vc4
->bo_cache
.time_timer
, vc4_bo_cache_time_timer
, 0);
1049 void vc4_bo_cache_destroy(struct drm_device
*dev
)
1051 struct vc4_dev
*vc4
= to_vc4_dev(dev
);
1054 del_timer(&vc4
->bo_cache
.time_timer
);
1055 cancel_work_sync(&vc4
->bo_cache
.time_work
);
1057 vc4_bo_cache_purge(dev
);
1059 for (i
= 0; i
< vc4
->num_labels
; i
++) {
1060 if (vc4
->bo_labels
[i
].num_allocated
) {
1061 DRM_ERROR("Destroying BO cache with %d %s "
1062 "BOs still allocated\n",
1063 vc4
->bo_labels
[i
].num_allocated
,
1064 vc4
->bo_labels
[i
].name
);
1067 if (is_user_label(i
))
1068 kfree(vc4
->bo_labels
[i
].name
);
1070 kfree(vc4
->bo_labels
);
1073 int vc4_label_bo_ioctl(struct drm_device
*dev
, void *data
,
1074 struct drm_file
*file_priv
)
1076 struct vc4_dev
*vc4
= to_vc4_dev(dev
);
1077 struct drm_vc4_label_bo
*args
= data
;
1079 struct drm_gem_object
*gem_obj
;
1085 name
= strndup_user(u64_to_user_ptr(args
->name
), args
->len
+ 1);
1087 return PTR_ERR(name
);
1089 gem_obj
= drm_gem_object_lookup(file_priv
, args
->handle
);
1091 DRM_ERROR("Failed to look up GEM BO %d\n", args
->handle
);
1096 mutex_lock(&vc4
->bo_lock
);
1097 label
= vc4_get_user_label(vc4
, name
);
1099 vc4_bo_set_label(gem_obj
, label
);
1102 mutex_unlock(&vc4
->bo_lock
);
1104 drm_gem_object_put_unlocked(gem_obj
);