2 * Copyright 2015 Advanced Micro Devices, Inc.
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sub license, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19 * USE OR OTHER DEALINGS IN THE SOFTWARE.
21 * The above copyright notice and this permission notice (including the
22 * next paragraph) shall be included in all copies or substantial portions
28 * Christian König <deathsimple@vodafone.de>
33 #include "amdgpu_trace.h"
35 #define AMDGPU_BO_LIST_MAX_PRIORITY 32u
36 #define AMDGPU_BO_LIST_NUM_BUCKETS (AMDGPU_BO_LIST_MAX_PRIORITY + 1)
38 static void amdgpu_bo_list_free_rcu(struct rcu_head
*rcu
)
40 struct amdgpu_bo_list
*list
= container_of(rcu
, struct amdgpu_bo_list
,
46 static void amdgpu_bo_list_free(struct kref
*ref
)
48 struct amdgpu_bo_list
*list
= container_of(ref
, struct amdgpu_bo_list
,
50 struct amdgpu_bo_list_entry
*e
;
52 amdgpu_bo_list_for_each_entry(e
, list
)
53 amdgpu_bo_unref(&e
->robj
);
55 call_rcu(&list
->rhead
, amdgpu_bo_list_free_rcu
);
58 int amdgpu_bo_list_create(struct amdgpu_device
*adev
, struct drm_file
*filp
,
59 struct drm_amdgpu_bo_list_entry
*info
,
60 unsigned num_entries
, struct amdgpu_bo_list
**result
)
62 unsigned last_entry
= 0, first_userptr
= num_entries
;
63 struct amdgpu_bo_list_entry
*array
;
64 struct amdgpu_bo_list
*list
;
65 uint64_t total_size
= 0;
70 if (num_entries
> (SIZE_MAX
- sizeof(struct amdgpu_bo_list
))
71 / sizeof(struct amdgpu_bo_list_entry
))
74 size
= sizeof(struct amdgpu_bo_list
);
75 size
+= num_entries
* sizeof(struct amdgpu_bo_list_entry
);
76 list
= kvmalloc(size
, GFP_KERNEL
);
80 kref_init(&list
->refcount
);
81 list
->gds_obj
= adev
->gds
.gds_gfx_bo
;
82 list
->gws_obj
= adev
->gds
.gws_gfx_bo
;
83 list
->oa_obj
= adev
->gds
.oa_gfx_bo
;
85 array
= amdgpu_bo_list_array_entry(list
, 0);
86 memset(array
, 0, num_entries
* sizeof(struct amdgpu_bo_list_entry
));
88 for (i
= 0; i
< num_entries
; ++i
) {
89 struct amdgpu_bo_list_entry
*entry
;
90 struct drm_gem_object
*gobj
;
92 struct mm_struct
*usermm
;
94 gobj
= drm_gem_object_lookup(filp
, info
[i
].bo_handle
);
100 bo
= amdgpu_bo_ref(gem_to_amdgpu_bo(gobj
));
101 drm_gem_object_put_unlocked(gobj
);
103 usermm
= amdgpu_ttm_tt_get_usermm(bo
->tbo
.ttm
);
105 if (usermm
!= current
->mm
) {
106 amdgpu_bo_unref(&bo
);
110 entry
= &array
[--first_userptr
];
112 entry
= &array
[last_entry
++];
116 entry
->priority
= min(info
[i
].bo_priority
,
117 AMDGPU_BO_LIST_MAX_PRIORITY
);
118 entry
->tv
.bo
= &entry
->robj
->tbo
;
119 entry
->tv
.shared
= !entry
->robj
->prime_shared_count
;
121 if (entry
->robj
->preferred_domains
== AMDGPU_GEM_DOMAIN_GDS
)
122 list
->gds_obj
= entry
->robj
;
123 if (entry
->robj
->preferred_domains
== AMDGPU_GEM_DOMAIN_GWS
)
124 list
->gws_obj
= entry
->robj
;
125 if (entry
->robj
->preferred_domains
== AMDGPU_GEM_DOMAIN_OA
)
126 list
->oa_obj
= entry
->robj
;
128 total_size
+= amdgpu_bo_size(entry
->robj
);
129 trace_amdgpu_bo_list_set(list
, entry
->robj
);
132 list
->first_userptr
= first_userptr
;
133 list
->num_entries
= num_entries
;
135 trace_amdgpu_cs_bo_status(list
->num_entries
, total_size
);
142 amdgpu_bo_unref(&array
[i
].robj
);
148 static void amdgpu_bo_list_destroy(struct amdgpu_fpriv
*fpriv
, int id
)
150 struct amdgpu_bo_list
*list
;
152 mutex_lock(&fpriv
->bo_list_lock
);
153 list
= idr_remove(&fpriv
->bo_list_handles
, id
);
154 mutex_unlock(&fpriv
->bo_list_lock
);
156 kref_put(&list
->refcount
, amdgpu_bo_list_free
);
159 int amdgpu_bo_list_get(struct amdgpu_fpriv
*fpriv
, int id
,
160 struct amdgpu_bo_list
**result
)
163 *result
= idr_find(&fpriv
->bo_list_handles
, id
);
165 if (*result
&& kref_get_unless_zero(&(*result
)->refcount
)) {
174 void amdgpu_bo_list_get_list(struct amdgpu_bo_list
*list
,
175 struct list_head
*validated
)
177 /* This is based on the bucket sort with O(n) time complexity.
178 * An item with priority "i" is added to bucket[i]. The lists are then
179 * concatenated in descending order.
181 struct list_head bucket
[AMDGPU_BO_LIST_NUM_BUCKETS
];
182 struct amdgpu_bo_list_entry
*e
;
185 for (i
= 0; i
< AMDGPU_BO_LIST_NUM_BUCKETS
; i
++)
186 INIT_LIST_HEAD(&bucket
[i
]);
188 /* Since buffers which appear sooner in the relocation list are
189 * likely to be used more often than buffers which appear later
190 * in the list, the sort mustn't change the ordering of buffers
191 * with the same priority, i.e. it must be stable.
193 amdgpu_bo_list_for_each_entry(e
, list
) {
194 unsigned priority
= e
->priority
;
196 if (!e
->robj
->parent
)
197 list_add_tail(&e
->tv
.head
, &bucket
[priority
]);
199 e
->user_pages
= NULL
;
202 /* Connect the sorted buckets in the output list. */
203 for (i
= 0; i
< AMDGPU_BO_LIST_NUM_BUCKETS
; i
++)
204 list_splice(&bucket
[i
], validated
);
207 void amdgpu_bo_list_put(struct amdgpu_bo_list
*list
)
209 kref_put(&list
->refcount
, amdgpu_bo_list_free
);
212 int amdgpu_bo_create_list_entry_array(struct drm_amdgpu_bo_list_in
*in
,
213 struct drm_amdgpu_bo_list_entry
**info_param
)
215 const void __user
*uptr
= u64_to_user_ptr(in
->bo_info_ptr
);
216 const uint32_t info_size
= sizeof(struct drm_amdgpu_bo_list_entry
);
217 struct drm_amdgpu_bo_list_entry
*info
;
220 info
= kvmalloc_array(in
->bo_number
, info_size
, GFP_KERNEL
);
224 /* copy the handle array from userspace to a kernel buffer */
226 if (likely(info_size
== in
->bo_info_size
)) {
227 unsigned long bytes
= in
->bo_number
*
230 if (copy_from_user(info
, uptr
, bytes
))
234 unsigned long bytes
= min(in
->bo_info_size
, info_size
);
237 memset(info
, 0, in
->bo_number
* info_size
);
238 for (i
= 0; i
< in
->bo_number
; ++i
) {
239 if (copy_from_user(&info
[i
], uptr
, bytes
))
242 uptr
+= in
->bo_info_size
;
254 int amdgpu_bo_list_ioctl(struct drm_device
*dev
, void *data
,
255 struct drm_file
*filp
)
257 struct amdgpu_device
*adev
= dev
->dev_private
;
258 struct amdgpu_fpriv
*fpriv
= filp
->driver_priv
;
259 union drm_amdgpu_bo_list
*args
= data
;
260 uint32_t handle
= args
->in
.list_handle
;
261 struct drm_amdgpu_bo_list_entry
*info
= NULL
;
262 struct amdgpu_bo_list
*list
, *old
;
265 r
= amdgpu_bo_create_list_entry_array(&args
->in
, &info
);
269 switch (args
->in
.operation
) {
270 case AMDGPU_BO_LIST_OP_CREATE
:
271 r
= amdgpu_bo_list_create(adev
, filp
, info
, args
->in
.bo_number
,
276 mutex_lock(&fpriv
->bo_list_lock
);
277 r
= idr_alloc(&fpriv
->bo_list_handles
, list
, 1, 0, GFP_KERNEL
);
278 mutex_unlock(&fpriv
->bo_list_lock
);
286 case AMDGPU_BO_LIST_OP_DESTROY
:
287 amdgpu_bo_list_destroy(fpriv
, handle
);
291 case AMDGPU_BO_LIST_OP_UPDATE
:
292 r
= amdgpu_bo_list_create(adev
, filp
, info
, args
->in
.bo_number
,
297 mutex_lock(&fpriv
->bo_list_lock
);
298 old
= idr_replace(&fpriv
->bo_list_handles
, list
, handle
);
299 mutex_unlock(&fpriv
->bo_list_lock
);
306 amdgpu_bo_list_put(old
);
314 memset(args
, 0, sizeof(*args
));
315 args
->out
.list_handle
= handle
;
321 amdgpu_bo_list_put(list
);