2 * Copyright (C) 2012-2014 Canonical Ltd (Maarten Lankhorst)
4 * Based on bo.c which bears the following copyright notice,
5 * but is dual licensed:
7 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
10 * Permission is hereby granted, free of charge, to any person obtaining a
11 * copy of this software and associated documentation files (the
12 * "Software"), to deal in the Software without restriction, including
13 * without limitation the rights to use, copy, modify, merge, publish,
14 * distribute, sub license, and/or sell copies of the Software, and to
15 * permit persons to whom the Software is furnished to do so, subject to
16 * the following conditions:
18 * The above copyright notice and this permission notice (including the
19 * next paragraph) shall be included in all copies or substantial portions
22 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
23 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
24 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
25 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
26 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
27 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
28 * USE OR OTHER DEALINGS IN THE SOFTWARE.
30 **************************************************************************/
32 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
35 #include <linux/reservation.h>
36 #include <linux/export.h>
39 * DOC: Reservation Object Overview
41 * The reservation object provides a mechanism to manage shared and
42 * exclusive fences associated with a buffer. A reservation object
43 * can have attached one exclusive fence (normally associated with
44 * write operations) or N shared fences (read operations). The RCU
45 * mechanism is used to protect read access to fences from locked
49 DEFINE_WW_CLASS(reservation_ww_class
);
50 EXPORT_SYMBOL(reservation_ww_class
);
52 struct lock_class_key reservation_seqcount_class
;
53 EXPORT_SYMBOL(reservation_seqcount_class
);
55 const char reservation_seqcount_string
[] = "reservation_seqcount";
56 EXPORT_SYMBOL(reservation_seqcount_string
);
59 * reservation_object_reserve_shared - Reserve space to add a shared
60 * fence to a reservation_object.
61 * @obj: reservation object
63 * Should be called before reservation_object_add_shared_fence(). Must
64 * be called with obj->lock held.
67 * Zero for success, or -errno
69 int reservation_object_reserve_shared(struct reservation_object
*obj
)
71 struct reservation_object_list
*fobj
, *old
;
74 old
= reservation_object_get_list(obj
);
76 if (old
&& old
->shared_max
) {
77 if (old
->shared_count
< old
->shared_max
) {
78 /* perform an in-place update */
83 max
= old
->shared_max
* 2;
88 * resize obj->staged or allocate if it doesn't exist,
89 * noop if already correct size
91 fobj
= krealloc(obj
->staged
, offsetof(typeof(*fobj
), shared
[max
]),
97 fobj
->shared_max
= max
;
100 EXPORT_SYMBOL(reservation_object_reserve_shared
);
103 reservation_object_add_shared_inplace(struct reservation_object
*obj
,
104 struct reservation_object_list
*fobj
,
105 struct dma_fence
*fence
)
109 dma_fence_get(fence
);
112 write_seqcount_begin(&obj
->seq
);
114 for (i
= 0; i
< fobj
->shared_count
; ++i
) {
115 struct dma_fence
*old_fence
;
117 old_fence
= rcu_dereference_protected(fobj
->shared
[i
],
118 reservation_object_held(obj
));
120 if (old_fence
->context
== fence
->context
) {
121 /* memory barrier is added by write_seqcount_begin */
122 RCU_INIT_POINTER(fobj
->shared
[i
], fence
);
123 write_seqcount_end(&obj
->seq
);
126 dma_fence_put(old_fence
);
132 * memory barrier is added by write_seqcount_begin,
133 * fobj->shared_count is protected by this lock too
135 RCU_INIT_POINTER(fobj
->shared
[fobj
->shared_count
], fence
);
136 fobj
->shared_count
++;
138 write_seqcount_end(&obj
->seq
);
143 reservation_object_add_shared_replace(struct reservation_object
*obj
,
144 struct reservation_object_list
*old
,
145 struct reservation_object_list
*fobj
,
146 struct dma_fence
*fence
)
149 struct dma_fence
*old_fence
= NULL
;
151 dma_fence_get(fence
);
154 RCU_INIT_POINTER(fobj
->shared
[0], fence
);
155 fobj
->shared_count
= 1;
160 * no need to bump fence refcounts, rcu_read access
161 * requires the use of kref_get_unless_zero, and the
162 * references from the old struct are carried over to
165 fobj
->shared_count
= old
->shared_count
;
167 for (i
= 0; i
< old
->shared_count
; ++i
) {
168 struct dma_fence
*check
;
170 check
= rcu_dereference_protected(old
->shared
[i
],
171 reservation_object_held(obj
));
173 if (!old_fence
&& check
->context
== fence
->context
) {
175 RCU_INIT_POINTER(fobj
->shared
[i
], fence
);
177 RCU_INIT_POINTER(fobj
->shared
[i
], check
);
180 RCU_INIT_POINTER(fobj
->shared
[fobj
->shared_count
], fence
);
181 fobj
->shared_count
++;
186 write_seqcount_begin(&obj
->seq
);
188 * RCU_INIT_POINTER can be used here,
189 * seqcount provides the necessary barriers
191 RCU_INIT_POINTER(obj
->fence
, fobj
);
192 write_seqcount_end(&obj
->seq
);
198 dma_fence_put(old_fence
);
202 * reservation_object_add_shared_fence - Add a fence to a shared slot
203 * @obj: the reservation object
204 * @fence: the shared fence to add
206 * Add a fence to a shared slot, obj->lock must be held, and
207 * reservation_object_reserve_shared() has been called.
209 void reservation_object_add_shared_fence(struct reservation_object
*obj
,
210 struct dma_fence
*fence
)
212 struct reservation_object_list
*old
, *fobj
= obj
->staged
;
214 old
= reservation_object_get_list(obj
);
218 BUG_ON(old
->shared_count
>= old
->shared_max
);
219 reservation_object_add_shared_inplace(obj
, old
, fence
);
221 reservation_object_add_shared_replace(obj
, old
, fobj
, fence
);
223 EXPORT_SYMBOL(reservation_object_add_shared_fence
);
226 * reservation_object_add_excl_fence - Add an exclusive fence.
227 * @obj: the reservation object
228 * @fence: the shared fence to add
230 * Add a fence to the exclusive slot. The obj->lock must be held.
232 void reservation_object_add_excl_fence(struct reservation_object
*obj
,
233 struct dma_fence
*fence
)
235 struct dma_fence
*old_fence
= reservation_object_get_excl(obj
);
236 struct reservation_object_list
*old
;
239 old
= reservation_object_get_list(obj
);
241 i
= old
->shared_count
;
244 dma_fence_get(fence
);
247 write_seqcount_begin(&obj
->seq
);
248 /* write_seqcount_begin provides the necessary memory barrier */
249 RCU_INIT_POINTER(obj
->fence_excl
, fence
);
251 old
->shared_count
= 0;
252 write_seqcount_end(&obj
->seq
);
255 /* inplace update, no shared fences */
257 dma_fence_put(rcu_dereference_protected(old
->shared
[i
],
258 reservation_object_held(obj
)));
260 dma_fence_put(old_fence
);
262 EXPORT_SYMBOL(reservation_object_add_excl_fence
);
265 * reservation_object_copy_fences - Copy all fences from src to dst.
266 * @dst: the destination reservation object
267 * @src: the source reservation object
269 * Copy all fences from src to dst. dst-lock must be held.
271 int reservation_object_copy_fences(struct reservation_object
*dst
,
272 struct reservation_object
*src
)
274 struct reservation_object_list
*src_list
, *dst_list
;
275 struct dma_fence
*old
, *new;
280 src_list
= rcu_dereference(src
->fence
);
284 unsigned shared_count
= src_list
->shared_count
;
286 size
= offsetof(typeof(*src_list
), shared
[shared_count
]);
289 dst_list
= kmalloc(size
, GFP_KERNEL
);
294 src_list
= rcu_dereference(src
->fence
);
295 if (!src_list
|| src_list
->shared_count
> shared_count
) {
300 dst_list
->shared_count
= 0;
301 dst_list
->shared_max
= shared_count
;
302 for (i
= 0; i
< src_list
->shared_count
; ++i
) {
303 struct dma_fence
*fence
;
305 fence
= rcu_dereference(src_list
->shared
[i
]);
306 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT
,
310 if (!dma_fence_get_rcu(fence
)) {
312 src_list
= rcu_dereference(src
->fence
);
316 if (dma_fence_is_signaled(fence
)) {
317 dma_fence_put(fence
);
321 dst_list
->shared
[dst_list
->shared_count
++] = fence
;
327 new = dma_fence_get_rcu_safe(&src
->fence_excl
);
333 src_list
= reservation_object_get_list(dst
);
334 old
= reservation_object_get_excl(dst
);
337 write_seqcount_begin(&dst
->seq
);
338 /* write_seqcount_begin provides the necessary memory barrier */
339 RCU_INIT_POINTER(dst
->fence_excl
, new);
340 RCU_INIT_POINTER(dst
->fence
, dst_list
);
341 write_seqcount_end(&dst
->seq
);
345 kfree_rcu(src_list
, rcu
);
350 EXPORT_SYMBOL(reservation_object_copy_fences
);
353 * reservation_object_get_fences_rcu - Get an object's shared and exclusive
354 * fences without update side lock held
355 * @obj: the reservation object
356 * @pfence_excl: the returned exclusive fence (or NULL)
357 * @pshared_count: the number of shared fences returned
358 * @pshared: the array of shared fence ptrs returned (array is krealloc'd to
359 * the required size, and must be freed by caller)
364 int reservation_object_get_fences_rcu(struct reservation_object
*obj
,
365 struct dma_fence
**pfence_excl
,
366 unsigned *pshared_count
,
367 struct dma_fence
***pshared
)
369 struct dma_fence
**shared
= NULL
;
370 struct dma_fence
*fence_excl
;
371 unsigned int shared_count
;
375 struct reservation_object_list
*fobj
;
379 shared_count
= i
= 0;
382 seq
= read_seqcount_begin(&obj
->seq
);
384 fence_excl
= rcu_dereference(obj
->fence_excl
);
385 if (fence_excl
&& !dma_fence_get_rcu(fence_excl
))
388 fobj
= rcu_dereference(obj
->fence
);
390 struct dma_fence
**nshared
;
391 size_t sz
= sizeof(*shared
) * fobj
->shared_max
;
393 nshared
= krealloc(shared
, sz
,
394 GFP_NOWAIT
| __GFP_NOWARN
);
397 nshared
= krealloc(shared
, sz
, GFP_KERNEL
);
407 shared_count
= fobj
->shared_count
;
409 for (i
= 0; i
< shared_count
; ++i
) {
410 shared
[i
] = rcu_dereference(fobj
->shared
[i
]);
411 if (!dma_fence_get_rcu(shared
[i
]))
416 if (i
!= shared_count
|| read_seqcount_retry(&obj
->seq
, seq
)) {
418 dma_fence_put(shared
[i
]);
419 dma_fence_put(fence_excl
);
433 *pshared_count
= shared_count
;
435 *pfence_excl
= fence_excl
;
439 EXPORT_SYMBOL_GPL(reservation_object_get_fences_rcu
);
442 * reservation_object_wait_timeout_rcu - Wait on reservation's objects
443 * shared and/or exclusive fences.
444 * @obj: the reservation object
445 * @wait_all: if true, wait on all fences, else wait on just exclusive fence
446 * @intr: if true, do interruptible wait
447 * @timeout: timeout value in jiffies or zero to return immediately
450 * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or
451 * greater than zer on success.
453 long reservation_object_wait_timeout_rcu(struct reservation_object
*obj
,
454 bool wait_all
, bool intr
,
455 unsigned long timeout
)
457 struct dma_fence
*fence
;
458 unsigned seq
, shared_count
;
459 long ret
= timeout
? timeout
: 1;
464 seq
= read_seqcount_begin(&obj
->seq
);
468 fence
= rcu_dereference(obj
->fence_excl
);
469 if (fence
&& !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT
, &fence
->flags
)) {
470 if (!dma_fence_get_rcu(fence
))
473 if (dma_fence_is_signaled(fence
)) {
474 dma_fence_put(fence
);
483 struct reservation_object_list
*fobj
=
484 rcu_dereference(obj
->fence
);
487 shared_count
= fobj
->shared_count
;
489 for (i
= 0; !fence
&& i
< shared_count
; ++i
) {
490 struct dma_fence
*lfence
= rcu_dereference(fobj
->shared
[i
]);
492 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT
,
496 if (!dma_fence_get_rcu(lfence
))
499 if (dma_fence_is_signaled(lfence
)) {
500 dma_fence_put(lfence
);
511 if (read_seqcount_retry(&obj
->seq
, seq
)) {
512 dma_fence_put(fence
);
516 ret
= dma_fence_wait_timeout(fence
, intr
, ret
);
517 dma_fence_put(fence
);
518 if (ret
> 0 && wait_all
&& (i
+ 1 < shared_count
))
527 EXPORT_SYMBOL_GPL(reservation_object_wait_timeout_rcu
);
531 reservation_object_test_signaled_single(struct dma_fence
*passed_fence
)
533 struct dma_fence
*fence
, *lfence
= passed_fence
;
536 if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT
, &lfence
->flags
)) {
537 fence
= dma_fence_get_rcu(lfence
);
541 ret
= !!dma_fence_is_signaled(fence
);
542 dma_fence_put(fence
);
548 * reservation_object_test_signaled_rcu - Test if a reservation object's
549 * fences have been signaled.
550 * @obj: the reservation object
551 * @test_all: if true, test all fences, otherwise only test the exclusive
555 * true if all fences signaled, else false
557 bool reservation_object_test_signaled_rcu(struct reservation_object
*obj
,
560 unsigned seq
, shared_count
;
567 seq
= read_seqcount_begin(&obj
->seq
);
572 struct reservation_object_list
*fobj
=
573 rcu_dereference(obj
->fence
);
576 shared_count
= fobj
->shared_count
;
578 for (i
= 0; i
< shared_count
; ++i
) {
579 struct dma_fence
*fence
= rcu_dereference(fobj
->shared
[i
]);
581 ret
= reservation_object_test_signaled_single(fence
);
588 if (read_seqcount_retry(&obj
->seq
, seq
))
593 struct dma_fence
*fence_excl
= rcu_dereference(obj
->fence_excl
);
596 ret
= reservation_object_test_signaled_single(
601 if (read_seqcount_retry(&obj
->seq
, seq
))
609 EXPORT_SYMBOL_GPL(reservation_object_test_signaled_rcu
);