2 * Copyright (C) 2012-2014 Canonical Ltd (Maarten Lankhorst)
4 * Based on bo.c which bears the following copyright notice,
5 * but is dual licensed:
7 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
10 * Permission is hereby granted, free of charge, to any person obtaining a
11 * copy of this software and associated documentation files (the
12 * "Software"), to deal in the Software without restriction, including
13 * without limitation the rights to use, copy, modify, merge, publish,
14 * distribute, sub license, and/or sell copies of the Software, and to
15 * permit persons to whom the Software is furnished to do so, subject to
16 * the following conditions:
18 * The above copyright notice and this permission notice (including the
19 * next paragraph) shall be included in all copies or substantial portions
22 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
23 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
24 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
25 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
26 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
27 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
28 * USE OR OTHER DEALINGS IN THE SOFTWARE.
30 **************************************************************************/
32 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
35 #include <linux/reservation.h>
36 #include <linux/export.h>
39 * DOC: Reservation Object Overview
41 * The reservation object provides a mechanism to manage shared and
42 * exclusive fences associated with a buffer. A reservation object
43 * can have attached one exclusive fence (normally associated with
44 * write operations) or N shared fences (read operations). The RCU
45 * mechanism is used to protect read access to fences from locked
49 DEFINE_WW_CLASS(reservation_ww_class
);
50 EXPORT_SYMBOL(reservation_ww_class
);
52 struct lock_class_key reservation_seqcount_class
;
53 EXPORT_SYMBOL(reservation_seqcount_class
);
55 const char reservation_seqcount_string
[] = "reservation_seqcount";
56 EXPORT_SYMBOL(reservation_seqcount_string
);
59 * reservation_object_reserve_shared - Reserve space to add a shared
60 * fence to a reservation_object.
61 * @obj: reservation object
63 * Should be called before reservation_object_add_shared_fence(). Must
64 * be called with obj->lock held.
67 * Zero for success, or -errno
69 int reservation_object_reserve_shared(struct reservation_object
*obj
)
71 struct reservation_object_list
*fobj
, *old
;
74 old
= reservation_object_get_list(obj
);
76 if (old
&& old
->shared_max
) {
77 if (old
->shared_count
< old
->shared_max
) {
78 /* perform an in-place update */
83 max
= old
->shared_max
* 2;
88 * resize obj->staged or allocate if it doesn't exist,
89 * noop if already correct size
91 fobj
= krealloc(obj
->staged
, offsetof(typeof(*fobj
), shared
[max
]),
97 fobj
->shared_max
= max
;
100 EXPORT_SYMBOL(reservation_object_reserve_shared
);
103 reservation_object_add_shared_inplace(struct reservation_object
*obj
,
104 struct reservation_object_list
*fobj
,
105 struct dma_fence
*fence
)
107 struct dma_fence
*signaled
= NULL
;
110 dma_fence_get(fence
);
113 write_seqcount_begin(&obj
->seq
);
115 for (i
= 0; i
< fobj
->shared_count
; ++i
) {
116 struct dma_fence
*old_fence
;
118 old_fence
= rcu_dereference_protected(fobj
->shared
[i
],
119 reservation_object_held(obj
));
121 if (old_fence
->context
== fence
->context
) {
122 /* memory barrier is added by write_seqcount_begin */
123 RCU_INIT_POINTER(fobj
->shared
[i
], fence
);
124 write_seqcount_end(&obj
->seq
);
127 dma_fence_put(old_fence
);
131 if (!signaled
&& dma_fence_is_signaled(old_fence
)) {
132 signaled
= old_fence
;
138 * memory barrier is added by write_seqcount_begin,
139 * fobj->shared_count is protected by this lock too
142 RCU_INIT_POINTER(fobj
->shared
[signaled_idx
], fence
);
144 RCU_INIT_POINTER(fobj
->shared
[fobj
->shared_count
], fence
);
145 fobj
->shared_count
++;
148 write_seqcount_end(&obj
->seq
);
151 dma_fence_put(signaled
);
155 reservation_object_add_shared_replace(struct reservation_object
*obj
,
156 struct reservation_object_list
*old
,
157 struct reservation_object_list
*fobj
,
158 struct dma_fence
*fence
)
162 dma_fence_get(fence
);
165 RCU_INIT_POINTER(fobj
->shared
[0], fence
);
166 fobj
->shared_count
= 1;
171 * no need to bump fence refcounts, rcu_read access
172 * requires the use of kref_get_unless_zero, and the
173 * references from the old struct are carried over to
176 for (i
= 0, j
= 0, k
= fobj
->shared_max
; i
< old
->shared_count
; ++i
) {
177 struct dma_fence
*check
;
179 check
= rcu_dereference_protected(old
->shared
[i
],
180 reservation_object_held(obj
));
182 if (check
->context
== fence
->context
||
183 dma_fence_is_signaled(check
))
184 RCU_INIT_POINTER(fobj
->shared
[--k
], check
);
186 RCU_INIT_POINTER(fobj
->shared
[j
++], check
);
188 fobj
->shared_count
= j
;
189 RCU_INIT_POINTER(fobj
->shared
[fobj
->shared_count
], fence
);
190 fobj
->shared_count
++;
194 write_seqcount_begin(&obj
->seq
);
196 * RCU_INIT_POINTER can be used here,
197 * seqcount provides the necessary barriers
199 RCU_INIT_POINTER(obj
->fence
, fobj
);
200 write_seqcount_end(&obj
->seq
);
206 /* Drop the references to the signaled fences */
207 for (i
= k
; i
< fobj
->shared_max
; ++i
) {
210 f
= rcu_dereference_protected(fobj
->shared
[i
],
211 reservation_object_held(obj
));
218 * reservation_object_add_shared_fence - Add a fence to a shared slot
219 * @obj: the reservation object
220 * @fence: the shared fence to add
222 * Add a fence to a shared slot, obj->lock must be held, and
223 * reservation_object_reserve_shared() has been called.
225 void reservation_object_add_shared_fence(struct reservation_object
*obj
,
226 struct dma_fence
*fence
)
228 struct reservation_object_list
*old
, *fobj
= obj
->staged
;
230 old
= reservation_object_get_list(obj
);
234 BUG_ON(old
->shared_count
>= old
->shared_max
);
235 reservation_object_add_shared_inplace(obj
, old
, fence
);
237 reservation_object_add_shared_replace(obj
, old
, fobj
, fence
);
239 EXPORT_SYMBOL(reservation_object_add_shared_fence
);
242 * reservation_object_add_excl_fence - Add an exclusive fence.
243 * @obj: the reservation object
244 * @fence: the shared fence to add
246 * Add a fence to the exclusive slot. The obj->lock must be held.
248 void reservation_object_add_excl_fence(struct reservation_object
*obj
,
249 struct dma_fence
*fence
)
251 struct dma_fence
*old_fence
= reservation_object_get_excl(obj
);
252 struct reservation_object_list
*old
;
255 old
= reservation_object_get_list(obj
);
257 i
= old
->shared_count
;
260 dma_fence_get(fence
);
263 write_seqcount_begin(&obj
->seq
);
264 /* write_seqcount_begin provides the necessary memory barrier */
265 RCU_INIT_POINTER(obj
->fence_excl
, fence
);
267 old
->shared_count
= 0;
268 write_seqcount_end(&obj
->seq
);
271 /* inplace update, no shared fences */
273 dma_fence_put(rcu_dereference_protected(old
->shared
[i
],
274 reservation_object_held(obj
)));
276 dma_fence_put(old_fence
);
278 EXPORT_SYMBOL(reservation_object_add_excl_fence
);
281 * reservation_object_copy_fences - Copy all fences from src to dst.
282 * @dst: the destination reservation object
283 * @src: the source reservation object
285 * Copy all fences from src to dst. dst-lock must be held.
287 int reservation_object_copy_fences(struct reservation_object
*dst
,
288 struct reservation_object
*src
)
290 struct reservation_object_list
*src_list
, *dst_list
;
291 struct dma_fence
*old
, *new;
296 src_list
= rcu_dereference(src
->fence
);
300 unsigned shared_count
= src_list
->shared_count
;
302 size
= offsetof(typeof(*src_list
), shared
[shared_count
]);
305 dst_list
= kmalloc(size
, GFP_KERNEL
);
310 src_list
= rcu_dereference(src
->fence
);
311 if (!src_list
|| src_list
->shared_count
> shared_count
) {
316 dst_list
->shared_count
= 0;
317 dst_list
->shared_max
= shared_count
;
318 for (i
= 0; i
< src_list
->shared_count
; ++i
) {
319 struct dma_fence
*fence
;
321 fence
= rcu_dereference(src_list
->shared
[i
]);
322 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT
,
326 if (!dma_fence_get_rcu(fence
)) {
328 src_list
= rcu_dereference(src
->fence
);
332 if (dma_fence_is_signaled(fence
)) {
333 dma_fence_put(fence
);
337 rcu_assign_pointer(dst_list
->shared
[dst_list
->shared_count
++], fence
);
343 new = dma_fence_get_rcu_safe(&src
->fence_excl
);
349 src_list
= reservation_object_get_list(dst
);
350 old
= reservation_object_get_excl(dst
);
353 write_seqcount_begin(&dst
->seq
);
354 /* write_seqcount_begin provides the necessary memory barrier */
355 RCU_INIT_POINTER(dst
->fence_excl
, new);
356 RCU_INIT_POINTER(dst
->fence
, dst_list
);
357 write_seqcount_end(&dst
->seq
);
361 kfree_rcu(src_list
, rcu
);
366 EXPORT_SYMBOL(reservation_object_copy_fences
);
369 * reservation_object_get_fences_rcu - Get an object's shared and exclusive
370 * fences without update side lock held
371 * @obj: the reservation object
372 * @pfence_excl: the returned exclusive fence (or NULL)
373 * @pshared_count: the number of shared fences returned
374 * @pshared: the array of shared fence ptrs returned (array is krealloc'd to
375 * the required size, and must be freed by caller)
377 * Retrieve all fences from the reservation object. If the pointer for the
378 * exclusive fence is not specified the fence is put into the array of the
379 * shared fences as well. Returns either zero or -ENOMEM.
381 int reservation_object_get_fences_rcu(struct reservation_object
*obj
,
382 struct dma_fence
**pfence_excl
,
383 unsigned *pshared_count
,
384 struct dma_fence
***pshared
)
386 struct dma_fence
**shared
= NULL
;
387 struct dma_fence
*fence_excl
;
388 unsigned int shared_count
;
392 struct reservation_object_list
*fobj
;
396 shared_count
= i
= 0;
399 seq
= read_seqcount_begin(&obj
->seq
);
401 fence_excl
= rcu_dereference(obj
->fence_excl
);
402 if (fence_excl
&& !dma_fence_get_rcu(fence_excl
))
405 fobj
= rcu_dereference(obj
->fence
);
407 sz
+= sizeof(*shared
) * fobj
->shared_max
;
409 if (!pfence_excl
&& fence_excl
)
410 sz
+= sizeof(*shared
);
413 struct dma_fence
**nshared
;
415 nshared
= krealloc(shared
, sz
,
416 GFP_NOWAIT
| __GFP_NOWARN
);
419 nshared
= krealloc(shared
, sz
, GFP_KERNEL
);
429 shared_count
= fobj
? fobj
->shared_count
: 0;
430 for (i
= 0; i
< shared_count
; ++i
) {
431 shared
[i
] = rcu_dereference(fobj
->shared
[i
]);
432 if (!dma_fence_get_rcu(shared
[i
]))
436 if (!pfence_excl
&& fence_excl
) {
437 shared
[i
] = fence_excl
;
444 if (i
!= shared_count
|| read_seqcount_retry(&obj
->seq
, seq
)) {
446 dma_fence_put(shared
[i
]);
447 dma_fence_put(fence_excl
);
461 *pshared_count
= shared_count
;
464 *pfence_excl
= fence_excl
;
468 EXPORT_SYMBOL_GPL(reservation_object_get_fences_rcu
);
471 * reservation_object_wait_timeout_rcu - Wait on reservation's objects
472 * shared and/or exclusive fences.
473 * @obj: the reservation object
474 * @wait_all: if true, wait on all fences, else wait on just exclusive fence
475 * @intr: if true, do interruptible wait
476 * @timeout: timeout value in jiffies or zero to return immediately
479 * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or
480 * greater than zer on success.
482 long reservation_object_wait_timeout_rcu(struct reservation_object
*obj
,
483 bool wait_all
, bool intr
,
484 unsigned long timeout
)
486 struct dma_fence
*fence
;
487 unsigned seq
, shared_count
;
488 long ret
= timeout
? timeout
: 1;
493 seq
= read_seqcount_begin(&obj
->seq
);
497 fence
= rcu_dereference(obj
->fence_excl
);
498 if (fence
&& !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT
, &fence
->flags
)) {
499 if (!dma_fence_get_rcu(fence
))
502 if (dma_fence_is_signaled(fence
)) {
503 dma_fence_put(fence
);
512 struct reservation_object_list
*fobj
=
513 rcu_dereference(obj
->fence
);
516 shared_count
= fobj
->shared_count
;
518 for (i
= 0; !fence
&& i
< shared_count
; ++i
) {
519 struct dma_fence
*lfence
= rcu_dereference(fobj
->shared
[i
]);
521 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT
,
525 if (!dma_fence_get_rcu(lfence
))
528 if (dma_fence_is_signaled(lfence
)) {
529 dma_fence_put(lfence
);
540 if (read_seqcount_retry(&obj
->seq
, seq
)) {
541 dma_fence_put(fence
);
545 ret
= dma_fence_wait_timeout(fence
, intr
, ret
);
546 dma_fence_put(fence
);
547 if (ret
> 0 && wait_all
&& (i
+ 1 < shared_count
))
556 EXPORT_SYMBOL_GPL(reservation_object_wait_timeout_rcu
);
560 reservation_object_test_signaled_single(struct dma_fence
*passed_fence
)
562 struct dma_fence
*fence
, *lfence
= passed_fence
;
565 if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT
, &lfence
->flags
)) {
566 fence
= dma_fence_get_rcu(lfence
);
570 ret
= !!dma_fence_is_signaled(fence
);
571 dma_fence_put(fence
);
577 * reservation_object_test_signaled_rcu - Test if a reservation object's
578 * fences have been signaled.
579 * @obj: the reservation object
580 * @test_all: if true, test all fences, otherwise only test the exclusive
584 * true if all fences signaled, else false
586 bool reservation_object_test_signaled_rcu(struct reservation_object
*obj
,
589 unsigned seq
, shared_count
;
596 seq
= read_seqcount_begin(&obj
->seq
);
601 struct reservation_object_list
*fobj
=
602 rcu_dereference(obj
->fence
);
605 shared_count
= fobj
->shared_count
;
607 for (i
= 0; i
< shared_count
; ++i
) {
608 struct dma_fence
*fence
= rcu_dereference(fobj
->shared
[i
]);
610 ret
= reservation_object_test_signaled_single(fence
);
617 if (read_seqcount_retry(&obj
->seq
, seq
))
622 struct dma_fence
*fence_excl
= rcu_dereference(obj
->fence_excl
);
625 ret
= reservation_object_test_signaled_single(
630 if (read_seqcount_retry(&obj
->seq
, seq
))
638 EXPORT_SYMBOL_GPL(reservation_object_test_signaled_rcu
);