2 * Copyright (C) 2007 Ben Skeggs.
5 * Permission is hereby granted, free of charge, to any person obtaining
6 * a copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sublicense, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
13 * The above copyright notice and this permission notice (including the
14 * next paragraph) shall be included in all copies or substantial
15 * portions of the Software.
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
20 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
29 #include <linux/ktime.h>
30 #include <linux/hrtimer.h>
31 #include <trace/events/dma_fence.h>
33 #include <nvif/cl826e.h>
34 #include <nvif/notify.h>
35 #include <nvif/event.h>
37 #include "nouveau_drv.h"
38 #include "nouveau_dma.h"
39 #include "nouveau_fence.h"
41 static const struct dma_fence_ops nouveau_fence_ops_uevent
;
42 static const struct dma_fence_ops nouveau_fence_ops_legacy
;
44 static inline struct nouveau_fence
*
45 from_fence(struct dma_fence
*fence
)
47 return container_of(fence
, struct nouveau_fence
, base
);
50 static inline struct nouveau_fence_chan
*
51 nouveau_fctx(struct nouveau_fence
*fence
)
53 return container_of(fence
->base
.lock
, struct nouveau_fence_chan
, lock
);
57 nouveau_fence_signal(struct nouveau_fence
*fence
)
61 dma_fence_signal_locked(&fence
->base
);
62 list_del(&fence
->head
);
63 rcu_assign_pointer(fence
->channel
, NULL
);
65 if (test_bit(DMA_FENCE_FLAG_USER_BITS
, &fence
->base
.flags
)) {
66 struct nouveau_fence_chan
*fctx
= nouveau_fctx(fence
);
68 if (!--fctx
->notify_ref
)
72 dma_fence_put(&fence
->base
);
76 static struct nouveau_fence
*
77 nouveau_local_fence(struct dma_fence
*fence
, struct nouveau_drm
*drm
) {
78 struct nouveau_fence_priv
*priv
= (void*)drm
->fence
;
80 if (fence
->ops
!= &nouveau_fence_ops_legacy
&&
81 fence
->ops
!= &nouveau_fence_ops_uevent
)
84 if (fence
->context
< priv
->context_base
||
85 fence
->context
>= priv
->context_base
+ priv
->contexts
)
88 return from_fence(fence
);
92 nouveau_fence_context_del(struct nouveau_fence_chan
*fctx
)
94 struct nouveau_fence
*fence
;
96 spin_lock_irq(&fctx
->lock
);
97 while (!list_empty(&fctx
->pending
)) {
98 fence
= list_entry(fctx
->pending
.next
, typeof(*fence
), head
);
100 if (nouveau_fence_signal(fence
))
101 nvif_notify_put(&fctx
->notify
);
103 spin_unlock_irq(&fctx
->lock
);
105 nvif_notify_fini(&fctx
->notify
);
109 * Ensure that all accesses to fence->channel complete before freeing
116 nouveau_fence_context_put(struct kref
*fence_ref
)
118 kfree(container_of(fence_ref
, struct nouveau_fence_chan
, fence_ref
));
122 nouveau_fence_context_free(struct nouveau_fence_chan
*fctx
)
124 kref_put(&fctx
->fence_ref
, nouveau_fence_context_put
);
128 nouveau_fence_update(struct nouveau_channel
*chan
, struct nouveau_fence_chan
*fctx
)
130 struct nouveau_fence
*fence
;
132 u32 seq
= fctx
->read(chan
);
134 while (!list_empty(&fctx
->pending
)) {
135 fence
= list_entry(fctx
->pending
.next
, typeof(*fence
), head
);
137 if ((int)(seq
- fence
->base
.seqno
) < 0)
140 drop
|= nouveau_fence_signal(fence
);
147 nouveau_fence_wait_uevent_handler(struct nvif_notify
*notify
)
149 struct nouveau_fence_chan
*fctx
=
150 container_of(notify
, typeof(*fctx
), notify
);
152 int ret
= NVIF_NOTIFY_KEEP
;
154 spin_lock_irqsave(&fctx
->lock
, flags
);
155 if (!list_empty(&fctx
->pending
)) {
156 struct nouveau_fence
*fence
;
157 struct nouveau_channel
*chan
;
159 fence
= list_entry(fctx
->pending
.next
, typeof(*fence
), head
);
160 chan
= rcu_dereference_protected(fence
->channel
, lockdep_is_held(&fctx
->lock
));
161 if (nouveau_fence_update(fence
->channel
, fctx
))
162 ret
= NVIF_NOTIFY_DROP
;
164 spin_unlock_irqrestore(&fctx
->lock
, flags
);
170 nouveau_fence_context_new(struct nouveau_channel
*chan
, struct nouveau_fence_chan
*fctx
)
172 struct nouveau_fence_priv
*priv
= (void*)chan
->drm
->fence
;
173 struct nouveau_cli
*cli
= (void *)chan
->user
.client
;
176 INIT_LIST_HEAD(&fctx
->flip
);
177 INIT_LIST_HEAD(&fctx
->pending
);
178 spin_lock_init(&fctx
->lock
);
179 fctx
->context
= priv
->context_base
+ chan
->chid
;
181 if (chan
== chan
->drm
->cechan
)
182 strcpy(fctx
->name
, "copy engine channel");
183 else if (chan
== chan
->drm
->channel
)
184 strcpy(fctx
->name
, "generic kernel channel");
186 strcpy(fctx
->name
, nvxx_client(&cli
->base
)->name
);
188 kref_init(&fctx
->fence_ref
);
192 ret
= nvif_notify_init(&chan
->user
, nouveau_fence_wait_uevent_handler
,
193 false, NV826E_V0_NTFY_NON_STALL_INTERRUPT
,
194 &(struct nvif_notify_uevent_req
) { },
195 sizeof(struct nvif_notify_uevent_req
),
196 sizeof(struct nvif_notify_uevent_rep
),
203 nouveau_fence_emit(struct nouveau_fence
*fence
, struct nouveau_channel
*chan
)
205 struct nouveau_fence_chan
*fctx
= chan
->fence
;
206 struct nouveau_fence_priv
*priv
= (void*)chan
->drm
->fence
;
209 fence
->channel
= chan
;
210 fence
->timeout
= jiffies
+ (15 * HZ
);
213 dma_fence_init(&fence
->base
, &nouveau_fence_ops_uevent
,
214 &fctx
->lock
, fctx
->context
, ++fctx
->sequence
);
216 dma_fence_init(&fence
->base
, &nouveau_fence_ops_legacy
,
217 &fctx
->lock
, fctx
->context
, ++fctx
->sequence
);
218 kref_get(&fctx
->fence_ref
);
220 trace_dma_fence_emit(&fence
->base
);
221 ret
= fctx
->emit(fence
);
223 dma_fence_get(&fence
->base
);
224 spin_lock_irq(&fctx
->lock
);
226 if (nouveau_fence_update(chan
, fctx
))
227 nvif_notify_put(&fctx
->notify
);
229 list_add_tail(&fence
->head
, &fctx
->pending
);
230 spin_unlock_irq(&fctx
->lock
);
237 nouveau_fence_done(struct nouveau_fence
*fence
)
239 if (fence
->base
.ops
== &nouveau_fence_ops_legacy
||
240 fence
->base
.ops
== &nouveau_fence_ops_uevent
) {
241 struct nouveau_fence_chan
*fctx
= nouveau_fctx(fence
);
242 struct nouveau_channel
*chan
;
245 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT
, &fence
->base
.flags
))
248 spin_lock_irqsave(&fctx
->lock
, flags
);
249 chan
= rcu_dereference_protected(fence
->channel
, lockdep_is_held(&fctx
->lock
));
250 if (chan
&& nouveau_fence_update(chan
, fctx
))
251 nvif_notify_put(&fctx
->notify
);
252 spin_unlock_irqrestore(&fctx
->lock
, flags
);
254 return dma_fence_is_signaled(&fence
->base
);
258 nouveau_fence_wait_legacy(struct dma_fence
*f
, bool intr
, long wait
)
260 struct nouveau_fence
*fence
= from_fence(f
);
261 unsigned long sleep_time
= NSEC_PER_MSEC
/ 1000;
262 unsigned long t
= jiffies
, timeout
= t
+ wait
;
264 while (!nouveau_fence_done(fence
)) {
269 if (wait
!= MAX_SCHEDULE_TIMEOUT
&& time_after_eq(t
, timeout
)) {
270 __set_current_state(TASK_RUNNING
);
274 __set_current_state(intr
? TASK_INTERRUPTIBLE
:
275 TASK_UNINTERRUPTIBLE
);
278 schedule_hrtimeout(&kt
, HRTIMER_MODE_REL
);
280 if (sleep_time
> NSEC_PER_MSEC
)
281 sleep_time
= NSEC_PER_MSEC
;
283 if (intr
&& signal_pending(current
))
287 __set_current_state(TASK_RUNNING
);
293 nouveau_fence_wait_busy(struct nouveau_fence
*fence
, bool intr
)
297 while (!nouveau_fence_done(fence
)) {
298 if (time_after_eq(jiffies
, fence
->timeout
)) {
303 __set_current_state(intr
?
305 TASK_UNINTERRUPTIBLE
);
307 if (intr
&& signal_pending(current
)) {
313 __set_current_state(TASK_RUNNING
);
318 nouveau_fence_wait(struct nouveau_fence
*fence
, bool lazy
, bool intr
)
323 return nouveau_fence_wait_busy(fence
, intr
);
325 ret
= dma_fence_wait_timeout(&fence
->base
, intr
, 15 * HZ
);
335 nouveau_fence_sync(struct nouveau_bo
*nvbo
, struct nouveau_channel
*chan
, bool exclusive
, bool intr
)
337 struct nouveau_fence_chan
*fctx
= chan
->fence
;
338 struct dma_fence
*fence
;
339 struct reservation_object
*resv
= nvbo
->bo
.resv
;
340 struct reservation_object_list
*fobj
;
341 struct nouveau_fence
*f
;
345 ret
= reservation_object_reserve_shared(resv
);
351 fobj
= reservation_object_get_list(resv
);
352 fence
= reservation_object_get_excl(resv
);
354 if (fence
&& (!exclusive
|| !fobj
|| !fobj
->shared_count
)) {
355 struct nouveau_channel
*prev
= NULL
;
356 bool must_wait
= true;
358 f
= nouveau_local_fence(fence
, chan
->drm
);
361 prev
= rcu_dereference(f
->channel
);
362 if (prev
&& (prev
== chan
|| fctx
->sync(f
, prev
, chan
) == 0))
368 ret
= dma_fence_wait(fence
, intr
);
373 if (!exclusive
|| !fobj
)
376 for (i
= 0; i
< fobj
->shared_count
&& !ret
; ++i
) {
377 struct nouveau_channel
*prev
= NULL
;
378 bool must_wait
= true;
380 fence
= rcu_dereference_protected(fobj
->shared
[i
],
381 reservation_object_held(resv
));
383 f
= nouveau_local_fence(fence
, chan
->drm
);
386 prev
= rcu_dereference(f
->channel
);
387 if (prev
&& (prev
== chan
|| fctx
->sync(f
, prev
, chan
) == 0))
393 ret
= dma_fence_wait(fence
, intr
);
400 nouveau_fence_unref(struct nouveau_fence
**pfence
)
403 dma_fence_put(&(*pfence
)->base
);
408 nouveau_fence_new(struct nouveau_channel
*chan
, bool sysmem
,
409 struct nouveau_fence
**pfence
)
411 struct nouveau_fence
*fence
;
414 if (unlikely(!chan
->fence
))
417 fence
= kzalloc(sizeof(*fence
), GFP_KERNEL
);
421 ret
= nouveau_fence_emit(fence
, chan
);
423 nouveau_fence_unref(&fence
);
429 static const char *nouveau_fence_get_get_driver_name(struct dma_fence
*fence
)
434 static const char *nouveau_fence_get_timeline_name(struct dma_fence
*f
)
436 struct nouveau_fence
*fence
= from_fence(f
);
437 struct nouveau_fence_chan
*fctx
= nouveau_fctx(fence
);
439 return !fctx
->dead
? fctx
->name
: "dead channel";
443 * In an ideal world, read would not assume the channel context is still alive.
444 * This function may be called from another device, running into free memory as a
445 * result. The drm node should still be there, so we can derive the index from
448 static bool nouveau_fence_is_signaled(struct dma_fence
*f
)
450 struct nouveau_fence
*fence
= from_fence(f
);
451 struct nouveau_fence_chan
*fctx
= nouveau_fctx(fence
);
452 struct nouveau_channel
*chan
;
456 chan
= rcu_dereference(fence
->channel
);
458 ret
= (int)(fctx
->read(chan
) - fence
->base
.seqno
) >= 0;
464 static bool nouveau_fence_no_signaling(struct dma_fence
*f
)
466 struct nouveau_fence
*fence
= from_fence(f
);
469 * caller should have a reference on the fence,
470 * else fence could get freed here
472 WARN_ON(kref_read(&fence
->base
.refcount
) <= 1);
475 * This needs uevents to work correctly, but dma_fence_add_callback relies on
476 * being able to enable signaling. It will still get signaled eventually,
477 * just not right away.
479 if (nouveau_fence_is_signaled(f
)) {
480 list_del(&fence
->head
);
482 dma_fence_put(&fence
->base
);
489 static void nouveau_fence_release(struct dma_fence
*f
)
491 struct nouveau_fence
*fence
= from_fence(f
);
492 struct nouveau_fence_chan
*fctx
= nouveau_fctx(fence
);
494 kref_put(&fctx
->fence_ref
, nouveau_fence_context_put
);
495 dma_fence_free(&fence
->base
);
498 static const struct dma_fence_ops nouveau_fence_ops_legacy
= {
499 .get_driver_name
= nouveau_fence_get_get_driver_name
,
500 .get_timeline_name
= nouveau_fence_get_timeline_name
,
501 .enable_signaling
= nouveau_fence_no_signaling
,
502 .signaled
= nouveau_fence_is_signaled
,
503 .wait
= nouveau_fence_wait_legacy
,
504 .release
= nouveau_fence_release
507 static bool nouveau_fence_enable_signaling(struct dma_fence
*f
)
509 struct nouveau_fence
*fence
= from_fence(f
);
510 struct nouveau_fence_chan
*fctx
= nouveau_fctx(fence
);
513 if (!fctx
->notify_ref
++)
514 nvif_notify_get(&fctx
->notify
);
516 ret
= nouveau_fence_no_signaling(f
);
518 set_bit(DMA_FENCE_FLAG_USER_BITS
, &fence
->base
.flags
);
519 else if (!--fctx
->notify_ref
)
520 nvif_notify_put(&fctx
->notify
);
525 static const struct dma_fence_ops nouveau_fence_ops_uevent
= {
526 .get_driver_name
= nouveau_fence_get_get_driver_name
,
527 .get_timeline_name
= nouveau_fence_get_timeline_name
,
528 .enable_signaling
= nouveau_fence_enable_signaling
,
529 .signaled
= nouveau_fence_is_signaled
,
530 .wait
= dma_fence_default_wait
,
531 .release
= nouveau_fence_release