Merge tag 'v3.3.7' into 3.3/master
[zen-stable.git] / drivers / gpu / drm / nouveau / nouveau_fence.c
blob2f6daae68b9d7e24830bb9f2cc5e5af09cbe4f8b
1 /*
2 * Copyright (C) 2007 Ben Skeggs.
3 * All Rights Reserved.
5 * Permission is hereby granted, free of charge, to any person obtaining
6 * a copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sublicense, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
13 * The above copyright notice and this permission notice (including the
14 * next paragraph) shall be included in all copies or substantial
15 * portions of the Software.
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
20 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
27 #include "drmP.h"
28 #include "drm.h"
30 #include <linux/ktime.h>
31 #include <linux/hrtimer.h>
33 #include "nouveau_drv.h"
34 #include "nouveau_ramht.h"
35 #include "nouveau_dma.h"
37 #define USE_REFCNT(dev) (nouveau_private(dev)->chipset >= 0x10)
38 #define USE_SEMA(dev) (nouveau_private(dev)->chipset >= 0x17)
40 struct nouveau_fence {
41 struct nouveau_channel *channel;
42 struct kref refcount;
43 struct list_head entry;
45 uint32_t sequence;
46 bool signalled;
48 void (*work)(void *priv, bool signalled);
49 void *priv;
52 struct nouveau_semaphore {
53 struct kref ref;
54 struct drm_device *dev;
55 struct drm_mm_node *mem;
58 static inline struct nouveau_fence *
59 nouveau_fence(void *sync_obj)
61 return (struct nouveau_fence *)sync_obj;
64 static void
65 nouveau_fence_del(struct kref *ref)
67 struct nouveau_fence *fence =
68 container_of(ref, struct nouveau_fence, refcount);
70 nouveau_channel_ref(NULL, &fence->channel);
71 kfree(fence);
74 void
75 nouveau_fence_update(struct nouveau_channel *chan)
77 struct drm_device *dev = chan->dev;
78 struct nouveau_fence *tmp, *fence;
79 uint32_t sequence;
81 spin_lock(&chan->fence.lock);
83 /* Fetch the last sequence if the channel is still up and running */
84 if (likely(!list_empty(&chan->fence.pending))) {
85 if (USE_REFCNT(dev))
86 sequence = nvchan_rd32(chan, 0x48);
87 else
88 sequence = atomic_read(&chan->fence.last_sequence_irq);
90 if (chan->fence.sequence_ack == sequence)
91 goto out;
92 chan->fence.sequence_ack = sequence;
95 list_for_each_entry_safe(fence, tmp, &chan->fence.pending, entry) {
96 sequence = fence->sequence;
97 fence->signalled = true;
98 list_del(&fence->entry);
100 if (unlikely(fence->work))
101 fence->work(fence->priv, true);
103 kref_put(&fence->refcount, nouveau_fence_del);
105 if (sequence == chan->fence.sequence_ack)
106 break;
108 out:
109 spin_unlock(&chan->fence.lock);
113 nouveau_fence_new(struct nouveau_channel *chan, struct nouveau_fence **pfence,
114 bool emit)
116 struct nouveau_fence *fence;
117 int ret = 0;
119 fence = kzalloc(sizeof(*fence), GFP_KERNEL);
120 if (!fence)
121 return -ENOMEM;
122 kref_init(&fence->refcount);
123 nouveau_channel_ref(chan, &fence->channel);
125 if (emit)
126 ret = nouveau_fence_emit(fence);
128 if (ret)
129 nouveau_fence_unref(&fence);
130 *pfence = fence;
131 return ret;
134 struct nouveau_channel *
135 nouveau_fence_channel(struct nouveau_fence *fence)
137 return fence ? nouveau_channel_get_unlocked(fence->channel) : NULL;
141 nouveau_fence_emit(struct nouveau_fence *fence)
143 struct nouveau_channel *chan = fence->channel;
144 struct drm_device *dev = chan->dev;
145 struct drm_nouveau_private *dev_priv = dev->dev_private;
146 int ret;
148 ret = RING_SPACE(chan, 2);
149 if (ret)
150 return ret;
152 if (unlikely(chan->fence.sequence == chan->fence.sequence_ack - 1)) {
153 nouveau_fence_update(chan);
155 BUG_ON(chan->fence.sequence ==
156 chan->fence.sequence_ack - 1);
159 fence->sequence = ++chan->fence.sequence;
161 kref_get(&fence->refcount);
162 spin_lock(&chan->fence.lock);
163 list_add_tail(&fence->entry, &chan->fence.pending);
164 spin_unlock(&chan->fence.lock);
166 if (USE_REFCNT(dev)) {
167 if (dev_priv->card_type < NV_C0)
168 BEGIN_RING(chan, NvSubSw, 0x0050, 1);
169 else
170 BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0050, 1);
171 } else {
172 BEGIN_RING(chan, NvSubSw, 0x0150, 1);
174 OUT_RING (chan, fence->sequence);
175 FIRE_RING(chan);
177 return 0;
180 void
181 nouveau_fence_work(struct nouveau_fence *fence,
182 void (*work)(void *priv, bool signalled),
183 void *priv)
185 BUG_ON(fence->work);
187 spin_lock(&fence->channel->fence.lock);
189 if (fence->signalled) {
190 work(priv, true);
191 } else {
192 fence->work = work;
193 fence->priv = priv;
196 spin_unlock(&fence->channel->fence.lock);
199 void
200 __nouveau_fence_unref(void **sync_obj)
202 struct nouveau_fence *fence = nouveau_fence(*sync_obj);
204 if (fence)
205 kref_put(&fence->refcount, nouveau_fence_del);
206 *sync_obj = NULL;
209 void *
210 __nouveau_fence_ref(void *sync_obj)
212 struct nouveau_fence *fence = nouveau_fence(sync_obj);
214 kref_get(&fence->refcount);
215 return sync_obj;
218 bool
219 __nouveau_fence_signalled(void *sync_obj, void *sync_arg)
221 struct nouveau_fence *fence = nouveau_fence(sync_obj);
222 struct nouveau_channel *chan = fence->channel;
224 if (fence->signalled)
225 return true;
227 nouveau_fence_update(chan);
228 return fence->signalled;
232 __nouveau_fence_wait(void *sync_obj, void *sync_arg, bool lazy, bool intr)
234 unsigned long timeout = jiffies + (3 * DRM_HZ);
235 unsigned long sleep_time = NSEC_PER_MSEC / 1000;
236 ktime_t t;
237 int ret = 0;
239 while (1) {
240 if (__nouveau_fence_signalled(sync_obj, sync_arg))
241 break;
243 if (time_after_eq(jiffies, timeout)) {
244 ret = -EBUSY;
245 break;
248 __set_current_state(intr ? TASK_INTERRUPTIBLE
249 : TASK_UNINTERRUPTIBLE);
250 if (lazy) {
251 t = ktime_set(0, sleep_time);
252 schedule_hrtimeout(&t, HRTIMER_MODE_REL);
253 sleep_time *= 2;
254 if (sleep_time > NSEC_PER_MSEC)
255 sleep_time = NSEC_PER_MSEC;
258 if (intr && signal_pending(current)) {
259 ret = -ERESTARTSYS;
260 break;
264 __set_current_state(TASK_RUNNING);
266 return ret;
269 static struct nouveau_semaphore *
270 semaphore_alloc(struct drm_device *dev)
272 struct drm_nouveau_private *dev_priv = dev->dev_private;
273 struct nouveau_semaphore *sema;
274 int size = (dev_priv->chipset < 0x84) ? 4 : 16;
275 int ret, i;
277 if (!USE_SEMA(dev))
278 return NULL;
280 sema = kmalloc(sizeof(*sema), GFP_KERNEL);
281 if (!sema)
282 goto fail;
284 ret = drm_mm_pre_get(&dev_priv->fence.heap);
285 if (ret)
286 goto fail;
288 spin_lock(&dev_priv->fence.lock);
289 sema->mem = drm_mm_search_free(&dev_priv->fence.heap, size, 0, 0);
290 if (sema->mem)
291 sema->mem = drm_mm_get_block_atomic(sema->mem, size, 0);
292 spin_unlock(&dev_priv->fence.lock);
294 if (!sema->mem)
295 goto fail;
297 kref_init(&sema->ref);
298 sema->dev = dev;
299 for (i = sema->mem->start; i < sema->mem->start + size; i += 4)
300 nouveau_bo_wr32(dev_priv->fence.bo, i / 4, 0);
302 return sema;
303 fail:
304 kfree(sema);
305 return NULL;
308 static void
309 semaphore_free(struct kref *ref)
311 struct nouveau_semaphore *sema =
312 container_of(ref, struct nouveau_semaphore, ref);
313 struct drm_nouveau_private *dev_priv = sema->dev->dev_private;
315 spin_lock(&dev_priv->fence.lock);
316 drm_mm_put_block(sema->mem);
317 spin_unlock(&dev_priv->fence.lock);
319 kfree(sema);
322 static void
323 semaphore_work(void *priv, bool signalled)
325 struct nouveau_semaphore *sema = priv;
326 struct drm_nouveau_private *dev_priv = sema->dev->dev_private;
328 if (unlikely(!signalled))
329 nouveau_bo_wr32(dev_priv->fence.bo, sema->mem->start / 4, 1);
331 kref_put(&sema->ref, semaphore_free);
334 static int
335 semaphore_acquire(struct nouveau_channel *chan, struct nouveau_semaphore *sema)
337 struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
338 struct nouveau_fence *fence = NULL;
339 u64 offset = chan->fence.vma.offset + sema->mem->start;
340 int ret;
342 if (dev_priv->chipset < 0x84) {
343 ret = RING_SPACE(chan, 4);
344 if (ret)
345 return ret;
347 BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 3);
348 OUT_RING (chan, NvSema);
349 OUT_RING (chan, offset);
350 OUT_RING (chan, 1);
351 } else
352 if (dev_priv->chipset < 0xc0) {
353 ret = RING_SPACE(chan, 7);
354 if (ret)
355 return ret;
357 BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 1);
358 OUT_RING (chan, chan->vram_handle);
359 BEGIN_RING(chan, NvSubSw, 0x0010, 4);
360 OUT_RING (chan, upper_32_bits(offset));
361 OUT_RING (chan, lower_32_bits(offset));
362 OUT_RING (chan, 1);
363 OUT_RING (chan, 1); /* ACQUIRE_EQ */
364 } else {
365 ret = RING_SPACE(chan, 5);
366 if (ret)
367 return ret;
369 BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0010, 4);
370 OUT_RING (chan, upper_32_bits(offset));
371 OUT_RING (chan, lower_32_bits(offset));
372 OUT_RING (chan, 1);
373 OUT_RING (chan, 0x1001); /* ACQUIRE_EQ */
376 /* Delay semaphore destruction until its work is done */
377 ret = nouveau_fence_new(chan, &fence, true);
378 if (ret)
379 return ret;
381 kref_get(&sema->ref);
382 nouveau_fence_work(fence, semaphore_work, sema);
383 nouveau_fence_unref(&fence);
384 return 0;
387 static int
388 semaphore_release(struct nouveau_channel *chan, struct nouveau_semaphore *sema)
390 struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
391 struct nouveau_fence *fence = NULL;
392 u64 offset = chan->fence.vma.offset + sema->mem->start;
393 int ret;
395 if (dev_priv->chipset < 0x84) {
396 ret = RING_SPACE(chan, 5);
397 if (ret)
398 return ret;
400 BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 2);
401 OUT_RING (chan, NvSema);
402 OUT_RING (chan, offset);
403 BEGIN_RING(chan, NvSubSw, NV_SW_SEMAPHORE_RELEASE, 1);
404 OUT_RING (chan, 1);
405 } else
406 if (dev_priv->chipset < 0xc0) {
407 ret = RING_SPACE(chan, 7);
408 if (ret)
409 return ret;
411 BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 1);
412 OUT_RING (chan, chan->vram_handle);
413 BEGIN_RING(chan, NvSubSw, 0x0010, 4);
414 OUT_RING (chan, upper_32_bits(offset));
415 OUT_RING (chan, lower_32_bits(offset));
416 OUT_RING (chan, 1);
417 OUT_RING (chan, 2); /* RELEASE */
418 } else {
419 ret = RING_SPACE(chan, 5);
420 if (ret)
421 return ret;
423 BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0010, 4);
424 OUT_RING (chan, upper_32_bits(offset));
425 OUT_RING (chan, lower_32_bits(offset));
426 OUT_RING (chan, 1);
427 OUT_RING (chan, 0x1002); /* RELEASE */
430 /* Delay semaphore destruction until its work is done */
431 ret = nouveau_fence_new(chan, &fence, true);
432 if (ret)
433 return ret;
435 kref_get(&sema->ref);
436 nouveau_fence_work(fence, semaphore_work, sema);
437 nouveau_fence_unref(&fence);
438 return 0;
442 nouveau_fence_sync(struct nouveau_fence *fence,
443 struct nouveau_channel *wchan)
445 struct nouveau_channel *chan = nouveau_fence_channel(fence);
446 struct drm_device *dev = wchan->dev;
447 struct nouveau_semaphore *sema;
448 int ret = 0;
450 if (likely(!chan || chan == wchan ||
451 nouveau_fence_signalled(fence)))
452 goto out;
454 sema = semaphore_alloc(dev);
455 if (!sema) {
456 /* Early card or broken userspace, fall back to
457 * software sync. */
458 ret = nouveau_fence_wait(fence, true, false);
459 goto out;
462 /* try to take chan's mutex, if we can't take it right away
463 * we have to fallback to software sync to prevent locking
464 * order issues
466 if (!mutex_trylock(&chan->mutex)) {
467 ret = nouveau_fence_wait(fence, true, false);
468 goto out_unref;
471 /* Make wchan wait until it gets signalled */
472 ret = semaphore_acquire(wchan, sema);
473 if (ret)
474 goto out_unlock;
476 /* Signal the semaphore from chan */
477 ret = semaphore_release(chan, sema);
479 out_unlock:
480 mutex_unlock(&chan->mutex);
481 out_unref:
482 kref_put(&sema->ref, semaphore_free);
483 out:
484 if (chan)
485 nouveau_channel_put_unlocked(&chan);
486 return ret;
490 __nouveau_fence_flush(void *sync_obj, void *sync_arg)
492 return 0;
496 nouveau_fence_channel_init(struct nouveau_channel *chan)
498 struct drm_device *dev = chan->dev;
499 struct drm_nouveau_private *dev_priv = dev->dev_private;
500 struct nouveau_gpuobj *obj = NULL;
501 int ret;
503 if (dev_priv->card_type < NV_C0) {
504 /* Create an NV_SW object for various sync purposes */
505 ret = nouveau_gpuobj_gr_new(chan, NvSw, NV_SW);
506 if (ret)
507 return ret;
509 ret = RING_SPACE(chan, 2);
510 if (ret)
511 return ret;
513 BEGIN_RING(chan, NvSubSw, 0, 1);
514 OUT_RING (chan, NvSw);
515 FIRE_RING (chan);
518 /* Setup area of memory shared between all channels for x-chan sync */
519 if (USE_SEMA(dev) && dev_priv->chipset < 0x84) {
520 struct ttm_mem_reg *mem = &dev_priv->fence.bo->bo.mem;
522 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_FROM_MEMORY,
523 mem->start << PAGE_SHIFT,
524 mem->size, NV_MEM_ACCESS_RW,
525 NV_MEM_TARGET_VRAM, &obj);
526 if (ret)
527 return ret;
529 ret = nouveau_ramht_insert(chan, NvSema, obj);
530 nouveau_gpuobj_ref(NULL, &obj);
531 if (ret)
532 return ret;
533 } else
534 if (USE_SEMA(dev)) {
535 /* map fence bo into channel's vm */
536 ret = nouveau_bo_vma_add(dev_priv->fence.bo, chan->vm,
537 &chan->fence.vma);
538 if (ret)
539 return ret;
542 atomic_set(&chan->fence.last_sequence_irq, 0);
543 return 0;
546 void
547 nouveau_fence_channel_fini(struct nouveau_channel *chan)
549 struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
550 struct nouveau_fence *tmp, *fence;
552 spin_lock(&chan->fence.lock);
553 list_for_each_entry_safe(fence, tmp, &chan->fence.pending, entry) {
554 fence->signalled = true;
555 list_del(&fence->entry);
557 if (unlikely(fence->work))
558 fence->work(fence->priv, false);
560 kref_put(&fence->refcount, nouveau_fence_del);
562 spin_unlock(&chan->fence.lock);
564 nouveau_bo_vma_del(dev_priv->fence.bo, &chan->fence.vma);
568 nouveau_fence_init(struct drm_device *dev)
570 struct drm_nouveau_private *dev_priv = dev->dev_private;
571 int size = (dev_priv->chipset < 0x84) ? 4096 : 16384;
572 int ret;
574 /* Create a shared VRAM heap for cross-channel sync. */
575 if (USE_SEMA(dev)) {
576 ret = nouveau_bo_new(dev, size, 0, TTM_PL_FLAG_VRAM,
577 0, 0, &dev_priv->fence.bo);
578 if (ret)
579 return ret;
581 ret = nouveau_bo_pin(dev_priv->fence.bo, TTM_PL_FLAG_VRAM);
582 if (ret)
583 goto fail;
585 ret = nouveau_bo_map(dev_priv->fence.bo);
586 if (ret)
587 goto fail;
589 ret = drm_mm_init(&dev_priv->fence.heap, 0,
590 dev_priv->fence.bo->bo.mem.size);
591 if (ret)
592 goto fail;
594 spin_lock_init(&dev_priv->fence.lock);
597 return 0;
598 fail:
599 nouveau_bo_unmap(dev_priv->fence.bo);
600 nouveau_bo_ref(NULL, &dev_priv->fence.bo);
601 return ret;
604 void
605 nouveau_fence_fini(struct drm_device *dev)
607 struct drm_nouveau_private *dev_priv = dev->dev_private;
609 if (USE_SEMA(dev)) {
610 drm_mm_takedown(&dev_priv->fence.heap);
611 nouveau_bo_unmap(dev_priv->fence.bo);
612 nouveau_bo_unpin(dev_priv->fence.bo);
613 nouveau_bo_ref(NULL, &dev_priv->fence.bo);