drivers/gpu/drm/nouveau/nouveau_fence.c

   1 /*
   2  * Copyright (C) 2007 Ben Skeggs.
   3  * All Rights Reserved.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining
   6  * a copy of this software and associated documentation files (the
   7  * "Software"), to deal in the Software without restriction, including
   8  * without limitation the rights to use, copy, modify, merge, publish,
   9  * distribute, sublicense, and/or sell copies of the Software, and to
  10  * permit persons to whom the Software is furnished to do so, subject to
  11  * the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the
  14  * next paragraph) shall be included in all copies or substantial
  15  * portions of the Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  18  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  19  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  20  * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  21  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  22  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  23  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  24  *
  25  */
  26
  27 #include "drmP.h"
  28 #include "drm.h"
  29
  30 #include <linux/ktime.h>
  31 #include <linux/hrtimer.h>
  32
  33 #include "nouveau_drv.h"
  34 #include "nouveau_ramht.h"
  35 #include "nouveau_dma.h"
  36
  37 #define USE_REFCNT(dev) (nouveau_private(dev)->chipset >= 0x10)
  38 #define USE_SEMA(dev) (nouveau_private(dev)->chipset >= 0x17)
  39
  40 struct nouveau_fence {
  41         struct nouveau_channel *channel;
  42         struct kref refcount;
  43         struct list_head entry;
  44
  45         uint32_t sequence;
  46         bool signalled;
  47
  48         void (*work)(void *priv, bool signalled);
  49         void *priv;
  50 };
  51
  52 struct nouveau_semaphore {
  53         struct kref ref;
  54         struct drm_device *dev;
  55         struct drm_mm_node *mem;
  56 };
  57
  58 static inline struct nouveau_fence *
  59 nouveau_fence(void *sync_obj)
  60 {
  61         return (struct nouveau_fence *)sync_obj;
  62 }
  63
  64 static void
  65 nouveau_fence_del(struct kref *ref)
  66 {
  67         struct nouveau_fence *fence =
  68                 container_of(ref, struct nouveau_fence, refcount);
  69
  70         nouveau_channel_ref(NULL, &fence->channel);
  71         kfree(fence);
  72 }
  73
  74 void
  75 nouveau_fence_update(struct nouveau_channel *chan)
  76 {
  77         struct drm_device *dev = chan->dev;
  78         struct nouveau_fence *tmp, *fence;
  79         uint32_t sequence;
  80
  81         spin_lock(&chan->fence.lock);
  82
  83         /* Fetch the last sequence if the channel is still up and running */
  84         if (likely(!list_empty(&chan->fence.pending))) {
  85                 if (USE_REFCNT(dev))
  86                         sequence = nvchan_rd32(chan, 0x48);
  87                 else
  88                         sequence = atomic_read(&chan->fence.last_sequence_irq);
  89
  90                 if (chan->fence.sequence_ack == sequence)
  91                         goto out;
  92                 chan->fence.sequence_ack = sequence;
  93         }
  94
  95         list_for_each_entry_safe(fence, tmp, &chan->fence.pending, entry) {
  96                 sequence = fence->sequence;
  97                 fence->signalled = true;
  98                 list_del(&fence->entry);
  99
 100                 if (unlikely(fence->work))
 101                         fence->work(fence->priv, true);
 102
 103                 kref_put(&fence->refcount, nouveau_fence_del);
 104
 105                 if (sequence == chan->fence.sequence_ack)
 106                         break;
 107         }
 108 out:
 109         spin_unlock(&chan->fence.lock);
 110 }
 111
 112 int
 113 nouveau_fence_new(struct nouveau_channel *chan, struct nouveau_fence **pfence,
 114                   bool emit)
 115 {
 116         struct nouveau_fence *fence;
 117         int ret = 0;
 118
 119         fence = kzalloc(sizeof(*fence), GFP_KERNEL);
 120         if (!fence)
 121                 return -ENOMEM;
 122         kref_init(&fence->refcount);
 123         nouveau_channel_ref(chan, &fence->channel);
 124
 125         if (emit)
 126                 ret = nouveau_fence_emit(fence);
 127
 128         if (ret)
 129                 nouveau_fence_unref(&fence);
 130         *pfence = fence;
 131         return ret;
 132 }
 133
 134 struct nouveau_channel *
 135 nouveau_fence_channel(struct nouveau_fence *fence)
 136 {
 137         return fence ? nouveau_channel_get_unlocked(fence->channel) : NULL;
 138 }
 139
 140 int
 141 nouveau_fence_emit(struct nouveau_fence *fence)
 142 {
 143         struct nouveau_channel *chan = fence->channel;
 144         struct drm_device *dev = chan->dev;
 145         struct drm_nouveau_private *dev_priv = dev->dev_private;
 146         int ret;
 147
 148         ret = RING_SPACE(chan, 2);
 149         if (ret)
 150                 return ret;
 151
 152         if (unlikely(chan->fence.sequence == chan->fence.sequence_ack - 1)) {
 153                 nouveau_fence_update(chan);
 154
 155                 BUG_ON(chan->fence.sequence ==
 156                        chan->fence.sequence_ack - 1);
 157         }
 158
 159         fence->sequence = ++chan->fence.sequence;
 160
 161         kref_get(&fence->refcount);
 162         spin_lock(&chan->fence.lock);
 163         list_add_tail(&fence->entry, &chan->fence.pending);
 164         spin_unlock(&chan->fence.lock);
 165
 166         if (USE_REFCNT(dev)) {
 167                 if (dev_priv->card_type < NV_C0)
 168                         BEGIN_RING(chan, NvSubSw, 0x0050, 1);
 169                 else
 170                         BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0050, 1);
 171         } else {
 172                 BEGIN_RING(chan, NvSubSw, 0x0150, 1);
 173         }
 174         OUT_RING (chan, fence->sequence);
 175         FIRE_RING(chan);
 176
 177         return 0;
 178 }
 179
 180 void
 181 nouveau_fence_work(struct nouveau_fence *fence,
 182                    void (*work)(void *priv, bool signalled),
 183                    void *priv)
 184 {
 185         BUG_ON(fence->work);
 186
 187         spin_lock(&fence->channel->fence.lock);
 188
 189         if (fence->signalled) {
 190                 work(priv, true);
 191         } else {
 192                 fence->work = work;
 193                 fence->priv = priv;
 194         }
 195
 196         spin_unlock(&fence->channel->fence.lock);
 197 }
 198
 199 void
 200 __nouveau_fence_unref(void **sync_obj)
 201 {
 202         struct nouveau_fence *fence = nouveau_fence(*sync_obj);
 203
 204         if (fence)
 205                 kref_put(&fence->refcount, nouveau_fence_del);
 206         *sync_obj = NULL;
 207 }
 208
 209 void *
 210 __nouveau_fence_ref(void *sync_obj)
 211 {
 212         struct nouveau_fence *fence = nouveau_fence(sync_obj);
 213
 214         kref_get(&fence->refcount);
 215         return sync_obj;
 216 }
 217
 218 bool
 219 __nouveau_fence_signalled(void *sync_obj, void *sync_arg)
 220 {
 221         struct nouveau_fence *fence = nouveau_fence(sync_obj);
 222         struct nouveau_channel *chan = fence->channel;
 223
 224         if (fence->signalled)
 225                 return true;
 226
 227         nouveau_fence_update(chan);
 228         return fence->signalled;
 229 }
 230
 231 int
 232 __nouveau_fence_wait(void *sync_obj, void *sync_arg, bool lazy, bool intr)
 233 {
 234         unsigned long timeout = jiffies + (3 * DRM_HZ);
 235         unsigned long sleep_time = NSEC_PER_MSEC / 1000;
 236         ktime_t t;
 237         int ret = 0;
 238
 239         while (1) {
 240                 if (__nouveau_fence_signalled(sync_obj, sync_arg))
 241                         break;
 242
 243                 if (time_after_eq(jiffies, timeout)) {
 244                         ret = -EBUSY;
 245                         break;
 246                 }
 247
 248                 __set_current_state(intr ? TASK_INTERRUPTIBLE
 249                         : TASK_UNINTERRUPTIBLE);
 250                 if (lazy) {
 251                         t = ktime_set(0, sleep_time);
 252                         schedule_hrtimeout(&t, HRTIMER_MODE_REL);
 253                         sleep_time *= 2;
 254                         if (sleep_time > NSEC_PER_MSEC)
 255                                 sleep_time = NSEC_PER_MSEC;
 256                 }
 257
 258                 if (intr && signal_pending(current)) {
 259                         ret = -ERESTARTSYS;
 260                         break;
 261                 }
 262         }
 263
 264         __set_current_state(TASK_RUNNING);
 265
 266         return ret;
 267 }
 268
 269 static struct nouveau_semaphore *
 270 semaphore_alloc(struct drm_device *dev)
 271 {
 272         struct drm_nouveau_private *dev_priv = dev->dev_private;
 273         struct nouveau_semaphore *sema;
 274         int size = (dev_priv->chipset < 0x84) ? 4 : 16;
 275         int ret, i;
 276
 277         if (!USE_SEMA(dev))
 278                 return NULL;
 279
 280         sema = kmalloc(sizeof(*sema), GFP_KERNEL);
 281         if (!sema)
 282                 goto fail;
 283
 284         ret = drm_mm_pre_get(&dev_priv->fence.heap);
 285         if (ret)
 286                 goto fail;
 287
 288         spin_lock(&dev_priv->fence.lock);
 289         sema->mem = drm_mm_search_free(&dev_priv->fence.heap, size, 0, 0);
 290         if (sema->mem)
 291                 sema->mem = drm_mm_get_block_atomic(sema->mem, size, 0);
 292         spin_unlock(&dev_priv->fence.lock);
 293
 294         if (!sema->mem)
 295                 goto fail;
 296
 297         kref_init(&sema->ref);
 298         sema->dev = dev;
 299         for (i = sema->mem->start; i < sema->mem->start + size; i += 4)
 300                 nouveau_bo_wr32(dev_priv->fence.bo, i / 4, 0);
 301
 302         return sema;
 303 fail:
 304         kfree(sema);
 305         return NULL;
 306 }
 307
 308 static void
 309 semaphore_free(struct kref *ref)
 310 {
 311         struct nouveau_semaphore *sema =
 312                 container_of(ref, struct nouveau_semaphore, ref);
 313         struct drm_nouveau_private *dev_priv = sema->dev->dev_private;
 314
 315         spin_lock(&dev_priv->fence.lock);
 316         drm_mm_put_block(sema->mem);
 317         spin_unlock(&dev_priv->fence.lock);
 318
 319         kfree(sema);
 320 }
 321
 322 static void
 323 semaphore_work(void *priv, bool signalled)
 324 {
 325         struct nouveau_semaphore *sema = priv;
 326         struct drm_nouveau_private *dev_priv = sema->dev->dev_private;
 327
 328         if (unlikely(!signalled))
 329                 nouveau_bo_wr32(dev_priv->fence.bo, sema->mem->start / 4, 1);
 330
 331         kref_put(&sema->ref, semaphore_free);
 332 }
 333
 334 static int
 335 semaphore_acquire(struct nouveau_channel *chan, struct nouveau_semaphore *sema)
 336 {
 337         struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
 338         struct nouveau_fence *fence = NULL;
 339         int ret;
 340
 341         if (dev_priv->chipset < 0x84) {
 342                 ret = RING_SPACE(chan, 3);
 343                 if (ret)
 344                         return ret;
 345
 346                 BEGIN_RING(chan, NvSubSw, NV_SW_SEMAPHORE_OFFSET, 2);
 347                 OUT_RING  (chan, sema->mem->start);
 348                 OUT_RING  (chan, 1);
 349         } else
 350         if (dev_priv->chipset < 0xc0) {
 351                 struct nouveau_vma *vma = &dev_priv->fence.bo->vma;
 352                 u64 offset = vma->offset + sema->mem->start;
 353
 354                 ret = RING_SPACE(chan, 5);
 355                 if (ret)
 356                         return ret;
 357
 358                 BEGIN_RING(chan, NvSubSw, 0x0010, 4);
 359                 OUT_RING  (chan, upper_32_bits(offset));
 360                 OUT_RING  (chan, lower_32_bits(offset));
 361                 OUT_RING  (chan, 1);
 362                 OUT_RING  (chan, 1); /* ACQUIRE_EQ */
 363         } else {
 364                 struct nouveau_vma *vma = &dev_priv->fence.bo->vma;
 365                 u64 offset = vma->offset + sema->mem->start;
 366
 367                 ret = RING_SPACE(chan, 5);
 368                 if (ret)
 369                         return ret;
 370
 371                 BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0010, 4);
 372                 OUT_RING  (chan, upper_32_bits(offset));
 373                 OUT_RING  (chan, lower_32_bits(offset));
 374                 OUT_RING  (chan, 1);
 375                 OUT_RING  (chan, 0x1001); /* ACQUIRE_EQ */
 376         }
 377
 378         /* Delay semaphore destruction until its work is done */
 379         ret = nouveau_fence_new(chan, &fence, true);
 380         if (ret)
 381                 return ret;
 382
 383         kref_get(&sema->ref);
 384         nouveau_fence_work(fence, semaphore_work, sema);
 385         nouveau_fence_unref(&fence);
 386         return 0;
 387 }
 388
 389 static int
 390 semaphore_release(struct nouveau_channel *chan, struct nouveau_semaphore *sema)
 391 {
 392         struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
 393         struct nouveau_fence *fence = NULL;
 394         int ret;
 395
 396         if (dev_priv->chipset < 0x84) {
 397                 ret = RING_SPACE(chan, 4);
 398                 if (ret)
 399                         return ret;
 400
 401                 BEGIN_RING(chan, NvSubSw, NV_SW_SEMAPHORE_OFFSET, 1);
 402                 OUT_RING  (chan, sema->mem->start);
 403                 BEGIN_RING(chan, NvSubSw, NV_SW_SEMAPHORE_RELEASE, 1);
 404                 OUT_RING  (chan, 1);
 405         } else
 406         if (dev_priv->chipset < 0xc0) {
 407                 struct nouveau_vma *vma = &dev_priv->fence.bo->vma;
 408                 u64 offset = vma->offset + sema->mem->start;
 409
 410                 ret = RING_SPACE(chan, 5);
 411                 if (ret)
 412                         return ret;
 413
 414                 BEGIN_RING(chan, NvSubSw, 0x0010, 4);
 415                 OUT_RING  (chan, upper_32_bits(offset));
 416                 OUT_RING  (chan, lower_32_bits(offset));
 417                 OUT_RING  (chan, 1);
 418                 OUT_RING  (chan, 2); /* RELEASE */
 419         } else {
 420                 struct nouveau_vma *vma = &dev_priv->fence.bo->vma;
 421                 u64 offset = vma->offset + sema->mem->start;
 422
 423                 ret = RING_SPACE(chan, 5);
 424                 if (ret)
 425                         return ret;
 426
 427                 BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0010, 4);
 428                 OUT_RING  (chan, upper_32_bits(offset));
 429                 OUT_RING  (chan, lower_32_bits(offset));
 430                 OUT_RING  (chan, 1);
 431                 OUT_RING  (chan, 0x1002); /* RELEASE */
 432         }
 433
 434         /* Delay semaphore destruction until its work is done */
 435         ret = nouveau_fence_new(chan, &fence, true);
 436         if (ret)
 437                 return ret;
 438
 439         kref_get(&sema->ref);
 440         nouveau_fence_work(fence, semaphore_work, sema);
 441         nouveau_fence_unref(&fence);
 442         return 0;
 443 }
 444
 445 int
 446 nouveau_fence_sync(struct nouveau_fence *fence,
 447                    struct nouveau_channel *wchan)
 448 {
 449         struct nouveau_channel *chan = nouveau_fence_channel(fence);
 450         struct drm_device *dev = wchan->dev;
 451         struct nouveau_semaphore *sema;
 452         int ret = 0;
 453
 454         if (likely(!chan || chan == wchan ||
 455                    nouveau_fence_signalled(fence)))
 456                 goto out;
 457
 458         sema = semaphore_alloc(dev);
 459         if (!sema) {
 460                 /* Early card or broken userspace, fall back to
 461                  * software sync. */
 462                 ret = nouveau_fence_wait(fence, true, false);
 463                 goto out;
 464         }
 465
 466         /* try to take chan's mutex, if we can't take it right away
 467          * we have to fallback to software sync to prevent locking
 468          * order issues
 469          */
 470         if (!mutex_trylock(&chan->mutex)) {
 471                 ret = nouveau_fence_wait(fence, true, false);
 472                 goto out_unref;
 473         }
 474
 475         /* Make wchan wait until it gets signalled */
 476         ret = semaphore_acquire(wchan, sema);
 477         if (ret)
 478                 goto out_unlock;
 479
 480         /* Signal the semaphore from chan */
 481         ret = semaphore_release(chan, sema);
 482
 483 out_unlock:
 484         mutex_unlock(&chan->mutex);
 485 out_unref:
 486         kref_put(&sema->ref, semaphore_free);
 487 out:
 488         if (chan)
 489                 nouveau_channel_put_unlocked(&chan);
 490         return ret;
 491 }
 492
 493 int
 494 __nouveau_fence_flush(void *sync_obj, void *sync_arg)
 495 {
 496         return 0;
 497 }
 498
 499 int
 500 nouveau_fence_channel_init(struct nouveau_channel *chan)
 501 {
 502         struct drm_device *dev = chan->dev;
 503         struct drm_nouveau_private *dev_priv = dev->dev_private;
 504         struct nouveau_gpuobj *obj = NULL;
 505         int ret;
 506
 507         if (dev_priv->card_type >= NV_C0)
 508                 goto out_initialised;
 509
 510         /* Create an NV_SW object for various sync purposes */
 511         ret = nouveau_gpuobj_gr_new(chan, NvSw, NV_SW);
 512         if (ret)
 513                 return ret;
 514
 515         /* we leave subchannel empty for nvc0 */
 516         ret = RING_SPACE(chan, 2);
 517         if (ret)
 518                 return ret;
 519         BEGIN_RING(chan, NvSubSw, 0, 1);
 520         OUT_RING(chan, NvSw);
 521
 522         /* Create a DMA object for the shared cross-channel sync area. */
 523         if (USE_SEMA(dev) && dev_priv->chipset < 0x84) {
 524                 struct ttm_mem_reg *mem = &dev_priv->fence.bo->bo.mem;
 525
 526                 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
 527                                              mem->start << PAGE_SHIFT,
 528                                              mem->size, NV_MEM_ACCESS_RW,
 529                                              NV_MEM_TARGET_VRAM, &obj);
 530                 if (ret)
 531                         return ret;
 532
 533                 ret = nouveau_ramht_insert(chan, NvSema, obj);
 534                 nouveau_gpuobj_ref(NULL, &obj);
 535                 if (ret)
 536                         return ret;
 537
 538                 ret = RING_SPACE(chan, 2);
 539                 if (ret)
 540                         return ret;
 541                 BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 1);
 542                 OUT_RING(chan, NvSema);
 543         } else {
 544                 ret = RING_SPACE(chan, 2);
 545                 if (ret)
 546                         return ret;
 547                 BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 1);
 548                 OUT_RING  (chan, chan->vram_handle); /* whole VM */
 549         }
 550
 551         FIRE_RING(chan);
 552
 553 out_initialised:
 554         INIT_LIST_HEAD(&chan->fence.pending);
 555         spin_lock_init(&chan->fence.lock);
 556         atomic_set(&chan->fence.last_sequence_irq, 0);
 557         return 0;
 558 }
 559
 560 void
 561 nouveau_fence_channel_fini(struct nouveau_channel *chan)
 562 {
 563         struct nouveau_fence *tmp, *fence;
 564
 565         spin_lock(&chan->fence.lock);
 566
 567         list_for_each_entry_safe(fence, tmp, &chan->fence.pending, entry) {
 568                 fence->signalled = true;
 569                 list_del(&fence->entry);
 570
 571                 if (unlikely(fence->work))
 572                         fence->work(fence->priv, false);
 573
 574                 kref_put(&fence->refcount, nouveau_fence_del);
 575         }
 576
 577         spin_unlock(&chan->fence.lock);
 578 }
 579
 580 int
 581 nouveau_fence_init(struct drm_device *dev)
 582 {
 583         struct drm_nouveau_private *dev_priv = dev->dev_private;
 584         int size = (dev_priv->chipset < 0x84) ? 4096 : 16384;
 585         int ret;
 586
 587         /* Create a shared VRAM heap for cross-channel sync. */
 588         if (USE_SEMA(dev)) {
 589                 ret = nouveau_bo_new(dev, NULL, size, 0, TTM_PL_FLAG_VRAM,
 590                                      0, 0, &dev_priv->fence.bo);
 591                 if (ret)
 592                         return ret;
 593
 594                 ret = nouveau_bo_pin(dev_priv->fence.bo, TTM_PL_FLAG_VRAM);
 595                 if (ret)
 596                         goto fail;
 597
 598                 ret = nouveau_bo_map(dev_priv->fence.bo);
 599                 if (ret)
 600                         goto fail;
 601
 602                 ret = drm_mm_init(&dev_priv->fence.heap, 0,
 603                                   dev_priv->fence.bo->bo.mem.size);
 604                 if (ret)
 605                         goto fail;
 606
 607                 spin_lock_init(&dev_priv->fence.lock);
 608         }
 609
 610         return 0;
 611 fail:
 612         nouveau_bo_unmap(dev_priv->fence.bo);
 613         nouveau_bo_ref(NULL, &dev_priv->fence.bo);
 614         return ret;
 615 }
 616
 617 void
 618 nouveau_fence_fini(struct drm_device *dev)
 619 {
 620         struct drm_nouveau_private *dev_priv = dev->dev_private;
 621
 622         if (USE_SEMA(dev)) {
 623                 drm_mm_takedown(&dev_priv->fence.heap);
 624                 nouveau_bo_unmap(dev_priv->fence.bo);
 625                 nouveau_bo_unpin(dev_priv->fence.bo);
 626                 nouveau_bo_ref(NULL, &dev_priv->fence.bo);
 627         }
 628 }