drivers/gpu/drm/nouveau/nouveau_fence.c

   1 /*
   2  * Copyright (C) 2007 Ben Skeggs.
   3  * All Rights Reserved.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining
   6  * a copy of this software and associated documentation files (the
   7  * "Software"), to deal in the Software without restriction, including
   8  * without limitation the rights to use, copy, modify, merge, publish,
   9  * distribute, sublicense, and/or sell copies of the Software, and to
  10  * permit persons to whom the Software is furnished to do so, subject to
  11  * the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the
  14  * next paragraph) shall be included in all copies or substantial
  15  * portions of the Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  18  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  19  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  20  * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  21  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  22  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  23  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  24  *
  25  */
  26
  27 #include <drm/drmP.h>
  28
  29 #include <linux/ktime.h>
  30 #include <linux/hrtimer.h>
  31 #include <trace/events/dma_fence.h>
  32
  33 #include <nvif/cl826e.h>
  34 #include <nvif/notify.h>
  35 #include <nvif/event.h>
  36
  37 #include "nouveau_drv.h"
  38 #include "nouveau_dma.h"
  39 #include "nouveau_fence.h"
  40
  41 static const struct dma_fence_ops nouveau_fence_ops_uevent;
  42 static const struct dma_fence_ops nouveau_fence_ops_legacy;
  43
  44 static inline struct nouveau_fence *
  45 from_fence(struct dma_fence *fence)
  46 {
  47         return container_of(fence, struct nouveau_fence, base);
  48 }
  49
  50 static inline struct nouveau_fence_chan *
  51 nouveau_fctx(struct nouveau_fence *fence)
  52 {
  53         return container_of(fence->base.lock, struct nouveau_fence_chan, lock);
  54 }
  55
  56 static int
  57 nouveau_fence_signal(struct nouveau_fence *fence)
  58 {
  59         int drop = 0;
  60
  61         dma_fence_signal_locked(&fence->base);
  62         list_del(&fence->head);
  63         rcu_assign_pointer(fence->channel, NULL);
  64
  65         if (test_bit(DMA_FENCE_FLAG_USER_BITS, &fence->base.flags)) {
  66                 struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
  67
  68                 if (!--fctx->notify_ref)
  69                         drop = 1;
  70         }
  71
  72         dma_fence_put(&fence->base);
  73         return drop;
  74 }
  75
  76 static struct nouveau_fence *
  77 nouveau_local_fence(struct dma_fence *fence, struct nouveau_drm *drm)
  78 {
  79         if (fence->ops != &nouveau_fence_ops_legacy &&
  80             fence->ops != &nouveau_fence_ops_uevent)
  81                 return NULL;
  82
  83         if (fence->context < drm->chan.context_base ||
  84             fence->context >= drm->chan.context_base + drm->chan.nr)
  85                 return NULL;
  86
  87         return from_fence(fence);
  88 }
  89
  90 void
  91 nouveau_fence_context_del(struct nouveau_fence_chan *fctx)
  92 {
  93         struct nouveau_fence *fence;
  94
  95         spin_lock_irq(&fctx->lock);
  96         while (!list_empty(&fctx->pending)) {
  97                 fence = list_entry(fctx->pending.next, typeof(*fence), head);
  98
  99                 if (nouveau_fence_signal(fence))
 100                         nvif_notify_put(&fctx->notify);
 101         }
 102         spin_unlock_irq(&fctx->lock);
 103
 104         nvif_notify_fini(&fctx->notify);
 105         fctx->dead = 1;
 106
 107         /*
 108          * Ensure that all accesses to fence->channel complete before freeing
 109          * the channel.
 110          */
 111         synchronize_rcu();
 112 }
 113
 114 static void
 115 nouveau_fence_context_put(struct kref *fence_ref)
 116 {
 117         kfree(container_of(fence_ref, struct nouveau_fence_chan, fence_ref));
 118 }
 119
 120 void
 121 nouveau_fence_context_free(struct nouveau_fence_chan *fctx)
 122 {
 123         kref_put(&fctx->fence_ref, nouveau_fence_context_put);
 124 }
 125
 126 static int
 127 nouveau_fence_update(struct nouveau_channel *chan, struct nouveau_fence_chan *fctx)
 128 {
 129         struct nouveau_fence *fence;
 130         int drop = 0;
 131         u32 seq = fctx->read(chan);
 132
 133         while (!list_empty(&fctx->pending)) {
 134                 fence = list_entry(fctx->pending.next, typeof(*fence), head);
 135
 136                 if ((int)(seq - fence->base.seqno) < 0)
 137                         break;
 138
 139                 drop |= nouveau_fence_signal(fence);
 140         }
 141
 142         return drop;
 143 }
 144
 145 static int
 146 nouveau_fence_wait_uevent_handler(struct nvif_notify *notify)
 147 {
 148         struct nouveau_fence_chan *fctx =
 149                 container_of(notify, typeof(*fctx), notify);
 150         unsigned long flags;
 151         int ret = NVIF_NOTIFY_KEEP;
 152
 153         spin_lock_irqsave(&fctx->lock, flags);
 154         if (!list_empty(&fctx->pending)) {
 155                 struct nouveau_fence *fence;
 156                 struct nouveau_channel *chan;
 157
 158                 fence = list_entry(fctx->pending.next, typeof(*fence), head);
 159                 chan = rcu_dereference_protected(fence->channel, lockdep_is_held(&fctx->lock));
 160                 if (nouveau_fence_update(fence->channel, fctx))
 161                         ret = NVIF_NOTIFY_DROP;
 162         }
 163         spin_unlock_irqrestore(&fctx->lock, flags);
 164
 165         return ret;
 166 }
 167
 168 void
 169 nouveau_fence_context_new(struct nouveau_channel *chan, struct nouveau_fence_chan *fctx)
 170 {
 171         struct nouveau_fence_priv *priv = (void*)chan->drm->fence;
 172         struct nouveau_cli *cli = (void *)chan->user.client;
 173         int ret;
 174
 175         INIT_LIST_HEAD(&fctx->flip);
 176         INIT_LIST_HEAD(&fctx->pending);
 177         spin_lock_init(&fctx->lock);
 178         fctx->context = chan->drm->chan.context_base + chan->chid;
 179
 180         if (chan == chan->drm->cechan)
 181                 strcpy(fctx->name, "copy engine channel");
 182         else if (chan == chan->drm->channel)
 183                 strcpy(fctx->name, "generic kernel channel");
 184         else
 185                 strcpy(fctx->name, nvxx_client(&cli->base)->name);
 186
 187         kref_init(&fctx->fence_ref);
 188         if (!priv->uevent)
 189                 return;
 190
 191         ret = nvif_notify_init(&chan->user, nouveau_fence_wait_uevent_handler,
 192                                false, NV826E_V0_NTFY_NON_STALL_INTERRUPT,
 193                                &(struct nvif_notify_uevent_req) { },
 194                                sizeof(struct nvif_notify_uevent_req),
 195                                sizeof(struct nvif_notify_uevent_rep),
 196                                &fctx->notify);
 197
 198         WARN_ON(ret);
 199 }
 200
 201 int
 202 nouveau_fence_emit(struct nouveau_fence *fence, struct nouveau_channel *chan)
 203 {
 204         struct nouveau_fence_chan *fctx = chan->fence;
 205         struct nouveau_fence_priv *priv = (void*)chan->drm->fence;
 206         int ret;
 207
 208         fence->channel  = chan;
 209         fence->timeout  = jiffies + (15 * HZ);
 210
 211         if (priv->uevent)
 212                 dma_fence_init(&fence->base, &nouveau_fence_ops_uevent,
 213                                &fctx->lock, fctx->context, ++fctx->sequence);
 214         else
 215                 dma_fence_init(&fence->base, &nouveau_fence_ops_legacy,
 216                                &fctx->lock, fctx->context, ++fctx->sequence);
 217         kref_get(&fctx->fence_ref);
 218
 219         trace_dma_fence_emit(&fence->base);
 220         ret = fctx->emit(fence);
 221         if (!ret) {
 222                 dma_fence_get(&fence->base);
 223                 spin_lock_irq(&fctx->lock);
 224
 225                 if (nouveau_fence_update(chan, fctx))
 226                         nvif_notify_put(&fctx->notify);
 227
 228                 list_add_tail(&fence->head, &fctx->pending);
 229                 spin_unlock_irq(&fctx->lock);
 230         }
 231
 232         return ret;
 233 }
 234
 235 bool
 236 nouveau_fence_done(struct nouveau_fence *fence)
 237 {
 238         if (fence->base.ops == &nouveau_fence_ops_legacy ||
 239             fence->base.ops == &nouveau_fence_ops_uevent) {
 240                 struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
 241                 struct nouveau_channel *chan;
 242                 unsigned long flags;
 243
 244                 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->base.flags))
 245                         return true;
 246
 247                 spin_lock_irqsave(&fctx->lock, flags);
 248                 chan = rcu_dereference_protected(fence->channel, lockdep_is_held(&fctx->lock));
 249                 if (chan && nouveau_fence_update(chan, fctx))
 250                         nvif_notify_put(&fctx->notify);
 251                 spin_unlock_irqrestore(&fctx->lock, flags);
 252         }
 253         return dma_fence_is_signaled(&fence->base);
 254 }
 255
 256 static long
 257 nouveau_fence_wait_legacy(struct dma_fence *f, bool intr, long wait)
 258 {
 259         struct nouveau_fence *fence = from_fence(f);
 260         unsigned long sleep_time = NSEC_PER_MSEC / 1000;
 261         unsigned long t = jiffies, timeout = t + wait;
 262
 263         while (!nouveau_fence_done(fence)) {
 264                 ktime_t kt;
 265
 266                 t = jiffies;
 267
 268                 if (wait != MAX_SCHEDULE_TIMEOUT && time_after_eq(t, timeout)) {
 269                         __set_current_state(TASK_RUNNING);
 270                         return 0;
 271                 }
 272
 273                 __set_current_state(intr ? TASK_INTERRUPTIBLE :
 274                                            TASK_UNINTERRUPTIBLE);
 275
 276                 kt = sleep_time;
 277                 schedule_hrtimeout(&kt, HRTIMER_MODE_REL);
 278                 sleep_time *= 2;
 279                 if (sleep_time > NSEC_PER_MSEC)
 280                         sleep_time = NSEC_PER_MSEC;
 281
 282                 if (intr && signal_pending(current))
 283                         return -ERESTARTSYS;
 284         }
 285
 286         __set_current_state(TASK_RUNNING);
 287
 288         return timeout - t;
 289 }
 290
 291 static int
 292 nouveau_fence_wait_busy(struct nouveau_fence *fence, bool intr)
 293 {
 294         int ret = 0;
 295
 296         while (!nouveau_fence_done(fence)) {
 297                 if (time_after_eq(jiffies, fence->timeout)) {
 298                         ret = -EBUSY;
 299                         break;
 300                 }
 301
 302                 __set_current_state(intr ?
 303                                     TASK_INTERRUPTIBLE :
 304                                     TASK_UNINTERRUPTIBLE);
 305
 306                 if (intr && signal_pending(current)) {
 307                         ret = -ERESTARTSYS;
 308                         break;
 309                 }
 310         }
 311
 312         __set_current_state(TASK_RUNNING);
 313         return ret;
 314 }
 315
 316 int
 317 nouveau_fence_wait(struct nouveau_fence *fence, bool lazy, bool intr)
 318 {
 319         long ret;
 320
 321         if (!lazy)
 322                 return nouveau_fence_wait_busy(fence, intr);
 323
 324         ret = dma_fence_wait_timeout(&fence->base, intr, 15 * HZ);
 325         if (ret < 0)
 326                 return ret;
 327         else if (!ret)
 328                 return -EBUSY;
 329         else
 330                 return 0;
 331 }
 332
 333 int
 334 nouveau_fence_sync(struct nouveau_bo *nvbo, struct nouveau_channel *chan, bool exclusive, bool intr)
 335 {
 336         struct nouveau_fence_chan *fctx = chan->fence;
 337         struct dma_fence *fence;
 338         struct reservation_object *resv = nvbo->bo.resv;
 339         struct reservation_object_list *fobj;
 340         struct nouveau_fence *f;
 341         int ret = 0, i;
 342
 343         if (!exclusive) {
 344                 ret = reservation_object_reserve_shared(resv, 1);
 345
 346                 if (ret)
 347                         return ret;
 348         }
 349
 350         fobj = reservation_object_get_list(resv);
 351         fence = reservation_object_get_excl(resv);
 352
 353         if (fence && (!exclusive || !fobj || !fobj->shared_count)) {
 354                 struct nouveau_channel *prev = NULL;
 355                 bool must_wait = true;
 356
 357                 f = nouveau_local_fence(fence, chan->drm);
 358                 if (f) {
 359                         rcu_read_lock();
 360                         prev = rcu_dereference(f->channel);
 361                         if (prev && (prev == chan || fctx->sync(f, prev, chan) == 0))
 362                                 must_wait = false;
 363                         rcu_read_unlock();
 364                 }
 365
 366                 if (must_wait)
 367                         ret = dma_fence_wait(fence, intr);
 368
 369                 return ret;
 370         }
 371
 372         if (!exclusive || !fobj)
 373                 return ret;
 374
 375         for (i = 0; i < fobj->shared_count && !ret; ++i) {
 376                 struct nouveau_channel *prev = NULL;
 377                 bool must_wait = true;
 378
 379                 fence = rcu_dereference_protected(fobj->shared[i],
 380                                                 reservation_object_held(resv));
 381
 382                 f = nouveau_local_fence(fence, chan->drm);
 383                 if (f) {
 384                         rcu_read_lock();
 385                         prev = rcu_dereference(f->channel);
 386                         if (prev && (prev == chan || fctx->sync(f, prev, chan) == 0))
 387                                 must_wait = false;
 388                         rcu_read_unlock();
 389                 }
 390
 391                 if (must_wait)
 392                         ret = dma_fence_wait(fence, intr);
 393         }
 394
 395         return ret;
 396 }
 397
 398 void
 399 nouveau_fence_unref(struct nouveau_fence **pfence)
 400 {
 401         if (*pfence)
 402                 dma_fence_put(&(*pfence)->base);
 403         *pfence = NULL;
 404 }
 405
 406 int
 407 nouveau_fence_new(struct nouveau_channel *chan, bool sysmem,
 408                   struct nouveau_fence **pfence)
 409 {
 410         struct nouveau_fence *fence;
 411         int ret = 0;
 412
 413         if (unlikely(!chan->fence))
 414                 return -ENODEV;
 415
 416         fence = kzalloc(sizeof(*fence), GFP_KERNEL);
 417         if (!fence)
 418                 return -ENOMEM;
 419
 420         ret = nouveau_fence_emit(fence, chan);
 421         if (ret)
 422                 nouveau_fence_unref(&fence);
 423
 424         *pfence = fence;
 425         return ret;
 426 }
 427
 428 static const char *nouveau_fence_get_get_driver_name(struct dma_fence *fence)
 429 {
 430         return "nouveau";
 431 }
 432
 433 static const char *nouveau_fence_get_timeline_name(struct dma_fence *f)
 434 {
 435         struct nouveau_fence *fence = from_fence(f);
 436         struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
 437
 438         return !fctx->dead ? fctx->name : "dead channel";
 439 }
 440
 441 /*
 442  * In an ideal world, read would not assume the channel context is still alive.
 443  * This function may be called from another device, running into free memory as a
 444  * result. The drm node should still be there, so we can derive the index from
 445  * the fence context.
 446  */
 447 static bool nouveau_fence_is_signaled(struct dma_fence *f)
 448 {
 449         struct nouveau_fence *fence = from_fence(f);
 450         struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
 451         struct nouveau_channel *chan;
 452         bool ret = false;
 453
 454         rcu_read_lock();
 455         chan = rcu_dereference(fence->channel);
 456         if (chan)
 457                 ret = (int)(fctx->read(chan) - fence->base.seqno) >= 0;
 458         rcu_read_unlock();
 459
 460         return ret;
 461 }
 462
 463 static bool nouveau_fence_no_signaling(struct dma_fence *f)
 464 {
 465         struct nouveau_fence *fence = from_fence(f);
 466
 467         /*
 468          * caller should have a reference on the fence,
 469          * else fence could get freed here
 470          */
 471         WARN_ON(kref_read(&fence->base.refcount) <= 1);
 472
 473         /*
 474          * This needs uevents to work correctly, but dma_fence_add_callback relies on
 475          * being able to enable signaling. It will still get signaled eventually,
 476          * just not right away.
 477          */
 478         if (nouveau_fence_is_signaled(f)) {
 479                 list_del(&fence->head);
 480
 481                 dma_fence_put(&fence->base);
 482                 return false;
 483         }
 484
 485         return true;
 486 }
 487
 488 static void nouveau_fence_release(struct dma_fence *f)
 489 {
 490         struct nouveau_fence *fence = from_fence(f);
 491         struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
 492
 493         kref_put(&fctx->fence_ref, nouveau_fence_context_put);
 494         dma_fence_free(&fence->base);
 495 }
 496
 497 static const struct dma_fence_ops nouveau_fence_ops_legacy = {
 498         .get_driver_name = nouveau_fence_get_get_driver_name,
 499         .get_timeline_name = nouveau_fence_get_timeline_name,
 500         .enable_signaling = nouveau_fence_no_signaling,
 501         .signaled = nouveau_fence_is_signaled,
 502         .wait = nouveau_fence_wait_legacy,
 503         .release = nouveau_fence_release
 504 };
 505
 506 static bool nouveau_fence_enable_signaling(struct dma_fence *f)
 507 {
 508         struct nouveau_fence *fence = from_fence(f);
 509         struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
 510         bool ret;
 511
 512         if (!fctx->notify_ref++)
 513                 nvif_notify_get(&fctx->notify);
 514
 515         ret = nouveau_fence_no_signaling(f);
 516         if (ret)
 517                 set_bit(DMA_FENCE_FLAG_USER_BITS, &fence->base.flags);
 518         else if (!--fctx->notify_ref)
 519                 nvif_notify_put(&fctx->notify);
 520
 521         return ret;
 522 }
 523
 524 static const struct dma_fence_ops nouveau_fence_ops_uevent = {
 525         .get_driver_name = nouveau_fence_get_get_driver_name,
 526         .get_timeline_name = nouveau_fence_get_timeline_name,
 527         .enable_signaling = nouveau_fence_enable_signaling,
 528         .signaled = nouveau_fence_is_signaled,
 529         .release = nouveau_fence_release
 530 };