drivers/gpu/drm/i915/gt/intel_context.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2019 Intel Corporation
   5  */
   6
   7 #include "gem/i915_gem_context.h"
   8 #include "gem/i915_gem_pm.h"
   9
  10 #include "i915_drv.h"
  11 #include "i915_globals.h"
  12
  13 #include "intel_context.h"
  14 #include "intel_engine.h"
  15 #include "intel_engine_pm.h"
  16 #include "intel_ring.h"
  17
  18 static struct i915_global_context {
  19         struct i915_global base;
  20         struct kmem_cache *slab_ce;
  21 } global;
  22
  23 static struct intel_context *intel_context_alloc(void)
  24 {
  25         return kmem_cache_zalloc(global.slab_ce, GFP_KERNEL);
  26 }
  27
  28 static void rcu_context_free(struct rcu_head *rcu)
  29 {
  30         struct intel_context *ce = container_of(rcu, typeof(*ce), rcu);
  31
  32         kmem_cache_free(global.slab_ce, ce);
  33 }
  34
  35 void intel_context_free(struct intel_context *ce)
  36 {
  37         call_rcu(&ce->rcu, rcu_context_free);
  38 }
  39
  40 struct intel_context *
  41 intel_context_create(struct intel_engine_cs *engine)
  42 {
  43         struct intel_context *ce;
  44
  45         ce = intel_context_alloc();
  46         if (!ce)
  47                 return ERR_PTR(-ENOMEM);
  48
  49         intel_context_init(ce, engine);
  50         return ce;
  51 }
  52
  53 int intel_context_alloc_state(struct intel_context *ce)
  54 {
  55         int err = 0;
  56
  57         if (mutex_lock_interruptible(&ce->pin_mutex))
  58                 return -EINTR;
  59
  60         if (!test_bit(CONTEXT_ALLOC_BIT, &ce->flags)) {
  61                 if (intel_context_is_banned(ce)) {
  62                         err = -EIO;
  63                         goto unlock;
  64                 }
  65
  66                 err = ce->ops->alloc(ce);
  67                 if (unlikely(err))
  68                         goto unlock;
  69
  70                 set_bit(CONTEXT_ALLOC_BIT, &ce->flags);
  71         }
  72
  73 unlock:
  74         mutex_unlock(&ce->pin_mutex);
  75         return err;
  76 }
  77
  78 static int intel_context_active_acquire(struct intel_context *ce)
  79 {
  80         int err;
  81
  82         __i915_active_acquire(&ce->active);
  83
  84         if (intel_context_is_barrier(ce))
  85                 return 0;
  86
  87         /* Preallocate tracking nodes */
  88         err = i915_active_acquire_preallocate_barrier(&ce->active,
  89                                                       ce->engine);
  90         if (err)
  91                 i915_active_release(&ce->active);
  92
  93         return err;
  94 }
  95
  96 static void intel_context_active_release(struct intel_context *ce)
  97 {
  98         /* Nodes preallocated in intel_context_active() */
  99         i915_active_acquire_barrier(&ce->active);
 100         i915_active_release(&ce->active);
 101 }
 102
 103 static int __context_pin_state(struct i915_vma *vma, struct i915_gem_ww_ctx *ww)
 104 {
 105         unsigned int bias = i915_ggtt_pin_bias(vma) | PIN_OFFSET_BIAS;
 106         int err;
 107
 108         err = i915_ggtt_pin(vma, ww, 0, bias | PIN_HIGH);
 109         if (err)
 110                 return err;
 111
 112         err = i915_active_acquire(&vma->active);
 113         if (err)
 114                 goto err_unpin;
 115
 116         /*
 117          * And mark it as a globally pinned object to let the shrinker know
 118          * it cannot reclaim the object until we release it.
 119          */
 120         i915_vma_make_unshrinkable(vma);
 121         vma->obj->mm.dirty = true;
 122
 123         return 0;
 124
 125 err_unpin:
 126         i915_vma_unpin(vma);
 127         return err;
 128 }
 129
 130 static void __context_unpin_state(struct i915_vma *vma)
 131 {
 132         i915_vma_make_shrinkable(vma);
 133         i915_active_release(&vma->active);
 134         __i915_vma_unpin(vma);
 135 }
 136
 137 static int __ring_active(struct intel_ring *ring,
 138                          struct i915_gem_ww_ctx *ww)
 139 {
 140         int err;
 141
 142         err = intel_ring_pin(ring, ww);
 143         if (err)
 144                 return err;
 145
 146         err = i915_active_acquire(&ring->vma->active);
 147         if (err)
 148                 goto err_pin;
 149
 150         return 0;
 151
 152 err_pin:
 153         intel_ring_unpin(ring);
 154         return err;
 155 }
 156
 157 static void __ring_retire(struct intel_ring *ring)
 158 {
 159         i915_active_release(&ring->vma->active);
 160         intel_ring_unpin(ring);
 161 }
 162
 163 static int intel_context_pre_pin(struct intel_context *ce,
 164                                  struct i915_gem_ww_ctx *ww)
 165 {
 166         int err;
 167
 168         CE_TRACE(ce, "active\n");
 169
 170         err = __ring_active(ce->ring, ww);
 171         if (err)
 172                 return err;
 173
 174         err = intel_timeline_pin(ce->timeline, ww);
 175         if (err)
 176                 goto err_ring;
 177
 178         if (!ce->state)
 179                 return 0;
 180
 181         err = __context_pin_state(ce->state, ww);
 182         if (err)
 183                 goto err_timeline;
 184
 185
 186         return 0;
 187
 188 err_timeline:
 189         intel_timeline_unpin(ce->timeline);
 190 err_ring:
 191         __ring_retire(ce->ring);
 192         return err;
 193 }
 194
 195 static void intel_context_post_unpin(struct intel_context *ce)
 196 {
 197         if (ce->state)
 198                 __context_unpin_state(ce->state);
 199
 200         intel_timeline_unpin(ce->timeline);
 201         __ring_retire(ce->ring);
 202 }
 203
 204 int __intel_context_do_pin_ww(struct intel_context *ce,
 205                               struct i915_gem_ww_ctx *ww)
 206 {
 207         bool handoff = false;
 208         void *vaddr;
 209         int err = 0;
 210
 211         if (unlikely(!test_bit(CONTEXT_ALLOC_BIT, &ce->flags))) {
 212                 err = intel_context_alloc_state(ce);
 213                 if (err)
 214                         return err;
 215         }
 216
 217         /*
 218          * We always pin the context/ring/timeline here, to ensure a pin
 219          * refcount for __intel_context_active(), which prevent a lock
 220          * inversion of ce->pin_mutex vs dma_resv_lock().
 221          */
 222
 223         err = i915_gem_object_lock(ce->timeline->hwsp_ggtt->obj, ww);
 224         if (!err && ce->ring->vma->obj)
 225                 err = i915_gem_object_lock(ce->ring->vma->obj, ww);
 226         if (!err && ce->state)
 227                 err = i915_gem_object_lock(ce->state->obj, ww);
 228         if (!err)
 229                 err = intel_context_pre_pin(ce, ww);
 230         if (err)
 231                 return err;
 232
 233         err = i915_active_acquire(&ce->active);
 234         if (err)
 235                 goto err_ctx_unpin;
 236
 237         err = ce->ops->pre_pin(ce, ww, &vaddr);
 238         if (err)
 239                 goto err_release;
 240
 241         err = mutex_lock_interruptible(&ce->pin_mutex);
 242         if (err)
 243                 goto err_post_unpin;
 244
 245         if (unlikely(intel_context_is_closed(ce))) {
 246                 err = -ENOENT;
 247                 goto err_unlock;
 248         }
 249
 250         if (likely(!atomic_add_unless(&ce->pin_count, 1, 0))) {
 251                 err = intel_context_active_acquire(ce);
 252                 if (unlikely(err))
 253                         goto err_unlock;
 254
 255                 err = ce->ops->pin(ce, vaddr);
 256                 if (err) {
 257                         intel_context_active_release(ce);
 258                         goto err_unlock;
 259                 }
 260
 261                 CE_TRACE(ce, "pin ring:{start:%08x, head:%04x, tail:%04x}\n",
 262                          i915_ggtt_offset(ce->ring->vma),
 263                          ce->ring->head, ce->ring->tail);
 264
 265                 handoff = true;
 266                 smp_mb__before_atomic(); /* flush pin before it is visible */
 267                 atomic_inc(&ce->pin_count);
 268         }
 269
 270         GEM_BUG_ON(!intel_context_is_pinned(ce)); /* no overflow! */
 271
 272 err_unlock:
 273         mutex_unlock(&ce->pin_mutex);
 274 err_post_unpin:
 275         if (!handoff)
 276                 ce->ops->post_unpin(ce);
 277 err_release:
 278         i915_active_release(&ce->active);
 279 err_ctx_unpin:
 280         intel_context_post_unpin(ce);
 281
 282         /*
 283          * Unlock the hwsp_ggtt object since it's shared.
 284          * In principle we can unlock all the global state locked above
 285          * since it's pinned and doesn't need fencing, and will
 286          * thus remain resident until it is explicitly unpinned.
 287          */
 288         i915_gem_ww_unlock_single(ce->timeline->hwsp_ggtt->obj);
 289
 290         return err;
 291 }
 292
 293 int __intel_context_do_pin(struct intel_context *ce)
 294 {
 295         struct i915_gem_ww_ctx ww;
 296         int err;
 297
 298         i915_gem_ww_ctx_init(&ww, true);
 299 retry:
 300         err = __intel_context_do_pin_ww(ce, &ww);
 301         if (err == -EDEADLK) {
 302                 err = i915_gem_ww_ctx_backoff(&ww);
 303                 if (!err)
 304                         goto retry;
 305         }
 306         i915_gem_ww_ctx_fini(&ww);
 307         return err;
 308 }
 309
 310 void intel_context_unpin(struct intel_context *ce)
 311 {
 312         if (!atomic_dec_and_test(&ce->pin_count))
 313                 return;
 314
 315         CE_TRACE(ce, "unpin\n");
 316         ce->ops->unpin(ce);
 317         ce->ops->post_unpin(ce);
 318
 319         /*
 320          * Once released, we may asynchronously drop the active reference.
 321          * As that may be the only reference keeping the context alive,
 322          * take an extra now so that it is not freed before we finish
 323          * dereferencing it.
 324          */
 325         intel_context_get(ce);
 326         intel_context_active_release(ce);
 327         intel_context_put(ce);
 328 }
 329
 330 __i915_active_call
 331 static void __intel_context_retire(struct i915_active *active)
 332 {
 333         struct intel_context *ce = container_of(active, typeof(*ce), active);
 334
 335         CE_TRACE(ce, "retire runtime: { total:%lluns, avg:%lluns }\n",
 336                  intel_context_get_total_runtime_ns(ce),
 337                  intel_context_get_avg_runtime_ns(ce));
 338
 339         set_bit(CONTEXT_VALID_BIT, &ce->flags);
 340         intel_context_post_unpin(ce);
 341         intel_context_put(ce);
 342 }
 343
 344 static int __intel_context_active(struct i915_active *active)
 345 {
 346         struct intel_context *ce = container_of(active, typeof(*ce), active);
 347
 348         intel_context_get(ce);
 349
 350         /* everything should already be activated by intel_context_pre_pin() */
 351         GEM_WARN_ON(!i915_active_acquire_if_busy(&ce->ring->vma->active));
 352         __intel_ring_pin(ce->ring);
 353
 354         __intel_timeline_pin(ce->timeline);
 355
 356         if (ce->state) {
 357                 GEM_WARN_ON(!i915_active_acquire_if_busy(&ce->state->active));
 358                 __i915_vma_pin(ce->state);
 359                 i915_vma_make_unshrinkable(ce->state);
 360         }
 361
 362         return 0;
 363 }
 364
 365 void
 366 intel_context_init(struct intel_context *ce, struct intel_engine_cs *engine)
 367 {
 368         GEM_BUG_ON(!engine->cops);
 369         GEM_BUG_ON(!engine->gt->vm);
 370
 371         kref_init(&ce->ref);
 372
 373         ce->engine = engine;
 374         ce->ops = engine->cops;
 375         ce->sseu = engine->sseu;
 376         ce->ring = __intel_context_ring_size(SZ_4K);
 377
 378         ewma_runtime_init(&ce->runtime.avg);
 379
 380         ce->vm = i915_vm_get(engine->gt->vm);
 381
 382         /* NB ce->signal_link/lock is used under RCU */
 383         spin_lock_init(&ce->signal_lock);
 384         INIT_LIST_HEAD(&ce->signals);
 385
 386         mutex_init(&ce->pin_mutex);
 387
 388         i915_active_init(&ce->active,
 389                          __intel_context_active, __intel_context_retire);
 390 }
 391
 392 void intel_context_fini(struct intel_context *ce)
 393 {
 394         if (ce->timeline)
 395                 intel_timeline_put(ce->timeline);
 396         i915_vm_put(ce->vm);
 397
 398         mutex_destroy(&ce->pin_mutex);
 399         i915_active_fini(&ce->active);
 400 }
 401
 402 static void i915_global_context_shrink(void)
 403 {
 404         kmem_cache_shrink(global.slab_ce);
 405 }
 406
 407 static void i915_global_context_exit(void)
 408 {
 409         kmem_cache_destroy(global.slab_ce);
 410 }
 411
 412 static struct i915_global_context global = { {
 413         .shrink = i915_global_context_shrink,
 414         .exit = i915_global_context_exit,
 415 } };
 416
 417 int __init i915_global_context_init(void)
 418 {
 419         global.slab_ce = KMEM_CACHE(intel_context, SLAB_HWCACHE_ALIGN);
 420         if (!global.slab_ce)
 421                 return -ENOMEM;
 422
 423         i915_global_register(&global.base);
 424         return 0;
 425 }
 426
 427 void intel_context_enter_engine(struct intel_context *ce)
 428 {
 429         intel_engine_pm_get(ce->engine);
 430         intel_timeline_enter(ce->timeline);
 431 }
 432
 433 void intel_context_exit_engine(struct intel_context *ce)
 434 {
 435         intel_timeline_exit(ce->timeline);
 436         intel_engine_pm_put(ce->engine);
 437 }
 438
 439 int intel_context_prepare_remote_request(struct intel_context *ce,
 440                                          struct i915_request *rq)
 441 {
 442         struct intel_timeline *tl = ce->timeline;
 443         int err;
 444
 445         /* Only suitable for use in remotely modifying this context */
 446         GEM_BUG_ON(rq->context == ce);
 447
 448         if (rcu_access_pointer(rq->timeline) != tl) { /* timeline sharing! */
 449                 /* Queue this switch after current activity by this context. */
 450                 err = i915_active_fence_set(&tl->last_request, rq);
 451                 if (err)
 452                         return err;
 453         }
 454
 455         /*
 456          * Guarantee context image and the timeline remains pinned until the
 457          * modifying request is retired by setting the ce activity tracker.
 458          *
 459          * But we only need to take one pin on the account of it. Or in other
 460          * words transfer the pinned ce object to tracked active request.
 461          */
 462         GEM_BUG_ON(i915_active_is_idle(&ce->active));
 463         return i915_active_add_request(&ce->active, rq);
 464 }
 465
 466 struct i915_request *intel_context_create_request(struct intel_context *ce)
 467 {
 468         struct i915_gem_ww_ctx ww;
 469         struct i915_request *rq;
 470         int err;
 471
 472         i915_gem_ww_ctx_init(&ww, true);
 473 retry:
 474         err = intel_context_pin_ww(ce, &ww);
 475         if (!err) {
 476                 rq = i915_request_create(ce);
 477                 intel_context_unpin(ce);
 478         } else if (err == -EDEADLK) {
 479                 err = i915_gem_ww_ctx_backoff(&ww);
 480                 if (!err)
 481                         goto retry;
 482                 rq = ERR_PTR(err);
 483         } else {
 484                 rq = ERR_PTR(err);
 485         }
 486
 487         i915_gem_ww_ctx_fini(&ww);
 488
 489         if (IS_ERR(rq))
 490                 return rq;
 491
 492         /*
 493          * timeline->mutex should be the inner lock, but is used as outer lock.
 494          * Hack around this to shut up lockdep in selftests..
 495          */
 496         lockdep_unpin_lock(&ce->timeline->mutex, rq->cookie);
 497         mutex_release(&ce->timeline->mutex.dep_map, _RET_IP_);
 498         mutex_acquire(&ce->timeline->mutex.dep_map, SINGLE_DEPTH_NESTING, 0, _RET_IP_);
 499         rq->cookie = lockdep_pin_lock(&ce->timeline->mutex);
 500
 501         return rq;
 502 }
 503
 504 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 505 #include "selftest_context.c"
 506 #endif