drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2019 Intel Corporation
   5  */
   6
   7 #include "i915_drv.h"
   8 #include "i915_request.h"
   9
  10 #include "intel_context.h"
  11 #include "intel_engine_heartbeat.h"
  12 #include "intel_engine_pm.h"
  13 #include "intel_engine.h"
  14 #include "intel_gt.h"
  15 #include "intel_reset.h"
  16
  17 /*
  18  * While the engine is active, we send a periodic pulse along the engine
  19  * to check on its health and to flush any idle-barriers. If that request
  20  * is stuck, and we fail to preempt it, we declare the engine hung and
  21  * issue a reset -- in the hope that restores progress.
  22  */
  23
  24 static bool next_heartbeat(struct intel_engine_cs *engine)
  25 {
  26         long delay;
  27
  28         delay = READ_ONCE(engine->props.heartbeat_interval_ms);
  29         if (!delay)
  30                 return false;
  31
  32         delay = msecs_to_jiffies_timeout(delay);
  33         if (delay >= HZ)
  34                 delay = round_jiffies_up_relative(delay);
  35         mod_delayed_work(system_highpri_wq, &engine->heartbeat.work, delay);
  36
  37         return true;
  38 }
  39
  40 static void idle_pulse(struct intel_engine_cs *engine, struct i915_request *rq)
  41 {
  42         engine->wakeref_serial = READ_ONCE(engine->serial) + 1;
  43         i915_request_add_active_barriers(rq);
  44         if (!engine->heartbeat.systole && intel_engine_has_heartbeat(engine))
  45                 engine->heartbeat.systole = i915_request_get(rq);
  46 }
  47
  48 static void show_heartbeat(const struct i915_request *rq,
  49                            struct intel_engine_cs *engine)
  50 {
  51         struct drm_printer p = drm_debug_printer("heartbeat");
  52
  53         intel_engine_dump(engine, &p,
  54                           "%s heartbeat {seqno:%llx:%lld, prio:%d} not ticking\n",
  55                           engine->name,
  56                           rq->fence.context,
  57                           rq->fence.seqno,
  58                           rq->sched.attr.priority);
  59 }
  60
  61 static void heartbeat(struct work_struct *wrk)
  62 {
  63         struct i915_sched_attr attr = {
  64                 .priority = I915_USER_PRIORITY(I915_PRIORITY_MIN),
  65         };
  66         struct intel_engine_cs *engine =
  67                 container_of(wrk, typeof(*engine), heartbeat.work.work);
  68         struct intel_context *ce = engine->kernel_context;
  69         struct i915_request *rq;
  70         unsigned long serial;
  71
  72         /* Just in case everything has gone horribly wrong, give it a kick */
  73         intel_engine_flush_submission(engine);
  74
  75         rq = engine->heartbeat.systole;
  76         if (rq && i915_request_completed(rq)) {
  77                 i915_request_put(rq);
  78                 engine->heartbeat.systole = NULL;
  79         }
  80
  81         if (!intel_engine_pm_get_if_awake(engine))
  82                 return;
  83
  84         if (intel_gt_is_wedged(engine->gt))
  85                 goto out;
  86
  87         if (engine->heartbeat.systole) {
  88                 if (!i915_sw_fence_signaled(&rq->submit)) {
  89                         /*
  90                          * Not yet submitted, system is stalled.
  91                          *
  92                          * This more often happens for ring submission,
  93                          * where all contexts are funnelled into a common
  94                          * ringbuffer. If one context is blocked on an
  95                          * external fence, not only is it not submitted,
  96                          * but all other contexts, including the kernel
  97                          * context are stuck waiting for the signal.
  98                          */
  99                 } else if (engine->schedule &&
 100                            rq->sched.attr.priority < I915_PRIORITY_BARRIER) {
 101                         /*
 102                          * Gradually raise the priority of the heartbeat to
 103                          * give high priority work [which presumably desires
 104                          * low latency and no jitter] the chance to naturally
 105                          * complete before being preempted.
 106                          */
 107                         attr.priority = I915_PRIORITY_MASK;
 108                         if (rq->sched.attr.priority >= attr.priority)
 109                                 attr.priority |= I915_USER_PRIORITY(I915_PRIORITY_HEARTBEAT);
 110                         if (rq->sched.attr.priority >= attr.priority)
 111                                 attr.priority = I915_PRIORITY_BARRIER;
 112
 113                         local_bh_disable();
 114                         engine->schedule(rq, &attr);
 115                         local_bh_enable();
 116                 } else {
 117                         if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
 118                                 show_heartbeat(rq, engine);
 119
 120                         intel_gt_handle_error(engine->gt, engine->mask,
 121                                               I915_ERROR_CAPTURE,
 122                                               "stopped heartbeat on %s",
 123                                               engine->name);
 124                 }
 125                 goto out;
 126         }
 127
 128         serial = READ_ONCE(engine->serial);
 129         if (engine->wakeref_serial == serial)
 130                 goto out;
 131
 132         if (!mutex_trylock(&ce->timeline->mutex)) {
 133                 /* Unable to lock the kernel timeline, is the engine stuck? */
 134                 if (xchg(&engine->heartbeat.blocked, serial) == serial)
 135                         intel_gt_handle_error(engine->gt, engine->mask,
 136                                               I915_ERROR_CAPTURE,
 137                                               "no heartbeat on %s",
 138                                               engine->name);
 139                 goto out;
 140         }
 141
 142         intel_context_enter(ce);
 143         rq = __i915_request_create(ce, GFP_NOWAIT | __GFP_NOWARN);
 144         intel_context_exit(ce);
 145         if (IS_ERR(rq))
 146                 goto unlock;
 147
 148         idle_pulse(engine, rq);
 149
 150         __i915_request_commit(rq);
 151         __i915_request_queue(rq, &attr);
 152
 153 unlock:
 154         mutex_unlock(&ce->timeline->mutex);
 155 out:
 156         if (!engine->i915->params.enable_hangcheck || !next_heartbeat(engine))
 157                 i915_request_put(fetch_and_zero(&engine->heartbeat.systole));
 158         intel_engine_pm_put(engine);
 159 }
 160
 161 void intel_engine_unpark_heartbeat(struct intel_engine_cs *engine)
 162 {
 163         if (!IS_ACTIVE(CONFIG_DRM_I915_HEARTBEAT_INTERVAL))
 164                 return;
 165
 166         next_heartbeat(engine);
 167 }
 168
 169 void intel_engine_park_heartbeat(struct intel_engine_cs *engine)
 170 {
 171         if (cancel_delayed_work(&engine->heartbeat.work))
 172                 i915_request_put(fetch_and_zero(&engine->heartbeat.systole));
 173 }
 174
 175 void intel_engine_init_heartbeat(struct intel_engine_cs *engine)
 176 {
 177         INIT_DELAYED_WORK(&engine->heartbeat.work, heartbeat);
 178 }
 179
 180 static int __intel_engine_pulse(struct intel_engine_cs *engine)
 181 {
 182         struct i915_sched_attr attr = { .priority = I915_PRIORITY_BARRIER };
 183         struct intel_context *ce = engine->kernel_context;
 184         struct i915_request *rq;
 185
 186         lockdep_assert_held(&ce->timeline->mutex);
 187         GEM_BUG_ON(!intel_engine_has_preemption(engine));
 188         GEM_BUG_ON(!intel_engine_pm_is_awake(engine));
 189
 190         intel_context_enter(ce);
 191         rq = __i915_request_create(ce, GFP_NOWAIT | __GFP_NOWARN);
 192         intel_context_exit(ce);
 193         if (IS_ERR(rq))
 194                 return PTR_ERR(rq);
 195
 196         __set_bit(I915_FENCE_FLAG_SENTINEL, &rq->fence.flags);
 197         idle_pulse(engine, rq);
 198
 199         __i915_request_commit(rq);
 200         __i915_request_queue(rq, &attr);
 201         GEM_BUG_ON(rq->sched.attr.priority < I915_PRIORITY_BARRIER);
 202
 203         return 0;
 204 }
 205
 206 static unsigned long set_heartbeat(struct intel_engine_cs *engine,
 207                                    unsigned long delay)
 208 {
 209         unsigned long old;
 210
 211         old = xchg(&engine->props.heartbeat_interval_ms, delay);
 212         if (delay)
 213                 intel_engine_unpark_heartbeat(engine);
 214         else
 215                 intel_engine_park_heartbeat(engine);
 216
 217         return old;
 218 }
 219
 220 int intel_engine_set_heartbeat(struct intel_engine_cs *engine,
 221                                unsigned long delay)
 222 {
 223         struct intel_context *ce = engine->kernel_context;
 224         int err = 0;
 225
 226         if (!delay && !intel_engine_has_preempt_reset(engine))
 227                 return -ENODEV;
 228
 229         intel_engine_pm_get(engine);
 230
 231         err = mutex_lock_interruptible(&ce->timeline->mutex);
 232         if (err)
 233                 goto out_rpm;
 234
 235         if (delay != engine->props.heartbeat_interval_ms) {
 236                 unsigned long saved = set_heartbeat(engine, delay);
 237
 238                 /* recheck current execution */
 239                 if (intel_engine_has_preemption(engine)) {
 240                         err = __intel_engine_pulse(engine);
 241                         if (err)
 242                                 set_heartbeat(engine, saved);
 243                 }
 244         }
 245
 246         mutex_unlock(&ce->timeline->mutex);
 247
 248 out_rpm:
 249         intel_engine_pm_put(engine);
 250         return err;
 251 }
 252
 253 int intel_engine_pulse(struct intel_engine_cs *engine)
 254 {
 255         struct intel_context *ce = engine->kernel_context;
 256         int err;
 257
 258         if (!intel_engine_has_preemption(engine))
 259                 return -ENODEV;
 260
 261         if (!intel_engine_pm_get_if_awake(engine))
 262                 return 0;
 263
 264         err = -EINTR;
 265         if (!mutex_lock_interruptible(&ce->timeline->mutex)) {
 266                 err = __intel_engine_pulse(engine);
 267                 mutex_unlock(&ce->timeline->mutex);
 268         }
 269
 270         intel_engine_pm_put(engine);
 271         return err;
 272 }
 273
 274 int intel_engine_flush_barriers(struct intel_engine_cs *engine)
 275 {
 276         struct i915_request *rq;
 277         int err = 0;
 278
 279         if (llist_empty(&engine->barrier_tasks))
 280                 return 0;
 281
 282         if (!intel_engine_pm_get_if_awake(engine))
 283                 return 0;
 284
 285         rq = i915_request_create(engine->kernel_context);
 286         if (IS_ERR(rq)) {
 287                 err = PTR_ERR(rq);
 288                 goto out_rpm;
 289         }
 290
 291         idle_pulse(engine, rq);
 292         i915_request_add(rq);
 293
 294 out_rpm:
 295         intel_engine_pm_put(engine);
 296         return err;
 297 }
 298
 299 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 300 #include "selftest_engine_heartbeat.c"
 301 #endif