drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2019 Intel Corporation
   5  */
   6
   7 #include "i915_request.h"
   8
   9 #include "intel_context.h"
  10 #include "intel_engine_heartbeat.h"
  11 #include "intel_engine_pm.h"
  12 #include "intel_engine.h"
  13 #include "intel_gt.h"
  14 #include "intel_reset.h"
  15
  16 /*
  17  * While the engine is active, we send a periodic pulse along the engine
  18  * to check on its health and to flush any idle-barriers. If that request
  19  * is stuck, and we fail to preempt it, we declare the engine hung and
  20  * issue a reset -- in the hope that restores progress.
  21  */
  22
  23 static bool next_heartbeat(struct intel_engine_cs *engine)
  24 {
  25         long delay;
  26
  27         delay = READ_ONCE(engine->props.heartbeat_interval_ms);
  28         if (!delay)
  29                 return false;
  30
  31         delay = msecs_to_jiffies_timeout(delay);
  32         if (delay >= HZ)
  33                 delay = round_jiffies_up_relative(delay);
  34         schedule_delayed_work(&engine->heartbeat.work, delay);
  35
  36         return true;
  37 }
  38
  39 static void idle_pulse(struct intel_engine_cs *engine, struct i915_request *rq)
  40 {
  41         engine->wakeref_serial = READ_ONCE(engine->serial) + 1;
  42         i915_request_add_active_barriers(rq);
  43 }
  44
  45 static void show_heartbeat(const struct i915_request *rq,
  46                            struct intel_engine_cs *engine)
  47 {
  48         struct drm_printer p = drm_debug_printer("heartbeat");
  49
  50         intel_engine_dump(engine, &p,
  51                           "%s heartbeat {prio:%d} not ticking\n",
  52                           engine->name,
  53                           rq->sched.attr.priority);
  54 }
  55
  56 static void heartbeat(struct work_struct *wrk)
  57 {
  58         struct i915_sched_attr attr = {
  59                 .priority = I915_USER_PRIORITY(I915_PRIORITY_MIN),
  60         };
  61         struct intel_engine_cs *engine =
  62                 container_of(wrk, typeof(*engine), heartbeat.work.work);
  63         struct intel_context *ce = engine->kernel_context;
  64         struct i915_request *rq;
  65
  66         rq = engine->heartbeat.systole;
  67         if (rq && i915_request_completed(rq)) {
  68                 i915_request_put(rq);
  69                 engine->heartbeat.systole = NULL;
  70         }
  71
  72         if (!intel_engine_pm_get_if_awake(engine))
  73                 return;
  74
  75         if (intel_gt_is_wedged(engine->gt))
  76                 goto out;
  77
  78         if (engine->heartbeat.systole) {
  79                 if (engine->schedule &&
  80                     rq->sched.attr.priority < I915_PRIORITY_BARRIER) {
  81                         /*
  82                          * Gradually raise the priority of the heartbeat to
  83                          * give high priority work [which presumably desires
  84                          * low latency and no jitter] the chance to naturally
  85                          * complete before being preempted.
  86                          */
  87                         attr.priority = I915_PRIORITY_MASK;
  88                         if (rq->sched.attr.priority >= attr.priority)
  89                                 attr.priority |= I915_USER_PRIORITY(I915_PRIORITY_HEARTBEAT);
  90                         if (rq->sched.attr.priority >= attr.priority)
  91                                 attr.priority = I915_PRIORITY_BARRIER;
  92
  93                         local_bh_disable();
  94                         engine->schedule(rq, &attr);
  95                         local_bh_enable();
  96                 } else {
  97                         if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
  98                                 show_heartbeat(rq, engine);
  99
 100                         intel_gt_handle_error(engine->gt, engine->mask,
 101                                               I915_ERROR_CAPTURE,
 102                                               "stopped heartbeat on %s",
 103                                               engine->name);
 104                 }
 105                 goto out;
 106         }
 107
 108         if (engine->wakeref_serial == engine->serial)
 109                 goto out;
 110
 111         mutex_lock(&ce->timeline->mutex);
 112
 113         intel_context_enter(ce);
 114         rq = __i915_request_create(ce, GFP_NOWAIT | __GFP_NOWARN);
 115         intel_context_exit(ce);
 116         if (IS_ERR(rq))
 117                 goto unlock;
 118
 119         idle_pulse(engine, rq);
 120         if (i915_modparams.enable_hangcheck)
 121                 engine->heartbeat.systole = i915_request_get(rq);
 122
 123         __i915_request_commit(rq);
 124         __i915_request_queue(rq, &attr);
 125
 126 unlock:
 127         mutex_unlock(&ce->timeline->mutex);
 128 out:
 129         if (!next_heartbeat(engine))
 130                 i915_request_put(fetch_and_zero(&engine->heartbeat.systole));
 131         intel_engine_pm_put(engine);
 132 }
 133
 134 void intel_engine_unpark_heartbeat(struct intel_engine_cs *engine)
 135 {
 136         if (!IS_ACTIVE(CONFIG_DRM_I915_HEARTBEAT_INTERVAL))
 137                 return;
 138
 139         next_heartbeat(engine);
 140 }
 141
 142 void intel_engine_park_heartbeat(struct intel_engine_cs *engine)
 143 {
 144         if (cancel_delayed_work(&engine->heartbeat.work))
 145                 i915_request_put(fetch_and_zero(&engine->heartbeat.systole));
 146 }
 147
 148 void intel_engine_init_heartbeat(struct intel_engine_cs *engine)
 149 {
 150         INIT_DELAYED_WORK(&engine->heartbeat.work, heartbeat);
 151 }
 152
 153 int intel_engine_set_heartbeat(struct intel_engine_cs *engine,
 154                                unsigned long delay)
 155 {
 156         int err;
 157
 158         /* Send one last pulse before to cleanup persistent hogs */
 159         if (!delay && IS_ACTIVE(CONFIG_DRM_I915_PREEMPT_TIMEOUT)) {
 160                 err = intel_engine_pulse(engine);
 161                 if (err)
 162                         return err;
 163         }
 164
 165         WRITE_ONCE(engine->props.heartbeat_interval_ms, delay);
 166
 167         if (intel_engine_pm_get_if_awake(engine)) {
 168                 if (delay)
 169                         intel_engine_unpark_heartbeat(engine);
 170                 else
 171                         intel_engine_park_heartbeat(engine);
 172                 intel_engine_pm_put(engine);
 173         }
 174
 175         return 0;
 176 }
 177
 178 int intel_engine_pulse(struct intel_engine_cs *engine)
 179 {
 180         struct i915_sched_attr attr = { .priority = I915_PRIORITY_BARRIER };
 181         struct intel_context *ce = engine->kernel_context;
 182         struct i915_request *rq;
 183         int err = 0;
 184
 185         if (!intel_engine_has_preemption(engine))
 186                 return -ENODEV;
 187
 188         if (!intel_engine_pm_get_if_awake(engine))
 189                 return 0;
 190
 191         if (mutex_lock_interruptible(&ce->timeline->mutex))
 192                 goto out_rpm;
 193
 194         intel_context_enter(ce);
 195         rq = __i915_request_create(ce, GFP_NOWAIT | __GFP_NOWARN);
 196         intel_context_exit(ce);
 197         if (IS_ERR(rq)) {
 198                 err = PTR_ERR(rq);
 199                 goto out_unlock;
 200         }
 201
 202         __set_bit(I915_FENCE_FLAG_SENTINEL, &rq->fence.flags);
 203         idle_pulse(engine, rq);
 204
 205         __i915_request_commit(rq);
 206         __i915_request_queue(rq, &attr);
 207
 208 out_unlock:
 209         mutex_unlock(&ce->timeline->mutex);
 210 out_rpm:
 211         intel_engine_pm_put(engine);
 212         return err;
 213 }
 214
 215 int intel_engine_flush_barriers(struct intel_engine_cs *engine)
 216 {
 217         struct i915_request *rq;
 218         int err = 0;
 219
 220         if (llist_empty(&engine->barrier_tasks))
 221                 return 0;
 222
 223         if (!intel_engine_pm_get_if_awake(engine))
 224                 return 0;
 225
 226         rq = i915_request_create(engine->kernel_context);
 227         if (IS_ERR(rq)) {
 228                 err = PTR_ERR(rq);
 229                 goto out_rpm;
 230         }
 231
 232         idle_pulse(engine, rq);
 233         i915_request_add(rq);
 234
 235 out_rpm:
 236         intel_engine_pm_put(engine);
 237         return err;
 238 }
 239
 240 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 241 #include "selftest_engine_heartbeat.c"
 242 #endif