drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2018 Intel Corporation
   5  */
   6
   7 #include <linux/sort.h>
   8
   9 #include "i915_drv.h"
  10
  11 #include "intel_gt_requests.h"
  12 #include "i915_selftest.h"
  13 #include "selftest_engine_heartbeat.h"
  14
  15 static int timeline_sync(struct intel_timeline *tl)
  16 {
  17         struct dma_fence *fence;
  18         long timeout;
  19
  20         fence = i915_active_fence_get(&tl->last_request);
  21         if (!fence)
  22                 return 0;
  23
  24         timeout = dma_fence_wait_timeout(fence, true, HZ / 2);
  25         dma_fence_put(fence);
  26         if (timeout < 0)
  27                 return timeout;
  28
  29         return 0;
  30 }
  31
  32 static int engine_sync_barrier(struct intel_engine_cs *engine)
  33 {
  34         return timeline_sync(engine->kernel_context->timeline);
  35 }
  36
  37 struct pulse {
  38         struct i915_active active;
  39         struct kref kref;
  40 };
  41
  42 static int pulse_active(struct i915_active *active)
  43 {
  44         kref_get(&container_of(active, struct pulse, active)->kref);
  45         return 0;
  46 }
  47
  48 static void pulse_free(struct kref *kref)
  49 {
  50         struct pulse *p = container_of(kref, typeof(*p), kref);
  51
  52         i915_active_fini(&p->active);
  53         kfree(p);
  54 }
  55
  56 static void pulse_put(struct pulse *p)
  57 {
  58         kref_put(&p->kref, pulse_free);
  59 }
  60
  61 static void pulse_retire(struct i915_active *active)
  62 {
  63         pulse_put(container_of(active, struct pulse, active));
  64 }
  65
  66 static struct pulse *pulse_create(void)
  67 {
  68         struct pulse *p;
  69
  70         p = kmalloc(sizeof(*p), GFP_KERNEL);
  71         if (!p)
  72                 return p;
  73
  74         kref_init(&p->kref);
  75         i915_active_init(&p->active, pulse_active, pulse_retire);
  76
  77         return p;
  78 }
  79
  80 static void pulse_unlock_wait(struct pulse *p)
  81 {
  82         i915_active_unlock_wait(&p->active);
  83 }
  84
  85 static int __live_idle_pulse(struct intel_engine_cs *engine,
  86                              int (*fn)(struct intel_engine_cs *cs))
  87 {
  88         struct pulse *p;
  89         int err;
  90
  91         GEM_BUG_ON(!intel_engine_pm_is_awake(engine));
  92
  93         p = pulse_create();
  94         if (!p)
  95                 return -ENOMEM;
  96
  97         err = i915_active_acquire(&p->active);
  98         if (err)
  99                 goto out;
 100
 101         err = i915_active_acquire_preallocate_barrier(&p->active, engine);
 102         if (err) {
 103                 i915_active_release(&p->active);
 104                 goto out;
 105         }
 106
 107         i915_active_acquire_barrier(&p->active);
 108         i915_active_release(&p->active);
 109
 110         GEM_BUG_ON(i915_active_is_idle(&p->active));
 111         GEM_BUG_ON(llist_empty(&engine->barrier_tasks));
 112
 113         err = fn(engine);
 114         if (err)
 115                 goto out;
 116
 117         GEM_BUG_ON(!llist_empty(&engine->barrier_tasks));
 118
 119         if (engine_sync_barrier(engine)) {
 120                 struct drm_printer m = drm_err_printer("pulse");
 121
 122                 pr_err("%s: no heartbeat pulse?\n", engine->name);
 123                 intel_engine_dump(engine, &m, "%s", engine->name);
 124
 125                 err = -ETIME;
 126                 goto out;
 127         }
 128
 129         GEM_BUG_ON(READ_ONCE(engine->serial) != engine->wakeref_serial);
 130
 131         pulse_unlock_wait(p); /* synchronize with the retirement callback */
 132
 133         if (!i915_active_is_idle(&p->active)) {
 134                 struct drm_printer m = drm_err_printer("pulse");
 135
 136                 pr_err("%s: heartbeat pulse did not flush idle tasks\n",
 137                        engine->name);
 138                 i915_active_print(&p->active, &m);
 139
 140                 err = -EINVAL;
 141                 goto out;
 142         }
 143
 144 out:
 145         pulse_put(p);
 146         return err;
 147 }
 148
 149 static int live_idle_flush(void *arg)
 150 {
 151         struct intel_gt *gt = arg;
 152         struct intel_engine_cs *engine;
 153         enum intel_engine_id id;
 154         int err = 0;
 155
 156         /* Check that we can flush the idle barriers */
 157
 158         for_each_engine(engine, gt, id) {
 159                 st_engine_heartbeat_disable(engine);
 160                 err = __live_idle_pulse(engine, intel_engine_flush_barriers);
 161                 st_engine_heartbeat_enable(engine);
 162                 if (err)
 163                         break;
 164         }
 165
 166         return err;
 167 }
 168
 169 static int live_idle_pulse(void *arg)
 170 {
 171         struct intel_gt *gt = arg;
 172         struct intel_engine_cs *engine;
 173         enum intel_engine_id id;
 174         int err = 0;
 175
 176         /* Check that heartbeat pulses flush the idle barriers */
 177
 178         for_each_engine(engine, gt, id) {
 179                 st_engine_heartbeat_disable(engine);
 180                 err = __live_idle_pulse(engine, intel_engine_pulse);
 181                 st_engine_heartbeat_enable(engine);
 182                 if (err && err != -ENODEV)
 183                         break;
 184
 185                 err = 0;
 186         }
 187
 188         return err;
 189 }
 190
 191 static int cmp_u32(const void *_a, const void *_b)
 192 {
 193         const u32 *a = _a, *b = _b;
 194
 195         return *a - *b;
 196 }
 197
 198 static int __live_heartbeat_fast(struct intel_engine_cs *engine)
 199 {
 200         struct intel_context *ce;
 201         struct i915_request *rq;
 202         ktime_t t0, t1;
 203         u32 times[5];
 204         int err;
 205         int i;
 206
 207         ce = intel_context_create(engine);
 208         if (IS_ERR(ce))
 209                 return PTR_ERR(ce);
 210
 211         intel_engine_pm_get(engine);
 212
 213         err = intel_engine_set_heartbeat(engine, 1);
 214         if (err)
 215                 goto err_pm;
 216
 217         for (i = 0; i < ARRAY_SIZE(times); i++) {
 218                 do {
 219                         /* Manufacture a tick */
 220                         intel_engine_park_heartbeat(engine);
 221                         GEM_BUG_ON(engine->heartbeat.systole);
 222                         engine->serial++; /*  pretend we are not idle! */
 223                         intel_engine_unpark_heartbeat(engine);
 224
 225                         flush_delayed_work(&engine->heartbeat.work);
 226                         if (!delayed_work_pending(&engine->heartbeat.work)) {
 227                                 pr_err("%s: heartbeat %d did not start\n",
 228                                        engine->name, i);
 229                                 err = -EINVAL;
 230                                 goto err_pm;
 231                         }
 232
 233                         rcu_read_lock();
 234                         rq = READ_ONCE(engine->heartbeat.systole);
 235                         if (rq)
 236                                 rq = i915_request_get_rcu(rq);
 237                         rcu_read_unlock();
 238                 } while (!rq);
 239
 240                 t0 = ktime_get();
 241                 while (rq == READ_ONCE(engine->heartbeat.systole))
 242                         yield(); /* work is on the local cpu! */
 243                 t1 = ktime_get();
 244
 245                 i915_request_put(rq);
 246                 times[i] = ktime_us_delta(t1, t0);
 247         }
 248
 249         sort(times, ARRAY_SIZE(times), sizeof(times[0]), cmp_u32, NULL);
 250
 251         pr_info("%s: Heartbeat delay: %uus [%u, %u]\n",
 252                 engine->name,
 253                 times[ARRAY_SIZE(times) / 2],
 254                 times[0],
 255                 times[ARRAY_SIZE(times) - 1]);
 256
 257         /* Min work delay is 2 * 2 (worst), +1 for scheduling, +1 for slack */
 258         if (times[ARRAY_SIZE(times) / 2] > jiffies_to_usecs(6)) {
 259                 pr_err("%s: Heartbeat delay was %uus, expected less than %dus\n",
 260                        engine->name,
 261                        times[ARRAY_SIZE(times) / 2],
 262                        jiffies_to_usecs(6));
 263                 err = -EINVAL;
 264         }
 265
 266         intel_engine_set_heartbeat(engine, CONFIG_DRM_I915_HEARTBEAT_INTERVAL);
 267 err_pm:
 268         intel_engine_pm_put(engine);
 269         intel_context_put(ce);
 270         return err;
 271 }
 272
 273 static int live_heartbeat_fast(void *arg)
 274 {
 275         struct intel_gt *gt = arg;
 276         struct intel_engine_cs *engine;
 277         enum intel_engine_id id;
 278         int err = 0;
 279
 280         /* Check that the heartbeat ticks at the desired rate. */
 281         if (!CONFIG_DRM_I915_HEARTBEAT_INTERVAL)
 282                 return 0;
 283
 284         for_each_engine(engine, gt, id) {
 285                 err = __live_heartbeat_fast(engine);
 286                 if (err)
 287                         break;
 288         }
 289
 290         return err;
 291 }
 292
 293 static int __live_heartbeat_off(struct intel_engine_cs *engine)
 294 {
 295         int err;
 296
 297         intel_engine_pm_get(engine);
 298
 299         engine->serial++;
 300         flush_delayed_work(&engine->heartbeat.work);
 301         if (!delayed_work_pending(&engine->heartbeat.work)) {
 302                 pr_err("%s: heartbeat not running\n",
 303                        engine->name);
 304                 err = -EINVAL;
 305                 goto err_pm;
 306         }
 307
 308         err = intel_engine_set_heartbeat(engine, 0);
 309         if (err)
 310                 goto err_pm;
 311
 312         engine->serial++;
 313         flush_delayed_work(&engine->heartbeat.work);
 314         if (delayed_work_pending(&engine->heartbeat.work)) {
 315                 pr_err("%s: heartbeat still running\n",
 316                        engine->name);
 317                 err = -EINVAL;
 318                 goto err_beat;
 319         }
 320
 321         if (READ_ONCE(engine->heartbeat.systole)) {
 322                 pr_err("%s: heartbeat still allocated\n",
 323                        engine->name);
 324                 err = -EINVAL;
 325                 goto err_beat;
 326         }
 327
 328 err_beat:
 329         intel_engine_set_heartbeat(engine, CONFIG_DRM_I915_HEARTBEAT_INTERVAL);
 330 err_pm:
 331         intel_engine_pm_put(engine);
 332         return err;
 333 }
 334
 335 static int live_heartbeat_off(void *arg)
 336 {
 337         struct intel_gt *gt = arg;
 338         struct intel_engine_cs *engine;
 339         enum intel_engine_id id;
 340         int err = 0;
 341
 342         /* Check that we can turn off heartbeat and not interrupt VIP */
 343         if (!CONFIG_DRM_I915_HEARTBEAT_INTERVAL)
 344                 return 0;
 345
 346         for_each_engine(engine, gt, id) {
 347                 if (!intel_engine_has_preemption(engine))
 348                         continue;
 349
 350                 err = __live_heartbeat_off(engine);
 351                 if (err)
 352                         break;
 353         }
 354
 355         return err;
 356 }
 357
 358 int intel_heartbeat_live_selftests(struct drm_i915_private *i915)
 359 {
 360         static const struct i915_subtest tests[] = {
 361                 SUBTEST(live_idle_flush),
 362                 SUBTEST(live_idle_pulse),
 363                 SUBTEST(live_heartbeat_fast),
 364                 SUBTEST(live_heartbeat_off),
 365         };
 366         int saved_hangcheck;
 367         int err;
 368
 369         if (intel_gt_is_wedged(&i915->gt))
 370                 return 0;
 371
 372         saved_hangcheck = i915->params.enable_hangcheck;
 373         i915->params.enable_hangcheck = INT_MAX;
 374
 375         err = intel_gt_live_subtests(tests, &i915->gt);
 376
 377         i915->params.enable_hangcheck = saved_hangcheck;
 378         return err;
 379 }
 380
 381 void st_engine_heartbeat_disable(struct intel_engine_cs *engine)
 382 {
 383         engine->props.heartbeat_interval_ms = 0;
 384
 385         intel_engine_pm_get(engine);
 386         intel_engine_park_heartbeat(engine);
 387 }
 388
 389 void st_engine_heartbeat_enable(struct intel_engine_cs *engine)
 390 {
 391         intel_engine_pm_put(engine);
 392
 393         engine->props.heartbeat_interval_ms =
 394                 engine->defaults.heartbeat_interval_ms;
 395 }