drivers/gpu/drm/i915/gt/intel_gt_irq.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2019 Intel Corporation
   5  */
   6
   7 #include <linux/sched/clock.h>
   8
   9 #include "i915_drv.h"
  10 #include "i915_irq.h"
  11 #include "intel_gt.h"
  12 #include "intel_gt_irq.h"
  13 #include "intel_uncore.h"
  14 #include "intel_rps.h"
  15
  16 static void guc_irq_handler(struct intel_guc *guc, u16 iir)
  17 {
  18         if (iir & GUC_INTR_GUC2HOST)
  19                 intel_guc_to_host_event_handler(guc);
  20 }
  21
  22 static void
  23 cs_irq_handler(struct intel_engine_cs *engine, u32 iir)
  24 {
  25         bool tasklet = false;
  26
  27         if (iir & GT_CONTEXT_SWITCH_INTERRUPT)
  28                 tasklet = true;
  29
  30         if (iir & GT_RENDER_USER_INTERRUPT) {
  31                 intel_engine_signal_breadcrumbs(engine);
  32                 tasklet |= intel_engine_needs_breadcrumb_tasklet(engine);
  33         }
  34
  35         if (tasklet)
  36                 tasklet_hi_schedule(&engine->execlists.tasklet);
  37 }
  38
  39 static u32
  40 gen11_gt_engine_identity(struct intel_gt *gt,
  41                          const unsigned int bank, const unsigned int bit)
  42 {
  43         void __iomem * const regs = gt->uncore->regs;
  44         u32 timeout_ts;
  45         u32 ident;
  46
  47         lockdep_assert_held(&gt->irq_lock);
  48
  49         raw_reg_write(regs, GEN11_IIR_REG_SELECTOR(bank), BIT(bit));
  50
  51         /*
  52          * NB: Specs do not specify how long to spin wait,
  53          * so we do ~100us as an educated guess.
  54          */
  55         timeout_ts = (local_clock() >> 10) + 100;
  56         do {
  57                 ident = raw_reg_read(regs, GEN11_INTR_IDENTITY_REG(bank));
  58         } while (!(ident & GEN11_INTR_DATA_VALID) &&
  59                  !time_after32(local_clock() >> 10, timeout_ts));
  60
  61         if (unlikely(!(ident & GEN11_INTR_DATA_VALID))) {
  62                 DRM_ERROR("INTR_IDENTITY_REG%u:%u 0x%08x not valid!\n",
  63                           bank, bit, ident);
  64                 return 0;
  65         }
  66
  67         raw_reg_write(regs, GEN11_INTR_IDENTITY_REG(bank),
  68                       GEN11_INTR_DATA_VALID);
  69
  70         return ident;
  71 }
  72
  73 static void
  74 gen11_other_irq_handler(struct intel_gt *gt, const u8 instance,
  75                         const u16 iir)
  76 {
  77         if (instance == OTHER_GUC_INSTANCE)
  78                 return guc_irq_handler(&gt->uc.guc, iir);
  79
  80         if (instance == OTHER_GTPM_INSTANCE)
  81                 return gen11_rps_irq_handler(&gt->rps, iir);
  82
  83         WARN_ONCE(1, "unhandled other interrupt instance=0x%x, iir=0x%x\n",
  84                   instance, iir);
  85 }
  86
  87 static void
  88 gen11_engine_irq_handler(struct intel_gt *gt, const u8 class,
  89                          const u8 instance, const u16 iir)
  90 {
  91         struct intel_engine_cs *engine;
  92
  93         if (instance <= MAX_ENGINE_INSTANCE)
  94                 engine = gt->engine_class[class][instance];
  95         else
  96                 engine = NULL;
  97
  98         if (likely(engine))
  99                 return cs_irq_handler(engine, iir);
 100
 101         WARN_ONCE(1, "unhandled engine interrupt class=0x%x, instance=0x%x\n",
 102                   class, instance);
 103 }
 104
 105 static void
 106 gen11_gt_identity_handler(struct intel_gt *gt, const u32 identity)
 107 {
 108         const u8 class = GEN11_INTR_ENGINE_CLASS(identity);
 109         const u8 instance = GEN11_INTR_ENGINE_INSTANCE(identity);
 110         const u16 intr = GEN11_INTR_ENGINE_INTR(identity);
 111
 112         if (unlikely(!intr))
 113                 return;
 114
 115         if (class <= COPY_ENGINE_CLASS)
 116                 return gen11_engine_irq_handler(gt, class, instance, intr);
 117
 118         if (class == OTHER_CLASS)
 119                 return gen11_other_irq_handler(gt, instance, intr);
 120
 121         WARN_ONCE(1, "unknown interrupt class=0x%x, instance=0x%x, intr=0x%x\n",
 122                   class, instance, intr);
 123 }
 124
 125 static void
 126 gen11_gt_bank_handler(struct intel_gt *gt, const unsigned int bank)
 127 {
 128         void __iomem * const regs = gt->uncore->regs;
 129         unsigned long intr_dw;
 130         unsigned int bit;
 131
 132         lockdep_assert_held(&gt->irq_lock);
 133
 134         intr_dw = raw_reg_read(regs, GEN11_GT_INTR_DW(bank));
 135
 136         for_each_set_bit(bit, &intr_dw, 32) {
 137                 const u32 ident = gen11_gt_engine_identity(gt, bank, bit);
 138
 139                 gen11_gt_identity_handler(gt, ident);
 140         }
 141
 142         /* Clear must be after shared has been served for engine */
 143         raw_reg_write(regs, GEN11_GT_INTR_DW(bank), intr_dw);
 144 }
 145
 146 void gen11_gt_irq_handler(struct intel_gt *gt, const u32 master_ctl)
 147 {
 148         unsigned int bank;
 149
 150         spin_lock(&gt->irq_lock);
 151
 152         for (bank = 0; bank < 2; bank++) {
 153                 if (master_ctl & GEN11_GT_DW_IRQ(bank))
 154                         gen11_gt_bank_handler(gt, bank);
 155         }
 156
 157         spin_unlock(&gt->irq_lock);
 158 }
 159
 160 bool gen11_gt_reset_one_iir(struct intel_gt *gt,
 161                             const unsigned int bank, const unsigned int bit)
 162 {
 163         void __iomem * const regs = gt->uncore->regs;
 164         u32 dw;
 165
 166         lockdep_assert_held(&gt->irq_lock);
 167
 168         dw = raw_reg_read(regs, GEN11_GT_INTR_DW(bank));
 169         if (dw & BIT(bit)) {
 170                 /*
 171                  * According to the BSpec, DW_IIR bits cannot be cleared without
 172                  * first servicing the Selector & Shared IIR registers.
 173                  */
 174                 gen11_gt_engine_identity(gt, bank, bit);
 175
 176                 /*
 177                  * We locked GT INT DW by reading it. If we want to (try
 178                  * to) recover from this successfully, we need to clear
 179                  * our bit, otherwise we are locking the register for
 180                  * everybody.
 181                  */
 182                 raw_reg_write(regs, GEN11_GT_INTR_DW(bank), BIT(bit));
 183
 184                 return true;
 185         }
 186
 187         return false;
 188 }
 189
 190 void gen11_gt_irq_reset(struct intel_gt *gt)
 191 {
 192         struct intel_uncore *uncore = gt->uncore;
 193
 194         /* Disable RCS, BCS, VCS and VECS class engines. */
 195         intel_uncore_write(uncore, GEN11_RENDER_COPY_INTR_ENABLE, 0);
 196         intel_uncore_write(uncore, GEN11_VCS_VECS_INTR_ENABLE,    0);
 197
 198         /* Restore masks irqs on RCS, BCS, VCS and VECS engines. */
 199         intel_uncore_write(uncore, GEN11_RCS0_RSVD_INTR_MASK,   ~0);
 200         intel_uncore_write(uncore, GEN11_BCS_RSVD_INTR_MASK,    ~0);
 201         intel_uncore_write(uncore, GEN11_VCS0_VCS1_INTR_MASK,   ~0);
 202         intel_uncore_write(uncore, GEN11_VCS2_VCS3_INTR_MASK,   ~0);
 203         intel_uncore_write(uncore, GEN11_VECS0_VECS1_INTR_MASK, ~0);
 204
 205         intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE, 0);
 206         intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_MASK,  ~0);
 207         intel_uncore_write(uncore, GEN11_GUC_SG_INTR_ENABLE, 0);
 208         intel_uncore_write(uncore, GEN11_GUC_SG_INTR_MASK,  ~0);
 209 }
 210
 211 void gen11_gt_irq_postinstall(struct intel_gt *gt)
 212 {
 213         const u32 irqs = GT_RENDER_USER_INTERRUPT | GT_CONTEXT_SWITCH_INTERRUPT;
 214         struct intel_uncore *uncore = gt->uncore;
 215         const u32 dmask = irqs << 16 | irqs;
 216         const u32 smask = irqs << 16;
 217
 218         BUILD_BUG_ON(irqs & 0xffff0000);
 219
 220         /* Enable RCS, BCS, VCS and VECS class interrupts. */
 221         intel_uncore_write(uncore, GEN11_RENDER_COPY_INTR_ENABLE, dmask);
 222         intel_uncore_write(uncore, GEN11_VCS_VECS_INTR_ENABLE, dmask);
 223
 224         /* Unmask irqs on RCS, BCS, VCS and VECS engines. */
 225         intel_uncore_write(uncore, GEN11_RCS0_RSVD_INTR_MASK, ~smask);
 226         intel_uncore_write(uncore, GEN11_BCS_RSVD_INTR_MASK, ~smask);
 227         intel_uncore_write(uncore, GEN11_VCS0_VCS1_INTR_MASK, ~dmask);
 228         intel_uncore_write(uncore, GEN11_VCS2_VCS3_INTR_MASK, ~dmask);
 229         intel_uncore_write(uncore, GEN11_VECS0_VECS1_INTR_MASK, ~dmask);
 230
 231         /*
 232          * RPS interrupts will get enabled/disabled on demand when RPS itself
 233          * is enabled/disabled.
 234          */
 235         gt->pm_ier = 0x0;
 236         gt->pm_imr = ~gt->pm_ier;
 237         intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE, 0);
 238         intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_MASK,  ~0);
 239
 240         /* Same thing for GuC interrupts */
 241         intel_uncore_write(uncore, GEN11_GUC_SG_INTR_ENABLE, 0);
 242         intel_uncore_write(uncore, GEN11_GUC_SG_INTR_MASK,  ~0);
 243 }
 244
 245 void gen5_gt_irq_handler(struct intel_gt *gt, u32 gt_iir)
 246 {
 247         if (gt_iir & GT_RENDER_USER_INTERRUPT)
 248                 intel_engine_signal_breadcrumbs(gt->engine_class[RENDER_CLASS][0]);
 249         if (gt_iir & ILK_BSD_USER_INTERRUPT)
 250                 intel_engine_signal_breadcrumbs(gt->engine_class[VIDEO_DECODE_CLASS][0]);
 251 }
 252
 253 static void gen7_parity_error_irq_handler(struct intel_gt *gt, u32 iir)
 254 {
 255         if (!HAS_L3_DPF(gt->i915))
 256                 return;
 257
 258         spin_lock(&gt->irq_lock);
 259         gen5_gt_disable_irq(gt, GT_PARITY_ERROR(gt->i915));
 260         spin_unlock(&gt->irq_lock);
 261
 262         if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1)
 263                 gt->i915->l3_parity.which_slice |= 1 << 1;
 264
 265         if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
 266                 gt->i915->l3_parity.which_slice |= 1 << 0;
 267
 268         schedule_work(&gt->i915->l3_parity.error_work);
 269 }
 270
 271 void gen6_gt_irq_handler(struct intel_gt *gt, u32 gt_iir)
 272 {
 273         if (gt_iir & GT_RENDER_USER_INTERRUPT)
 274                 intel_engine_signal_breadcrumbs(gt->engine_class[RENDER_CLASS][0]);
 275         if (gt_iir & GT_BSD_USER_INTERRUPT)
 276                 intel_engine_signal_breadcrumbs(gt->engine_class[VIDEO_DECODE_CLASS][0]);
 277         if (gt_iir & GT_BLT_USER_INTERRUPT)
 278                 intel_engine_signal_breadcrumbs(gt->engine_class[COPY_ENGINE_CLASS][0]);
 279
 280         if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
 281                       GT_BSD_CS_ERROR_INTERRUPT |
 282                       GT_RENDER_CS_MASTER_ERROR_INTERRUPT))
 283                 DRM_DEBUG("Command parser error, gt_iir 0x%08x\n", gt_iir);
 284
 285         if (gt_iir & GT_PARITY_ERROR(gt->i915))
 286                 gen7_parity_error_irq_handler(gt, gt_iir);
 287 }
 288
 289 void gen8_gt_irq_ack(struct intel_gt *gt, u32 master_ctl, u32 gt_iir[4])
 290 {
 291         void __iomem * const regs = gt->uncore->regs;
 292
 293         if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
 294                 gt_iir[0] = raw_reg_read(regs, GEN8_GT_IIR(0));
 295                 if (likely(gt_iir[0]))
 296                         raw_reg_write(regs, GEN8_GT_IIR(0), gt_iir[0]);
 297         }
 298
 299         if (master_ctl & (GEN8_GT_VCS0_IRQ | GEN8_GT_VCS1_IRQ)) {
 300                 gt_iir[1] = raw_reg_read(regs, GEN8_GT_IIR(1));
 301                 if (likely(gt_iir[1]))
 302                         raw_reg_write(regs, GEN8_GT_IIR(1), gt_iir[1]);
 303         }
 304
 305         if (master_ctl & (GEN8_GT_PM_IRQ | GEN8_GT_GUC_IRQ)) {
 306                 gt_iir[2] = raw_reg_read(regs, GEN8_GT_IIR(2));
 307                 if (likely(gt_iir[2]))
 308                         raw_reg_write(regs, GEN8_GT_IIR(2), gt_iir[2]);
 309         }
 310
 311         if (master_ctl & GEN8_GT_VECS_IRQ) {
 312                 gt_iir[3] = raw_reg_read(regs, GEN8_GT_IIR(3));
 313                 if (likely(gt_iir[3]))
 314                         raw_reg_write(regs, GEN8_GT_IIR(3), gt_iir[3]);
 315         }
 316 }
 317
 318 void gen8_gt_irq_handler(struct intel_gt *gt, u32 master_ctl, u32 gt_iir[4])
 319 {
 320         if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
 321                 cs_irq_handler(gt->engine_class[RENDER_CLASS][0],
 322                                gt_iir[0] >> GEN8_RCS_IRQ_SHIFT);
 323                 cs_irq_handler(gt->engine_class[COPY_ENGINE_CLASS][0],
 324                                gt_iir[0] >> GEN8_BCS_IRQ_SHIFT);
 325         }
 326
 327         if (master_ctl & (GEN8_GT_VCS0_IRQ | GEN8_GT_VCS1_IRQ)) {
 328                 cs_irq_handler(gt->engine_class[VIDEO_DECODE_CLASS][0],
 329                                gt_iir[1] >> GEN8_VCS0_IRQ_SHIFT);
 330                 cs_irq_handler(gt->engine_class[VIDEO_DECODE_CLASS][1],
 331                                gt_iir[1] >> GEN8_VCS1_IRQ_SHIFT);
 332         }
 333
 334         if (master_ctl & GEN8_GT_VECS_IRQ) {
 335                 cs_irq_handler(gt->engine_class[VIDEO_ENHANCEMENT_CLASS][0],
 336                                gt_iir[3] >> GEN8_VECS_IRQ_SHIFT);
 337         }
 338
 339         if (master_ctl & (GEN8_GT_PM_IRQ | GEN8_GT_GUC_IRQ)) {
 340                 gen6_rps_irq_handler(&gt->rps, gt_iir[2]);
 341                 guc_irq_handler(&gt->uc.guc, gt_iir[2] >> 16);
 342         }
 343 }
 344
 345 void gen8_gt_irq_reset(struct intel_gt *gt)
 346 {
 347         struct intel_uncore *uncore = gt->uncore;
 348
 349         GEN8_IRQ_RESET_NDX(uncore, GT, 0);
 350         GEN8_IRQ_RESET_NDX(uncore, GT, 1);
 351         GEN8_IRQ_RESET_NDX(uncore, GT, 2);
 352         GEN8_IRQ_RESET_NDX(uncore, GT, 3);
 353 }
 354
 355 void gen8_gt_irq_postinstall(struct intel_gt *gt)
 356 {
 357         struct intel_uncore *uncore = gt->uncore;
 358
 359         /* These are interrupts we'll toggle with the ring mask register */
 360         u32 gt_interrupts[] = {
 361                 (GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
 362                  GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
 363                  GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT |
 364                  GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT),
 365
 366                 (GT_RENDER_USER_INTERRUPT << GEN8_VCS0_IRQ_SHIFT |
 367                  GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS0_IRQ_SHIFT |
 368                  GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
 369                  GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT),
 370
 371                 0,
 372
 373                 (GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT |
 374                  GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT)
 375         };
 376
 377         gt->pm_ier = 0x0;
 378         gt->pm_imr = ~gt->pm_ier;
 379         GEN8_IRQ_INIT_NDX(uncore, GT, 0, ~gt_interrupts[0], gt_interrupts[0]);
 380         GEN8_IRQ_INIT_NDX(uncore, GT, 1, ~gt_interrupts[1], gt_interrupts[1]);
 381         /*
 382          * RPS interrupts will get enabled/disabled on demand when RPS itself
 383          * is enabled/disabled. Same wil be the case for GuC interrupts.
 384          */
 385         GEN8_IRQ_INIT_NDX(uncore, GT, 2, gt->pm_imr, gt->pm_ier);
 386         GEN8_IRQ_INIT_NDX(uncore, GT, 3, ~gt_interrupts[3], gt_interrupts[3]);
 387 }
 388
 389 static void gen5_gt_update_irq(struct intel_gt *gt,
 390                                u32 interrupt_mask,
 391                                u32 enabled_irq_mask)
 392 {
 393         lockdep_assert_held(&gt->irq_lock);
 394
 395         GEM_BUG_ON(enabled_irq_mask & ~interrupt_mask);
 396
 397         gt->gt_imr &= ~interrupt_mask;
 398         gt->gt_imr |= (~enabled_irq_mask & interrupt_mask);
 399         intel_uncore_write(gt->uncore, GTIMR, gt->gt_imr);
 400 }
 401
 402 void gen5_gt_enable_irq(struct intel_gt *gt, u32 mask)
 403 {
 404         gen5_gt_update_irq(gt, mask, mask);
 405         intel_uncore_posting_read_fw(gt->uncore, GTIMR);
 406 }
 407
 408 void gen5_gt_disable_irq(struct intel_gt *gt, u32 mask)
 409 {
 410         gen5_gt_update_irq(gt, mask, 0);
 411 }
 412
 413 void gen5_gt_irq_reset(struct intel_gt *gt)
 414 {
 415         struct intel_uncore *uncore = gt->uncore;
 416
 417         GEN3_IRQ_RESET(uncore, GT);
 418         if (INTEL_GEN(gt->i915) >= 6)
 419                 GEN3_IRQ_RESET(uncore, GEN6_PM);
 420 }
 421
 422 void gen5_gt_irq_postinstall(struct intel_gt *gt)
 423 {
 424         struct intel_uncore *uncore = gt->uncore;
 425         u32 pm_irqs = 0;
 426         u32 gt_irqs = 0;
 427
 428         gt->gt_imr = ~0;
 429         if (HAS_L3_DPF(gt->i915)) {
 430                 /* L3 parity interrupt is always unmasked. */
 431                 gt->gt_imr = ~GT_PARITY_ERROR(gt->i915);
 432                 gt_irqs |= GT_PARITY_ERROR(gt->i915);
 433         }
 434
 435         gt_irqs |= GT_RENDER_USER_INTERRUPT;
 436         if (IS_GEN(gt->i915, 5))
 437                 gt_irqs |= ILK_BSD_USER_INTERRUPT;
 438         else
 439                 gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
 440
 441         GEN3_IRQ_INIT(uncore, GT, gt->gt_imr, gt_irqs);
 442
 443         if (INTEL_GEN(gt->i915) >= 6) {
 444                 /*
 445                  * RPS interrupts will get enabled/disabled on demand when RPS
 446                  * itself is enabled/disabled.
 447                  */
 448                 if (HAS_ENGINE(gt->i915, VECS0)) {
 449                         pm_irqs |= PM_VEBOX_USER_INTERRUPT;
 450                         gt->pm_ier |= PM_VEBOX_USER_INTERRUPT;
 451                 }
 452
 453                 gt->pm_imr = 0xffffffff;
 454                 GEN3_IRQ_INIT(uncore, GEN6_PM, gt->pm_imr, pm_irqs);
 455         }
 456 }