2 * SPDX-License-Identifier: MIT
4 * Copyright © 2019 Intel Corporation
7 #include <linux/sched/clock.h>
11 #include "intel_breadcrumbs.h"
13 #include "intel_gt_irq.h"
14 #include "intel_uncore.h"
15 #include "intel_rps.h"
17 static void guc_irq_handler(struct intel_guc
*guc
, u16 iir
)
19 if (iir
& GUC_INTR_GUC2HOST
)
20 intel_guc_to_host_event_handler(guc
);
24 cs_irq_handler(struct intel_engine_cs
*engine
, u32 iir
)
28 if (unlikely(iir
& GT_CS_MASTER_ERROR_INTERRUPT
)) {
31 /* Upper 16b are the enabling mask, rsvd for internal errors */
32 eir
= ENGINE_READ(engine
, RING_EIR
) & GENMASK(15, 0);
33 ENGINE_TRACE(engine
, "CS error: %x\n", eir
);
35 /* Disable the error interrupt until after the reset */
37 ENGINE_WRITE(engine
, RING_EMR
, ~0u);
38 ENGINE_WRITE(engine
, RING_EIR
, eir
);
39 WRITE_ONCE(engine
->execlists
.error_interrupt
, eir
);
44 if (iir
& GT_WAIT_SEMAPHORE_INTERRUPT
) {
45 WRITE_ONCE(engine
->execlists
.yield
,
46 ENGINE_READ_FW(engine
, RING_EXECLIST_STATUS_HI
));
47 ENGINE_TRACE(engine
, "semaphore yield: %08x\n",
48 engine
->execlists
.yield
);
49 if (del_timer(&engine
->execlists
.timer
))
53 if (iir
& GT_CONTEXT_SWITCH_INTERRUPT
)
56 if (iir
& GT_RENDER_USER_INTERRUPT
) {
57 intel_engine_signal_breadcrumbs(engine
);
58 tasklet
|= intel_engine_needs_breadcrumb_tasklet(engine
);
62 tasklet_hi_schedule(&engine
->execlists
.tasklet
);
66 gen11_gt_engine_identity(struct intel_gt
*gt
,
67 const unsigned int bank
, const unsigned int bit
)
69 void __iomem
* const regs
= gt
->uncore
->regs
;
73 lockdep_assert_held(>
->irq_lock
);
75 raw_reg_write(regs
, GEN11_IIR_REG_SELECTOR(bank
), BIT(bit
));
78 * NB: Specs do not specify how long to spin wait,
79 * so we do ~100us as an educated guess.
81 timeout_ts
= (local_clock() >> 10) + 100;
83 ident
= raw_reg_read(regs
, GEN11_INTR_IDENTITY_REG(bank
));
84 } while (!(ident
& GEN11_INTR_DATA_VALID
) &&
85 !time_after32(local_clock() >> 10, timeout_ts
));
87 if (unlikely(!(ident
& GEN11_INTR_DATA_VALID
))) {
88 DRM_ERROR("INTR_IDENTITY_REG%u:%u 0x%08x not valid!\n",
93 raw_reg_write(regs
, GEN11_INTR_IDENTITY_REG(bank
),
94 GEN11_INTR_DATA_VALID
);
100 gen11_other_irq_handler(struct intel_gt
*gt
, const u8 instance
,
103 if (instance
== OTHER_GUC_INSTANCE
)
104 return guc_irq_handler(>
->uc
.guc
, iir
);
106 if (instance
== OTHER_GTPM_INSTANCE
)
107 return gen11_rps_irq_handler(>
->rps
, iir
);
109 WARN_ONCE(1, "unhandled other interrupt instance=0x%x, iir=0x%x\n",
114 gen11_engine_irq_handler(struct intel_gt
*gt
, const u8
class,
115 const u8 instance
, const u16 iir
)
117 struct intel_engine_cs
*engine
;
119 if (instance
<= MAX_ENGINE_INSTANCE
)
120 engine
= gt
->engine_class
[class][instance
];
125 return cs_irq_handler(engine
, iir
);
127 WARN_ONCE(1, "unhandled engine interrupt class=0x%x, instance=0x%x\n",
132 gen11_gt_identity_handler(struct intel_gt
*gt
, const u32 identity
)
134 const u8
class = GEN11_INTR_ENGINE_CLASS(identity
);
135 const u8 instance
= GEN11_INTR_ENGINE_INSTANCE(identity
);
136 const u16 intr
= GEN11_INTR_ENGINE_INTR(identity
);
141 if (class <= COPY_ENGINE_CLASS
)
142 return gen11_engine_irq_handler(gt
, class, instance
, intr
);
144 if (class == OTHER_CLASS
)
145 return gen11_other_irq_handler(gt
, instance
, intr
);
147 WARN_ONCE(1, "unknown interrupt class=0x%x, instance=0x%x, intr=0x%x\n",
148 class, instance
, intr
);
152 gen11_gt_bank_handler(struct intel_gt
*gt
, const unsigned int bank
)
154 void __iomem
* const regs
= gt
->uncore
->regs
;
155 unsigned long intr_dw
;
158 lockdep_assert_held(>
->irq_lock
);
160 intr_dw
= raw_reg_read(regs
, GEN11_GT_INTR_DW(bank
));
162 for_each_set_bit(bit
, &intr_dw
, 32) {
163 const u32 ident
= gen11_gt_engine_identity(gt
, bank
, bit
);
165 gen11_gt_identity_handler(gt
, ident
);
168 /* Clear must be after shared has been served for engine */
169 raw_reg_write(regs
, GEN11_GT_INTR_DW(bank
), intr_dw
);
172 void gen11_gt_irq_handler(struct intel_gt
*gt
, const u32 master_ctl
)
176 spin_lock(>
->irq_lock
);
178 for (bank
= 0; bank
< 2; bank
++) {
179 if (master_ctl
& GEN11_GT_DW_IRQ(bank
))
180 gen11_gt_bank_handler(gt
, bank
);
183 spin_unlock(>
->irq_lock
);
186 bool gen11_gt_reset_one_iir(struct intel_gt
*gt
,
187 const unsigned int bank
, const unsigned int bit
)
189 void __iomem
* const regs
= gt
->uncore
->regs
;
192 lockdep_assert_held(>
->irq_lock
);
194 dw
= raw_reg_read(regs
, GEN11_GT_INTR_DW(bank
));
197 * According to the BSpec, DW_IIR bits cannot be cleared without
198 * first servicing the Selector & Shared IIR registers.
200 gen11_gt_engine_identity(gt
, bank
, bit
);
203 * We locked GT INT DW by reading it. If we want to (try
204 * to) recover from this successfully, we need to clear
205 * our bit, otherwise we are locking the register for
208 raw_reg_write(regs
, GEN11_GT_INTR_DW(bank
), BIT(bit
));
216 void gen11_gt_irq_reset(struct intel_gt
*gt
)
218 struct intel_uncore
*uncore
= gt
->uncore
;
220 /* Disable RCS, BCS, VCS and VECS class engines. */
221 intel_uncore_write(uncore
, GEN11_RENDER_COPY_INTR_ENABLE
, 0);
222 intel_uncore_write(uncore
, GEN11_VCS_VECS_INTR_ENABLE
, 0);
224 /* Restore masks irqs on RCS, BCS, VCS and VECS engines. */
225 intel_uncore_write(uncore
, GEN11_RCS0_RSVD_INTR_MASK
, ~0);
226 intel_uncore_write(uncore
, GEN11_BCS_RSVD_INTR_MASK
, ~0);
227 intel_uncore_write(uncore
, GEN11_VCS0_VCS1_INTR_MASK
, ~0);
228 intel_uncore_write(uncore
, GEN11_VCS2_VCS3_INTR_MASK
, ~0);
229 intel_uncore_write(uncore
, GEN11_VECS0_VECS1_INTR_MASK
, ~0);
231 intel_uncore_write(uncore
, GEN11_GPM_WGBOXPERF_INTR_ENABLE
, 0);
232 intel_uncore_write(uncore
, GEN11_GPM_WGBOXPERF_INTR_MASK
, ~0);
233 intel_uncore_write(uncore
, GEN11_GUC_SG_INTR_ENABLE
, 0);
234 intel_uncore_write(uncore
, GEN11_GUC_SG_INTR_MASK
, ~0);
237 void gen11_gt_irq_postinstall(struct intel_gt
*gt
)
240 GT_CS_MASTER_ERROR_INTERRUPT
|
241 GT_RENDER_USER_INTERRUPT
|
242 GT_CONTEXT_SWITCH_INTERRUPT
|
243 GT_WAIT_SEMAPHORE_INTERRUPT
;
244 struct intel_uncore
*uncore
= gt
->uncore
;
245 const u32 dmask
= irqs
<< 16 | irqs
;
246 const u32 smask
= irqs
<< 16;
248 BUILD_BUG_ON(irqs
& 0xffff0000);
250 /* Enable RCS, BCS, VCS and VECS class interrupts. */
251 intel_uncore_write(uncore
, GEN11_RENDER_COPY_INTR_ENABLE
, dmask
);
252 intel_uncore_write(uncore
, GEN11_VCS_VECS_INTR_ENABLE
, dmask
);
254 /* Unmask irqs on RCS, BCS, VCS and VECS engines. */
255 intel_uncore_write(uncore
, GEN11_RCS0_RSVD_INTR_MASK
, ~smask
);
256 intel_uncore_write(uncore
, GEN11_BCS_RSVD_INTR_MASK
, ~smask
);
257 intel_uncore_write(uncore
, GEN11_VCS0_VCS1_INTR_MASK
, ~dmask
);
258 intel_uncore_write(uncore
, GEN11_VCS2_VCS3_INTR_MASK
, ~dmask
);
259 intel_uncore_write(uncore
, GEN11_VECS0_VECS1_INTR_MASK
, ~dmask
);
262 * RPS interrupts will get enabled/disabled on demand when RPS itself
263 * is enabled/disabled.
266 gt
->pm_imr
= ~gt
->pm_ier
;
267 intel_uncore_write(uncore
, GEN11_GPM_WGBOXPERF_INTR_ENABLE
, 0);
268 intel_uncore_write(uncore
, GEN11_GPM_WGBOXPERF_INTR_MASK
, ~0);
270 /* Same thing for GuC interrupts */
271 intel_uncore_write(uncore
, GEN11_GUC_SG_INTR_ENABLE
, 0);
272 intel_uncore_write(uncore
, GEN11_GUC_SG_INTR_MASK
, ~0);
275 void gen5_gt_irq_handler(struct intel_gt
*gt
, u32 gt_iir
)
277 if (gt_iir
& GT_RENDER_USER_INTERRUPT
)
278 intel_engine_signal_breadcrumbs(gt
->engine_class
[RENDER_CLASS
][0]);
279 if (gt_iir
& ILK_BSD_USER_INTERRUPT
)
280 intel_engine_signal_breadcrumbs(gt
->engine_class
[VIDEO_DECODE_CLASS
][0]);
283 static void gen7_parity_error_irq_handler(struct intel_gt
*gt
, u32 iir
)
285 if (!HAS_L3_DPF(gt
->i915
))
288 spin_lock(>
->irq_lock
);
289 gen5_gt_disable_irq(gt
, GT_PARITY_ERROR(gt
->i915
));
290 spin_unlock(>
->irq_lock
);
292 if (iir
& GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1
)
293 gt
->i915
->l3_parity
.which_slice
|= 1 << 1;
295 if (iir
& GT_RENDER_L3_PARITY_ERROR_INTERRUPT
)
296 gt
->i915
->l3_parity
.which_slice
|= 1 << 0;
298 schedule_work(>
->i915
->l3_parity
.error_work
);
301 void gen6_gt_irq_handler(struct intel_gt
*gt
, u32 gt_iir
)
303 if (gt_iir
& GT_RENDER_USER_INTERRUPT
)
304 intel_engine_signal_breadcrumbs(gt
->engine_class
[RENDER_CLASS
][0]);
305 if (gt_iir
& GT_BSD_USER_INTERRUPT
)
306 intel_engine_signal_breadcrumbs(gt
->engine_class
[VIDEO_DECODE_CLASS
][0]);
307 if (gt_iir
& GT_BLT_USER_INTERRUPT
)
308 intel_engine_signal_breadcrumbs(gt
->engine_class
[COPY_ENGINE_CLASS
][0]);
310 if (gt_iir
& (GT_BLT_CS_ERROR_INTERRUPT
|
311 GT_BSD_CS_ERROR_INTERRUPT
|
312 GT_CS_MASTER_ERROR_INTERRUPT
))
313 DRM_DEBUG("Command parser error, gt_iir 0x%08x\n", gt_iir
);
315 if (gt_iir
& GT_PARITY_ERROR(gt
->i915
))
316 gen7_parity_error_irq_handler(gt
, gt_iir
);
319 void gen8_gt_irq_handler(struct intel_gt
*gt
, u32 master_ctl
)
321 void __iomem
* const regs
= gt
->uncore
->regs
;
324 if (master_ctl
& (GEN8_GT_RCS_IRQ
| GEN8_GT_BCS_IRQ
)) {
325 iir
= raw_reg_read(regs
, GEN8_GT_IIR(0));
327 cs_irq_handler(gt
->engine_class
[RENDER_CLASS
][0],
328 iir
>> GEN8_RCS_IRQ_SHIFT
);
329 cs_irq_handler(gt
->engine_class
[COPY_ENGINE_CLASS
][0],
330 iir
>> GEN8_BCS_IRQ_SHIFT
);
331 raw_reg_write(regs
, GEN8_GT_IIR(0), iir
);
335 if (master_ctl
& (GEN8_GT_VCS0_IRQ
| GEN8_GT_VCS1_IRQ
)) {
336 iir
= raw_reg_read(regs
, GEN8_GT_IIR(1));
338 cs_irq_handler(gt
->engine_class
[VIDEO_DECODE_CLASS
][0],
339 iir
>> GEN8_VCS0_IRQ_SHIFT
);
340 cs_irq_handler(gt
->engine_class
[VIDEO_DECODE_CLASS
][1],
341 iir
>> GEN8_VCS1_IRQ_SHIFT
);
342 raw_reg_write(regs
, GEN8_GT_IIR(1), iir
);
346 if (master_ctl
& GEN8_GT_VECS_IRQ
) {
347 iir
= raw_reg_read(regs
, GEN8_GT_IIR(3));
349 cs_irq_handler(gt
->engine_class
[VIDEO_ENHANCEMENT_CLASS
][0],
350 iir
>> GEN8_VECS_IRQ_SHIFT
);
351 raw_reg_write(regs
, GEN8_GT_IIR(3), iir
);
355 if (master_ctl
& (GEN8_GT_PM_IRQ
| GEN8_GT_GUC_IRQ
)) {
356 iir
= raw_reg_read(regs
, GEN8_GT_IIR(2));
358 gen6_rps_irq_handler(>
->rps
, iir
);
359 guc_irq_handler(>
->uc
.guc
, iir
>> 16);
360 raw_reg_write(regs
, GEN8_GT_IIR(2), iir
);
365 void gen8_gt_irq_reset(struct intel_gt
*gt
)
367 struct intel_uncore
*uncore
= gt
->uncore
;
369 GEN8_IRQ_RESET_NDX(uncore
, GT
, 0);
370 GEN8_IRQ_RESET_NDX(uncore
, GT
, 1);
371 GEN8_IRQ_RESET_NDX(uncore
, GT
, 2);
372 GEN8_IRQ_RESET_NDX(uncore
, GT
, 3);
375 void gen8_gt_irq_postinstall(struct intel_gt
*gt
)
377 /* These are interrupts we'll toggle with the ring mask register */
379 GT_CS_MASTER_ERROR_INTERRUPT
|
380 GT_RENDER_USER_INTERRUPT
|
381 GT_CONTEXT_SWITCH_INTERRUPT
|
382 GT_WAIT_SEMAPHORE_INTERRUPT
;
383 const u32 gt_interrupts
[] = {
384 irqs
<< GEN8_RCS_IRQ_SHIFT
| irqs
<< GEN8_BCS_IRQ_SHIFT
,
385 irqs
<< GEN8_VCS0_IRQ_SHIFT
| irqs
<< GEN8_VCS1_IRQ_SHIFT
,
387 irqs
<< GEN8_VECS_IRQ_SHIFT
,
389 struct intel_uncore
*uncore
= gt
->uncore
;
392 gt
->pm_imr
= ~gt
->pm_ier
;
393 GEN8_IRQ_INIT_NDX(uncore
, GT
, 0, ~gt_interrupts
[0], gt_interrupts
[0]);
394 GEN8_IRQ_INIT_NDX(uncore
, GT
, 1, ~gt_interrupts
[1], gt_interrupts
[1]);
396 * RPS interrupts will get enabled/disabled on demand when RPS itself
397 * is enabled/disabled. Same wil be the case for GuC interrupts.
399 GEN8_IRQ_INIT_NDX(uncore
, GT
, 2, gt
->pm_imr
, gt
->pm_ier
);
400 GEN8_IRQ_INIT_NDX(uncore
, GT
, 3, ~gt_interrupts
[3], gt_interrupts
[3]);
403 static void gen5_gt_update_irq(struct intel_gt
*gt
,
405 u32 enabled_irq_mask
)
407 lockdep_assert_held(>
->irq_lock
);
409 GEM_BUG_ON(enabled_irq_mask
& ~interrupt_mask
);
411 gt
->gt_imr
&= ~interrupt_mask
;
412 gt
->gt_imr
|= (~enabled_irq_mask
& interrupt_mask
);
413 intel_uncore_write(gt
->uncore
, GTIMR
, gt
->gt_imr
);
416 void gen5_gt_enable_irq(struct intel_gt
*gt
, u32 mask
)
418 gen5_gt_update_irq(gt
, mask
, mask
);
419 intel_uncore_posting_read_fw(gt
->uncore
, GTIMR
);
422 void gen5_gt_disable_irq(struct intel_gt
*gt
, u32 mask
)
424 gen5_gt_update_irq(gt
, mask
, 0);
427 void gen5_gt_irq_reset(struct intel_gt
*gt
)
429 struct intel_uncore
*uncore
= gt
->uncore
;
431 GEN3_IRQ_RESET(uncore
, GT
);
432 if (INTEL_GEN(gt
->i915
) >= 6)
433 GEN3_IRQ_RESET(uncore
, GEN6_PM
);
436 void gen5_gt_irq_postinstall(struct intel_gt
*gt
)
438 struct intel_uncore
*uncore
= gt
->uncore
;
443 if (HAS_L3_DPF(gt
->i915
)) {
444 /* L3 parity interrupt is always unmasked. */
445 gt
->gt_imr
= ~GT_PARITY_ERROR(gt
->i915
);
446 gt_irqs
|= GT_PARITY_ERROR(gt
->i915
);
449 gt_irqs
|= GT_RENDER_USER_INTERRUPT
;
450 if (IS_GEN(gt
->i915
, 5))
451 gt_irqs
|= ILK_BSD_USER_INTERRUPT
;
453 gt_irqs
|= GT_BLT_USER_INTERRUPT
| GT_BSD_USER_INTERRUPT
;
455 GEN3_IRQ_INIT(uncore
, GT
, gt
->gt_imr
, gt_irqs
);
457 if (INTEL_GEN(gt
->i915
) >= 6) {
459 * RPS interrupts will get enabled/disabled on demand when RPS
460 * itself is enabled/disabled.
462 if (HAS_ENGINE(gt
, VECS0
)) {
463 pm_irqs
|= PM_VEBOX_USER_INTERRUPT
;
464 gt
->pm_ier
|= PM_VEBOX_USER_INTERRUPT
;
467 gt
->pm_imr
= 0xffffffff;
468 GEN3_IRQ_INIT(uncore
, GEN6_PM
, gt
->pm_imr
, pm_irqs
);