?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'io_uring-5.11-2021-01-16' of git://git.kernel.dk/linux-block
[linux/fpc-iii.git] / drivers / gpu / drm / i915 / i915_irq.c
blob6cdb052e385050c1e74f45c0a0fd066649d6a3e2
1 /* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
2 */
3 /*
4 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
5 * All Rights Reserved.
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the
9 * "Software"), to deal in the Software without restriction, including
10 * without limitation the rights to use, copy, modify, merge, publish,
11 * distribute, sub license, and/or sell copies of the Software, and to
12 * permit persons to whom the Software is furnished to do so, subject to
13 * the following conditions:
14 *
15 * The above copyright notice and this permission notice (including the
16 * next paragraph) shall be included in all copies or substantial portions
17 * of the Software.
18 *
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
20 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
21 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
22 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
23 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
24 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
25 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 *
27 */
28
29 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
30
31 #include <linux/circ_buf.h>
32 #include <linux/slab.h>
33 #include <linux/sysrq.h>
34
35 #include <drm/drm_drv.h>
36 #include <drm/drm_irq.h>
37
38 #include "display/intel_display_types.h"
39 #include "display/intel_fifo_underrun.h"
40 #include "display/intel_hotplug.h"
41 #include "display/intel_lpe_audio.h"
42 #include "display/intel_psr.h"
43
44 #include "gt/intel_breadcrumbs.h"
45 #include "gt/intel_gt.h"
46 #include "gt/intel_gt_irq.h"
47 #include "gt/intel_gt_pm_irq.h"
48 #include "gt/intel_rps.h"
49
50 #include "i915_drv.h"
51 #include "i915_irq.h"
52 #include "i915_trace.h"
53 #include "intel_pm.h"
54
55 /**
56 * DOC: interrupt handling
57 *
58 * These functions provide the basic support for enabling and disabling the
59 * interrupt handling support. There's a lot more functionality in i915_irq.c
60 * and related files, but that will be described in separate chapters.
61 */
62
63 /*
64 * Interrupt statistic for PMU. Increments the counter only if the
65 * interrupt originated from the the GPU so interrupts from a device which
66 * shares the interrupt line are not accounted.
67 */
68 static inline void pmu_irq_stats(struct drm_i915_private *i915,
69 irqreturn_t res)
70 {
71 if (unlikely(res != IRQ_HANDLED))
72 return;
73
74 /*
75 * A clever compiler translates that into INC. A not so clever one
76 * should at least prevent store tearing.
77 */
78 WRITE_ONCE(i915->pmu.irq_count, i915->pmu.irq_count + 1);
79 }
80
81 typedef bool (*long_pulse_detect_func)(enum hpd_pin pin, u32 val);
82 typedef u32 (*hotplug_enables_func)(struct drm_i915_private *i915,
83 enum hpd_pin pin);
84
85 static const u32 hpd_ilk[HPD_NUM_PINS] = {
86 [HPD_PORT_A] = DE_DP_A_HOTPLUG,
87 };
88
89 static const u32 hpd_ivb[HPD_NUM_PINS] = {
90 [HPD_PORT_A] = DE_DP_A_HOTPLUG_IVB,
91 };
92
93 static const u32 hpd_bdw[HPD_NUM_PINS] = {
94 [HPD_PORT_A] = GEN8_DE_PORT_HOTPLUG(HPD_PORT_A),
95 };
96
97 static const u32 hpd_ibx[HPD_NUM_PINS] = {
98 [HPD_CRT] = SDE_CRT_HOTPLUG,
99 [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG,
100 [HPD_PORT_B] = SDE_PORTB_HOTPLUG,
101 [HPD_PORT_C] = SDE_PORTC_HOTPLUG,
102 [HPD_PORT_D] = SDE_PORTD_HOTPLUG,
103 };
104
105 static const u32 hpd_cpt[HPD_NUM_PINS] = {
106 [HPD_CRT] = SDE_CRT_HOTPLUG_CPT,
107 [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT,
108 [HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
109 [HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
110 [HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT,
111 };
112
113 static const u32 hpd_spt[HPD_NUM_PINS] = {
114 [HPD_PORT_A] = SDE_PORTA_HOTPLUG_SPT,
115 [HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
116 [HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
117 [HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT,
118 [HPD_PORT_E] = SDE_PORTE_HOTPLUG_SPT,
119 };
120
121 static const u32 hpd_mask_i915[HPD_NUM_PINS] = {
122 [HPD_CRT] = CRT_HOTPLUG_INT_EN,
123 [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN,
124 [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN,
125 [HPD_PORT_B] = PORTB_HOTPLUG_INT_EN,
126 [HPD_PORT_C] = PORTC_HOTPLUG_INT_EN,
127 [HPD_PORT_D] = PORTD_HOTPLUG_INT_EN,
128 };
129
130 static const u32 hpd_status_g4x[HPD_NUM_PINS] = {
131 [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
132 [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X,
133 [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X,
134 [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
135 [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
136 [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS,
137 };
138
139 static const u32 hpd_status_i915[HPD_NUM_PINS] = {
140 [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
141 [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915,
142 [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915,
143 [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
144 [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
145 [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS,
146 };
147
148 static const u32 hpd_bxt[HPD_NUM_PINS] = {
149 [HPD_PORT_A] = GEN8_DE_PORT_HOTPLUG(HPD_PORT_A),
150 [HPD_PORT_B] = GEN8_DE_PORT_HOTPLUG(HPD_PORT_B),
151 [HPD_PORT_C] = GEN8_DE_PORT_HOTPLUG(HPD_PORT_C),
152 };
153
154 static const u32 hpd_gen11[HPD_NUM_PINS] = {
155 [HPD_PORT_TC1] = GEN11_TC_HOTPLUG(HPD_PORT_TC1) | GEN11_TBT_HOTPLUG(HPD_PORT_TC1),
156 [HPD_PORT_TC2] = GEN11_TC_HOTPLUG(HPD_PORT_TC2) | GEN11_TBT_HOTPLUG(HPD_PORT_TC2),
157 [HPD_PORT_TC3] = GEN11_TC_HOTPLUG(HPD_PORT_TC3) | GEN11_TBT_HOTPLUG(HPD_PORT_TC3),
158 [HPD_PORT_TC4] = GEN11_TC_HOTPLUG(HPD_PORT_TC4) | GEN11_TBT_HOTPLUG(HPD_PORT_TC4),
159 [HPD_PORT_TC5] = GEN11_TC_HOTPLUG(HPD_PORT_TC5) | GEN11_TBT_HOTPLUG(HPD_PORT_TC5),
160 [HPD_PORT_TC6] = GEN11_TC_HOTPLUG(HPD_PORT_TC6) | GEN11_TBT_HOTPLUG(HPD_PORT_TC6),
161 };
162
163 static const u32 hpd_icp[HPD_NUM_PINS] = {
164 [HPD_PORT_A] = SDE_DDI_HOTPLUG_ICP(HPD_PORT_A),
165 [HPD_PORT_B] = SDE_DDI_HOTPLUG_ICP(HPD_PORT_B),
166 [HPD_PORT_C] = SDE_DDI_HOTPLUG_ICP(HPD_PORT_C),
167 [HPD_PORT_TC1] = SDE_TC_HOTPLUG_ICP(HPD_PORT_TC1),
168 [HPD_PORT_TC2] = SDE_TC_HOTPLUG_ICP(HPD_PORT_TC2),
169 [HPD_PORT_TC3] = SDE_TC_HOTPLUG_ICP(HPD_PORT_TC3),
170 [HPD_PORT_TC4] = SDE_TC_HOTPLUG_ICP(HPD_PORT_TC4),
171 [HPD_PORT_TC5] = SDE_TC_HOTPLUG_ICP(HPD_PORT_TC5),
172 [HPD_PORT_TC6] = SDE_TC_HOTPLUG_ICP(HPD_PORT_TC6),
173 };
174
175 static const u32 hpd_sde_dg1[HPD_NUM_PINS] = {
176 [HPD_PORT_A] = SDE_DDI_HOTPLUG_ICP(HPD_PORT_A),
177 [HPD_PORT_B] = SDE_DDI_HOTPLUG_ICP(HPD_PORT_B),
178 [HPD_PORT_C] = SDE_DDI_HOTPLUG_ICP(HPD_PORT_C),
179 [HPD_PORT_D] = SDE_DDI_HOTPLUG_ICP(HPD_PORT_D),
180 };
181
182 static void intel_hpd_init_pins(struct drm_i915_private *dev_priv)
183 {
184 struct i915_hotplug *hpd = &dev_priv->hotplug;
185
186 if (HAS_GMCH(dev_priv)) {
187 if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
188 IS_CHERRYVIEW(dev_priv))
189 hpd->hpd = hpd_status_g4x;
190 else
191 hpd->hpd = hpd_status_i915;
192 return;
193 }
194
195 if (INTEL_GEN(dev_priv) >= 11)
196 hpd->hpd = hpd_gen11;
197 else if (IS_GEN9_LP(dev_priv))
198 hpd->hpd = hpd_bxt;
199 else if (INTEL_GEN(dev_priv) >= 8)
200 hpd->hpd = hpd_bdw;
201 else if (INTEL_GEN(dev_priv) >= 7)
202 hpd->hpd = hpd_ivb;
203 else
204 hpd->hpd = hpd_ilk;
205
206 if ((INTEL_PCH_TYPE(dev_priv) < PCH_DG1) &&
207 (!HAS_PCH_SPLIT(dev_priv) || HAS_PCH_NOP(dev_priv)))
208 return;
209
210 if (HAS_PCH_DG1(dev_priv))
211 hpd->pch_hpd = hpd_sde_dg1;
212 else if (HAS_PCH_TGP(dev_priv) || HAS_PCH_JSP(dev_priv) ||
213 HAS_PCH_ICP(dev_priv) || HAS_PCH_MCC(dev_priv))
214 hpd->pch_hpd = hpd_icp;
215 else if (HAS_PCH_CNP(dev_priv) || HAS_PCH_SPT(dev_priv))
216 hpd->pch_hpd = hpd_spt;
217 else if (HAS_PCH_LPT(dev_priv) || HAS_PCH_CPT(dev_priv))
218 hpd->pch_hpd = hpd_cpt;
219 else if (HAS_PCH_IBX(dev_priv))
220 hpd->pch_hpd = hpd_ibx;
221 else
222 MISSING_CASE(INTEL_PCH_TYPE(dev_priv));
223 }
224
225 static void
226 intel_handle_vblank(struct drm_i915_private *dev_priv, enum pipe pipe)
227 {
228 struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
229
230 drm_crtc_handle_vblank(&crtc->base);
231 }
232
233 void gen3_irq_reset(struct intel_uncore *uncore, i915_reg_t imr,
234 i915_reg_t iir, i915_reg_t ier)
235 {
236 intel_uncore_write(uncore, imr, 0xffffffff);
237 intel_uncore_posting_read(uncore, imr);
238
239 intel_uncore_write(uncore, ier, 0);
240
241 /* IIR can theoretically queue up two events. Be paranoid. */
242 intel_uncore_write(uncore, iir, 0xffffffff);
243 intel_uncore_posting_read(uncore, iir);
244 intel_uncore_write(uncore, iir, 0xffffffff);
245 intel_uncore_posting_read(uncore, iir);
246 }
247
248 void gen2_irq_reset(struct intel_uncore *uncore)
249 {
250 intel_uncore_write16(uncore, GEN2_IMR, 0xffff);
251 intel_uncore_posting_read16(uncore, GEN2_IMR);
252
253 intel_uncore_write16(uncore, GEN2_IER, 0);
254
255 /* IIR can theoretically queue up two events. Be paranoid. */
256 intel_uncore_write16(uncore, GEN2_IIR, 0xffff);
257 intel_uncore_posting_read16(uncore, GEN2_IIR);
258 intel_uncore_write16(uncore, GEN2_IIR, 0xffff);
259 intel_uncore_posting_read16(uncore, GEN2_IIR);
260 }
261
262 /*
263 * We should clear IMR at preinstall/uninstall, and just check at postinstall.
264 */
265 static void gen3_assert_iir_is_zero(struct intel_uncore *uncore, i915_reg_t reg)
266 {
267 u32 val = intel_uncore_read(uncore, reg);
268
269 if (val == 0)
270 return;
271
272 drm_WARN(&uncore->i915->drm, 1,
273 "Interrupt register 0x%x is not zero: 0x%08x\n",
274 i915_mmio_reg_offset(reg), val);
275 intel_uncore_write(uncore, reg, 0xffffffff);
276 intel_uncore_posting_read(uncore, reg);
277 intel_uncore_write(uncore, reg, 0xffffffff);
278 intel_uncore_posting_read(uncore, reg);
279 }
280
281 static void gen2_assert_iir_is_zero(struct intel_uncore *uncore)
282 {
283 u16 val = intel_uncore_read16(uncore, GEN2_IIR);
284
285 if (val == 0)
286 return;
287
288 drm_WARN(&uncore->i915->drm, 1,
289 "Interrupt register 0x%x is not zero: 0x%08x\n",
290 i915_mmio_reg_offset(GEN2_IIR), val);
291 intel_uncore_write16(uncore, GEN2_IIR, 0xffff);
292 intel_uncore_posting_read16(uncore, GEN2_IIR);
293 intel_uncore_write16(uncore, GEN2_IIR, 0xffff);
294 intel_uncore_posting_read16(uncore, GEN2_IIR);
295 }
296
297 void gen3_irq_init(struct intel_uncore *uncore,
298 i915_reg_t imr, u32 imr_val,
299 i915_reg_t ier, u32 ier_val,
300 i915_reg_t iir)
301 {
302 gen3_assert_iir_is_zero(uncore, iir);
303
304 intel_uncore_write(uncore, ier, ier_val);
305 intel_uncore_write(uncore, imr, imr_val);
306 intel_uncore_posting_read(uncore, imr);
307 }
308
309 void gen2_irq_init(struct intel_uncore *uncore,
310 u32 imr_val, u32 ier_val)
311 {
312 gen2_assert_iir_is_zero(uncore);
313
314 intel_uncore_write16(uncore, GEN2_IER, ier_val);
315 intel_uncore_write16(uncore, GEN2_IMR, imr_val);
316 intel_uncore_posting_read16(uncore, GEN2_IMR);
317 }
318
319 /* For display hotplug interrupt */
320 static inline void
321 i915_hotplug_interrupt_update_locked(struct drm_i915_private *dev_priv,
322 u32 mask,
323 u32 bits)
324 {
325 u32 val;
326
327 lockdep_assert_held(&dev_priv->irq_lock);
328 drm_WARN_ON(&dev_priv->drm, bits & ~mask);
329
330 val = I915_READ(PORT_HOTPLUG_EN);
331 val &= ~mask;
332 val |= bits;
333 I915_WRITE(PORT_HOTPLUG_EN, val);
334 }
335
336 /**
337 * i915_hotplug_interrupt_update - update hotplug interrupt enable
338 * @dev_priv: driver private
339 * @mask: bits to update
340 * @bits: bits to enable
341 * NOTE: the HPD enable bits are modified both inside and outside
342 * of an interrupt context. To avoid that read-modify-write cycles
343 * interfer, these bits are protected by a spinlock. Since this
344 * function is usually not called from a context where the lock is
345 * held already, this function acquires the lock itself. A non-locking
346 * version is also available.
347 */
348 void i915_hotplug_interrupt_update(struct drm_i915_private *dev_priv,
349 u32 mask,
350 u32 bits)
351 {
352 spin_lock_irq(&dev_priv->irq_lock);
353 i915_hotplug_interrupt_update_locked(dev_priv, mask, bits);
354 spin_unlock_irq(&dev_priv->irq_lock);
355 }
356
357 /**
358 * ilk_update_display_irq - update DEIMR
359 * @dev_priv: driver private
360 * @interrupt_mask: mask of interrupt bits to update
361 * @enabled_irq_mask: mask of interrupt bits to enable
362 */
363 void ilk_update_display_irq(struct drm_i915_private *dev_priv,
364 u32 interrupt_mask,
365 u32 enabled_irq_mask)
366 {
367 u32 new_val;
368
369 lockdep_assert_held(&dev_priv->irq_lock);
370 drm_WARN_ON(&dev_priv->drm, enabled_irq_mask & ~interrupt_mask);
371
372 new_val = dev_priv->irq_mask;
373 new_val &= ~interrupt_mask;
374 new_val |= (~enabled_irq_mask & interrupt_mask);
375
376 if (new_val != dev_priv->irq_mask &&
377 !drm_WARN_ON(&dev_priv->drm, !intel_irqs_enabled(dev_priv))) {
378 dev_priv->irq_mask = new_val;
379 I915_WRITE(DEIMR, dev_priv->irq_mask);
380 POSTING_READ(DEIMR);
381 }
382 }
383
384 /**
385 * bdw_update_port_irq - update DE port interrupt
386 * @dev_priv: driver private
387 * @interrupt_mask: mask of interrupt bits to update
388 * @enabled_irq_mask: mask of interrupt bits to enable
389 */
390 static void bdw_update_port_irq(struct drm_i915_private *dev_priv,
391 u32 interrupt_mask,
392 u32 enabled_irq_mask)
393 {
394 u32 new_val;
395 u32 old_val;
396
397 lockdep_assert_held(&dev_priv->irq_lock);
398
399 drm_WARN_ON(&dev_priv->drm, enabled_irq_mask & ~interrupt_mask);
400
401 if (drm_WARN_ON(&dev_priv->drm, !intel_irqs_enabled(dev_priv)))
402 return;
403
404 old_val = I915_READ(GEN8_DE_PORT_IMR);
405
406 new_val = old_val;
407 new_val &= ~interrupt_mask;
408 new_val |= (~enabled_irq_mask & interrupt_mask);
409
410 if (new_val != old_val) {
411 I915_WRITE(GEN8_DE_PORT_IMR, new_val);
412 POSTING_READ(GEN8_DE_PORT_IMR);
413 }
414 }
415
416 /**
417 * bdw_update_pipe_irq - update DE pipe interrupt
418 * @dev_priv: driver private
419 * @pipe: pipe whose interrupt to update
420 * @interrupt_mask: mask of interrupt bits to update
421 * @enabled_irq_mask: mask of interrupt bits to enable
422 */
423 void bdw_update_pipe_irq(struct drm_i915_private *dev_priv,
424 enum pipe pipe,
425 u32 interrupt_mask,
426 u32 enabled_irq_mask)
427 {
428 u32 new_val;
429
430 lockdep_assert_held(&dev_priv->irq_lock);
431
432 drm_WARN_ON(&dev_priv->drm, enabled_irq_mask & ~interrupt_mask);
433
434 if (drm_WARN_ON(&dev_priv->drm, !intel_irqs_enabled(dev_priv)))
435 return;
436
437 new_val = dev_priv->de_irq_mask[pipe];
438 new_val &= ~interrupt_mask;
439 new_val |= (~enabled_irq_mask & interrupt_mask);
440
441 if (new_val != dev_priv->de_irq_mask[pipe]) {
442 dev_priv->de_irq_mask[pipe] = new_val;
443 I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
444 POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
445 }
446 }
447
448 /**
449 * ibx_display_interrupt_update - update SDEIMR
450 * @dev_priv: driver private
451 * @interrupt_mask: mask of interrupt bits to update
452 * @enabled_irq_mask: mask of interrupt bits to enable
453 */
454 void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
455 u32 interrupt_mask,
456 u32 enabled_irq_mask)
457 {
458 u32 sdeimr = I915_READ(SDEIMR);
459 sdeimr &= ~interrupt_mask;
460 sdeimr |= (~enabled_irq_mask & interrupt_mask);
461
462 drm_WARN_ON(&dev_priv->drm, enabled_irq_mask & ~interrupt_mask);
463
464 lockdep_assert_held(&dev_priv->irq_lock);
465
466 if (drm_WARN_ON(&dev_priv->drm, !intel_irqs_enabled(dev_priv)))
467 return;
468
469 I915_WRITE(SDEIMR, sdeimr);
470 POSTING_READ(SDEIMR);
471 }
472
473 u32 i915_pipestat_enable_mask(struct drm_i915_private *dev_priv,
474 enum pipe pipe)
475 {
476 u32 status_mask = dev_priv->pipestat_irq_mask[pipe];
477 u32 enable_mask = status_mask << 16;
478
479 lockdep_assert_held(&dev_priv->irq_lock);
480
481 if (INTEL_GEN(dev_priv) < 5)
482 goto out;
483
484 /*
485 * On pipe A we don't support the PSR interrupt yet,
486 * on pipe B and C the same bit MBZ.
487 */
488 if (drm_WARN_ON_ONCE(&dev_priv->drm,
489 status_mask & PIPE_A_PSR_STATUS_VLV))
490 return 0;
491 /*
492 * On pipe B and C we don't support the PSR interrupt yet, on pipe
493 * A the same bit is for perf counters which we don't use either.
494 */
495 if (drm_WARN_ON_ONCE(&dev_priv->drm,
496 status_mask & PIPE_B_PSR_STATUS_VLV))
497 return 0;
498
499 enable_mask &= ~(PIPE_FIFO_UNDERRUN_STATUS |
500 SPRITE0_FLIP_DONE_INT_EN_VLV |
501 SPRITE1_FLIP_DONE_INT_EN_VLV);
502 if (status_mask & SPRITE0_FLIP_DONE_INT_STATUS_VLV)
503 enable_mask |= SPRITE0_FLIP_DONE_INT_EN_VLV;
504 if (status_mask & SPRITE1_FLIP_DONE_INT_STATUS_VLV)
505 enable_mask |= SPRITE1_FLIP_DONE_INT_EN_VLV;
506
507 out:
508 drm_WARN_ONCE(&dev_priv->drm,
509 enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
510 status_mask & ~PIPESTAT_INT_STATUS_MASK,
511 "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
512 pipe_name(pipe), enable_mask, status_mask);
513
514 return enable_mask;
515 }
516
517 void i915_enable_pipestat(struct drm_i915_private *dev_priv,
518 enum pipe pipe, u32 status_mask)
519 {
520 i915_reg_t reg = PIPESTAT(pipe);
521 u32 enable_mask;
522
523 drm_WARN_ONCE(&dev_priv->drm, status_mask & ~PIPESTAT_INT_STATUS_MASK,
524 "pipe %c: status_mask=0x%x\n",
525 pipe_name(pipe), status_mask);
526
527 lockdep_assert_held(&dev_priv->irq_lock);
528 drm_WARN_ON(&dev_priv->drm, !intel_irqs_enabled(dev_priv));
529
530 if ((dev_priv->pipestat_irq_mask[pipe] & status_mask) == status_mask)
531 return;
532
533 dev_priv->pipestat_irq_mask[pipe] |= status_mask;
534 enable_mask = i915_pipestat_enable_mask(dev_priv, pipe);
535
536 I915_WRITE(reg, enable_mask | status_mask);
537 POSTING_READ(reg);
538 }
539
540 void i915_disable_pipestat(struct drm_i915_private *dev_priv,
541 enum pipe pipe, u32 status_mask)
542 {
543 i915_reg_t reg = PIPESTAT(pipe);
544 u32 enable_mask;
545
546 drm_WARN_ONCE(&dev_priv->drm, status_mask & ~PIPESTAT_INT_STATUS_MASK,
547 "pipe %c: status_mask=0x%x\n",
548 pipe_name(pipe), status_mask);
549
550 lockdep_assert_held(&dev_priv->irq_lock);
551 drm_WARN_ON(&dev_priv->drm, !intel_irqs_enabled(dev_priv));
552
553 if ((dev_priv->pipestat_irq_mask[pipe] & status_mask) == 0)
554 return;
555
556 dev_priv->pipestat_irq_mask[pipe] &= ~status_mask;
557 enable_mask = i915_pipestat_enable_mask(dev_priv, pipe);
558
559 I915_WRITE(reg, enable_mask | status_mask);
560 POSTING_READ(reg);
561 }
562
563 static bool i915_has_asle(struct drm_i915_private *dev_priv)
564 {
565 if (!dev_priv->opregion.asle)
566 return false;
567
568 return IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv);
569 }
570
571 /**
572 * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
573 * @dev_priv: i915 device private
574 */
575 static void i915_enable_asle_pipestat(struct drm_i915_private *dev_priv)
576 {
577 if (!i915_has_asle(dev_priv))
578 return;
579
580 spin_lock_irq(&dev_priv->irq_lock);
581
582 i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS);
583 if (INTEL_GEN(dev_priv) >= 4)
584 i915_enable_pipestat(dev_priv, PIPE_A,
585 PIPE_LEGACY_BLC_EVENT_STATUS);
586
587 spin_unlock_irq(&dev_priv->irq_lock);
588 }
589
590 /*
591 * This timing diagram depicts the video signal in and
592 * around the vertical blanking period.
593 *
594 * Assumptions about the fictitious mode used in this example:
595 * vblank_start >= 3
596 * vsync_start = vblank_start + 1
597 * vsync_end = vblank_start + 2
598 * vtotal = vblank_start + 3
599 *
600 * start of vblank:
601 * latch double buffered registers
602 * increment frame counter (ctg+)
603 * generate start of vblank interrupt (gen4+)
604 * |
605 * | frame start:
606 * | generate frame start interrupt (aka. vblank interrupt) (gmch)
607 * | may be shifted forward 1-3 extra lines via PIPECONF
608 * | |
609 * | | start of vsync:
610 * | | generate vsync interrupt
611 * | | |
612 * ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx
613 * . \hs/ . \hs/ \hs/ \hs/ . \hs/
614 * ----va---> <-----------------vb--------------------> <--------va-------------
615 * | | <----vs-----> |
616 * -vbs-----> <---vbs+1---> <---vbs+2---> <-----0-----> <-----1-----> <-----2--- (scanline counter gen2)
617 * -vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2---> <-----0--- (scanline counter gen3+)
618 * -vbs-2---> <---vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2- (scanline counter hsw+ hdmi)
619 * | | |
620 * last visible pixel first visible pixel
621 * | increment frame counter (gen3/4)
622 * pixel counter = vblank_start * htotal pixel counter = 0 (gen3/4)
623 *
624 * x = horizontal active
625 * _ = horizontal blanking
626 * hs = horizontal sync
627 * va = vertical active
628 * vb = vertical blanking
629 * vs = vertical sync
630 * vbs = vblank_start (number)
631 *
632 * Summary:
633 * - most events happen at the start of horizontal sync
634 * - frame start happens at the start of horizontal blank, 1-4 lines
635 * (depending on PIPECONF settings) after the start of vblank
636 * - gen3/4 pixel and frame counter are synchronized with the start
637 * of horizontal active on the first line of vertical active
638 */
639
640 /* Called from drm generic code, passed a 'crtc', which
641 * we use as a pipe index
642 */
643 u32 i915_get_vblank_counter(struct drm_crtc *crtc)
644 {
645 struct drm_i915_private *dev_priv = to_i915(crtc->dev);
646 struct drm_vblank_crtc *vblank = &dev_priv->drm.vblank[drm_crtc_index(crtc)];
647 const struct drm_display_mode *mode = &vblank->hwmode;
648 enum pipe pipe = to_intel_crtc(crtc)->pipe;
649 i915_reg_t high_frame, low_frame;
650 u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal;
651 unsigned long irqflags;
652
653 /*
654 * On i965gm TV output the frame counter only works up to
655 * the point when we enable the TV encoder. After that the
656 * frame counter ceases to work and reads zero. We need a
657 * vblank wait before enabling the TV encoder and so we
658 * have to enable vblank interrupts while the frame counter
659 * is still in a working state. However the core vblank code
660 * does not like us returning non-zero frame counter values
661 * when we've told it that we don't have a working frame
662 * counter. Thus we must stop non-zero values leaking out.
663 */
664 if (!vblank->max_vblank_count)
665 return 0;
666
667 htotal = mode->crtc_htotal;
668 hsync_start = mode->crtc_hsync_start;
669 vbl_start = mode->crtc_vblank_start;
670 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
671 vbl_start = DIV_ROUND_UP(vbl_start, 2);
672
673 /* Convert to pixel count */
674 vbl_start *= htotal;
675
676 /* Start of vblank event occurs at start of hsync */
677 vbl_start -= htotal - hsync_start;
678
679 high_frame = PIPEFRAME(pipe);
680 low_frame = PIPEFRAMEPIXEL(pipe);
681
682 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
683
684 /*
685 * High & low register fields aren't synchronized, so make sure
686 * we get a low value that's stable across two reads of the high
687 * register.
688 */
689 do {
690 high1 = intel_de_read_fw(dev_priv, high_frame) & PIPE_FRAME_HIGH_MASK;
691 low = intel_de_read_fw(dev_priv, low_frame);
692 high2 = intel_de_read_fw(dev_priv, high_frame) & PIPE_FRAME_HIGH_MASK;
693 } while (high1 != high2);
694
695 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
696
697 high1 >>= PIPE_FRAME_HIGH_SHIFT;
698 pixel = low & PIPE_PIXEL_MASK;
699 low >>= PIPE_FRAME_LOW_SHIFT;
700
701 /*
702 * The frame counter increments at beginning of active.
703 * Cook up a vblank counter by also checking the pixel
704 * counter against vblank start.
705 */
706 return (((high1 << 8) | low) + (pixel >= vbl_start)) & 0xffffff;
707 }
708
709 u32 g4x_get_vblank_counter(struct drm_crtc *crtc)
710 {
711 struct drm_i915_private *dev_priv = to_i915(crtc->dev);
712 struct drm_vblank_crtc *vblank = &dev_priv->drm.vblank[drm_crtc_index(crtc)];
713 enum pipe pipe = to_intel_crtc(crtc)->pipe;
714
715 if (!vblank->max_vblank_count)
716 return 0;
717
718 return I915_READ(PIPE_FRMCOUNT_G4X(pipe));
719 }
720
721 /*
722 * On certain encoders on certain platforms, pipe
723 * scanline register will not work to get the scanline,
724 * since the timings are driven from the PORT or issues
725 * with scanline register updates.
726 * This function will use Framestamp and current
727 * timestamp registers to calculate the scanline.
728 */
729 static u32 __intel_get_crtc_scanline_from_timestamp(struct intel_crtc *crtc)
730 {
731 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
732 struct drm_vblank_crtc *vblank =
733 &crtc->base.dev->vblank[drm_crtc_index(&crtc->base)];
734 const struct drm_display_mode *mode = &vblank->hwmode;
735 u32 vblank_start = mode->crtc_vblank_start;
736 u32 vtotal = mode->crtc_vtotal;
737 u32 htotal = mode->crtc_htotal;
738 u32 clock = mode->crtc_clock;
739 u32 scanline, scan_prev_time, scan_curr_time, scan_post_time;
740
741 /*
742 * To avoid the race condition where we might cross into the
743 * next vblank just between the PIPE_FRMTMSTMP and TIMESTAMP_CTR
744 * reads. We make sure we read PIPE_FRMTMSTMP and TIMESTAMP_CTR
745 * during the same frame.
746 */
747 do {
748 /*
749 * This field provides read back of the display
750 * pipe frame time stamp. The time stamp value
751 * is sampled at every start of vertical blank.
752 */
753 scan_prev_time = intel_de_read_fw(dev_priv,
754 PIPE_FRMTMSTMP(crtc->pipe));
755
756 /*
757 * The TIMESTAMP_CTR register has the current
758 * time stamp value.
759 */
760 scan_curr_time = intel_de_read_fw(dev_priv, IVB_TIMESTAMP_CTR);
761
762 scan_post_time = intel_de_read_fw(dev_priv,
763 PIPE_FRMTMSTMP(crtc->pipe));
764 } while (scan_post_time != scan_prev_time);
765
766 scanline = div_u64(mul_u32_u32(scan_curr_time - scan_prev_time,
767 clock), 1000 * htotal);
768 scanline = min(scanline, vtotal - 1);
769 scanline = (scanline + vblank_start) % vtotal;
770
771 return scanline;
772 }
773
774 /*
775 * intel_de_read_fw(), only for fast reads of display block, no need for
776 * forcewake etc.
777 */
778 static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
779 {
780 struct drm_device *dev = crtc->base.dev;
781 struct drm_i915_private *dev_priv = to_i915(dev);
782 const struct drm_display_mode *mode;
783 struct drm_vblank_crtc *vblank;
784 enum pipe pipe = crtc->pipe;
785 int position, vtotal;
786
787 if (!crtc->active)
788 return -1;
789
790 vblank = &crtc->base.dev->vblank[drm_crtc_index(&crtc->base)];
791 mode = &vblank->hwmode;
792
793 if (crtc->mode_flags & I915_MODE_FLAG_GET_SCANLINE_FROM_TIMESTAMP)
794 return __intel_get_crtc_scanline_from_timestamp(crtc);
795
796 vtotal = mode->crtc_vtotal;
797 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
798 vtotal /= 2;
799
800 if (IS_GEN(dev_priv, 2))
801 position = intel_de_read_fw(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN2;
802 else
803 position = intel_de_read_fw(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
804
805 /*
806 * On HSW, the DSL reg (0x70000) appears to return 0 if we
807 * read it just before the start of vblank. So try it again
808 * so we don't accidentally end up spanning a vblank frame
809 * increment, causing the pipe_update_end() code to squak at us.
810 *
811 * The nature of this problem means we can't simply check the ISR
812 * bit and return the vblank start value; nor can we use the scanline
813 * debug register in the transcoder as it appears to have the same
814 * problem. We may need to extend this to include other platforms,
815 * but so far testing only shows the problem on HSW.
816 */
817 if (HAS_DDI(dev_priv) && !position) {
818 int i, temp;
819
820 for (i = 0; i < 100; i++) {
821 udelay(1);
822 temp = intel_de_read_fw(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
823 if (temp != position) {
824 position = temp;
825 break;
826 }
827 }
828 }
829
830 /*
831 * See update_scanline_offset() for the details on the
832 * scanline_offset adjustment.
833 */
834 return (position + crtc->scanline_offset) % vtotal;
835 }
836
837 static bool i915_get_crtc_scanoutpos(struct drm_crtc *_crtc,
838 bool in_vblank_irq,
839 int *vpos, int *hpos,
840 ktime_t *stime, ktime_t *etime,
841 const struct drm_display_mode *mode)
842 {
843 struct drm_device *dev = _crtc->dev;
844 struct drm_i915_private *dev_priv = to_i915(dev);
845 struct intel_crtc *crtc = to_intel_crtc(_crtc);
846 enum pipe pipe = crtc->pipe;
847 int position;
848 int vbl_start, vbl_end, hsync_start, htotal, vtotal;
849 unsigned long irqflags;
850 bool use_scanline_counter = INTEL_GEN(dev_priv) >= 5 ||
851 IS_G4X(dev_priv) || IS_GEN(dev_priv, 2) ||
852 crtc->mode_flags & I915_MODE_FLAG_USE_SCANLINE_COUNTER;
853
854 if (drm_WARN_ON(&dev_priv->drm, !mode->crtc_clock)) {
855 drm_dbg(&dev_priv->drm,
856 "trying to get scanoutpos for disabled "
857 "pipe %c\n", pipe_name(pipe));
858 return false;
859 }
860
861 htotal = mode->crtc_htotal;
862 hsync_start = mode->crtc_hsync_start;
863 vtotal = mode->crtc_vtotal;
864 vbl_start = mode->crtc_vblank_start;
865 vbl_end = mode->crtc_vblank_end;
866
867 if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
868 vbl_start = DIV_ROUND_UP(vbl_start, 2);
869 vbl_end /= 2;
870 vtotal /= 2;
871 }
872
873 /*
874 * Lock uncore.lock, as we will do multiple timing critical raw
875 * register reads, potentially with preemption disabled, so the
876 * following code must not block on uncore.lock.
877 */
878 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
879
880 /* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */
881
882 /* Get optional system timestamp before query. */
883 if (stime)
884 *stime = ktime_get();
885
886 if (use_scanline_counter) {
887 /* No obvious pixelcount register. Only query vertical
888 * scanout position from Display scan line register.
889 */
890 position = __intel_get_crtc_scanline(crtc);
891 } else {
892 /* Have access to pixelcount since start of frame.
893 * We can split this into vertical and horizontal
894 * scanout position.
895 */
896 position = (intel_de_read_fw(dev_priv, PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
897
898 /* convert to pixel counts */
899 vbl_start *= htotal;
900 vbl_end *= htotal;
901 vtotal *= htotal;
902
903 /*
904 * In interlaced modes, the pixel counter counts all pixels,
905 * so one field will have htotal more pixels. In order to avoid
906 * the reported position from jumping backwards when the pixel
907 * counter is beyond the length of the shorter field, just
908 * clamp the position the length of the shorter field. This
909 * matches how the scanline counter based position works since
910 * the scanline counter doesn't count the two half lines.
911 */
912 if (position >= vtotal)
913 position = vtotal - 1;
914
915 /*
916 * Start of vblank interrupt is triggered at start of hsync,
917 * just prior to the first active line of vblank. However we
918 * consider lines to start at the leading edge of horizontal
919 * active. So, should we get here before we've crossed into
920 * the horizontal active of the first line in vblank, we would
921 * not set the DRM_SCANOUTPOS_INVBL flag. In order to fix that,
922 * always add htotal-hsync_start to the current pixel position.
923 */
924 position = (position + htotal - hsync_start) % vtotal;
925 }
926
927 /* Get optional system timestamp after query. */
928 if (etime)
929 *etime = ktime_get();
930
931 /* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */
932
933 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
934
935 /*
936 * While in vblank, position will be negative
937 * counting up towards 0 at vbl_end. And outside
938 * vblank, position will be positive counting
939 * up since vbl_end.
940 */
941 if (position >= vbl_start)
942 position -= vbl_end;
943 else
944 position += vtotal - vbl_end;
945
946 if (use_scanline_counter) {
947 *vpos = position;
948 *hpos = 0;
949 } else {
950 *vpos = position / htotal;
951 *hpos = position - (*vpos * htotal);
952 }
953
954 return true;
955 }
956
957 bool intel_crtc_get_vblank_timestamp(struct drm_crtc *crtc, int *max_error,
958 ktime_t *vblank_time, bool in_vblank_irq)
959 {
960 return drm_crtc_vblank_helper_get_vblank_timestamp_internal(
961 crtc, max_error, vblank_time, in_vblank_irq,
962 i915_get_crtc_scanoutpos);
963 }
964
965 int intel_get_crtc_scanline(struct intel_crtc *crtc)
966 {
967 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
968 unsigned long irqflags;
969 int position;
970
971 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
972 position = __intel_get_crtc_scanline(crtc);
973 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
974
975 return position;
976 }
977
978 /**
979 * ivb_parity_work - Workqueue called when a parity error interrupt
980 * occurred.
981 * @work: workqueue struct
982 *
983 * Doesn't actually do anything except notify userspace. As a consequence of
984 * this event, userspace should try to remap the bad rows since statistically
985 * it is likely the same row is more likely to go bad again.
986 */
987 static void ivb_parity_work(struct work_struct *work)
988 {
989 struct drm_i915_private *dev_priv =
990 container_of(work, typeof(*dev_priv), l3_parity.error_work);
991 struct intel_gt *gt = &dev_priv->gt;
992 u32 error_status, row, bank, subbank;
993 char *parity_event[6];
994 u32 misccpctl;
995 u8 slice = 0;
996
997 /* We must turn off DOP level clock gating to access the L3 registers.
998 * In order to prevent a get/put style interface, acquire struct mutex
999 * any time we access those registers.
1000 */
1001 mutex_lock(&dev_priv->drm.struct_mutex);
1002
1003 /* If we've screwed up tracking, just let the interrupt fire again */
1004 if (drm_WARN_ON(&dev_priv->drm, !dev_priv->l3_parity.which_slice))
1005 goto out;
1006
1007 misccpctl = I915_READ(GEN7_MISCCPCTL);
1008 I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
1009 POSTING_READ(GEN7_MISCCPCTL);
1010
1011 while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) {
1012 i915_reg_t reg;
1013
1014 slice--;
1015 if (drm_WARN_ON_ONCE(&dev_priv->drm,
1016 slice >= NUM_L3_SLICES(dev_priv)))
1017 break;
1018
1019 dev_priv->l3_parity.which_slice &= ~(1<<slice);
1020
1021 reg = GEN7_L3CDERRST1(slice);
1022
1023 error_status = I915_READ(reg);
1024 row = GEN7_PARITY_ERROR_ROW(error_status);
1025 bank = GEN7_PARITY_ERROR_BANK(error_status);
1026 subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);
1027
1028 I915_WRITE(reg, GEN7_PARITY_ERROR_VALID | GEN7_L3CDERRST1_ENABLE);
1029 POSTING_READ(reg);
1030
1031 parity_event[0] = I915_L3_PARITY_UEVENT "=1";
1032 parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row);
1033 parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank);
1034 parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank);
1035 parity_event[4] = kasprintf(GFP_KERNEL, "SLICE=%d", slice);
1036 parity_event[5] = NULL;
1037
1038 kobject_uevent_env(&dev_priv->drm.primary->kdev->kobj,
1039 KOBJ_CHANGE, parity_event);
1040
1041 DRM_DEBUG("Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n",
1042 slice, row, bank, subbank);
1043
1044 kfree(parity_event[4]);
1045 kfree(parity_event[3]);
1046 kfree(parity_event[2]);
1047 kfree(parity_event[1]);
1048 }
1049
1050 I915_WRITE(GEN7_MISCCPCTL, misccpctl);
1051
1052 out:
1053 drm_WARN_ON(&dev_priv->drm, dev_priv->l3_parity.which_slice);
1054 spin_lock_irq(&gt->irq_lock);
1055 gen5_gt_enable_irq(gt, GT_PARITY_ERROR(dev_priv));
1056 spin_unlock_irq(&gt->irq_lock);
1057
1058 mutex_unlock(&dev_priv->drm.struct_mutex);
1059 }
1060
1061 static bool gen11_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1062 {
1063 switch (pin) {
1064 case HPD_PORT_TC1:
1065 return val & GEN11_HOTPLUG_CTL_LONG_DETECT(HPD_PORT_TC1);
1066 case HPD_PORT_TC2:
1067 return val & GEN11_HOTPLUG_CTL_LONG_DETECT(HPD_PORT_TC2);
1068 case HPD_PORT_TC3:
1069 return val & GEN11_HOTPLUG_CTL_LONG_DETECT(HPD_PORT_TC3);
1070 case HPD_PORT_TC4:
1071 return val & GEN11_HOTPLUG_CTL_LONG_DETECT(HPD_PORT_TC4);
1072 case HPD_PORT_TC5:
1073 return val & GEN11_HOTPLUG_CTL_LONG_DETECT(HPD_PORT_TC5);
1074 case HPD_PORT_TC6:
1075 return val & GEN11_HOTPLUG_CTL_LONG_DETECT(HPD_PORT_TC6);
1076 default:
1077 return false;
1078 }
1079 }
1080
1081 static bool bxt_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1082 {
1083 switch (pin) {
1084 case HPD_PORT_A:
1085 return val & PORTA_HOTPLUG_LONG_DETECT;
1086 case HPD_PORT_B:
1087 return val & PORTB_HOTPLUG_LONG_DETECT;
1088 case HPD_PORT_C:
1089 return val & PORTC_HOTPLUG_LONG_DETECT;
1090 default:
1091 return false;
1092 }
1093 }
1094
1095 static bool icp_ddi_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1096 {
1097 switch (pin) {
1098 case HPD_PORT_A:
1099 return val & SHOTPLUG_CTL_DDI_HPD_LONG_DETECT(HPD_PORT_A);
1100 case HPD_PORT_B:
1101 return val & SHOTPLUG_CTL_DDI_HPD_LONG_DETECT(HPD_PORT_B);
1102 case HPD_PORT_C:
1103 return val & SHOTPLUG_CTL_DDI_HPD_LONG_DETECT(HPD_PORT_C);
1104 case HPD_PORT_D:
1105 return val & SHOTPLUG_CTL_DDI_HPD_LONG_DETECT(HPD_PORT_D);
1106 default:
1107 return false;
1108 }
1109 }
1110
1111 static bool icp_tc_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1112 {
1113 switch (pin) {
1114 case HPD_PORT_TC1:
1115 return val & ICP_TC_HPD_LONG_DETECT(HPD_PORT_TC1);
1116 case HPD_PORT_TC2:
1117 return val & ICP_TC_HPD_LONG_DETECT(HPD_PORT_TC2);
1118 case HPD_PORT_TC3:
1119 return val & ICP_TC_HPD_LONG_DETECT(HPD_PORT_TC3);
1120 case HPD_PORT_TC4:
1121 return val & ICP_TC_HPD_LONG_DETECT(HPD_PORT_TC4);
1122 case HPD_PORT_TC5:
1123 return val & ICP_TC_HPD_LONG_DETECT(HPD_PORT_TC5);
1124 case HPD_PORT_TC6:
1125 return val & ICP_TC_HPD_LONG_DETECT(HPD_PORT_TC6);
1126 default:
1127 return false;
1128 }
1129 }
1130
1131 static bool spt_port_hotplug2_long_detect(enum hpd_pin pin, u32 val)
1132 {
1133 switch (pin) {
1134 case HPD_PORT_E:
1135 return val & PORTE_HOTPLUG_LONG_DETECT;
1136 default:
1137 return false;
1138 }
1139 }
1140
1141 static bool spt_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1142 {
1143 switch (pin) {
1144 case HPD_PORT_A:
1145 return val & PORTA_HOTPLUG_LONG_DETECT;
1146 case HPD_PORT_B:
1147 return val & PORTB_HOTPLUG_LONG_DETECT;
1148 case HPD_PORT_C:
1149 return val & PORTC_HOTPLUG_LONG_DETECT;
1150 case HPD_PORT_D:
1151 return val & PORTD_HOTPLUG_LONG_DETECT;
1152 default:
1153 return false;
1154 }
1155 }
1156
1157 static bool ilk_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1158 {
1159 switch (pin) {
1160 case HPD_PORT_A:
1161 return val & DIGITAL_PORTA_HOTPLUG_LONG_DETECT;
1162 default:
1163 return false;
1164 }
1165 }
1166
1167 static bool pch_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1168 {
1169 switch (pin) {
1170 case HPD_PORT_B:
1171 return val & PORTB_HOTPLUG_LONG_DETECT;
1172 case HPD_PORT_C:
1173 return val & PORTC_HOTPLUG_LONG_DETECT;
1174 case HPD_PORT_D:
1175 return val & PORTD_HOTPLUG_LONG_DETECT;
1176 default:
1177 return false;
1178 }
1179 }
1180
1181 static bool i9xx_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1182 {
1183 switch (pin) {
1184 case HPD_PORT_B:
1185 return val & PORTB_HOTPLUG_INT_LONG_PULSE;
1186 case HPD_PORT_C:
1187 return val & PORTC_HOTPLUG_INT_LONG_PULSE;
1188 case HPD_PORT_D:
1189 return val & PORTD_HOTPLUG_INT_LONG_PULSE;
1190 default:
1191 return false;
1192 }
1193 }
1194
1195 /*
1196 * Get a bit mask of pins that have triggered, and which ones may be long.
1197 * This can be called multiple times with the same masks to accumulate
1198 * hotplug detection results from several registers.
1199 *
1200 * Note that the caller is expected to zero out the masks initially.
1201 */
1202 static void intel_get_hpd_pins(struct drm_i915_private *dev_priv,
1203 u32 *pin_mask, u32 *long_mask,
1204 u32 hotplug_trigger, u32 dig_hotplug_reg,
1205 const u32 hpd[HPD_NUM_PINS],
1206 bool long_pulse_detect(enum hpd_pin pin, u32 val))
1207 {
1208 enum hpd_pin pin;
1209
1210 BUILD_BUG_ON(BITS_PER_TYPE(*pin_mask) < HPD_NUM_PINS);
1211
1212 for_each_hpd_pin(pin) {
1213 if ((hpd[pin] & hotplug_trigger) == 0)
1214 continue;
1215
1216 *pin_mask |= BIT(pin);
1217
1218 if (long_pulse_detect(pin, dig_hotplug_reg))
1219 *long_mask |= BIT(pin);
1220 }
1221
1222 drm_dbg(&dev_priv->drm,
1223 "hotplug event received, stat 0x%08x, dig 0x%08x, pins 0x%08x, long 0x%08x\n",
1224 hotplug_trigger, dig_hotplug_reg, *pin_mask, *long_mask);
1225
1226 }
1227
1228 static u32 intel_hpd_enabled_irqs(struct drm_i915_private *dev_priv,
1229 const u32 hpd[HPD_NUM_PINS])
1230 {
1231 struct intel_encoder *encoder;
1232 u32 enabled_irqs = 0;
1233
1234 for_each_intel_encoder(&dev_priv->drm, encoder)
1235 if (dev_priv->hotplug.stats[encoder->hpd_pin].state == HPD_ENABLED)
1236 enabled_irqs |= hpd[encoder->hpd_pin];
1237
1238 return enabled_irqs;
1239 }
1240
1241 static u32 intel_hpd_hotplug_irqs(struct drm_i915_private *dev_priv,
1242 const u32 hpd[HPD_NUM_PINS])
1243 {
1244 struct intel_encoder *encoder;
1245 u32 hotplug_irqs = 0;
1246
1247 for_each_intel_encoder(&dev_priv->drm, encoder)
1248 hotplug_irqs |= hpd[encoder->hpd_pin];
1249
1250 return hotplug_irqs;
1251 }
1252
1253 static u32 intel_hpd_hotplug_enables(struct drm_i915_private *i915,
1254 hotplug_enables_func hotplug_enables)
1255 {
1256 struct intel_encoder *encoder;
1257 u32 hotplug = 0;
1258
1259 for_each_intel_encoder(&i915->drm, encoder)
1260 hotplug |= hotplug_enables(i915, encoder->hpd_pin);
1261
1262 return hotplug;
1263 }
1264
1265 static void gmbus_irq_handler(struct drm_i915_private *dev_priv)
1266 {
1267 wake_up_all(&dev_priv->gmbus_wait_queue);
1268 }
1269
1270 static void dp_aux_irq_handler(struct drm_i915_private *dev_priv)
1271 {
1272 wake_up_all(&dev_priv->gmbus_wait_queue);
1273 }
1274
1275 #if defined(CONFIG_DEBUG_FS)
1276 static void display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
1277 enum pipe pipe,
1278 u32 crc0, u32 crc1,
1279 u32 crc2, u32 crc3,
1280 u32 crc4)
1281 {
1282 struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
1283 struct intel_pipe_crc *pipe_crc = &crtc->pipe_crc;
1284 u32 crcs[5] = { crc0, crc1, crc2, crc3, crc4 };
1285
1286 trace_intel_pipe_crc(crtc, crcs);
1287
1288 spin_lock(&pipe_crc->lock);
1289 /*
1290 * For some not yet identified reason, the first CRC is
1291 * bonkers. So let's just wait for the next vblank and read
1292 * out the buggy result.
1293 *
1294 * On GEN8+ sometimes the second CRC is bonkers as well, so
1295 * don't trust that one either.
1296 */
1297 if (pipe_crc->skipped <= 0 ||
1298 (INTEL_GEN(dev_priv) >= 8 && pipe_crc->skipped == 1)) {
1299 pipe_crc->skipped++;
1300 spin_unlock(&pipe_crc->lock);
1301 return;
1302 }
1303 spin_unlock(&pipe_crc->lock);
1304
1305 drm_crtc_add_crc_entry(&crtc->base, true,
1306 drm_crtc_accurate_vblank_count(&crtc->base),
1307 crcs);
1308 }
1309 #else
1310 static inline void
1311 display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
1312 enum pipe pipe,
1313 u32 crc0, u32 crc1,
1314 u32 crc2, u32 crc3,
1315 u32 crc4) {}
1316 #endif
1317
1318 static void flip_done_handler(struct drm_i915_private *i915,
1319 enum pipe pipe)
1320 {
1321 struct intel_crtc *crtc = intel_get_crtc_for_pipe(i915, pipe);
1322 struct drm_crtc_state *crtc_state = crtc->base.state;
1323 struct drm_pending_vblank_event *e = crtc_state->event;
1324 struct drm_device *dev = &i915->drm;
1325 unsigned long irqflags;
1326
1327 spin_lock_irqsave(&dev->event_lock, irqflags);
1328
1329 crtc_state->event = NULL;
1330
1331 drm_crtc_send_vblank_event(&crtc->base, e);
1332
1333 spin_unlock_irqrestore(&dev->event_lock, irqflags);
1334 }
1335
1336 static void hsw_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
1337 enum pipe pipe)
1338 {
1339 display_pipe_crc_irq_handler(dev_priv, pipe,
1340 I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
1341 0, 0, 0, 0);
1342 }
1343
1344 static void ivb_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
1345 enum pipe pipe)
1346 {
1347 display_pipe_crc_irq_handler(dev_priv, pipe,
1348 I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
1349 I915_READ(PIPE_CRC_RES_2_IVB(pipe)),
1350 I915_READ(PIPE_CRC_RES_3_IVB(pipe)),
1351 I915_READ(PIPE_CRC_RES_4_IVB(pipe)),
1352 I915_READ(PIPE_CRC_RES_5_IVB(pipe)));
1353 }
1354
1355 static void i9xx_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
1356 enum pipe pipe)
1357 {
1358 u32 res1, res2;
1359
1360 if (INTEL_GEN(dev_priv) >= 3)
1361 res1 = I915_READ(PIPE_CRC_RES_RES1_I915(pipe));
1362 else
1363 res1 = 0;
1364
1365 if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
1366 res2 = I915_READ(PIPE_CRC_RES_RES2_G4X(pipe));
1367 else
1368 res2 = 0;
1369
1370 display_pipe_crc_irq_handler(dev_priv, pipe,
1371 I915_READ(PIPE_CRC_RES_RED(pipe)),
1372 I915_READ(PIPE_CRC_RES_GREEN(pipe)),
1373 I915_READ(PIPE_CRC_RES_BLUE(pipe)),
1374 res1, res2);
1375 }
1376
1377 static void i9xx_pipestat_irq_reset(struct drm_i915_private *dev_priv)
1378 {
1379 enum pipe pipe;
1380
1381 for_each_pipe(dev_priv, pipe) {
1382 I915_WRITE(PIPESTAT(pipe),
1383 PIPESTAT_INT_STATUS_MASK |
1384 PIPE_FIFO_UNDERRUN_STATUS);
1385
1386 dev_priv->pipestat_irq_mask[pipe] = 0;
1387 }
1388 }
1389
1390 static void i9xx_pipestat_irq_ack(struct drm_i915_private *dev_priv,
1391 u32 iir, u32 pipe_stats[I915_MAX_PIPES])
1392 {
1393 enum pipe pipe;
1394
1395 spin_lock(&dev_priv->irq_lock);
1396
1397 if (!dev_priv->display_irqs_enabled) {
1398 spin_unlock(&dev_priv->irq_lock);
1399 return;
1400 }
1401
1402 for_each_pipe(dev_priv, pipe) {
1403 i915_reg_t reg;
1404 u32 status_mask, enable_mask, iir_bit = 0;
1405
1406 /*
1407 * PIPESTAT bits get signalled even when the interrupt is
1408 * disabled with the mask bits, and some of the status bits do
1409 * not generate interrupts at all (like the underrun bit). Hence
1410 * we need to be careful that we only handle what we want to
1411 * handle.
1412 */
1413
1414 /* fifo underruns are filterered in the underrun handler. */
1415 status_mask = PIPE_FIFO_UNDERRUN_STATUS;
1416
1417 switch (pipe) {
1418 default:
1419 case PIPE_A:
1420 iir_bit = I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
1421 break;
1422 case PIPE_B:
1423 iir_bit = I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
1424 break;
1425 case PIPE_C:
1426 iir_bit = I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
1427 break;
1428 }
1429 if (iir & iir_bit)
1430 status_mask |= dev_priv->pipestat_irq_mask[pipe];
1431
1432 if (!status_mask)
1433 continue;
1434
1435 reg = PIPESTAT(pipe);
1436 pipe_stats[pipe] = I915_READ(reg) & status_mask;
1437 enable_mask = i915_pipestat_enable_mask(dev_priv, pipe);
1438
1439 /*
1440 * Clear the PIPE*STAT regs before the IIR
1441 *
1442 * Toggle the enable bits to make sure we get an
1443 * edge in the ISR pipe event bit if we don't clear
1444 * all the enabled status bits. Otherwise the edge
1445 * triggered IIR on i965/g4x wouldn't notice that
1446 * an interrupt is still pending.
1447 */
1448 if (pipe_stats[pipe]) {
1449 I915_WRITE(reg, pipe_stats[pipe]);
1450 I915_WRITE(reg, enable_mask);
1451 }
1452 }
1453 spin_unlock(&dev_priv->irq_lock);
1454 }
1455
1456 static void i8xx_pipestat_irq_handler(struct drm_i915_private *dev_priv,
1457 u16 iir, u32 pipe_stats[I915_MAX_PIPES])
1458 {
1459 enum pipe pipe;
1460
1461 for_each_pipe(dev_priv, pipe) {
1462 if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS)
1463 intel_handle_vblank(dev_priv, pipe);
1464
1465 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
1466 i9xx_pipe_crc_irq_handler(dev_priv, pipe);
1467
1468 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1469 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1470 }
1471 }
1472
1473 static void i915_pipestat_irq_handler(struct drm_i915_private *dev_priv,
1474 u32 iir, u32 pipe_stats[I915_MAX_PIPES])
1475 {
1476 bool blc_event = false;
1477 enum pipe pipe;
1478
1479 for_each_pipe(dev_priv, pipe) {
1480 if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS)
1481 intel_handle_vblank(dev_priv, pipe);
1482
1483 if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
1484 blc_event = true;
1485
1486 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
1487 i9xx_pipe_crc_irq_handler(dev_priv, pipe);
1488
1489 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1490 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1491 }
1492
1493 if (blc_event || (iir & I915_ASLE_INTERRUPT))
1494 intel_opregion_asle_intr(dev_priv);
1495 }
1496
1497 static void i965_pipestat_irq_handler(struct drm_i915_private *dev_priv,
1498 u32 iir, u32 pipe_stats[I915_MAX_PIPES])
1499 {
1500 bool blc_event = false;
1501 enum pipe pipe;
1502
1503 for_each_pipe(dev_priv, pipe) {
1504 if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS)
1505 intel_handle_vblank(dev_priv, pipe);
1506
1507 if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
1508 blc_event = true;
1509
1510 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
1511 i9xx_pipe_crc_irq_handler(dev_priv, pipe);
1512
1513 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1514 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1515 }
1516
1517 if (blc_event || (iir & I915_ASLE_INTERRUPT))
1518 intel_opregion_asle_intr(dev_priv);
1519
1520 if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
1521 gmbus_irq_handler(dev_priv);
1522 }
1523
1524 static void valleyview_pipestat_irq_handler(struct drm_i915_private *dev_priv,
1525 u32 pipe_stats[I915_MAX_PIPES])
1526 {
1527 enum pipe pipe;
1528
1529 for_each_pipe(dev_priv, pipe) {
1530 if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS)
1531 intel_handle_vblank(dev_priv, pipe);
1532
1533 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
1534 i9xx_pipe_crc_irq_handler(dev_priv, pipe);
1535
1536 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1537 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1538 }
1539
1540 if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
1541 gmbus_irq_handler(dev_priv);
1542 }
1543
1544 static u32 i9xx_hpd_irq_ack(struct drm_i915_private *dev_priv)
1545 {
1546 u32 hotplug_status = 0, hotplug_status_mask;
1547 int i;
1548
1549 if (IS_G4X(dev_priv) ||
1550 IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
1551 hotplug_status_mask = HOTPLUG_INT_STATUS_G4X |
1552 DP_AUX_CHANNEL_MASK_INT_STATUS_G4X;
1553 else
1554 hotplug_status_mask = HOTPLUG_INT_STATUS_I915;
1555
1556 /*
1557 * We absolutely have to clear all the pending interrupt
1558 * bits in PORT_HOTPLUG_STAT. Otherwise the ISR port
1559 * interrupt bit won't have an edge, and the i965/g4x
1560 * edge triggered IIR will not notice that an interrupt
1561 * is still pending. We can't use PORT_HOTPLUG_EN to
1562 * guarantee the edge as the act of toggling the enable
1563 * bits can itself generate a new hotplug interrupt :(
1564 */
1565 for (i = 0; i < 10; i++) {
1566 u32 tmp = I915_READ(PORT_HOTPLUG_STAT) & hotplug_status_mask;
1567
1568 if (tmp == 0)
1569 return hotplug_status;
1570
1571 hotplug_status |= tmp;
1572 I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
1573 }
1574
1575 drm_WARN_ONCE(&dev_priv->drm, 1,
1576 "PORT_HOTPLUG_STAT did not clear (0x%08x)\n",
1577 I915_READ(PORT_HOTPLUG_STAT));
1578
1579 return hotplug_status;
1580 }
1581
1582 static void i9xx_hpd_irq_handler(struct drm_i915_private *dev_priv,
1583 u32 hotplug_status)
1584 {
1585 u32 pin_mask = 0, long_mask = 0;
1586 u32 hotplug_trigger;
1587
1588 if (IS_G4X(dev_priv) ||
1589 IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
1590 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X;
1591 else
1592 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
1593
1594 if (hotplug_trigger) {
1595 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
1596 hotplug_trigger, hotplug_trigger,
1597 dev_priv->hotplug.hpd,
1598 i9xx_port_hotplug_long_detect);
1599
1600 intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
1601 }
1602
1603 if ((IS_G4X(dev_priv) ||
1604 IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
1605 hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X)
1606 dp_aux_irq_handler(dev_priv);
1607 }
1608
1609 static irqreturn_t valleyview_irq_handler(int irq, void *arg)
1610 {
1611 struct drm_i915_private *dev_priv = arg;
1612 irqreturn_t ret = IRQ_NONE;
1613
1614 if (!intel_irqs_enabled(dev_priv))
1615 return IRQ_NONE;
1616
1617 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
1618 disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1619
1620 do {
1621 u32 iir, gt_iir, pm_iir;
1622 u32 pipe_stats[I915_MAX_PIPES] = {};
1623 u32 hotplug_status = 0;
1624 u32 ier = 0;
1625
1626 gt_iir = I915_READ(GTIIR);
1627 pm_iir = I915_READ(GEN6_PMIIR);
1628 iir = I915_READ(VLV_IIR);
1629
1630 if (gt_iir == 0 && pm_iir == 0 && iir == 0)
1631 break;
1632
1633 ret = IRQ_HANDLED;
1634
1635 /*
1636 * Theory on interrupt generation, based on empirical evidence:
1637 *
1638 * x = ((VLV_IIR & VLV_IER) ||
1639 * (((GT_IIR & GT_IER) || (GEN6_PMIIR & GEN6_PMIER)) &&
1640 * (VLV_MASTER_IER & MASTER_INTERRUPT_ENABLE)));
1641 *
1642 * A CPU interrupt will only be raised when 'x' has a 0->1 edge.
1643 * Hence we clear MASTER_INTERRUPT_ENABLE and VLV_IER to
1644 * guarantee the CPU interrupt will be raised again even if we
1645 * don't end up clearing all the VLV_IIR, GT_IIR, GEN6_PMIIR
1646 * bits this time around.
1647 */
1648 I915_WRITE(VLV_MASTER_IER, 0);
1649 ier = I915_READ(VLV_IER);
1650 I915_WRITE(VLV_IER, 0);
1651
1652 if (gt_iir)
1653 I915_WRITE(GTIIR, gt_iir);
1654 if (pm_iir)
1655 I915_WRITE(GEN6_PMIIR, pm_iir);
1656
1657 if (iir & I915_DISPLAY_PORT_INTERRUPT)
1658 hotplug_status = i9xx_hpd_irq_ack(dev_priv);
1659
1660 /* Call regardless, as some status bits might not be
1661 * signalled in iir */
1662 i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
1663
1664 if (iir & (I915_LPE_PIPE_A_INTERRUPT |
1665 I915_LPE_PIPE_B_INTERRUPT))
1666 intel_lpe_audio_irq_handler(dev_priv);
1667
1668 /*
1669 * VLV_IIR is single buffered, and reflects the level
1670 * from PIPESTAT/PORT_HOTPLUG_STAT, hence clear it last.
1671 */
1672 if (iir)
1673 I915_WRITE(VLV_IIR, iir);
1674
1675 I915_WRITE(VLV_IER, ier);
1676 I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
1677
1678 if (gt_iir)
1679 gen6_gt_irq_handler(&dev_priv->gt, gt_iir);
1680 if (pm_iir)
1681 gen6_rps_irq_handler(&dev_priv->gt.rps, pm_iir);
1682
1683 if (hotplug_status)
1684 i9xx_hpd_irq_handler(dev_priv, hotplug_status);
1685
1686 valleyview_pipestat_irq_handler(dev_priv, pipe_stats);
1687 } while (0);
1688
1689 pmu_irq_stats(dev_priv, ret);
1690
1691 enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1692
1693 return ret;
1694 }
1695
1696 static irqreturn_t cherryview_irq_handler(int irq, void *arg)
1697 {
1698 struct drm_i915_private *dev_priv = arg;
1699 irqreturn_t ret = IRQ_NONE;
1700
1701 if (!intel_irqs_enabled(dev_priv))
1702 return IRQ_NONE;
1703
1704 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
1705 disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1706
1707 do {
1708 u32 master_ctl, iir;
1709 u32 pipe_stats[I915_MAX_PIPES] = {};
1710 u32 hotplug_status = 0;
1711 u32 ier = 0;
1712
1713 master_ctl = I915_READ(GEN8_MASTER_IRQ) & ~GEN8_MASTER_IRQ_CONTROL;
1714 iir = I915_READ(VLV_IIR);
1715
1716 if (master_ctl == 0 && iir == 0)
1717 break;
1718
1719 ret = IRQ_HANDLED;
1720
1721 /*
1722 * Theory on interrupt generation, based on empirical evidence:
1723 *
1724 * x = ((VLV_IIR & VLV_IER) ||
1725 * ((GEN8_MASTER_IRQ & ~GEN8_MASTER_IRQ_CONTROL) &&
1726 * (GEN8_MASTER_IRQ & GEN8_MASTER_IRQ_CONTROL)));
1727 *
1728 * A CPU interrupt will only be raised when 'x' has a 0->1 edge.
1729 * Hence we clear GEN8_MASTER_IRQ_CONTROL and VLV_IER to
1730 * guarantee the CPU interrupt will be raised again even if we
1731 * don't end up clearing all the VLV_IIR and GEN8_MASTER_IRQ_CONTROL
1732 * bits this time around.
1733 */
1734 I915_WRITE(GEN8_MASTER_IRQ, 0);
1735 ier = I915_READ(VLV_IER);
1736 I915_WRITE(VLV_IER, 0);
1737
1738 gen8_gt_irq_handler(&dev_priv->gt, master_ctl);
1739
1740 if (iir & I915_DISPLAY_PORT_INTERRUPT)
1741 hotplug_status = i9xx_hpd_irq_ack(dev_priv);
1742
1743 /* Call regardless, as some status bits might not be
1744 * signalled in iir */
1745 i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
1746
1747 if (iir & (I915_LPE_PIPE_A_INTERRUPT |
1748 I915_LPE_PIPE_B_INTERRUPT |
1749 I915_LPE_PIPE_C_INTERRUPT))
1750 intel_lpe_audio_irq_handler(dev_priv);
1751
1752 /*
1753 * VLV_IIR is single buffered, and reflects the level
1754 * from PIPESTAT/PORT_HOTPLUG_STAT, hence clear it last.
1755 */
1756 if (iir)
1757 I915_WRITE(VLV_IIR, iir);
1758
1759 I915_WRITE(VLV_IER, ier);
1760 I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
1761
1762 if (hotplug_status)
1763 i9xx_hpd_irq_handler(dev_priv, hotplug_status);
1764
1765 valleyview_pipestat_irq_handler(dev_priv, pipe_stats);
1766 } while (0);
1767
1768 pmu_irq_stats(dev_priv, ret);
1769
1770 enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1771
1772 return ret;
1773 }
1774
1775 static void ibx_hpd_irq_handler(struct drm_i915_private *dev_priv,
1776 u32 hotplug_trigger)
1777 {
1778 u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
1779
1780 /*
1781 * Somehow the PCH doesn't seem to really ack the interrupt to the CPU
1782 * unless we touch the hotplug register, even if hotplug_trigger is
1783 * zero. Not acking leads to "The master control interrupt lied (SDE)!"
1784 * errors.
1785 */
1786 dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
1787 if (!hotplug_trigger) {
1788 u32 mask = PORTA_HOTPLUG_STATUS_MASK |
1789 PORTD_HOTPLUG_STATUS_MASK |
1790 PORTC_HOTPLUG_STATUS_MASK |
1791 PORTB_HOTPLUG_STATUS_MASK;
1792 dig_hotplug_reg &= ~mask;
1793 }
1794
1795 I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
1796 if (!hotplug_trigger)
1797 return;
1798
1799 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
1800 hotplug_trigger, dig_hotplug_reg,
1801 dev_priv->hotplug.pch_hpd,
1802 pch_port_hotplug_long_detect);
1803
1804 intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
1805 }
1806
1807 static void ibx_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
1808 {
1809 enum pipe pipe;
1810 u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
1811
1812 ibx_hpd_irq_handler(dev_priv, hotplug_trigger);
1813
1814 if (pch_iir & SDE_AUDIO_POWER_MASK) {
1815 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
1816 SDE_AUDIO_POWER_SHIFT);
1817 drm_dbg(&dev_priv->drm, "PCH audio power change on port %d\n",
1818 port_name(port));
1819 }
1820
1821 if (pch_iir & SDE_AUX_MASK)
1822 dp_aux_irq_handler(dev_priv);
1823
1824 if (pch_iir & SDE_GMBUS)
1825 gmbus_irq_handler(dev_priv);
1826
1827 if (pch_iir & SDE_AUDIO_HDCP_MASK)
1828 drm_dbg(&dev_priv->drm, "PCH HDCP audio interrupt\n");
1829
1830 if (pch_iir & SDE_AUDIO_TRANS_MASK)
1831 drm_dbg(&dev_priv->drm, "PCH transcoder audio interrupt\n");
1832
1833 if (pch_iir & SDE_POISON)
1834 drm_err(&dev_priv->drm, "PCH poison interrupt\n");
1835
1836 if (pch_iir & SDE_FDI_MASK) {
1837 for_each_pipe(dev_priv, pipe)
1838 drm_dbg(&dev_priv->drm, " pipe %c FDI IIR: 0x%08x\n",
1839 pipe_name(pipe),
1840 I915_READ(FDI_RX_IIR(pipe)));
1841 }
1842
1843 if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
1844 drm_dbg(&dev_priv->drm, "PCH transcoder CRC done interrupt\n");
1845
1846 if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
1847 drm_dbg(&dev_priv->drm,
1848 "PCH transcoder CRC error interrupt\n");
1849
1850 if (pch_iir & SDE_TRANSA_FIFO_UNDER)
1851 intel_pch_fifo_underrun_irq_handler(dev_priv, PIPE_A);
1852
1853 if (pch_iir & SDE_TRANSB_FIFO_UNDER)
1854 intel_pch_fifo_underrun_irq_handler(dev_priv, PIPE_B);
1855 }
1856
1857 static void ivb_err_int_handler(struct drm_i915_private *dev_priv)
1858 {
1859 u32 err_int = I915_READ(GEN7_ERR_INT);
1860 enum pipe pipe;
1861
1862 if (err_int & ERR_INT_POISON)
1863 drm_err(&dev_priv->drm, "Poison interrupt\n");
1864
1865 for_each_pipe(dev_priv, pipe) {
1866 if (err_int & ERR_INT_FIFO_UNDERRUN(pipe))
1867 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1868
1869 if (err_int & ERR_INT_PIPE_CRC_DONE(pipe)) {
1870 if (IS_IVYBRIDGE(dev_priv))
1871 ivb_pipe_crc_irq_handler(dev_priv, pipe);
1872 else
1873 hsw_pipe_crc_irq_handler(dev_priv, pipe);
1874 }
1875 }
1876
1877 I915_WRITE(GEN7_ERR_INT, err_int);
1878 }
1879
1880 static void cpt_serr_int_handler(struct drm_i915_private *dev_priv)
1881 {
1882 u32 serr_int = I915_READ(SERR_INT);
1883 enum pipe pipe;
1884
1885 if (serr_int & SERR_INT_POISON)
1886 drm_err(&dev_priv->drm, "PCH poison interrupt\n");
1887
1888 for_each_pipe(dev_priv, pipe)
1889 if (serr_int & SERR_INT_TRANS_FIFO_UNDERRUN(pipe))
1890 intel_pch_fifo_underrun_irq_handler(dev_priv, pipe);
1891
1892 I915_WRITE(SERR_INT, serr_int);
1893 }
1894
1895 static void cpt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
1896 {
1897 enum pipe pipe;
1898 u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
1899
1900 ibx_hpd_irq_handler(dev_priv, hotplug_trigger);
1901
1902 if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
1903 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
1904 SDE_AUDIO_POWER_SHIFT_CPT);
1905 drm_dbg(&dev_priv->drm, "PCH audio power change on port %c\n",
1906 port_name(port));
1907 }
1908
1909 if (pch_iir & SDE_AUX_MASK_CPT)
1910 dp_aux_irq_handler(dev_priv);
1911
1912 if (pch_iir & SDE_GMBUS_CPT)
1913 gmbus_irq_handler(dev_priv);
1914
1915 if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
1916 drm_dbg(&dev_priv->drm, "Audio CP request interrupt\n");
1917
1918 if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
1919 drm_dbg(&dev_priv->drm, "Audio CP change interrupt\n");
1920
1921 if (pch_iir & SDE_FDI_MASK_CPT) {
1922 for_each_pipe(dev_priv, pipe)
1923 drm_dbg(&dev_priv->drm, " pipe %c FDI IIR: 0x%08x\n",
1924 pipe_name(pipe),
1925 I915_READ(FDI_RX_IIR(pipe)));
1926 }
1927
1928 if (pch_iir & SDE_ERROR_CPT)
1929 cpt_serr_int_handler(dev_priv);
1930 }
1931
1932 static void icp_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
1933 {
1934 u32 ddi_hotplug_trigger = pch_iir & SDE_DDI_HOTPLUG_MASK_ICP;
1935 u32 tc_hotplug_trigger = pch_iir & SDE_TC_HOTPLUG_MASK_ICP;
1936 u32 pin_mask = 0, long_mask = 0;
1937
1938 if (ddi_hotplug_trigger) {
1939 u32 dig_hotplug_reg;
1940
1941 dig_hotplug_reg = I915_READ(SHOTPLUG_CTL_DDI);
1942 I915_WRITE(SHOTPLUG_CTL_DDI, dig_hotplug_reg);
1943
1944 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
1945 ddi_hotplug_trigger, dig_hotplug_reg,
1946 dev_priv->hotplug.pch_hpd,
1947 icp_ddi_port_hotplug_long_detect);
1948 }
1949
1950 if (tc_hotplug_trigger) {
1951 u32 dig_hotplug_reg;
1952
1953 dig_hotplug_reg = I915_READ(SHOTPLUG_CTL_TC);
1954 I915_WRITE(SHOTPLUG_CTL_TC, dig_hotplug_reg);
1955
1956 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
1957 tc_hotplug_trigger, dig_hotplug_reg,
1958 dev_priv->hotplug.pch_hpd,
1959 icp_tc_port_hotplug_long_detect);
1960 }
1961
1962 if (pin_mask)
1963 intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
1964
1965 if (pch_iir & SDE_GMBUS_ICP)
1966 gmbus_irq_handler(dev_priv);
1967 }
1968
1969 static void spt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
1970 {
1971 u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_SPT &
1972 ~SDE_PORTE_HOTPLUG_SPT;
1973 u32 hotplug2_trigger = pch_iir & SDE_PORTE_HOTPLUG_SPT;
1974 u32 pin_mask = 0, long_mask = 0;
1975
1976 if (hotplug_trigger) {
1977 u32 dig_hotplug_reg;
1978
1979 dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
1980 I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
1981
1982 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
1983 hotplug_trigger, dig_hotplug_reg,
1984 dev_priv->hotplug.pch_hpd,
1985 spt_port_hotplug_long_detect);
1986 }
1987
1988 if (hotplug2_trigger) {
1989 u32 dig_hotplug_reg;
1990
1991 dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG2);
1992 I915_WRITE(PCH_PORT_HOTPLUG2, dig_hotplug_reg);
1993
1994 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
1995 hotplug2_trigger, dig_hotplug_reg,
1996 dev_priv->hotplug.pch_hpd,
1997 spt_port_hotplug2_long_detect);
1998 }
1999
2000 if (pin_mask)
2001 intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2002
2003 if (pch_iir & SDE_GMBUS_CPT)
2004 gmbus_irq_handler(dev_priv);
2005 }
2006
2007 static void ilk_hpd_irq_handler(struct drm_i915_private *dev_priv,
2008 u32 hotplug_trigger)
2009 {
2010 u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
2011
2012 dig_hotplug_reg = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
2013 I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, dig_hotplug_reg);
2014
2015 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
2016 hotplug_trigger, dig_hotplug_reg,
2017 dev_priv->hotplug.hpd,
2018 ilk_port_hotplug_long_detect);
2019
2020 intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2021 }
2022
2023 static void ilk_display_irq_handler(struct drm_i915_private *dev_priv,
2024 u32 de_iir)
2025 {
2026 enum pipe pipe;
2027 u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG;
2028
2029 if (hotplug_trigger)
2030 ilk_hpd_irq_handler(dev_priv, hotplug_trigger);
2031
2032 if (de_iir & DE_AUX_CHANNEL_A)
2033 dp_aux_irq_handler(dev_priv);
2034
2035 if (de_iir & DE_GSE)
2036 intel_opregion_asle_intr(dev_priv);
2037
2038 if (de_iir & DE_POISON)
2039 drm_err(&dev_priv->drm, "Poison interrupt\n");
2040
2041 for_each_pipe(dev_priv, pipe) {
2042 if (de_iir & DE_PIPE_VBLANK(pipe))
2043 intel_handle_vblank(dev_priv, pipe);
2044
2045 if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe))
2046 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2047
2048 if (de_iir & DE_PIPE_CRC_DONE(pipe))
2049 i9xx_pipe_crc_irq_handler(dev_priv, pipe);
2050 }
2051
2052 /* check event from PCH */
2053 if (de_iir & DE_PCH_EVENT) {
2054 u32 pch_iir = I915_READ(SDEIIR);
2055
2056 if (HAS_PCH_CPT(dev_priv))
2057 cpt_irq_handler(dev_priv, pch_iir);
2058 else
2059 ibx_irq_handler(dev_priv, pch_iir);
2060
2061 /* should clear PCH hotplug event before clear CPU irq */
2062 I915_WRITE(SDEIIR, pch_iir);
2063 }
2064
2065 if (IS_GEN(dev_priv, 5) && de_iir & DE_PCU_EVENT)
2066 gen5_rps_irq_handler(&dev_priv->gt.rps);
2067 }
2068
2069 static void ivb_display_irq_handler(struct drm_i915_private *dev_priv,
2070 u32 de_iir)
2071 {
2072 enum pipe pipe;
2073 u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG_IVB;
2074
2075 if (hotplug_trigger)
2076 ilk_hpd_irq_handler(dev_priv, hotplug_trigger);
2077
2078 if (de_iir & DE_ERR_INT_IVB)
2079 ivb_err_int_handler(dev_priv);
2080
2081 if (de_iir & DE_EDP_PSR_INT_HSW) {
2082 u32 psr_iir = I915_READ(EDP_PSR_IIR);
2083
2084 intel_psr_irq_handler(dev_priv, psr_iir);
2085 I915_WRITE(EDP_PSR_IIR, psr_iir);
2086 }
2087
2088 if (de_iir & DE_AUX_CHANNEL_A_IVB)
2089 dp_aux_irq_handler(dev_priv);
2090
2091 if (de_iir & DE_GSE_IVB)
2092 intel_opregion_asle_intr(dev_priv);
2093
2094 for_each_pipe(dev_priv, pipe) {
2095 if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)))
2096 intel_handle_vblank(dev_priv, pipe);
2097 }
2098
2099 /* check event from PCH */
2100 if (!HAS_PCH_NOP(dev_priv) && (de_iir & DE_PCH_EVENT_IVB)) {
2101 u32 pch_iir = I915_READ(SDEIIR);
2102
2103 cpt_irq_handler(dev_priv, pch_iir);
2104
2105 /* clear PCH hotplug event before clear CPU irq */
2106 I915_WRITE(SDEIIR, pch_iir);
2107 }
2108 }
2109
2110 /*
2111 * To handle irqs with the minimum potential races with fresh interrupts, we:
2112 * 1 - Disable Master Interrupt Control.
2113 * 2 - Find the source(s) of the interrupt.
2114 * 3 - Clear the Interrupt Identity bits (IIR).
2115 * 4 - Process the interrupt(s) that had bits set in the IIRs.
2116 * 5 - Re-enable Master Interrupt Control.
2117 */
2118 static irqreturn_t ilk_irq_handler(int irq, void *arg)
2119 {
2120 struct drm_i915_private *i915 = arg;
2121 void __iomem * const regs = i915->uncore.regs;
2122 u32 de_iir, gt_iir, de_ier, sde_ier = 0;
2123 irqreturn_t ret = IRQ_NONE;
2124
2125 if (unlikely(!intel_irqs_enabled(i915)))
2126 return IRQ_NONE;
2127
2128 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
2129 disable_rpm_wakeref_asserts(&i915->runtime_pm);
2130
2131 /* disable master interrupt before clearing iir */
2132 de_ier = raw_reg_read(regs, DEIER);
2133 raw_reg_write(regs, DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
2134
2135 /* Disable south interrupts. We'll only write to SDEIIR once, so further
2136 * interrupts will will be stored on its back queue, and then we'll be
2137 * able to process them after we restore SDEIER (as soon as we restore
2138 * it, we'll get an interrupt if SDEIIR still has something to process
2139 * due to its back queue). */
2140 if (!HAS_PCH_NOP(i915)) {
2141 sde_ier = raw_reg_read(regs, SDEIER);
2142 raw_reg_write(regs, SDEIER, 0);
2143 }
2144
2145 /* Find, clear, then process each source of interrupt */
2146
2147 gt_iir = raw_reg_read(regs, GTIIR);
2148 if (gt_iir) {
2149 raw_reg_write(regs, GTIIR, gt_iir);
2150 if (INTEL_GEN(i915) >= 6)
2151 gen6_gt_irq_handler(&i915->gt, gt_iir);
2152 else
2153 gen5_gt_irq_handler(&i915->gt, gt_iir);
2154 ret = IRQ_HANDLED;
2155 }
2156
2157 de_iir = raw_reg_read(regs, DEIIR);
2158 if (de_iir) {
2159 raw_reg_write(regs, DEIIR, de_iir);
2160 if (INTEL_GEN(i915) >= 7)
2161 ivb_display_irq_handler(i915, de_iir);
2162 else
2163 ilk_display_irq_handler(i915, de_iir);
2164 ret = IRQ_HANDLED;
2165 }
2166
2167 if (INTEL_GEN(i915) >= 6) {
2168 u32 pm_iir = raw_reg_read(regs, GEN6_PMIIR);
2169 if (pm_iir) {
2170 raw_reg_write(regs, GEN6_PMIIR, pm_iir);
2171 gen6_rps_irq_handler(&i915->gt.rps, pm_iir);
2172 ret = IRQ_HANDLED;
2173 }
2174 }
2175
2176 raw_reg_write(regs, DEIER, de_ier);
2177 if (sde_ier)
2178 raw_reg_write(regs, SDEIER, sde_ier);
2179
2180 pmu_irq_stats(i915, ret);
2181
2182 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
2183 enable_rpm_wakeref_asserts(&i915->runtime_pm);
2184
2185 return ret;
2186 }
2187
2188 static void bxt_hpd_irq_handler(struct drm_i915_private *dev_priv,
2189 u32 hotplug_trigger)
2190 {
2191 u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
2192
2193 dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
2194 I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
2195
2196 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
2197 hotplug_trigger, dig_hotplug_reg,
2198 dev_priv->hotplug.hpd,
2199 bxt_port_hotplug_long_detect);
2200
2201 intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2202 }
2203
2204 static void gen11_hpd_irq_handler(struct drm_i915_private *dev_priv, u32 iir)
2205 {
2206 u32 pin_mask = 0, long_mask = 0;
2207 u32 trigger_tc = iir & GEN11_DE_TC_HOTPLUG_MASK;
2208 u32 trigger_tbt = iir & GEN11_DE_TBT_HOTPLUG_MASK;
2209
2210 if (trigger_tc) {
2211 u32 dig_hotplug_reg;
2212
2213 dig_hotplug_reg = I915_READ(GEN11_TC_HOTPLUG_CTL);
2214 I915_WRITE(GEN11_TC_HOTPLUG_CTL, dig_hotplug_reg);
2215
2216 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
2217 trigger_tc, dig_hotplug_reg,
2218 dev_priv->hotplug.hpd,
2219 gen11_port_hotplug_long_detect);
2220 }
2221
2222 if (trigger_tbt) {
2223 u32 dig_hotplug_reg;
2224
2225 dig_hotplug_reg = I915_READ(GEN11_TBT_HOTPLUG_CTL);
2226 I915_WRITE(GEN11_TBT_HOTPLUG_CTL, dig_hotplug_reg);
2227
2228 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
2229 trigger_tbt, dig_hotplug_reg,
2230 dev_priv->hotplug.hpd,
2231 gen11_port_hotplug_long_detect);
2232 }
2233
2234 if (pin_mask)
2235 intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2236 else
2237 drm_err(&dev_priv->drm,
2238 "Unexpected DE HPD interrupt 0x%08x\n", iir);
2239 }
2240
2241 static u32 gen8_de_port_aux_mask(struct drm_i915_private *dev_priv)
2242 {
2243 u32 mask;
2244
2245 if (INTEL_GEN(dev_priv) >= 12)
2246 return TGL_DE_PORT_AUX_DDIA |
2247 TGL_DE_PORT_AUX_DDIB |
2248 TGL_DE_PORT_AUX_DDIC |
2249 TGL_DE_PORT_AUX_USBC1 |
2250 TGL_DE_PORT_AUX_USBC2 |
2251 TGL_DE_PORT_AUX_USBC3 |
2252 TGL_DE_PORT_AUX_USBC4 |
2253 TGL_DE_PORT_AUX_USBC5 |
2254 TGL_DE_PORT_AUX_USBC6;
2255
2256
2257 mask = GEN8_AUX_CHANNEL_A;
2258 if (INTEL_GEN(dev_priv) >= 9)
2259 mask |= GEN9_AUX_CHANNEL_B |
2260 GEN9_AUX_CHANNEL_C |
2261 GEN9_AUX_CHANNEL_D;
2262
2263 if (IS_CNL_WITH_PORT_F(dev_priv) || IS_GEN(dev_priv, 11))
2264 mask |= CNL_AUX_CHANNEL_F;
2265
2266 if (IS_GEN(dev_priv, 11))
2267 mask |= ICL_AUX_CHANNEL_E;
2268
2269 return mask;
2270 }
2271
2272 static u32 gen8_de_pipe_fault_mask(struct drm_i915_private *dev_priv)
2273 {
2274 if (IS_ROCKETLAKE(dev_priv))
2275 return RKL_DE_PIPE_IRQ_FAULT_ERRORS;
2276 else if (INTEL_GEN(dev_priv) >= 11)
2277 return GEN11_DE_PIPE_IRQ_FAULT_ERRORS;
2278 else if (INTEL_GEN(dev_priv) >= 9)
2279 return GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
2280 else
2281 return GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
2282 }
2283
2284 static void
2285 gen8_de_misc_irq_handler(struct drm_i915_private *dev_priv, u32 iir)
2286 {
2287 bool found = false;
2288
2289 if (iir & GEN8_DE_MISC_GSE) {
2290 intel_opregion_asle_intr(dev_priv);
2291 found = true;
2292 }
2293
2294 if (iir & GEN8_DE_EDP_PSR) {
2295 u32 psr_iir;
2296 i915_reg_t iir_reg;
2297
2298 if (INTEL_GEN(dev_priv) >= 12)
2299 iir_reg = TRANS_PSR_IIR(dev_priv->psr.transcoder);
2300 else
2301 iir_reg = EDP_PSR_IIR;
2302
2303 psr_iir = I915_READ(iir_reg);
2304 I915_WRITE(iir_reg, psr_iir);
2305
2306 if (psr_iir)
2307 found = true;
2308
2309 intel_psr_irq_handler(dev_priv, psr_iir);
2310 }
2311
2312 if (!found)
2313 drm_err(&dev_priv->drm, "Unexpected DE Misc interrupt\n");
2314 }
2315
2316 static void gen11_dsi_te_interrupt_handler(struct drm_i915_private *dev_priv,
2317 u32 te_trigger)
2318 {
2319 enum pipe pipe = INVALID_PIPE;
2320 enum transcoder dsi_trans;
2321 enum port port;
2322 u32 val, tmp;
2323
2324 /*
2325 * Incase of dual link, TE comes from DSI_1
2326 * this is to check if dual link is enabled
2327 */
2328 val = I915_READ(TRANS_DDI_FUNC_CTL2(TRANSCODER_DSI_0));
2329 val &= PORT_SYNC_MODE_ENABLE;
2330
2331 /*
2332 * if dual link is enabled, then read DSI_0
2333 * transcoder registers
2334 */
2335 port = ((te_trigger & DSI1_TE && val) || (te_trigger & DSI0_TE)) ?
2336 PORT_A : PORT_B;
2337 dsi_trans = (port == PORT_A) ? TRANSCODER_DSI_0 : TRANSCODER_DSI_1;
2338
2339 /* Check if DSI configured in command mode */
2340 val = I915_READ(DSI_TRANS_FUNC_CONF(dsi_trans));
2341 val = val & OP_MODE_MASK;
2342
2343 if (val != CMD_MODE_NO_GATE && val != CMD_MODE_TE_GATE) {
2344 drm_err(&dev_priv->drm, "DSI trancoder not configured in command mode\n");
2345 return;
2346 }
2347
2348 /* Get PIPE for handling VBLANK event */
2349 val = I915_READ(TRANS_DDI_FUNC_CTL(dsi_trans));
2350 switch (val & TRANS_DDI_EDP_INPUT_MASK) {
2351 case TRANS_DDI_EDP_INPUT_A_ON:
2352 pipe = PIPE_A;
2353 break;
2354 case TRANS_DDI_EDP_INPUT_B_ONOFF:
2355 pipe = PIPE_B;
2356 break;
2357 case TRANS_DDI_EDP_INPUT_C_ONOFF:
2358 pipe = PIPE_C;
2359 break;
2360 default:
2361 drm_err(&dev_priv->drm, "Invalid PIPE\n");
2362 return;
2363 }
2364
2365 intel_handle_vblank(dev_priv, pipe);
2366
2367 /* clear TE in dsi IIR */
2368 port = (te_trigger & DSI1_TE) ? PORT_B : PORT_A;
2369 tmp = I915_READ(DSI_INTR_IDENT_REG(port));
2370 I915_WRITE(DSI_INTR_IDENT_REG(port), tmp);
2371 }
2372
2373 static irqreturn_t
2374 gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl)
2375 {
2376 irqreturn_t ret = IRQ_NONE;
2377 u32 iir;
2378 enum pipe pipe;
2379
2380 if (master_ctl & GEN8_DE_MISC_IRQ) {
2381 iir = I915_READ(GEN8_DE_MISC_IIR);
2382 if (iir) {
2383 I915_WRITE(GEN8_DE_MISC_IIR, iir);
2384 ret = IRQ_HANDLED;
2385 gen8_de_misc_irq_handler(dev_priv, iir);
2386 } else {
2387 drm_err(&dev_priv->drm,
2388 "The master control interrupt lied (DE MISC)!\n");
2389 }
2390 }
2391
2392 if (INTEL_GEN(dev_priv) >= 11 && (master_ctl & GEN11_DE_HPD_IRQ)) {
2393 iir = I915_READ(GEN11_DE_HPD_IIR);
2394 if (iir) {
2395 I915_WRITE(GEN11_DE_HPD_IIR, iir);
2396 ret = IRQ_HANDLED;
2397 gen11_hpd_irq_handler(dev_priv, iir);
2398 } else {
2399 drm_err(&dev_priv->drm,
2400 "The master control interrupt lied, (DE HPD)!\n");
2401 }
2402 }
2403
2404 if (master_ctl & GEN8_DE_PORT_IRQ) {
2405 iir = I915_READ(GEN8_DE_PORT_IIR);
2406 if (iir) {
2407 bool found = false;
2408
2409 I915_WRITE(GEN8_DE_PORT_IIR, iir);
2410 ret = IRQ_HANDLED;
2411
2412 if (iir & gen8_de_port_aux_mask(dev_priv)) {
2413 dp_aux_irq_handler(dev_priv);
2414 found = true;
2415 }
2416
2417 if (IS_GEN9_LP(dev_priv)) {
2418 u32 hotplug_trigger = iir & BXT_DE_PORT_HOTPLUG_MASK;
2419
2420 if (hotplug_trigger) {
2421 bxt_hpd_irq_handler(dev_priv, hotplug_trigger);
2422 found = true;
2423 }
2424 } else if (IS_BROADWELL(dev_priv)) {
2425 u32 hotplug_trigger = iir & BDW_DE_PORT_HOTPLUG_MASK;
2426
2427 if (hotplug_trigger) {
2428 ilk_hpd_irq_handler(dev_priv, hotplug_trigger);
2429 found = true;
2430 }
2431 }
2432
2433 if (IS_GEN9_LP(dev_priv) && (iir & BXT_DE_PORT_GMBUS)) {
2434 gmbus_irq_handler(dev_priv);
2435 found = true;
2436 }
2437
2438 if (INTEL_GEN(dev_priv) >= 11) {
2439 u32 te_trigger = iir & (DSI0_TE | DSI1_TE);
2440
2441 if (te_trigger) {
2442 gen11_dsi_te_interrupt_handler(dev_priv, te_trigger);
2443 found = true;
2444 }
2445 }
2446
2447 if (!found)
2448 drm_err(&dev_priv->drm,
2449 "Unexpected DE Port interrupt\n");
2450 }
2451 else
2452 drm_err(&dev_priv->drm,
2453 "The master control interrupt lied (DE PORT)!\n");
2454 }
2455
2456 for_each_pipe(dev_priv, pipe) {
2457 u32 fault_errors;
2458
2459 if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe)))
2460 continue;
2461
2462 iir = I915_READ(GEN8_DE_PIPE_IIR(pipe));
2463 if (!iir) {
2464 drm_err(&dev_priv->drm,
2465 "The master control interrupt lied (DE PIPE)!\n");
2466 continue;
2467 }
2468
2469 ret = IRQ_HANDLED;
2470 I915_WRITE(GEN8_DE_PIPE_IIR(pipe), iir);
2471
2472 if (iir & GEN8_PIPE_VBLANK)
2473 intel_handle_vblank(dev_priv, pipe);
2474
2475 if (iir & GEN9_PIPE_PLANE1_FLIP_DONE)
2476 flip_done_handler(dev_priv, pipe);
2477
2478 if (iir & GEN8_PIPE_CDCLK_CRC_DONE)
2479 hsw_pipe_crc_irq_handler(dev_priv, pipe);
2480
2481 if (iir & GEN8_PIPE_FIFO_UNDERRUN)
2482 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2483
2484 fault_errors = iir & gen8_de_pipe_fault_mask(dev_priv);
2485 if (fault_errors)
2486 drm_err(&dev_priv->drm,
2487 "Fault errors on pipe %c: 0x%08x\n",
2488 pipe_name(pipe),
2489 fault_errors);
2490 }
2491
2492 if (HAS_PCH_SPLIT(dev_priv) && !HAS_PCH_NOP(dev_priv) &&
2493 master_ctl & GEN8_DE_PCH_IRQ) {
2494 /*
2495 * FIXME(BDW): Assume for now that the new interrupt handling
2496 * scheme also closed the SDE interrupt handling race we've seen
2497 * on older pch-split platforms. But this needs testing.
2498 */
2499 iir = I915_READ(SDEIIR);
2500 if (iir) {
2501 I915_WRITE(SDEIIR, iir);
2502 ret = IRQ_HANDLED;
2503
2504 if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
2505 icp_irq_handler(dev_priv, iir);
2506 else if (INTEL_PCH_TYPE(dev_priv) >= PCH_SPT)
2507 spt_irq_handler(dev_priv, iir);
2508 else
2509 cpt_irq_handler(dev_priv, iir);
2510 } else {
2511 /*
2512 * Like on previous PCH there seems to be something
2513 * fishy going on with forwarding PCH interrupts.
2514 */
2515 drm_dbg(&dev_priv->drm,
2516 "The master control interrupt lied (SDE)!\n");
2517 }
2518 }
2519
2520 return ret;
2521 }
2522
2523 static inline u32 gen8_master_intr_disable(void __iomem * const regs)
2524 {
2525 raw_reg_write(regs, GEN8_MASTER_IRQ, 0);
2526
2527 /*
2528 * Now with master disabled, get a sample of level indications
2529 * for this interrupt. Indications will be cleared on related acks.
2530 * New indications can and will light up during processing,
2531 * and will generate new interrupt after enabling master.
2532 */
2533 return raw_reg_read(regs, GEN8_MASTER_IRQ);
2534 }
2535
2536 static inline void gen8_master_intr_enable(void __iomem * const regs)
2537 {
2538 raw_reg_write(regs, GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
2539 }
2540
2541 static irqreturn_t gen8_irq_handler(int irq, void *arg)
2542 {
2543 struct drm_i915_private *dev_priv = arg;
2544 void __iomem * const regs = dev_priv->uncore.regs;
2545 u32 master_ctl;
2546
2547 if (!intel_irqs_enabled(dev_priv))
2548 return IRQ_NONE;
2549
2550 master_ctl = gen8_master_intr_disable(regs);
2551 if (!master_ctl) {
2552 gen8_master_intr_enable(regs);
2553 return IRQ_NONE;
2554 }
2555
2556 /* Find, queue (onto bottom-halves), then clear each source */
2557 gen8_gt_irq_handler(&dev_priv->gt, master_ctl);
2558
2559 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
2560 if (master_ctl & ~GEN8_GT_IRQS) {
2561 disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
2562 gen8_de_irq_handler(dev_priv, master_ctl);
2563 enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
2564 }
2565
2566 gen8_master_intr_enable(regs);
2567
2568 pmu_irq_stats(dev_priv, IRQ_HANDLED);
2569
2570 return IRQ_HANDLED;
2571 }
2572
2573 static u32
2574 gen11_gu_misc_irq_ack(struct intel_gt *gt, const u32 master_ctl)
2575 {
2576 void __iomem * const regs = gt->uncore->regs;
2577 u32 iir;
2578
2579 if (!(master_ctl & GEN11_GU_MISC_IRQ))
2580 return 0;
2581
2582 iir = raw_reg_read(regs, GEN11_GU_MISC_IIR);
2583 if (likely(iir))
2584 raw_reg_write(regs, GEN11_GU_MISC_IIR, iir);
2585
2586 return iir;
2587 }
2588
2589 static void
2590 gen11_gu_misc_irq_handler(struct intel_gt *gt, const u32 iir)
2591 {
2592 if (iir & GEN11_GU_MISC_GSE)
2593 intel_opregion_asle_intr(gt->i915);
2594 }
2595
2596 static inline u32 gen11_master_intr_disable(void __iomem * const regs)
2597 {
2598 raw_reg_write(regs, GEN11_GFX_MSTR_IRQ, 0);
2599
2600 /*
2601 * Now with master disabled, get a sample of level indications
2602 * for this interrupt. Indications will be cleared on related acks.
2603 * New indications can and will light up during processing,
2604 * and will generate new interrupt after enabling master.
2605 */
2606 return raw_reg_read(regs, GEN11_GFX_MSTR_IRQ);
2607 }
2608
2609 static inline void gen11_master_intr_enable(void __iomem * const regs)
2610 {
2611 raw_reg_write(regs, GEN11_GFX_MSTR_IRQ, GEN11_MASTER_IRQ);
2612 }
2613
2614 static void
2615 gen11_display_irq_handler(struct drm_i915_private *i915)
2616 {
2617 void __iomem * const regs = i915->uncore.regs;
2618 const u32 disp_ctl = raw_reg_read(regs, GEN11_DISPLAY_INT_CTL);
2619
2620 disable_rpm_wakeref_asserts(&i915->runtime_pm);
2621 /*
2622 * GEN11_DISPLAY_INT_CTL has same format as GEN8_MASTER_IRQ
2623 * for the display related bits.
2624 */
2625 raw_reg_write(regs, GEN11_DISPLAY_INT_CTL, 0x0);
2626 gen8_de_irq_handler(i915, disp_ctl);
2627 raw_reg_write(regs, GEN11_DISPLAY_INT_CTL,
2628 GEN11_DISPLAY_IRQ_ENABLE);
2629
2630 enable_rpm_wakeref_asserts(&i915->runtime_pm);
2631 }
2632
2633 static __always_inline irqreturn_t
2634 __gen11_irq_handler(struct drm_i915_private * const i915,
2635 u32 (*intr_disable)(void __iomem * const regs),
2636 void (*intr_enable)(void __iomem * const regs))
2637 {
2638 void __iomem * const regs = i915->uncore.regs;
2639 struct intel_gt *gt = &i915->gt;
2640 u32 master_ctl;
2641 u32 gu_misc_iir;
2642
2643 if (!intel_irqs_enabled(i915))
2644 return IRQ_NONE;
2645
2646 master_ctl = intr_disable(regs);
2647 if (!master_ctl) {
2648 intr_enable(regs);
2649 return IRQ_NONE;
2650 }
2651
2652 /* Find, queue (onto bottom-halves), then clear each source */
2653 gen11_gt_irq_handler(gt, master_ctl);
2654
2655 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
2656 if (master_ctl & GEN11_DISPLAY_IRQ)
2657 gen11_display_irq_handler(i915);
2658
2659 gu_misc_iir = gen11_gu_misc_irq_ack(gt, master_ctl);
2660
2661 intr_enable(regs);
2662
2663 gen11_gu_misc_irq_handler(gt, gu_misc_iir);
2664
2665 pmu_irq_stats(i915, IRQ_HANDLED);
2666
2667 return IRQ_HANDLED;
2668 }
2669
2670 static irqreturn_t gen11_irq_handler(int irq, void *arg)
2671 {
2672 return __gen11_irq_handler(arg,
2673 gen11_master_intr_disable,
2674 gen11_master_intr_enable);
2675 }
2676
2677 static u32 dg1_master_intr_disable_and_ack(void __iomem * const regs)
2678 {
2679 u32 val;
2680
2681 /* First disable interrupts */
2682 raw_reg_write(regs, DG1_MSTR_UNIT_INTR, 0);
2683
2684 /* Get the indication levels and ack the master unit */
2685 val = raw_reg_read(regs, DG1_MSTR_UNIT_INTR);
2686 if (unlikely(!val))
2687 return 0;
2688
2689 raw_reg_write(regs, DG1_MSTR_UNIT_INTR, val);
2690
2691 /*
2692 * Now with master disabled, get a sample of level indications
2693 * for this interrupt and ack them right away - we keep GEN11_MASTER_IRQ
2694 * out as this bit doesn't exist anymore for DG1
2695 */
2696 val = raw_reg_read(regs, GEN11_GFX_MSTR_IRQ) & ~GEN11_MASTER_IRQ;
2697 if (unlikely(!val))
2698 return 0;
2699
2700 raw_reg_write(regs, GEN11_GFX_MSTR_IRQ, val);
2701
2702 return val;
2703 }
2704
2705 static inline void dg1_master_intr_enable(void __iomem * const regs)
2706 {
2707 raw_reg_write(regs, DG1_MSTR_UNIT_INTR, DG1_MSTR_IRQ);
2708 }
2709
2710 static irqreturn_t dg1_irq_handler(int irq, void *arg)
2711 {
2712 return __gen11_irq_handler(arg,
2713 dg1_master_intr_disable_and_ack,
2714 dg1_master_intr_enable);
2715 }
2716
2717 /* Called from drm generic code, passed 'crtc' which
2718 * we use as a pipe index
2719 */
2720 int i8xx_enable_vblank(struct drm_crtc *crtc)
2721 {
2722 struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2723 enum pipe pipe = to_intel_crtc(crtc)->pipe;
2724 unsigned long irqflags;
2725
2726 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2727 i915_enable_pipestat(dev_priv, pipe, PIPE_VBLANK_INTERRUPT_STATUS);
2728 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2729
2730 return 0;
2731 }
2732
2733 int i915gm_enable_vblank(struct drm_crtc *crtc)
2734 {
2735 struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2736
2737 /*
2738 * Vblank interrupts fail to wake the device up from C2+.
2739 * Disabling render clock gating during C-states avoids
2740 * the problem. There is a small power cost so we do this
2741 * only when vblank interrupts are actually enabled.
2742 */
2743 if (dev_priv->vblank_enabled++ == 0)
2744 I915_WRITE(SCPD0, _MASKED_BIT_ENABLE(CSTATE_RENDER_CLOCK_GATE_DISABLE));
2745
2746 return i8xx_enable_vblank(crtc);
2747 }
2748
2749 int i965_enable_vblank(struct drm_crtc *crtc)
2750 {
2751 struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2752 enum pipe pipe = to_intel_crtc(crtc)->pipe;
2753 unsigned long irqflags;
2754
2755 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2756 i915_enable_pipestat(dev_priv, pipe,
2757 PIPE_START_VBLANK_INTERRUPT_STATUS);
2758 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2759
2760 return 0;
2761 }
2762
2763 int ilk_enable_vblank(struct drm_crtc *crtc)
2764 {
2765 struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2766 enum pipe pipe = to_intel_crtc(crtc)->pipe;
2767 unsigned long irqflags;
2768 u32 bit = INTEL_GEN(dev_priv) >= 7 ?
2769 DE_PIPE_VBLANK_IVB(pipe) : DE_PIPE_VBLANK(pipe);
2770
2771 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2772 ilk_enable_display_irq(dev_priv, bit);
2773 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2774
2775 /* Even though there is no DMC, frame counter can get stuck when
2776 * PSR is active as no frames are generated.
2777 */
2778 if (HAS_PSR(dev_priv))
2779 drm_crtc_vblank_restore(crtc);
2780
2781 return 0;
2782 }
2783
2784 static bool gen11_dsi_configure_te(struct intel_crtc *intel_crtc,
2785 bool enable)
2786 {
2787 struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
2788 enum port port;
2789 u32 tmp;
2790
2791 if (!(intel_crtc->mode_flags &
2792 (I915_MODE_FLAG_DSI_USE_TE1 | I915_MODE_FLAG_DSI_USE_TE0)))
2793 return false;
2794
2795 /* for dual link cases we consider TE from slave */
2796 if (intel_crtc->mode_flags & I915_MODE_FLAG_DSI_USE_TE1)
2797 port = PORT_B;
2798 else
2799 port = PORT_A;
2800
2801 tmp = I915_READ(DSI_INTR_MASK_REG(port));
2802 if (enable)
2803 tmp &= ~DSI_TE_EVENT;
2804 else
2805 tmp |= DSI_TE_EVENT;
2806
2807 I915_WRITE(DSI_INTR_MASK_REG(port), tmp);
2808
2809 tmp = I915_READ(DSI_INTR_IDENT_REG(port));
2810 I915_WRITE(DSI_INTR_IDENT_REG(port), tmp);
2811
2812 return true;
2813 }
2814
2815 int bdw_enable_vblank(struct drm_crtc *crtc)
2816 {
2817 struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2818 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2819 enum pipe pipe = intel_crtc->pipe;
2820 unsigned long irqflags;
2821
2822 if (gen11_dsi_configure_te(intel_crtc, true))
2823 return 0;
2824
2825 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2826 bdw_enable_pipe_irq(dev_priv, pipe, GEN8_PIPE_VBLANK);
2827 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2828
2829 /* Even if there is no DMC, frame counter can get stuck when
2830 * PSR is active as no frames are generated, so check only for PSR.
2831 */
2832 if (HAS_PSR(dev_priv))
2833 drm_crtc_vblank_restore(crtc);
2834
2835 return 0;
2836 }
2837
2838 void skl_enable_flip_done(struct intel_crtc *crtc)
2839 {
2840 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2841 enum pipe pipe = crtc->pipe;
2842 unsigned long irqflags;
2843
2844 spin_lock_irqsave(&i915->irq_lock, irqflags);
2845
2846 bdw_enable_pipe_irq(i915, pipe, GEN9_PIPE_PLANE1_FLIP_DONE);
2847
2848 spin_unlock_irqrestore(&i915->irq_lock, irqflags);
2849 }
2850
2851 /* Called from drm generic code, passed 'crtc' which
2852 * we use as a pipe index
2853 */
2854 void i8xx_disable_vblank(struct drm_crtc *crtc)
2855 {
2856 struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2857 enum pipe pipe = to_intel_crtc(crtc)->pipe;
2858 unsigned long irqflags;
2859
2860 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2861 i915_disable_pipestat(dev_priv, pipe, PIPE_VBLANK_INTERRUPT_STATUS);
2862 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2863 }
2864
2865 void i915gm_disable_vblank(struct drm_crtc *crtc)
2866 {
2867 struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2868
2869 i8xx_disable_vblank(crtc);
2870
2871 if (--dev_priv->vblank_enabled == 0)
2872 I915_WRITE(SCPD0, _MASKED_BIT_DISABLE(CSTATE_RENDER_CLOCK_GATE_DISABLE));
2873 }
2874
2875 void i965_disable_vblank(struct drm_crtc *crtc)
2876 {
2877 struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2878 enum pipe pipe = to_intel_crtc(crtc)->pipe;
2879 unsigned long irqflags;
2880
2881 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2882 i915_disable_pipestat(dev_priv, pipe,
2883 PIPE_START_VBLANK_INTERRUPT_STATUS);
2884 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2885 }
2886
2887 void ilk_disable_vblank(struct drm_crtc *crtc)
2888 {
2889 struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2890 enum pipe pipe = to_intel_crtc(crtc)->pipe;
2891 unsigned long irqflags;
2892 u32 bit = INTEL_GEN(dev_priv) >= 7 ?
2893 DE_PIPE_VBLANK_IVB(pipe) : DE_PIPE_VBLANK(pipe);
2894
2895 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2896 ilk_disable_display_irq(dev_priv, bit);
2897 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2898 }
2899
2900 void bdw_disable_vblank(struct drm_crtc *crtc)
2901 {
2902 struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2903 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2904 enum pipe pipe = intel_crtc->pipe;
2905 unsigned long irqflags;
2906
2907 if (gen11_dsi_configure_te(intel_crtc, false))
2908 return;
2909
2910 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2911 bdw_disable_pipe_irq(dev_priv, pipe, GEN8_PIPE_VBLANK);
2912 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2913 }
2914
2915 void skl_disable_flip_done(struct intel_crtc *crtc)
2916 {
2917 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2918 enum pipe pipe = crtc->pipe;
2919 unsigned long irqflags;
2920
2921 spin_lock_irqsave(&i915->irq_lock, irqflags);
2922
2923 bdw_disable_pipe_irq(i915, pipe, GEN9_PIPE_PLANE1_FLIP_DONE);
2924
2925 spin_unlock_irqrestore(&i915->irq_lock, irqflags);
2926 }
2927
2928 static void ibx_irq_reset(struct drm_i915_private *dev_priv)
2929 {
2930 struct intel_uncore *uncore = &dev_priv->uncore;
2931
2932 if (HAS_PCH_NOP(dev_priv))
2933 return;
2934
2935 GEN3_IRQ_RESET(uncore, SDE);
2936
2937 if (HAS_PCH_CPT(dev_priv) || HAS_PCH_LPT(dev_priv))
2938 I915_WRITE(SERR_INT, 0xffffffff);
2939 }
2940
2941 static void vlv_display_irq_reset(struct drm_i915_private *dev_priv)
2942 {
2943 struct intel_uncore *uncore = &dev_priv->uncore;
2944
2945 if (IS_CHERRYVIEW(dev_priv))
2946 intel_uncore_write(uncore, DPINVGTT, DPINVGTT_STATUS_MASK_CHV);
2947 else
2948 intel_uncore_write(uncore, DPINVGTT, DPINVGTT_STATUS_MASK);
2949
2950 i915_hotplug_interrupt_update_locked(dev_priv, 0xffffffff, 0);
2951 intel_uncore_write(uncore, PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2952
2953 i9xx_pipestat_irq_reset(dev_priv);
2954
2955 GEN3_IRQ_RESET(uncore, VLV_);
2956 dev_priv->irq_mask = ~0u;
2957 }
2958
2959 static void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv)
2960 {
2961 struct intel_uncore *uncore = &dev_priv->uncore;
2962
2963 u32 pipestat_mask;
2964 u32 enable_mask;
2965 enum pipe pipe;
2966
2967 pipestat_mask = PIPE_CRC_DONE_INTERRUPT_STATUS;
2968
2969 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
2970 for_each_pipe(dev_priv, pipe)
2971 i915_enable_pipestat(dev_priv, pipe, pipestat_mask);
2972
2973 enable_mask = I915_DISPLAY_PORT_INTERRUPT |
2974 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2975 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2976 I915_LPE_PIPE_A_INTERRUPT |
2977 I915_LPE_PIPE_B_INTERRUPT;
2978
2979 if (IS_CHERRYVIEW(dev_priv))
2980 enable_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT |
2981 I915_LPE_PIPE_C_INTERRUPT;
2982
2983 drm_WARN_ON(&dev_priv->drm, dev_priv->irq_mask != ~0u);
2984
2985 dev_priv->irq_mask = ~enable_mask;
2986
2987 GEN3_IRQ_INIT(uncore, VLV_, dev_priv->irq_mask, enable_mask);
2988 }
2989
2990 /* drm_dma.h hooks
2991 */
2992 static void ilk_irq_reset(struct drm_i915_private *dev_priv)
2993 {
2994 struct intel_uncore *uncore = &dev_priv->uncore;
2995
2996 GEN3_IRQ_RESET(uncore, DE);
2997 dev_priv->irq_mask = ~0u;
2998
2999 if (IS_GEN(dev_priv, 7))
3000 intel_uncore_write(uncore, GEN7_ERR_INT, 0xffffffff);
3001
3002 if (IS_HASWELL(dev_priv)) {
3003 intel_uncore_write(uncore, EDP_PSR_IMR, 0xffffffff);
3004 intel_uncore_write(uncore, EDP_PSR_IIR, 0xffffffff);
3005 }
3006
3007 gen5_gt_irq_reset(&dev_priv->gt);
3008
3009 ibx_irq_reset(dev_priv);
3010 }
3011
3012 static void valleyview_irq_reset(struct drm_i915_private *dev_priv)
3013 {
3014 I915_WRITE(VLV_MASTER_IER, 0);
3015 POSTING_READ(VLV_MASTER_IER);
3016
3017 gen5_gt_irq_reset(&dev_priv->gt);
3018
3019 spin_lock_irq(&dev_priv->irq_lock);
3020 if (dev_priv->display_irqs_enabled)
3021 vlv_display_irq_reset(dev_priv);
3022 spin_unlock_irq(&dev_priv->irq_lock);
3023 }
3024
3025 static void gen8_irq_reset(struct drm_i915_private *dev_priv)
3026 {
3027 struct intel_uncore *uncore = &dev_priv->uncore;
3028 enum pipe pipe;
3029
3030 gen8_master_intr_disable(dev_priv->uncore.regs);
3031
3032 gen8_gt_irq_reset(&dev_priv->gt);
3033
3034 intel_uncore_write(uncore, EDP_PSR_IMR, 0xffffffff);
3035 intel_uncore_write(uncore, EDP_PSR_IIR, 0xffffffff);
3036
3037 for_each_pipe(dev_priv, pipe)
3038 if (intel_display_power_is_enabled(dev_priv,
3039 POWER_DOMAIN_PIPE(pipe)))
3040 GEN8_IRQ_RESET_NDX(uncore, DE_PIPE, pipe);
3041
3042 GEN3_IRQ_RESET(uncore, GEN8_DE_PORT_);
3043 GEN3_IRQ_RESET(uncore, GEN8_DE_MISC_);
3044 GEN3_IRQ_RESET(uncore, GEN8_PCU_);
3045
3046 if (HAS_PCH_SPLIT(dev_priv))
3047 ibx_irq_reset(dev_priv);
3048 }
3049
3050 static void gen11_display_irq_reset(struct drm_i915_private *dev_priv)
3051 {
3052 struct intel_uncore *uncore = &dev_priv->uncore;
3053 enum pipe pipe;
3054 u32 trans_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) |
3055 BIT(TRANSCODER_C) | BIT(TRANSCODER_D);
3056
3057 intel_uncore_write(uncore, GEN11_DISPLAY_INT_CTL, 0);
3058
3059 if (INTEL_GEN(dev_priv) >= 12) {
3060 enum transcoder trans;
3061
3062 for_each_cpu_transcoder_masked(dev_priv, trans, trans_mask) {
3063 enum intel_display_power_domain domain;
3064
3065 domain = POWER_DOMAIN_TRANSCODER(trans);
3066 if (!intel_display_power_is_enabled(dev_priv, domain))
3067 continue;
3068
3069 intel_uncore_write(uncore, TRANS_PSR_IMR(trans), 0xffffffff);
3070 intel_uncore_write(uncore, TRANS_PSR_IIR(trans), 0xffffffff);
3071 }
3072 } else {
3073 intel_uncore_write(uncore, EDP_PSR_IMR, 0xffffffff);
3074 intel_uncore_write(uncore, EDP_PSR_IIR, 0xffffffff);
3075 }
3076
3077 for_each_pipe(dev_priv, pipe)
3078 if (intel_display_power_is_enabled(dev_priv,
3079 POWER_DOMAIN_PIPE(pipe)))
3080 GEN8_IRQ_RESET_NDX(uncore, DE_PIPE, pipe);
3081
3082 GEN3_IRQ_RESET(uncore, GEN8_DE_PORT_);
3083 GEN3_IRQ_RESET(uncore, GEN8_DE_MISC_);
3084 GEN3_IRQ_RESET(uncore, GEN11_DE_HPD_);
3085
3086 if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
3087 GEN3_IRQ_RESET(uncore, SDE);
3088
3089 /* Wa_14010685332:cnp/cmp,tgp,adp */
3090 if (INTEL_PCH_TYPE(dev_priv) == PCH_CNP ||
3091 (INTEL_PCH_TYPE(dev_priv) >= PCH_TGP &&
3092 INTEL_PCH_TYPE(dev_priv) < PCH_DG1)) {
3093 intel_uncore_rmw(uncore, SOUTH_CHICKEN1,
3094 SBCLK_RUN_REFCLK_DIS, SBCLK_RUN_REFCLK_DIS);
3095 intel_uncore_rmw(uncore, SOUTH_CHICKEN1,
3096 SBCLK_RUN_REFCLK_DIS, 0);
3097 }
3098 }
3099
3100 static void gen11_irq_reset(struct drm_i915_private *dev_priv)
3101 {
3102 struct intel_uncore *uncore = &dev_priv->uncore;
3103
3104 if (HAS_MASTER_UNIT_IRQ(dev_priv))
3105 dg1_master_intr_disable_and_ack(dev_priv->uncore.regs);
3106 else
3107 gen11_master_intr_disable(dev_priv->uncore.regs);
3108
3109 gen11_gt_irq_reset(&dev_priv->gt);
3110 gen11_display_irq_reset(dev_priv);
3111
3112 GEN3_IRQ_RESET(uncore, GEN11_GU_MISC_);
3113 GEN3_IRQ_RESET(uncore, GEN8_PCU_);
3114 }
3115
3116 void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv,
3117 u8 pipe_mask)
3118 {
3119 struct intel_uncore *uncore = &dev_priv->uncore;
3120
3121 u32 extra_ier = GEN8_PIPE_VBLANK | GEN8_PIPE_FIFO_UNDERRUN;
3122 enum pipe pipe;
3123
3124 if (INTEL_GEN(dev_priv) >= 9)
3125 extra_ier |= GEN9_PIPE_PLANE1_FLIP_DONE;
3126
3127 spin_lock_irq(&dev_priv->irq_lock);
3128
3129 if (!intel_irqs_enabled(dev_priv)) {
3130 spin_unlock_irq(&dev_priv->irq_lock);
3131 return;
3132 }
3133
3134 for_each_pipe_masked(dev_priv, pipe, pipe_mask)
3135 GEN8_IRQ_INIT_NDX(uncore, DE_PIPE, pipe,
3136 dev_priv->de_irq_mask[pipe],
3137 ~dev_priv->de_irq_mask[pipe] | extra_ier);
3138
3139 spin_unlock_irq(&dev_priv->irq_lock);
3140 }
3141
3142 void gen8_irq_power_well_pre_disable(struct drm_i915_private *dev_priv,
3143 u8 pipe_mask)
3144 {
3145 struct intel_uncore *uncore = &dev_priv->uncore;
3146 enum pipe pipe;
3147
3148 spin_lock_irq(&dev_priv->irq_lock);
3149
3150 if (!intel_irqs_enabled(dev_priv)) {
3151 spin_unlock_irq(&dev_priv->irq_lock);
3152 return;
3153 }
3154
3155 for_each_pipe_masked(dev_priv, pipe, pipe_mask)
3156 GEN8_IRQ_RESET_NDX(uncore, DE_PIPE, pipe);
3157
3158 spin_unlock_irq(&dev_priv->irq_lock);
3159
3160 /* make sure we're done processing display irqs */
3161 intel_synchronize_irq(dev_priv);
3162 }
3163
3164 static void cherryview_irq_reset(struct drm_i915_private *dev_priv)
3165 {
3166 struct intel_uncore *uncore = &dev_priv->uncore;
3167
3168 I915_WRITE(GEN8_MASTER_IRQ, 0);
3169 POSTING_READ(GEN8_MASTER_IRQ);
3170
3171 gen8_gt_irq_reset(&dev_priv->gt);
3172
3173 GEN3_IRQ_RESET(uncore, GEN8_PCU_);
3174
3175 spin_lock_irq(&dev_priv->irq_lock);
3176 if (dev_priv->display_irqs_enabled)
3177 vlv_display_irq_reset(dev_priv);
3178 spin_unlock_irq(&dev_priv->irq_lock);
3179 }
3180
3181 static u32 ibx_hotplug_enables(struct drm_i915_private *i915,
3182 enum hpd_pin pin)
3183 {
3184 switch (pin) {
3185 case HPD_PORT_A:
3186 /*
3187 * When CPU and PCH are on the same package, port A
3188 * HPD must be enabled in both north and south.
3189 */
3190 return HAS_PCH_LPT_LP(i915) ?
3191 PORTA_HOTPLUG_ENABLE : 0;
3192 case HPD_PORT_B:
3193 return PORTB_HOTPLUG_ENABLE |
3194 PORTB_PULSE_DURATION_2ms;
3195 case HPD_PORT_C:
3196 return PORTC_HOTPLUG_ENABLE |
3197 PORTC_PULSE_DURATION_2ms;
3198 case HPD_PORT_D:
3199 return PORTD_HOTPLUG_ENABLE |
3200 PORTD_PULSE_DURATION_2ms;
3201 default:
3202 return 0;
3203 }
3204 }
3205
3206 static void ibx_hpd_detection_setup(struct drm_i915_private *dev_priv)
3207 {
3208 u32 hotplug;
3209
3210 /*
3211 * Enable digital hotplug on the PCH, and configure the DP short pulse
3212 * duration to 2ms (which is the minimum in the Display Port spec).
3213 * The pulse duration bits are reserved on LPT+.
3214 */
3215 hotplug = I915_READ(PCH_PORT_HOTPLUG);
3216 hotplug &= ~(PORTA_HOTPLUG_ENABLE |
3217 PORTB_HOTPLUG_ENABLE |
3218 PORTC_HOTPLUG_ENABLE |
3219 PORTD_HOTPLUG_ENABLE |
3220 PORTB_PULSE_DURATION_MASK |
3221 PORTC_PULSE_DURATION_MASK |
3222 PORTD_PULSE_DURATION_MASK);
3223 hotplug |= intel_hpd_hotplug_enables(dev_priv, ibx_hotplug_enables);
3224 I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3225 }
3226
3227 static void ibx_hpd_irq_setup(struct drm_i915_private *dev_priv)
3228 {
3229 u32 hotplug_irqs, enabled_irqs;
3230
3231 enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->hotplug.pch_hpd);
3232 hotplug_irqs = intel_hpd_hotplug_irqs(dev_priv, dev_priv->hotplug.pch_hpd);
3233
3234 ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
3235
3236 ibx_hpd_detection_setup(dev_priv);
3237 }
3238
3239 static u32 icp_ddi_hotplug_enables(struct drm_i915_private *i915,
3240 enum hpd_pin pin)
3241 {
3242 switch (pin) {
3243 case HPD_PORT_A:
3244 case HPD_PORT_B:
3245 case HPD_PORT_C:
3246 case HPD_PORT_D:
3247 return SHOTPLUG_CTL_DDI_HPD_ENABLE(pin);
3248 default:
3249 return 0;
3250 }
3251 }
3252
3253 static u32 icp_tc_hotplug_enables(struct drm_i915_private *i915,
3254 enum hpd_pin pin)
3255 {
3256 switch (pin) {
3257 case HPD_PORT_TC1:
3258 case HPD_PORT_TC2:
3259 case HPD_PORT_TC3:
3260 case HPD_PORT_TC4:
3261 case HPD_PORT_TC5:
3262 case HPD_PORT_TC6:
3263 return ICP_TC_HPD_ENABLE(pin);
3264 default:
3265 return 0;
3266 }
3267 }
3268
3269 static void icp_ddi_hpd_detection_setup(struct drm_i915_private *dev_priv)
3270 {
3271 u32 hotplug;
3272
3273 hotplug = I915_READ(SHOTPLUG_CTL_DDI);
3274 hotplug &= ~(SHOTPLUG_CTL_DDI_HPD_ENABLE(HPD_PORT_A) |
3275 SHOTPLUG_CTL_DDI_HPD_ENABLE(HPD_PORT_B) |
3276 SHOTPLUG_CTL_DDI_HPD_ENABLE(HPD_PORT_C) |
3277 SHOTPLUG_CTL_DDI_HPD_ENABLE(HPD_PORT_D));
3278 hotplug |= intel_hpd_hotplug_enables(dev_priv, icp_ddi_hotplug_enables);
3279 I915_WRITE(SHOTPLUG_CTL_DDI, hotplug);
3280 }
3281
3282 static void icp_tc_hpd_detection_setup(struct drm_i915_private *dev_priv)
3283 {
3284 u32 hotplug;
3285
3286 hotplug = I915_READ(SHOTPLUG_CTL_TC);
3287 hotplug &= ~(ICP_TC_HPD_ENABLE(HPD_PORT_TC1) |
3288 ICP_TC_HPD_ENABLE(HPD_PORT_TC2) |
3289 ICP_TC_HPD_ENABLE(HPD_PORT_TC3) |
3290 ICP_TC_HPD_ENABLE(HPD_PORT_TC4) |
3291 ICP_TC_HPD_ENABLE(HPD_PORT_TC5) |
3292 ICP_TC_HPD_ENABLE(HPD_PORT_TC6));
3293 hotplug |= intel_hpd_hotplug_enables(dev_priv, icp_tc_hotplug_enables);
3294 I915_WRITE(SHOTPLUG_CTL_TC, hotplug);
3295 }
3296
3297 static void icp_hpd_irq_setup(struct drm_i915_private *dev_priv)
3298 {
3299 u32 hotplug_irqs, enabled_irqs;
3300
3301 enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->hotplug.pch_hpd);
3302 hotplug_irqs = intel_hpd_hotplug_irqs(dev_priv, dev_priv->hotplug.pch_hpd);
3303
3304 if (INTEL_PCH_TYPE(dev_priv) <= PCH_TGP)
3305 I915_WRITE(SHPD_FILTER_CNT, SHPD_FILTER_CNT_500_ADJ);
3306
3307 ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
3308
3309 icp_ddi_hpd_detection_setup(dev_priv);
3310 icp_tc_hpd_detection_setup(dev_priv);
3311 }
3312
3313 static u32 gen11_hotplug_enables(struct drm_i915_private *i915,
3314 enum hpd_pin pin)
3315 {
3316 switch (pin) {
3317 case HPD_PORT_TC1:
3318 case HPD_PORT_TC2:
3319 case HPD_PORT_TC3:
3320 case HPD_PORT_TC4:
3321 case HPD_PORT_TC5:
3322 case HPD_PORT_TC6:
3323 return GEN11_HOTPLUG_CTL_ENABLE(pin);
3324 default:
3325 return 0;
3326 }
3327 }
3328
3329 static void dg1_hpd_irq_setup(struct drm_i915_private *dev_priv)
3330 {
3331 u32 val;
3332
3333 val = I915_READ(SOUTH_CHICKEN1);
3334 val |= (INVERT_DDIA_HPD |
3335 INVERT_DDIB_HPD |
3336 INVERT_DDIC_HPD |
3337 INVERT_DDID_HPD);
3338 I915_WRITE(SOUTH_CHICKEN1, val);
3339
3340 icp_hpd_irq_setup(dev_priv);
3341 }
3342
3343 static void gen11_tc_hpd_detection_setup(struct drm_i915_private *dev_priv)
3344 {
3345 u32 hotplug;
3346
3347 hotplug = I915_READ(GEN11_TC_HOTPLUG_CTL);
3348 hotplug &= ~(GEN11_HOTPLUG_CTL_ENABLE(HPD_PORT_TC1) |
3349 GEN11_HOTPLUG_CTL_ENABLE(HPD_PORT_TC2) |
3350 GEN11_HOTPLUG_CTL_ENABLE(HPD_PORT_TC3) |
3351 GEN11_HOTPLUG_CTL_ENABLE(HPD_PORT_TC4) |
3352 GEN11_HOTPLUG_CTL_ENABLE(HPD_PORT_TC5) |
3353 GEN11_HOTPLUG_CTL_ENABLE(HPD_PORT_TC6));
3354 hotplug |= intel_hpd_hotplug_enables(dev_priv, gen11_hotplug_enables);
3355 I915_WRITE(GEN11_TC_HOTPLUG_CTL, hotplug);
3356 }
3357
3358 static void gen11_tbt_hpd_detection_setup(struct drm_i915_private *dev_priv)
3359 {
3360 u32 hotplug;
3361
3362 hotplug = I915_READ(GEN11_TBT_HOTPLUG_CTL);
3363 hotplug &= ~(GEN11_HOTPLUG_CTL_ENABLE(HPD_PORT_TC1) |
3364 GEN11_HOTPLUG_CTL_ENABLE(HPD_PORT_TC2) |
3365 GEN11_HOTPLUG_CTL_ENABLE(HPD_PORT_TC3) |
3366 GEN11_HOTPLUG_CTL_ENABLE(HPD_PORT_TC4) |
3367 GEN11_HOTPLUG_CTL_ENABLE(HPD_PORT_TC5) |
3368 GEN11_HOTPLUG_CTL_ENABLE(HPD_PORT_TC6));
3369 hotplug |= intel_hpd_hotplug_enables(dev_priv, gen11_hotplug_enables);
3370 I915_WRITE(GEN11_TBT_HOTPLUG_CTL, hotplug);
3371 }
3372
3373 static void gen11_hpd_irq_setup(struct drm_i915_private *dev_priv)
3374 {
3375 u32 hotplug_irqs, enabled_irqs;
3376 u32 val;
3377
3378 enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->hotplug.hpd);
3379 hotplug_irqs = intel_hpd_hotplug_irqs(dev_priv, dev_priv->hotplug.hpd);
3380
3381 val = I915_READ(GEN11_DE_HPD_IMR);
3382 val &= ~hotplug_irqs;
3383 val |= ~enabled_irqs & hotplug_irqs;
3384 I915_WRITE(GEN11_DE_HPD_IMR, val);
3385 POSTING_READ(GEN11_DE_HPD_IMR);
3386
3387 gen11_tc_hpd_detection_setup(dev_priv);
3388 gen11_tbt_hpd_detection_setup(dev_priv);
3389
3390 if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
3391 icp_hpd_irq_setup(dev_priv);
3392 }
3393
3394 static u32 spt_hotplug_enables(struct drm_i915_private *i915,
3395 enum hpd_pin pin)
3396 {
3397 switch (pin) {
3398 case HPD_PORT_A:
3399 return PORTA_HOTPLUG_ENABLE;
3400 case HPD_PORT_B:
3401 return PORTB_HOTPLUG_ENABLE;
3402 case HPD_PORT_C:
3403 return PORTC_HOTPLUG_ENABLE;
3404 case HPD_PORT_D:
3405 return PORTD_HOTPLUG_ENABLE;
3406 default:
3407 return 0;
3408 }
3409 }
3410
3411 static u32 spt_hotplug2_enables(struct drm_i915_private *i915,
3412 enum hpd_pin pin)
3413 {
3414 switch (pin) {
3415 case HPD_PORT_E:
3416 return PORTE_HOTPLUG_ENABLE;
3417 default:
3418 return 0;
3419 }
3420 }
3421
3422 static void spt_hpd_detection_setup(struct drm_i915_private *dev_priv)
3423 {
3424 u32 val, hotplug;
3425
3426 /* Display WA #1179 WaHardHangonHotPlug: cnp */
3427 if (HAS_PCH_CNP(dev_priv)) {
3428 val = I915_READ(SOUTH_CHICKEN1);
3429 val &= ~CHASSIS_CLK_REQ_DURATION_MASK;
3430 val |= CHASSIS_CLK_REQ_DURATION(0xf);
3431 I915_WRITE(SOUTH_CHICKEN1, val);
3432 }
3433
3434 /* Enable digital hotplug on the PCH */
3435 hotplug = I915_READ(PCH_PORT_HOTPLUG);
3436 hotplug &= ~(PORTA_HOTPLUG_ENABLE |
3437 PORTB_HOTPLUG_ENABLE |
3438 PORTC_HOTPLUG_ENABLE |
3439 PORTD_HOTPLUG_ENABLE);
3440 hotplug |= intel_hpd_hotplug_enables(dev_priv, spt_hotplug_enables);
3441 I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3442
3443 hotplug = I915_READ(PCH_PORT_HOTPLUG2);
3444 hotplug &= ~PORTE_HOTPLUG_ENABLE;
3445 hotplug |= intel_hpd_hotplug_enables(dev_priv, spt_hotplug2_enables);
3446 I915_WRITE(PCH_PORT_HOTPLUG2, hotplug);
3447 }
3448
3449 static void spt_hpd_irq_setup(struct drm_i915_private *dev_priv)
3450 {
3451 u32 hotplug_irqs, enabled_irqs;
3452
3453 if (INTEL_PCH_TYPE(dev_priv) >= PCH_CNP)
3454 I915_WRITE(SHPD_FILTER_CNT, SHPD_FILTER_CNT_500_ADJ);
3455
3456 enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->hotplug.pch_hpd);
3457 hotplug_irqs = intel_hpd_hotplug_irqs(dev_priv, dev_priv->hotplug.pch_hpd);
3458
3459 ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
3460
3461 spt_hpd_detection_setup(dev_priv);
3462 }
3463
3464 static u32 ilk_hotplug_enables(struct drm_i915_private *i915,
3465 enum hpd_pin pin)
3466 {
3467 switch (pin) {
3468 case HPD_PORT_A:
3469 return DIGITAL_PORTA_HOTPLUG_ENABLE |
3470 DIGITAL_PORTA_PULSE_DURATION_2ms;
3471 default:
3472 return 0;
3473 }
3474 }
3475
3476 static void ilk_hpd_detection_setup(struct drm_i915_private *dev_priv)
3477 {
3478 u32 hotplug;
3479
3480 /*
3481 * Enable digital hotplug on the CPU, and configure the DP short pulse
3482 * duration to 2ms (which is the minimum in the Display Port spec)
3483 * The pulse duration bits are reserved on HSW+.
3484 */
3485 hotplug = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
3486 hotplug &= ~(DIGITAL_PORTA_HOTPLUG_ENABLE |
3487 DIGITAL_PORTA_PULSE_DURATION_MASK);
3488 hotplug |= intel_hpd_hotplug_enables(dev_priv, ilk_hotplug_enables);
3489 I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, hotplug);
3490 }
3491
3492 static void ilk_hpd_irq_setup(struct drm_i915_private *dev_priv)
3493 {
3494 u32 hotplug_irqs, enabled_irqs;
3495
3496 enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->hotplug.hpd);
3497 hotplug_irqs = intel_hpd_hotplug_irqs(dev_priv, dev_priv->hotplug.hpd);
3498
3499 if (INTEL_GEN(dev_priv) >= 8)
3500 bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
3501 else
3502 ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
3503
3504 ilk_hpd_detection_setup(dev_priv);
3505
3506 ibx_hpd_irq_setup(dev_priv);
3507 }
3508
3509 static u32 bxt_hotplug_enables(struct drm_i915_private *i915,
3510 enum hpd_pin pin)
3511 {
3512 u32 hotplug;
3513
3514 switch (pin) {
3515 case HPD_PORT_A:
3516 hotplug = PORTA_HOTPLUG_ENABLE;
3517 if (intel_bios_is_port_hpd_inverted(i915, PORT_A))
3518 hotplug |= BXT_DDIA_HPD_INVERT;
3519 return hotplug;
3520 case HPD_PORT_B:
3521 hotplug = PORTB_HOTPLUG_ENABLE;
3522 if (intel_bios_is_port_hpd_inverted(i915, PORT_B))
3523 hotplug |= BXT_DDIB_HPD_INVERT;
3524 return hotplug;
3525 case HPD_PORT_C:
3526 hotplug = PORTC_HOTPLUG_ENABLE;
3527 if (intel_bios_is_port_hpd_inverted(i915, PORT_C))
3528 hotplug |= BXT_DDIC_HPD_INVERT;
3529 return hotplug;
3530 default:
3531 return 0;
3532 }
3533 }
3534
3535 static void bxt_hpd_detection_setup(struct drm_i915_private *dev_priv)
3536 {
3537 u32 hotplug;
3538
3539 hotplug = I915_READ(PCH_PORT_HOTPLUG);
3540 hotplug &= ~(PORTA_HOTPLUG_ENABLE |
3541 PORTB_HOTPLUG_ENABLE |
3542 PORTC_HOTPLUG_ENABLE |
3543 BXT_DDIA_HPD_INVERT |
3544 BXT_DDIB_HPD_INVERT |
3545 BXT_DDIC_HPD_INVERT);
3546 hotplug |= intel_hpd_hotplug_enables(dev_priv, bxt_hotplug_enables);
3547 I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3548 }
3549
3550 static void bxt_hpd_irq_setup(struct drm_i915_private *dev_priv)
3551 {
3552 u32 hotplug_irqs, enabled_irqs;
3553
3554 enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->hotplug.hpd);
3555 hotplug_irqs = intel_hpd_hotplug_irqs(dev_priv, dev_priv->hotplug.hpd);
3556
3557 bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
3558
3559 bxt_hpd_detection_setup(dev_priv);
3560 }
3561
3562 /*
3563 * SDEIER is also touched by the interrupt handler to work around missed PCH
3564 * interrupts. Hence we can't update it after the interrupt handler is enabled -
3565 * instead we unconditionally enable all PCH interrupt sources here, but then
3566 * only unmask them as needed with SDEIMR.
3567 *
3568 * Note that we currently do this after installing the interrupt handler,
3569 * but before we enable the master interrupt. That should be sufficient
3570 * to avoid races with the irq handler, assuming we have MSI. Shared legacy
3571 * interrupts could still race.
3572 */
3573 static void ibx_irq_postinstall(struct drm_i915_private *dev_priv)
3574 {
3575 struct intel_uncore *uncore = &dev_priv->uncore;
3576 u32 mask;
3577
3578 if (HAS_PCH_NOP(dev_priv))
3579 return;
3580
3581 if (HAS_PCH_IBX(dev_priv))
3582 mask = SDE_GMBUS | SDE_AUX_MASK | SDE_POISON;
3583 else if (HAS_PCH_CPT(dev_priv) || HAS_PCH_LPT(dev_priv))
3584 mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT;
3585 else
3586 mask = SDE_GMBUS_CPT;
3587
3588 GEN3_IRQ_INIT(uncore, SDE, ~mask, 0xffffffff);
3589 }
3590
3591 static void ilk_irq_postinstall(struct drm_i915_private *dev_priv)
3592 {
3593 struct intel_uncore *uncore = &dev_priv->uncore;
3594 u32 display_mask, extra_mask;
3595
3596 if (INTEL_GEN(dev_priv) >= 7) {
3597 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
3598 DE_PCH_EVENT_IVB | DE_AUX_CHANNEL_A_IVB);
3599 extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB |
3600 DE_PIPEA_VBLANK_IVB | DE_ERR_INT_IVB |
3601 DE_DP_A_HOTPLUG_IVB);
3602 } else {
3603 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
3604 DE_AUX_CHANNEL_A | DE_PIPEB_CRC_DONE |
3605 DE_PIPEA_CRC_DONE | DE_POISON);
3606 extra_mask = (DE_PIPEA_VBLANK | DE_PIPEB_VBLANK |
3607 DE_PIPEB_FIFO_UNDERRUN | DE_PIPEA_FIFO_UNDERRUN |
3608 DE_DP_A_HOTPLUG);
3609 }
3610
3611 if (IS_HASWELL(dev_priv)) {
3612 gen3_assert_iir_is_zero(uncore, EDP_PSR_IIR);
3613 display_mask |= DE_EDP_PSR_INT_HSW;
3614 }
3615
3616 if (IS_IRONLAKE_M(dev_priv))
3617 extra_mask |= DE_PCU_EVENT;
3618
3619 dev_priv->irq_mask = ~display_mask;
3620
3621 ibx_irq_postinstall(dev_priv);
3622
3623 gen5_gt_irq_postinstall(&dev_priv->gt);
3624
3625 GEN3_IRQ_INIT(uncore, DE, dev_priv->irq_mask,
3626 display_mask | extra_mask);
3627 }
3628
3629 void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv)
3630 {
3631 lockdep_assert_held(&dev_priv->irq_lock);
3632
3633 if (dev_priv->display_irqs_enabled)
3634 return;
3635
3636 dev_priv->display_irqs_enabled = true;
3637
3638 if (intel_irqs_enabled(dev_priv)) {
3639 vlv_display_irq_reset(dev_priv);
3640 vlv_display_irq_postinstall(dev_priv);
3641 }
3642 }
3643
3644 void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv)
3645 {
3646 lockdep_assert_held(&dev_priv->irq_lock);
3647
3648 if (!dev_priv->display_irqs_enabled)
3649 return;
3650
3651 dev_priv->display_irqs_enabled = false;
3652
3653 if (intel_irqs_enabled(dev_priv))
3654 vlv_display_irq_reset(dev_priv);
3655 }
3656
3657
3658 static void valleyview_irq_postinstall(struct drm_i915_private *dev_priv)
3659 {
3660 gen5_gt_irq_postinstall(&dev_priv->gt);
3661
3662 spin_lock_irq(&dev_priv->irq_lock);
3663 if (dev_priv->display_irqs_enabled)
3664 vlv_display_irq_postinstall(dev_priv);
3665 spin_unlock_irq(&dev_priv->irq_lock);
3666
3667 I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
3668 POSTING_READ(VLV_MASTER_IER);
3669 }
3670
3671 static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
3672 {
3673 struct intel_uncore *uncore = &dev_priv->uncore;
3674
3675 u32 de_pipe_masked = gen8_de_pipe_fault_mask(dev_priv) |
3676 GEN8_PIPE_CDCLK_CRC_DONE;
3677 u32 de_pipe_enables;
3678 u32 de_port_masked = gen8_de_port_aux_mask(dev_priv);
3679 u32 de_port_enables;
3680 u32 de_misc_masked = GEN8_DE_EDP_PSR;
3681 u32 trans_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) |
3682 BIT(TRANSCODER_C) | BIT(TRANSCODER_D);
3683 enum pipe pipe;
3684
3685 if (INTEL_GEN(dev_priv) <= 10)
3686 de_misc_masked |= GEN8_DE_MISC_GSE;
3687
3688 if (IS_GEN9_LP(dev_priv))
3689 de_port_masked |= BXT_DE_PORT_GMBUS;
3690
3691 if (INTEL_GEN(dev_priv) >= 11) {
3692 enum port port;
3693
3694 if (intel_bios_is_dsi_present(dev_priv, &port))
3695 de_port_masked |= DSI0_TE | DSI1_TE;
3696 }
3697
3698 de_pipe_enables = de_pipe_masked | GEN8_PIPE_VBLANK |
3699 GEN8_PIPE_FIFO_UNDERRUN;
3700
3701 if (INTEL_GEN(dev_priv) >= 9)
3702 de_pipe_enables |= GEN9_PIPE_PLANE1_FLIP_DONE;
3703
3704 de_port_enables = de_port_masked;
3705 if (IS_GEN9_LP(dev_priv))
3706 de_port_enables |= BXT_DE_PORT_HOTPLUG_MASK;
3707 else if (IS_BROADWELL(dev_priv))
3708 de_port_enables |= BDW_DE_PORT_HOTPLUG_MASK;
3709
3710 if (INTEL_GEN(dev_priv) >= 12) {
3711 enum transcoder trans;
3712
3713 for_each_cpu_transcoder_masked(dev_priv, trans, trans_mask) {
3714 enum intel_display_power_domain domain;
3715
3716 domain = POWER_DOMAIN_TRANSCODER(trans);
3717 if (!intel_display_power_is_enabled(dev_priv, domain))
3718 continue;
3719
3720 gen3_assert_iir_is_zero(uncore, TRANS_PSR_IIR(trans));
3721 }
3722 } else {
3723 gen3_assert_iir_is_zero(uncore, EDP_PSR_IIR);
3724 }
3725
3726 for_each_pipe(dev_priv, pipe) {
3727 dev_priv->de_irq_mask[pipe] = ~de_pipe_masked;
3728
3729 if (intel_display_power_is_enabled(dev_priv,
3730 POWER_DOMAIN_PIPE(pipe)))
3731 GEN8_IRQ_INIT_NDX(uncore, DE_PIPE, pipe,
3732 dev_priv->de_irq_mask[pipe],
3733 de_pipe_enables);
3734 }
3735
3736 GEN3_IRQ_INIT(uncore, GEN8_DE_PORT_, ~de_port_masked, de_port_enables);
3737 GEN3_IRQ_INIT(uncore, GEN8_DE_MISC_, ~de_misc_masked, de_misc_masked);
3738
3739 if (INTEL_GEN(dev_priv) >= 11) {
3740 u32 de_hpd_masked = 0;
3741 u32 de_hpd_enables = GEN11_DE_TC_HOTPLUG_MASK |
3742 GEN11_DE_TBT_HOTPLUG_MASK;
3743
3744 GEN3_IRQ_INIT(uncore, GEN11_DE_HPD_, ~de_hpd_masked,
3745 de_hpd_enables);
3746 }
3747 }
3748
3749 static void gen8_irq_postinstall(struct drm_i915_private *dev_priv)
3750 {
3751 if (HAS_PCH_SPLIT(dev_priv))
3752 ibx_irq_postinstall(dev_priv);
3753
3754 gen8_gt_irq_postinstall(&dev_priv->gt);
3755 gen8_de_irq_postinstall(dev_priv);
3756
3757 gen8_master_intr_enable(dev_priv->uncore.regs);
3758 }
3759
3760 static void icp_irq_postinstall(struct drm_i915_private *dev_priv)
3761 {
3762 struct intel_uncore *uncore = &dev_priv->uncore;
3763 u32 mask = SDE_GMBUS_ICP;
3764
3765 GEN3_IRQ_INIT(uncore, SDE, ~mask, 0xffffffff);
3766 }
3767
3768 static void gen11_irq_postinstall(struct drm_i915_private *dev_priv)
3769 {
3770 struct intel_uncore *uncore = &dev_priv->uncore;
3771 u32 gu_misc_masked = GEN11_GU_MISC_GSE;
3772
3773 if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
3774 icp_irq_postinstall(dev_priv);
3775
3776 gen11_gt_irq_postinstall(&dev_priv->gt);
3777 gen8_de_irq_postinstall(dev_priv);
3778
3779 GEN3_IRQ_INIT(uncore, GEN11_GU_MISC_, ~gu_misc_masked, gu_misc_masked);
3780
3781 I915_WRITE(GEN11_DISPLAY_INT_CTL, GEN11_DISPLAY_IRQ_ENABLE);
3782
3783 if (HAS_MASTER_UNIT_IRQ(dev_priv)) {
3784 dg1_master_intr_enable(uncore->regs);
3785 POSTING_READ(DG1_MSTR_UNIT_INTR);
3786 } else {
3787 gen11_master_intr_enable(uncore->regs);
3788 POSTING_READ(GEN11_GFX_MSTR_IRQ);
3789 }
3790 }
3791
3792 static void cherryview_irq_postinstall(struct drm_i915_private *dev_priv)
3793 {
3794 gen8_gt_irq_postinstall(&dev_priv->gt);
3795
3796 spin_lock_irq(&dev_priv->irq_lock);
3797 if (dev_priv->display_irqs_enabled)
3798 vlv_display_irq_postinstall(dev_priv);
3799 spin_unlock_irq(&dev_priv->irq_lock);
3800
3801 I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
3802 POSTING_READ(GEN8_MASTER_IRQ);
3803 }
3804
3805 static void i8xx_irq_reset(struct drm_i915_private *dev_priv)
3806 {
3807 struct intel_uncore *uncore = &dev_priv->uncore;
3808
3809 i9xx_pipestat_irq_reset(dev_priv);
3810
3811 GEN2_IRQ_RESET(uncore);
3812 dev_priv->irq_mask = ~0u;
3813 }
3814
3815 static void i8xx_irq_postinstall(struct drm_i915_private *dev_priv)
3816 {
3817 struct intel_uncore *uncore = &dev_priv->uncore;
3818 u16 enable_mask;
3819
3820 intel_uncore_write16(uncore,
3821 EMR,
3822 ~(I915_ERROR_PAGE_TABLE |
3823 I915_ERROR_MEMORY_REFRESH));
3824
3825 /* Unmask the interrupts that we always want on. */
3826 dev_priv->irq_mask =
3827 ~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3828 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3829 I915_MASTER_ERROR_INTERRUPT);
3830
3831 enable_mask =
3832 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3833 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3834 I915_MASTER_ERROR_INTERRUPT |
3835 I915_USER_INTERRUPT;
3836
3837 GEN2_IRQ_INIT(uncore, dev_priv->irq_mask, enable_mask);
3838
3839 /* Interrupt setup is already guaranteed to be single-threaded, this is
3840 * just to make the assert_spin_locked check happy. */
3841 spin_lock_irq(&dev_priv->irq_lock);
3842 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
3843 i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3844 spin_unlock_irq(&dev_priv->irq_lock);
3845 }
3846
3847 static void i8xx_error_irq_ack(struct drm_i915_private *i915,
3848 u16 *eir, u16 *eir_stuck)
3849 {
3850 struct intel_uncore *uncore = &i915->uncore;
3851 u16 emr;
3852
3853 *eir = intel_uncore_read16(uncore, EIR);
3854
3855 if (*eir)
3856 intel_uncore_write16(uncore, EIR, *eir);
3857
3858 *eir_stuck = intel_uncore_read16(uncore, EIR);
3859 if (*eir_stuck == 0)
3860 return;
3861
3862 /*
3863 * Toggle all EMR bits to make sure we get an edge
3864 * in the ISR master error bit if we don't clear
3865 * all the EIR bits. Otherwise the edge triggered
3866 * IIR on i965/g4x wouldn't notice that an interrupt
3867 * is still pending. Also some EIR bits can't be
3868 * cleared except by handling the underlying error
3869 * (or by a GPU reset) so we mask any bit that
3870 * remains set.
3871 */
3872 emr = intel_uncore_read16(uncore, EMR);
3873 intel_uncore_write16(uncore, EMR, 0xffff);
3874 intel_uncore_write16(uncore, EMR, emr | *eir_stuck);
3875 }
3876
3877 static void i8xx_error_irq_handler(struct drm_i915_private *dev_priv,
3878 u16 eir, u16 eir_stuck)
3879 {
3880 DRM_DEBUG("Master Error: EIR 0x%04x\n", eir);
3881
3882 if (eir_stuck)
3883 drm_dbg(&dev_priv->drm, "EIR stuck: 0x%04x, masked\n",
3884 eir_stuck);
3885 }
3886
3887 static void i9xx_error_irq_ack(struct drm_i915_private *dev_priv,
3888 u32 *eir, u32 *eir_stuck)
3889 {
3890 u32 emr;
3891
3892 *eir = I915_READ(EIR);
3893
3894 I915_WRITE(EIR, *eir);
3895
3896 *eir_stuck = I915_READ(EIR);
3897 if (*eir_stuck == 0)
3898 return;
3899
3900 /*
3901 * Toggle all EMR bits to make sure we get an edge
3902 * in the ISR master error bit if we don't clear
3903 * all the EIR bits. Otherwise the edge triggered
3904 * IIR on i965/g4x wouldn't notice that an interrupt
3905 * is still pending. Also some EIR bits can't be
3906 * cleared except by handling the underlying error
3907 * (or by a GPU reset) so we mask any bit that
3908 * remains set.
3909 */
3910 emr = I915_READ(EMR);
3911 I915_WRITE(EMR, 0xffffffff);
3912 I915_WRITE(EMR, emr | *eir_stuck);
3913 }
3914
3915 static void i9xx_error_irq_handler(struct drm_i915_private *dev_priv,
3916 u32 eir, u32 eir_stuck)
3917 {
3918 DRM_DEBUG("Master Error, EIR 0x%08x\n", eir);
3919
3920 if (eir_stuck)
3921 drm_dbg(&dev_priv->drm, "EIR stuck: 0x%08x, masked\n",
3922 eir_stuck);
3923 }
3924
3925 static irqreturn_t i8xx_irq_handler(int irq, void *arg)
3926 {
3927 struct drm_i915_private *dev_priv = arg;
3928 irqreturn_t ret = IRQ_NONE;
3929
3930 if (!intel_irqs_enabled(dev_priv))
3931 return IRQ_NONE;
3932
3933 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
3934 disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
3935
3936 do {
3937 u32 pipe_stats[I915_MAX_PIPES] = {};
3938 u16 eir = 0, eir_stuck = 0;
3939 u16 iir;
3940
3941 iir = intel_uncore_read16(&dev_priv->uncore, GEN2_IIR);
3942 if (iir == 0)
3943 break;
3944
3945 ret = IRQ_HANDLED;
3946
3947 /* Call regardless, as some status bits might not be
3948 * signalled in iir */
3949 i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
3950
3951 if (iir & I915_MASTER_ERROR_INTERRUPT)
3952 i8xx_error_irq_ack(dev_priv, &eir, &eir_stuck);
3953
3954 intel_uncore_write16(&dev_priv->uncore, GEN2_IIR, iir);
3955
3956 if (iir & I915_USER_INTERRUPT)
3957 intel_engine_signal_breadcrumbs(dev_priv->gt.engine[RCS0]);
3958
3959 if (iir & I915_MASTER_ERROR_INTERRUPT)
3960 i8xx_error_irq_handler(dev_priv, eir, eir_stuck);
3961
3962 i8xx_pipestat_irq_handler(dev_priv, iir, pipe_stats);
3963 } while (0);
3964
3965 pmu_irq_stats(dev_priv, ret);
3966
3967 enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
3968
3969 return ret;
3970 }
3971
3972 static void i915_irq_reset(struct drm_i915_private *dev_priv)
3973 {
3974 struct intel_uncore *uncore = &dev_priv->uncore;
3975
3976 if (I915_HAS_HOTPLUG(dev_priv)) {
3977 i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3978 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3979 }
3980
3981 i9xx_pipestat_irq_reset(dev_priv);
3982
3983 GEN3_IRQ_RESET(uncore, GEN2_);
3984 dev_priv->irq_mask = ~0u;
3985 }
3986
3987 static void i915_irq_postinstall(struct drm_i915_private *dev_priv)
3988 {
3989 struct intel_uncore *uncore = &dev_priv->uncore;
3990 u32 enable_mask;
3991
3992 I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE |
3993 I915_ERROR_MEMORY_REFRESH));
3994
3995 /* Unmask the interrupts that we always want on. */
3996 dev_priv->irq_mask =
3997 ~(I915_ASLE_INTERRUPT |
3998 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3999 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
4000 I915_MASTER_ERROR_INTERRUPT);
4001
4002 enable_mask =
4003 I915_ASLE_INTERRUPT |
4004 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
4005 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
4006 I915_MASTER_ERROR_INTERRUPT |
4007 I915_USER_INTERRUPT;
4008
4009 if (I915_HAS_HOTPLUG(dev_priv)) {
4010 /* Enable in IER... */
4011 enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
4012 /* and unmask in IMR */
4013 dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
4014 }
4015
4016 GEN3_IRQ_INIT(uncore, GEN2_, dev_priv->irq_mask, enable_mask);
4017
4018 /* Interrupt setup is already guaranteed to be single-threaded, this is
4019 * just to make the assert_spin_locked check happy. */
4020 spin_lock_irq(&dev_priv->irq_lock);
4021 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
4022 i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
4023 spin_unlock_irq(&dev_priv->irq_lock);
4024
4025 i915_enable_asle_pipestat(dev_priv);
4026 }
4027
4028 static irqreturn_t i915_irq_handler(int irq, void *arg)
4029 {
4030 struct drm_i915_private *dev_priv = arg;
4031 irqreturn_t ret = IRQ_NONE;
4032
4033 if (!intel_irqs_enabled(dev_priv))
4034 return IRQ_NONE;
4035
4036 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
4037 disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
4038
4039 do {
4040 u32 pipe_stats[I915_MAX_PIPES] = {};
4041 u32 eir = 0, eir_stuck = 0;
4042 u32 hotplug_status = 0;
4043 u32 iir;
4044
4045 iir = I915_READ(GEN2_IIR);
4046 if (iir == 0)
4047 break;
4048
4049 ret = IRQ_HANDLED;
4050
4051 if (I915_HAS_HOTPLUG(dev_priv) &&
4052 iir & I915_DISPLAY_PORT_INTERRUPT)
4053 hotplug_status = i9xx_hpd_irq_ack(dev_priv);
4054
4055 /* Call regardless, as some status bits might not be
4056 * signalled in iir */
4057 i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
4058
4059 if (iir & I915_MASTER_ERROR_INTERRUPT)
4060 i9xx_error_irq_ack(dev_priv, &eir, &eir_stuck);
4061
4062 I915_WRITE(GEN2_IIR, iir);
4063
4064 if (iir & I915_USER_INTERRUPT)
4065 intel_engine_signal_breadcrumbs(dev_priv->gt.engine[RCS0]);
4066
4067 if (iir & I915_MASTER_ERROR_INTERRUPT)
4068 i9xx_error_irq_handler(dev_priv, eir, eir_stuck);
4069
4070 if (hotplug_status)
4071 i9xx_hpd_irq_handler(dev_priv, hotplug_status);
4072
4073 i915_pipestat_irq_handler(dev_priv, iir, pipe_stats);
4074 } while (0);
4075
4076 pmu_irq_stats(dev_priv, ret);
4077
4078 enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
4079
4080 return ret;
4081 }
4082
4083 static void i965_irq_reset(struct drm_i915_private *dev_priv)
4084 {
4085 struct intel_uncore *uncore = &dev_priv->uncore;
4086
4087 i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4088 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4089
4090 i9xx_pipestat_irq_reset(dev_priv);
4091
4092 GEN3_IRQ_RESET(uncore, GEN2_);
4093 dev_priv->irq_mask = ~0u;
4094 }
4095
4096 static void i965_irq_postinstall(struct drm_i915_private *dev_priv)
4097 {
4098 struct intel_uncore *uncore = &dev_priv->uncore;
4099 u32 enable_mask;
4100 u32 error_mask;
4101
4102 /*
4103 * Enable some error detection, note the instruction error mask
4104 * bit is reserved, so we leave it masked.
4105 */
4106 if (IS_G4X(dev_priv)) {
4107 error_mask = ~(GM45_ERROR_PAGE_TABLE |
4108 GM45_ERROR_MEM_PRIV |
4109 GM45_ERROR_CP_PRIV |
4110 I915_ERROR_MEMORY_REFRESH);
4111 } else {
4112 error_mask = ~(I915_ERROR_PAGE_TABLE |
4113 I915_ERROR_MEMORY_REFRESH);
4114 }
4115 I915_WRITE(EMR, error_mask);
4116
4117 /* Unmask the interrupts that we always want on. */
4118 dev_priv->irq_mask =
4119 ~(I915_ASLE_INTERRUPT |
4120 I915_DISPLAY_PORT_INTERRUPT |
4121 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
4122 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
4123 I915_MASTER_ERROR_INTERRUPT);
4124
4125 enable_mask =
4126 I915_ASLE_INTERRUPT |
4127 I915_DISPLAY_PORT_INTERRUPT |
4128 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
4129 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
4130 I915_MASTER_ERROR_INTERRUPT |
4131 I915_USER_INTERRUPT;
4132
4133 if (IS_G4X(dev_priv))
4134 enable_mask |= I915_BSD_USER_INTERRUPT;
4135
4136 GEN3_IRQ_INIT(uncore, GEN2_, dev_priv->irq_mask, enable_mask);
4137
4138 /* Interrupt setup is already guaranteed to be single-threaded, this is
4139 * just to make the assert_spin_locked check happy. */
4140 spin_lock_irq(&dev_priv->irq_lock);
4141 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
4142 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
4143 i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
4144 spin_unlock_irq(&dev_priv->irq_lock);
4145
4146 i915_enable_asle_pipestat(dev_priv);
4147 }
4148
4149 static void i915_hpd_irq_setup(struct drm_i915_private *dev_priv)
4150 {
4151 u32 hotplug_en;
4152
4153 lockdep_assert_held(&dev_priv->irq_lock);
4154
4155 /* Note HDMI and DP share hotplug bits */
4156 /* enable bits are the same for all generations */
4157 hotplug_en = intel_hpd_enabled_irqs(dev_priv, hpd_mask_i915);
4158 /* Programming the CRT detection parameters tends
4159 to generate a spurious hotplug event about three
4160 seconds later. So just do it once.
4161 */
4162 if (IS_G4X(dev_priv))
4163 hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
4164 hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;
4165
4166 /* Ignore TV since it's buggy */
4167 i915_hotplug_interrupt_update_locked(dev_priv,
4168 HOTPLUG_INT_EN_MASK |
4169 CRT_HOTPLUG_VOLTAGE_COMPARE_MASK |
4170 CRT_HOTPLUG_ACTIVATION_PERIOD_64,
4171 hotplug_en);
4172 }
4173
4174 static irqreturn_t i965_irq_handler(int irq, void *arg)
4175 {
4176 struct drm_i915_private *dev_priv = arg;
4177 irqreturn_t ret = IRQ_NONE;
4178
4179 if (!intel_irqs_enabled(dev_priv))
4180 return IRQ_NONE;
4181
4182 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
4183 disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
4184
4185 do {
4186 u32 pipe_stats[I915_MAX_PIPES] = {};
4187 u32 eir = 0, eir_stuck = 0;
4188 u32 hotplug_status = 0;
4189 u32 iir;
4190
4191 iir = I915_READ(GEN2_IIR);
4192 if (iir == 0)
4193 break;
4194
4195 ret = IRQ_HANDLED;
4196
4197 if (iir & I915_DISPLAY_PORT_INTERRUPT)
4198 hotplug_status = i9xx_hpd_irq_ack(dev_priv);
4199
4200 /* Call regardless, as some status bits might not be
4201 * signalled in iir */
4202 i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
4203
4204 if (iir & I915_MASTER_ERROR_INTERRUPT)
4205 i9xx_error_irq_ack(dev_priv, &eir, &eir_stuck);
4206
4207 I915_WRITE(GEN2_IIR, iir);
4208
4209 if (iir & I915_USER_INTERRUPT)
4210 intel_engine_signal_breadcrumbs(dev_priv->gt.engine[RCS0]);
4211
4212 if (iir & I915_BSD_USER_INTERRUPT)
4213 intel_engine_signal_breadcrumbs(dev_priv->gt.engine[VCS0]);
4214
4215 if (iir & I915_MASTER_ERROR_INTERRUPT)
4216 i9xx_error_irq_handler(dev_priv, eir, eir_stuck);
4217
4218 if (hotplug_status)
4219 i9xx_hpd_irq_handler(dev_priv, hotplug_status);
4220
4221 i965_pipestat_irq_handler(dev_priv, iir, pipe_stats);
4222 } while (0);
4223
4224 pmu_irq_stats(dev_priv, IRQ_HANDLED);
4225
4226 enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
4227
4228 return ret;
4229 }
4230
4231 /**
4232 * intel_irq_init - initializes irq support
4233 * @dev_priv: i915 device instance
4234 *
4235 * This function initializes all the irq support including work items, timers
4236 * and all the vtables. It does not setup the interrupt itself though.
4237 */
4238 void intel_irq_init(struct drm_i915_private *dev_priv)
4239 {
4240 struct drm_device *dev = &dev_priv->drm;
4241 int i;
4242
4243 INIT_WORK(&dev_priv->l3_parity.error_work, ivb_parity_work);
4244 for (i = 0; i < MAX_L3_SLICES; ++i)
4245 dev_priv->l3_parity.remap_info[i] = NULL;
4246
4247 /* pre-gen11 the guc irqs bits are in the upper 16 bits of the pm reg */
4248 if (HAS_GT_UC(dev_priv) && INTEL_GEN(dev_priv) < 11)
4249 dev_priv->gt.pm_guc_events = GUC_INTR_GUC2HOST << 16;
4250
4251 if (!HAS_DISPLAY(dev_priv))
4252 return;
4253
4254 intel_hpd_init_pins(dev_priv);
4255
4256 intel_hpd_init_work(dev_priv);
4257
4258 dev->vblank_disable_immediate = true;
4259
4260 /* Most platforms treat the display irq block as an always-on
4261 * power domain. vlv/chv can disable it at runtime and need
4262 * special care to avoid writing any of the display block registers
4263 * outside of the power domain. We defer setting up the display irqs
4264 * in this case to the runtime pm.
4265 */
4266 dev_priv->display_irqs_enabled = true;
4267 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
4268 dev_priv->display_irqs_enabled = false;
4269
4270 dev_priv->hotplug.hpd_storm_threshold = HPD_STORM_DEFAULT_THRESHOLD;
4271 /* If we have MST support, we want to avoid doing short HPD IRQ storm
4272 * detection, as short HPD storms will occur as a natural part of
4273 * sideband messaging with MST.
4274 * On older platforms however, IRQ storms can occur with both long and
4275 * short pulses, as seen on some G4x systems.
4276 */
4277 dev_priv->hotplug.hpd_short_storm_enabled = !HAS_DP_MST(dev_priv);
4278
4279 if (HAS_GMCH(dev_priv)) {
4280 if (I915_HAS_HOTPLUG(dev_priv))
4281 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4282 } else {
4283 if (HAS_PCH_DG1(dev_priv))
4284 dev_priv->display.hpd_irq_setup = dg1_hpd_irq_setup;
4285 else if (INTEL_GEN(dev_priv) >= 11)
4286 dev_priv->display.hpd_irq_setup = gen11_hpd_irq_setup;
4287 else if (IS_GEN9_LP(dev_priv))
4288 dev_priv->display.hpd_irq_setup = bxt_hpd_irq_setup;
4289 else if (INTEL_PCH_TYPE(dev_priv) >= PCH_SPT)
4290 dev_priv->display.hpd_irq_setup = spt_hpd_irq_setup;
4291 else
4292 dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
4293 }
4294 }
4295
4296 /**
4297 * intel_irq_fini - deinitializes IRQ support
4298 * @i915: i915 device instance
4299 *
4300 * This function deinitializes all the IRQ support.
4301 */
4302 void intel_irq_fini(struct drm_i915_private *i915)
4303 {
4304 int i;
4305
4306 for (i = 0; i < MAX_L3_SLICES; ++i)
4307 kfree(i915->l3_parity.remap_info[i]);
4308 }
4309
4310 static irq_handler_t intel_irq_handler(struct drm_i915_private *dev_priv)
4311 {
4312 if (HAS_GMCH(dev_priv)) {
4313 if (IS_CHERRYVIEW(dev_priv))
4314 return cherryview_irq_handler;
4315 else if (IS_VALLEYVIEW(dev_priv))
4316 return valleyview_irq_handler;
4317 else if (IS_GEN(dev_priv, 4))
4318 return i965_irq_handler;
4319 else if (IS_GEN(dev_priv, 3))
4320 return i915_irq_handler;
4321 else
4322 return i8xx_irq_handler;
4323 } else {
4324 if (HAS_MASTER_UNIT_IRQ(dev_priv))
4325 return dg1_irq_handler;
4326 if (INTEL_GEN(dev_priv) >= 11)
4327 return gen11_irq_handler;
4328 else if (INTEL_GEN(dev_priv) >= 8)
4329 return gen8_irq_handler;
4330 else
4331 return ilk_irq_handler;
4332 }
4333 }
4334
4335 static void intel_irq_reset(struct drm_i915_private *dev_priv)
4336 {
4337 if (HAS_GMCH(dev_priv)) {
4338 if (IS_CHERRYVIEW(dev_priv))
4339 cherryview_irq_reset(dev_priv);
4340 else if (IS_VALLEYVIEW(dev_priv))
4341 valleyview_irq_reset(dev_priv);
4342 else if (IS_GEN(dev_priv, 4))
4343 i965_irq_reset(dev_priv);
4344 else if (IS_GEN(dev_priv, 3))
4345 i915_irq_reset(dev_priv);
4346 else
4347 i8xx_irq_reset(dev_priv);
4348 } else {
4349 if (INTEL_GEN(dev_priv) >= 11)
4350 gen11_irq_reset(dev_priv);
4351 else if (INTEL_GEN(dev_priv) >= 8)
4352 gen8_irq_reset(dev_priv);
4353 else
4354 ilk_irq_reset(dev_priv);
4355 }
4356 }
4357
4358 static void intel_irq_postinstall(struct drm_i915_private *dev_priv)
4359 {
4360 if (HAS_GMCH(dev_priv)) {
4361 if (IS_CHERRYVIEW(dev_priv))
4362 cherryview_irq_postinstall(dev_priv);
4363 else if (IS_VALLEYVIEW(dev_priv))
4364 valleyview_irq_postinstall(dev_priv);
4365 else if (IS_GEN(dev_priv, 4))
4366 i965_irq_postinstall(dev_priv);
4367 else if (IS_GEN(dev_priv, 3))
4368 i915_irq_postinstall(dev_priv);
4369 else
4370 i8xx_irq_postinstall(dev_priv);
4371 } else {
4372 if (INTEL_GEN(dev_priv) >= 11)
4373 gen11_irq_postinstall(dev_priv);
4374 else if (INTEL_GEN(dev_priv) >= 8)
4375 gen8_irq_postinstall(dev_priv);
4376 else
4377 ilk_irq_postinstall(dev_priv);
4378 }
4379 }
4380
4381 /**
4382 * intel_irq_install - enables the hardware interrupt
4383 * @dev_priv: i915 device instance
4384 *
4385 * This function enables the hardware interrupt handling, but leaves the hotplug
4386 * handling still disabled. It is called after intel_irq_init().
4387 *
4388 * In the driver load and resume code we need working interrupts in a few places
4389 * but don't want to deal with the hassle of concurrent probe and hotplug
4390 * workers. Hence the split into this two-stage approach.
4391 */
4392 int intel_irq_install(struct drm_i915_private *dev_priv)
4393 {
4394 int irq = dev_priv->drm.pdev->irq;
4395 int ret;
4396
4397 /*
4398 * We enable some interrupt sources in our postinstall hooks, so mark
4399 * interrupts as enabled _before_ actually enabling them to avoid
4400 * special cases in our ordering checks.
4401 */
4402 dev_priv->runtime_pm.irqs_enabled = true;
4403
4404 dev_priv->drm.irq_enabled = true;
4405
4406 intel_irq_reset(dev_priv);
4407
4408 ret = request_irq(irq, intel_irq_handler(dev_priv),
4409 IRQF_SHARED, DRIVER_NAME, dev_priv);
4410 if (ret < 0) {
4411 dev_priv->drm.irq_enabled = false;
4412 return ret;
4413 }
4414
4415 intel_irq_postinstall(dev_priv);
4416
4417 return ret;
4418 }
4419
4420 /**
4421 * intel_irq_uninstall - finilizes all irq handling
4422 * @dev_priv: i915 device instance
4423 *
4424 * This stops interrupt and hotplug handling and unregisters and frees all
4425 * resources acquired in the init functions.
4426 */
4427 void intel_irq_uninstall(struct drm_i915_private *dev_priv)
4428 {
4429 int irq = dev_priv->drm.pdev->irq;
4430
4431 /*
4432 * FIXME we can get called twice during driver probe
4433 * error handling as well as during driver remove due to
4434 * intel_modeset_driver_remove() calling us out of sequence.
4435 * Would be nice if it didn't do that...
4436 */
4437 if (!dev_priv->drm.irq_enabled)
4438 return;
4439
4440 dev_priv->drm.irq_enabled = false;
4441
4442 intel_irq_reset(dev_priv);
4443
4444 free_irq(irq, dev_priv);
4445
4446 intel_hpd_cancel_work(dev_priv);
4447 dev_priv->runtime_pm.irqs_enabled = false;
4448 }
4449
4450 /**
4451 * intel_runtime_pm_disable_interrupts - runtime interrupt disabling
4452 * @dev_priv: i915 device instance
4453 *
4454 * This function is used to disable interrupts at runtime, both in the runtime
4455 * pm and the system suspend/resume code.
4456 */
4457 void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv)
4458 {
4459 intel_irq_reset(dev_priv);
4460 dev_priv->runtime_pm.irqs_enabled = false;
4461 intel_synchronize_irq(dev_priv);
4462 }
4463
4464 /**
4465 * intel_runtime_pm_enable_interrupts - runtime interrupt enabling
4466 * @dev_priv: i915 device instance
4467 *
4468 * This function is used to enable interrupts at runtime, both in the runtime
4469 * pm and the system suspend/resume code.
4470 */
4471 void intel_runtime_pm_enable_interrupts(struct drm_i915_private *dev_priv)
4472 {
4473 dev_priv->runtime_pm.irqs_enabled = true;
4474 intel_irq_reset(dev_priv);
4475 intel_irq_postinstall(dev_priv);
4476 }
4477
4478 bool intel_irqs_enabled(struct drm_i915_private *dev_priv)
4479 {
4480 /*
4481 * We only use drm_irq_uninstall() at unload and VT switch, so
4482 * this is the only thing we need to check.
4483 */
4484 return dev_priv->runtime_pm.irqs_enabled;
4485 }
4486
4487 void intel_synchronize_irq(struct drm_i915_private *i915)
4488 {
4489 synchronize_irq(i915->drm.pdev->irq);
4490 }
Linux with added FPC-III support