?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'block-5.11-2021-01-10' of git://git.kernel.dk/linux-block
[linux/fpc-iii.git] / drivers / gpu / drm / i915 / gt / intel_workarounds.c
blobadc9a8ea410a3a2242c454c2347a41728375b0da
1 /*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright © 2014-2018 Intel Corporation
5 */
6
7 #include "i915_drv.h"
8 #include "intel_context.h"
9 #include "intel_engine_pm.h"
10 #include "intel_gt.h"
11 #include "intel_ring.h"
12 #include "intel_workarounds.h"
13
14 /**
15 * DOC: Hardware workarounds
16 *
17 * This file is intended as a central place to implement most [1]_ of the
18 * required workarounds for hardware to work as originally intended. They fall
19 * in five basic categories depending on how/when they are applied:
20 *
21 * - Workarounds that touch registers that are saved/restored to/from the HW
22 * context image. The list is emitted (via Load Register Immediate commands)
23 * everytime a new context is created.
24 * - GT workarounds. The list of these WAs is applied whenever these registers
25 * revert to default values (on GPU reset, suspend/resume [2]_, etc..).
26 * - Display workarounds. The list is applied during display clock-gating
27 * initialization.
28 * - Workarounds that whitelist a privileged register, so that UMDs can manage
29 * them directly. This is just a special case of a MMMIO workaround (as we
30 * write the list of these to/be-whitelisted registers to some special HW
31 * registers).
32 * - Workaround batchbuffers, that get executed automatically by the hardware
33 * on every HW context restore.
34 *
35 * .. [1] Please notice that there are other WAs that, due to their nature,
36 * cannot be applied from a central place. Those are peppered around the rest
37 * of the code, as needed.
38 *
39 * .. [2] Technically, some registers are powercontext saved & restored, so they
40 * survive a suspend/resume. In practice, writing them again is not too
41 * costly and simplifies things. We can revisit this in the future.
42 *
43 * Layout
44 * ~~~~~~
45 *
46 * Keep things in this file ordered by WA type, as per the above (context, GT,
47 * display, register whitelist, batchbuffer). Then, inside each type, keep the
48 * following order:
49 *
50 * - Infrastructure functions and macros
51 * - WAs per platform in standard gen/chrono order
52 * - Public functions to init or apply the given workaround type.
53 */
54
55 /*
56 * KBL revision ID ordering is bizarre; higher revision ID's map to lower
57 * steppings in some cases. So rather than test against the revision ID
58 * directly, let's map that into our own range of increasing ID's that we
59 * can test against in a regular manner.
60 */
61
62 const struct i915_rev_steppings kbl_revids[] = {
63 [0] = { .gt_stepping = KBL_REVID_A0, .disp_stepping = KBL_REVID_A0 },
64 [1] = { .gt_stepping = KBL_REVID_B0, .disp_stepping = KBL_REVID_B0 },
65 [2] = { .gt_stepping = KBL_REVID_C0, .disp_stepping = KBL_REVID_B0 },
66 [3] = { .gt_stepping = KBL_REVID_D0, .disp_stepping = KBL_REVID_B0 },
67 [4] = { .gt_stepping = KBL_REVID_F0, .disp_stepping = KBL_REVID_C0 },
68 [5] = { .gt_stepping = KBL_REVID_C0, .disp_stepping = KBL_REVID_B1 },
69 [6] = { .gt_stepping = KBL_REVID_D1, .disp_stepping = KBL_REVID_B1 },
70 [7] = { .gt_stepping = KBL_REVID_G0, .disp_stepping = KBL_REVID_C0 },
71 };
72
73 const struct i915_rev_steppings tgl_uy_revids[] = {
74 [0] = { .gt_stepping = TGL_REVID_A0, .disp_stepping = TGL_REVID_A0 },
75 [1] = { .gt_stepping = TGL_REVID_B0, .disp_stepping = TGL_REVID_C0 },
76 [2] = { .gt_stepping = TGL_REVID_B1, .disp_stepping = TGL_REVID_C0 },
77 [3] = { .gt_stepping = TGL_REVID_C0, .disp_stepping = TGL_REVID_D0 },
78 };
79
80 /* Same GT stepping between tgl_uy_revids and tgl_revids don't mean the same HW */
81 const struct i915_rev_steppings tgl_revids[] = {
82 [0] = { .gt_stepping = TGL_REVID_A0, .disp_stepping = TGL_REVID_B0 },
83 [1] = { .gt_stepping = TGL_REVID_B0, .disp_stepping = TGL_REVID_D0 },
84 };
85
86 static void wa_init_start(struct i915_wa_list *wal, const char *name, const char *engine_name)
87 {
88 wal->name = name;
89 wal->engine_name = engine_name;
90 }
91
92 #define WA_LIST_CHUNK (1 << 4)
93
94 static void wa_init_finish(struct i915_wa_list *wal)
95 {
96 /* Trim unused entries. */
97 if (!IS_ALIGNED(wal->count, WA_LIST_CHUNK)) {
98 struct i915_wa *list = kmemdup(wal->list,
99 wal->count * sizeof(*list),
100 GFP_KERNEL);
101
102 if (list) {
103 kfree(wal->list);
104 wal->list = list;
105 }
106 }
107
108 if (!wal->count)
109 return;
110
111 DRM_DEBUG_DRIVER("Initialized %u %s workarounds on %s\n",
112 wal->wa_count, wal->name, wal->engine_name);
113 }
114
115 static void _wa_add(struct i915_wa_list *wal, const struct i915_wa *wa)
116 {
117 unsigned int addr = i915_mmio_reg_offset(wa->reg);
118 unsigned int start = 0, end = wal->count;
119 const unsigned int grow = WA_LIST_CHUNK;
120 struct i915_wa *wa_;
121
122 GEM_BUG_ON(!is_power_of_2(grow));
123
124 if (IS_ALIGNED(wal->count, grow)) { /* Either uninitialized or full. */
125 struct i915_wa *list;
126
127 list = kmalloc_array(ALIGN(wal->count + 1, grow), sizeof(*wa),
128 GFP_KERNEL);
129 if (!list) {
130 DRM_ERROR("No space for workaround init!\n");
131 return;
132 }
133
134 if (wal->list) {
135 memcpy(list, wal->list, sizeof(*wa) * wal->count);
136 kfree(wal->list);
137 }
138
139 wal->list = list;
140 }
141
142 while (start < end) {
143 unsigned int mid = start + (end - start) / 2;
144
145 if (i915_mmio_reg_offset(wal->list[mid].reg) < addr) {
146 start = mid + 1;
147 } else if (i915_mmio_reg_offset(wal->list[mid].reg) > addr) {
148 end = mid;
149 } else {
150 wa_ = &wal->list[mid];
151
152 if ((wa->clr | wa_->clr) && !(wa->clr & ~wa_->clr)) {
153 DRM_ERROR("Discarding overwritten w/a for reg %04x (clear: %08x, set: %08x)\n",
154 i915_mmio_reg_offset(wa_->reg),
155 wa_->clr, wa_->set);
156
157 wa_->set &= ~wa->clr;
158 }
159
160 wal->wa_count++;
161 wa_->set |= wa->set;
162 wa_->clr |= wa->clr;
163 wa_->read |= wa->read;
164 return;
165 }
166 }
167
168 wal->wa_count++;
169 wa_ = &wal->list[wal->count++];
170 *wa_ = *wa;
171
172 while (wa_-- > wal->list) {
173 GEM_BUG_ON(i915_mmio_reg_offset(wa_[0].reg) ==
174 i915_mmio_reg_offset(wa_[1].reg));
175 if (i915_mmio_reg_offset(wa_[1].reg) >
176 i915_mmio_reg_offset(wa_[0].reg))
177 break;
178
179 swap(wa_[1], wa_[0]);
180 }
181 }
182
183 static void wa_add(struct i915_wa_list *wal, i915_reg_t reg,
184 u32 clear, u32 set, u32 read_mask)
185 {
186 struct i915_wa wa = {
187 .reg = reg,
188 .clr = clear,
189 .set = set,
190 .read = read_mask,
191 };
192
193 _wa_add(wal, &wa);
194 }
195
196 static void
197 wa_write_masked_or(struct i915_wa_list *wal, i915_reg_t reg, u32 clear, u32 set)
198 {
199 wa_add(wal, reg, clear, set, clear);
200 }
201
202 static void
203 wa_write(struct i915_wa_list *wal, i915_reg_t reg, u32 set)
204 {
205 wa_write_masked_or(wal, reg, ~0, set);
206 }
207
208 static void
209 wa_write_or(struct i915_wa_list *wal, i915_reg_t reg, u32 set)
210 {
211 wa_write_masked_or(wal, reg, set, set);
212 }
213
214 static void
215 wa_write_clr(struct i915_wa_list *wal, i915_reg_t reg, u32 clr)
216 {
217 wa_write_masked_or(wal, reg, clr, 0);
218 }
219
220 static void
221 wa_masked_en(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
222 {
223 wa_add(wal, reg, 0, _MASKED_BIT_ENABLE(val), val);
224 }
225
226 static void
227 wa_masked_dis(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
228 {
229 wa_add(wal, reg, 0, _MASKED_BIT_DISABLE(val), val);
230 }
231
232 #define WA_SET_BIT_MASKED(addr, mask) \
233 wa_masked_en(wal, (addr), (mask))
234
235 #define WA_CLR_BIT_MASKED(addr, mask) \
236 wa_masked_dis(wal, (addr), (mask))
237
238 #define WA_SET_FIELD_MASKED(addr, mask, value) \
239 wa_write_masked_or(wal, (addr), 0, _MASKED_FIELD((mask), (value)))
240
241 static void gen6_ctx_workarounds_init(struct intel_engine_cs *engine,
242 struct i915_wa_list *wal)
243 {
244 WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING);
245 }
246
247 static void gen7_ctx_workarounds_init(struct intel_engine_cs *engine,
248 struct i915_wa_list *wal)
249 {
250 WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING);
251 }
252
253 static void gen8_ctx_workarounds_init(struct intel_engine_cs *engine,
254 struct i915_wa_list *wal)
255 {
256 WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING);
257
258 /* WaDisableAsyncFlipPerfMode:bdw,chv */
259 WA_SET_BIT_MASKED(MI_MODE, ASYNC_FLIP_PERF_DISABLE);
260
261 /* WaDisablePartialInstShootdown:bdw,chv */
262 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
263 PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
264
265 /* Use Force Non-Coherent whenever executing a 3D context. This is a
266 * workaround for for a possible hang in the unlikely event a TLB
267 * invalidation occurs during a PSD flush.
268 */
269 /* WaForceEnableNonCoherent:bdw,chv */
270 /* WaHdcDisableFetchWhenMasked:bdw,chv */
271 WA_SET_BIT_MASKED(HDC_CHICKEN0,
272 HDC_DONOT_FETCH_MEM_WHEN_MASKED |
273 HDC_FORCE_NON_COHERENT);
274
275 /* From the Haswell PRM, Command Reference: Registers, CACHE_MODE_0:
276 * "The Hierarchical Z RAW Stall Optimization allows non-overlapping
277 * polygons in the same 8x4 pixel/sample area to be processed without
278 * stalling waiting for the earlier ones to write to Hierarchical Z
279 * buffer."
280 *
281 * This optimization is off by default for BDW and CHV; turn it on.
282 */
283 WA_CLR_BIT_MASKED(CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE);
284
285 /* Wa4x4STCOptimizationDisable:bdw,chv */
286 WA_SET_BIT_MASKED(CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE);
287
288 /*
289 * BSpec recommends 8x4 when MSAA is used,
290 * however in practice 16x4 seems fastest.
291 *
292 * Note that PS/WM thread counts depend on the WIZ hashing
293 * disable bit, which we don't touch here, but it's good
294 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
295 */
296 WA_SET_FIELD_MASKED(GEN7_GT_MODE,
297 GEN6_WIZ_HASHING_MASK,
298 GEN6_WIZ_HASHING_16x4);
299 }
300
301 static void bdw_ctx_workarounds_init(struct intel_engine_cs *engine,
302 struct i915_wa_list *wal)
303 {
304 struct drm_i915_private *i915 = engine->i915;
305
306 gen8_ctx_workarounds_init(engine, wal);
307
308 /* WaDisableThreadStallDopClockGating:bdw (pre-production) */
309 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
310
311 /* WaDisableDopClockGating:bdw
312 *
313 * Also see the related UCGTCL1 write in bdw_init_clock_gating()
314 * to disable EUTC clock gating.
315 */
316 WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
317 DOP_CLOCK_GATING_DISABLE);
318
319 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
320 GEN8_SAMPLER_POWER_BYPASS_DIS);
321
322 WA_SET_BIT_MASKED(HDC_CHICKEN0,
323 /* WaForceContextSaveRestoreNonCoherent:bdw */
324 HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
325 /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */
326 (IS_BDW_GT3(i915) ? HDC_FENCE_DEST_SLM_DISABLE : 0));
327 }
328
329 static void chv_ctx_workarounds_init(struct intel_engine_cs *engine,
330 struct i915_wa_list *wal)
331 {
332 gen8_ctx_workarounds_init(engine, wal);
333
334 /* WaDisableThreadStallDopClockGating:chv */
335 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
336
337 /* Improve HiZ throughput on CHV. */
338 WA_SET_BIT_MASKED(HIZ_CHICKEN, CHV_HZ_8X8_MODE_IN_1X);
339 }
340
341 static void gen9_ctx_workarounds_init(struct intel_engine_cs *engine,
342 struct i915_wa_list *wal)
343 {
344 struct drm_i915_private *i915 = engine->i915;
345
346 if (HAS_LLC(i915)) {
347 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
348 *
349 * Must match Display Engine. See
350 * WaCompressedResourceDisplayNewHashMode.
351 */
352 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
353 GEN9_PBE_COMPRESSED_HASH_SELECTION);
354 WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
355 GEN9_SAMPLER_HASH_COMPRESSED_READ_ADDR);
356 }
357
358 /* WaClearFlowControlGpgpuContextSave:skl,bxt,kbl,glk,cfl */
359 /* WaDisablePartialInstShootdown:skl,bxt,kbl,glk,cfl */
360 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
361 FLOW_CONTROL_ENABLE |
362 PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
363
364 /* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt,kbl,glk,cfl */
365 /* WaEnableSamplerGPGPUPreemptionSupport:skl,bxt,kbl,cfl */
366 WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
367 GEN9_ENABLE_YV12_BUGFIX |
368 GEN9_ENABLE_GPGPU_PREEMPTION);
369
370 /* Wa4x4STCOptimizationDisable:skl,bxt,kbl,glk,cfl */
371 /* WaDisablePartialResolveInVc:skl,bxt,kbl,cfl */
372 WA_SET_BIT_MASKED(CACHE_MODE_1,
373 GEN8_4x4_STC_OPTIMIZATION_DISABLE |
374 GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE);
375
376 /* WaCcsTlbPrefetchDisable:skl,bxt,kbl,glk,cfl */
377 WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
378 GEN9_CCS_TLB_PREFETCH_ENABLE);
379
380 /* WaForceContextSaveRestoreNonCoherent:skl,bxt,kbl,cfl */
381 WA_SET_BIT_MASKED(HDC_CHICKEN0,
382 HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
383 HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE);
384
385 /* WaForceEnableNonCoherent and WaDisableHDCInvalidation are
386 * both tied to WaForceContextSaveRestoreNonCoherent
387 * in some hsds for skl. We keep the tie for all gen9. The
388 * documentation is a bit hazy and so we want to get common behaviour,
389 * even though there is no clear evidence we would need both on kbl/bxt.
390 * This area has been source of system hangs so we play it safe
391 * and mimic the skl regardless of what bspec says.
392 *
393 * Use Force Non-Coherent whenever executing a 3D context. This
394 * is a workaround for a possible hang in the unlikely event
395 * a TLB invalidation occurs during a PSD flush.
396 */
397
398 /* WaForceEnableNonCoherent:skl,bxt,kbl,cfl */
399 WA_SET_BIT_MASKED(HDC_CHICKEN0,
400 HDC_FORCE_NON_COHERENT);
401
402 /* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt,kbl,cfl */
403 if (IS_SKYLAKE(i915) ||
404 IS_KABYLAKE(i915) ||
405 IS_COFFEELAKE(i915) ||
406 IS_COMETLAKE(i915))
407 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
408 GEN8_SAMPLER_POWER_BYPASS_DIS);
409
410 /* WaDisableSTUnitPowerOptimization:skl,bxt,kbl,glk,cfl */
411 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE);
412
413 /*
414 * Supporting preemption with fine-granularity requires changes in the
415 * batch buffer programming. Since we can't break old userspace, we
416 * need to set our default preemption level to safe value. Userspace is
417 * still able to use more fine-grained preemption levels, since in
418 * WaEnablePreemptionGranularityControlByUMD we're whitelisting the
419 * per-ctx register. As such, WaDisable{3D,GPGPU}MidCmdPreemption are
420 * not real HW workarounds, but merely a way to start using preemption
421 * while maintaining old contract with userspace.
422 */
423
424 /* WaDisable3DMidCmdPreemption:skl,bxt,glk,cfl,[cnl] */
425 WA_CLR_BIT_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
426
427 /* WaDisableGPGPUMidCmdPreemption:skl,bxt,blk,cfl,[cnl] */
428 WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1,
429 GEN9_PREEMPT_GPGPU_LEVEL_MASK,
430 GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
431
432 /* WaClearHIZ_WM_CHICKEN3:bxt,glk */
433 if (IS_GEN9_LP(i915))
434 WA_SET_BIT_MASKED(GEN9_WM_CHICKEN3, GEN9_FACTOR_IN_CLR_VAL_HIZ);
435 }
436
437 static void skl_tune_iz_hashing(struct intel_engine_cs *engine,
438 struct i915_wa_list *wal)
439 {
440 struct intel_gt *gt = engine->gt;
441 u8 vals[3] = { 0, 0, 0 };
442 unsigned int i;
443
444 for (i = 0; i < 3; i++) {
445 u8 ss;
446
447 /*
448 * Only consider slices where one, and only one, subslice has 7
449 * EUs
450 */
451 if (!is_power_of_2(gt->info.sseu.subslice_7eu[i]))
452 continue;
453
454 /*
455 * subslice_7eu[i] != 0 (because of the check above) and
456 * ss_max == 4 (maximum number of subslices possible per slice)
457 *
458 * -> 0 <= ss <= 3;
459 */
460 ss = ffs(gt->info.sseu.subslice_7eu[i]) - 1;
461 vals[i] = 3 - ss;
462 }
463
464 if (vals[0] == 0 && vals[1] == 0 && vals[2] == 0)
465 return;
466
467 /* Tune IZ hashing. See intel_device_info_runtime_init() */
468 WA_SET_FIELD_MASKED(GEN7_GT_MODE,
469 GEN9_IZ_HASHING_MASK(2) |
470 GEN9_IZ_HASHING_MASK(1) |
471 GEN9_IZ_HASHING_MASK(0),
472 GEN9_IZ_HASHING(2, vals[2]) |
473 GEN9_IZ_HASHING(1, vals[1]) |
474 GEN9_IZ_HASHING(0, vals[0]));
475 }
476
477 static void skl_ctx_workarounds_init(struct intel_engine_cs *engine,
478 struct i915_wa_list *wal)
479 {
480 gen9_ctx_workarounds_init(engine, wal);
481 skl_tune_iz_hashing(engine, wal);
482 }
483
484 static void bxt_ctx_workarounds_init(struct intel_engine_cs *engine,
485 struct i915_wa_list *wal)
486 {
487 gen9_ctx_workarounds_init(engine, wal);
488
489 /* WaDisableThreadStallDopClockGating:bxt */
490 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
491 STALL_DOP_GATING_DISABLE);
492
493 /* WaToEnableHwFixForPushConstHWBug:bxt */
494 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
495 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
496 }
497
498 static void kbl_ctx_workarounds_init(struct intel_engine_cs *engine,
499 struct i915_wa_list *wal)
500 {
501 struct drm_i915_private *i915 = engine->i915;
502
503 gen9_ctx_workarounds_init(engine, wal);
504
505 /* WaToEnableHwFixForPushConstHWBug:kbl */
506 if (IS_KBL_GT_REVID(i915, KBL_REVID_C0, REVID_FOREVER))
507 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
508 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
509
510 /* WaDisableSbeCacheDispatchPortSharing:kbl */
511 WA_SET_BIT_MASKED(GEN7_HALF_SLICE_CHICKEN1,
512 GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
513 }
514
515 static void glk_ctx_workarounds_init(struct intel_engine_cs *engine,
516 struct i915_wa_list *wal)
517 {
518 gen9_ctx_workarounds_init(engine, wal);
519
520 /* WaToEnableHwFixForPushConstHWBug:glk */
521 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
522 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
523 }
524
525 static void cfl_ctx_workarounds_init(struct intel_engine_cs *engine,
526 struct i915_wa_list *wal)
527 {
528 gen9_ctx_workarounds_init(engine, wal);
529
530 /* WaToEnableHwFixForPushConstHWBug:cfl */
531 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
532 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
533
534 /* WaDisableSbeCacheDispatchPortSharing:cfl */
535 WA_SET_BIT_MASKED(GEN7_HALF_SLICE_CHICKEN1,
536 GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
537 }
538
539 static void cnl_ctx_workarounds_init(struct intel_engine_cs *engine,
540 struct i915_wa_list *wal)
541 {
542 /* WaForceContextSaveRestoreNonCoherent:cnl */
543 WA_SET_BIT_MASKED(CNL_HDC_CHICKEN0,
544 HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT);
545
546 /* WaDisableReplayBufferBankArbitrationOptimization:cnl */
547 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
548 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
549
550 /* WaPushConstantDereferenceHoldDisable:cnl */
551 WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2, PUSH_CONSTANT_DEREF_DISABLE);
552
553 /* FtrEnableFastAnisoL1BankingFix:cnl */
554 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3, CNL_FAST_ANISO_L1_BANKING_FIX);
555
556 /* WaDisable3DMidCmdPreemption:cnl */
557 WA_CLR_BIT_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
558
559 /* WaDisableGPGPUMidCmdPreemption:cnl */
560 WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1,
561 GEN9_PREEMPT_GPGPU_LEVEL_MASK,
562 GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
563
564 /* WaDisableEarlyEOT:cnl */
565 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, DISABLE_EARLY_EOT);
566 }
567
568 static void icl_ctx_workarounds_init(struct intel_engine_cs *engine,
569 struct i915_wa_list *wal)
570 {
571 struct drm_i915_private *i915 = engine->i915;
572
573 /* WaDisableBankHangMode:icl */
574 wa_write(wal,
575 GEN8_L3CNTLREG,
576 intel_uncore_read(engine->uncore, GEN8_L3CNTLREG) |
577 GEN8_ERRDETBCTRL);
578
579 /* Wa_1604370585:icl (pre-prod)
580 * Formerly known as WaPushConstantDereferenceHoldDisable
581 */
582 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0))
583 WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
584 PUSH_CONSTANT_DEREF_DISABLE);
585
586 /* WaForceEnableNonCoherent:icl
587 * This is not the same workaround as in early Gen9 platforms, where
588 * lacking this could cause system hangs, but coherency performance
589 * overhead is high and only a few compute workloads really need it
590 * (the register is whitelisted in hardware now, so UMDs can opt in
591 * for coherency if they have a good reason).
592 */
593 WA_SET_BIT_MASKED(ICL_HDC_MODE, HDC_FORCE_NON_COHERENT);
594
595 /* Wa_2006611047:icl (pre-prod)
596 * Formerly known as WaDisableImprovedTdlClkGating
597 */
598 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0))
599 WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
600 GEN11_TDL_CLOCK_GATING_FIX_DISABLE);
601
602 /* Wa_2006665173:icl (pre-prod) */
603 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0))
604 WA_SET_BIT_MASKED(GEN11_COMMON_SLICE_CHICKEN3,
605 GEN11_BLEND_EMB_FIX_DISABLE_IN_RCC);
606
607 /* WaEnableFloatBlendOptimization:icl */
608 wa_write_masked_or(wal,
609 GEN10_CACHE_MODE_SS,
610 0, /* write-only, so skip validation */
611 _MASKED_BIT_ENABLE(FLOAT_BLEND_OPTIMIZATION_ENABLE));
612
613 /* WaDisableGPGPUMidThreadPreemption:icl */
614 WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1,
615 GEN9_PREEMPT_GPGPU_LEVEL_MASK,
616 GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL);
617
618 /* allow headerless messages for preemptible GPGPU context */
619 WA_SET_BIT_MASKED(GEN10_SAMPLER_MODE,
620 GEN11_SAMPLER_ENABLE_HEADLESS_MSG);
621
622 /* Wa_1604278689:icl,ehl */
623 wa_write(wal, IVB_FBC_RT_BASE, 0xFFFFFFFF & ~ILK_FBC_RT_VALID);
624 wa_write_masked_or(wal, IVB_FBC_RT_BASE_UPPER,
625 0, /* write-only register; skip validation */
626 0xFFFFFFFF);
627
628 /* Wa_1406306137:icl,ehl */
629 wa_masked_en(wal, GEN9_ROW_CHICKEN4, GEN11_DIS_PICK_2ND_EU);
630 }
631
632 static void gen12_ctx_workarounds_init(struct intel_engine_cs *engine,
633 struct i915_wa_list *wal)
634 {
635 /*
636 * Wa_1409142259:tgl
637 * Wa_1409347922:tgl
638 * Wa_1409252684:tgl
639 * Wa_1409217633:tgl
640 * Wa_1409207793:tgl
641 * Wa_1409178076:tgl
642 * Wa_1408979724:tgl
643 * Wa_14010443199:rkl
644 * Wa_14010698770:rkl
645 */
646 WA_SET_BIT_MASKED(GEN11_COMMON_SLICE_CHICKEN3,
647 GEN12_DISABLE_CPS_AWARE_COLOR_PIPE);
648
649 /* WaDisableGPGPUMidThreadPreemption:gen12 */
650 WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1,
651 GEN9_PREEMPT_GPGPU_LEVEL_MASK,
652 GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL);
653 }
654
655 static void tgl_ctx_workarounds_init(struct intel_engine_cs *engine,
656 struct i915_wa_list *wal)
657 {
658 gen12_ctx_workarounds_init(engine, wal);
659
660 /*
661 * Wa_1604555607:tgl,rkl
662 *
663 * Note that the implementation of this workaround is further modified
664 * according to the FF_MODE2 guidance given by Wa_1608008084:gen12.
665 * FF_MODE2 register will return the wrong value when read. The default
666 * value for this register is zero for all fields and there are no bit
667 * masks. So instead of doing a RMW we should just write the GS Timer
668 * and TDS timer values for Wa_1604555607 and Wa_16011163337.
669 */
670 wa_add(wal,
671 FF_MODE2,
672 FF_MODE2_GS_TIMER_MASK | FF_MODE2_TDS_TIMER_MASK,
673 FF_MODE2_GS_TIMER_224 | FF_MODE2_TDS_TIMER_128,
674 0);
675 }
676
677 static void dg1_ctx_workarounds_init(struct intel_engine_cs *engine,
678 struct i915_wa_list *wal)
679 {
680 gen12_ctx_workarounds_init(engine, wal);
681
682 /* Wa_1409044764 */
683 WA_CLR_BIT_MASKED(GEN11_COMMON_SLICE_CHICKEN3,
684 DG1_FLOAT_POINT_BLEND_OPT_STRICT_MODE_EN);
685
686 /* Wa_22010493298 */
687 WA_SET_BIT_MASKED(HIZ_CHICKEN,
688 DG1_HZ_READ_SUPPRESSION_OPTIMIZATION_DISABLE);
689 }
690
691 static void
692 __intel_engine_init_ctx_wa(struct intel_engine_cs *engine,
693 struct i915_wa_list *wal,
694 const char *name)
695 {
696 struct drm_i915_private *i915 = engine->i915;
697
698 if (engine->class != RENDER_CLASS)
699 return;
700
701 wa_init_start(wal, name, engine->name);
702
703 if (IS_DG1(i915))
704 dg1_ctx_workarounds_init(engine, wal);
705 else if (IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915))
706 tgl_ctx_workarounds_init(engine, wal);
707 else if (IS_GEN(i915, 12))
708 gen12_ctx_workarounds_init(engine, wal);
709 else if (IS_GEN(i915, 11))
710 icl_ctx_workarounds_init(engine, wal);
711 else if (IS_CANNONLAKE(i915))
712 cnl_ctx_workarounds_init(engine, wal);
713 else if (IS_COFFEELAKE(i915) || IS_COMETLAKE(i915))
714 cfl_ctx_workarounds_init(engine, wal);
715 else if (IS_GEMINILAKE(i915))
716 glk_ctx_workarounds_init(engine, wal);
717 else if (IS_KABYLAKE(i915))
718 kbl_ctx_workarounds_init(engine, wal);
719 else if (IS_BROXTON(i915))
720 bxt_ctx_workarounds_init(engine, wal);
721 else if (IS_SKYLAKE(i915))
722 skl_ctx_workarounds_init(engine, wal);
723 else if (IS_CHERRYVIEW(i915))
724 chv_ctx_workarounds_init(engine, wal);
725 else if (IS_BROADWELL(i915))
726 bdw_ctx_workarounds_init(engine, wal);
727 else if (IS_GEN(i915, 7))
728 gen7_ctx_workarounds_init(engine, wal);
729 else if (IS_GEN(i915, 6))
730 gen6_ctx_workarounds_init(engine, wal);
731 else if (INTEL_GEN(i915) < 8)
732 return;
733 else
734 MISSING_CASE(INTEL_GEN(i915));
735
736 wa_init_finish(wal);
737 }
738
739 void intel_engine_init_ctx_wa(struct intel_engine_cs *engine)
740 {
741 __intel_engine_init_ctx_wa(engine, &engine->ctx_wa_list, "context");
742 }
743
744 int intel_engine_emit_ctx_wa(struct i915_request *rq)
745 {
746 struct i915_wa_list *wal = &rq->engine->ctx_wa_list;
747 struct i915_wa *wa;
748 unsigned int i;
749 u32 *cs;
750 int ret;
751
752 if (wal->count == 0)
753 return 0;
754
755 ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
756 if (ret)
757 return ret;
758
759 cs = intel_ring_begin(rq, (wal->count * 2 + 2));
760 if (IS_ERR(cs))
761 return PTR_ERR(cs);
762
763 *cs++ = MI_LOAD_REGISTER_IMM(wal->count);
764 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
765 *cs++ = i915_mmio_reg_offset(wa->reg);
766 *cs++ = wa->set;
767 }
768 *cs++ = MI_NOOP;
769
770 intel_ring_advance(rq, cs);
771
772 ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
773 if (ret)
774 return ret;
775
776 return 0;
777 }
778
779 static void
780 gen4_gt_workarounds_init(struct drm_i915_private *i915,
781 struct i915_wa_list *wal)
782 {
783 /* WaDisable_RenderCache_OperationalFlush:gen4,ilk */
784 wa_masked_dis(wal, CACHE_MODE_0, RC_OP_FLUSH_ENABLE);
785 }
786
787 static void
788 g4x_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
789 {
790 gen4_gt_workarounds_init(i915, wal);
791
792 /* WaDisableRenderCachePipelinedFlush:g4x,ilk */
793 wa_masked_en(wal, CACHE_MODE_0, CM0_PIPELINED_RENDER_FLUSH_DISABLE);
794 }
795
796 static void
797 ilk_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
798 {
799 g4x_gt_workarounds_init(i915, wal);
800
801 wa_masked_en(wal, _3D_CHICKEN2, _3D_CHICKEN2_WM_READ_PIPELINED);
802 }
803
804 static void
805 snb_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
806 {
807 /* WaDisableHiZPlanesWhenMSAAEnabled:snb */
808 wa_masked_en(wal,
809 _3D_CHICKEN,
810 _3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB);
811
812 /* WaDisable_RenderCache_OperationalFlush:snb */
813 wa_masked_dis(wal, CACHE_MODE_0, RC_OP_FLUSH_ENABLE);
814
815 /*
816 * BSpec recommends 8x4 when MSAA is used,
817 * however in practice 16x4 seems fastest.
818 *
819 * Note that PS/WM thread counts depend on the WIZ hashing
820 * disable bit, which we don't touch here, but it's good
821 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
822 */
823 wa_add(wal,
824 GEN6_GT_MODE, 0,
825 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4),
826 GEN6_WIZ_HASHING_16x4);
827
828 wa_masked_dis(wal, CACHE_MODE_0, CM0_STC_EVICT_DISABLE_LRA_SNB);
829
830 wa_masked_en(wal,
831 _3D_CHICKEN3,
832 /* WaStripsFansDisableFastClipPerformanceFix:snb */
833 _3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL |
834 /*
835 * Bspec says:
836 * "This bit must be set if 3DSTATE_CLIP clip mode is set
837 * to normal and 3DSTATE_SF number of SF output attributes
838 * is more than 16."
839 */
840 _3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH);
841 }
842
843 static void
844 ivb_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
845 {
846 /* WaDisableEarlyCull:ivb */
847 wa_masked_en(wal, _3D_CHICKEN3, _3D_CHICKEN_SF_DISABLE_OBJEND_CULL);
848
849 /* WaDisablePSDDualDispatchEnable:ivb */
850 if (IS_IVB_GT1(i915))
851 wa_masked_en(wal,
852 GEN7_HALF_SLICE_CHICKEN1,
853 GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE);
854
855 /* WaDisable_RenderCache_OperationalFlush:ivb */
856 wa_masked_dis(wal, CACHE_MODE_0_GEN7, RC_OP_FLUSH_ENABLE);
857
858 /* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
859 wa_masked_dis(wal,
860 GEN7_COMMON_SLICE_CHICKEN1,
861 GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
862
863 /* WaApplyL3ControlAndL3ChickenMode:ivb */
864 wa_write(wal, GEN7_L3CNTLREG1, GEN7_WA_FOR_GEN7_L3_CONTROL);
865 wa_write(wal, GEN7_L3_CHICKEN_MODE_REGISTER, GEN7_WA_L3_CHICKEN_MODE);
866
867 /* WaForceL3Serialization:ivb */
868 wa_write_clr(wal, GEN7_L3SQCREG4, L3SQ_URB_READ_CAM_MATCH_DISABLE);
869
870 /*
871 * WaVSThreadDispatchOverride:ivb,vlv
872 *
873 * This actually overrides the dispatch
874 * mode for all thread types.
875 */
876 wa_write_masked_or(wal, GEN7_FF_THREAD_MODE,
877 GEN7_FF_SCHED_MASK,
878 GEN7_FF_TS_SCHED_HW |
879 GEN7_FF_VS_SCHED_HW |
880 GEN7_FF_DS_SCHED_HW);
881
882 if (0) { /* causes HiZ corruption on ivb:gt1 */
883 /* enable HiZ Raw Stall Optimization */
884 wa_masked_dis(wal, CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE);
885 }
886
887 /* WaDisable4x2SubspanOptimization:ivb */
888 wa_masked_en(wal, CACHE_MODE_1, PIXEL_SUBSPAN_COLLECT_OPT_DISABLE);
889
890 /*
891 * BSpec recommends 8x4 when MSAA is used,
892 * however in practice 16x4 seems fastest.
893 *
894 * Note that PS/WM thread counts depend on the WIZ hashing
895 * disable bit, which we don't touch here, but it's good
896 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
897 */
898 wa_add(wal, GEN7_GT_MODE, 0,
899 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4),
900 GEN6_WIZ_HASHING_16x4);
901 }
902
903 static void
904 vlv_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
905 {
906 /* WaDisableEarlyCull:vlv */
907 wa_masked_en(wal, _3D_CHICKEN3, _3D_CHICKEN_SF_DISABLE_OBJEND_CULL);
908
909 /* WaPsdDispatchEnable:vlv */
910 /* WaDisablePSDDualDispatchEnable:vlv */
911 wa_masked_en(wal,
912 GEN7_HALF_SLICE_CHICKEN1,
913 GEN7_MAX_PS_THREAD_DEP |
914 GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE);
915
916 /* WaDisable_RenderCache_OperationalFlush:vlv */
917 wa_masked_dis(wal, CACHE_MODE_0_GEN7, RC_OP_FLUSH_ENABLE);
918
919 /* WaForceL3Serialization:vlv */
920 wa_write_clr(wal, GEN7_L3SQCREG4, L3SQ_URB_READ_CAM_MATCH_DISABLE);
921
922 /*
923 * WaVSThreadDispatchOverride:ivb,vlv
924 *
925 * This actually overrides the dispatch
926 * mode for all thread types.
927 */
928 wa_write_masked_or(wal,
929 GEN7_FF_THREAD_MODE,
930 GEN7_FF_SCHED_MASK,
931 GEN7_FF_TS_SCHED_HW |
932 GEN7_FF_VS_SCHED_HW |
933 GEN7_FF_DS_SCHED_HW);
934
935 /*
936 * BSpec says this must be set, even though
937 * WaDisable4x2SubspanOptimization isn't listed for VLV.
938 */
939 wa_masked_en(wal, CACHE_MODE_1, PIXEL_SUBSPAN_COLLECT_OPT_DISABLE);
940
941 /*
942 * BSpec recommends 8x4 when MSAA is used,
943 * however in practice 16x4 seems fastest.
944 *
945 * Note that PS/WM thread counts depend on the WIZ hashing
946 * disable bit, which we don't touch here, but it's good
947 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
948 */
949 wa_add(wal, GEN7_GT_MODE, 0,
950 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4),
951 GEN6_WIZ_HASHING_16x4);
952
953 /*
954 * WaIncreaseL3CreditsForVLVB0:vlv
955 * This is the hardware default actually.
956 */
957 wa_write(wal, GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE);
958 }
959
960 static void
961 hsw_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
962 {
963 /* L3 caching of data atomics doesn't work -- disable it. */
964 wa_write(wal, HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
965
966 wa_add(wal,
967 HSW_ROW_CHICKEN3, 0,
968 _MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE),
969 0 /* XXX does this reg exist? */);
970
971 /* WaVSRefCountFullforceMissDisable:hsw */
972 wa_write_clr(wal, GEN7_FF_THREAD_MODE, GEN7_FF_VS_REF_CNT_FFME);
973
974 wa_masked_dis(wal,
975 CACHE_MODE_0_GEN7,
976 /* WaDisable_RenderCache_OperationalFlush:hsw */
977 RC_OP_FLUSH_ENABLE |
978 /* enable HiZ Raw Stall Optimization */
979 HIZ_RAW_STALL_OPT_DISABLE);
980
981 /* WaDisable4x2SubspanOptimization:hsw */
982 wa_masked_en(wal, CACHE_MODE_1, PIXEL_SUBSPAN_COLLECT_OPT_DISABLE);
983
984 /*
985 * BSpec recommends 8x4 when MSAA is used,
986 * however in practice 16x4 seems fastest.
987 *
988 * Note that PS/WM thread counts depend on the WIZ hashing
989 * disable bit, which we don't touch here, but it's good
990 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
991 */
992 wa_add(wal, GEN7_GT_MODE, 0,
993 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4),
994 GEN6_WIZ_HASHING_16x4);
995
996 /* WaSampleCChickenBitEnable:hsw */
997 wa_masked_en(wal, HALF_SLICE_CHICKEN3, HSW_SAMPLE_C_PERFORMANCE);
998 }
999
1000 static void
1001 gen9_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1002 {
1003 /* WaDisableKillLogic:bxt,skl,kbl */
1004 if (!IS_COFFEELAKE(i915) && !IS_COMETLAKE(i915))
1005 wa_write_or(wal,
1006 GAM_ECOCHK,
1007 ECOCHK_DIS_TLB);
1008
1009 if (HAS_LLC(i915)) {
1010 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
1011 *
1012 * Must match Display Engine. See
1013 * WaCompressedResourceDisplayNewHashMode.
1014 */
1015 wa_write_or(wal,
1016 MMCD_MISC_CTRL,
1017 MMCD_PCLA | MMCD_HOTSPOT_EN);
1018 }
1019
1020 /* WaDisableHDCInvalidation:skl,bxt,kbl,cfl */
1021 wa_write_or(wal,
1022 GAM_ECOCHK,
1023 BDW_DISABLE_HDC_INVALIDATION);
1024 }
1025
1026 static void
1027 skl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1028 {
1029 gen9_gt_workarounds_init(i915, wal);
1030
1031 /* WaDisableGafsUnitClkGating:skl */
1032 wa_write_or(wal,
1033 GEN7_UCGCTL4,
1034 GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
1035
1036 /* WaInPlaceDecompressionHang:skl */
1037 if (IS_SKL_REVID(i915, SKL_REVID_H0, REVID_FOREVER))
1038 wa_write_or(wal,
1039 GEN9_GAMT_ECO_REG_RW_IA,
1040 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1041 }
1042
1043 static void
1044 bxt_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1045 {
1046 gen9_gt_workarounds_init(i915, wal);
1047
1048 /* WaInPlaceDecompressionHang:bxt */
1049 wa_write_or(wal,
1050 GEN9_GAMT_ECO_REG_RW_IA,
1051 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1052 }
1053
1054 static void
1055 kbl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1056 {
1057 gen9_gt_workarounds_init(i915, wal);
1058
1059 /* WaDisableDynamicCreditSharing:kbl */
1060 if (IS_KBL_GT_REVID(i915, 0, KBL_REVID_B0))
1061 wa_write_or(wal,
1062 GAMT_CHKN_BIT_REG,
1063 GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING);
1064
1065 /* WaDisableGafsUnitClkGating:kbl */
1066 wa_write_or(wal,
1067 GEN7_UCGCTL4,
1068 GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
1069
1070 /* WaInPlaceDecompressionHang:kbl */
1071 wa_write_or(wal,
1072 GEN9_GAMT_ECO_REG_RW_IA,
1073 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1074 }
1075
1076 static void
1077 glk_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1078 {
1079 gen9_gt_workarounds_init(i915, wal);
1080 }
1081
1082 static void
1083 cfl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1084 {
1085 gen9_gt_workarounds_init(i915, wal);
1086
1087 /* WaDisableGafsUnitClkGating:cfl */
1088 wa_write_or(wal,
1089 GEN7_UCGCTL4,
1090 GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
1091
1092 /* WaInPlaceDecompressionHang:cfl */
1093 wa_write_or(wal,
1094 GEN9_GAMT_ECO_REG_RW_IA,
1095 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1096 }
1097
1098 static void
1099 wa_init_mcr(struct drm_i915_private *i915, struct i915_wa_list *wal)
1100 {
1101 const struct sseu_dev_info *sseu = &i915->gt.info.sseu;
1102 unsigned int slice, subslice;
1103 u32 l3_en, mcr, mcr_mask;
1104
1105 GEM_BUG_ON(INTEL_GEN(i915) < 10);
1106
1107 /*
1108 * WaProgramMgsrForL3BankSpecificMmioReads: cnl,icl
1109 * L3Banks could be fused off in single slice scenario. If that is
1110 * the case, we might need to program MCR select to a valid L3Bank
1111 * by default, to make sure we correctly read certain registers
1112 * later on (in the range 0xB100 - 0xB3FF).
1113 *
1114 * WaProgramMgsrForCorrectSliceSpecificMmioReads:cnl,icl
1115 * Before any MMIO read into slice/subslice specific registers, MCR
1116 * packet control register needs to be programmed to point to any
1117 * enabled s/ss pair. Otherwise, incorrect values will be returned.
1118 * This means each subsequent MMIO read will be forwarded to an
1119 * specific s/ss combination, but this is OK since these registers
1120 * are consistent across s/ss in almost all cases. In the rare
1121 * occasions, such as INSTDONE, where this value is dependent
1122 * on s/ss combo, the read should be done with read_subslice_reg.
1123 *
1124 * Since GEN8_MCR_SELECTOR contains dual-purpose bits which select both
1125 * to which subslice, or to which L3 bank, the respective mmio reads
1126 * will go, we have to find a common index which works for both
1127 * accesses.
1128 *
1129 * Case where we cannot find a common index fortunately should not
1130 * happen in production hardware, so we only emit a warning instead of
1131 * implementing something more complex that requires checking the range
1132 * of every MMIO read.
1133 */
1134
1135 if (INTEL_GEN(i915) >= 10 && is_power_of_2(sseu->slice_mask)) {
1136 u32 l3_fuse =
1137 intel_uncore_read(&i915->uncore, GEN10_MIRROR_FUSE3) &
1138 GEN10_L3BANK_MASK;
1139
1140 drm_dbg(&i915->drm, "L3 fuse = %x\n", l3_fuse);
1141 l3_en = ~(l3_fuse << GEN10_L3BANK_PAIR_COUNT | l3_fuse);
1142 } else {
1143 l3_en = ~0;
1144 }
1145
1146 slice = fls(sseu->slice_mask) - 1;
1147 subslice = fls(l3_en & intel_sseu_get_subslices(sseu, slice));
1148 if (!subslice) {
1149 drm_warn(&i915->drm,
1150 "No common index found between subslice mask %x and L3 bank mask %x!\n",
1151 intel_sseu_get_subslices(sseu, slice), l3_en);
1152 subslice = fls(l3_en);
1153 drm_WARN_ON(&i915->drm, !subslice);
1154 }
1155 subslice--;
1156
1157 if (INTEL_GEN(i915) >= 11) {
1158 mcr = GEN11_MCR_SLICE(slice) | GEN11_MCR_SUBSLICE(subslice);
1159 mcr_mask = GEN11_MCR_SLICE_MASK | GEN11_MCR_SUBSLICE_MASK;
1160 } else {
1161 mcr = GEN8_MCR_SLICE(slice) | GEN8_MCR_SUBSLICE(subslice);
1162 mcr_mask = GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK;
1163 }
1164
1165 drm_dbg(&i915->drm, "MCR slice/subslice = %x\n", mcr);
1166
1167 wa_write_masked_or(wal, GEN8_MCR_SELECTOR, mcr_mask, mcr);
1168 }
1169
1170 static void
1171 cnl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1172 {
1173 wa_init_mcr(i915, wal);
1174
1175 /* WaInPlaceDecompressionHang:cnl */
1176 wa_write_or(wal,
1177 GEN9_GAMT_ECO_REG_RW_IA,
1178 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1179 }
1180
1181 static void
1182 icl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1183 {
1184 wa_init_mcr(i915, wal);
1185
1186 /* WaInPlaceDecompressionHang:icl */
1187 wa_write_or(wal,
1188 GEN9_GAMT_ECO_REG_RW_IA,
1189 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1190
1191 /* WaModifyGamTlbPartitioning:icl */
1192 wa_write_masked_or(wal,
1193 GEN11_GACB_PERF_CTRL,
1194 GEN11_HASH_CTRL_MASK,
1195 GEN11_HASH_CTRL_BIT0 | GEN11_HASH_CTRL_BIT4);
1196
1197 /* Wa_1405766107:icl
1198 * Formerly known as WaCL2SFHalfMaxAlloc
1199 */
1200 wa_write_or(wal,
1201 GEN11_LSN_UNSLCVC,
1202 GEN11_LSN_UNSLCVC_GAFS_HALF_SF_MAXALLOC |
1203 GEN11_LSN_UNSLCVC_GAFS_HALF_CL2_MAXALLOC);
1204
1205 /* Wa_220166154:icl
1206 * Formerly known as WaDisCtxReload
1207 */
1208 wa_write_or(wal,
1209 GEN8_GAMW_ECO_DEV_RW_IA,
1210 GAMW_ECO_DEV_CTX_RELOAD_DISABLE);
1211
1212 /* Wa_1405779004:icl (pre-prod) */
1213 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0))
1214 wa_write_or(wal,
1215 SLICE_UNIT_LEVEL_CLKGATE,
1216 MSCUNIT_CLKGATE_DIS);
1217
1218 /* Wa_1406838659:icl (pre-prod) */
1219 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0))
1220 wa_write_or(wal,
1221 INF_UNIT_LEVEL_CLKGATE,
1222 CGPSF_CLKGATE_DIS);
1223
1224 /* Wa_1406463099:icl
1225 * Formerly known as WaGamTlbPendError
1226 */
1227 wa_write_or(wal,
1228 GAMT_CHKN_BIT_REG,
1229 GAMT_CHKN_DISABLE_L3_COH_PIPE);
1230
1231 /* Wa_1607087056:icl,ehl,jsl */
1232 if (IS_ICELAKE(i915) ||
1233 IS_JSL_EHL_REVID(i915, EHL_REVID_A0, EHL_REVID_A0)) {
1234 wa_write_or(wal,
1235 SLICE_UNIT_LEVEL_CLKGATE,
1236 L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
1237 }
1238 }
1239
1240 static void
1241 gen12_gt_workarounds_init(struct drm_i915_private *i915,
1242 struct i915_wa_list *wal)
1243 {
1244 wa_init_mcr(i915, wal);
1245 }
1246
1247 static void
1248 tgl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1249 {
1250 gen12_gt_workarounds_init(i915, wal);
1251
1252 /* Wa_1409420604:tgl */
1253 if (IS_TGL_UY_GT_REVID(i915, TGL_REVID_A0, TGL_REVID_A0))
1254 wa_write_or(wal,
1255 SUBSLICE_UNIT_LEVEL_CLKGATE2,
1256 CPSSUNIT_CLKGATE_DIS);
1257
1258 /* Wa_1607087056:tgl also know as BUG:1409180338 */
1259 if (IS_TGL_UY_GT_REVID(i915, TGL_REVID_A0, TGL_REVID_A0))
1260 wa_write_or(wal,
1261 SLICE_UNIT_LEVEL_CLKGATE,
1262 L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
1263 }
1264
1265 static void
1266 dg1_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1267 {
1268 gen12_gt_workarounds_init(i915, wal);
1269
1270 /* Wa_1607087056:dg1 */
1271 if (IS_DG1_REVID(i915, DG1_REVID_A0, DG1_REVID_A0))
1272 wa_write_or(wal,
1273 SLICE_UNIT_LEVEL_CLKGATE,
1274 L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
1275
1276 /* Wa_1409420604:dg1 */
1277 if (IS_DG1(i915))
1278 wa_write_or(wal,
1279 SUBSLICE_UNIT_LEVEL_CLKGATE2,
1280 CPSSUNIT_CLKGATE_DIS);
1281
1282 /* Wa_1408615072:dg1 */
1283 /* Empirical testing shows this register is unaffected by engine reset. */
1284 if (IS_DG1(i915))
1285 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2,
1286 VSUNIT_CLKGATE_DIS_TGL);
1287 }
1288
1289 static void
1290 gt_init_workarounds(struct drm_i915_private *i915, struct i915_wa_list *wal)
1291 {
1292 if (IS_DG1(i915))
1293 dg1_gt_workarounds_init(i915, wal);
1294 else if (IS_TIGERLAKE(i915))
1295 tgl_gt_workarounds_init(i915, wal);
1296 else if (IS_GEN(i915, 12))
1297 gen12_gt_workarounds_init(i915, wal);
1298 else if (IS_GEN(i915, 11))
1299 icl_gt_workarounds_init(i915, wal);
1300 else if (IS_CANNONLAKE(i915))
1301 cnl_gt_workarounds_init(i915, wal);
1302 else if (IS_COFFEELAKE(i915) || IS_COMETLAKE(i915))
1303 cfl_gt_workarounds_init(i915, wal);
1304 else if (IS_GEMINILAKE(i915))
1305 glk_gt_workarounds_init(i915, wal);
1306 else if (IS_KABYLAKE(i915))
1307 kbl_gt_workarounds_init(i915, wal);
1308 else if (IS_BROXTON(i915))
1309 bxt_gt_workarounds_init(i915, wal);
1310 else if (IS_SKYLAKE(i915))
1311 skl_gt_workarounds_init(i915, wal);
1312 else if (IS_HASWELL(i915))
1313 hsw_gt_workarounds_init(i915, wal);
1314 else if (IS_VALLEYVIEW(i915))
1315 vlv_gt_workarounds_init(i915, wal);
1316 else if (IS_IVYBRIDGE(i915))
1317 ivb_gt_workarounds_init(i915, wal);
1318 else if (IS_GEN(i915, 6))
1319 snb_gt_workarounds_init(i915, wal);
1320 else if (IS_GEN(i915, 5))
1321 ilk_gt_workarounds_init(i915, wal);
1322 else if (IS_G4X(i915))
1323 g4x_gt_workarounds_init(i915, wal);
1324 else if (IS_GEN(i915, 4))
1325 gen4_gt_workarounds_init(i915, wal);
1326 else if (INTEL_GEN(i915) <= 8)
1327 return;
1328 else
1329 MISSING_CASE(INTEL_GEN(i915));
1330 }
1331
1332 void intel_gt_init_workarounds(struct drm_i915_private *i915)
1333 {
1334 struct i915_wa_list *wal = &i915->gt_wa_list;
1335
1336 wa_init_start(wal, "GT", "global");
1337 gt_init_workarounds(i915, wal);
1338 wa_init_finish(wal);
1339 }
1340
1341 static enum forcewake_domains
1342 wal_get_fw_for_rmw(struct intel_uncore *uncore, const struct i915_wa_list *wal)
1343 {
1344 enum forcewake_domains fw = 0;
1345 struct i915_wa *wa;
1346 unsigned int i;
1347
1348 for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1349 fw |= intel_uncore_forcewake_for_reg(uncore,
1350 wa->reg,
1351 FW_REG_READ |
1352 FW_REG_WRITE);
1353
1354 return fw;
1355 }
1356
1357 static bool
1358 wa_verify(const struct i915_wa *wa, u32 cur, const char *name, const char *from)
1359 {
1360 if ((cur ^ wa->set) & wa->read) {
1361 DRM_ERROR("%s workaround lost on %s! (%x=%x/%x, expected %x)\n",
1362 name, from, i915_mmio_reg_offset(wa->reg),
1363 cur, cur & wa->read, wa->set);
1364
1365 return false;
1366 }
1367
1368 return true;
1369 }
1370
1371 static void
1372 wa_list_apply(struct intel_uncore *uncore, const struct i915_wa_list *wal)
1373 {
1374 enum forcewake_domains fw;
1375 unsigned long flags;
1376 struct i915_wa *wa;
1377 unsigned int i;
1378
1379 if (!wal->count)
1380 return;
1381
1382 fw = wal_get_fw_for_rmw(uncore, wal);
1383
1384 spin_lock_irqsave(&uncore->lock, flags);
1385 intel_uncore_forcewake_get__locked(uncore, fw);
1386
1387 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
1388 if (wa->clr)
1389 intel_uncore_rmw_fw(uncore, wa->reg, wa->clr, wa->set);
1390 else
1391 intel_uncore_write_fw(uncore, wa->reg, wa->set);
1392 if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
1393 wa_verify(wa,
1394 intel_uncore_read_fw(uncore, wa->reg),
1395 wal->name, "application");
1396 }
1397
1398 intel_uncore_forcewake_put__locked(uncore, fw);
1399 spin_unlock_irqrestore(&uncore->lock, flags);
1400 }
1401
1402 void intel_gt_apply_workarounds(struct intel_gt *gt)
1403 {
1404 wa_list_apply(gt->uncore, &gt->i915->gt_wa_list);
1405 }
1406
1407 static bool wa_list_verify(struct intel_uncore *uncore,
1408 const struct i915_wa_list *wal,
1409 const char *from)
1410 {
1411 struct i915_wa *wa;
1412 unsigned int i;
1413 bool ok = true;
1414
1415 for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1416 ok &= wa_verify(wa,
1417 intel_uncore_read(uncore, wa->reg),
1418 wal->name, from);
1419
1420 return ok;
1421 }
1422
1423 bool intel_gt_verify_workarounds(struct intel_gt *gt, const char *from)
1424 {
1425 return wa_list_verify(gt->uncore, &gt->i915->gt_wa_list, from);
1426 }
1427
1428 static inline bool is_nonpriv_flags_valid(u32 flags)
1429 {
1430 /* Check only valid flag bits are set */
1431 if (flags & ~RING_FORCE_TO_NONPRIV_MASK_VALID)
1432 return false;
1433
1434 /* NB: Only 3 out of 4 enum values are valid for access field */
1435 if ((flags & RING_FORCE_TO_NONPRIV_ACCESS_MASK) ==
1436 RING_FORCE_TO_NONPRIV_ACCESS_INVALID)
1437 return false;
1438
1439 return true;
1440 }
1441
1442 static void
1443 whitelist_reg_ext(struct i915_wa_list *wal, i915_reg_t reg, u32 flags)
1444 {
1445 struct i915_wa wa = {
1446 .reg = reg
1447 };
1448
1449 if (GEM_DEBUG_WARN_ON(wal->count >= RING_MAX_NONPRIV_SLOTS))
1450 return;
1451
1452 if (GEM_DEBUG_WARN_ON(!is_nonpriv_flags_valid(flags)))
1453 return;
1454
1455 wa.reg.reg |= flags;
1456 _wa_add(wal, &wa);
1457 }
1458
1459 static void
1460 whitelist_reg(struct i915_wa_list *wal, i915_reg_t reg)
1461 {
1462 whitelist_reg_ext(wal, reg, RING_FORCE_TO_NONPRIV_ACCESS_RW);
1463 }
1464
1465 static void gen9_whitelist_build(struct i915_wa_list *w)
1466 {
1467 /* WaVFEStateAfterPipeControlwithMediaStateClear:skl,bxt,glk,cfl */
1468 whitelist_reg(w, GEN9_CTX_PREEMPT_REG);
1469
1470 /* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl,cfl,[cnl] */
1471 whitelist_reg(w, GEN8_CS_CHICKEN1);
1472
1473 /* WaAllowUMDToModifyHDCChicken1:skl,bxt,kbl,glk,cfl */
1474 whitelist_reg(w, GEN8_HDC_CHICKEN1);
1475
1476 /* WaSendPushConstantsFromMMIO:skl,bxt */
1477 whitelist_reg(w, COMMON_SLICE_CHICKEN2);
1478 }
1479
1480 static void skl_whitelist_build(struct intel_engine_cs *engine)
1481 {
1482 struct i915_wa_list *w = &engine->whitelist;
1483
1484 if (engine->class != RENDER_CLASS)
1485 return;
1486
1487 gen9_whitelist_build(w);
1488
1489 /* WaDisableLSQCROPERFforOCL:skl */
1490 whitelist_reg(w, GEN8_L3SQCREG4);
1491 }
1492
1493 static void bxt_whitelist_build(struct intel_engine_cs *engine)
1494 {
1495 if (engine->class != RENDER_CLASS)
1496 return;
1497
1498 gen9_whitelist_build(&engine->whitelist);
1499 }
1500
1501 static void kbl_whitelist_build(struct intel_engine_cs *engine)
1502 {
1503 struct i915_wa_list *w = &engine->whitelist;
1504
1505 if (engine->class != RENDER_CLASS)
1506 return;
1507
1508 gen9_whitelist_build(w);
1509
1510 /* WaDisableLSQCROPERFforOCL:kbl */
1511 whitelist_reg(w, GEN8_L3SQCREG4);
1512 }
1513
1514 static void glk_whitelist_build(struct intel_engine_cs *engine)
1515 {
1516 struct i915_wa_list *w = &engine->whitelist;
1517
1518 if (engine->class != RENDER_CLASS)
1519 return;
1520
1521 gen9_whitelist_build(w);
1522
1523 /* WA #0862: Userspace has to set "Barrier Mode" to avoid hangs. */
1524 whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1);
1525 }
1526
1527 static void cfl_whitelist_build(struct intel_engine_cs *engine)
1528 {
1529 struct i915_wa_list *w = &engine->whitelist;
1530
1531 if (engine->class != RENDER_CLASS)
1532 return;
1533
1534 gen9_whitelist_build(w);
1535
1536 /*
1537 * WaAllowPMDepthAndInvocationCountAccessFromUMD:cfl,whl,cml,aml
1538 *
1539 * This covers 4 register which are next to one another :
1540 * - PS_INVOCATION_COUNT
1541 * - PS_INVOCATION_COUNT_UDW
1542 * - PS_DEPTH_COUNT
1543 * - PS_DEPTH_COUNT_UDW
1544 */
1545 whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1546 RING_FORCE_TO_NONPRIV_ACCESS_RD |
1547 RING_FORCE_TO_NONPRIV_RANGE_4);
1548 }
1549
1550 static void cml_whitelist_build(struct intel_engine_cs *engine)
1551 {
1552 struct i915_wa_list *w = &engine->whitelist;
1553
1554 if (engine->class != RENDER_CLASS)
1555 whitelist_reg_ext(w,
1556 RING_CTX_TIMESTAMP(engine->mmio_base),
1557 RING_FORCE_TO_NONPRIV_ACCESS_RD);
1558
1559 cfl_whitelist_build(engine);
1560 }
1561
1562 static void cnl_whitelist_build(struct intel_engine_cs *engine)
1563 {
1564 struct i915_wa_list *w = &engine->whitelist;
1565
1566 if (engine->class != RENDER_CLASS)
1567 return;
1568
1569 /* WaEnablePreemptionGranularityControlByUMD:cnl */
1570 whitelist_reg(w, GEN8_CS_CHICKEN1);
1571 }
1572
1573 static void icl_whitelist_build(struct intel_engine_cs *engine)
1574 {
1575 struct i915_wa_list *w = &engine->whitelist;
1576
1577 switch (engine->class) {
1578 case RENDER_CLASS:
1579 /* WaAllowUMDToModifyHalfSliceChicken7:icl */
1580 whitelist_reg(w, GEN9_HALF_SLICE_CHICKEN7);
1581
1582 /* WaAllowUMDToModifySamplerMode:icl */
1583 whitelist_reg(w, GEN10_SAMPLER_MODE);
1584
1585 /* WaEnableStateCacheRedirectToCS:icl */
1586 whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1);
1587
1588 /*
1589 * WaAllowPMDepthAndInvocationCountAccessFromUMD:icl
1590 *
1591 * This covers 4 register which are next to one another :
1592 * - PS_INVOCATION_COUNT
1593 * - PS_INVOCATION_COUNT_UDW
1594 * - PS_DEPTH_COUNT
1595 * - PS_DEPTH_COUNT_UDW
1596 */
1597 whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1598 RING_FORCE_TO_NONPRIV_ACCESS_RD |
1599 RING_FORCE_TO_NONPRIV_RANGE_4);
1600 break;
1601
1602 case VIDEO_DECODE_CLASS:
1603 /* hucStatusRegOffset */
1604 whitelist_reg_ext(w, _MMIO(0x2000 + engine->mmio_base),
1605 RING_FORCE_TO_NONPRIV_ACCESS_RD);
1606 /* hucUKernelHdrInfoRegOffset */
1607 whitelist_reg_ext(w, _MMIO(0x2014 + engine->mmio_base),
1608 RING_FORCE_TO_NONPRIV_ACCESS_RD);
1609 /* hucStatus2RegOffset */
1610 whitelist_reg_ext(w, _MMIO(0x23B0 + engine->mmio_base),
1611 RING_FORCE_TO_NONPRIV_ACCESS_RD);
1612 whitelist_reg_ext(w,
1613 RING_CTX_TIMESTAMP(engine->mmio_base),
1614 RING_FORCE_TO_NONPRIV_ACCESS_RD);
1615 break;
1616
1617 default:
1618 whitelist_reg_ext(w,
1619 RING_CTX_TIMESTAMP(engine->mmio_base),
1620 RING_FORCE_TO_NONPRIV_ACCESS_RD);
1621 break;
1622 }
1623 }
1624
1625 static void tgl_whitelist_build(struct intel_engine_cs *engine)
1626 {
1627 struct i915_wa_list *w = &engine->whitelist;
1628
1629 switch (engine->class) {
1630 case RENDER_CLASS:
1631 /*
1632 * WaAllowPMDepthAndInvocationCountAccessFromUMD:tgl
1633 * Wa_1408556865:tgl
1634 *
1635 * This covers 4 registers which are next to one another :
1636 * - PS_INVOCATION_COUNT
1637 * - PS_INVOCATION_COUNT_UDW
1638 * - PS_DEPTH_COUNT
1639 * - PS_DEPTH_COUNT_UDW
1640 */
1641 whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1642 RING_FORCE_TO_NONPRIV_ACCESS_RD |
1643 RING_FORCE_TO_NONPRIV_RANGE_4);
1644
1645 /* Wa_1808121037:tgl */
1646 whitelist_reg(w, GEN7_COMMON_SLICE_CHICKEN1);
1647
1648 /* Wa_1806527549:tgl */
1649 whitelist_reg(w, HIZ_CHICKEN);
1650 break;
1651 default:
1652 whitelist_reg_ext(w,
1653 RING_CTX_TIMESTAMP(engine->mmio_base),
1654 RING_FORCE_TO_NONPRIV_ACCESS_RD);
1655 break;
1656 }
1657 }
1658
1659 static void dg1_whitelist_build(struct intel_engine_cs *engine)
1660 {
1661 struct i915_wa_list *w = &engine->whitelist;
1662
1663 tgl_whitelist_build(engine);
1664
1665 /* GEN:BUG:1409280441:dg1 */
1666 if (IS_DG1_REVID(engine->i915, DG1_REVID_A0, DG1_REVID_A0) &&
1667 (engine->class == RENDER_CLASS ||
1668 engine->class == COPY_ENGINE_CLASS))
1669 whitelist_reg_ext(w, RING_ID(engine->mmio_base),
1670 RING_FORCE_TO_NONPRIV_ACCESS_RD);
1671 }
1672
1673 void intel_engine_init_whitelist(struct intel_engine_cs *engine)
1674 {
1675 struct drm_i915_private *i915 = engine->i915;
1676 struct i915_wa_list *w = &engine->whitelist;
1677
1678 wa_init_start(w, "whitelist", engine->name);
1679
1680 if (IS_DG1(i915))
1681 dg1_whitelist_build(engine);
1682 else if (IS_GEN(i915, 12))
1683 tgl_whitelist_build(engine);
1684 else if (IS_GEN(i915, 11))
1685 icl_whitelist_build(engine);
1686 else if (IS_CANNONLAKE(i915))
1687 cnl_whitelist_build(engine);
1688 else if (IS_COMETLAKE(i915))
1689 cml_whitelist_build(engine);
1690 else if (IS_COFFEELAKE(i915))
1691 cfl_whitelist_build(engine);
1692 else if (IS_GEMINILAKE(i915))
1693 glk_whitelist_build(engine);
1694 else if (IS_KABYLAKE(i915))
1695 kbl_whitelist_build(engine);
1696 else if (IS_BROXTON(i915))
1697 bxt_whitelist_build(engine);
1698 else if (IS_SKYLAKE(i915))
1699 skl_whitelist_build(engine);
1700 else if (INTEL_GEN(i915) <= 8)
1701 return;
1702 else
1703 MISSING_CASE(INTEL_GEN(i915));
1704
1705 wa_init_finish(w);
1706 }
1707
1708 void intel_engine_apply_whitelist(struct intel_engine_cs *engine)
1709 {
1710 const struct i915_wa_list *wal = &engine->whitelist;
1711 struct intel_uncore *uncore = engine->uncore;
1712 const u32 base = engine->mmio_base;
1713 struct i915_wa *wa;
1714 unsigned int i;
1715
1716 if (!wal->count)
1717 return;
1718
1719 for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1720 intel_uncore_write(uncore,
1721 RING_FORCE_TO_NONPRIV(base, i),
1722 i915_mmio_reg_offset(wa->reg));
1723
1724 /* And clear the rest just in case of garbage */
1725 for (; i < RING_MAX_NONPRIV_SLOTS; i++)
1726 intel_uncore_write(uncore,
1727 RING_FORCE_TO_NONPRIV(base, i),
1728 i915_mmio_reg_offset(RING_NOPID(base)));
1729 }
1730
1731 static void
1732 rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
1733 {
1734 struct drm_i915_private *i915 = engine->i915;
1735
1736 if (IS_DG1_REVID(i915, DG1_REVID_A0, DG1_REVID_A0) ||
1737 IS_TGL_UY_GT_REVID(i915, TGL_REVID_A0, TGL_REVID_A0)) {
1738 /*
1739 * Wa_1607138336:tgl[a0],dg1[a0]
1740 * Wa_1607063988:tgl[a0],dg1[a0]
1741 */
1742 wa_write_or(wal,
1743 GEN9_CTX_PREEMPT_REG,
1744 GEN12_DISABLE_POSH_BUSY_FF_DOP_CG);
1745 }
1746
1747 if (IS_TGL_UY_GT_REVID(i915, TGL_REVID_A0, TGL_REVID_A0)) {
1748 /*
1749 * Wa_1606679103:tgl
1750 * (see also Wa_1606682166:icl)
1751 */
1752 wa_write_or(wal,
1753 GEN7_SARCHKMD,
1754 GEN7_DISABLE_SAMPLER_PREFETCH);
1755
1756 /* Wa_1408615072:tgl */
1757 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2,
1758 VSUNIT_CLKGATE_DIS_TGL);
1759 }
1760
1761 if (IS_DG1(i915) || IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) {
1762 /* Wa_1606931601:tgl,rkl,dg1 */
1763 wa_masked_en(wal, GEN7_ROW_CHICKEN2, GEN12_DISABLE_EARLY_READ);
1764
1765 /*
1766 * Wa_1407928979:tgl A*
1767 * Wa_18011464164:tgl[B0+],dg1[B0+]
1768 * Wa_22010931296:tgl[B0+],dg1[B0+]
1769 * Wa_14010919138:rkl, dg1
1770 */
1771 wa_write_or(wal, GEN7_FF_THREAD_MODE,
1772 GEN12_FF_TESSELATION_DOP_GATE_DISABLE);
1773 }
1774
1775 if (IS_DG1_REVID(i915, DG1_REVID_A0, DG1_REVID_A0) ||
1776 IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) {
1777 /* Wa_1409804808:tgl,rkl,dg1[a0] */
1778 wa_masked_en(wal, GEN7_ROW_CHICKEN2,
1779 GEN12_PUSH_CONST_DEREF_HOLD_DIS);
1780
1781 /*
1782 * Wa_1409085225:tgl
1783 * Wa_14010229206:tgl,rkl,dg1[a0]
1784 */
1785 wa_masked_en(wal, GEN9_ROW_CHICKEN4, GEN12_DISABLE_TDL_PUSH);
1786
1787 /*
1788 * Wa_1607030317:tgl
1789 * Wa_1607186500:tgl
1790 * Wa_1607297627:tgl,rkl,dg1[a0]
1791 *
1792 * On TGL and RKL there are multiple entries for this WA in the
1793 * BSpec; some indicate this is an A0-only WA, others indicate
1794 * it applies to all steppings so we trust the "all steppings."
1795 * For DG1 this only applies to A0.
1796 */
1797 wa_masked_en(wal,
1798 GEN6_RC_SLEEP_PSMI_CONTROL,
1799 GEN12_WAIT_FOR_EVENT_POWER_DOWN_DISABLE |
1800 GEN8_RC_SEMA_IDLE_MSG_DISABLE);
1801
1802 /*
1803 * Wa_1606700617:tgl
1804 * Wa_22010271021:tgl,rkl
1805 */
1806 wa_masked_en(wal,
1807 GEN9_CS_DEBUG_MODE1,
1808 FF_DOP_CLOCK_GATE_DISABLE);
1809 }
1810
1811 if (IS_GEN(i915, 12)) {
1812 /* Wa_1406941453:gen12 */
1813 wa_masked_en(wal,
1814 GEN10_SAMPLER_MODE,
1815 ENABLE_SMALLPL);
1816 }
1817
1818 if (IS_GEN(i915, 11)) {
1819 /* This is not an Wa. Enable for better image quality */
1820 wa_masked_en(wal,
1821 _3D_CHICKEN3,
1822 _3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE);
1823
1824 /* WaPipelineFlushCoherentLines:icl */
1825 wa_write_or(wal,
1826 GEN8_L3SQCREG4,
1827 GEN8_LQSC_FLUSH_COHERENT_LINES);
1828
1829 /*
1830 * Wa_1405543622:icl
1831 * Formerly known as WaGAPZPriorityScheme
1832 */
1833 wa_write_or(wal,
1834 GEN8_GARBCNTL,
1835 GEN11_ARBITRATION_PRIO_ORDER_MASK);
1836
1837 /*
1838 * Wa_1604223664:icl
1839 * Formerly known as WaL3BankAddressHashing
1840 */
1841 wa_write_masked_or(wal,
1842 GEN8_GARBCNTL,
1843 GEN11_HASH_CTRL_EXCL_MASK,
1844 GEN11_HASH_CTRL_EXCL_BIT0);
1845 wa_write_masked_or(wal,
1846 GEN11_GLBLINVL,
1847 GEN11_BANK_HASH_ADDR_EXCL_MASK,
1848 GEN11_BANK_HASH_ADDR_EXCL_BIT0);
1849
1850 /*
1851 * Wa_1405733216:icl
1852 * Formerly known as WaDisableCleanEvicts
1853 */
1854 wa_write_or(wal,
1855 GEN8_L3SQCREG4,
1856 GEN11_LQSC_CLEAN_EVICT_DISABLE);
1857
1858 /* WaForwardProgressSoftReset:icl */
1859 wa_write_or(wal,
1860 GEN10_SCRATCH_LNCF2,
1861 PMFLUSHDONE_LNICRSDROP |
1862 PMFLUSH_GAPL3UNBLOCK |
1863 PMFLUSHDONE_LNEBLK);
1864
1865 /* Wa_1406609255:icl (pre-prod) */
1866 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0))
1867 wa_write_or(wal,
1868 GEN7_SARCHKMD,
1869 GEN7_DISABLE_DEMAND_PREFETCH);
1870
1871 /* Wa_1606682166:icl */
1872 wa_write_or(wal,
1873 GEN7_SARCHKMD,
1874 GEN7_DISABLE_SAMPLER_PREFETCH);
1875
1876 /* Wa_1409178092:icl */
1877 wa_write_masked_or(wal,
1878 GEN11_SCRATCH2,
1879 GEN11_COHERENT_PARTIAL_WRITE_MERGE_ENABLE,
1880 0);
1881
1882 /* WaEnable32PlaneMode:icl */
1883 wa_masked_en(wal, GEN9_CSFE_CHICKEN1_RCS,
1884 GEN11_ENABLE_32_PLANE_MODE);
1885
1886 /*
1887 * Wa_1408615072:icl,ehl (vsunit)
1888 * Wa_1407596294:icl,ehl (hsunit)
1889 */
1890 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE,
1891 VSUNIT_CLKGATE_DIS | HSUNIT_CLKGATE_DIS);
1892
1893 /* Wa_1407352427:icl,ehl */
1894 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2,
1895 PSDUNIT_CLKGATE_DIS);
1896
1897 /* Wa_1406680159:icl,ehl */
1898 wa_write_or(wal,
1899 SUBSLICE_UNIT_LEVEL_CLKGATE,
1900 GWUNIT_CLKGATE_DIS);
1901
1902 /*
1903 * Wa_1408767742:icl[a2..forever],ehl[all]
1904 * Wa_1605460711:icl[a0..c0]
1905 */
1906 wa_write_or(wal,
1907 GEN7_FF_THREAD_MODE,
1908 GEN12_FF_TESSELATION_DOP_GATE_DISABLE);
1909
1910 /* Wa_22010271021:ehl */
1911 if (IS_JSL_EHL(i915))
1912 wa_masked_en(wal,
1913 GEN9_CS_DEBUG_MODE1,
1914 FF_DOP_CLOCK_GATE_DISABLE);
1915 }
1916
1917 if (IS_GEN_RANGE(i915, 9, 12)) {
1918 /* FtrPerCtxtPreemptionGranularityControl:skl,bxt,kbl,cfl,cnl,icl,tgl */
1919 wa_masked_en(wal,
1920 GEN7_FF_SLICE_CS_CHICKEN1,
1921 GEN9_FFSC_PERCTX_PREEMPT_CTRL);
1922 }
1923
1924 if (IS_SKYLAKE(i915) ||
1925 IS_KABYLAKE(i915) ||
1926 IS_COFFEELAKE(i915) ||
1927 IS_COMETLAKE(i915)) {
1928 /* WaEnableGapsTsvCreditFix:skl,kbl,cfl */
1929 wa_write_or(wal,
1930 GEN8_GARBCNTL,
1931 GEN9_GAPS_TSV_CREDIT_DISABLE);
1932 }
1933
1934 if (IS_BROXTON(i915)) {
1935 /* WaDisablePooledEuLoadBalancingFix:bxt */
1936 wa_masked_en(wal,
1937 FF_SLICE_CS_CHICKEN2,
1938 GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE);
1939 }
1940
1941 if (IS_GEN(i915, 9)) {
1942 /* WaContextSwitchWithConcurrentTLBInvalidate:skl,bxt,kbl,glk,cfl */
1943 wa_masked_en(wal,
1944 GEN9_CSFE_CHICKEN1_RCS,
1945 GEN9_PREEMPT_GPGPU_SYNC_SWITCH_DISABLE);
1946
1947 /* WaEnableLbsSlaRetryTimerDecrement:skl,bxt,kbl,glk,cfl */
1948 wa_write_or(wal,
1949 BDW_SCRATCH1,
1950 GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE);
1951
1952 /* WaProgramL3SqcReg1DefaultForPerf:bxt,glk */
1953 if (IS_GEN9_LP(i915))
1954 wa_write_masked_or(wal,
1955 GEN8_L3SQCREG1,
1956 L3_PRIO_CREDITS_MASK,
1957 L3_GENERAL_PRIO_CREDITS(62) |
1958 L3_HIGH_PRIO_CREDITS(2));
1959
1960 /* WaOCLCoherentLineFlush:skl,bxt,kbl,cfl */
1961 wa_write_or(wal,
1962 GEN8_L3SQCREG4,
1963 GEN8_LQSC_FLUSH_COHERENT_LINES);
1964 }
1965
1966 if (IS_GEN(i915, 7))
1967 /* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */
1968 wa_masked_en(wal,
1969 GFX_MODE_GEN7,
1970 GFX_TLB_INVALIDATE_EXPLICIT | GFX_REPLAY_MODE);
1971
1972 if (IS_GEN_RANGE(i915, 6, 7))
1973 /*
1974 * We need to disable the AsyncFlip performance optimisations in
1975 * order to use MI_WAIT_FOR_EVENT within the CS. It should
1976 * already be programmed to '1' on all products.
1977 *
1978 * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv
1979 */
1980 wa_masked_en(wal,
1981 MI_MODE,
1982 ASYNC_FLIP_PERF_DISABLE);
1983
1984 if (IS_GEN(i915, 6)) {
1985 /*
1986 * Required for the hardware to program scanline values for
1987 * waiting
1988 * WaEnableFlushTlbInvalidationMode:snb
1989 */
1990 wa_masked_en(wal,
1991 GFX_MODE,
1992 GFX_TLB_INVALIDATE_EXPLICIT);
1993
1994 /*
1995 * From the Sandybridge PRM, volume 1 part 3, page 24:
1996 * "If this bit is set, STCunit will have LRA as replacement
1997 * policy. [...] This bit must be reset. LRA replacement
1998 * policy is not supported."
1999 */
2000 wa_masked_dis(wal,
2001 CACHE_MODE_0,
2002 CM0_STC_EVICT_DISABLE_LRA_SNB);
2003 }
2004
2005 if (IS_GEN_RANGE(i915, 4, 6))
2006 /* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */
2007 wa_add(wal, MI_MODE,
2008 0, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH),
2009 /* XXX bit doesn't stick on Broadwater */
2010 IS_I965G(i915) ? 0 : VS_TIMER_DISPATCH);
2011
2012 if (IS_GEN(i915, 4))
2013 /*
2014 * Disable CONSTANT_BUFFER before it is loaded from the context
2015 * image. For as it is loaded, it is executed and the stored
2016 * address may no longer be valid, leading to a GPU hang.
2017 *
2018 * This imposes the requirement that userspace reload their
2019 * CONSTANT_BUFFER on every batch, fortunately a requirement
2020 * they are already accustomed to from before contexts were
2021 * enabled.
2022 */
2023 wa_add(wal, ECOSKPD,
2024 0, _MASKED_BIT_ENABLE(ECO_CONSTANT_BUFFER_SR_DISABLE),
2025 0 /* XXX bit doesn't stick on Broadwater */);
2026 }
2027
2028 static void
2029 xcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
2030 {
2031 struct drm_i915_private *i915 = engine->i915;
2032
2033 /* WaKBLVECSSemaphoreWaitPoll:kbl */
2034 if (IS_KBL_GT_REVID(i915, KBL_REVID_A0, KBL_REVID_E0)) {
2035 wa_write(wal,
2036 RING_SEMA_WAIT_POLL(engine->mmio_base),
2037 1);
2038 }
2039 }
2040
2041 static void
2042 engine_init_workarounds(struct intel_engine_cs *engine, struct i915_wa_list *wal)
2043 {
2044 if (I915_SELFTEST_ONLY(INTEL_GEN(engine->i915) < 4))
2045 return;
2046
2047 if (engine->class == RENDER_CLASS)
2048 rcs_engine_wa_init(engine, wal);
2049 else
2050 xcs_engine_wa_init(engine, wal);
2051 }
2052
2053 void intel_engine_init_workarounds(struct intel_engine_cs *engine)
2054 {
2055 struct i915_wa_list *wal = &engine->wa_list;
2056
2057 if (INTEL_GEN(engine->i915) < 4)
2058 return;
2059
2060 wa_init_start(wal, "engine", engine->name);
2061 engine_init_workarounds(engine, wal);
2062 wa_init_finish(wal);
2063 }
2064
2065 void intel_engine_apply_workarounds(struct intel_engine_cs *engine)
2066 {
2067 wa_list_apply(engine->uncore, &engine->wa_list);
2068 }
2069
2070 static struct i915_vma *
2071 create_scratch(struct i915_address_space *vm, int count)
2072 {
2073 struct drm_i915_gem_object *obj;
2074 struct i915_vma *vma;
2075 unsigned int size;
2076 int err;
2077
2078 size = round_up(count * sizeof(u32), PAGE_SIZE);
2079 obj = i915_gem_object_create_internal(vm->i915, size);
2080 if (IS_ERR(obj))
2081 return ERR_CAST(obj);
2082
2083 i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
2084
2085 vma = i915_vma_instance(obj, vm, NULL);
2086 if (IS_ERR(vma)) {
2087 err = PTR_ERR(vma);
2088 goto err_obj;
2089 }
2090
2091 err = i915_vma_pin(vma, 0, 0,
2092 i915_vma_is_ggtt(vma) ? PIN_GLOBAL : PIN_USER);
2093 if (err)
2094 goto err_obj;
2095
2096 return vma;
2097
2098 err_obj:
2099 i915_gem_object_put(obj);
2100 return ERR_PTR(err);
2101 }
2102
2103 struct mcr_range {
2104 u32 start;
2105 u32 end;
2106 };
2107
2108 static const struct mcr_range mcr_ranges_gen8[] = {
2109 { .start = 0x5500, .end = 0x55ff },
2110 { .start = 0x7000, .end = 0x7fff },
2111 { .start = 0x9400, .end = 0x97ff },
2112 { .start = 0xb000, .end = 0xb3ff },
2113 { .start = 0xe000, .end = 0xe7ff },
2114 {},
2115 };
2116
2117 static const struct mcr_range mcr_ranges_gen12[] = {
2118 { .start = 0x8150, .end = 0x815f },
2119 { .start = 0x9520, .end = 0x955f },
2120 { .start = 0xb100, .end = 0xb3ff },
2121 { .start = 0xde80, .end = 0xe8ff },
2122 { .start = 0x24a00, .end = 0x24a7f },
2123 {},
2124 };
2125
2126 static bool mcr_range(struct drm_i915_private *i915, u32 offset)
2127 {
2128 const struct mcr_range *mcr_ranges;
2129 int i;
2130
2131 if (INTEL_GEN(i915) >= 12)
2132 mcr_ranges = mcr_ranges_gen12;
2133 else if (INTEL_GEN(i915) >= 8)
2134 mcr_ranges = mcr_ranges_gen8;
2135 else
2136 return false;
2137
2138 /*
2139 * Registers in these ranges are affected by the MCR selector
2140 * which only controls CPU initiated MMIO. Routing does not
2141 * work for CS access so we cannot verify them on this path.
2142 */
2143 for (i = 0; mcr_ranges[i].start; i++)
2144 if (offset >= mcr_ranges[i].start &&
2145 offset <= mcr_ranges[i].end)
2146 return true;
2147
2148 return false;
2149 }
2150
2151 static int
2152 wa_list_srm(struct i915_request *rq,
2153 const struct i915_wa_list *wal,
2154 struct i915_vma *vma)
2155 {
2156 struct drm_i915_private *i915 = rq->engine->i915;
2157 unsigned int i, count = 0;
2158 const struct i915_wa *wa;
2159 u32 srm, *cs;
2160
2161 srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
2162 if (INTEL_GEN(i915) >= 8)
2163 srm++;
2164
2165 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
2166 if (!mcr_range(i915, i915_mmio_reg_offset(wa->reg)))
2167 count++;
2168 }
2169
2170 cs = intel_ring_begin(rq, 4 * count);
2171 if (IS_ERR(cs))
2172 return PTR_ERR(cs);
2173
2174 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
2175 u32 offset = i915_mmio_reg_offset(wa->reg);
2176
2177 if (mcr_range(i915, offset))
2178 continue;
2179
2180 *cs++ = srm;
2181 *cs++ = offset;
2182 *cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i;
2183 *cs++ = 0;
2184 }
2185 intel_ring_advance(rq, cs);
2186
2187 return 0;
2188 }
2189
2190 static int engine_wa_list_verify(struct intel_context *ce,
2191 const struct i915_wa_list * const wal,
2192 const char *from)
2193 {
2194 const struct i915_wa *wa;
2195 struct i915_request *rq;
2196 struct i915_vma *vma;
2197 struct i915_gem_ww_ctx ww;
2198 unsigned int i;
2199 u32 *results;
2200 int err;
2201
2202 if (!wal->count)
2203 return 0;
2204
2205 vma = create_scratch(&ce->engine->gt->ggtt->vm, wal->count);
2206 if (IS_ERR(vma))
2207 return PTR_ERR(vma);
2208
2209 intel_engine_pm_get(ce->engine);
2210 i915_gem_ww_ctx_init(&ww, false);
2211 retry:
2212 err = i915_gem_object_lock(vma->obj, &ww);
2213 if (err == 0)
2214 err = intel_context_pin_ww(ce, &ww);
2215 if (err)
2216 goto err_pm;
2217
2218 rq = i915_request_create(ce);
2219 if (IS_ERR(rq)) {
2220 err = PTR_ERR(rq);
2221 goto err_unpin;
2222 }
2223
2224 err = i915_request_await_object(rq, vma->obj, true);
2225 if (err == 0)
2226 err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
2227 if (err == 0)
2228 err = wa_list_srm(rq, wal, vma);
2229
2230 i915_request_get(rq);
2231 if (err)
2232 i915_request_set_error_once(rq, err);
2233 i915_request_add(rq);
2234
2235 if (err)
2236 goto err_rq;
2237
2238 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
2239 err = -ETIME;
2240 goto err_rq;
2241 }
2242
2243 results = i915_gem_object_pin_map(vma->obj, I915_MAP_WB);
2244 if (IS_ERR(results)) {
2245 err = PTR_ERR(results);
2246 goto err_rq;
2247 }
2248
2249 err = 0;
2250 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
2251 if (mcr_range(rq->engine->i915, i915_mmio_reg_offset(wa->reg)))
2252 continue;
2253
2254 if (!wa_verify(wa, results[i], wal->name, from))
2255 err = -ENXIO;
2256 }
2257
2258 i915_gem_object_unpin_map(vma->obj);
2259
2260 err_rq:
2261 i915_request_put(rq);
2262 err_unpin:
2263 intel_context_unpin(ce);
2264 err_pm:
2265 if (err == -EDEADLK) {
2266 err = i915_gem_ww_ctx_backoff(&ww);
2267 if (!err)
2268 goto retry;
2269 }
2270 i915_gem_ww_ctx_fini(&ww);
2271 intel_engine_pm_put(ce->engine);
2272 i915_vma_unpin(vma);
2273 i915_vma_put(vma);
2274 return err;
2275 }
2276
2277 int intel_engine_verify_workarounds(struct intel_engine_cs *engine,
2278 const char *from)
2279 {
2280 return engine_wa_list_verify(engine->kernel_context,
2281 &engine->wa_list,
2282 from);
2283 }
2284
2285 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
2286 #include "selftest_workarounds.c"
2287 #endif
Linux with added FPC-III support