?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
treewide: remove redundant IS_ERR() before error code check
[linux/fpc-iii.git] / drivers / gpu / drm / i915 / gt / intel_workarounds.c
blob4e292d4bf7b9d2aeaf79dccb8b99992b62a3c046
1 /*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright © 2014-2018 Intel Corporation
5 */
6
7 #include "i915_drv.h"
8 #include "intel_context.h"
9 #include "intel_engine_pm.h"
10 #include "intel_gt.h"
11 #include "intel_ring.h"
12 #include "intel_workarounds.h"
13
14 /**
15 * DOC: Hardware workarounds
16 *
17 * This file is intended as a central place to implement most [1]_ of the
18 * required workarounds for hardware to work as originally intended. They fall
19 * in five basic categories depending on how/when they are applied:
20 *
21 * - Workarounds that touch registers that are saved/restored to/from the HW
22 * context image. The list is emitted (via Load Register Immediate commands)
23 * everytime a new context is created.
24 * - GT workarounds. The list of these WAs is applied whenever these registers
25 * revert to default values (on GPU reset, suspend/resume [2]_, etc..).
26 * - Display workarounds. The list is applied during display clock-gating
27 * initialization.
28 * - Workarounds that whitelist a privileged register, so that UMDs can manage
29 * them directly. This is just a special case of a MMMIO workaround (as we
30 * write the list of these to/be-whitelisted registers to some special HW
31 * registers).
32 * - Workaround batchbuffers, that get executed automatically by the hardware
33 * on every HW context restore.
34 *
35 * .. [1] Please notice that there are other WAs that, due to their nature,
36 * cannot be applied from a central place. Those are peppered around the rest
37 * of the code, as needed.
38 *
39 * .. [2] Technically, some registers are powercontext saved & restored, so they
40 * survive a suspend/resume. In practice, writing them again is not too
41 * costly and simplifies things. We can revisit this in the future.
42 *
43 * Layout
44 * ~~~~~~
45 *
46 * Keep things in this file ordered by WA type, as per the above (context, GT,
47 * display, register whitelist, batchbuffer). Then, inside each type, keep the
48 * following order:
49 *
50 * - Infrastructure functions and macros
51 * - WAs per platform in standard gen/chrono order
52 * - Public functions to init or apply the given workaround type.
53 */
54
55 static void wa_init_start(struct i915_wa_list *wal, const char *name, const char *engine_name)
56 {
57 wal->name = name;
58 wal->engine_name = engine_name;
59 }
60
61 #define WA_LIST_CHUNK (1 << 4)
62
63 static void wa_init_finish(struct i915_wa_list *wal)
64 {
65 /* Trim unused entries. */
66 if (!IS_ALIGNED(wal->count, WA_LIST_CHUNK)) {
67 struct i915_wa *list = kmemdup(wal->list,
68 wal->count * sizeof(*list),
69 GFP_KERNEL);
70
71 if (list) {
72 kfree(wal->list);
73 wal->list = list;
74 }
75 }
76
77 if (!wal->count)
78 return;
79
80 DRM_DEBUG_DRIVER("Initialized %u %s workarounds on %s\n",
81 wal->wa_count, wal->name, wal->engine_name);
82 }
83
84 static void _wa_add(struct i915_wa_list *wal, const struct i915_wa *wa)
85 {
86 unsigned int addr = i915_mmio_reg_offset(wa->reg);
87 unsigned int start = 0, end = wal->count;
88 const unsigned int grow = WA_LIST_CHUNK;
89 struct i915_wa *wa_;
90
91 GEM_BUG_ON(!is_power_of_2(grow));
92
93 if (IS_ALIGNED(wal->count, grow)) { /* Either uninitialized or full. */
94 struct i915_wa *list;
95
96 list = kmalloc_array(ALIGN(wal->count + 1, grow), sizeof(*wa),
97 GFP_KERNEL);
98 if (!list) {
99 DRM_ERROR("No space for workaround init!\n");
100 return;
101 }
102
103 if (wal->list)
104 memcpy(list, wal->list, sizeof(*wa) * wal->count);
105
106 wal->list = list;
107 }
108
109 while (start < end) {
110 unsigned int mid = start + (end - start) / 2;
111
112 if (i915_mmio_reg_offset(wal->list[mid].reg) < addr) {
113 start = mid + 1;
114 } else if (i915_mmio_reg_offset(wal->list[mid].reg) > addr) {
115 end = mid;
116 } else {
117 wa_ = &wal->list[mid];
118
119 if ((wa->mask & ~wa_->mask) == 0) {
120 DRM_ERROR("Discarding overwritten w/a for reg %04x (mask: %08x, value: %08x)\n",
121 i915_mmio_reg_offset(wa_->reg),
122 wa_->mask, wa_->val);
123
124 wa_->val &= ~wa->mask;
125 }
126
127 wal->wa_count++;
128 wa_->val |= wa->val;
129 wa_->mask |= wa->mask;
130 wa_->read |= wa->read;
131 return;
132 }
133 }
134
135 wal->wa_count++;
136 wa_ = &wal->list[wal->count++];
137 *wa_ = *wa;
138
139 while (wa_-- > wal->list) {
140 GEM_BUG_ON(i915_mmio_reg_offset(wa_[0].reg) ==
141 i915_mmio_reg_offset(wa_[1].reg));
142 if (i915_mmio_reg_offset(wa_[1].reg) >
143 i915_mmio_reg_offset(wa_[0].reg))
144 break;
145
146 swap(wa_[1], wa_[0]);
147 }
148 }
149
150 static void wa_add(struct i915_wa_list *wal, i915_reg_t reg, u32 mask,
151 u32 val, u32 read_mask)
152 {
153 struct i915_wa wa = {
154 .reg = reg,
155 .mask = mask,
156 .val = val,
157 .read = read_mask,
158 };
159
160 _wa_add(wal, &wa);
161 }
162
163 static void
164 wa_write_masked_or(struct i915_wa_list *wal, i915_reg_t reg, u32 mask,
165 u32 val)
166 {
167 wa_add(wal, reg, mask, val, mask);
168 }
169
170 static void
171 wa_masked_en(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
172 {
173 wa_write_masked_or(wal, reg, val, _MASKED_BIT_ENABLE(val));
174 }
175
176 static void
177 wa_write(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
178 {
179 wa_write_masked_or(wal, reg, ~0, val);
180 }
181
182 static void
183 wa_write_or(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
184 {
185 wa_write_masked_or(wal, reg, val, val);
186 }
187
188 #define WA_SET_BIT_MASKED(addr, mask) \
189 wa_write_masked_or(wal, (addr), (mask), _MASKED_BIT_ENABLE(mask))
190
191 #define WA_CLR_BIT_MASKED(addr, mask) \
192 wa_write_masked_or(wal, (addr), (mask), _MASKED_BIT_DISABLE(mask))
193
194 #define WA_SET_FIELD_MASKED(addr, mask, value) \
195 wa_write_masked_or(wal, (addr), (mask), _MASKED_FIELD((mask), (value)))
196
197 static void gen8_ctx_workarounds_init(struct intel_engine_cs *engine,
198 struct i915_wa_list *wal)
199 {
200 WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING);
201
202 /* WaDisableAsyncFlipPerfMode:bdw,chv */
203 WA_SET_BIT_MASKED(MI_MODE, ASYNC_FLIP_PERF_DISABLE);
204
205 /* WaDisablePartialInstShootdown:bdw,chv */
206 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
207 PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
208
209 /* Use Force Non-Coherent whenever executing a 3D context. This is a
210 * workaround for for a possible hang in the unlikely event a TLB
211 * invalidation occurs during a PSD flush.
212 */
213 /* WaForceEnableNonCoherent:bdw,chv */
214 /* WaHdcDisableFetchWhenMasked:bdw,chv */
215 WA_SET_BIT_MASKED(HDC_CHICKEN0,
216 HDC_DONOT_FETCH_MEM_WHEN_MASKED |
217 HDC_FORCE_NON_COHERENT);
218
219 /* From the Haswell PRM, Command Reference: Registers, CACHE_MODE_0:
220 * "The Hierarchical Z RAW Stall Optimization allows non-overlapping
221 * polygons in the same 8x4 pixel/sample area to be processed without
222 * stalling waiting for the earlier ones to write to Hierarchical Z
223 * buffer."
224 *
225 * This optimization is off by default for BDW and CHV; turn it on.
226 */
227 WA_CLR_BIT_MASKED(CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE);
228
229 /* Wa4x4STCOptimizationDisable:bdw,chv */
230 WA_SET_BIT_MASKED(CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE);
231
232 /*
233 * BSpec recommends 8x4 when MSAA is used,
234 * however in practice 16x4 seems fastest.
235 *
236 * Note that PS/WM thread counts depend on the WIZ hashing
237 * disable bit, which we don't touch here, but it's good
238 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
239 */
240 WA_SET_FIELD_MASKED(GEN7_GT_MODE,
241 GEN6_WIZ_HASHING_MASK,
242 GEN6_WIZ_HASHING_16x4);
243 }
244
245 static void bdw_ctx_workarounds_init(struct intel_engine_cs *engine,
246 struct i915_wa_list *wal)
247 {
248 struct drm_i915_private *i915 = engine->i915;
249
250 gen8_ctx_workarounds_init(engine, wal);
251
252 /* WaDisableThreadStallDopClockGating:bdw (pre-production) */
253 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
254
255 /* WaDisableDopClockGating:bdw
256 *
257 * Also see the related UCGTCL1 write in bdw_init_clock_gating()
258 * to disable EUTC clock gating.
259 */
260 WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
261 DOP_CLOCK_GATING_DISABLE);
262
263 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
264 GEN8_SAMPLER_POWER_BYPASS_DIS);
265
266 WA_SET_BIT_MASKED(HDC_CHICKEN0,
267 /* WaForceContextSaveRestoreNonCoherent:bdw */
268 HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
269 /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */
270 (IS_BDW_GT3(i915) ? HDC_FENCE_DEST_SLM_DISABLE : 0));
271 }
272
273 static void chv_ctx_workarounds_init(struct intel_engine_cs *engine,
274 struct i915_wa_list *wal)
275 {
276 gen8_ctx_workarounds_init(engine, wal);
277
278 /* WaDisableThreadStallDopClockGating:chv */
279 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
280
281 /* Improve HiZ throughput on CHV. */
282 WA_SET_BIT_MASKED(HIZ_CHICKEN, CHV_HZ_8X8_MODE_IN_1X);
283 }
284
285 static void gen9_ctx_workarounds_init(struct intel_engine_cs *engine,
286 struct i915_wa_list *wal)
287 {
288 struct drm_i915_private *i915 = engine->i915;
289
290 if (HAS_LLC(i915)) {
291 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
292 *
293 * Must match Display Engine. See
294 * WaCompressedResourceDisplayNewHashMode.
295 */
296 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
297 GEN9_PBE_COMPRESSED_HASH_SELECTION);
298 WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
299 GEN9_SAMPLER_HASH_COMPRESSED_READ_ADDR);
300 }
301
302 /* WaClearFlowControlGpgpuContextSave:skl,bxt,kbl,glk,cfl */
303 /* WaDisablePartialInstShootdown:skl,bxt,kbl,glk,cfl */
304 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
305 FLOW_CONTROL_ENABLE |
306 PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
307
308 /* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt,kbl,glk,cfl */
309 /* WaEnableSamplerGPGPUPreemptionSupport:skl,bxt,kbl,cfl */
310 WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
311 GEN9_ENABLE_YV12_BUGFIX |
312 GEN9_ENABLE_GPGPU_PREEMPTION);
313
314 /* Wa4x4STCOptimizationDisable:skl,bxt,kbl,glk,cfl */
315 /* WaDisablePartialResolveInVc:skl,bxt,kbl,cfl */
316 WA_SET_BIT_MASKED(CACHE_MODE_1,
317 GEN8_4x4_STC_OPTIMIZATION_DISABLE |
318 GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE);
319
320 /* WaCcsTlbPrefetchDisable:skl,bxt,kbl,glk,cfl */
321 WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
322 GEN9_CCS_TLB_PREFETCH_ENABLE);
323
324 /* WaForceContextSaveRestoreNonCoherent:skl,bxt,kbl,cfl */
325 WA_SET_BIT_MASKED(HDC_CHICKEN0,
326 HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
327 HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE);
328
329 /* WaForceEnableNonCoherent and WaDisableHDCInvalidation are
330 * both tied to WaForceContextSaveRestoreNonCoherent
331 * in some hsds for skl. We keep the tie for all gen9. The
332 * documentation is a bit hazy and so we want to get common behaviour,
333 * even though there is no clear evidence we would need both on kbl/bxt.
334 * This area has been source of system hangs so we play it safe
335 * and mimic the skl regardless of what bspec says.
336 *
337 * Use Force Non-Coherent whenever executing a 3D context. This
338 * is a workaround for a possible hang in the unlikely event
339 * a TLB invalidation occurs during a PSD flush.
340 */
341
342 /* WaForceEnableNonCoherent:skl,bxt,kbl,cfl */
343 WA_SET_BIT_MASKED(HDC_CHICKEN0,
344 HDC_FORCE_NON_COHERENT);
345
346 /* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt,kbl,cfl */
347 if (IS_SKYLAKE(i915) || IS_KABYLAKE(i915) || IS_COFFEELAKE(i915))
348 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
349 GEN8_SAMPLER_POWER_BYPASS_DIS);
350
351 /* WaDisableSTUnitPowerOptimization:skl,bxt,kbl,glk,cfl */
352 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE);
353
354 /*
355 * Supporting preemption with fine-granularity requires changes in the
356 * batch buffer programming. Since we can't break old userspace, we
357 * need to set our default preemption level to safe value. Userspace is
358 * still able to use more fine-grained preemption levels, since in
359 * WaEnablePreemptionGranularityControlByUMD we're whitelisting the
360 * per-ctx register. As such, WaDisable{3D,GPGPU}MidCmdPreemption are
361 * not real HW workarounds, but merely a way to start using preemption
362 * while maintaining old contract with userspace.
363 */
364
365 /* WaDisable3DMidCmdPreemption:skl,bxt,glk,cfl,[cnl] */
366 WA_CLR_BIT_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
367
368 /* WaDisableGPGPUMidCmdPreemption:skl,bxt,blk,cfl,[cnl] */
369 WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1,
370 GEN9_PREEMPT_GPGPU_LEVEL_MASK,
371 GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
372
373 /* WaClearHIZ_WM_CHICKEN3:bxt,glk */
374 if (IS_GEN9_LP(i915))
375 WA_SET_BIT_MASKED(GEN9_WM_CHICKEN3, GEN9_FACTOR_IN_CLR_VAL_HIZ);
376 }
377
378 static void skl_tune_iz_hashing(struct intel_engine_cs *engine,
379 struct i915_wa_list *wal)
380 {
381 struct drm_i915_private *i915 = engine->i915;
382 u8 vals[3] = { 0, 0, 0 };
383 unsigned int i;
384
385 for (i = 0; i < 3; i++) {
386 u8 ss;
387
388 /*
389 * Only consider slices where one, and only one, subslice has 7
390 * EUs
391 */
392 if (!is_power_of_2(RUNTIME_INFO(i915)->sseu.subslice_7eu[i]))
393 continue;
394
395 /*
396 * subslice_7eu[i] != 0 (because of the check above) and
397 * ss_max == 4 (maximum number of subslices possible per slice)
398 *
399 * -> 0 <= ss <= 3;
400 */
401 ss = ffs(RUNTIME_INFO(i915)->sseu.subslice_7eu[i]) - 1;
402 vals[i] = 3 - ss;
403 }
404
405 if (vals[0] == 0 && vals[1] == 0 && vals[2] == 0)
406 return;
407
408 /* Tune IZ hashing. See intel_device_info_runtime_init() */
409 WA_SET_FIELD_MASKED(GEN7_GT_MODE,
410 GEN9_IZ_HASHING_MASK(2) |
411 GEN9_IZ_HASHING_MASK(1) |
412 GEN9_IZ_HASHING_MASK(0),
413 GEN9_IZ_HASHING(2, vals[2]) |
414 GEN9_IZ_HASHING(1, vals[1]) |
415 GEN9_IZ_HASHING(0, vals[0]));
416 }
417
418 static void skl_ctx_workarounds_init(struct intel_engine_cs *engine,
419 struct i915_wa_list *wal)
420 {
421 gen9_ctx_workarounds_init(engine, wal);
422 skl_tune_iz_hashing(engine, wal);
423 }
424
425 static void bxt_ctx_workarounds_init(struct intel_engine_cs *engine,
426 struct i915_wa_list *wal)
427 {
428 gen9_ctx_workarounds_init(engine, wal);
429
430 /* WaDisableThreadStallDopClockGating:bxt */
431 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
432 STALL_DOP_GATING_DISABLE);
433
434 /* WaToEnableHwFixForPushConstHWBug:bxt */
435 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
436 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
437 }
438
439 static void kbl_ctx_workarounds_init(struct intel_engine_cs *engine,
440 struct i915_wa_list *wal)
441 {
442 struct drm_i915_private *i915 = engine->i915;
443
444 gen9_ctx_workarounds_init(engine, wal);
445
446 /* WaToEnableHwFixForPushConstHWBug:kbl */
447 if (IS_KBL_REVID(i915, KBL_REVID_C0, REVID_FOREVER))
448 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
449 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
450
451 /* WaDisableSbeCacheDispatchPortSharing:kbl */
452 WA_SET_BIT_MASKED(GEN7_HALF_SLICE_CHICKEN1,
453 GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
454 }
455
456 static void glk_ctx_workarounds_init(struct intel_engine_cs *engine,
457 struct i915_wa_list *wal)
458 {
459 gen9_ctx_workarounds_init(engine, wal);
460
461 /* WaToEnableHwFixForPushConstHWBug:glk */
462 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
463 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
464 }
465
466 static void cfl_ctx_workarounds_init(struct intel_engine_cs *engine,
467 struct i915_wa_list *wal)
468 {
469 gen9_ctx_workarounds_init(engine, wal);
470
471 /* WaToEnableHwFixForPushConstHWBug:cfl */
472 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
473 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
474
475 /* WaDisableSbeCacheDispatchPortSharing:cfl */
476 WA_SET_BIT_MASKED(GEN7_HALF_SLICE_CHICKEN1,
477 GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
478 }
479
480 static void cnl_ctx_workarounds_init(struct intel_engine_cs *engine,
481 struct i915_wa_list *wal)
482 {
483 struct drm_i915_private *i915 = engine->i915;
484
485 /* WaForceContextSaveRestoreNonCoherent:cnl */
486 WA_SET_BIT_MASKED(CNL_HDC_CHICKEN0,
487 HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT);
488
489 /* WaThrottleEUPerfToAvoidTDBackPressure:cnl(pre-prod) */
490 if (IS_CNL_REVID(i915, CNL_REVID_B0, CNL_REVID_B0))
491 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, THROTTLE_12_5);
492
493 /* WaDisableReplayBufferBankArbitrationOptimization:cnl */
494 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
495 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
496
497 /* WaDisableEnhancedSBEVertexCaching:cnl (pre-prod) */
498 if (IS_CNL_REVID(i915, 0, CNL_REVID_B0))
499 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
500 GEN8_CSC2_SBE_VUE_CACHE_CONSERVATIVE);
501
502 /* WaPushConstantDereferenceHoldDisable:cnl */
503 WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2, PUSH_CONSTANT_DEREF_DISABLE);
504
505 /* FtrEnableFastAnisoL1BankingFix:cnl */
506 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3, CNL_FAST_ANISO_L1_BANKING_FIX);
507
508 /* WaDisable3DMidCmdPreemption:cnl */
509 WA_CLR_BIT_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
510
511 /* WaDisableGPGPUMidCmdPreemption:cnl */
512 WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1,
513 GEN9_PREEMPT_GPGPU_LEVEL_MASK,
514 GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
515
516 /* WaDisableEarlyEOT:cnl */
517 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, DISABLE_EARLY_EOT);
518 }
519
520 static void icl_ctx_workarounds_init(struct intel_engine_cs *engine,
521 struct i915_wa_list *wal)
522 {
523 struct drm_i915_private *i915 = engine->i915;
524
525 /* WaDisableBankHangMode:icl */
526 wa_write(wal,
527 GEN8_L3CNTLREG,
528 intel_uncore_read(engine->uncore, GEN8_L3CNTLREG) |
529 GEN8_ERRDETBCTRL);
530
531 /* Wa_1604370585:icl (pre-prod)
532 * Formerly known as WaPushConstantDereferenceHoldDisable
533 */
534 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0))
535 WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
536 PUSH_CONSTANT_DEREF_DISABLE);
537
538 /* WaForceEnableNonCoherent:icl
539 * This is not the same workaround as in early Gen9 platforms, where
540 * lacking this could cause system hangs, but coherency performance
541 * overhead is high and only a few compute workloads really need it
542 * (the register is whitelisted in hardware now, so UMDs can opt in
543 * for coherency if they have a good reason).
544 */
545 WA_SET_BIT_MASKED(ICL_HDC_MODE, HDC_FORCE_NON_COHERENT);
546
547 /* Wa_2006611047:icl (pre-prod)
548 * Formerly known as WaDisableImprovedTdlClkGating
549 */
550 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0))
551 WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
552 GEN11_TDL_CLOCK_GATING_FIX_DISABLE);
553
554 /* Wa_2006665173:icl (pre-prod) */
555 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0))
556 WA_SET_BIT_MASKED(GEN11_COMMON_SLICE_CHICKEN3,
557 GEN11_BLEND_EMB_FIX_DISABLE_IN_RCC);
558
559 /* WaEnableFloatBlendOptimization:icl */
560 wa_write_masked_or(wal,
561 GEN10_CACHE_MODE_SS,
562 0, /* write-only, so skip validation */
563 _MASKED_BIT_ENABLE(FLOAT_BLEND_OPTIMIZATION_ENABLE));
564
565 /* WaDisableGPGPUMidThreadPreemption:icl */
566 WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1,
567 GEN9_PREEMPT_GPGPU_LEVEL_MASK,
568 GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL);
569
570 /* allow headerless messages for preemptible GPGPU context */
571 WA_SET_BIT_MASKED(GEN10_SAMPLER_MODE,
572 GEN11_SAMPLER_ENABLE_HEADLESS_MSG);
573 }
574
575 static void tgl_ctx_workarounds_init(struct intel_engine_cs *engine,
576 struct i915_wa_list *wal)
577 {
578 u32 val;
579
580 /* Wa_1409142259:tgl */
581 WA_SET_BIT_MASKED(GEN11_COMMON_SLICE_CHICKEN3,
582 GEN12_DISABLE_CPS_AWARE_COLOR_PIPE);
583
584 /* Wa_1604555607:tgl */
585 val = intel_uncore_read(engine->uncore, FF_MODE2);
586 val &= ~FF_MODE2_TDS_TIMER_MASK;
587 val |= FF_MODE2_TDS_TIMER_128;
588 /*
589 * FIXME: FF_MODE2 register is not readable till TGL B0. We can
590 * enable verification of WA from the later steppings, which enables
591 * the read of FF_MODE2.
592 */
593 wa_add(wal, FF_MODE2, FF_MODE2_TDS_TIMER_MASK, val,
594 IS_TGL_REVID(engine->i915, TGL_REVID_A0, TGL_REVID_A0) ? 0 :
595 FF_MODE2_TDS_TIMER_MASK);
596 }
597
598 static void
599 __intel_engine_init_ctx_wa(struct intel_engine_cs *engine,
600 struct i915_wa_list *wal,
601 const char *name)
602 {
603 struct drm_i915_private *i915 = engine->i915;
604
605 if (engine->class != RENDER_CLASS)
606 return;
607
608 wa_init_start(wal, name, engine->name);
609
610 if (IS_GEN(i915, 12))
611 tgl_ctx_workarounds_init(engine, wal);
612 else if (IS_GEN(i915, 11))
613 icl_ctx_workarounds_init(engine, wal);
614 else if (IS_CANNONLAKE(i915))
615 cnl_ctx_workarounds_init(engine, wal);
616 else if (IS_COFFEELAKE(i915))
617 cfl_ctx_workarounds_init(engine, wal);
618 else if (IS_GEMINILAKE(i915))
619 glk_ctx_workarounds_init(engine, wal);
620 else if (IS_KABYLAKE(i915))
621 kbl_ctx_workarounds_init(engine, wal);
622 else if (IS_BROXTON(i915))
623 bxt_ctx_workarounds_init(engine, wal);
624 else if (IS_SKYLAKE(i915))
625 skl_ctx_workarounds_init(engine, wal);
626 else if (IS_CHERRYVIEW(i915))
627 chv_ctx_workarounds_init(engine, wal);
628 else if (IS_BROADWELL(i915))
629 bdw_ctx_workarounds_init(engine, wal);
630 else if (INTEL_GEN(i915) < 8)
631 return;
632 else
633 MISSING_CASE(INTEL_GEN(i915));
634
635 wa_init_finish(wal);
636 }
637
638 void intel_engine_init_ctx_wa(struct intel_engine_cs *engine)
639 {
640 __intel_engine_init_ctx_wa(engine, &engine->ctx_wa_list, "context");
641 }
642
643 int intel_engine_emit_ctx_wa(struct i915_request *rq)
644 {
645 struct i915_wa_list *wal = &rq->engine->ctx_wa_list;
646 struct i915_wa *wa;
647 unsigned int i;
648 u32 *cs;
649 int ret;
650
651 if (wal->count == 0)
652 return 0;
653
654 ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
655 if (ret)
656 return ret;
657
658 cs = intel_ring_begin(rq, (wal->count * 2 + 2));
659 if (IS_ERR(cs))
660 return PTR_ERR(cs);
661
662 *cs++ = MI_LOAD_REGISTER_IMM(wal->count);
663 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
664 *cs++ = i915_mmio_reg_offset(wa->reg);
665 *cs++ = wa->val;
666 }
667 *cs++ = MI_NOOP;
668
669 intel_ring_advance(rq, cs);
670
671 ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
672 if (ret)
673 return ret;
674
675 return 0;
676 }
677
678 static void
679 gen9_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
680 {
681 /* WaDisableKillLogic:bxt,skl,kbl */
682 if (!IS_COFFEELAKE(i915))
683 wa_write_or(wal,
684 GAM_ECOCHK,
685 ECOCHK_DIS_TLB);
686
687 if (HAS_LLC(i915)) {
688 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
689 *
690 * Must match Display Engine. See
691 * WaCompressedResourceDisplayNewHashMode.
692 */
693 wa_write_or(wal,
694 MMCD_MISC_CTRL,
695 MMCD_PCLA | MMCD_HOTSPOT_EN);
696 }
697
698 /* WaDisableHDCInvalidation:skl,bxt,kbl,cfl */
699 wa_write_or(wal,
700 GAM_ECOCHK,
701 BDW_DISABLE_HDC_INVALIDATION);
702 }
703
704 static void
705 skl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
706 {
707 gen9_gt_workarounds_init(i915, wal);
708
709 /* WaDisableGafsUnitClkGating:skl */
710 wa_write_or(wal,
711 GEN7_UCGCTL4,
712 GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
713
714 /* WaInPlaceDecompressionHang:skl */
715 if (IS_SKL_REVID(i915, SKL_REVID_H0, REVID_FOREVER))
716 wa_write_or(wal,
717 GEN9_GAMT_ECO_REG_RW_IA,
718 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
719 }
720
721 static void
722 bxt_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
723 {
724 gen9_gt_workarounds_init(i915, wal);
725
726 /* WaInPlaceDecompressionHang:bxt */
727 wa_write_or(wal,
728 GEN9_GAMT_ECO_REG_RW_IA,
729 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
730 }
731
732 static void
733 kbl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
734 {
735 gen9_gt_workarounds_init(i915, wal);
736
737 /* WaDisableDynamicCreditSharing:kbl */
738 if (IS_KBL_REVID(i915, 0, KBL_REVID_B0))
739 wa_write_or(wal,
740 GAMT_CHKN_BIT_REG,
741 GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING);
742
743 /* WaDisableGafsUnitClkGating:kbl */
744 wa_write_or(wal,
745 GEN7_UCGCTL4,
746 GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
747
748 /* WaInPlaceDecompressionHang:kbl */
749 wa_write_or(wal,
750 GEN9_GAMT_ECO_REG_RW_IA,
751 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
752 }
753
754 static void
755 glk_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
756 {
757 gen9_gt_workarounds_init(i915, wal);
758 }
759
760 static void
761 cfl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
762 {
763 gen9_gt_workarounds_init(i915, wal);
764
765 /* WaDisableGafsUnitClkGating:cfl */
766 wa_write_or(wal,
767 GEN7_UCGCTL4,
768 GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
769
770 /* WaInPlaceDecompressionHang:cfl */
771 wa_write_or(wal,
772 GEN9_GAMT_ECO_REG_RW_IA,
773 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
774 }
775
776 static void
777 wa_init_mcr(struct drm_i915_private *i915, struct i915_wa_list *wal)
778 {
779 const struct sseu_dev_info *sseu = &RUNTIME_INFO(i915)->sseu;
780 unsigned int slice, subslice;
781 u32 l3_en, mcr, mcr_mask;
782
783 GEM_BUG_ON(INTEL_GEN(i915) < 10);
784
785 /*
786 * WaProgramMgsrForL3BankSpecificMmioReads: cnl,icl
787 * L3Banks could be fused off in single slice scenario. If that is
788 * the case, we might need to program MCR select to a valid L3Bank
789 * by default, to make sure we correctly read certain registers
790 * later on (in the range 0xB100 - 0xB3FF).
791 *
792 * WaProgramMgsrForCorrectSliceSpecificMmioReads:cnl,icl
793 * Before any MMIO read into slice/subslice specific registers, MCR
794 * packet control register needs to be programmed to point to any
795 * enabled s/ss pair. Otherwise, incorrect values will be returned.
796 * This means each subsequent MMIO read will be forwarded to an
797 * specific s/ss combination, but this is OK since these registers
798 * are consistent across s/ss in almost all cases. In the rare
799 * occasions, such as INSTDONE, where this value is dependent
800 * on s/ss combo, the read should be done with read_subslice_reg.
801 *
802 * Since GEN8_MCR_SELECTOR contains dual-purpose bits which select both
803 * to which subslice, or to which L3 bank, the respective mmio reads
804 * will go, we have to find a common index which works for both
805 * accesses.
806 *
807 * Case where we cannot find a common index fortunately should not
808 * happen in production hardware, so we only emit a warning instead of
809 * implementing something more complex that requires checking the range
810 * of every MMIO read.
811 */
812
813 if (INTEL_GEN(i915) >= 10 && is_power_of_2(sseu->slice_mask)) {
814 u32 l3_fuse =
815 intel_uncore_read(&i915->uncore, GEN10_MIRROR_FUSE3) &
816 GEN10_L3BANK_MASK;
817
818 DRM_DEBUG_DRIVER("L3 fuse = %x\n", l3_fuse);
819 l3_en = ~(l3_fuse << GEN10_L3BANK_PAIR_COUNT | l3_fuse);
820 } else {
821 l3_en = ~0;
822 }
823
824 slice = fls(sseu->slice_mask) - 1;
825 subslice = fls(l3_en & intel_sseu_get_subslices(sseu, slice));
826 if (!subslice) {
827 DRM_WARN("No common index found between subslice mask %x and L3 bank mask %x!\n",
828 intel_sseu_get_subslices(sseu, slice), l3_en);
829 subslice = fls(l3_en);
830 WARN_ON(!subslice);
831 }
832 subslice--;
833
834 if (INTEL_GEN(i915) >= 11) {
835 mcr = GEN11_MCR_SLICE(slice) | GEN11_MCR_SUBSLICE(subslice);
836 mcr_mask = GEN11_MCR_SLICE_MASK | GEN11_MCR_SUBSLICE_MASK;
837 } else {
838 mcr = GEN8_MCR_SLICE(slice) | GEN8_MCR_SUBSLICE(subslice);
839 mcr_mask = GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK;
840 }
841
842 DRM_DEBUG_DRIVER("MCR slice/subslice = %x\n", mcr);
843
844 wa_write_masked_or(wal, GEN8_MCR_SELECTOR, mcr_mask, mcr);
845 }
846
847 static void
848 cnl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
849 {
850 wa_init_mcr(i915, wal);
851
852 /* WaDisableI2mCycleOnWRPort:cnl (pre-prod) */
853 if (IS_CNL_REVID(i915, CNL_REVID_B0, CNL_REVID_B0))
854 wa_write_or(wal,
855 GAMT_CHKN_BIT_REG,
856 GAMT_CHKN_DISABLE_I2M_CYCLE_ON_WR_PORT);
857
858 /* WaInPlaceDecompressionHang:cnl */
859 wa_write_or(wal,
860 GEN9_GAMT_ECO_REG_RW_IA,
861 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
862 }
863
864 static void
865 icl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
866 {
867 wa_init_mcr(i915, wal);
868
869 /* WaInPlaceDecompressionHang:icl */
870 wa_write_or(wal,
871 GEN9_GAMT_ECO_REG_RW_IA,
872 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
873
874 /* WaModifyGamTlbPartitioning:icl */
875 wa_write_masked_or(wal,
876 GEN11_GACB_PERF_CTRL,
877 GEN11_HASH_CTRL_MASK,
878 GEN11_HASH_CTRL_BIT0 | GEN11_HASH_CTRL_BIT4);
879
880 /* Wa_1405766107:icl
881 * Formerly known as WaCL2SFHalfMaxAlloc
882 */
883 wa_write_or(wal,
884 GEN11_LSN_UNSLCVC,
885 GEN11_LSN_UNSLCVC_GAFS_HALF_SF_MAXALLOC |
886 GEN11_LSN_UNSLCVC_GAFS_HALF_CL2_MAXALLOC);
887
888 /* Wa_220166154:icl
889 * Formerly known as WaDisCtxReload
890 */
891 wa_write_or(wal,
892 GEN8_GAMW_ECO_DEV_RW_IA,
893 GAMW_ECO_DEV_CTX_RELOAD_DISABLE);
894
895 /* Wa_1405779004:icl (pre-prod) */
896 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0))
897 wa_write_or(wal,
898 SLICE_UNIT_LEVEL_CLKGATE,
899 MSCUNIT_CLKGATE_DIS);
900
901 /* Wa_1406680159:icl */
902 wa_write_or(wal,
903 SUBSLICE_UNIT_LEVEL_CLKGATE,
904 GWUNIT_CLKGATE_DIS);
905
906 /* Wa_1406838659:icl (pre-prod) */
907 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0))
908 wa_write_or(wal,
909 INF_UNIT_LEVEL_CLKGATE,
910 CGPSF_CLKGATE_DIS);
911
912 /* Wa_1406463099:icl
913 * Formerly known as WaGamTlbPendError
914 */
915 wa_write_or(wal,
916 GAMT_CHKN_BIT_REG,
917 GAMT_CHKN_DISABLE_L3_COH_PIPE);
918
919 /* Wa_1607087056:icl */
920 wa_write_or(wal,
921 SLICE_UNIT_LEVEL_CLKGATE,
922 L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
923 }
924
925 static void
926 tgl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
927 {
928 /* Wa_1409420604:tgl */
929 if (IS_TGL_REVID(i915, TGL_REVID_A0, TGL_REVID_A0))
930 wa_write_or(wal,
931 SUBSLICE_UNIT_LEVEL_CLKGATE2,
932 CPSSUNIT_CLKGATE_DIS);
933
934 /* Wa_1409180338:tgl */
935 if (IS_TGL_REVID(i915, TGL_REVID_A0, TGL_REVID_A0))
936 wa_write_or(wal,
937 SLICE_UNIT_LEVEL_CLKGATE,
938 L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
939 }
940
941 static void
942 gt_init_workarounds(struct drm_i915_private *i915, struct i915_wa_list *wal)
943 {
944 if (IS_GEN(i915, 12))
945 tgl_gt_workarounds_init(i915, wal);
946 else if (IS_GEN(i915, 11))
947 icl_gt_workarounds_init(i915, wal);
948 else if (IS_CANNONLAKE(i915))
949 cnl_gt_workarounds_init(i915, wal);
950 else if (IS_COFFEELAKE(i915))
951 cfl_gt_workarounds_init(i915, wal);
952 else if (IS_GEMINILAKE(i915))
953 glk_gt_workarounds_init(i915, wal);
954 else if (IS_KABYLAKE(i915))
955 kbl_gt_workarounds_init(i915, wal);
956 else if (IS_BROXTON(i915))
957 bxt_gt_workarounds_init(i915, wal);
958 else if (IS_SKYLAKE(i915))
959 skl_gt_workarounds_init(i915, wal);
960 else if (INTEL_GEN(i915) <= 8)
961 return;
962 else
963 MISSING_CASE(INTEL_GEN(i915));
964 }
965
966 void intel_gt_init_workarounds(struct drm_i915_private *i915)
967 {
968 struct i915_wa_list *wal = &i915->gt_wa_list;
969
970 wa_init_start(wal, "GT", "global");
971 gt_init_workarounds(i915, wal);
972 wa_init_finish(wal);
973 }
974
975 static enum forcewake_domains
976 wal_get_fw_for_rmw(struct intel_uncore *uncore, const struct i915_wa_list *wal)
977 {
978 enum forcewake_domains fw = 0;
979 struct i915_wa *wa;
980 unsigned int i;
981
982 for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
983 fw |= intel_uncore_forcewake_for_reg(uncore,
984 wa->reg,
985 FW_REG_READ |
986 FW_REG_WRITE);
987
988 return fw;
989 }
990
991 static bool
992 wa_verify(const struct i915_wa *wa, u32 cur, const char *name, const char *from)
993 {
994 if ((cur ^ wa->val) & wa->read) {
995 DRM_ERROR("%s workaround lost on %s! (%x=%x/%x, expected %x, mask=%x)\n",
996 name, from, i915_mmio_reg_offset(wa->reg),
997 cur, cur & wa->read,
998 wa->val, wa->mask);
999
1000 return false;
1001 }
1002
1003 return true;
1004 }
1005
1006 static void
1007 wa_list_apply(struct intel_uncore *uncore, const struct i915_wa_list *wal)
1008 {
1009 enum forcewake_domains fw;
1010 unsigned long flags;
1011 struct i915_wa *wa;
1012 unsigned int i;
1013
1014 if (!wal->count)
1015 return;
1016
1017 fw = wal_get_fw_for_rmw(uncore, wal);
1018
1019 spin_lock_irqsave(&uncore->lock, flags);
1020 intel_uncore_forcewake_get__locked(uncore, fw);
1021
1022 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
1023 intel_uncore_rmw_fw(uncore, wa->reg, wa->mask, wa->val);
1024 if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
1025 wa_verify(wa,
1026 intel_uncore_read_fw(uncore, wa->reg),
1027 wal->name, "application");
1028 }
1029
1030 intel_uncore_forcewake_put__locked(uncore, fw);
1031 spin_unlock_irqrestore(&uncore->lock, flags);
1032 }
1033
1034 void intel_gt_apply_workarounds(struct intel_gt *gt)
1035 {
1036 wa_list_apply(gt->uncore, &gt->i915->gt_wa_list);
1037 }
1038
1039 static bool wa_list_verify(struct intel_uncore *uncore,
1040 const struct i915_wa_list *wal,
1041 const char *from)
1042 {
1043 struct i915_wa *wa;
1044 unsigned int i;
1045 bool ok = true;
1046
1047 for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1048 ok &= wa_verify(wa,
1049 intel_uncore_read(uncore, wa->reg),
1050 wal->name, from);
1051
1052 return ok;
1053 }
1054
1055 bool intel_gt_verify_workarounds(struct intel_gt *gt, const char *from)
1056 {
1057 return wa_list_verify(gt->uncore, &gt->i915->gt_wa_list, from);
1058 }
1059
1060 static inline bool is_nonpriv_flags_valid(u32 flags)
1061 {
1062 /* Check only valid flag bits are set */
1063 if (flags & ~RING_FORCE_TO_NONPRIV_MASK_VALID)
1064 return false;
1065
1066 /* NB: Only 3 out of 4 enum values are valid for access field */
1067 if ((flags & RING_FORCE_TO_NONPRIV_ACCESS_MASK) ==
1068 RING_FORCE_TO_NONPRIV_ACCESS_INVALID)
1069 return false;
1070
1071 return true;
1072 }
1073
1074 static void
1075 whitelist_reg_ext(struct i915_wa_list *wal, i915_reg_t reg, u32 flags)
1076 {
1077 struct i915_wa wa = {
1078 .reg = reg
1079 };
1080
1081 if (GEM_DEBUG_WARN_ON(wal->count >= RING_MAX_NONPRIV_SLOTS))
1082 return;
1083
1084 if (GEM_DEBUG_WARN_ON(!is_nonpriv_flags_valid(flags)))
1085 return;
1086
1087 wa.reg.reg |= flags;
1088 _wa_add(wal, &wa);
1089 }
1090
1091 static void
1092 whitelist_reg(struct i915_wa_list *wal, i915_reg_t reg)
1093 {
1094 whitelist_reg_ext(wal, reg, RING_FORCE_TO_NONPRIV_ACCESS_RW);
1095 }
1096
1097 static void gen9_whitelist_build(struct i915_wa_list *w)
1098 {
1099 /* WaVFEStateAfterPipeControlwithMediaStateClear:skl,bxt,glk,cfl */
1100 whitelist_reg(w, GEN9_CTX_PREEMPT_REG);
1101
1102 /* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl,cfl,[cnl] */
1103 whitelist_reg(w, GEN8_CS_CHICKEN1);
1104
1105 /* WaAllowUMDToModifyHDCChicken1:skl,bxt,kbl,glk,cfl */
1106 whitelist_reg(w, GEN8_HDC_CHICKEN1);
1107
1108 /* WaSendPushConstantsFromMMIO:skl,bxt */
1109 whitelist_reg(w, COMMON_SLICE_CHICKEN2);
1110 }
1111
1112 static void skl_whitelist_build(struct intel_engine_cs *engine)
1113 {
1114 struct i915_wa_list *w = &engine->whitelist;
1115
1116 if (engine->class != RENDER_CLASS)
1117 return;
1118
1119 gen9_whitelist_build(w);
1120
1121 /* WaDisableLSQCROPERFforOCL:skl */
1122 whitelist_reg(w, GEN8_L3SQCREG4);
1123 }
1124
1125 static void bxt_whitelist_build(struct intel_engine_cs *engine)
1126 {
1127 if (engine->class != RENDER_CLASS)
1128 return;
1129
1130 gen9_whitelist_build(&engine->whitelist);
1131 }
1132
1133 static void kbl_whitelist_build(struct intel_engine_cs *engine)
1134 {
1135 struct i915_wa_list *w = &engine->whitelist;
1136
1137 if (engine->class != RENDER_CLASS)
1138 return;
1139
1140 gen9_whitelist_build(w);
1141
1142 /* WaDisableLSQCROPERFforOCL:kbl */
1143 whitelist_reg(w, GEN8_L3SQCREG4);
1144 }
1145
1146 static void glk_whitelist_build(struct intel_engine_cs *engine)
1147 {
1148 struct i915_wa_list *w = &engine->whitelist;
1149
1150 if (engine->class != RENDER_CLASS)
1151 return;
1152
1153 gen9_whitelist_build(w);
1154
1155 /* WA #0862: Userspace has to set "Barrier Mode" to avoid hangs. */
1156 whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1);
1157 }
1158
1159 static void cfl_whitelist_build(struct intel_engine_cs *engine)
1160 {
1161 struct i915_wa_list *w = &engine->whitelist;
1162
1163 if (engine->class != RENDER_CLASS)
1164 return;
1165
1166 gen9_whitelist_build(w);
1167
1168 /*
1169 * WaAllowPMDepthAndInvocationCountAccessFromUMD:cfl,whl,cml,aml
1170 *
1171 * This covers 4 register which are next to one another :
1172 * - PS_INVOCATION_COUNT
1173 * - PS_INVOCATION_COUNT_UDW
1174 * - PS_DEPTH_COUNT
1175 * - PS_DEPTH_COUNT_UDW
1176 */
1177 whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1178 RING_FORCE_TO_NONPRIV_ACCESS_RD |
1179 RING_FORCE_TO_NONPRIV_RANGE_4);
1180 }
1181
1182 static void cnl_whitelist_build(struct intel_engine_cs *engine)
1183 {
1184 struct i915_wa_list *w = &engine->whitelist;
1185
1186 if (engine->class != RENDER_CLASS)
1187 return;
1188
1189 /* WaEnablePreemptionGranularityControlByUMD:cnl */
1190 whitelist_reg(w, GEN8_CS_CHICKEN1);
1191 }
1192
1193 static void icl_whitelist_build(struct intel_engine_cs *engine)
1194 {
1195 struct i915_wa_list *w = &engine->whitelist;
1196
1197 switch (engine->class) {
1198 case RENDER_CLASS:
1199 /* WaAllowUMDToModifyHalfSliceChicken7:icl */
1200 whitelist_reg(w, GEN9_HALF_SLICE_CHICKEN7);
1201
1202 /* WaAllowUMDToModifySamplerMode:icl */
1203 whitelist_reg(w, GEN10_SAMPLER_MODE);
1204
1205 /* WaEnableStateCacheRedirectToCS:icl */
1206 whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1);
1207
1208 /*
1209 * WaAllowPMDepthAndInvocationCountAccessFromUMD:icl
1210 *
1211 * This covers 4 register which are next to one another :
1212 * - PS_INVOCATION_COUNT
1213 * - PS_INVOCATION_COUNT_UDW
1214 * - PS_DEPTH_COUNT
1215 * - PS_DEPTH_COUNT_UDW
1216 */
1217 whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1218 RING_FORCE_TO_NONPRIV_ACCESS_RD |
1219 RING_FORCE_TO_NONPRIV_RANGE_4);
1220 break;
1221
1222 case VIDEO_DECODE_CLASS:
1223 /* hucStatusRegOffset */
1224 whitelist_reg_ext(w, _MMIO(0x2000 + engine->mmio_base),
1225 RING_FORCE_TO_NONPRIV_ACCESS_RD);
1226 /* hucUKernelHdrInfoRegOffset */
1227 whitelist_reg_ext(w, _MMIO(0x2014 + engine->mmio_base),
1228 RING_FORCE_TO_NONPRIV_ACCESS_RD);
1229 /* hucStatus2RegOffset */
1230 whitelist_reg_ext(w, _MMIO(0x23B0 + engine->mmio_base),
1231 RING_FORCE_TO_NONPRIV_ACCESS_RD);
1232 break;
1233
1234 default:
1235 break;
1236 }
1237 }
1238
1239 static void tgl_whitelist_build(struct intel_engine_cs *engine)
1240 {
1241 struct i915_wa_list *w = &engine->whitelist;
1242
1243 switch (engine->class) {
1244 case RENDER_CLASS:
1245 /*
1246 * WaAllowPMDepthAndInvocationCountAccessFromUMD:tgl
1247 *
1248 * This covers 4 registers which are next to one another :
1249 * - PS_INVOCATION_COUNT
1250 * - PS_INVOCATION_COUNT_UDW
1251 * - PS_DEPTH_COUNT
1252 * - PS_DEPTH_COUNT_UDW
1253 */
1254 whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1255 RING_FORCE_TO_NONPRIV_ACCESS_RD |
1256 RING_FORCE_TO_NONPRIV_RANGE_4);
1257 break;
1258 default:
1259 break;
1260 }
1261 }
1262
1263 void intel_engine_init_whitelist(struct intel_engine_cs *engine)
1264 {
1265 struct drm_i915_private *i915 = engine->i915;
1266 struct i915_wa_list *w = &engine->whitelist;
1267
1268 wa_init_start(w, "whitelist", engine->name);
1269
1270 if (IS_GEN(i915, 12))
1271 tgl_whitelist_build(engine);
1272 else if (IS_GEN(i915, 11))
1273 icl_whitelist_build(engine);
1274 else if (IS_CANNONLAKE(i915))
1275 cnl_whitelist_build(engine);
1276 else if (IS_COFFEELAKE(i915))
1277 cfl_whitelist_build(engine);
1278 else if (IS_GEMINILAKE(i915))
1279 glk_whitelist_build(engine);
1280 else if (IS_KABYLAKE(i915))
1281 kbl_whitelist_build(engine);
1282 else if (IS_BROXTON(i915))
1283 bxt_whitelist_build(engine);
1284 else if (IS_SKYLAKE(i915))
1285 skl_whitelist_build(engine);
1286 else if (INTEL_GEN(i915) <= 8)
1287 return;
1288 else
1289 MISSING_CASE(INTEL_GEN(i915));
1290
1291 wa_init_finish(w);
1292 }
1293
1294 void intel_engine_apply_whitelist(struct intel_engine_cs *engine)
1295 {
1296 const struct i915_wa_list *wal = &engine->whitelist;
1297 struct intel_uncore *uncore = engine->uncore;
1298 const u32 base = engine->mmio_base;
1299 struct i915_wa *wa;
1300 unsigned int i;
1301
1302 if (!wal->count)
1303 return;
1304
1305 for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1306 intel_uncore_write(uncore,
1307 RING_FORCE_TO_NONPRIV(base, i),
1308 i915_mmio_reg_offset(wa->reg));
1309
1310 /* And clear the rest just in case of garbage */
1311 for (; i < RING_MAX_NONPRIV_SLOTS; i++)
1312 intel_uncore_write(uncore,
1313 RING_FORCE_TO_NONPRIV(base, i),
1314 i915_mmio_reg_offset(RING_NOPID(base)));
1315 }
1316
1317 static void
1318 rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
1319 {
1320 struct drm_i915_private *i915 = engine->i915;
1321
1322 if (IS_TGL_REVID(i915, TGL_REVID_A0, TGL_REVID_A0)) {
1323 /* Wa_1606700617:tgl */
1324 wa_masked_en(wal,
1325 GEN9_CS_DEBUG_MODE1,
1326 FF_DOP_CLOCK_GATE_DISABLE);
1327
1328 /* Wa_1607138336:tgl */
1329 wa_write_or(wal,
1330 GEN9_CTX_PREEMPT_REG,
1331 GEN12_DISABLE_POSH_BUSY_FF_DOP_CG);
1332
1333 /* Wa_1607030317:tgl */
1334 /* Wa_1607186500:tgl */
1335 /* Wa_1607297627:tgl */
1336 wa_masked_en(wal,
1337 GEN6_RC_SLEEP_PSMI_CONTROL,
1338 GEN12_WAIT_FOR_EVENT_POWER_DOWN_DISABLE |
1339 GEN8_RC_SEMA_IDLE_MSG_DISABLE);
1340
1341 /*
1342 * Wa_1606679103:tgl
1343 * (see also Wa_1606682166:icl)
1344 */
1345 wa_write_or(wal,
1346 GEN7_SARCHKMD,
1347 GEN7_DISABLE_SAMPLER_PREFETCH);
1348 }
1349
1350 if (IS_GEN(i915, 11)) {
1351 /* This is not an Wa. Enable for better image quality */
1352 wa_masked_en(wal,
1353 _3D_CHICKEN3,
1354 _3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE);
1355
1356 /* WaPipelineFlushCoherentLines:icl */
1357 wa_write_or(wal,
1358 GEN8_L3SQCREG4,
1359 GEN8_LQSC_FLUSH_COHERENT_LINES);
1360
1361 /*
1362 * Wa_1405543622:icl
1363 * Formerly known as WaGAPZPriorityScheme
1364 */
1365 wa_write_or(wal,
1366 GEN8_GARBCNTL,
1367 GEN11_ARBITRATION_PRIO_ORDER_MASK);
1368
1369 /*
1370 * Wa_1604223664:icl
1371 * Formerly known as WaL3BankAddressHashing
1372 */
1373 wa_write_masked_or(wal,
1374 GEN8_GARBCNTL,
1375 GEN11_HASH_CTRL_EXCL_MASK,
1376 GEN11_HASH_CTRL_EXCL_BIT0);
1377 wa_write_masked_or(wal,
1378 GEN11_GLBLINVL,
1379 GEN11_BANK_HASH_ADDR_EXCL_MASK,
1380 GEN11_BANK_HASH_ADDR_EXCL_BIT0);
1381
1382 /*
1383 * Wa_1405733216:icl
1384 * Formerly known as WaDisableCleanEvicts
1385 */
1386 wa_write_or(wal,
1387 GEN8_L3SQCREG4,
1388 GEN11_LQSC_CLEAN_EVICT_DISABLE);
1389
1390 /* WaForwardProgressSoftReset:icl */
1391 wa_write_or(wal,
1392 GEN10_SCRATCH_LNCF2,
1393 PMFLUSHDONE_LNICRSDROP |
1394 PMFLUSH_GAPL3UNBLOCK |
1395 PMFLUSHDONE_LNEBLK);
1396
1397 /* Wa_1406609255:icl (pre-prod) */
1398 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0))
1399 wa_write_or(wal,
1400 GEN7_SARCHKMD,
1401 GEN7_DISABLE_DEMAND_PREFETCH);
1402
1403 /* Wa_1606682166:icl */
1404 wa_write_or(wal,
1405 GEN7_SARCHKMD,
1406 GEN7_DISABLE_SAMPLER_PREFETCH);
1407
1408 /* Wa_1409178092:icl */
1409 wa_write_masked_or(wal,
1410 GEN11_SCRATCH2,
1411 GEN11_COHERENT_PARTIAL_WRITE_MERGE_ENABLE,
1412 0);
1413 }
1414
1415 if (IS_GEN_RANGE(i915, 9, 11)) {
1416 /* FtrPerCtxtPreemptionGranularityControl:skl,bxt,kbl,cfl,cnl,icl */
1417 wa_masked_en(wal,
1418 GEN7_FF_SLICE_CS_CHICKEN1,
1419 GEN9_FFSC_PERCTX_PREEMPT_CTRL);
1420 }
1421
1422 if (IS_SKYLAKE(i915) || IS_KABYLAKE(i915) || IS_COFFEELAKE(i915)) {
1423 /* WaEnableGapsTsvCreditFix:skl,kbl,cfl */
1424 wa_write_or(wal,
1425 GEN8_GARBCNTL,
1426 GEN9_GAPS_TSV_CREDIT_DISABLE);
1427 }
1428
1429 if (IS_BROXTON(i915)) {
1430 /* WaDisablePooledEuLoadBalancingFix:bxt */
1431 wa_masked_en(wal,
1432 FF_SLICE_CS_CHICKEN2,
1433 GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE);
1434 }
1435
1436 if (IS_GEN(i915, 9)) {
1437 /* WaContextSwitchWithConcurrentTLBInvalidate:skl,bxt,kbl,glk,cfl */
1438 wa_masked_en(wal,
1439 GEN9_CSFE_CHICKEN1_RCS,
1440 GEN9_PREEMPT_GPGPU_SYNC_SWITCH_DISABLE);
1441
1442 /* WaEnableLbsSlaRetryTimerDecrement:skl,bxt,kbl,glk,cfl */
1443 wa_write_or(wal,
1444 BDW_SCRATCH1,
1445 GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE);
1446
1447 /* WaProgramL3SqcReg1DefaultForPerf:bxt,glk */
1448 if (IS_GEN9_LP(i915))
1449 wa_write_masked_or(wal,
1450 GEN8_L3SQCREG1,
1451 L3_PRIO_CREDITS_MASK,
1452 L3_GENERAL_PRIO_CREDITS(62) |
1453 L3_HIGH_PRIO_CREDITS(2));
1454
1455 /* WaOCLCoherentLineFlush:skl,bxt,kbl,cfl */
1456 wa_write_or(wal,
1457 GEN8_L3SQCREG4,
1458 GEN8_LQSC_FLUSH_COHERENT_LINES);
1459 }
1460 }
1461
1462 static void
1463 xcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
1464 {
1465 struct drm_i915_private *i915 = engine->i915;
1466
1467 /* WaKBLVECSSemaphoreWaitPoll:kbl */
1468 if (IS_KBL_REVID(i915, KBL_REVID_A0, KBL_REVID_E0)) {
1469 wa_write(wal,
1470 RING_SEMA_WAIT_POLL(engine->mmio_base),
1471 1);
1472 }
1473 }
1474
1475 static void
1476 engine_init_workarounds(struct intel_engine_cs *engine, struct i915_wa_list *wal)
1477 {
1478 if (I915_SELFTEST_ONLY(INTEL_GEN(engine->i915) < 8))
1479 return;
1480
1481 if (engine->class == RENDER_CLASS)
1482 rcs_engine_wa_init(engine, wal);
1483 else
1484 xcs_engine_wa_init(engine, wal);
1485 }
1486
1487 void intel_engine_init_workarounds(struct intel_engine_cs *engine)
1488 {
1489 struct i915_wa_list *wal = &engine->wa_list;
1490
1491 if (INTEL_GEN(engine->i915) < 8)
1492 return;
1493
1494 wa_init_start(wal, "engine", engine->name);
1495 engine_init_workarounds(engine, wal);
1496 wa_init_finish(wal);
1497 }
1498
1499 void intel_engine_apply_workarounds(struct intel_engine_cs *engine)
1500 {
1501 wa_list_apply(engine->uncore, &engine->wa_list);
1502 }
1503
1504 static struct i915_vma *
1505 create_scratch(struct i915_address_space *vm, int count)
1506 {
1507 struct drm_i915_gem_object *obj;
1508 struct i915_vma *vma;
1509 unsigned int size;
1510 int err;
1511
1512 size = round_up(count * sizeof(u32), PAGE_SIZE);
1513 obj = i915_gem_object_create_internal(vm->i915, size);
1514 if (IS_ERR(obj))
1515 return ERR_CAST(obj);
1516
1517 i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
1518
1519 vma = i915_vma_instance(obj, vm, NULL);
1520 if (IS_ERR(vma)) {
1521 err = PTR_ERR(vma);
1522 goto err_obj;
1523 }
1524
1525 err = i915_vma_pin(vma, 0, 0,
1526 i915_vma_is_ggtt(vma) ? PIN_GLOBAL : PIN_USER);
1527 if (err)
1528 goto err_obj;
1529
1530 return vma;
1531
1532 err_obj:
1533 i915_gem_object_put(obj);
1534 return ERR_PTR(err);
1535 }
1536
1537 static bool mcr_range(struct drm_i915_private *i915, u32 offset)
1538 {
1539 /*
1540 * Registers in this range are affected by the MCR selector
1541 * which only controls CPU initiated MMIO. Routing does not
1542 * work for CS access so we cannot verify them on this path.
1543 */
1544 if (INTEL_GEN(i915) >= 8 && (offset >= 0xb000 && offset <= 0xb4ff))
1545 return true;
1546
1547 return false;
1548 }
1549
1550 static int
1551 wa_list_srm(struct i915_request *rq,
1552 const struct i915_wa_list *wal,
1553 struct i915_vma *vma)
1554 {
1555 struct drm_i915_private *i915 = rq->i915;
1556 unsigned int i, count = 0;
1557 const struct i915_wa *wa;
1558 u32 srm, *cs;
1559
1560 srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
1561 if (INTEL_GEN(i915) >= 8)
1562 srm++;
1563
1564 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
1565 if (!mcr_range(i915, i915_mmio_reg_offset(wa->reg)))
1566 count++;
1567 }
1568
1569 cs = intel_ring_begin(rq, 4 * count);
1570 if (IS_ERR(cs))
1571 return PTR_ERR(cs);
1572
1573 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
1574 u32 offset = i915_mmio_reg_offset(wa->reg);
1575
1576 if (mcr_range(i915, offset))
1577 continue;
1578
1579 *cs++ = srm;
1580 *cs++ = offset;
1581 *cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i;
1582 *cs++ = 0;
1583 }
1584 intel_ring_advance(rq, cs);
1585
1586 return 0;
1587 }
1588
1589 static int engine_wa_list_verify(struct intel_context *ce,
1590 const struct i915_wa_list * const wal,
1591 const char *from)
1592 {
1593 const struct i915_wa *wa;
1594 struct i915_request *rq;
1595 struct i915_vma *vma;
1596 unsigned int i;
1597 u32 *results;
1598 int err;
1599
1600 if (!wal->count)
1601 return 0;
1602
1603 vma = create_scratch(&ce->engine->gt->ggtt->vm, wal->count);
1604 if (IS_ERR(vma))
1605 return PTR_ERR(vma);
1606
1607 intel_engine_pm_get(ce->engine);
1608 rq = intel_context_create_request(ce);
1609 intel_engine_pm_put(ce->engine);
1610 if (IS_ERR(rq)) {
1611 err = PTR_ERR(rq);
1612 goto err_vma;
1613 }
1614
1615 err = wa_list_srm(rq, wal, vma);
1616 if (err)
1617 goto err_vma;
1618
1619 i915_request_get(rq);
1620 i915_request_add(rq);
1621 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
1622 err = -ETIME;
1623 goto err_rq;
1624 }
1625
1626 results = i915_gem_object_pin_map(vma->obj, I915_MAP_WB);
1627 if (IS_ERR(results)) {
1628 err = PTR_ERR(results);
1629 goto err_rq;
1630 }
1631
1632 err = 0;
1633 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
1634 if (mcr_range(rq->i915, i915_mmio_reg_offset(wa->reg)))
1635 continue;
1636
1637 if (!wa_verify(wa, results[i], wal->name, from))
1638 err = -ENXIO;
1639 }
1640
1641 i915_gem_object_unpin_map(vma->obj);
1642
1643 err_rq:
1644 i915_request_put(rq);
1645 err_vma:
1646 i915_vma_unpin(vma);
1647 i915_vma_put(vma);
1648 return err;
1649 }
1650
1651 int intel_engine_verify_workarounds(struct intel_engine_cs *engine,
1652 const char *from)
1653 {
1654 return engine_wa_list_verify(engine->kernel_context,
1655 &engine->wa_list,
1656 from);
1657 }
1658
1659 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1660 #include "selftest_workarounds.c"
1661 #endif
Linux with added FPC-III support