?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
treewide: remove redundant IS_ERR() before error code check
[linux/fpc-iii.git] / drivers / gpu / drm / i915 / gt / selftest_lrc.c
blob15cda024e3e4577424a4c8580b03bfee2f3027d3
1 /*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright © 2018 Intel Corporation
5 */
6
7 #include <linux/prime_numbers.h>
8
9 #include "gem/i915_gem_pm.h"
10 #include "gt/intel_engine_heartbeat.h"
11 #include "gt/intel_reset.h"
12
13 #include "i915_selftest.h"
14 #include "selftests/i915_random.h"
15 #include "selftests/igt_flush_test.h"
16 #include "selftests/igt_live_test.h"
17 #include "selftests/igt_spinner.h"
18 #include "selftests/lib_sw_fence.h"
19
20 #include "gem/selftests/igt_gem_utils.h"
21 #include "gem/selftests/mock_context.h"
22
23 #define CS_GPR(engine, n) ((engine)->mmio_base + 0x600 + (n) * 4)
24 #define NUM_GPR_DW (16 * 2) /* each GPR is 2 dwords */
25
26 static struct i915_vma *create_scratch(struct intel_gt *gt)
27 {
28 struct drm_i915_gem_object *obj;
29 struct i915_vma *vma;
30 int err;
31
32 obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
33 if (IS_ERR(obj))
34 return ERR_CAST(obj);
35
36 i915_gem_object_set_cache_coherency(obj, I915_CACHING_CACHED);
37
38 vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);
39 if (IS_ERR(vma)) {
40 i915_gem_object_put(obj);
41 return vma;
42 }
43
44 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
45 if (err) {
46 i915_gem_object_put(obj);
47 return ERR_PTR(err);
48 }
49
50 return vma;
51 }
52
53 static void engine_heartbeat_disable(struct intel_engine_cs *engine,
54 unsigned long *saved)
55 {
56 *saved = engine->props.heartbeat_interval_ms;
57 engine->props.heartbeat_interval_ms = 0;
58
59 intel_engine_pm_get(engine);
60 intel_engine_park_heartbeat(engine);
61 }
62
63 static void engine_heartbeat_enable(struct intel_engine_cs *engine,
64 unsigned long saved)
65 {
66 intel_engine_pm_put(engine);
67
68 engine->props.heartbeat_interval_ms = saved;
69 }
70
71 static int live_sanitycheck(void *arg)
72 {
73 struct intel_gt *gt = arg;
74 struct intel_engine_cs *engine;
75 enum intel_engine_id id;
76 struct igt_spinner spin;
77 int err = 0;
78
79 if (!HAS_LOGICAL_RING_CONTEXTS(gt->i915))
80 return 0;
81
82 if (igt_spinner_init(&spin, gt))
83 return -ENOMEM;
84
85 for_each_engine(engine, gt, id) {
86 struct intel_context *ce;
87 struct i915_request *rq;
88
89 ce = intel_context_create(engine);
90 if (IS_ERR(ce)) {
91 err = PTR_ERR(ce);
92 break;
93 }
94
95 rq = igt_spinner_create_request(&spin, ce, MI_NOOP);
96 if (IS_ERR(rq)) {
97 err = PTR_ERR(rq);
98 goto out_ctx;
99 }
100
101 i915_request_add(rq);
102 if (!igt_wait_for_spinner(&spin, rq)) {
103 GEM_TRACE("spinner failed to start\n");
104 GEM_TRACE_DUMP();
105 intel_gt_set_wedged(gt);
106 err = -EIO;
107 goto out_ctx;
108 }
109
110 igt_spinner_end(&spin);
111 if (igt_flush_test(gt->i915)) {
112 err = -EIO;
113 goto out_ctx;
114 }
115
116 out_ctx:
117 intel_context_put(ce);
118 if (err)
119 break;
120 }
121
122 igt_spinner_fini(&spin);
123 return err;
124 }
125
126 static int live_unlite_restore(struct intel_gt *gt, int prio)
127 {
128 struct intel_engine_cs *engine;
129 enum intel_engine_id id;
130 struct igt_spinner spin;
131 int err = -ENOMEM;
132
133 /*
134 * Check that we can correctly context switch between 2 instances
135 * on the same engine from the same parent context.
136 */
137
138 if (igt_spinner_init(&spin, gt))
139 return err;
140
141 err = 0;
142 for_each_engine(engine, gt, id) {
143 struct intel_context *ce[2] = {};
144 struct i915_request *rq[2];
145 struct igt_live_test t;
146 unsigned long saved;
147 int n;
148
149 if (prio && !intel_engine_has_preemption(engine))
150 continue;
151
152 if (!intel_engine_can_store_dword(engine))
153 continue;
154
155 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
156 err = -EIO;
157 break;
158 }
159 engine_heartbeat_disable(engine, &saved);
160
161 for (n = 0; n < ARRAY_SIZE(ce); n++) {
162 struct intel_context *tmp;
163
164 tmp = intel_context_create(engine);
165 if (IS_ERR(tmp)) {
166 err = PTR_ERR(tmp);
167 goto err_ce;
168 }
169
170 err = intel_context_pin(tmp);
171 if (err) {
172 intel_context_put(tmp);
173 goto err_ce;
174 }
175
176 /*
177 * Setup the pair of contexts such that if we
178 * lite-restore using the RING_TAIL from ce[1] it
179 * will execute garbage from ce[0]->ring.
180 */
181 memset(tmp->ring->vaddr,
182 POISON_INUSE, /* IPEHR: 0x5a5a5a5a [hung!] */
183 tmp->ring->vma->size);
184
185 ce[n] = tmp;
186 }
187 GEM_BUG_ON(!ce[1]->ring->size);
188 intel_ring_reset(ce[1]->ring, ce[1]->ring->size / 2);
189 __execlists_update_reg_state(ce[1], engine);
190
191 rq[0] = igt_spinner_create_request(&spin, ce[0], MI_ARB_CHECK);
192 if (IS_ERR(rq[0])) {
193 err = PTR_ERR(rq[0]);
194 goto err_ce;
195 }
196
197 i915_request_get(rq[0]);
198 i915_request_add(rq[0]);
199 GEM_BUG_ON(rq[0]->postfix > ce[1]->ring->emit);
200
201 if (!igt_wait_for_spinner(&spin, rq[0])) {
202 i915_request_put(rq[0]);
203 goto err_ce;
204 }
205
206 rq[1] = i915_request_create(ce[1]);
207 if (IS_ERR(rq[1])) {
208 err = PTR_ERR(rq[1]);
209 i915_request_put(rq[0]);
210 goto err_ce;
211 }
212
213 if (!prio) {
214 /*
215 * Ensure we do the switch to ce[1] on completion.
216 *
217 * rq[0] is already submitted, so this should reduce
218 * to a no-op (a wait on a request on the same engine
219 * uses the submit fence, not the completion fence),
220 * but it will install a dependency on rq[1] for rq[0]
221 * that will prevent the pair being reordered by
222 * timeslicing.
223 */
224 i915_request_await_dma_fence(rq[1], &rq[0]->fence);
225 }
226
227 i915_request_get(rq[1]);
228 i915_request_add(rq[1]);
229 GEM_BUG_ON(rq[1]->postfix <= rq[0]->postfix);
230 i915_request_put(rq[0]);
231
232 if (prio) {
233 struct i915_sched_attr attr = {
234 .priority = prio,
235 };
236
237 /* Alternatively preempt the spinner with ce[1] */
238 engine->schedule(rq[1], &attr);
239 }
240
241 /* And switch back to ce[0] for good measure */
242 rq[0] = i915_request_create(ce[0]);
243 if (IS_ERR(rq[0])) {
244 err = PTR_ERR(rq[0]);
245 i915_request_put(rq[1]);
246 goto err_ce;
247 }
248
249 i915_request_await_dma_fence(rq[0], &rq[1]->fence);
250 i915_request_get(rq[0]);
251 i915_request_add(rq[0]);
252 GEM_BUG_ON(rq[0]->postfix > rq[1]->postfix);
253 i915_request_put(rq[1]);
254 i915_request_put(rq[0]);
255
256 err_ce:
257 tasklet_kill(&engine->execlists.tasklet); /* flush submission */
258 igt_spinner_end(&spin);
259 for (n = 0; n < ARRAY_SIZE(ce); n++) {
260 if (IS_ERR_OR_NULL(ce[n]))
261 break;
262
263 intel_context_unpin(ce[n]);
264 intel_context_put(ce[n]);
265 }
266
267 engine_heartbeat_enable(engine, saved);
268 if (igt_live_test_end(&t))
269 err = -EIO;
270 if (err)
271 break;
272 }
273
274 igt_spinner_fini(&spin);
275 return err;
276 }
277
278 static int live_unlite_switch(void *arg)
279 {
280 return live_unlite_restore(arg, 0);
281 }
282
283 static int live_unlite_preempt(void *arg)
284 {
285 return live_unlite_restore(arg, I915_USER_PRIORITY(I915_PRIORITY_MAX));
286 }
287
288 static int
289 emit_semaphore_chain(struct i915_request *rq, struct i915_vma *vma, int idx)
290 {
291 u32 *cs;
292
293 cs = intel_ring_begin(rq, 10);
294 if (IS_ERR(cs))
295 return PTR_ERR(cs);
296
297 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
298
299 *cs++ = MI_SEMAPHORE_WAIT |
300 MI_SEMAPHORE_GLOBAL_GTT |
301 MI_SEMAPHORE_POLL |
302 MI_SEMAPHORE_SAD_NEQ_SDD;
303 *cs++ = 0;
304 *cs++ = i915_ggtt_offset(vma) + 4 * idx;
305 *cs++ = 0;
306
307 if (idx > 0) {
308 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
309 *cs++ = i915_ggtt_offset(vma) + 4 * (idx - 1);
310 *cs++ = 0;
311 *cs++ = 1;
312 } else {
313 *cs++ = MI_NOOP;
314 *cs++ = MI_NOOP;
315 *cs++ = MI_NOOP;
316 *cs++ = MI_NOOP;
317 }
318
319 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
320
321 intel_ring_advance(rq, cs);
322 return 0;
323 }
324
325 static struct i915_request *
326 semaphore_queue(struct intel_engine_cs *engine, struct i915_vma *vma, int idx)
327 {
328 struct intel_context *ce;
329 struct i915_request *rq;
330 int err;
331
332 ce = intel_context_create(engine);
333 if (IS_ERR(ce))
334 return ERR_CAST(ce);
335
336 rq = intel_context_create_request(ce);
337 if (IS_ERR(rq))
338 goto out_ce;
339
340 err = 0;
341 if (rq->engine->emit_init_breadcrumb)
342 err = rq->engine->emit_init_breadcrumb(rq);
343 if (err == 0)
344 err = emit_semaphore_chain(rq, vma, idx);
345 if (err == 0)
346 i915_request_get(rq);
347 i915_request_add(rq);
348 if (err)
349 rq = ERR_PTR(err);
350
351 out_ce:
352 intel_context_put(ce);
353 return rq;
354 }
355
356 static int
357 release_queue(struct intel_engine_cs *engine,
358 struct i915_vma *vma,
359 int idx, int prio)
360 {
361 struct i915_sched_attr attr = {
362 .priority = prio,
363 };
364 struct i915_request *rq;
365 u32 *cs;
366
367 rq = intel_engine_create_kernel_request(engine);
368 if (IS_ERR(rq))
369 return PTR_ERR(rq);
370
371 cs = intel_ring_begin(rq, 4);
372 if (IS_ERR(cs)) {
373 i915_request_add(rq);
374 return PTR_ERR(cs);
375 }
376
377 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
378 *cs++ = i915_ggtt_offset(vma) + 4 * (idx - 1);
379 *cs++ = 0;
380 *cs++ = 1;
381
382 intel_ring_advance(rq, cs);
383
384 i915_request_get(rq);
385 i915_request_add(rq);
386
387 local_bh_disable();
388 engine->schedule(rq, &attr);
389 local_bh_enable(); /* kick tasklet */
390
391 i915_request_put(rq);
392
393 return 0;
394 }
395
396 static int
397 slice_semaphore_queue(struct intel_engine_cs *outer,
398 struct i915_vma *vma,
399 int count)
400 {
401 struct intel_engine_cs *engine;
402 struct i915_request *head;
403 enum intel_engine_id id;
404 int err, i, n = 0;
405
406 head = semaphore_queue(outer, vma, n++);
407 if (IS_ERR(head))
408 return PTR_ERR(head);
409
410 for_each_engine(engine, outer->gt, id) {
411 for (i = 0; i < count; i++) {
412 struct i915_request *rq;
413
414 rq = semaphore_queue(engine, vma, n++);
415 if (IS_ERR(rq)) {
416 err = PTR_ERR(rq);
417 goto out;
418 }
419
420 i915_request_put(rq);
421 }
422 }
423
424 err = release_queue(outer, vma, n, INT_MAX);
425 if (err)
426 goto out;
427
428 if (i915_request_wait(head, 0,
429 2 * RUNTIME_INFO(outer->i915)->num_engines * (count + 2) * (count + 3)) < 0) {
430 pr_err("Failed to slice along semaphore chain of length (%d, %d)!\n",
431 count, n);
432 GEM_TRACE_DUMP();
433 intel_gt_set_wedged(outer->gt);
434 err = -EIO;
435 }
436
437 out:
438 i915_request_put(head);
439 return err;
440 }
441
442 static int live_timeslice_preempt(void *arg)
443 {
444 struct intel_gt *gt = arg;
445 struct drm_i915_gem_object *obj;
446 struct i915_vma *vma;
447 void *vaddr;
448 int err = 0;
449 int count;
450
451 /*
452 * If a request takes too long, we would like to give other users
453 * a fair go on the GPU. In particular, users may create batches
454 * that wait upon external input, where that input may even be
455 * supplied by another GPU job. To avoid blocking forever, we
456 * need to preempt the current task and replace it with another
457 * ready task.
458 */
459 if (!IS_ACTIVE(CONFIG_DRM_I915_TIMESLICE_DURATION))
460 return 0;
461
462 obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
463 if (IS_ERR(obj))
464 return PTR_ERR(obj);
465
466 vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);
467 if (IS_ERR(vma)) {
468 err = PTR_ERR(vma);
469 goto err_obj;
470 }
471
472 vaddr = i915_gem_object_pin_map(obj, I915_MAP_WC);
473 if (IS_ERR(vaddr)) {
474 err = PTR_ERR(vaddr);
475 goto err_obj;
476 }
477
478 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
479 if (err)
480 goto err_map;
481
482 for_each_prime_number_from(count, 1, 16) {
483 struct intel_engine_cs *engine;
484 enum intel_engine_id id;
485
486 for_each_engine(engine, gt, id) {
487 unsigned long saved;
488
489 if (!intel_engine_has_preemption(engine))
490 continue;
491
492 memset(vaddr, 0, PAGE_SIZE);
493
494 engine_heartbeat_disable(engine, &saved);
495 err = slice_semaphore_queue(engine, vma, count);
496 engine_heartbeat_enable(engine, saved);
497 if (err)
498 goto err_pin;
499
500 if (igt_flush_test(gt->i915)) {
501 err = -EIO;
502 goto err_pin;
503 }
504 }
505 }
506
507 err_pin:
508 i915_vma_unpin(vma);
509 err_map:
510 i915_gem_object_unpin_map(obj);
511 err_obj:
512 i915_gem_object_put(obj);
513 return err;
514 }
515
516 static struct i915_request *nop_request(struct intel_engine_cs *engine)
517 {
518 struct i915_request *rq;
519
520 rq = intel_engine_create_kernel_request(engine);
521 if (IS_ERR(rq))
522 return rq;
523
524 i915_request_get(rq);
525 i915_request_add(rq);
526
527 return rq;
528 }
529
530 static int wait_for_submit(struct intel_engine_cs *engine,
531 struct i915_request *rq,
532 unsigned long timeout)
533 {
534 timeout += jiffies;
535 do {
536 cond_resched();
537 intel_engine_flush_submission(engine);
538 if (i915_request_is_active(rq))
539 return 0;
540 } while (time_before(jiffies, timeout));
541
542 return -ETIME;
543 }
544
545 static long timeslice_threshold(const struct intel_engine_cs *engine)
546 {
547 return 2 * msecs_to_jiffies_timeout(timeslice(engine)) + 1;
548 }
549
550 static int live_timeslice_queue(void *arg)
551 {
552 struct intel_gt *gt = arg;
553 struct drm_i915_gem_object *obj;
554 struct intel_engine_cs *engine;
555 enum intel_engine_id id;
556 struct i915_vma *vma;
557 void *vaddr;
558 int err = 0;
559
560 /*
561 * Make sure that even if ELSP[0] and ELSP[1] are filled with
562 * timeslicing between them disabled, we *do* enable timeslicing
563 * if the queue demands it. (Normally, we do not submit if
564 * ELSP[1] is already occupied, so must rely on timeslicing to
565 * eject ELSP[0] in favour of the queue.)
566 */
567 if (!IS_ACTIVE(CONFIG_DRM_I915_TIMESLICE_DURATION))
568 return 0;
569
570 obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
571 if (IS_ERR(obj))
572 return PTR_ERR(obj);
573
574 vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);
575 if (IS_ERR(vma)) {
576 err = PTR_ERR(vma);
577 goto err_obj;
578 }
579
580 vaddr = i915_gem_object_pin_map(obj, I915_MAP_WC);
581 if (IS_ERR(vaddr)) {
582 err = PTR_ERR(vaddr);
583 goto err_obj;
584 }
585
586 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
587 if (err)
588 goto err_map;
589
590 for_each_engine(engine, gt, id) {
591 struct i915_sched_attr attr = {
592 .priority = I915_USER_PRIORITY(I915_PRIORITY_MAX),
593 };
594 struct i915_request *rq, *nop;
595 unsigned long saved;
596
597 if (!intel_engine_has_preemption(engine))
598 continue;
599
600 engine_heartbeat_disable(engine, &saved);
601 memset(vaddr, 0, PAGE_SIZE);
602
603 /* ELSP[0]: semaphore wait */
604 rq = semaphore_queue(engine, vma, 0);
605 if (IS_ERR(rq)) {
606 err = PTR_ERR(rq);
607 goto err_heartbeat;
608 }
609 engine->schedule(rq, &attr);
610 err = wait_for_submit(engine, rq, HZ / 2);
611 if (err) {
612 pr_err("%s: Timed out trying to submit semaphores\n",
613 engine->name);
614 goto err_rq;
615 }
616
617 /* ELSP[1]: nop request */
618 nop = nop_request(engine);
619 if (IS_ERR(nop)) {
620 err = PTR_ERR(nop);
621 goto err_rq;
622 }
623 err = wait_for_submit(engine, nop, HZ / 2);
624 i915_request_put(nop);
625 if (err) {
626 pr_err("%s: Timed out trying to submit nop\n",
627 engine->name);
628 goto err_rq;
629 }
630
631 GEM_BUG_ON(i915_request_completed(rq));
632 GEM_BUG_ON(execlists_active(&engine->execlists) != rq);
633
634 /* Queue: semaphore signal, matching priority as semaphore */
635 err = release_queue(engine, vma, 1, effective_prio(rq));
636 if (err)
637 goto err_rq;
638
639 intel_engine_flush_submission(engine);
640 if (!READ_ONCE(engine->execlists.timer.expires) &&
641 !i915_request_completed(rq)) {
642 struct drm_printer p =
643 drm_info_printer(gt->i915->drm.dev);
644
645 GEM_TRACE_ERR("%s: Failed to enable timeslicing!\n",
646 engine->name);
647 intel_engine_dump(engine, &p,
648 "%s\n", engine->name);
649 GEM_TRACE_DUMP();
650
651 memset(vaddr, 0xff, PAGE_SIZE);
652 err = -EINVAL;
653 }
654
655 /* Timeslice every jiffy, so within 2 we should signal */
656 if (i915_request_wait(rq, 0, timeslice_threshold(engine)) < 0) {
657 struct drm_printer p =
658 drm_info_printer(gt->i915->drm.dev);
659
660 pr_err("%s: Failed to timeslice into queue\n",
661 engine->name);
662 intel_engine_dump(engine, &p,
663 "%s\n", engine->name);
664
665 memset(vaddr, 0xff, PAGE_SIZE);
666 err = -EIO;
667 }
668 err_rq:
669 i915_request_put(rq);
670 err_heartbeat:
671 engine_heartbeat_enable(engine, saved);
672 if (err)
673 break;
674 }
675
676 i915_vma_unpin(vma);
677 err_map:
678 i915_gem_object_unpin_map(obj);
679 err_obj:
680 i915_gem_object_put(obj);
681 return err;
682 }
683
684 static int live_busywait_preempt(void *arg)
685 {
686 struct intel_gt *gt = arg;
687 struct i915_gem_context *ctx_hi, *ctx_lo;
688 struct intel_engine_cs *engine;
689 struct drm_i915_gem_object *obj;
690 struct i915_vma *vma;
691 enum intel_engine_id id;
692 int err = -ENOMEM;
693 u32 *map;
694
695 /*
696 * Verify that even without HAS_LOGICAL_RING_PREEMPTION, we can
697 * preempt the busywaits used to synchronise between rings.
698 */
699
700 ctx_hi = kernel_context(gt->i915);
701 if (!ctx_hi)
702 return -ENOMEM;
703 ctx_hi->sched.priority =
704 I915_USER_PRIORITY(I915_CONTEXT_MAX_USER_PRIORITY);
705
706 ctx_lo = kernel_context(gt->i915);
707 if (!ctx_lo)
708 goto err_ctx_hi;
709 ctx_lo->sched.priority =
710 I915_USER_PRIORITY(I915_CONTEXT_MIN_USER_PRIORITY);
711
712 obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
713 if (IS_ERR(obj)) {
714 err = PTR_ERR(obj);
715 goto err_ctx_lo;
716 }
717
718 map = i915_gem_object_pin_map(obj, I915_MAP_WC);
719 if (IS_ERR(map)) {
720 err = PTR_ERR(map);
721 goto err_obj;
722 }
723
724 vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);
725 if (IS_ERR(vma)) {
726 err = PTR_ERR(vma);
727 goto err_map;
728 }
729
730 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
731 if (err)
732 goto err_map;
733
734 for_each_engine(engine, gt, id) {
735 struct i915_request *lo, *hi;
736 struct igt_live_test t;
737 u32 *cs;
738
739 if (!intel_engine_has_preemption(engine))
740 continue;
741
742 if (!intel_engine_can_store_dword(engine))
743 continue;
744
745 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
746 err = -EIO;
747 goto err_vma;
748 }
749
750 /*
751 * We create two requests. The low priority request
752 * busywaits on a semaphore (inside the ringbuffer where
753 * is should be preemptible) and the high priority requests
754 * uses a MI_STORE_DWORD_IMM to update the semaphore value
755 * allowing the first request to complete. If preemption
756 * fails, we hang instead.
757 */
758
759 lo = igt_request_alloc(ctx_lo, engine);
760 if (IS_ERR(lo)) {
761 err = PTR_ERR(lo);
762 goto err_vma;
763 }
764
765 cs = intel_ring_begin(lo, 8);
766 if (IS_ERR(cs)) {
767 err = PTR_ERR(cs);
768 i915_request_add(lo);
769 goto err_vma;
770 }
771
772 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
773 *cs++ = i915_ggtt_offset(vma);
774 *cs++ = 0;
775 *cs++ = 1;
776
777 /* XXX Do we need a flush + invalidate here? */
778
779 *cs++ = MI_SEMAPHORE_WAIT |
780 MI_SEMAPHORE_GLOBAL_GTT |
781 MI_SEMAPHORE_POLL |
782 MI_SEMAPHORE_SAD_EQ_SDD;
783 *cs++ = 0;
784 *cs++ = i915_ggtt_offset(vma);
785 *cs++ = 0;
786
787 intel_ring_advance(lo, cs);
788
789 i915_request_get(lo);
790 i915_request_add(lo);
791
792 if (wait_for(READ_ONCE(*map), 10)) {
793 i915_request_put(lo);
794 err = -ETIMEDOUT;
795 goto err_vma;
796 }
797
798 /* Low priority request should be busywaiting now */
799 if (i915_request_wait(lo, 0, 1) != -ETIME) {
800 i915_request_put(lo);
801 pr_err("%s: Busywaiting request did not!\n",
802 engine->name);
803 err = -EIO;
804 goto err_vma;
805 }
806
807 hi = igt_request_alloc(ctx_hi, engine);
808 if (IS_ERR(hi)) {
809 err = PTR_ERR(hi);
810 i915_request_put(lo);
811 goto err_vma;
812 }
813
814 cs = intel_ring_begin(hi, 4);
815 if (IS_ERR(cs)) {
816 err = PTR_ERR(cs);
817 i915_request_add(hi);
818 i915_request_put(lo);
819 goto err_vma;
820 }
821
822 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
823 *cs++ = i915_ggtt_offset(vma);
824 *cs++ = 0;
825 *cs++ = 0;
826
827 intel_ring_advance(hi, cs);
828 i915_request_add(hi);
829
830 if (i915_request_wait(lo, 0, HZ / 5) < 0) {
831 struct drm_printer p = drm_info_printer(gt->i915->drm.dev);
832
833 pr_err("%s: Failed to preempt semaphore busywait!\n",
834 engine->name);
835
836 intel_engine_dump(engine, &p, "%s\n", engine->name);
837 GEM_TRACE_DUMP();
838
839 i915_request_put(lo);
840 intel_gt_set_wedged(gt);
841 err = -EIO;
842 goto err_vma;
843 }
844 GEM_BUG_ON(READ_ONCE(*map));
845 i915_request_put(lo);
846
847 if (igt_live_test_end(&t)) {
848 err = -EIO;
849 goto err_vma;
850 }
851 }
852
853 err = 0;
854 err_vma:
855 i915_vma_unpin(vma);
856 err_map:
857 i915_gem_object_unpin_map(obj);
858 err_obj:
859 i915_gem_object_put(obj);
860 err_ctx_lo:
861 kernel_context_close(ctx_lo);
862 err_ctx_hi:
863 kernel_context_close(ctx_hi);
864 return err;
865 }
866
867 static struct i915_request *
868 spinner_create_request(struct igt_spinner *spin,
869 struct i915_gem_context *ctx,
870 struct intel_engine_cs *engine,
871 u32 arb)
872 {
873 struct intel_context *ce;
874 struct i915_request *rq;
875
876 ce = i915_gem_context_get_engine(ctx, engine->legacy_idx);
877 if (IS_ERR(ce))
878 return ERR_CAST(ce);
879
880 rq = igt_spinner_create_request(spin, ce, arb);
881 intel_context_put(ce);
882 return rq;
883 }
884
885 static int live_preempt(void *arg)
886 {
887 struct intel_gt *gt = arg;
888 struct i915_gem_context *ctx_hi, *ctx_lo;
889 struct igt_spinner spin_hi, spin_lo;
890 struct intel_engine_cs *engine;
891 enum intel_engine_id id;
892 int err = -ENOMEM;
893
894 if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
895 return 0;
896
897 if (!(gt->i915->caps.scheduler & I915_SCHEDULER_CAP_PREEMPTION))
898 pr_err("Logical preemption supported, but not exposed\n");
899
900 if (igt_spinner_init(&spin_hi, gt))
901 return -ENOMEM;
902
903 if (igt_spinner_init(&spin_lo, gt))
904 goto err_spin_hi;
905
906 ctx_hi = kernel_context(gt->i915);
907 if (!ctx_hi)
908 goto err_spin_lo;
909 ctx_hi->sched.priority =
910 I915_USER_PRIORITY(I915_CONTEXT_MAX_USER_PRIORITY);
911
912 ctx_lo = kernel_context(gt->i915);
913 if (!ctx_lo)
914 goto err_ctx_hi;
915 ctx_lo->sched.priority =
916 I915_USER_PRIORITY(I915_CONTEXT_MIN_USER_PRIORITY);
917
918 for_each_engine(engine, gt, id) {
919 struct igt_live_test t;
920 struct i915_request *rq;
921
922 if (!intel_engine_has_preemption(engine))
923 continue;
924
925 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
926 err = -EIO;
927 goto err_ctx_lo;
928 }
929
930 rq = spinner_create_request(&spin_lo, ctx_lo, engine,
931 MI_ARB_CHECK);
932 if (IS_ERR(rq)) {
933 err = PTR_ERR(rq);
934 goto err_ctx_lo;
935 }
936
937 i915_request_add(rq);
938 if (!igt_wait_for_spinner(&spin_lo, rq)) {
939 GEM_TRACE("lo spinner failed to start\n");
940 GEM_TRACE_DUMP();
941 intel_gt_set_wedged(gt);
942 err = -EIO;
943 goto err_ctx_lo;
944 }
945
946 rq = spinner_create_request(&spin_hi, ctx_hi, engine,
947 MI_ARB_CHECK);
948 if (IS_ERR(rq)) {
949 igt_spinner_end(&spin_lo);
950 err = PTR_ERR(rq);
951 goto err_ctx_lo;
952 }
953
954 i915_request_add(rq);
955 if (!igt_wait_for_spinner(&spin_hi, rq)) {
956 GEM_TRACE("hi spinner failed to start\n");
957 GEM_TRACE_DUMP();
958 intel_gt_set_wedged(gt);
959 err = -EIO;
960 goto err_ctx_lo;
961 }
962
963 igt_spinner_end(&spin_hi);
964 igt_spinner_end(&spin_lo);
965
966 if (igt_live_test_end(&t)) {
967 err = -EIO;
968 goto err_ctx_lo;
969 }
970 }
971
972 err = 0;
973 err_ctx_lo:
974 kernel_context_close(ctx_lo);
975 err_ctx_hi:
976 kernel_context_close(ctx_hi);
977 err_spin_lo:
978 igt_spinner_fini(&spin_lo);
979 err_spin_hi:
980 igt_spinner_fini(&spin_hi);
981 return err;
982 }
983
984 static int live_late_preempt(void *arg)
985 {
986 struct intel_gt *gt = arg;
987 struct i915_gem_context *ctx_hi, *ctx_lo;
988 struct igt_spinner spin_hi, spin_lo;
989 struct intel_engine_cs *engine;
990 struct i915_sched_attr attr = {};
991 enum intel_engine_id id;
992 int err = -ENOMEM;
993
994 if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
995 return 0;
996
997 if (igt_spinner_init(&spin_hi, gt))
998 return -ENOMEM;
999
1000 if (igt_spinner_init(&spin_lo, gt))
1001 goto err_spin_hi;
1002
1003 ctx_hi = kernel_context(gt->i915);
1004 if (!ctx_hi)
1005 goto err_spin_lo;
1006
1007 ctx_lo = kernel_context(gt->i915);
1008 if (!ctx_lo)
1009 goto err_ctx_hi;
1010
1011 /* Make sure ctx_lo stays before ctx_hi until we trigger preemption. */
1012 ctx_lo->sched.priority = I915_USER_PRIORITY(1);
1013
1014 for_each_engine(engine, gt, id) {
1015 struct igt_live_test t;
1016 struct i915_request *rq;
1017
1018 if (!intel_engine_has_preemption(engine))
1019 continue;
1020
1021 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
1022 err = -EIO;
1023 goto err_ctx_lo;
1024 }
1025
1026 rq = spinner_create_request(&spin_lo, ctx_lo, engine,
1027 MI_ARB_CHECK);
1028 if (IS_ERR(rq)) {
1029 err = PTR_ERR(rq);
1030 goto err_ctx_lo;
1031 }
1032
1033 i915_request_add(rq);
1034 if (!igt_wait_for_spinner(&spin_lo, rq)) {
1035 pr_err("First context failed to start\n");
1036 goto err_wedged;
1037 }
1038
1039 rq = spinner_create_request(&spin_hi, ctx_hi, engine,
1040 MI_NOOP);
1041 if (IS_ERR(rq)) {
1042 igt_spinner_end(&spin_lo);
1043 err = PTR_ERR(rq);
1044 goto err_ctx_lo;
1045 }
1046
1047 i915_request_add(rq);
1048 if (igt_wait_for_spinner(&spin_hi, rq)) {
1049 pr_err("Second context overtook first?\n");
1050 goto err_wedged;
1051 }
1052
1053 attr.priority = I915_USER_PRIORITY(I915_PRIORITY_MAX);
1054 engine->schedule(rq, &attr);
1055
1056 if (!igt_wait_for_spinner(&spin_hi, rq)) {
1057 pr_err("High priority context failed to preempt the low priority context\n");
1058 GEM_TRACE_DUMP();
1059 goto err_wedged;
1060 }
1061
1062 igt_spinner_end(&spin_hi);
1063 igt_spinner_end(&spin_lo);
1064
1065 if (igt_live_test_end(&t)) {
1066 err = -EIO;
1067 goto err_ctx_lo;
1068 }
1069 }
1070
1071 err = 0;
1072 err_ctx_lo:
1073 kernel_context_close(ctx_lo);
1074 err_ctx_hi:
1075 kernel_context_close(ctx_hi);
1076 err_spin_lo:
1077 igt_spinner_fini(&spin_lo);
1078 err_spin_hi:
1079 igt_spinner_fini(&spin_hi);
1080 return err;
1081
1082 err_wedged:
1083 igt_spinner_end(&spin_hi);
1084 igt_spinner_end(&spin_lo);
1085 intel_gt_set_wedged(gt);
1086 err = -EIO;
1087 goto err_ctx_lo;
1088 }
1089
1090 struct preempt_client {
1091 struct igt_spinner spin;
1092 struct i915_gem_context *ctx;
1093 };
1094
1095 static int preempt_client_init(struct intel_gt *gt, struct preempt_client *c)
1096 {
1097 c->ctx = kernel_context(gt->i915);
1098 if (!c->ctx)
1099 return -ENOMEM;
1100
1101 if (igt_spinner_init(&c->spin, gt))
1102 goto err_ctx;
1103
1104 return 0;
1105
1106 err_ctx:
1107 kernel_context_close(c->ctx);
1108 return -ENOMEM;
1109 }
1110
1111 static void preempt_client_fini(struct preempt_client *c)
1112 {
1113 igt_spinner_fini(&c->spin);
1114 kernel_context_close(c->ctx);
1115 }
1116
1117 static int live_nopreempt(void *arg)
1118 {
1119 struct intel_gt *gt = arg;
1120 struct intel_engine_cs *engine;
1121 struct preempt_client a, b;
1122 enum intel_engine_id id;
1123 int err = -ENOMEM;
1124
1125 /*
1126 * Verify that we can disable preemption for an individual request
1127 * that may be being observed and not want to be interrupted.
1128 */
1129
1130 if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
1131 return 0;
1132
1133 if (preempt_client_init(gt, &a))
1134 return -ENOMEM;
1135 if (preempt_client_init(gt, &b))
1136 goto err_client_a;
1137 b.ctx->sched.priority = I915_USER_PRIORITY(I915_PRIORITY_MAX);
1138
1139 for_each_engine(engine, gt, id) {
1140 struct i915_request *rq_a, *rq_b;
1141
1142 if (!intel_engine_has_preemption(engine))
1143 continue;
1144
1145 engine->execlists.preempt_hang.count = 0;
1146
1147 rq_a = spinner_create_request(&a.spin,
1148 a.ctx, engine,
1149 MI_ARB_CHECK);
1150 if (IS_ERR(rq_a)) {
1151 err = PTR_ERR(rq_a);
1152 goto err_client_b;
1153 }
1154
1155 /* Low priority client, but unpreemptable! */
1156 __set_bit(I915_FENCE_FLAG_NOPREEMPT, &rq_a->fence.flags);
1157
1158 i915_request_add(rq_a);
1159 if (!igt_wait_for_spinner(&a.spin, rq_a)) {
1160 pr_err("First client failed to start\n");
1161 goto err_wedged;
1162 }
1163
1164 rq_b = spinner_create_request(&b.spin,
1165 b.ctx, engine,
1166 MI_ARB_CHECK);
1167 if (IS_ERR(rq_b)) {
1168 err = PTR_ERR(rq_b);
1169 goto err_client_b;
1170 }
1171
1172 i915_request_add(rq_b);
1173
1174 /* B is much more important than A! (But A is unpreemptable.) */
1175 GEM_BUG_ON(rq_prio(rq_b) <= rq_prio(rq_a));
1176
1177 /* Wait long enough for preemption and timeslicing */
1178 if (igt_wait_for_spinner(&b.spin, rq_b)) {
1179 pr_err("Second client started too early!\n");
1180 goto err_wedged;
1181 }
1182
1183 igt_spinner_end(&a.spin);
1184
1185 if (!igt_wait_for_spinner(&b.spin, rq_b)) {
1186 pr_err("Second client failed to start\n");
1187 goto err_wedged;
1188 }
1189
1190 igt_spinner_end(&b.spin);
1191
1192 if (engine->execlists.preempt_hang.count) {
1193 pr_err("Preemption recorded x%d; should have been suppressed!\n",
1194 engine->execlists.preempt_hang.count);
1195 err = -EINVAL;
1196 goto err_wedged;
1197 }
1198
1199 if (igt_flush_test(gt->i915))
1200 goto err_wedged;
1201 }
1202
1203 err = 0;
1204 err_client_b:
1205 preempt_client_fini(&b);
1206 err_client_a:
1207 preempt_client_fini(&a);
1208 return err;
1209
1210 err_wedged:
1211 igt_spinner_end(&b.spin);
1212 igt_spinner_end(&a.spin);
1213 intel_gt_set_wedged(gt);
1214 err = -EIO;
1215 goto err_client_b;
1216 }
1217
1218 struct live_preempt_cancel {
1219 struct intel_engine_cs *engine;
1220 struct preempt_client a, b;
1221 };
1222
1223 static int __cancel_active0(struct live_preempt_cancel *arg)
1224 {
1225 struct i915_request *rq;
1226 struct igt_live_test t;
1227 int err;
1228
1229 /* Preempt cancel of ELSP0 */
1230 GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);
1231 if (igt_live_test_begin(&t, arg->engine->i915,
1232 __func__, arg->engine->name))
1233 return -EIO;
1234
1235 rq = spinner_create_request(&arg->a.spin,
1236 arg->a.ctx, arg->engine,
1237 MI_ARB_CHECK);
1238 if (IS_ERR(rq))
1239 return PTR_ERR(rq);
1240
1241 clear_bit(CONTEXT_BANNED, &rq->context->flags);
1242 i915_request_get(rq);
1243 i915_request_add(rq);
1244 if (!igt_wait_for_spinner(&arg->a.spin, rq)) {
1245 err = -EIO;
1246 goto out;
1247 }
1248
1249 intel_context_set_banned(rq->context);
1250 err = intel_engine_pulse(arg->engine);
1251 if (err)
1252 goto out;
1253
1254 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
1255 err = -EIO;
1256 goto out;
1257 }
1258
1259 if (rq->fence.error != -EIO) {
1260 pr_err("Cancelled inflight0 request did not report -EIO\n");
1261 err = -EINVAL;
1262 goto out;
1263 }
1264
1265 out:
1266 i915_request_put(rq);
1267 if (igt_live_test_end(&t))
1268 err = -EIO;
1269 return err;
1270 }
1271
1272 static int __cancel_active1(struct live_preempt_cancel *arg)
1273 {
1274 struct i915_request *rq[2] = {};
1275 struct igt_live_test t;
1276 int err;
1277
1278 /* Preempt cancel of ELSP1 */
1279 GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);
1280 if (igt_live_test_begin(&t, arg->engine->i915,
1281 __func__, arg->engine->name))
1282 return -EIO;
1283
1284 rq[0] = spinner_create_request(&arg->a.spin,
1285 arg->a.ctx, arg->engine,
1286 MI_NOOP); /* no preemption */
1287 if (IS_ERR(rq[0]))
1288 return PTR_ERR(rq[0]);
1289
1290 clear_bit(CONTEXT_BANNED, &rq[0]->context->flags);
1291 i915_request_get(rq[0]);
1292 i915_request_add(rq[0]);
1293 if (!igt_wait_for_spinner(&arg->a.spin, rq[0])) {
1294 err = -EIO;
1295 goto out;
1296 }
1297
1298 rq[1] = spinner_create_request(&arg->b.spin,
1299 arg->b.ctx, arg->engine,
1300 MI_ARB_CHECK);
1301 if (IS_ERR(rq[1])) {
1302 err = PTR_ERR(rq[1]);
1303 goto out;
1304 }
1305
1306 clear_bit(CONTEXT_BANNED, &rq[1]->context->flags);
1307 i915_request_get(rq[1]);
1308 err = i915_request_await_dma_fence(rq[1], &rq[0]->fence);
1309 i915_request_add(rq[1]);
1310 if (err)
1311 goto out;
1312
1313 intel_context_set_banned(rq[1]->context);
1314 err = intel_engine_pulse(arg->engine);
1315 if (err)
1316 goto out;
1317
1318 igt_spinner_end(&arg->a.spin);
1319 if (i915_request_wait(rq[1], 0, HZ / 5) < 0) {
1320 err = -EIO;
1321 goto out;
1322 }
1323
1324 if (rq[0]->fence.error != 0) {
1325 pr_err("Normal inflight0 request did not complete\n");
1326 err = -EINVAL;
1327 goto out;
1328 }
1329
1330 if (rq[1]->fence.error != -EIO) {
1331 pr_err("Cancelled inflight1 request did not report -EIO\n");
1332 err = -EINVAL;
1333 goto out;
1334 }
1335
1336 out:
1337 i915_request_put(rq[1]);
1338 i915_request_put(rq[0]);
1339 if (igt_live_test_end(&t))
1340 err = -EIO;
1341 return err;
1342 }
1343
1344 static int __cancel_queued(struct live_preempt_cancel *arg)
1345 {
1346 struct i915_request *rq[3] = {};
1347 struct igt_live_test t;
1348 int err;
1349
1350 /* Full ELSP and one in the wings */
1351 GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);
1352 if (igt_live_test_begin(&t, arg->engine->i915,
1353 __func__, arg->engine->name))
1354 return -EIO;
1355
1356 rq[0] = spinner_create_request(&arg->a.spin,
1357 arg->a.ctx, arg->engine,
1358 MI_ARB_CHECK);
1359 if (IS_ERR(rq[0]))
1360 return PTR_ERR(rq[0]);
1361
1362 clear_bit(CONTEXT_BANNED, &rq[0]->context->flags);
1363 i915_request_get(rq[0]);
1364 i915_request_add(rq[0]);
1365 if (!igt_wait_for_spinner(&arg->a.spin, rq[0])) {
1366 err = -EIO;
1367 goto out;
1368 }
1369
1370 rq[1] = igt_request_alloc(arg->b.ctx, arg->engine);
1371 if (IS_ERR(rq[1])) {
1372 err = PTR_ERR(rq[1]);
1373 goto out;
1374 }
1375
1376 clear_bit(CONTEXT_BANNED, &rq[1]->context->flags);
1377 i915_request_get(rq[1]);
1378 err = i915_request_await_dma_fence(rq[1], &rq[0]->fence);
1379 i915_request_add(rq[1]);
1380 if (err)
1381 goto out;
1382
1383 rq[2] = spinner_create_request(&arg->b.spin,
1384 arg->a.ctx, arg->engine,
1385 MI_ARB_CHECK);
1386 if (IS_ERR(rq[2])) {
1387 err = PTR_ERR(rq[2]);
1388 goto out;
1389 }
1390
1391 i915_request_get(rq[2]);
1392 err = i915_request_await_dma_fence(rq[2], &rq[1]->fence);
1393 i915_request_add(rq[2]);
1394 if (err)
1395 goto out;
1396
1397 intel_context_set_banned(rq[2]->context);
1398 err = intel_engine_pulse(arg->engine);
1399 if (err)
1400 goto out;
1401
1402 if (i915_request_wait(rq[2], 0, HZ / 5) < 0) {
1403 err = -EIO;
1404 goto out;
1405 }
1406
1407 if (rq[0]->fence.error != -EIO) {
1408 pr_err("Cancelled inflight0 request did not report -EIO\n");
1409 err = -EINVAL;
1410 goto out;
1411 }
1412
1413 if (rq[1]->fence.error != 0) {
1414 pr_err("Normal inflight1 request did not complete\n");
1415 err = -EINVAL;
1416 goto out;
1417 }
1418
1419 if (rq[2]->fence.error != -EIO) {
1420 pr_err("Cancelled queued request did not report -EIO\n");
1421 err = -EINVAL;
1422 goto out;
1423 }
1424
1425 out:
1426 i915_request_put(rq[2]);
1427 i915_request_put(rq[1]);
1428 i915_request_put(rq[0]);
1429 if (igt_live_test_end(&t))
1430 err = -EIO;
1431 return err;
1432 }
1433
1434 static int __cancel_hostile(struct live_preempt_cancel *arg)
1435 {
1436 struct i915_request *rq;
1437 int err;
1438
1439 /* Preempt cancel non-preemptible spinner in ELSP0 */
1440 if (!IS_ACTIVE(CONFIG_DRM_I915_PREEMPT_TIMEOUT))
1441 return 0;
1442
1443 GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);
1444 rq = spinner_create_request(&arg->a.spin,
1445 arg->a.ctx, arg->engine,
1446 MI_NOOP); /* preemption disabled */
1447 if (IS_ERR(rq))
1448 return PTR_ERR(rq);
1449
1450 clear_bit(CONTEXT_BANNED, &rq->context->flags);
1451 i915_request_get(rq);
1452 i915_request_add(rq);
1453 if (!igt_wait_for_spinner(&arg->a.spin, rq)) {
1454 err = -EIO;
1455 goto out;
1456 }
1457
1458 intel_context_set_banned(rq->context);
1459 err = intel_engine_pulse(arg->engine); /* force reset */
1460 if (err)
1461 goto out;
1462
1463 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
1464 err = -EIO;
1465 goto out;
1466 }
1467
1468 if (rq->fence.error != -EIO) {
1469 pr_err("Cancelled inflight0 request did not report -EIO\n");
1470 err = -EINVAL;
1471 goto out;
1472 }
1473
1474 out:
1475 i915_request_put(rq);
1476 if (igt_flush_test(arg->engine->i915))
1477 err = -EIO;
1478 return err;
1479 }
1480
1481 static int live_preempt_cancel(void *arg)
1482 {
1483 struct intel_gt *gt = arg;
1484 struct live_preempt_cancel data;
1485 enum intel_engine_id id;
1486 int err = -ENOMEM;
1487
1488 /*
1489 * To cancel an inflight context, we need to first remove it from the
1490 * GPU. That sounds like preemption! Plus a little bit of bookkeeping.
1491 */
1492
1493 if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
1494 return 0;
1495
1496 if (preempt_client_init(gt, &data.a))
1497 return -ENOMEM;
1498 if (preempt_client_init(gt, &data.b))
1499 goto err_client_a;
1500
1501 for_each_engine(data.engine, gt, id) {
1502 if (!intel_engine_has_preemption(data.engine))
1503 continue;
1504
1505 err = __cancel_active0(&data);
1506 if (err)
1507 goto err_wedged;
1508
1509 err = __cancel_active1(&data);
1510 if (err)
1511 goto err_wedged;
1512
1513 err = __cancel_queued(&data);
1514 if (err)
1515 goto err_wedged;
1516
1517 err = __cancel_hostile(&data);
1518 if (err)
1519 goto err_wedged;
1520 }
1521
1522 err = 0;
1523 err_client_b:
1524 preempt_client_fini(&data.b);
1525 err_client_a:
1526 preempt_client_fini(&data.a);
1527 return err;
1528
1529 err_wedged:
1530 GEM_TRACE_DUMP();
1531 igt_spinner_end(&data.b.spin);
1532 igt_spinner_end(&data.a.spin);
1533 intel_gt_set_wedged(gt);
1534 goto err_client_b;
1535 }
1536
1537 static int live_suppress_self_preempt(void *arg)
1538 {
1539 struct intel_gt *gt = arg;
1540 struct intel_engine_cs *engine;
1541 struct i915_sched_attr attr = {
1542 .priority = I915_USER_PRIORITY(I915_PRIORITY_MAX)
1543 };
1544 struct preempt_client a, b;
1545 enum intel_engine_id id;
1546 int err = -ENOMEM;
1547
1548 /*
1549 * Verify that if a preemption request does not cause a change in
1550 * the current execution order, the preempt-to-idle injection is
1551 * skipped and that we do not accidentally apply it after the CS
1552 * completion event.
1553 */
1554
1555 if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
1556 return 0;
1557
1558 if (USES_GUC_SUBMISSION(gt->i915))
1559 return 0; /* presume black blox */
1560
1561 if (intel_vgpu_active(gt->i915))
1562 return 0; /* GVT forces single port & request submission */
1563
1564 if (preempt_client_init(gt, &a))
1565 return -ENOMEM;
1566 if (preempt_client_init(gt, &b))
1567 goto err_client_a;
1568
1569 for_each_engine(engine, gt, id) {
1570 struct i915_request *rq_a, *rq_b;
1571 int depth;
1572
1573 if (!intel_engine_has_preemption(engine))
1574 continue;
1575
1576 if (igt_flush_test(gt->i915))
1577 goto err_wedged;
1578
1579 intel_engine_pm_get(engine);
1580 engine->execlists.preempt_hang.count = 0;
1581
1582 rq_a = spinner_create_request(&a.spin,
1583 a.ctx, engine,
1584 MI_NOOP);
1585 if (IS_ERR(rq_a)) {
1586 err = PTR_ERR(rq_a);
1587 intel_engine_pm_put(engine);
1588 goto err_client_b;
1589 }
1590
1591 i915_request_add(rq_a);
1592 if (!igt_wait_for_spinner(&a.spin, rq_a)) {
1593 pr_err("First client failed to start\n");
1594 intel_engine_pm_put(engine);
1595 goto err_wedged;
1596 }
1597
1598 /* Keep postponing the timer to avoid premature slicing */
1599 mod_timer(&engine->execlists.timer, jiffies + HZ);
1600 for (depth = 0; depth < 8; depth++) {
1601 rq_b = spinner_create_request(&b.spin,
1602 b.ctx, engine,
1603 MI_NOOP);
1604 if (IS_ERR(rq_b)) {
1605 err = PTR_ERR(rq_b);
1606 intel_engine_pm_put(engine);
1607 goto err_client_b;
1608 }
1609 i915_request_add(rq_b);
1610
1611 GEM_BUG_ON(i915_request_completed(rq_a));
1612 engine->schedule(rq_a, &attr);
1613 igt_spinner_end(&a.spin);
1614
1615 if (!igt_wait_for_spinner(&b.spin, rq_b)) {
1616 pr_err("Second client failed to start\n");
1617 intel_engine_pm_put(engine);
1618 goto err_wedged;
1619 }
1620
1621 swap(a, b);
1622 rq_a = rq_b;
1623 }
1624 igt_spinner_end(&a.spin);
1625
1626 if (engine->execlists.preempt_hang.count) {
1627 pr_err("Preemption on %s recorded x%d, depth %d; should have been suppressed!\n",
1628 engine->name,
1629 engine->execlists.preempt_hang.count,
1630 depth);
1631 intel_engine_pm_put(engine);
1632 err = -EINVAL;
1633 goto err_client_b;
1634 }
1635
1636 intel_engine_pm_put(engine);
1637 if (igt_flush_test(gt->i915))
1638 goto err_wedged;
1639 }
1640
1641 err = 0;
1642 err_client_b:
1643 preempt_client_fini(&b);
1644 err_client_a:
1645 preempt_client_fini(&a);
1646 return err;
1647
1648 err_wedged:
1649 igt_spinner_end(&b.spin);
1650 igt_spinner_end(&a.spin);
1651 intel_gt_set_wedged(gt);
1652 err = -EIO;
1653 goto err_client_b;
1654 }
1655
1656 static int __i915_sw_fence_call
1657 dummy_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
1658 {
1659 return NOTIFY_DONE;
1660 }
1661
1662 static struct i915_request *dummy_request(struct intel_engine_cs *engine)
1663 {
1664 struct i915_request *rq;
1665
1666 rq = kzalloc(sizeof(*rq), GFP_KERNEL);
1667 if (!rq)
1668 return NULL;
1669
1670 rq->engine = engine;
1671
1672 spin_lock_init(&rq->lock);
1673 INIT_LIST_HEAD(&rq->fence.cb_list);
1674 rq->fence.lock = &rq->lock;
1675 rq->fence.ops = &i915_fence_ops;
1676
1677 i915_sched_node_init(&rq->sched);
1678
1679 /* mark this request as permanently incomplete */
1680 rq->fence.seqno = 1;
1681 BUILD_BUG_ON(sizeof(rq->fence.seqno) != 8); /* upper 32b == 0 */
1682 rq->hwsp_seqno = (u32 *)&rq->fence.seqno + 1;
1683 GEM_BUG_ON(i915_request_completed(rq));
1684
1685 i915_sw_fence_init(&rq->submit, dummy_notify);
1686 set_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags);
1687
1688 spin_lock_init(&rq->lock);
1689 rq->fence.lock = &rq->lock;
1690 INIT_LIST_HEAD(&rq->fence.cb_list);
1691
1692 return rq;
1693 }
1694
1695 static void dummy_request_free(struct i915_request *dummy)
1696 {
1697 /* We have to fake the CS interrupt to kick the next request */
1698 i915_sw_fence_commit(&dummy->submit);
1699
1700 i915_request_mark_complete(dummy);
1701 dma_fence_signal(&dummy->fence);
1702
1703 i915_sched_node_fini(&dummy->sched);
1704 i915_sw_fence_fini(&dummy->submit);
1705
1706 dma_fence_free(&dummy->fence);
1707 }
1708
1709 static int live_suppress_wait_preempt(void *arg)
1710 {
1711 struct intel_gt *gt = arg;
1712 struct preempt_client client[4];
1713 struct i915_request *rq[ARRAY_SIZE(client)] = {};
1714 struct intel_engine_cs *engine;
1715 enum intel_engine_id id;
1716 int err = -ENOMEM;
1717 int i;
1718
1719 /*
1720 * Waiters are given a little priority nudge, but not enough
1721 * to actually cause any preemption. Double check that we do
1722 * not needlessly generate preempt-to-idle cycles.
1723 */
1724
1725 if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
1726 return 0;
1727
1728 if (preempt_client_init(gt, &client[0])) /* ELSP[0] */
1729 return -ENOMEM;
1730 if (preempt_client_init(gt, &client[1])) /* ELSP[1] */
1731 goto err_client_0;
1732 if (preempt_client_init(gt, &client[2])) /* head of queue */
1733 goto err_client_1;
1734 if (preempt_client_init(gt, &client[3])) /* bystander */
1735 goto err_client_2;
1736
1737 for_each_engine(engine, gt, id) {
1738 int depth;
1739
1740 if (!intel_engine_has_preemption(engine))
1741 continue;
1742
1743 if (!engine->emit_init_breadcrumb)
1744 continue;
1745
1746 for (depth = 0; depth < ARRAY_SIZE(client); depth++) {
1747 struct i915_request *dummy;
1748
1749 engine->execlists.preempt_hang.count = 0;
1750
1751 dummy = dummy_request(engine);
1752 if (!dummy)
1753 goto err_client_3;
1754
1755 for (i = 0; i < ARRAY_SIZE(client); i++) {
1756 struct i915_request *this;
1757
1758 this = spinner_create_request(&client[i].spin,
1759 client[i].ctx, engine,
1760 MI_NOOP);
1761 if (IS_ERR(this)) {
1762 err = PTR_ERR(this);
1763 goto err_wedged;
1764 }
1765
1766 /* Disable NEWCLIENT promotion */
1767 __i915_active_fence_set(&i915_request_timeline(this)->last_request,
1768 &dummy->fence);
1769
1770 rq[i] = i915_request_get(this);
1771 i915_request_add(this);
1772 }
1773
1774 dummy_request_free(dummy);
1775
1776 GEM_BUG_ON(i915_request_completed(rq[0]));
1777 if (!igt_wait_for_spinner(&client[0].spin, rq[0])) {
1778 pr_err("%s: First client failed to start\n",
1779 engine->name);
1780 goto err_wedged;
1781 }
1782 GEM_BUG_ON(!i915_request_started(rq[0]));
1783
1784 if (i915_request_wait(rq[depth],
1785 I915_WAIT_PRIORITY,
1786 1) != -ETIME) {
1787 pr_err("%s: Waiter depth:%d completed!\n",
1788 engine->name, depth);
1789 goto err_wedged;
1790 }
1791
1792 for (i = 0; i < ARRAY_SIZE(client); i++) {
1793 igt_spinner_end(&client[i].spin);
1794 i915_request_put(rq[i]);
1795 rq[i] = NULL;
1796 }
1797
1798 if (igt_flush_test(gt->i915))
1799 goto err_wedged;
1800
1801 if (engine->execlists.preempt_hang.count) {
1802 pr_err("%s: Preemption recorded x%d, depth %d; should have been suppressed!\n",
1803 engine->name,
1804 engine->execlists.preempt_hang.count,
1805 depth);
1806 err = -EINVAL;
1807 goto err_client_3;
1808 }
1809 }
1810 }
1811
1812 err = 0;
1813 err_client_3:
1814 preempt_client_fini(&client[3]);
1815 err_client_2:
1816 preempt_client_fini(&client[2]);
1817 err_client_1:
1818 preempt_client_fini(&client[1]);
1819 err_client_0:
1820 preempt_client_fini(&client[0]);
1821 return err;
1822
1823 err_wedged:
1824 for (i = 0; i < ARRAY_SIZE(client); i++) {
1825 igt_spinner_end(&client[i].spin);
1826 i915_request_put(rq[i]);
1827 }
1828 intel_gt_set_wedged(gt);
1829 err = -EIO;
1830 goto err_client_3;
1831 }
1832
1833 static int live_chain_preempt(void *arg)
1834 {
1835 struct intel_gt *gt = arg;
1836 struct intel_engine_cs *engine;
1837 struct preempt_client hi, lo;
1838 enum intel_engine_id id;
1839 int err = -ENOMEM;
1840
1841 /*
1842 * Build a chain AB...BA between two contexts (A, B) and request
1843 * preemption of the last request. It should then complete before
1844 * the previously submitted spinner in B.
1845 */
1846
1847 if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
1848 return 0;
1849
1850 if (preempt_client_init(gt, &hi))
1851 return -ENOMEM;
1852
1853 if (preempt_client_init(gt, &lo))
1854 goto err_client_hi;
1855
1856 for_each_engine(engine, gt, id) {
1857 struct i915_sched_attr attr = {
1858 .priority = I915_USER_PRIORITY(I915_PRIORITY_MAX),
1859 };
1860 struct igt_live_test t;
1861 struct i915_request *rq;
1862 int ring_size, count, i;
1863
1864 if (!intel_engine_has_preemption(engine))
1865 continue;
1866
1867 rq = spinner_create_request(&lo.spin,
1868 lo.ctx, engine,
1869 MI_ARB_CHECK);
1870 if (IS_ERR(rq))
1871 goto err_wedged;
1872
1873 i915_request_get(rq);
1874 i915_request_add(rq);
1875
1876 ring_size = rq->wa_tail - rq->head;
1877 if (ring_size < 0)
1878 ring_size += rq->ring->size;
1879 ring_size = rq->ring->size / ring_size;
1880 pr_debug("%s(%s): Using maximum of %d requests\n",
1881 __func__, engine->name, ring_size);
1882
1883 igt_spinner_end(&lo.spin);
1884 if (i915_request_wait(rq, 0, HZ / 2) < 0) {
1885 pr_err("Timed out waiting to flush %s\n", engine->name);
1886 i915_request_put(rq);
1887 goto err_wedged;
1888 }
1889 i915_request_put(rq);
1890
1891 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
1892 err = -EIO;
1893 goto err_wedged;
1894 }
1895
1896 for_each_prime_number_from(count, 1, ring_size) {
1897 rq = spinner_create_request(&hi.spin,
1898 hi.ctx, engine,
1899 MI_ARB_CHECK);
1900 if (IS_ERR(rq))
1901 goto err_wedged;
1902 i915_request_add(rq);
1903 if (!igt_wait_for_spinner(&hi.spin, rq))
1904 goto err_wedged;
1905
1906 rq = spinner_create_request(&lo.spin,
1907 lo.ctx, engine,
1908 MI_ARB_CHECK);
1909 if (IS_ERR(rq))
1910 goto err_wedged;
1911 i915_request_add(rq);
1912
1913 for (i = 0; i < count; i++) {
1914 rq = igt_request_alloc(lo.ctx, engine);
1915 if (IS_ERR(rq))
1916 goto err_wedged;
1917 i915_request_add(rq);
1918 }
1919
1920 rq = igt_request_alloc(hi.ctx, engine);
1921 if (IS_ERR(rq))
1922 goto err_wedged;
1923
1924 i915_request_get(rq);
1925 i915_request_add(rq);
1926 engine->schedule(rq, &attr);
1927
1928 igt_spinner_end(&hi.spin);
1929 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
1930 struct drm_printer p =
1931 drm_info_printer(gt->i915->drm.dev);
1932
1933 pr_err("Failed to preempt over chain of %d\n",
1934 count);
1935 intel_engine_dump(engine, &p,
1936 "%s\n", engine->name);
1937 i915_request_put(rq);
1938 goto err_wedged;
1939 }
1940 igt_spinner_end(&lo.spin);
1941 i915_request_put(rq);
1942
1943 rq = igt_request_alloc(lo.ctx, engine);
1944 if (IS_ERR(rq))
1945 goto err_wedged;
1946
1947 i915_request_get(rq);
1948 i915_request_add(rq);
1949
1950 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
1951 struct drm_printer p =
1952 drm_info_printer(gt->i915->drm.dev);
1953
1954 pr_err("Failed to flush low priority chain of %d requests\n",
1955 count);
1956 intel_engine_dump(engine, &p,
1957 "%s\n", engine->name);
1958
1959 i915_request_put(rq);
1960 goto err_wedged;
1961 }
1962 i915_request_put(rq);
1963 }
1964
1965 if (igt_live_test_end(&t)) {
1966 err = -EIO;
1967 goto err_wedged;
1968 }
1969 }
1970
1971 err = 0;
1972 err_client_lo:
1973 preempt_client_fini(&lo);
1974 err_client_hi:
1975 preempt_client_fini(&hi);
1976 return err;
1977
1978 err_wedged:
1979 igt_spinner_end(&hi.spin);
1980 igt_spinner_end(&lo.spin);
1981 intel_gt_set_wedged(gt);
1982 err = -EIO;
1983 goto err_client_lo;
1984 }
1985
1986 static int create_gang(struct intel_engine_cs *engine,
1987 struct i915_request **prev)
1988 {
1989 struct drm_i915_gem_object *obj;
1990 struct intel_context *ce;
1991 struct i915_request *rq;
1992 struct i915_vma *vma;
1993 u32 *cs;
1994 int err;
1995
1996 ce = intel_context_create(engine);
1997 if (IS_ERR(ce))
1998 return PTR_ERR(ce);
1999
2000 obj = i915_gem_object_create_internal(engine->i915, 4096);
2001 if (IS_ERR(obj)) {
2002 err = PTR_ERR(obj);
2003 goto err_ce;
2004 }
2005
2006 vma = i915_vma_instance(obj, ce->vm, NULL);
2007 if (IS_ERR(vma)) {
2008 err = PTR_ERR(vma);
2009 goto err_obj;
2010 }
2011
2012 err = i915_vma_pin(vma, 0, 0, PIN_USER);
2013 if (err)
2014 goto err_obj;
2015
2016 cs = i915_gem_object_pin_map(obj, I915_MAP_WC);
2017 if (IS_ERR(cs))
2018 goto err_obj;
2019
2020 /* Semaphore target: spin until zero */
2021 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
2022
2023 *cs++ = MI_SEMAPHORE_WAIT |
2024 MI_SEMAPHORE_POLL |
2025 MI_SEMAPHORE_SAD_EQ_SDD;
2026 *cs++ = 0;
2027 *cs++ = lower_32_bits(vma->node.start);
2028 *cs++ = upper_32_bits(vma->node.start);
2029
2030 if (*prev) {
2031 u64 offset = (*prev)->batch->node.start;
2032
2033 /* Terminate the spinner in the next lower priority batch. */
2034 *cs++ = MI_STORE_DWORD_IMM_GEN4;
2035 *cs++ = lower_32_bits(offset);
2036 *cs++ = upper_32_bits(offset);
2037 *cs++ = 0;
2038 }
2039
2040 *cs++ = MI_BATCH_BUFFER_END;
2041 i915_gem_object_flush_map(obj);
2042 i915_gem_object_unpin_map(obj);
2043
2044 rq = intel_context_create_request(ce);
2045 if (IS_ERR(rq))
2046 goto err_obj;
2047
2048 rq->batch = vma;
2049 i915_request_get(rq);
2050
2051 i915_vma_lock(vma);
2052 err = i915_request_await_object(rq, vma->obj, false);
2053 if (!err)
2054 err = i915_vma_move_to_active(vma, rq, 0);
2055 if (!err)
2056 err = rq->engine->emit_bb_start(rq,
2057 vma->node.start,
2058 PAGE_SIZE, 0);
2059 i915_vma_unlock(vma);
2060 i915_request_add(rq);
2061 if (err)
2062 goto err_rq;
2063
2064 i915_gem_object_put(obj);
2065 intel_context_put(ce);
2066
2067 rq->client_link.next = &(*prev)->client_link;
2068 *prev = rq;
2069 return 0;
2070
2071 err_rq:
2072 i915_request_put(rq);
2073 err_obj:
2074 i915_gem_object_put(obj);
2075 err_ce:
2076 intel_context_put(ce);
2077 return err;
2078 }
2079
2080 static int live_preempt_gang(void *arg)
2081 {
2082 struct intel_gt *gt = arg;
2083 struct intel_engine_cs *engine;
2084 enum intel_engine_id id;
2085
2086 if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
2087 return 0;
2088
2089 /*
2090 * Build as long a chain of preempters as we can, with each
2091 * request higher priority than the last. Once we are ready, we release
2092 * the last batch which then precolates down the chain, each releasing
2093 * the next oldest in turn. The intent is to simply push as hard as we
2094 * can with the number of preemptions, trying to exceed narrow HW
2095 * limits. At a minimum, we insist that we can sort all the user
2096 * high priority levels into execution order.
2097 */
2098
2099 for_each_engine(engine, gt, id) {
2100 struct i915_request *rq = NULL;
2101 struct igt_live_test t;
2102 IGT_TIMEOUT(end_time);
2103 int prio = 0;
2104 int err = 0;
2105 u32 *cs;
2106
2107 if (!intel_engine_has_preemption(engine))
2108 continue;
2109
2110 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name))
2111 return -EIO;
2112
2113 do {
2114 struct i915_sched_attr attr = {
2115 .priority = I915_USER_PRIORITY(prio++),
2116 };
2117
2118 err = create_gang(engine, &rq);
2119 if (err)
2120 break;
2121
2122 /* Submit each spinner at increasing priority */
2123 engine->schedule(rq, &attr);
2124
2125 if (prio <= I915_PRIORITY_MAX)
2126 continue;
2127
2128 if (prio > (INT_MAX >> I915_USER_PRIORITY_SHIFT))
2129 break;
2130
2131 if (__igt_timeout(end_time, NULL))
2132 break;
2133 } while (1);
2134 pr_debug("%s: Preempt chain of %d requests\n",
2135 engine->name, prio);
2136
2137 /*
2138 * Such that the last spinner is the highest priority and
2139 * should execute first. When that spinner completes,
2140 * it will terminate the next lowest spinner until there
2141 * are no more spinners and the gang is complete.
2142 */
2143 cs = i915_gem_object_pin_map(rq->batch->obj, I915_MAP_WC);
2144 if (!IS_ERR(cs)) {
2145 *cs = 0;
2146 i915_gem_object_unpin_map(rq->batch->obj);
2147 } else {
2148 err = PTR_ERR(cs);
2149 intel_gt_set_wedged(gt);
2150 }
2151
2152 while (rq) { /* wait for each rq from highest to lowest prio */
2153 struct i915_request *n =
2154 list_next_entry(rq, client_link);
2155
2156 if (err == 0 && i915_request_wait(rq, 0, HZ / 5) < 0) {
2157 struct drm_printer p =
2158 drm_info_printer(engine->i915->drm.dev);
2159
2160 pr_err("Failed to flush chain of %d requests, at %d\n",
2161 prio, rq_prio(rq) >> I915_USER_PRIORITY_SHIFT);
2162 intel_engine_dump(engine, &p,
2163 "%s\n", engine->name);
2164
2165 err = -ETIME;
2166 }
2167
2168 i915_request_put(rq);
2169 rq = n;
2170 }
2171
2172 if (igt_live_test_end(&t))
2173 err = -EIO;
2174 if (err)
2175 return err;
2176 }
2177
2178 return 0;
2179 }
2180
2181 static int live_preempt_hang(void *arg)
2182 {
2183 struct intel_gt *gt = arg;
2184 struct i915_gem_context *ctx_hi, *ctx_lo;
2185 struct igt_spinner spin_hi, spin_lo;
2186 struct intel_engine_cs *engine;
2187 enum intel_engine_id id;
2188 int err = -ENOMEM;
2189
2190 if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
2191 return 0;
2192
2193 if (!intel_has_reset_engine(gt))
2194 return 0;
2195
2196 if (igt_spinner_init(&spin_hi, gt))
2197 return -ENOMEM;
2198
2199 if (igt_spinner_init(&spin_lo, gt))
2200 goto err_spin_hi;
2201
2202 ctx_hi = kernel_context(gt->i915);
2203 if (!ctx_hi)
2204 goto err_spin_lo;
2205 ctx_hi->sched.priority =
2206 I915_USER_PRIORITY(I915_CONTEXT_MAX_USER_PRIORITY);
2207
2208 ctx_lo = kernel_context(gt->i915);
2209 if (!ctx_lo)
2210 goto err_ctx_hi;
2211 ctx_lo->sched.priority =
2212 I915_USER_PRIORITY(I915_CONTEXT_MIN_USER_PRIORITY);
2213
2214 for_each_engine(engine, gt, id) {
2215 struct i915_request *rq;
2216
2217 if (!intel_engine_has_preemption(engine))
2218 continue;
2219
2220 rq = spinner_create_request(&spin_lo, ctx_lo, engine,
2221 MI_ARB_CHECK);
2222 if (IS_ERR(rq)) {
2223 err = PTR_ERR(rq);
2224 goto err_ctx_lo;
2225 }
2226
2227 i915_request_add(rq);
2228 if (!igt_wait_for_spinner(&spin_lo, rq)) {
2229 GEM_TRACE("lo spinner failed to start\n");
2230 GEM_TRACE_DUMP();
2231 intel_gt_set_wedged(gt);
2232 err = -EIO;
2233 goto err_ctx_lo;
2234 }
2235
2236 rq = spinner_create_request(&spin_hi, ctx_hi, engine,
2237 MI_ARB_CHECK);
2238 if (IS_ERR(rq)) {
2239 igt_spinner_end(&spin_lo);
2240 err = PTR_ERR(rq);
2241 goto err_ctx_lo;
2242 }
2243
2244 init_completion(&engine->execlists.preempt_hang.completion);
2245 engine->execlists.preempt_hang.inject_hang = true;
2246
2247 i915_request_add(rq);
2248
2249 if (!wait_for_completion_timeout(&engine->execlists.preempt_hang.completion,
2250 HZ / 10)) {
2251 pr_err("Preemption did not occur within timeout!");
2252 GEM_TRACE_DUMP();
2253 intel_gt_set_wedged(gt);
2254 err = -EIO;
2255 goto err_ctx_lo;
2256 }
2257
2258 set_bit(I915_RESET_ENGINE + id, &gt->reset.flags);
2259 intel_engine_reset(engine, NULL);
2260 clear_bit(I915_RESET_ENGINE + id, &gt->reset.flags);
2261
2262 engine->execlists.preempt_hang.inject_hang = false;
2263
2264 if (!igt_wait_for_spinner(&spin_hi, rq)) {
2265 GEM_TRACE("hi spinner failed to start\n");
2266 GEM_TRACE_DUMP();
2267 intel_gt_set_wedged(gt);
2268 err = -EIO;
2269 goto err_ctx_lo;
2270 }
2271
2272 igt_spinner_end(&spin_hi);
2273 igt_spinner_end(&spin_lo);
2274 if (igt_flush_test(gt->i915)) {
2275 err = -EIO;
2276 goto err_ctx_lo;
2277 }
2278 }
2279
2280 err = 0;
2281 err_ctx_lo:
2282 kernel_context_close(ctx_lo);
2283 err_ctx_hi:
2284 kernel_context_close(ctx_hi);
2285 err_spin_lo:
2286 igt_spinner_fini(&spin_lo);
2287 err_spin_hi:
2288 igt_spinner_fini(&spin_hi);
2289 return err;
2290 }
2291
2292 static int live_preempt_timeout(void *arg)
2293 {
2294 struct intel_gt *gt = arg;
2295 struct i915_gem_context *ctx_hi, *ctx_lo;
2296 struct igt_spinner spin_lo;
2297 struct intel_engine_cs *engine;
2298 enum intel_engine_id id;
2299 int err = -ENOMEM;
2300
2301 /*
2302 * Check that we force preemption to occur by cancelling the previous
2303 * context if it refuses to yield the GPU.
2304 */
2305 if (!IS_ACTIVE(CONFIG_DRM_I915_PREEMPT_TIMEOUT))
2306 return 0;
2307
2308 if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
2309 return 0;
2310
2311 if (!intel_has_reset_engine(gt))
2312 return 0;
2313
2314 if (igt_spinner_init(&spin_lo, gt))
2315 return -ENOMEM;
2316
2317 ctx_hi = kernel_context(gt->i915);
2318 if (!ctx_hi)
2319 goto err_spin_lo;
2320 ctx_hi->sched.priority =
2321 I915_USER_PRIORITY(I915_CONTEXT_MAX_USER_PRIORITY);
2322
2323 ctx_lo = kernel_context(gt->i915);
2324 if (!ctx_lo)
2325 goto err_ctx_hi;
2326 ctx_lo->sched.priority =
2327 I915_USER_PRIORITY(I915_CONTEXT_MIN_USER_PRIORITY);
2328
2329 for_each_engine(engine, gt, id) {
2330 unsigned long saved_timeout;
2331 struct i915_request *rq;
2332
2333 if (!intel_engine_has_preemption(engine))
2334 continue;
2335
2336 rq = spinner_create_request(&spin_lo, ctx_lo, engine,
2337 MI_NOOP); /* preemption disabled */
2338 if (IS_ERR(rq)) {
2339 err = PTR_ERR(rq);
2340 goto err_ctx_lo;
2341 }
2342
2343 i915_request_add(rq);
2344 if (!igt_wait_for_spinner(&spin_lo, rq)) {
2345 intel_gt_set_wedged(gt);
2346 err = -EIO;
2347 goto err_ctx_lo;
2348 }
2349
2350 rq = igt_request_alloc(ctx_hi, engine);
2351 if (IS_ERR(rq)) {
2352 igt_spinner_end(&spin_lo);
2353 err = PTR_ERR(rq);
2354 goto err_ctx_lo;
2355 }
2356
2357 /* Flush the previous CS ack before changing timeouts */
2358 while (READ_ONCE(engine->execlists.pending[0]))
2359 cpu_relax();
2360
2361 saved_timeout = engine->props.preempt_timeout_ms;
2362 engine->props.preempt_timeout_ms = 1; /* in ms, -> 1 jiffie */
2363
2364 i915_request_get(rq);
2365 i915_request_add(rq);
2366
2367 intel_engine_flush_submission(engine);
2368 engine->props.preempt_timeout_ms = saved_timeout;
2369
2370 if (i915_request_wait(rq, 0, HZ / 10) < 0) {
2371 intel_gt_set_wedged(gt);
2372 i915_request_put(rq);
2373 err = -ETIME;
2374 goto err_ctx_lo;
2375 }
2376
2377 igt_spinner_end(&spin_lo);
2378 i915_request_put(rq);
2379 }
2380
2381 err = 0;
2382 err_ctx_lo:
2383 kernel_context_close(ctx_lo);
2384 err_ctx_hi:
2385 kernel_context_close(ctx_hi);
2386 err_spin_lo:
2387 igt_spinner_fini(&spin_lo);
2388 return err;
2389 }
2390
2391 static int random_range(struct rnd_state *rnd, int min, int max)
2392 {
2393 return i915_prandom_u32_max_state(max - min, rnd) + min;
2394 }
2395
2396 static int random_priority(struct rnd_state *rnd)
2397 {
2398 return random_range(rnd, I915_PRIORITY_MIN, I915_PRIORITY_MAX);
2399 }
2400
2401 struct preempt_smoke {
2402 struct intel_gt *gt;
2403 struct i915_gem_context **contexts;
2404 struct intel_engine_cs *engine;
2405 struct drm_i915_gem_object *batch;
2406 unsigned int ncontext;
2407 struct rnd_state prng;
2408 unsigned long count;
2409 };
2410
2411 static struct i915_gem_context *smoke_context(struct preempt_smoke *smoke)
2412 {
2413 return smoke->contexts[i915_prandom_u32_max_state(smoke->ncontext,
2414 &smoke->prng)];
2415 }
2416
2417 static int smoke_submit(struct preempt_smoke *smoke,
2418 struct i915_gem_context *ctx, int prio,
2419 struct drm_i915_gem_object *batch)
2420 {
2421 struct i915_request *rq;
2422 struct i915_vma *vma = NULL;
2423 int err = 0;
2424
2425 if (batch) {
2426 struct i915_address_space *vm;
2427
2428 vm = i915_gem_context_get_vm_rcu(ctx);
2429 vma = i915_vma_instance(batch, vm, NULL);
2430 i915_vm_put(vm);
2431 if (IS_ERR(vma))
2432 return PTR_ERR(vma);
2433
2434 err = i915_vma_pin(vma, 0, 0, PIN_USER);
2435 if (err)
2436 return err;
2437 }
2438
2439 ctx->sched.priority = prio;
2440
2441 rq = igt_request_alloc(ctx, smoke->engine);
2442 if (IS_ERR(rq)) {
2443 err = PTR_ERR(rq);
2444 goto unpin;
2445 }
2446
2447 if (vma) {
2448 i915_vma_lock(vma);
2449 err = i915_request_await_object(rq, vma->obj, false);
2450 if (!err)
2451 err = i915_vma_move_to_active(vma, rq, 0);
2452 if (!err)
2453 err = rq->engine->emit_bb_start(rq,
2454 vma->node.start,
2455 PAGE_SIZE, 0);
2456 i915_vma_unlock(vma);
2457 }
2458
2459 i915_request_add(rq);
2460
2461 unpin:
2462 if (vma)
2463 i915_vma_unpin(vma);
2464
2465 return err;
2466 }
2467
2468 static int smoke_crescendo_thread(void *arg)
2469 {
2470 struct preempt_smoke *smoke = arg;
2471 IGT_TIMEOUT(end_time);
2472 unsigned long count;
2473
2474 count = 0;
2475 do {
2476 struct i915_gem_context *ctx = smoke_context(smoke);
2477 int err;
2478
2479 err = smoke_submit(smoke,
2480 ctx, count % I915_PRIORITY_MAX,
2481 smoke->batch);
2482 if (err)
2483 return err;
2484
2485 count++;
2486 } while (!__igt_timeout(end_time, NULL));
2487
2488 smoke->count = count;
2489 return 0;
2490 }
2491
2492 static int smoke_crescendo(struct preempt_smoke *smoke, unsigned int flags)
2493 #define BATCH BIT(0)
2494 {
2495 struct task_struct *tsk[I915_NUM_ENGINES] = {};
2496 struct preempt_smoke arg[I915_NUM_ENGINES];
2497 struct intel_engine_cs *engine;
2498 enum intel_engine_id id;
2499 unsigned long count;
2500 int err = 0;
2501
2502 for_each_engine(engine, smoke->gt, id) {
2503 arg[id] = *smoke;
2504 arg[id].engine = engine;
2505 if (!(flags & BATCH))
2506 arg[id].batch = NULL;
2507 arg[id].count = 0;
2508
2509 tsk[id] = kthread_run(smoke_crescendo_thread, &arg,
2510 "igt/smoke:%d", id);
2511 if (IS_ERR(tsk[id])) {
2512 err = PTR_ERR(tsk[id]);
2513 break;
2514 }
2515 get_task_struct(tsk[id]);
2516 }
2517
2518 yield(); /* start all threads before we kthread_stop() */
2519
2520 count = 0;
2521 for_each_engine(engine, smoke->gt, id) {
2522 int status;
2523
2524 if (IS_ERR_OR_NULL(tsk[id]))
2525 continue;
2526
2527 status = kthread_stop(tsk[id]);
2528 if (status && !err)
2529 err = status;
2530
2531 count += arg[id].count;
2532
2533 put_task_struct(tsk[id]);
2534 }
2535
2536 pr_info("Submitted %lu crescendo:%x requests across %d engines and %d contexts\n",
2537 count, flags,
2538 RUNTIME_INFO(smoke->gt->i915)->num_engines, smoke->ncontext);
2539 return 0;
2540 }
2541
2542 static int smoke_random(struct preempt_smoke *smoke, unsigned int flags)
2543 {
2544 enum intel_engine_id id;
2545 IGT_TIMEOUT(end_time);
2546 unsigned long count;
2547
2548 count = 0;
2549 do {
2550 for_each_engine(smoke->engine, smoke->gt, id) {
2551 struct i915_gem_context *ctx = smoke_context(smoke);
2552 int err;
2553
2554 err = smoke_submit(smoke,
2555 ctx, random_priority(&smoke->prng),
2556 flags & BATCH ? smoke->batch : NULL);
2557 if (err)
2558 return err;
2559
2560 count++;
2561 }
2562 } while (!__igt_timeout(end_time, NULL));
2563
2564 pr_info("Submitted %lu random:%x requests across %d engines and %d contexts\n",
2565 count, flags,
2566 RUNTIME_INFO(smoke->gt->i915)->num_engines, smoke->ncontext);
2567 return 0;
2568 }
2569
2570 static int live_preempt_smoke(void *arg)
2571 {
2572 struct preempt_smoke smoke = {
2573 .gt = arg,
2574 .prng = I915_RND_STATE_INITIALIZER(i915_selftest.random_seed),
2575 .ncontext = 1024,
2576 };
2577 const unsigned int phase[] = { 0, BATCH };
2578 struct igt_live_test t;
2579 int err = -ENOMEM;
2580 u32 *cs;
2581 int n;
2582
2583 if (!HAS_LOGICAL_RING_PREEMPTION(smoke.gt->i915))
2584 return 0;
2585
2586 smoke.contexts = kmalloc_array(smoke.ncontext,
2587 sizeof(*smoke.contexts),
2588 GFP_KERNEL);
2589 if (!smoke.contexts)
2590 return -ENOMEM;
2591
2592 smoke.batch =
2593 i915_gem_object_create_internal(smoke.gt->i915, PAGE_SIZE);
2594 if (IS_ERR(smoke.batch)) {
2595 err = PTR_ERR(smoke.batch);
2596 goto err_free;
2597 }
2598
2599 cs = i915_gem_object_pin_map(smoke.batch, I915_MAP_WB);
2600 if (IS_ERR(cs)) {
2601 err = PTR_ERR(cs);
2602 goto err_batch;
2603 }
2604 for (n = 0; n < PAGE_SIZE / sizeof(*cs) - 1; n++)
2605 cs[n] = MI_ARB_CHECK;
2606 cs[n] = MI_BATCH_BUFFER_END;
2607 i915_gem_object_flush_map(smoke.batch);
2608 i915_gem_object_unpin_map(smoke.batch);
2609
2610 if (igt_live_test_begin(&t, smoke.gt->i915, __func__, "all")) {
2611 err = -EIO;
2612 goto err_batch;
2613 }
2614
2615 for (n = 0; n < smoke.ncontext; n++) {
2616 smoke.contexts[n] = kernel_context(smoke.gt->i915);
2617 if (!smoke.contexts[n])
2618 goto err_ctx;
2619 }
2620
2621 for (n = 0; n < ARRAY_SIZE(phase); n++) {
2622 err = smoke_crescendo(&smoke, phase[n]);
2623 if (err)
2624 goto err_ctx;
2625
2626 err = smoke_random(&smoke, phase[n]);
2627 if (err)
2628 goto err_ctx;
2629 }
2630
2631 err_ctx:
2632 if (igt_live_test_end(&t))
2633 err = -EIO;
2634
2635 for (n = 0; n < smoke.ncontext; n++) {
2636 if (!smoke.contexts[n])
2637 break;
2638 kernel_context_close(smoke.contexts[n]);
2639 }
2640
2641 err_batch:
2642 i915_gem_object_put(smoke.batch);
2643 err_free:
2644 kfree(smoke.contexts);
2645
2646 return err;
2647 }
2648
2649 static int nop_virtual_engine(struct intel_gt *gt,
2650 struct intel_engine_cs **siblings,
2651 unsigned int nsibling,
2652 unsigned int nctx,
2653 unsigned int flags)
2654 #define CHAIN BIT(0)
2655 {
2656 IGT_TIMEOUT(end_time);
2657 struct i915_request *request[16] = {};
2658 struct intel_context *ve[16];
2659 unsigned long n, prime, nc;
2660 struct igt_live_test t;
2661 ktime_t times[2] = {};
2662 int err;
2663
2664 GEM_BUG_ON(!nctx || nctx > ARRAY_SIZE(ve));
2665
2666 for (n = 0; n < nctx; n++) {
2667 ve[n] = intel_execlists_create_virtual(siblings, nsibling);
2668 if (IS_ERR(ve[n])) {
2669 err = PTR_ERR(ve[n]);
2670 nctx = n;
2671 goto out;
2672 }
2673
2674 err = intel_context_pin(ve[n]);
2675 if (err) {
2676 intel_context_put(ve[n]);
2677 nctx = n;
2678 goto out;
2679 }
2680 }
2681
2682 err = igt_live_test_begin(&t, gt->i915, __func__, ve[0]->engine->name);
2683 if (err)
2684 goto out;
2685
2686 for_each_prime_number_from(prime, 1, 8192) {
2687 times[1] = ktime_get_raw();
2688
2689 if (flags & CHAIN) {
2690 for (nc = 0; nc < nctx; nc++) {
2691 for (n = 0; n < prime; n++) {
2692 struct i915_request *rq;
2693
2694 rq = i915_request_create(ve[nc]);
2695 if (IS_ERR(rq)) {
2696 err = PTR_ERR(rq);
2697 goto out;
2698 }
2699
2700 if (request[nc])
2701 i915_request_put(request[nc]);
2702 request[nc] = i915_request_get(rq);
2703 i915_request_add(rq);
2704 }
2705 }
2706 } else {
2707 for (n = 0; n < prime; n++) {
2708 for (nc = 0; nc < nctx; nc++) {
2709 struct i915_request *rq;
2710
2711 rq = i915_request_create(ve[nc]);
2712 if (IS_ERR(rq)) {
2713 err = PTR_ERR(rq);
2714 goto out;
2715 }
2716
2717 if (request[nc])
2718 i915_request_put(request[nc]);
2719 request[nc] = i915_request_get(rq);
2720 i915_request_add(rq);
2721 }
2722 }
2723 }
2724
2725 for (nc = 0; nc < nctx; nc++) {
2726 if (i915_request_wait(request[nc], 0, HZ / 10) < 0) {
2727 pr_err("%s(%s): wait for %llx:%lld timed out\n",
2728 __func__, ve[0]->engine->name,
2729 request[nc]->fence.context,
2730 request[nc]->fence.seqno);
2731
2732 GEM_TRACE("%s(%s) failed at request %llx:%lld\n",
2733 __func__, ve[0]->engine->name,
2734 request[nc]->fence.context,
2735 request[nc]->fence.seqno);
2736 GEM_TRACE_DUMP();
2737 intel_gt_set_wedged(gt);
2738 break;
2739 }
2740 }
2741
2742 times[1] = ktime_sub(ktime_get_raw(), times[1]);
2743 if (prime == 1)
2744 times[0] = times[1];
2745
2746 for (nc = 0; nc < nctx; nc++) {
2747 i915_request_put(request[nc]);
2748 request[nc] = NULL;
2749 }
2750
2751 if (__igt_timeout(end_time, NULL))
2752 break;
2753 }
2754
2755 err = igt_live_test_end(&t);
2756 if (err)
2757 goto out;
2758
2759 pr_info("Requestx%d latencies on %s: 1 = %lluns, %lu = %lluns\n",
2760 nctx, ve[0]->engine->name, ktime_to_ns(times[0]),
2761 prime, div64_u64(ktime_to_ns(times[1]), prime));
2762
2763 out:
2764 if (igt_flush_test(gt->i915))
2765 err = -EIO;
2766
2767 for (nc = 0; nc < nctx; nc++) {
2768 i915_request_put(request[nc]);
2769 intel_context_unpin(ve[nc]);
2770 intel_context_put(ve[nc]);
2771 }
2772 return err;
2773 }
2774
2775 static int live_virtual_engine(void *arg)
2776 {
2777 struct intel_gt *gt = arg;
2778 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
2779 struct intel_engine_cs *engine;
2780 enum intel_engine_id id;
2781 unsigned int class, inst;
2782 int err;
2783
2784 if (USES_GUC_SUBMISSION(gt->i915))
2785 return 0;
2786
2787 for_each_engine(engine, gt, id) {
2788 err = nop_virtual_engine(gt, &engine, 1, 1, 0);
2789 if (err) {
2790 pr_err("Failed to wrap engine %s: err=%d\n",
2791 engine->name, err);
2792 return err;
2793 }
2794 }
2795
2796 for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
2797 int nsibling, n;
2798
2799 nsibling = 0;
2800 for (inst = 0; inst <= MAX_ENGINE_INSTANCE; inst++) {
2801 if (!gt->engine_class[class][inst])
2802 continue;
2803
2804 siblings[nsibling++] = gt->engine_class[class][inst];
2805 }
2806 if (nsibling < 2)
2807 continue;
2808
2809 for (n = 1; n <= nsibling + 1; n++) {
2810 err = nop_virtual_engine(gt, siblings, nsibling,
2811 n, 0);
2812 if (err)
2813 return err;
2814 }
2815
2816 err = nop_virtual_engine(gt, siblings, nsibling, n, CHAIN);
2817 if (err)
2818 return err;
2819 }
2820
2821 return 0;
2822 }
2823
2824 static int mask_virtual_engine(struct intel_gt *gt,
2825 struct intel_engine_cs **siblings,
2826 unsigned int nsibling)
2827 {
2828 struct i915_request *request[MAX_ENGINE_INSTANCE + 1];
2829 struct intel_context *ve;
2830 struct igt_live_test t;
2831 unsigned int n;
2832 int err;
2833
2834 /*
2835 * Check that by setting the execution mask on a request, we can
2836 * restrict it to our desired engine within the virtual engine.
2837 */
2838
2839 ve = intel_execlists_create_virtual(siblings, nsibling);
2840 if (IS_ERR(ve)) {
2841 err = PTR_ERR(ve);
2842 goto out_close;
2843 }
2844
2845 err = intel_context_pin(ve);
2846 if (err)
2847 goto out_put;
2848
2849 err = igt_live_test_begin(&t, gt->i915, __func__, ve->engine->name);
2850 if (err)
2851 goto out_unpin;
2852
2853 for (n = 0; n < nsibling; n++) {
2854 request[n] = i915_request_create(ve);
2855 if (IS_ERR(request[n])) {
2856 err = PTR_ERR(request[n]);
2857 nsibling = n;
2858 goto out;
2859 }
2860
2861 /* Reverse order as it's more likely to be unnatural */
2862 request[n]->execution_mask = siblings[nsibling - n - 1]->mask;
2863
2864 i915_request_get(request[n]);
2865 i915_request_add(request[n]);
2866 }
2867
2868 for (n = 0; n < nsibling; n++) {
2869 if (i915_request_wait(request[n], 0, HZ / 10) < 0) {
2870 pr_err("%s(%s): wait for %llx:%lld timed out\n",
2871 __func__, ve->engine->name,
2872 request[n]->fence.context,
2873 request[n]->fence.seqno);
2874
2875 GEM_TRACE("%s(%s) failed at request %llx:%lld\n",
2876 __func__, ve->engine->name,
2877 request[n]->fence.context,
2878 request[n]->fence.seqno);
2879 GEM_TRACE_DUMP();
2880 intel_gt_set_wedged(gt);
2881 err = -EIO;
2882 goto out;
2883 }
2884
2885 if (request[n]->engine != siblings[nsibling - n - 1]) {
2886 pr_err("Executed on wrong sibling '%s', expected '%s'\n",
2887 request[n]->engine->name,
2888 siblings[nsibling - n - 1]->name);
2889 err = -EINVAL;
2890 goto out;
2891 }
2892 }
2893
2894 err = igt_live_test_end(&t);
2895 out:
2896 if (igt_flush_test(gt->i915))
2897 err = -EIO;
2898
2899 for (n = 0; n < nsibling; n++)
2900 i915_request_put(request[n]);
2901
2902 out_unpin:
2903 intel_context_unpin(ve);
2904 out_put:
2905 intel_context_put(ve);
2906 out_close:
2907 return err;
2908 }
2909
2910 static int live_virtual_mask(void *arg)
2911 {
2912 struct intel_gt *gt = arg;
2913 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
2914 unsigned int class, inst;
2915 int err;
2916
2917 if (USES_GUC_SUBMISSION(gt->i915))
2918 return 0;
2919
2920 for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
2921 unsigned int nsibling;
2922
2923 nsibling = 0;
2924 for (inst = 0; inst <= MAX_ENGINE_INSTANCE; inst++) {
2925 if (!gt->engine_class[class][inst])
2926 break;
2927
2928 siblings[nsibling++] = gt->engine_class[class][inst];
2929 }
2930 if (nsibling < 2)
2931 continue;
2932
2933 err = mask_virtual_engine(gt, siblings, nsibling);
2934 if (err)
2935 return err;
2936 }
2937
2938 return 0;
2939 }
2940
2941 static int preserved_virtual_engine(struct intel_gt *gt,
2942 struct intel_engine_cs **siblings,
2943 unsigned int nsibling)
2944 {
2945 struct i915_request *last = NULL;
2946 struct intel_context *ve;
2947 struct i915_vma *scratch;
2948 struct igt_live_test t;
2949 unsigned int n;
2950 int err = 0;
2951 u32 *cs;
2952
2953 scratch = create_scratch(siblings[0]->gt);
2954 if (IS_ERR(scratch))
2955 return PTR_ERR(scratch);
2956
2957 ve = intel_execlists_create_virtual(siblings, nsibling);
2958 if (IS_ERR(ve)) {
2959 err = PTR_ERR(ve);
2960 goto out_scratch;
2961 }
2962
2963 err = intel_context_pin(ve);
2964 if (err)
2965 goto out_put;
2966
2967 err = igt_live_test_begin(&t, gt->i915, __func__, ve->engine->name);
2968 if (err)
2969 goto out_unpin;
2970
2971 for (n = 0; n < NUM_GPR_DW; n++) {
2972 struct intel_engine_cs *engine = siblings[n % nsibling];
2973 struct i915_request *rq;
2974
2975 rq = i915_request_create(ve);
2976 if (IS_ERR(rq)) {
2977 err = PTR_ERR(rq);
2978 goto out_end;
2979 }
2980
2981 i915_request_put(last);
2982 last = i915_request_get(rq);
2983
2984 cs = intel_ring_begin(rq, 8);
2985 if (IS_ERR(cs)) {
2986 i915_request_add(rq);
2987 err = PTR_ERR(cs);
2988 goto out_end;
2989 }
2990
2991 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
2992 *cs++ = CS_GPR(engine, n);
2993 *cs++ = i915_ggtt_offset(scratch) + n * sizeof(u32);
2994 *cs++ = 0;
2995
2996 *cs++ = MI_LOAD_REGISTER_IMM(1);
2997 *cs++ = CS_GPR(engine, (n + 1) % NUM_GPR_DW);
2998 *cs++ = n + 1;
2999
3000 *cs++ = MI_NOOP;
3001 intel_ring_advance(rq, cs);
3002
3003 /* Restrict this request to run on a particular engine */
3004 rq->execution_mask = engine->mask;
3005 i915_request_add(rq);
3006 }
3007
3008 if (i915_request_wait(last, 0, HZ / 5) < 0) {
3009 err = -ETIME;
3010 goto out_end;
3011 }
3012
3013 cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
3014 if (IS_ERR(cs)) {
3015 err = PTR_ERR(cs);
3016 goto out_end;
3017 }
3018
3019 for (n = 0; n < NUM_GPR_DW; n++) {
3020 if (cs[n] != n) {
3021 pr_err("Incorrect value[%d] found for GPR[%d]\n",
3022 cs[n], n);
3023 err = -EINVAL;
3024 break;
3025 }
3026 }
3027
3028 i915_gem_object_unpin_map(scratch->obj);
3029
3030 out_end:
3031 if (igt_live_test_end(&t))
3032 err = -EIO;
3033 i915_request_put(last);
3034 out_unpin:
3035 intel_context_unpin(ve);
3036 out_put:
3037 intel_context_put(ve);
3038 out_scratch:
3039 i915_vma_unpin_and_release(&scratch, 0);
3040 return err;
3041 }
3042
3043 static int live_virtual_preserved(void *arg)
3044 {
3045 struct intel_gt *gt = arg;
3046 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
3047 unsigned int class, inst;
3048
3049 /*
3050 * Check that the context image retains non-privileged (user) registers
3051 * from one engine to the next. For this we check that the CS_GPR
3052 * are preserved.
3053 */
3054
3055 if (USES_GUC_SUBMISSION(gt->i915))
3056 return 0;
3057
3058 /* As we use CS_GPR we cannot run before they existed on all engines. */
3059 if (INTEL_GEN(gt->i915) < 9)
3060 return 0;
3061
3062 for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
3063 int nsibling, err;
3064
3065 nsibling = 0;
3066 for (inst = 0; inst <= MAX_ENGINE_INSTANCE; inst++) {
3067 if (!gt->engine_class[class][inst])
3068 continue;
3069
3070 siblings[nsibling++] = gt->engine_class[class][inst];
3071 }
3072 if (nsibling < 2)
3073 continue;
3074
3075 err = preserved_virtual_engine(gt, siblings, nsibling);
3076 if (err)
3077 return err;
3078 }
3079
3080 return 0;
3081 }
3082
3083 static int bond_virtual_engine(struct intel_gt *gt,
3084 unsigned int class,
3085 struct intel_engine_cs **siblings,
3086 unsigned int nsibling,
3087 unsigned int flags)
3088 #define BOND_SCHEDULE BIT(0)
3089 {
3090 struct intel_engine_cs *master;
3091 struct i915_request *rq[16];
3092 enum intel_engine_id id;
3093 struct igt_spinner spin;
3094 unsigned long n;
3095 int err;
3096
3097 /*
3098 * A set of bonded requests is intended to be run concurrently
3099 * across a number of engines. We use one request per-engine
3100 * and a magic fence to schedule each of the bonded requests
3101 * at the same time. A consequence of our current scheduler is that
3102 * we only move requests to the HW ready queue when the request
3103 * becomes ready, that is when all of its prerequisite fences have
3104 * been signaled. As one of those fences is the master submit fence,
3105 * there is a delay on all secondary fences as the HW may be
3106 * currently busy. Equally, as all the requests are independent,
3107 * they may have other fences that delay individual request
3108 * submission to HW. Ergo, we do not guarantee that all requests are
3109 * immediately submitted to HW at the same time, just that if the
3110 * rules are abided by, they are ready at the same time as the
3111 * first is submitted. Userspace can embed semaphores in its batch
3112 * to ensure parallel execution of its phases as it requires.
3113 * Though naturally it gets requested that perhaps the scheduler should
3114 * take care of parallel execution, even across preemption events on
3115 * different HW. (The proper answer is of course "lalalala".)
3116 *
3117 * With the submit-fence, we have identified three possible phases
3118 * of synchronisation depending on the master fence: queued (not
3119 * ready), executing, and signaled. The first two are quite simple
3120 * and checked below. However, the signaled master fence handling is
3121 * contentious. Currently we do not distinguish between a signaled
3122 * fence and an expired fence, as once signaled it does not convey
3123 * any information about the previous execution. It may even be freed
3124 * and hence checking later it may not exist at all. Ergo we currently
3125 * do not apply the bonding constraint for an already signaled fence,
3126 * as our expectation is that it should not constrain the secondaries
3127 * and is outside of the scope of the bonded request API (i.e. all
3128 * userspace requests are meant to be running in parallel). As
3129 * it imposes no constraint, and is effectively a no-op, we do not
3130 * check below as normal execution flows are checked extensively above.
3131 *
3132 * XXX Is the degenerate handling of signaled submit fences the
3133 * expected behaviour for userpace?
3134 */
3135
3136 GEM_BUG_ON(nsibling >= ARRAY_SIZE(rq) - 1);
3137
3138 if (igt_spinner_init(&spin, gt))
3139 return -ENOMEM;
3140
3141 err = 0;
3142 rq[0] = ERR_PTR(-ENOMEM);
3143 for_each_engine(master, gt, id) {
3144 struct i915_sw_fence fence = {};
3145
3146 if (master->class == class)
3147 continue;
3148
3149 memset_p((void *)rq, ERR_PTR(-EINVAL), ARRAY_SIZE(rq));
3150
3151 rq[0] = igt_spinner_create_request(&spin,
3152 master->kernel_context,
3153 MI_NOOP);
3154 if (IS_ERR(rq[0])) {
3155 err = PTR_ERR(rq[0]);
3156 goto out;
3157 }
3158 i915_request_get(rq[0]);
3159
3160 if (flags & BOND_SCHEDULE) {
3161 onstack_fence_init(&fence);
3162 err = i915_sw_fence_await_sw_fence_gfp(&rq[0]->submit,
3163 &fence,
3164 GFP_KERNEL);
3165 }
3166
3167 i915_request_add(rq[0]);
3168 if (err < 0)
3169 goto out;
3170
3171 if (!(flags & BOND_SCHEDULE) &&
3172 !igt_wait_for_spinner(&spin, rq[0])) {
3173 err = -EIO;
3174 goto out;
3175 }
3176
3177 for (n = 0; n < nsibling; n++) {
3178 struct intel_context *ve;
3179
3180 ve = intel_execlists_create_virtual(siblings, nsibling);
3181 if (IS_ERR(ve)) {
3182 err = PTR_ERR(ve);
3183 onstack_fence_fini(&fence);
3184 goto out;
3185 }
3186
3187 err = intel_virtual_engine_attach_bond(ve->engine,
3188 master,
3189 siblings[n]);
3190 if (err) {
3191 intel_context_put(ve);
3192 onstack_fence_fini(&fence);
3193 goto out;
3194 }
3195
3196 err = intel_context_pin(ve);
3197 intel_context_put(ve);
3198 if (err) {
3199 onstack_fence_fini(&fence);
3200 goto out;
3201 }
3202
3203 rq[n + 1] = i915_request_create(ve);
3204 intel_context_unpin(ve);
3205 if (IS_ERR(rq[n + 1])) {
3206 err = PTR_ERR(rq[n + 1]);
3207 onstack_fence_fini(&fence);
3208 goto out;
3209 }
3210 i915_request_get(rq[n + 1]);
3211
3212 err = i915_request_await_execution(rq[n + 1],
3213 &rq[0]->fence,
3214 ve->engine->bond_execute);
3215 i915_request_add(rq[n + 1]);
3216 if (err < 0) {
3217 onstack_fence_fini(&fence);
3218 goto out;
3219 }
3220 }
3221 onstack_fence_fini(&fence);
3222 intel_engine_flush_submission(master);
3223 igt_spinner_end(&spin);
3224
3225 if (i915_request_wait(rq[0], 0, HZ / 10) < 0) {
3226 pr_err("Master request did not execute (on %s)!\n",
3227 rq[0]->engine->name);
3228 err = -EIO;
3229 goto out;
3230 }
3231
3232 for (n = 0; n < nsibling; n++) {
3233 if (i915_request_wait(rq[n + 1], 0,
3234 MAX_SCHEDULE_TIMEOUT) < 0) {
3235 err = -EIO;
3236 goto out;
3237 }
3238
3239 if (rq[n + 1]->engine != siblings[n]) {
3240 pr_err("Bonded request did not execute on target engine: expected %s, used %s; master was %s\n",
3241 siblings[n]->name,
3242 rq[n + 1]->engine->name,
3243 rq[0]->engine->name);
3244 err = -EINVAL;
3245 goto out;
3246 }
3247 }
3248
3249 for (n = 0; !IS_ERR(rq[n]); n++)
3250 i915_request_put(rq[n]);
3251 rq[0] = ERR_PTR(-ENOMEM);
3252 }
3253
3254 out:
3255 for (n = 0; !IS_ERR(rq[n]); n++)
3256 i915_request_put(rq[n]);
3257 if (igt_flush_test(gt->i915))
3258 err = -EIO;
3259
3260 igt_spinner_fini(&spin);
3261 return err;
3262 }
3263
3264 static int live_virtual_bond(void *arg)
3265 {
3266 static const struct phase {
3267 const char *name;
3268 unsigned int flags;
3269 } phases[] = {
3270 { "", 0 },
3271 { "schedule", BOND_SCHEDULE },
3272 { },
3273 };
3274 struct intel_gt *gt = arg;
3275 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
3276 unsigned int class, inst;
3277 int err;
3278
3279 if (USES_GUC_SUBMISSION(gt->i915))
3280 return 0;
3281
3282 for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
3283 const struct phase *p;
3284 int nsibling;
3285
3286 nsibling = 0;
3287 for (inst = 0; inst <= MAX_ENGINE_INSTANCE; inst++) {
3288 if (!gt->engine_class[class][inst])
3289 break;
3290
3291 GEM_BUG_ON(nsibling == ARRAY_SIZE(siblings));
3292 siblings[nsibling++] = gt->engine_class[class][inst];
3293 }
3294 if (nsibling < 2)
3295 continue;
3296
3297 for (p = phases; p->name; p++) {
3298 err = bond_virtual_engine(gt,
3299 class, siblings, nsibling,
3300 p->flags);
3301 if (err) {
3302 pr_err("%s(%s): failed class=%d, nsibling=%d, err=%d\n",
3303 __func__, p->name, class, nsibling, err);
3304 return err;
3305 }
3306 }
3307 }
3308
3309 return 0;
3310 }
3311
3312 int intel_execlists_live_selftests(struct drm_i915_private *i915)
3313 {
3314 static const struct i915_subtest tests[] = {
3315 SUBTEST(live_sanitycheck),
3316 SUBTEST(live_unlite_switch),
3317 SUBTEST(live_unlite_preempt),
3318 SUBTEST(live_timeslice_preempt),
3319 SUBTEST(live_timeslice_queue),
3320 SUBTEST(live_busywait_preempt),
3321 SUBTEST(live_preempt),
3322 SUBTEST(live_late_preempt),
3323 SUBTEST(live_nopreempt),
3324 SUBTEST(live_preempt_cancel),
3325 SUBTEST(live_suppress_self_preempt),
3326 SUBTEST(live_suppress_wait_preempt),
3327 SUBTEST(live_chain_preempt),
3328 SUBTEST(live_preempt_gang),
3329 SUBTEST(live_preempt_hang),
3330 SUBTEST(live_preempt_timeout),
3331 SUBTEST(live_preempt_smoke),
3332 SUBTEST(live_virtual_engine),
3333 SUBTEST(live_virtual_mask),
3334 SUBTEST(live_virtual_preserved),
3335 SUBTEST(live_virtual_bond),
3336 };
3337
3338 if (!HAS_EXECLISTS(i915))
3339 return 0;
3340
3341 if (intel_gt_is_wedged(&i915->gt))
3342 return 0;
3343
3344 return intel_gt_live_subtests(tests, &i915->gt);
3345 }
3346
3347 static void hexdump(const void *buf, size_t len)
3348 {
3349 const size_t rowsize = 8 * sizeof(u32);
3350 const void *prev = NULL;
3351 bool skip = false;
3352 size_t pos;
3353
3354 for (pos = 0; pos < len; pos += rowsize) {
3355 char line[128];
3356
3357 if (prev && !memcmp(prev, buf + pos, rowsize)) {
3358 if (!skip) {
3359 pr_info("*\n");
3360 skip = true;
3361 }
3362 continue;
3363 }
3364
3365 WARN_ON_ONCE(hex_dump_to_buffer(buf + pos, len - pos,
3366 rowsize, sizeof(u32),
3367 line, sizeof(line),
3368 false) >= sizeof(line));
3369 pr_info("[%04zx] %s\n", pos, line);
3370
3371 prev = buf + pos;
3372 skip = false;
3373 }
3374 }
3375
3376 static int live_lrc_layout(void *arg)
3377 {
3378 struct intel_gt *gt = arg;
3379 struct intel_engine_cs *engine;
3380 enum intel_engine_id id;
3381 u32 *lrc;
3382 int err;
3383
3384 /*
3385 * Check the registers offsets we use to create the initial reg state
3386 * match the layout saved by HW.
3387 */
3388
3389 lrc = kmalloc(PAGE_SIZE, GFP_KERNEL);
3390 if (!lrc)
3391 return -ENOMEM;
3392
3393 err = 0;
3394 for_each_engine(engine, gt, id) {
3395 u32 *hw;
3396 int dw;
3397
3398 if (!engine->default_state)
3399 continue;
3400
3401 hw = i915_gem_object_pin_map(engine->default_state,
3402 I915_MAP_WB);
3403 if (IS_ERR(hw)) {
3404 err = PTR_ERR(hw);
3405 break;
3406 }
3407 hw += LRC_STATE_PN * PAGE_SIZE / sizeof(*hw);
3408
3409 execlists_init_reg_state(memset(lrc, POISON_INUSE, PAGE_SIZE),
3410 engine->kernel_context,
3411 engine,
3412 engine->kernel_context->ring,
3413 true);
3414
3415 dw = 0;
3416 do {
3417 u32 lri = hw[dw];
3418
3419 if (lri == 0) {
3420 dw++;
3421 continue;
3422 }
3423
3424 if (lrc[dw] == 0) {
3425 pr_debug("%s: skipped instruction %x at dword %d\n",
3426 engine->name, lri, dw);
3427 dw++;
3428 continue;
3429 }
3430
3431 if ((lri & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
3432 pr_err("%s: Expected LRI command at dword %d, found %08x\n",
3433 engine->name, dw, lri);
3434 err = -EINVAL;
3435 break;
3436 }
3437
3438 if (lrc[dw] != lri) {
3439 pr_err("%s: LRI command mismatch at dword %d, expected %08x found %08x\n",
3440 engine->name, dw, lri, lrc[dw]);
3441 err = -EINVAL;
3442 break;
3443 }
3444
3445 lri &= 0x7f;
3446 lri++;
3447 dw++;
3448
3449 while (lri) {
3450 if (hw[dw] != lrc[dw]) {
3451 pr_err("%s: Different registers found at dword %d, expected %x, found %x\n",
3452 engine->name, dw, hw[dw], lrc[dw]);
3453 err = -EINVAL;
3454 break;
3455 }
3456
3457 /*
3458 * Skip over the actual register value as we
3459 * expect that to differ.
3460 */
3461 dw += 2;
3462 lri -= 2;
3463 }
3464 } while ((lrc[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
3465
3466 if (err) {
3467 pr_info("%s: HW register image:\n", engine->name);
3468 hexdump(hw, PAGE_SIZE);
3469
3470 pr_info("%s: SW register image:\n", engine->name);
3471 hexdump(lrc, PAGE_SIZE);
3472 }
3473
3474 i915_gem_object_unpin_map(engine->default_state);
3475 if (err)
3476 break;
3477 }
3478
3479 kfree(lrc);
3480 return err;
3481 }
3482
3483 static int find_offset(const u32 *lri, u32 offset)
3484 {
3485 int i;
3486
3487 for (i = 0; i < PAGE_SIZE / sizeof(u32); i++)
3488 if (lri[i] == offset)
3489 return i;
3490
3491 return -1;
3492 }
3493
3494 static int live_lrc_fixed(void *arg)
3495 {
3496 struct intel_gt *gt = arg;
3497 struct intel_engine_cs *engine;
3498 enum intel_engine_id id;
3499 int err = 0;
3500
3501 /*
3502 * Check the assumed register offsets match the actual locations in
3503 * the context image.
3504 */
3505
3506 for_each_engine(engine, gt, id) {
3507 const struct {
3508 u32 reg;
3509 u32 offset;
3510 const char *name;
3511 } tbl[] = {
3512 {
3513 i915_mmio_reg_offset(RING_START(engine->mmio_base)),
3514 CTX_RING_START - 1,
3515 "RING_START"
3516 },
3517 {
3518 i915_mmio_reg_offset(RING_CTL(engine->mmio_base)),
3519 CTX_RING_CTL - 1,
3520 "RING_CTL"
3521 },
3522 {
3523 i915_mmio_reg_offset(RING_HEAD(engine->mmio_base)),
3524 CTX_RING_HEAD - 1,
3525 "RING_HEAD"
3526 },
3527 {
3528 i915_mmio_reg_offset(RING_TAIL(engine->mmio_base)),
3529 CTX_RING_TAIL - 1,
3530 "RING_TAIL"
3531 },
3532 {
3533 i915_mmio_reg_offset(RING_MI_MODE(engine->mmio_base)),
3534 lrc_ring_mi_mode(engine),
3535 "RING_MI_MODE"
3536 },
3537 {
3538 i915_mmio_reg_offset(RING_BBSTATE(engine->mmio_base)),
3539 CTX_BB_STATE - 1,
3540 "BB_STATE"
3541 },
3542 { },
3543 }, *t;
3544 u32 *hw;
3545
3546 if (!engine->default_state)
3547 continue;
3548
3549 hw = i915_gem_object_pin_map(engine->default_state,
3550 I915_MAP_WB);
3551 if (IS_ERR(hw)) {
3552 err = PTR_ERR(hw);
3553 break;
3554 }
3555 hw += LRC_STATE_PN * PAGE_SIZE / sizeof(*hw);
3556
3557 for (t = tbl; t->name; t++) {
3558 int dw = find_offset(hw, t->reg);
3559
3560 if (dw != t->offset) {
3561 pr_err("%s: Offset for %s [0x%x] mismatch, found %x, expected %x\n",
3562 engine->name,
3563 t->name,
3564 t->reg,
3565 dw,
3566 t->offset);
3567 err = -EINVAL;
3568 }
3569 }
3570
3571 i915_gem_object_unpin_map(engine->default_state);
3572 }
3573
3574 return err;
3575 }
3576
3577 static int __live_lrc_state(struct intel_engine_cs *engine,
3578 struct i915_vma *scratch)
3579 {
3580 struct intel_context *ce;
3581 struct i915_request *rq;
3582 enum {
3583 RING_START_IDX = 0,
3584 RING_TAIL_IDX,
3585 MAX_IDX
3586 };
3587 u32 expected[MAX_IDX];
3588 u32 *cs;
3589 int err;
3590 int n;
3591
3592 ce = intel_context_create(engine);
3593 if (IS_ERR(ce))
3594 return PTR_ERR(ce);
3595
3596 err = intel_context_pin(ce);
3597 if (err)
3598 goto err_put;
3599
3600 rq = i915_request_create(ce);
3601 if (IS_ERR(rq)) {
3602 err = PTR_ERR(rq);
3603 goto err_unpin;
3604 }
3605
3606 cs = intel_ring_begin(rq, 4 * MAX_IDX);
3607 if (IS_ERR(cs)) {
3608 err = PTR_ERR(cs);
3609 i915_request_add(rq);
3610 goto err_unpin;
3611 }
3612
3613 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
3614 *cs++ = i915_mmio_reg_offset(RING_START(engine->mmio_base));
3615 *cs++ = i915_ggtt_offset(scratch) + RING_START_IDX * sizeof(u32);
3616 *cs++ = 0;
3617
3618 expected[RING_START_IDX] = i915_ggtt_offset(ce->ring->vma);
3619
3620 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
3621 *cs++ = i915_mmio_reg_offset(RING_TAIL(engine->mmio_base));
3622 *cs++ = i915_ggtt_offset(scratch) + RING_TAIL_IDX * sizeof(u32);
3623 *cs++ = 0;
3624
3625 i915_request_get(rq);
3626 i915_request_add(rq);
3627
3628 intel_engine_flush_submission(engine);
3629 expected[RING_TAIL_IDX] = ce->ring->tail;
3630
3631 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
3632 err = -ETIME;
3633 goto err_rq;
3634 }
3635
3636 cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
3637 if (IS_ERR(cs)) {
3638 err = PTR_ERR(cs);
3639 goto err_rq;
3640 }
3641
3642 for (n = 0; n < MAX_IDX; n++) {
3643 if (cs[n] != expected[n]) {
3644 pr_err("%s: Stored register[%d] value[0x%x] did not match expected[0x%x]\n",
3645 engine->name, n, cs[n], expected[n]);
3646 err = -EINVAL;
3647 break;
3648 }
3649 }
3650
3651 i915_gem_object_unpin_map(scratch->obj);
3652
3653 err_rq:
3654 i915_request_put(rq);
3655 err_unpin:
3656 intel_context_unpin(ce);
3657 err_put:
3658 intel_context_put(ce);
3659 return err;
3660 }
3661
3662 static int live_lrc_state(void *arg)
3663 {
3664 struct intel_gt *gt = arg;
3665 struct intel_engine_cs *engine;
3666 struct i915_vma *scratch;
3667 enum intel_engine_id id;
3668 int err = 0;
3669
3670 /*
3671 * Check the live register state matches what we expect for this
3672 * intel_context.
3673 */
3674
3675 scratch = create_scratch(gt);
3676 if (IS_ERR(scratch))
3677 return PTR_ERR(scratch);
3678
3679 for_each_engine(engine, gt, id) {
3680 err = __live_lrc_state(engine, scratch);
3681 if (err)
3682 break;
3683 }
3684
3685 if (igt_flush_test(gt->i915))
3686 err = -EIO;
3687
3688 i915_vma_unpin_and_release(&scratch, 0);
3689 return err;
3690 }
3691
3692 static int gpr_make_dirty(struct intel_engine_cs *engine)
3693 {
3694 struct i915_request *rq;
3695 u32 *cs;
3696 int n;
3697
3698 rq = intel_engine_create_kernel_request(engine);
3699 if (IS_ERR(rq))
3700 return PTR_ERR(rq);
3701
3702 cs = intel_ring_begin(rq, 2 * NUM_GPR_DW + 2);
3703 if (IS_ERR(cs)) {
3704 i915_request_add(rq);
3705 return PTR_ERR(cs);
3706 }
3707
3708 *cs++ = MI_LOAD_REGISTER_IMM(NUM_GPR_DW);
3709 for (n = 0; n < NUM_GPR_DW; n++) {
3710 *cs++ = CS_GPR(engine, n);
3711 *cs++ = STACK_MAGIC;
3712 }
3713 *cs++ = MI_NOOP;
3714
3715 intel_ring_advance(rq, cs);
3716 i915_request_add(rq);
3717
3718 return 0;
3719 }
3720
3721 static int __live_gpr_clear(struct intel_engine_cs *engine,
3722 struct i915_vma *scratch)
3723 {
3724 struct intel_context *ce;
3725 struct i915_request *rq;
3726 u32 *cs;
3727 int err;
3728 int n;
3729
3730 if (INTEL_GEN(engine->i915) < 9 && engine->class != RENDER_CLASS)
3731 return 0; /* GPR only on rcs0 for gen8 */
3732
3733 err = gpr_make_dirty(engine);
3734 if (err)
3735 return err;
3736
3737 ce = intel_context_create(engine);
3738 if (IS_ERR(ce))
3739 return PTR_ERR(ce);
3740
3741 rq = intel_context_create_request(ce);
3742 if (IS_ERR(rq)) {
3743 err = PTR_ERR(rq);
3744 goto err_put;
3745 }
3746
3747 cs = intel_ring_begin(rq, 4 * NUM_GPR_DW);
3748 if (IS_ERR(cs)) {
3749 err = PTR_ERR(cs);
3750 i915_request_add(rq);
3751 goto err_put;
3752 }
3753
3754 for (n = 0; n < NUM_GPR_DW; n++) {
3755 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
3756 *cs++ = CS_GPR(engine, n);
3757 *cs++ = i915_ggtt_offset(scratch) + n * sizeof(u32);
3758 *cs++ = 0;
3759 }
3760
3761 i915_request_get(rq);
3762 i915_request_add(rq);
3763
3764 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
3765 err = -ETIME;
3766 goto err_rq;
3767 }
3768
3769 cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
3770 if (IS_ERR(cs)) {
3771 err = PTR_ERR(cs);
3772 goto err_rq;
3773 }
3774
3775 for (n = 0; n < NUM_GPR_DW; n++) {
3776 if (cs[n]) {
3777 pr_err("%s: GPR[%d].%s was not zero, found 0x%08x!\n",
3778 engine->name,
3779 n / 2, n & 1 ? "udw" : "ldw",
3780 cs[n]);
3781 err = -EINVAL;
3782 break;
3783 }
3784 }
3785
3786 i915_gem_object_unpin_map(scratch->obj);
3787
3788 err_rq:
3789 i915_request_put(rq);
3790 err_put:
3791 intel_context_put(ce);
3792 return err;
3793 }
3794
3795 static int live_gpr_clear(void *arg)
3796 {
3797 struct intel_gt *gt = arg;
3798 struct intel_engine_cs *engine;
3799 struct i915_vma *scratch;
3800 enum intel_engine_id id;
3801 int err = 0;
3802
3803 /*
3804 * Check that GPR registers are cleared in new contexts as we need
3805 * to avoid leaking any information from previous contexts.
3806 */
3807
3808 scratch = create_scratch(gt);
3809 if (IS_ERR(scratch))
3810 return PTR_ERR(scratch);
3811
3812 for_each_engine(engine, gt, id) {
3813 err = __live_gpr_clear(engine, scratch);
3814 if (err)
3815 break;
3816 }
3817
3818 if (igt_flush_test(gt->i915))
3819 err = -EIO;
3820
3821 i915_vma_unpin_and_release(&scratch, 0);
3822 return err;
3823 }
3824
3825 int intel_lrc_live_selftests(struct drm_i915_private *i915)
3826 {
3827 static const struct i915_subtest tests[] = {
3828 SUBTEST(live_lrc_layout),
3829 SUBTEST(live_lrc_fixed),
3830 SUBTEST(live_lrc_state),
3831 SUBTEST(live_gpr_clear),
3832 };
3833
3834 if (!HAS_LOGICAL_RING_CONTEXTS(i915))
3835 return 0;
3836
3837 return intel_gt_live_subtests(tests, &i915->gt);
3838 }
Linux with added FPC-III support