?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
WIP FPC-III support
[linux/fpc-iii.git] / drivers / gpu / drm / i915 / gt / selftest_lrc.c
blob95d41c01d0e04221f6f22733d3f1d5a65e2107a9
1 /*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright © 2018 Intel Corporation
5 */
6
7 #include <linux/prime_numbers.h>
8
9 #include "gem/i915_gem_pm.h"
10 #include "gt/intel_engine_heartbeat.h"
11 #include "gt/intel_reset.h"
12 #include "gt/selftest_engine_heartbeat.h"
13
14 #include "i915_selftest.h"
15 #include "selftests/i915_random.h"
16 #include "selftests/igt_flush_test.h"
17 #include "selftests/igt_live_test.h"
18 #include "selftests/igt_spinner.h"
19 #include "selftests/lib_sw_fence.h"
20
21 #include "gem/selftests/igt_gem_utils.h"
22 #include "gem/selftests/mock_context.h"
23
24 #define CS_GPR(engine, n) ((engine)->mmio_base + 0x600 + (n) * 4)
25 #define NUM_GPR 16
26 #define NUM_GPR_DW (NUM_GPR * 2) /* each GPR is 2 dwords */
27
28 static struct i915_vma *create_scratch(struct intel_gt *gt)
29 {
30 struct drm_i915_gem_object *obj;
31 struct i915_vma *vma;
32 int err;
33
34 obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
35 if (IS_ERR(obj))
36 return ERR_CAST(obj);
37
38 i915_gem_object_set_cache_coherency(obj, I915_CACHING_CACHED);
39
40 vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);
41 if (IS_ERR(vma)) {
42 i915_gem_object_put(obj);
43 return vma;
44 }
45
46 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
47 if (err) {
48 i915_gem_object_put(obj);
49 return ERR_PTR(err);
50 }
51
52 return vma;
53 }
54
55 static bool is_active(struct i915_request *rq)
56 {
57 if (i915_request_is_active(rq))
58 return true;
59
60 if (i915_request_on_hold(rq))
61 return true;
62
63 if (i915_request_has_initial_breadcrumb(rq) && i915_request_started(rq))
64 return true;
65
66 return false;
67 }
68
69 static int wait_for_submit(struct intel_engine_cs *engine,
70 struct i915_request *rq,
71 unsigned long timeout)
72 {
73 timeout += jiffies;
74 do {
75 bool done = time_after(jiffies, timeout);
76
77 if (i915_request_completed(rq)) /* that was quick! */
78 return 0;
79
80 /* Wait until the HW has acknowleged the submission (or err) */
81 intel_engine_flush_submission(engine);
82 if (!READ_ONCE(engine->execlists.pending[0]) && is_active(rq))
83 return 0;
84
85 if (done)
86 return -ETIME;
87
88 cond_resched();
89 } while (1);
90 }
91
92 static int wait_for_reset(struct intel_engine_cs *engine,
93 struct i915_request *rq,
94 unsigned long timeout)
95 {
96 timeout += jiffies;
97
98 do {
99 cond_resched();
100 intel_engine_flush_submission(engine);
101
102 if (READ_ONCE(engine->execlists.pending[0]))
103 continue;
104
105 if (i915_request_completed(rq))
106 break;
107
108 if (READ_ONCE(rq->fence.error))
109 break;
110 } while (time_before(jiffies, timeout));
111
112 flush_scheduled_work();
113
114 if (rq->fence.error != -EIO) {
115 pr_err("%s: hanging request %llx:%lld not reset\n",
116 engine->name,
117 rq->fence.context,
118 rq->fence.seqno);
119 return -EINVAL;
120 }
121
122 /* Give the request a jiffie to complete after flushing the worker */
123 if (i915_request_wait(rq, 0,
124 max(0l, (long)(timeout - jiffies)) + 1) < 0) {
125 pr_err("%s: hanging request %llx:%lld did not complete\n",
126 engine->name,
127 rq->fence.context,
128 rq->fence.seqno);
129 return -ETIME;
130 }
131
132 return 0;
133 }
134
135 static int live_sanitycheck(void *arg)
136 {
137 struct intel_gt *gt = arg;
138 struct intel_engine_cs *engine;
139 enum intel_engine_id id;
140 struct igt_spinner spin;
141 int err = 0;
142
143 if (!HAS_LOGICAL_RING_CONTEXTS(gt->i915))
144 return 0;
145
146 if (igt_spinner_init(&spin, gt))
147 return -ENOMEM;
148
149 for_each_engine(engine, gt, id) {
150 struct intel_context *ce;
151 struct i915_request *rq;
152
153 ce = intel_context_create(engine);
154 if (IS_ERR(ce)) {
155 err = PTR_ERR(ce);
156 break;
157 }
158
159 rq = igt_spinner_create_request(&spin, ce, MI_NOOP);
160 if (IS_ERR(rq)) {
161 err = PTR_ERR(rq);
162 goto out_ctx;
163 }
164
165 i915_request_add(rq);
166 if (!igt_wait_for_spinner(&spin, rq)) {
167 GEM_TRACE("spinner failed to start\n");
168 GEM_TRACE_DUMP();
169 intel_gt_set_wedged(gt);
170 err = -EIO;
171 goto out_ctx;
172 }
173
174 igt_spinner_end(&spin);
175 if (igt_flush_test(gt->i915)) {
176 err = -EIO;
177 goto out_ctx;
178 }
179
180 out_ctx:
181 intel_context_put(ce);
182 if (err)
183 break;
184 }
185
186 igt_spinner_fini(&spin);
187 return err;
188 }
189
190 static int live_unlite_restore(struct intel_gt *gt, int prio)
191 {
192 struct intel_engine_cs *engine;
193 enum intel_engine_id id;
194 struct igt_spinner spin;
195 int err = -ENOMEM;
196
197 /*
198 * Check that we can correctly context switch between 2 instances
199 * on the same engine from the same parent context.
200 */
201
202 if (igt_spinner_init(&spin, gt))
203 return err;
204
205 err = 0;
206 for_each_engine(engine, gt, id) {
207 struct intel_context *ce[2] = {};
208 struct i915_request *rq[2];
209 struct igt_live_test t;
210 int n;
211
212 if (prio && !intel_engine_has_preemption(engine))
213 continue;
214
215 if (!intel_engine_can_store_dword(engine))
216 continue;
217
218 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
219 err = -EIO;
220 break;
221 }
222 st_engine_heartbeat_disable(engine);
223
224 for (n = 0; n < ARRAY_SIZE(ce); n++) {
225 struct intel_context *tmp;
226
227 tmp = intel_context_create(engine);
228 if (IS_ERR(tmp)) {
229 err = PTR_ERR(tmp);
230 goto err_ce;
231 }
232
233 err = intel_context_pin(tmp);
234 if (err) {
235 intel_context_put(tmp);
236 goto err_ce;
237 }
238
239 /*
240 * Setup the pair of contexts such that if we
241 * lite-restore using the RING_TAIL from ce[1] it
242 * will execute garbage from ce[0]->ring.
243 */
244 memset(tmp->ring->vaddr,
245 POISON_INUSE, /* IPEHR: 0x5a5a5a5a [hung!] */
246 tmp->ring->vma->size);
247
248 ce[n] = tmp;
249 }
250 GEM_BUG_ON(!ce[1]->ring->size);
251 intel_ring_reset(ce[1]->ring, ce[1]->ring->size / 2);
252 __execlists_update_reg_state(ce[1], engine, ce[1]->ring->head);
253
254 rq[0] = igt_spinner_create_request(&spin, ce[0], MI_ARB_CHECK);
255 if (IS_ERR(rq[0])) {
256 err = PTR_ERR(rq[0]);
257 goto err_ce;
258 }
259
260 i915_request_get(rq[0]);
261 i915_request_add(rq[0]);
262 GEM_BUG_ON(rq[0]->postfix > ce[1]->ring->emit);
263
264 if (!igt_wait_for_spinner(&spin, rq[0])) {
265 i915_request_put(rq[0]);
266 goto err_ce;
267 }
268
269 rq[1] = i915_request_create(ce[1]);
270 if (IS_ERR(rq[1])) {
271 err = PTR_ERR(rq[1]);
272 i915_request_put(rq[0]);
273 goto err_ce;
274 }
275
276 if (!prio) {
277 /*
278 * Ensure we do the switch to ce[1] on completion.
279 *
280 * rq[0] is already submitted, so this should reduce
281 * to a no-op (a wait on a request on the same engine
282 * uses the submit fence, not the completion fence),
283 * but it will install a dependency on rq[1] for rq[0]
284 * that will prevent the pair being reordered by
285 * timeslicing.
286 */
287 i915_request_await_dma_fence(rq[1], &rq[0]->fence);
288 }
289
290 i915_request_get(rq[1]);
291 i915_request_add(rq[1]);
292 GEM_BUG_ON(rq[1]->postfix <= rq[0]->postfix);
293 i915_request_put(rq[0]);
294
295 if (prio) {
296 struct i915_sched_attr attr = {
297 .priority = prio,
298 };
299
300 /* Alternatively preempt the spinner with ce[1] */
301 engine->schedule(rq[1], &attr);
302 }
303
304 /* And switch back to ce[0] for good measure */
305 rq[0] = i915_request_create(ce[0]);
306 if (IS_ERR(rq[0])) {
307 err = PTR_ERR(rq[0]);
308 i915_request_put(rq[1]);
309 goto err_ce;
310 }
311
312 i915_request_await_dma_fence(rq[0], &rq[1]->fence);
313 i915_request_get(rq[0]);
314 i915_request_add(rq[0]);
315 GEM_BUG_ON(rq[0]->postfix > rq[1]->postfix);
316 i915_request_put(rq[1]);
317 i915_request_put(rq[0]);
318
319 err_ce:
320 intel_engine_flush_submission(engine);
321 igt_spinner_end(&spin);
322 for (n = 0; n < ARRAY_SIZE(ce); n++) {
323 if (IS_ERR_OR_NULL(ce[n]))
324 break;
325
326 intel_context_unpin(ce[n]);
327 intel_context_put(ce[n]);
328 }
329
330 st_engine_heartbeat_enable(engine);
331 if (igt_live_test_end(&t))
332 err = -EIO;
333 if (err)
334 break;
335 }
336
337 igt_spinner_fini(&spin);
338 return err;
339 }
340
341 static int live_unlite_switch(void *arg)
342 {
343 return live_unlite_restore(arg, 0);
344 }
345
346 static int live_unlite_preempt(void *arg)
347 {
348 return live_unlite_restore(arg, I915_USER_PRIORITY(I915_PRIORITY_MAX));
349 }
350
351 static int live_unlite_ring(void *arg)
352 {
353 struct intel_gt *gt = arg;
354 struct intel_engine_cs *engine;
355 struct igt_spinner spin;
356 enum intel_engine_id id;
357 int err = 0;
358
359 /*
360 * Setup a preemption event that will cause almost the entire ring
361 * to be unwound, potentially fooling our intel_ring_direction()
362 * into emitting a forward lite-restore instead of the rollback.
363 */
364
365 if (igt_spinner_init(&spin, gt))
366 return -ENOMEM;
367
368 for_each_engine(engine, gt, id) {
369 struct intel_context *ce[2] = {};
370 struct i915_request *rq;
371 struct igt_live_test t;
372 int n;
373
374 if (!intel_engine_has_preemption(engine))
375 continue;
376
377 if (!intel_engine_can_store_dword(engine))
378 continue;
379
380 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
381 err = -EIO;
382 break;
383 }
384 st_engine_heartbeat_disable(engine);
385
386 for (n = 0; n < ARRAY_SIZE(ce); n++) {
387 struct intel_context *tmp;
388
389 tmp = intel_context_create(engine);
390 if (IS_ERR(tmp)) {
391 err = PTR_ERR(tmp);
392 goto err_ce;
393 }
394
395 err = intel_context_pin(tmp);
396 if (err) {
397 intel_context_put(tmp);
398 goto err_ce;
399 }
400
401 memset32(tmp->ring->vaddr,
402 0xdeadbeef, /* trigger a hang if executed */
403 tmp->ring->vma->size / sizeof(u32));
404
405 ce[n] = tmp;
406 }
407
408 /* Create max prio spinner, followed by N low prio nops */
409 rq = igt_spinner_create_request(&spin, ce[0], MI_ARB_CHECK);
410 if (IS_ERR(rq)) {
411 err = PTR_ERR(rq);
412 goto err_ce;
413 }
414
415 i915_request_get(rq);
416 rq->sched.attr.priority = I915_PRIORITY_BARRIER;
417 i915_request_add(rq);
418
419 if (!igt_wait_for_spinner(&spin, rq)) {
420 intel_gt_set_wedged(gt);
421 i915_request_put(rq);
422 err = -ETIME;
423 goto err_ce;
424 }
425
426 /* Fill the ring, until we will cause a wrap */
427 n = 0;
428 while (intel_ring_direction(ce[0]->ring,
429 rq->wa_tail,
430 ce[0]->ring->tail) <= 0) {
431 struct i915_request *tmp;
432
433 tmp = intel_context_create_request(ce[0]);
434 if (IS_ERR(tmp)) {
435 err = PTR_ERR(tmp);
436 i915_request_put(rq);
437 goto err_ce;
438 }
439
440 i915_request_add(tmp);
441 intel_engine_flush_submission(engine);
442 n++;
443 }
444 intel_engine_flush_submission(engine);
445 pr_debug("%s: Filled ring with %d nop tails {size:%x, tail:%x, emit:%x, rq.tail:%x}\n",
446 engine->name, n,
447 ce[0]->ring->size,
448 ce[0]->ring->tail,
449 ce[0]->ring->emit,
450 rq->tail);
451 GEM_BUG_ON(intel_ring_direction(ce[0]->ring,
452 rq->tail,
453 ce[0]->ring->tail) <= 0);
454 i915_request_put(rq);
455
456 /* Create a second ring to preempt the first ring after rq[0] */
457 rq = intel_context_create_request(ce[1]);
458 if (IS_ERR(rq)) {
459 err = PTR_ERR(rq);
460 goto err_ce;
461 }
462
463 rq->sched.attr.priority = I915_PRIORITY_BARRIER;
464 i915_request_get(rq);
465 i915_request_add(rq);
466
467 err = wait_for_submit(engine, rq, HZ / 2);
468 i915_request_put(rq);
469 if (err) {
470 pr_err("%s: preemption request was not submitted\n",
471 engine->name);
472 err = -ETIME;
473 }
474
475 pr_debug("%s: ring[0]:{ tail:%x, emit:%x }, ring[1]:{ tail:%x, emit:%x }\n",
476 engine->name,
477 ce[0]->ring->tail, ce[0]->ring->emit,
478 ce[1]->ring->tail, ce[1]->ring->emit);
479
480 err_ce:
481 intel_engine_flush_submission(engine);
482 igt_spinner_end(&spin);
483 for (n = 0; n < ARRAY_SIZE(ce); n++) {
484 if (IS_ERR_OR_NULL(ce[n]))
485 break;
486
487 intel_context_unpin(ce[n]);
488 intel_context_put(ce[n]);
489 }
490 st_engine_heartbeat_enable(engine);
491 if (igt_live_test_end(&t))
492 err = -EIO;
493 if (err)
494 break;
495 }
496
497 igt_spinner_fini(&spin);
498 return err;
499 }
500
501 static int live_pin_rewind(void *arg)
502 {
503 struct intel_gt *gt = arg;
504 struct intel_engine_cs *engine;
505 enum intel_engine_id id;
506 int err = 0;
507
508 /*
509 * We have to be careful not to trust intel_ring too much, for example
510 * ring->head is updated upon retire which is out of sync with pinning
511 * the context. Thus we cannot use ring->head to set CTX_RING_HEAD,
512 * or else we risk writing an older, stale value.
513 *
514 * To simulate this, let's apply a bit of deliberate sabotague.
515 */
516
517 for_each_engine(engine, gt, id) {
518 struct intel_context *ce;
519 struct i915_request *rq;
520 struct intel_ring *ring;
521 struct igt_live_test t;
522
523 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
524 err = -EIO;
525 break;
526 }
527
528 ce = intel_context_create(engine);
529 if (IS_ERR(ce)) {
530 err = PTR_ERR(ce);
531 break;
532 }
533
534 err = intel_context_pin(ce);
535 if (err) {
536 intel_context_put(ce);
537 break;
538 }
539
540 /* Keep the context awake while we play games */
541 err = i915_active_acquire(&ce->active);
542 if (err) {
543 intel_context_unpin(ce);
544 intel_context_put(ce);
545 break;
546 }
547 ring = ce->ring;
548
549 /* Poison the ring, and offset the next request from HEAD */
550 memset32(ring->vaddr, STACK_MAGIC, ring->size / sizeof(u32));
551 ring->emit = ring->size / 2;
552 ring->tail = ring->emit;
553 GEM_BUG_ON(ring->head);
554
555 intel_context_unpin(ce);
556
557 /* Submit a simple nop request */
558 GEM_BUG_ON(intel_context_is_pinned(ce));
559 rq = intel_context_create_request(ce);
560 i915_active_release(&ce->active); /* e.g. async retire */
561 intel_context_put(ce);
562 if (IS_ERR(rq)) {
563 err = PTR_ERR(rq);
564 break;
565 }
566 GEM_BUG_ON(!rq->head);
567 i915_request_add(rq);
568
569 /* Expect not to hang! */
570 if (igt_live_test_end(&t)) {
571 err = -EIO;
572 break;
573 }
574 }
575
576 return err;
577 }
578
579 static int live_hold_reset(void *arg)
580 {
581 struct intel_gt *gt = arg;
582 struct intel_engine_cs *engine;
583 enum intel_engine_id id;
584 struct igt_spinner spin;
585 int err = 0;
586
587 /*
588 * In order to support offline error capture for fast preempt reset,
589 * we need to decouple the guilty request and ensure that it and its
590 * descendents are not executed while the capture is in progress.
591 */
592
593 if (!intel_has_reset_engine(gt))
594 return 0;
595
596 if (igt_spinner_init(&spin, gt))
597 return -ENOMEM;
598
599 for_each_engine(engine, gt, id) {
600 struct intel_context *ce;
601 struct i915_request *rq;
602
603 ce = intel_context_create(engine);
604 if (IS_ERR(ce)) {
605 err = PTR_ERR(ce);
606 break;
607 }
608
609 st_engine_heartbeat_disable(engine);
610
611 rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
612 if (IS_ERR(rq)) {
613 err = PTR_ERR(rq);
614 goto out;
615 }
616 i915_request_add(rq);
617
618 if (!igt_wait_for_spinner(&spin, rq)) {
619 intel_gt_set_wedged(gt);
620 err = -ETIME;
621 goto out;
622 }
623
624 /* We have our request executing, now remove it and reset */
625
626 if (test_and_set_bit(I915_RESET_ENGINE + id,
627 &gt->reset.flags)) {
628 intel_gt_set_wedged(gt);
629 err = -EBUSY;
630 goto out;
631 }
632 tasklet_disable(&engine->execlists.tasklet);
633
634 engine->execlists.tasklet.func(engine->execlists.tasklet.data);
635 GEM_BUG_ON(execlists_active(&engine->execlists) != rq);
636
637 i915_request_get(rq);
638 execlists_hold(engine, rq);
639 GEM_BUG_ON(!i915_request_on_hold(rq));
640
641 intel_engine_reset(engine, NULL);
642 GEM_BUG_ON(rq->fence.error != -EIO);
643
644 tasklet_enable(&engine->execlists.tasklet);
645 clear_and_wake_up_bit(I915_RESET_ENGINE + id,
646 &gt->reset.flags);
647
648 /* Check that we do not resubmit the held request */
649 if (!i915_request_wait(rq, 0, HZ / 5)) {
650 pr_err("%s: on hold request completed!\n",
651 engine->name);
652 i915_request_put(rq);
653 err = -EIO;
654 goto out;
655 }
656 GEM_BUG_ON(!i915_request_on_hold(rq));
657
658 /* But is resubmitted on release */
659 execlists_unhold(engine, rq);
660 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
661 pr_err("%s: held request did not complete!\n",
662 engine->name);
663 intel_gt_set_wedged(gt);
664 err = -ETIME;
665 }
666 i915_request_put(rq);
667
668 out:
669 st_engine_heartbeat_enable(engine);
670 intel_context_put(ce);
671 if (err)
672 break;
673 }
674
675 igt_spinner_fini(&spin);
676 return err;
677 }
678
679 static const char *error_repr(int err)
680 {
681 return err ? "bad" : "good";
682 }
683
684 static int live_error_interrupt(void *arg)
685 {
686 static const struct error_phase {
687 enum { GOOD = 0, BAD = -EIO } error[2];
688 } phases[] = {
689 { { BAD, GOOD } },
690 { { BAD, BAD } },
691 { { BAD, GOOD } },
692 { { GOOD, GOOD } }, /* sentinel */
693 };
694 struct intel_gt *gt = arg;
695 struct intel_engine_cs *engine;
696 enum intel_engine_id id;
697
698 /*
699 * We hook up the CS_MASTER_ERROR_INTERRUPT to have forewarning
700 * of invalid commands in user batches that will cause a GPU hang.
701 * This is a faster mechanism than using hangcheck/heartbeats, but
702 * only detects problems the HW knows about -- it will not warn when
703 * we kill the HW!
704 *
705 * To verify our detection and reset, we throw some invalid commands
706 * at the HW and wait for the interrupt.
707 */
708
709 if (!intel_has_reset_engine(gt))
710 return 0;
711
712 for_each_engine(engine, gt, id) {
713 const struct error_phase *p;
714 int err = 0;
715
716 st_engine_heartbeat_disable(engine);
717
718 for (p = phases; p->error[0] != GOOD; p++) {
719 struct i915_request *client[ARRAY_SIZE(phases->error)];
720 u32 *cs;
721 int i;
722
723 memset(client, 0, sizeof(*client));
724 for (i = 0; i < ARRAY_SIZE(client); i++) {
725 struct intel_context *ce;
726 struct i915_request *rq;
727
728 ce = intel_context_create(engine);
729 if (IS_ERR(ce)) {
730 err = PTR_ERR(ce);
731 goto out;
732 }
733
734 rq = intel_context_create_request(ce);
735 intel_context_put(ce);
736 if (IS_ERR(rq)) {
737 err = PTR_ERR(rq);
738 goto out;
739 }
740
741 if (rq->engine->emit_init_breadcrumb) {
742 err = rq->engine->emit_init_breadcrumb(rq);
743 if (err) {
744 i915_request_add(rq);
745 goto out;
746 }
747 }
748
749 cs = intel_ring_begin(rq, 2);
750 if (IS_ERR(cs)) {
751 i915_request_add(rq);
752 err = PTR_ERR(cs);
753 goto out;
754 }
755
756 if (p->error[i]) {
757 *cs++ = 0xdeadbeef;
758 *cs++ = 0xdeadbeef;
759 } else {
760 *cs++ = MI_NOOP;
761 *cs++ = MI_NOOP;
762 }
763
764 client[i] = i915_request_get(rq);
765 i915_request_add(rq);
766 }
767
768 err = wait_for_submit(engine, client[0], HZ / 2);
769 if (err) {
770 pr_err("%s: first request did not start within time!\n",
771 engine->name);
772 err = -ETIME;
773 goto out;
774 }
775
776 for (i = 0; i < ARRAY_SIZE(client); i++) {
777 if (i915_request_wait(client[i], 0, HZ / 5) < 0)
778 pr_debug("%s: %s request incomplete!\n",
779 engine->name,
780 error_repr(p->error[i]));
781
782 if (!i915_request_started(client[i])) {
783 pr_err("%s: %s request not started!\n",
784 engine->name,
785 error_repr(p->error[i]));
786 err = -ETIME;
787 goto out;
788 }
789
790 /* Kick the tasklet to process the error */
791 intel_engine_flush_submission(engine);
792 if (client[i]->fence.error != p->error[i]) {
793 pr_err("%s: %s request (%s) with wrong error code: %d\n",
794 engine->name,
795 error_repr(p->error[i]),
796 i915_request_completed(client[i]) ? "completed" : "running",
797 client[i]->fence.error);
798 err = -EINVAL;
799 goto out;
800 }
801 }
802
803 out:
804 for (i = 0; i < ARRAY_SIZE(client); i++)
805 if (client[i])
806 i915_request_put(client[i]);
807 if (err) {
808 pr_err("%s: failed at phase[%zd] { %d, %d }\n",
809 engine->name, p - phases,
810 p->error[0], p->error[1]);
811 break;
812 }
813 }
814
815 st_engine_heartbeat_enable(engine);
816 if (err) {
817 intel_gt_set_wedged(gt);
818 return err;
819 }
820 }
821
822 return 0;
823 }
824
825 static int
826 emit_semaphore_chain(struct i915_request *rq, struct i915_vma *vma, int idx)
827 {
828 u32 *cs;
829
830 cs = intel_ring_begin(rq, 10);
831 if (IS_ERR(cs))
832 return PTR_ERR(cs);
833
834 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
835
836 *cs++ = MI_SEMAPHORE_WAIT |
837 MI_SEMAPHORE_GLOBAL_GTT |
838 MI_SEMAPHORE_POLL |
839 MI_SEMAPHORE_SAD_NEQ_SDD;
840 *cs++ = 0;
841 *cs++ = i915_ggtt_offset(vma) + 4 * idx;
842 *cs++ = 0;
843
844 if (idx > 0) {
845 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
846 *cs++ = i915_ggtt_offset(vma) + 4 * (idx - 1);
847 *cs++ = 0;
848 *cs++ = 1;
849 } else {
850 *cs++ = MI_NOOP;
851 *cs++ = MI_NOOP;
852 *cs++ = MI_NOOP;
853 *cs++ = MI_NOOP;
854 }
855
856 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
857
858 intel_ring_advance(rq, cs);
859 return 0;
860 }
861
862 static struct i915_request *
863 semaphore_queue(struct intel_engine_cs *engine, struct i915_vma *vma, int idx)
864 {
865 struct intel_context *ce;
866 struct i915_request *rq;
867 int err;
868
869 ce = intel_context_create(engine);
870 if (IS_ERR(ce))
871 return ERR_CAST(ce);
872
873 rq = intel_context_create_request(ce);
874 if (IS_ERR(rq))
875 goto out_ce;
876
877 err = 0;
878 if (rq->engine->emit_init_breadcrumb)
879 err = rq->engine->emit_init_breadcrumb(rq);
880 if (err == 0)
881 err = emit_semaphore_chain(rq, vma, idx);
882 if (err == 0)
883 i915_request_get(rq);
884 i915_request_add(rq);
885 if (err)
886 rq = ERR_PTR(err);
887
888 out_ce:
889 intel_context_put(ce);
890 return rq;
891 }
892
893 static int
894 release_queue(struct intel_engine_cs *engine,
895 struct i915_vma *vma,
896 int idx, int prio)
897 {
898 struct i915_sched_attr attr = {
899 .priority = prio,
900 };
901 struct i915_request *rq;
902 u32 *cs;
903
904 rq = intel_engine_create_kernel_request(engine);
905 if (IS_ERR(rq))
906 return PTR_ERR(rq);
907
908 cs = intel_ring_begin(rq, 4);
909 if (IS_ERR(cs)) {
910 i915_request_add(rq);
911 return PTR_ERR(cs);
912 }
913
914 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
915 *cs++ = i915_ggtt_offset(vma) + 4 * (idx - 1);
916 *cs++ = 0;
917 *cs++ = 1;
918
919 intel_ring_advance(rq, cs);
920
921 i915_request_get(rq);
922 i915_request_add(rq);
923
924 local_bh_disable();
925 engine->schedule(rq, &attr);
926 local_bh_enable(); /* kick tasklet */
927
928 i915_request_put(rq);
929
930 return 0;
931 }
932
933 static int
934 slice_semaphore_queue(struct intel_engine_cs *outer,
935 struct i915_vma *vma,
936 int count)
937 {
938 struct intel_engine_cs *engine;
939 struct i915_request *head;
940 enum intel_engine_id id;
941 int err, i, n = 0;
942
943 head = semaphore_queue(outer, vma, n++);
944 if (IS_ERR(head))
945 return PTR_ERR(head);
946
947 for_each_engine(engine, outer->gt, id) {
948 for (i = 0; i < count; i++) {
949 struct i915_request *rq;
950
951 rq = semaphore_queue(engine, vma, n++);
952 if (IS_ERR(rq)) {
953 err = PTR_ERR(rq);
954 goto out;
955 }
956
957 i915_request_put(rq);
958 }
959 }
960
961 err = release_queue(outer, vma, n, I915_PRIORITY_BARRIER);
962 if (err)
963 goto out;
964
965 if (i915_request_wait(head, 0,
966 2 * outer->gt->info.num_engines * (count + 2) * (count + 3)) < 0) {
967 pr_err("Failed to slice along semaphore chain of length (%d, %d)!\n",
968 count, n);
969 GEM_TRACE_DUMP();
970 intel_gt_set_wedged(outer->gt);
971 err = -EIO;
972 }
973
974 out:
975 i915_request_put(head);
976 return err;
977 }
978
979 static int live_timeslice_preempt(void *arg)
980 {
981 struct intel_gt *gt = arg;
982 struct drm_i915_gem_object *obj;
983 struct intel_engine_cs *engine;
984 enum intel_engine_id id;
985 struct i915_vma *vma;
986 void *vaddr;
987 int err = 0;
988
989 /*
990 * If a request takes too long, we would like to give other users
991 * a fair go on the GPU. In particular, users may create batches
992 * that wait upon external input, where that input may even be
993 * supplied by another GPU job. To avoid blocking forever, we
994 * need to preempt the current task and replace it with another
995 * ready task.
996 */
997 if (!IS_ACTIVE(CONFIG_DRM_I915_TIMESLICE_DURATION))
998 return 0;
999
1000 obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
1001 if (IS_ERR(obj))
1002 return PTR_ERR(obj);
1003
1004 vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);
1005 if (IS_ERR(vma)) {
1006 err = PTR_ERR(vma);
1007 goto err_obj;
1008 }
1009
1010 vaddr = i915_gem_object_pin_map(obj, I915_MAP_WC);
1011 if (IS_ERR(vaddr)) {
1012 err = PTR_ERR(vaddr);
1013 goto err_obj;
1014 }
1015
1016 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
1017 if (err)
1018 goto err_map;
1019
1020 err = i915_vma_sync(vma);
1021 if (err)
1022 goto err_pin;
1023
1024 for_each_engine(engine, gt, id) {
1025 if (!intel_engine_has_preemption(engine))
1026 continue;
1027
1028 memset(vaddr, 0, PAGE_SIZE);
1029
1030 st_engine_heartbeat_disable(engine);
1031 err = slice_semaphore_queue(engine, vma, 5);
1032 st_engine_heartbeat_enable(engine);
1033 if (err)
1034 goto err_pin;
1035
1036 if (igt_flush_test(gt->i915)) {
1037 err = -EIO;
1038 goto err_pin;
1039 }
1040 }
1041
1042 err_pin:
1043 i915_vma_unpin(vma);
1044 err_map:
1045 i915_gem_object_unpin_map(obj);
1046 err_obj:
1047 i915_gem_object_put(obj);
1048 return err;
1049 }
1050
1051 static struct i915_request *
1052 create_rewinder(struct intel_context *ce,
1053 struct i915_request *wait,
1054 void *slot, int idx)
1055 {
1056 const u32 offset =
1057 i915_ggtt_offset(ce->engine->status_page.vma) +
1058 offset_in_page(slot);
1059 struct i915_request *rq;
1060 u32 *cs;
1061 int err;
1062
1063 rq = intel_context_create_request(ce);
1064 if (IS_ERR(rq))
1065 return rq;
1066
1067 if (wait) {
1068 err = i915_request_await_dma_fence(rq, &wait->fence);
1069 if (err)
1070 goto err;
1071 }
1072
1073 cs = intel_ring_begin(rq, 14);
1074 if (IS_ERR(cs)) {
1075 err = PTR_ERR(cs);
1076 goto err;
1077 }
1078
1079 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
1080 *cs++ = MI_NOOP;
1081
1082 *cs++ = MI_SEMAPHORE_WAIT |
1083 MI_SEMAPHORE_GLOBAL_GTT |
1084 MI_SEMAPHORE_POLL |
1085 MI_SEMAPHORE_SAD_GTE_SDD;
1086 *cs++ = idx;
1087 *cs++ = offset;
1088 *cs++ = 0;
1089
1090 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
1091 *cs++ = i915_mmio_reg_offset(RING_TIMESTAMP(rq->engine->mmio_base));
1092 *cs++ = offset + idx * sizeof(u32);
1093 *cs++ = 0;
1094
1095 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
1096 *cs++ = offset;
1097 *cs++ = 0;
1098 *cs++ = idx + 1;
1099
1100 intel_ring_advance(rq, cs);
1101
1102 rq->sched.attr.priority = I915_PRIORITY_MASK;
1103 err = 0;
1104 err:
1105 i915_request_get(rq);
1106 i915_request_add(rq);
1107 if (err) {
1108 i915_request_put(rq);
1109 return ERR_PTR(err);
1110 }
1111
1112 return rq;
1113 }
1114
1115 static int live_timeslice_rewind(void *arg)
1116 {
1117 struct intel_gt *gt = arg;
1118 struct intel_engine_cs *engine;
1119 enum intel_engine_id id;
1120
1121 /*
1122 * The usual presumption on timeslice expiration is that we replace
1123 * the active context with another. However, given a chain of
1124 * dependencies we may end up with replacing the context with itself,
1125 * but only a few of those requests, forcing us to rewind the
1126 * RING_TAIL of the original request.
1127 */
1128 if (!IS_ACTIVE(CONFIG_DRM_I915_TIMESLICE_DURATION))
1129 return 0;
1130
1131 for_each_engine(engine, gt, id) {
1132 enum { A1, A2, B1 };
1133 enum { X = 1, Z, Y };
1134 struct i915_request *rq[3] = {};
1135 struct intel_context *ce;
1136 unsigned long timeslice;
1137 int i, err = 0;
1138 u32 *slot;
1139
1140 if (!intel_engine_has_timeslices(engine))
1141 continue;
1142
1143 /*
1144 * A:rq1 -- semaphore wait, timestamp X
1145 * A:rq2 -- write timestamp Y
1146 *
1147 * B:rq1 [await A:rq1] -- write timestamp Z
1148 *
1149 * Force timeslice, release semaphore.
1150 *
1151 * Expect execution/evaluation order XZY
1152 */
1153
1154 st_engine_heartbeat_disable(engine);
1155 timeslice = xchg(&engine->props.timeslice_duration_ms, 1);
1156
1157 slot = memset32(engine->status_page.addr + 1000, 0, 4);
1158
1159 ce = intel_context_create(engine);
1160 if (IS_ERR(ce)) {
1161 err = PTR_ERR(ce);
1162 goto err;
1163 }
1164
1165 rq[A1] = create_rewinder(ce, NULL, slot, X);
1166 if (IS_ERR(rq[A1])) {
1167 intel_context_put(ce);
1168 goto err;
1169 }
1170
1171 rq[A2] = create_rewinder(ce, NULL, slot, Y);
1172 intel_context_put(ce);
1173 if (IS_ERR(rq[A2]))
1174 goto err;
1175
1176 err = wait_for_submit(engine, rq[A2], HZ / 2);
1177 if (err) {
1178 pr_err("%s: failed to submit first context\n",
1179 engine->name);
1180 goto err;
1181 }
1182
1183 ce = intel_context_create(engine);
1184 if (IS_ERR(ce)) {
1185 err = PTR_ERR(ce);
1186 goto err;
1187 }
1188
1189 rq[B1] = create_rewinder(ce, rq[A1], slot, Z);
1190 intel_context_put(ce);
1191 if (IS_ERR(rq[2]))
1192 goto err;
1193
1194 err = wait_for_submit(engine, rq[B1], HZ / 2);
1195 if (err) {
1196 pr_err("%s: failed to submit second context\n",
1197 engine->name);
1198 goto err;
1199 }
1200
1201 /* ELSP[] = { { A:rq1, A:rq2 }, { B:rq1 } } */
1202 ENGINE_TRACE(engine, "forcing tasklet for rewind\n");
1203 if (i915_request_is_active(rq[A2])) { /* semaphore yielded! */
1204 /* Wait for the timeslice to kick in */
1205 del_timer(&engine->execlists.timer);
1206 tasklet_hi_schedule(&engine->execlists.tasklet);
1207 intel_engine_flush_submission(engine);
1208 }
1209 /* -> ELSP[] = { { A:rq1 }, { B:rq1 } } */
1210 GEM_BUG_ON(!i915_request_is_active(rq[A1]));
1211 GEM_BUG_ON(!i915_request_is_active(rq[B1]));
1212 GEM_BUG_ON(i915_request_is_active(rq[A2]));
1213
1214 /* Release the hounds! */
1215 slot[0] = 1;
1216 wmb(); /* "pairs" with GPU; paranoid kick of internal CPU$ */
1217
1218 for (i = 1; i <= 3; i++) {
1219 unsigned long timeout = jiffies + HZ / 2;
1220
1221 while (!READ_ONCE(slot[i]) &&
1222 time_before(jiffies, timeout))
1223 ;
1224
1225 if (!time_before(jiffies, timeout)) {
1226 pr_err("%s: rq[%d] timed out\n",
1227 engine->name, i - 1);
1228 err = -ETIME;
1229 goto err;
1230 }
1231
1232 pr_debug("%s: slot[%d]:%x\n", engine->name, i, slot[i]);
1233 }
1234
1235 /* XZY: XZ < XY */
1236 if (slot[Z] - slot[X] >= slot[Y] - slot[X]) {
1237 pr_err("%s: timeslicing did not run context B [%u] before A [%u]!\n",
1238 engine->name,
1239 slot[Z] - slot[X],
1240 slot[Y] - slot[X]);
1241 err = -EINVAL;
1242 }
1243
1244 err:
1245 memset32(&slot[0], -1, 4);
1246 wmb();
1247
1248 engine->props.timeslice_duration_ms = timeslice;
1249 st_engine_heartbeat_enable(engine);
1250 for (i = 0; i < 3; i++)
1251 i915_request_put(rq[i]);
1252 if (igt_flush_test(gt->i915))
1253 err = -EIO;
1254 if (err)
1255 return err;
1256 }
1257
1258 return 0;
1259 }
1260
1261 static struct i915_request *nop_request(struct intel_engine_cs *engine)
1262 {
1263 struct i915_request *rq;
1264
1265 rq = intel_engine_create_kernel_request(engine);
1266 if (IS_ERR(rq))
1267 return rq;
1268
1269 i915_request_get(rq);
1270 i915_request_add(rq);
1271
1272 return rq;
1273 }
1274
1275 static long slice_timeout(struct intel_engine_cs *engine)
1276 {
1277 long timeout;
1278
1279 /* Enough time for a timeslice to kick in, and kick out */
1280 timeout = 2 * msecs_to_jiffies_timeout(timeslice(engine));
1281
1282 /* Enough time for the nop request to complete */
1283 timeout += HZ / 5;
1284
1285 return timeout + 1;
1286 }
1287
1288 static int live_timeslice_queue(void *arg)
1289 {
1290 struct intel_gt *gt = arg;
1291 struct drm_i915_gem_object *obj;
1292 struct intel_engine_cs *engine;
1293 enum intel_engine_id id;
1294 struct i915_vma *vma;
1295 void *vaddr;
1296 int err = 0;
1297
1298 /*
1299 * Make sure that even if ELSP[0] and ELSP[1] are filled with
1300 * timeslicing between them disabled, we *do* enable timeslicing
1301 * if the queue demands it. (Normally, we do not submit if
1302 * ELSP[1] is already occupied, so must rely on timeslicing to
1303 * eject ELSP[0] in favour of the queue.)
1304 */
1305 if (!IS_ACTIVE(CONFIG_DRM_I915_TIMESLICE_DURATION))
1306 return 0;
1307
1308 obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
1309 if (IS_ERR(obj))
1310 return PTR_ERR(obj);
1311
1312 vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);
1313 if (IS_ERR(vma)) {
1314 err = PTR_ERR(vma);
1315 goto err_obj;
1316 }
1317
1318 vaddr = i915_gem_object_pin_map(obj, I915_MAP_WC);
1319 if (IS_ERR(vaddr)) {
1320 err = PTR_ERR(vaddr);
1321 goto err_obj;
1322 }
1323
1324 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
1325 if (err)
1326 goto err_map;
1327
1328 err = i915_vma_sync(vma);
1329 if (err)
1330 goto err_pin;
1331
1332 for_each_engine(engine, gt, id) {
1333 struct i915_sched_attr attr = {
1334 .priority = I915_USER_PRIORITY(I915_PRIORITY_MAX),
1335 };
1336 struct i915_request *rq, *nop;
1337
1338 if (!intel_engine_has_preemption(engine))
1339 continue;
1340
1341 st_engine_heartbeat_disable(engine);
1342 memset(vaddr, 0, PAGE_SIZE);
1343
1344 /* ELSP[0]: semaphore wait */
1345 rq = semaphore_queue(engine, vma, 0);
1346 if (IS_ERR(rq)) {
1347 err = PTR_ERR(rq);
1348 goto err_heartbeat;
1349 }
1350 engine->schedule(rq, &attr);
1351 err = wait_for_submit(engine, rq, HZ / 2);
1352 if (err) {
1353 pr_err("%s: Timed out trying to submit semaphores\n",
1354 engine->name);
1355 goto err_rq;
1356 }
1357
1358 /* ELSP[1]: nop request */
1359 nop = nop_request(engine);
1360 if (IS_ERR(nop)) {
1361 err = PTR_ERR(nop);
1362 goto err_rq;
1363 }
1364 err = wait_for_submit(engine, nop, HZ / 2);
1365 i915_request_put(nop);
1366 if (err) {
1367 pr_err("%s: Timed out trying to submit nop\n",
1368 engine->name);
1369 goto err_rq;
1370 }
1371
1372 GEM_BUG_ON(i915_request_completed(rq));
1373 GEM_BUG_ON(execlists_active(&engine->execlists) != rq);
1374
1375 /* Queue: semaphore signal, matching priority as semaphore */
1376 err = release_queue(engine, vma, 1, effective_prio(rq));
1377 if (err)
1378 goto err_rq;
1379
1380 /* Wait until we ack the release_queue and start timeslicing */
1381 do {
1382 cond_resched();
1383 intel_engine_flush_submission(engine);
1384 } while (READ_ONCE(engine->execlists.pending[0]));
1385
1386 /* Timeslice every jiffy, so within 2 we should signal */
1387 if (i915_request_wait(rq, 0, slice_timeout(engine)) < 0) {
1388 struct drm_printer p =
1389 drm_info_printer(gt->i915->drm.dev);
1390
1391 pr_err("%s: Failed to timeslice into queue\n",
1392 engine->name);
1393 intel_engine_dump(engine, &p,
1394 "%s\n", engine->name);
1395
1396 memset(vaddr, 0xff, PAGE_SIZE);
1397 err = -EIO;
1398 }
1399 err_rq:
1400 i915_request_put(rq);
1401 err_heartbeat:
1402 st_engine_heartbeat_enable(engine);
1403 if (err)
1404 break;
1405 }
1406
1407 err_pin:
1408 i915_vma_unpin(vma);
1409 err_map:
1410 i915_gem_object_unpin_map(obj);
1411 err_obj:
1412 i915_gem_object_put(obj);
1413 return err;
1414 }
1415
1416 static int live_timeslice_nopreempt(void *arg)
1417 {
1418 struct intel_gt *gt = arg;
1419 struct intel_engine_cs *engine;
1420 enum intel_engine_id id;
1421 struct igt_spinner spin;
1422 int err = 0;
1423
1424 /*
1425 * We should not timeslice into a request that is marked with
1426 * I915_REQUEST_NOPREEMPT.
1427 */
1428 if (!IS_ACTIVE(CONFIG_DRM_I915_TIMESLICE_DURATION))
1429 return 0;
1430
1431 if (igt_spinner_init(&spin, gt))
1432 return -ENOMEM;
1433
1434 for_each_engine(engine, gt, id) {
1435 struct intel_context *ce;
1436 struct i915_request *rq;
1437 unsigned long timeslice;
1438
1439 if (!intel_engine_has_preemption(engine))
1440 continue;
1441
1442 ce = intel_context_create(engine);
1443 if (IS_ERR(ce)) {
1444 err = PTR_ERR(ce);
1445 break;
1446 }
1447
1448 st_engine_heartbeat_disable(engine);
1449 timeslice = xchg(&engine->props.timeslice_duration_ms, 1);
1450
1451 /* Create an unpreemptible spinner */
1452
1453 rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
1454 intel_context_put(ce);
1455 if (IS_ERR(rq)) {
1456 err = PTR_ERR(rq);
1457 goto out_heartbeat;
1458 }
1459
1460 i915_request_get(rq);
1461 i915_request_add(rq);
1462
1463 if (!igt_wait_for_spinner(&spin, rq)) {
1464 i915_request_put(rq);
1465 err = -ETIME;
1466 goto out_spin;
1467 }
1468
1469 set_bit(I915_FENCE_FLAG_NOPREEMPT, &rq->fence.flags);
1470 i915_request_put(rq);
1471
1472 /* Followed by a maximum priority barrier (heartbeat) */
1473
1474 ce = intel_context_create(engine);
1475 if (IS_ERR(ce)) {
1476 err = PTR_ERR(ce);
1477 goto out_spin;
1478 }
1479
1480 rq = intel_context_create_request(ce);
1481 intel_context_put(ce);
1482 if (IS_ERR(rq)) {
1483 err = PTR_ERR(rq);
1484 goto out_spin;
1485 }
1486
1487 rq->sched.attr.priority = I915_PRIORITY_BARRIER;
1488 i915_request_get(rq);
1489 i915_request_add(rq);
1490
1491 /*
1492 * Wait until the barrier is in ELSP, and we know timeslicing
1493 * will have been activated.
1494 */
1495 if (wait_for_submit(engine, rq, HZ / 2)) {
1496 i915_request_put(rq);
1497 err = -ETIME;
1498 goto out_spin;
1499 }
1500
1501 /*
1502 * Since the ELSP[0] request is unpreemptible, it should not
1503 * allow the maximum priority barrier through. Wait long
1504 * enough to see if it is timesliced in by mistake.
1505 */
1506 if (i915_request_wait(rq, 0, slice_timeout(engine)) >= 0) {
1507 pr_err("%s: I915_PRIORITY_BARRIER request completed, bypassing no-preempt request\n",
1508 engine->name);
1509 err = -EINVAL;
1510 }
1511 i915_request_put(rq);
1512
1513 out_spin:
1514 igt_spinner_end(&spin);
1515 out_heartbeat:
1516 xchg(&engine->props.timeslice_duration_ms, timeslice);
1517 st_engine_heartbeat_enable(engine);
1518 if (err)
1519 break;
1520
1521 if (igt_flush_test(gt->i915)) {
1522 err = -EIO;
1523 break;
1524 }
1525 }
1526
1527 igt_spinner_fini(&spin);
1528 return err;
1529 }
1530
1531 static int live_busywait_preempt(void *arg)
1532 {
1533 struct intel_gt *gt = arg;
1534 struct i915_gem_context *ctx_hi, *ctx_lo;
1535 struct intel_engine_cs *engine;
1536 struct drm_i915_gem_object *obj;
1537 struct i915_vma *vma;
1538 enum intel_engine_id id;
1539 int err = -ENOMEM;
1540 u32 *map;
1541
1542 /*
1543 * Verify that even without HAS_LOGICAL_RING_PREEMPTION, we can
1544 * preempt the busywaits used to synchronise between rings.
1545 */
1546
1547 ctx_hi = kernel_context(gt->i915);
1548 if (!ctx_hi)
1549 return -ENOMEM;
1550 ctx_hi->sched.priority =
1551 I915_USER_PRIORITY(I915_CONTEXT_MAX_USER_PRIORITY);
1552
1553 ctx_lo = kernel_context(gt->i915);
1554 if (!ctx_lo)
1555 goto err_ctx_hi;
1556 ctx_lo->sched.priority =
1557 I915_USER_PRIORITY(I915_CONTEXT_MIN_USER_PRIORITY);
1558
1559 obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
1560 if (IS_ERR(obj)) {
1561 err = PTR_ERR(obj);
1562 goto err_ctx_lo;
1563 }
1564
1565 map = i915_gem_object_pin_map(obj, I915_MAP_WC);
1566 if (IS_ERR(map)) {
1567 err = PTR_ERR(map);
1568 goto err_obj;
1569 }
1570
1571 vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);
1572 if (IS_ERR(vma)) {
1573 err = PTR_ERR(vma);
1574 goto err_map;
1575 }
1576
1577 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
1578 if (err)
1579 goto err_map;
1580
1581 err = i915_vma_sync(vma);
1582 if (err)
1583 goto err_vma;
1584
1585 for_each_engine(engine, gt, id) {
1586 struct i915_request *lo, *hi;
1587 struct igt_live_test t;
1588 u32 *cs;
1589
1590 if (!intel_engine_has_preemption(engine))
1591 continue;
1592
1593 if (!intel_engine_can_store_dword(engine))
1594 continue;
1595
1596 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
1597 err = -EIO;
1598 goto err_vma;
1599 }
1600
1601 /*
1602 * We create two requests. The low priority request
1603 * busywaits on a semaphore (inside the ringbuffer where
1604 * is should be preemptible) and the high priority requests
1605 * uses a MI_STORE_DWORD_IMM to update the semaphore value
1606 * allowing the first request to complete. If preemption
1607 * fails, we hang instead.
1608 */
1609
1610 lo = igt_request_alloc(ctx_lo, engine);
1611 if (IS_ERR(lo)) {
1612 err = PTR_ERR(lo);
1613 goto err_vma;
1614 }
1615
1616 cs = intel_ring_begin(lo, 8);
1617 if (IS_ERR(cs)) {
1618 err = PTR_ERR(cs);
1619 i915_request_add(lo);
1620 goto err_vma;
1621 }
1622
1623 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
1624 *cs++ = i915_ggtt_offset(vma);
1625 *cs++ = 0;
1626 *cs++ = 1;
1627
1628 /* XXX Do we need a flush + invalidate here? */
1629
1630 *cs++ = MI_SEMAPHORE_WAIT |
1631 MI_SEMAPHORE_GLOBAL_GTT |
1632 MI_SEMAPHORE_POLL |
1633 MI_SEMAPHORE_SAD_EQ_SDD;
1634 *cs++ = 0;
1635 *cs++ = i915_ggtt_offset(vma);
1636 *cs++ = 0;
1637
1638 intel_ring_advance(lo, cs);
1639
1640 i915_request_get(lo);
1641 i915_request_add(lo);
1642
1643 if (wait_for(READ_ONCE(*map), 10)) {
1644 i915_request_put(lo);
1645 err = -ETIMEDOUT;
1646 goto err_vma;
1647 }
1648
1649 /* Low priority request should be busywaiting now */
1650 if (i915_request_wait(lo, 0, 1) != -ETIME) {
1651 i915_request_put(lo);
1652 pr_err("%s: Busywaiting request did not!\n",
1653 engine->name);
1654 err = -EIO;
1655 goto err_vma;
1656 }
1657
1658 hi = igt_request_alloc(ctx_hi, engine);
1659 if (IS_ERR(hi)) {
1660 err = PTR_ERR(hi);
1661 i915_request_put(lo);
1662 goto err_vma;
1663 }
1664
1665 cs = intel_ring_begin(hi, 4);
1666 if (IS_ERR(cs)) {
1667 err = PTR_ERR(cs);
1668 i915_request_add(hi);
1669 i915_request_put(lo);
1670 goto err_vma;
1671 }
1672
1673 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
1674 *cs++ = i915_ggtt_offset(vma);
1675 *cs++ = 0;
1676 *cs++ = 0;
1677
1678 intel_ring_advance(hi, cs);
1679 i915_request_add(hi);
1680
1681 if (i915_request_wait(lo, 0, HZ / 5) < 0) {
1682 struct drm_printer p = drm_info_printer(gt->i915->drm.dev);
1683
1684 pr_err("%s: Failed to preempt semaphore busywait!\n",
1685 engine->name);
1686
1687 intel_engine_dump(engine, &p, "%s\n", engine->name);
1688 GEM_TRACE_DUMP();
1689
1690 i915_request_put(lo);
1691 intel_gt_set_wedged(gt);
1692 err = -EIO;
1693 goto err_vma;
1694 }
1695 GEM_BUG_ON(READ_ONCE(*map));
1696 i915_request_put(lo);
1697
1698 if (igt_live_test_end(&t)) {
1699 err = -EIO;
1700 goto err_vma;
1701 }
1702 }
1703
1704 err = 0;
1705 err_vma:
1706 i915_vma_unpin(vma);
1707 err_map:
1708 i915_gem_object_unpin_map(obj);
1709 err_obj:
1710 i915_gem_object_put(obj);
1711 err_ctx_lo:
1712 kernel_context_close(ctx_lo);
1713 err_ctx_hi:
1714 kernel_context_close(ctx_hi);
1715 return err;
1716 }
1717
1718 static struct i915_request *
1719 spinner_create_request(struct igt_spinner *spin,
1720 struct i915_gem_context *ctx,
1721 struct intel_engine_cs *engine,
1722 u32 arb)
1723 {
1724 struct intel_context *ce;
1725 struct i915_request *rq;
1726
1727 ce = i915_gem_context_get_engine(ctx, engine->legacy_idx);
1728 if (IS_ERR(ce))
1729 return ERR_CAST(ce);
1730
1731 rq = igt_spinner_create_request(spin, ce, arb);
1732 intel_context_put(ce);
1733 return rq;
1734 }
1735
1736 static int live_preempt(void *arg)
1737 {
1738 struct intel_gt *gt = arg;
1739 struct i915_gem_context *ctx_hi, *ctx_lo;
1740 struct igt_spinner spin_hi, spin_lo;
1741 struct intel_engine_cs *engine;
1742 enum intel_engine_id id;
1743 int err = -ENOMEM;
1744
1745 if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
1746 return 0;
1747
1748 if (!(gt->i915->caps.scheduler & I915_SCHEDULER_CAP_PREEMPTION))
1749 pr_err("Logical preemption supported, but not exposed\n");
1750
1751 if (igt_spinner_init(&spin_hi, gt))
1752 return -ENOMEM;
1753
1754 if (igt_spinner_init(&spin_lo, gt))
1755 goto err_spin_hi;
1756
1757 ctx_hi = kernel_context(gt->i915);
1758 if (!ctx_hi)
1759 goto err_spin_lo;
1760 ctx_hi->sched.priority =
1761 I915_USER_PRIORITY(I915_CONTEXT_MAX_USER_PRIORITY);
1762
1763 ctx_lo = kernel_context(gt->i915);
1764 if (!ctx_lo)
1765 goto err_ctx_hi;
1766 ctx_lo->sched.priority =
1767 I915_USER_PRIORITY(I915_CONTEXT_MIN_USER_PRIORITY);
1768
1769 for_each_engine(engine, gt, id) {
1770 struct igt_live_test t;
1771 struct i915_request *rq;
1772
1773 if (!intel_engine_has_preemption(engine))
1774 continue;
1775
1776 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
1777 err = -EIO;
1778 goto err_ctx_lo;
1779 }
1780
1781 rq = spinner_create_request(&spin_lo, ctx_lo, engine,
1782 MI_ARB_CHECK);
1783 if (IS_ERR(rq)) {
1784 err = PTR_ERR(rq);
1785 goto err_ctx_lo;
1786 }
1787
1788 i915_request_add(rq);
1789 if (!igt_wait_for_spinner(&spin_lo, rq)) {
1790 GEM_TRACE("lo spinner failed to start\n");
1791 GEM_TRACE_DUMP();
1792 intel_gt_set_wedged(gt);
1793 err = -EIO;
1794 goto err_ctx_lo;
1795 }
1796
1797 rq = spinner_create_request(&spin_hi, ctx_hi, engine,
1798 MI_ARB_CHECK);
1799 if (IS_ERR(rq)) {
1800 igt_spinner_end(&spin_lo);
1801 err = PTR_ERR(rq);
1802 goto err_ctx_lo;
1803 }
1804
1805 i915_request_add(rq);
1806 if (!igt_wait_for_spinner(&spin_hi, rq)) {
1807 GEM_TRACE("hi spinner failed to start\n");
1808 GEM_TRACE_DUMP();
1809 intel_gt_set_wedged(gt);
1810 err = -EIO;
1811 goto err_ctx_lo;
1812 }
1813
1814 igt_spinner_end(&spin_hi);
1815 igt_spinner_end(&spin_lo);
1816
1817 if (igt_live_test_end(&t)) {
1818 err = -EIO;
1819 goto err_ctx_lo;
1820 }
1821 }
1822
1823 err = 0;
1824 err_ctx_lo:
1825 kernel_context_close(ctx_lo);
1826 err_ctx_hi:
1827 kernel_context_close(ctx_hi);
1828 err_spin_lo:
1829 igt_spinner_fini(&spin_lo);
1830 err_spin_hi:
1831 igt_spinner_fini(&spin_hi);
1832 return err;
1833 }
1834
1835 static int live_late_preempt(void *arg)
1836 {
1837 struct intel_gt *gt = arg;
1838 struct i915_gem_context *ctx_hi, *ctx_lo;
1839 struct igt_spinner spin_hi, spin_lo;
1840 struct intel_engine_cs *engine;
1841 struct i915_sched_attr attr = {};
1842 enum intel_engine_id id;
1843 int err = -ENOMEM;
1844
1845 if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
1846 return 0;
1847
1848 if (igt_spinner_init(&spin_hi, gt))
1849 return -ENOMEM;
1850
1851 if (igt_spinner_init(&spin_lo, gt))
1852 goto err_spin_hi;
1853
1854 ctx_hi = kernel_context(gt->i915);
1855 if (!ctx_hi)
1856 goto err_spin_lo;
1857
1858 ctx_lo = kernel_context(gt->i915);
1859 if (!ctx_lo)
1860 goto err_ctx_hi;
1861
1862 /* Make sure ctx_lo stays before ctx_hi until we trigger preemption. */
1863 ctx_lo->sched.priority = I915_USER_PRIORITY(1);
1864
1865 for_each_engine(engine, gt, id) {
1866 struct igt_live_test t;
1867 struct i915_request *rq;
1868
1869 if (!intel_engine_has_preemption(engine))
1870 continue;
1871
1872 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
1873 err = -EIO;
1874 goto err_ctx_lo;
1875 }
1876
1877 rq = spinner_create_request(&spin_lo, ctx_lo, engine,
1878 MI_ARB_CHECK);
1879 if (IS_ERR(rq)) {
1880 err = PTR_ERR(rq);
1881 goto err_ctx_lo;
1882 }
1883
1884 i915_request_add(rq);
1885 if (!igt_wait_for_spinner(&spin_lo, rq)) {
1886 pr_err("First context failed to start\n");
1887 goto err_wedged;
1888 }
1889
1890 rq = spinner_create_request(&spin_hi, ctx_hi, engine,
1891 MI_NOOP);
1892 if (IS_ERR(rq)) {
1893 igt_spinner_end(&spin_lo);
1894 err = PTR_ERR(rq);
1895 goto err_ctx_lo;
1896 }
1897
1898 i915_request_add(rq);
1899 if (igt_wait_for_spinner(&spin_hi, rq)) {
1900 pr_err("Second context overtook first?\n");
1901 goto err_wedged;
1902 }
1903
1904 attr.priority = I915_USER_PRIORITY(I915_PRIORITY_MAX);
1905 engine->schedule(rq, &attr);
1906
1907 if (!igt_wait_for_spinner(&spin_hi, rq)) {
1908 pr_err("High priority context failed to preempt the low priority context\n");
1909 GEM_TRACE_DUMP();
1910 goto err_wedged;
1911 }
1912
1913 igt_spinner_end(&spin_hi);
1914 igt_spinner_end(&spin_lo);
1915
1916 if (igt_live_test_end(&t)) {
1917 err = -EIO;
1918 goto err_ctx_lo;
1919 }
1920 }
1921
1922 err = 0;
1923 err_ctx_lo:
1924 kernel_context_close(ctx_lo);
1925 err_ctx_hi:
1926 kernel_context_close(ctx_hi);
1927 err_spin_lo:
1928 igt_spinner_fini(&spin_lo);
1929 err_spin_hi:
1930 igt_spinner_fini(&spin_hi);
1931 return err;
1932
1933 err_wedged:
1934 igt_spinner_end(&spin_hi);
1935 igt_spinner_end(&spin_lo);
1936 intel_gt_set_wedged(gt);
1937 err = -EIO;
1938 goto err_ctx_lo;
1939 }
1940
1941 struct preempt_client {
1942 struct igt_spinner spin;
1943 struct i915_gem_context *ctx;
1944 };
1945
1946 static int preempt_client_init(struct intel_gt *gt, struct preempt_client *c)
1947 {
1948 c->ctx = kernel_context(gt->i915);
1949 if (!c->ctx)
1950 return -ENOMEM;
1951
1952 if (igt_spinner_init(&c->spin, gt))
1953 goto err_ctx;
1954
1955 return 0;
1956
1957 err_ctx:
1958 kernel_context_close(c->ctx);
1959 return -ENOMEM;
1960 }
1961
1962 static void preempt_client_fini(struct preempt_client *c)
1963 {
1964 igt_spinner_fini(&c->spin);
1965 kernel_context_close(c->ctx);
1966 }
1967
1968 static int live_nopreempt(void *arg)
1969 {
1970 struct intel_gt *gt = arg;
1971 struct intel_engine_cs *engine;
1972 struct preempt_client a, b;
1973 enum intel_engine_id id;
1974 int err = -ENOMEM;
1975
1976 /*
1977 * Verify that we can disable preemption for an individual request
1978 * that may be being observed and not want to be interrupted.
1979 */
1980
1981 if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
1982 return 0;
1983
1984 if (preempt_client_init(gt, &a))
1985 return -ENOMEM;
1986 if (preempt_client_init(gt, &b))
1987 goto err_client_a;
1988 b.ctx->sched.priority = I915_USER_PRIORITY(I915_PRIORITY_MAX);
1989
1990 for_each_engine(engine, gt, id) {
1991 struct i915_request *rq_a, *rq_b;
1992
1993 if (!intel_engine_has_preemption(engine))
1994 continue;
1995
1996 engine->execlists.preempt_hang.count = 0;
1997
1998 rq_a = spinner_create_request(&a.spin,
1999 a.ctx, engine,
2000 MI_ARB_CHECK);
2001 if (IS_ERR(rq_a)) {
2002 err = PTR_ERR(rq_a);
2003 goto err_client_b;
2004 }
2005
2006 /* Low priority client, but unpreemptable! */
2007 __set_bit(I915_FENCE_FLAG_NOPREEMPT, &rq_a->fence.flags);
2008
2009 i915_request_add(rq_a);
2010 if (!igt_wait_for_spinner(&a.spin, rq_a)) {
2011 pr_err("First client failed to start\n");
2012 goto err_wedged;
2013 }
2014
2015 rq_b = spinner_create_request(&b.spin,
2016 b.ctx, engine,
2017 MI_ARB_CHECK);
2018 if (IS_ERR(rq_b)) {
2019 err = PTR_ERR(rq_b);
2020 goto err_client_b;
2021 }
2022
2023 i915_request_add(rq_b);
2024
2025 /* B is much more important than A! (But A is unpreemptable.) */
2026 GEM_BUG_ON(rq_prio(rq_b) <= rq_prio(rq_a));
2027
2028 /* Wait long enough for preemption and timeslicing */
2029 if (igt_wait_for_spinner(&b.spin, rq_b)) {
2030 pr_err("Second client started too early!\n");
2031 goto err_wedged;
2032 }
2033
2034 igt_spinner_end(&a.spin);
2035
2036 if (!igt_wait_for_spinner(&b.spin, rq_b)) {
2037 pr_err("Second client failed to start\n");
2038 goto err_wedged;
2039 }
2040
2041 igt_spinner_end(&b.spin);
2042
2043 if (engine->execlists.preempt_hang.count) {
2044 pr_err("Preemption recorded x%d; should have been suppressed!\n",
2045 engine->execlists.preempt_hang.count);
2046 err = -EINVAL;
2047 goto err_wedged;
2048 }
2049
2050 if (igt_flush_test(gt->i915))
2051 goto err_wedged;
2052 }
2053
2054 err = 0;
2055 err_client_b:
2056 preempt_client_fini(&b);
2057 err_client_a:
2058 preempt_client_fini(&a);
2059 return err;
2060
2061 err_wedged:
2062 igt_spinner_end(&b.spin);
2063 igt_spinner_end(&a.spin);
2064 intel_gt_set_wedged(gt);
2065 err = -EIO;
2066 goto err_client_b;
2067 }
2068
2069 struct live_preempt_cancel {
2070 struct intel_engine_cs *engine;
2071 struct preempt_client a, b;
2072 };
2073
2074 static int __cancel_active0(struct live_preempt_cancel *arg)
2075 {
2076 struct i915_request *rq;
2077 struct igt_live_test t;
2078 int err;
2079
2080 /* Preempt cancel of ELSP0 */
2081 GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);
2082 if (igt_live_test_begin(&t, arg->engine->i915,
2083 __func__, arg->engine->name))
2084 return -EIO;
2085
2086 rq = spinner_create_request(&arg->a.spin,
2087 arg->a.ctx, arg->engine,
2088 MI_ARB_CHECK);
2089 if (IS_ERR(rq))
2090 return PTR_ERR(rq);
2091
2092 clear_bit(CONTEXT_BANNED, &rq->context->flags);
2093 i915_request_get(rq);
2094 i915_request_add(rq);
2095 if (!igt_wait_for_spinner(&arg->a.spin, rq)) {
2096 err = -EIO;
2097 goto out;
2098 }
2099
2100 intel_context_set_banned(rq->context);
2101 err = intel_engine_pulse(arg->engine);
2102 if (err)
2103 goto out;
2104
2105 err = wait_for_reset(arg->engine, rq, HZ / 2);
2106 if (err) {
2107 pr_err("Cancelled inflight0 request did not reset\n");
2108 goto out;
2109 }
2110
2111 out:
2112 i915_request_put(rq);
2113 if (igt_live_test_end(&t))
2114 err = -EIO;
2115 return err;
2116 }
2117
2118 static int __cancel_active1(struct live_preempt_cancel *arg)
2119 {
2120 struct i915_request *rq[2] = {};
2121 struct igt_live_test t;
2122 int err;
2123
2124 /* Preempt cancel of ELSP1 */
2125 GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);
2126 if (igt_live_test_begin(&t, arg->engine->i915,
2127 __func__, arg->engine->name))
2128 return -EIO;
2129
2130 rq[0] = spinner_create_request(&arg->a.spin,
2131 arg->a.ctx, arg->engine,
2132 MI_NOOP); /* no preemption */
2133 if (IS_ERR(rq[0]))
2134 return PTR_ERR(rq[0]);
2135
2136 clear_bit(CONTEXT_BANNED, &rq[0]->context->flags);
2137 i915_request_get(rq[0]);
2138 i915_request_add(rq[0]);
2139 if (!igt_wait_for_spinner(&arg->a.spin, rq[0])) {
2140 err = -EIO;
2141 goto out;
2142 }
2143
2144 rq[1] = spinner_create_request(&arg->b.spin,
2145 arg->b.ctx, arg->engine,
2146 MI_ARB_CHECK);
2147 if (IS_ERR(rq[1])) {
2148 err = PTR_ERR(rq[1]);
2149 goto out;
2150 }
2151
2152 clear_bit(CONTEXT_BANNED, &rq[1]->context->flags);
2153 i915_request_get(rq[1]);
2154 err = i915_request_await_dma_fence(rq[1], &rq[0]->fence);
2155 i915_request_add(rq[1]);
2156 if (err)
2157 goto out;
2158
2159 intel_context_set_banned(rq[1]->context);
2160 err = intel_engine_pulse(arg->engine);
2161 if (err)
2162 goto out;
2163
2164 igt_spinner_end(&arg->a.spin);
2165 err = wait_for_reset(arg->engine, rq[1], HZ / 2);
2166 if (err)
2167 goto out;
2168
2169 if (rq[0]->fence.error != 0) {
2170 pr_err("Normal inflight0 request did not complete\n");
2171 err = -EINVAL;
2172 goto out;
2173 }
2174
2175 if (rq[1]->fence.error != -EIO) {
2176 pr_err("Cancelled inflight1 request did not report -EIO\n");
2177 err = -EINVAL;
2178 goto out;
2179 }
2180
2181 out:
2182 i915_request_put(rq[1]);
2183 i915_request_put(rq[0]);
2184 if (igt_live_test_end(&t))
2185 err = -EIO;
2186 return err;
2187 }
2188
2189 static int __cancel_queued(struct live_preempt_cancel *arg)
2190 {
2191 struct i915_request *rq[3] = {};
2192 struct igt_live_test t;
2193 int err;
2194
2195 /* Full ELSP and one in the wings */
2196 GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);
2197 if (igt_live_test_begin(&t, arg->engine->i915,
2198 __func__, arg->engine->name))
2199 return -EIO;
2200
2201 rq[0] = spinner_create_request(&arg->a.spin,
2202 arg->a.ctx, arg->engine,
2203 MI_ARB_CHECK);
2204 if (IS_ERR(rq[0]))
2205 return PTR_ERR(rq[0]);
2206
2207 clear_bit(CONTEXT_BANNED, &rq[0]->context->flags);
2208 i915_request_get(rq[0]);
2209 i915_request_add(rq[0]);
2210 if (!igt_wait_for_spinner(&arg->a.spin, rq[0])) {
2211 err = -EIO;
2212 goto out;
2213 }
2214
2215 rq[1] = igt_request_alloc(arg->b.ctx, arg->engine);
2216 if (IS_ERR(rq[1])) {
2217 err = PTR_ERR(rq[1]);
2218 goto out;
2219 }
2220
2221 clear_bit(CONTEXT_BANNED, &rq[1]->context->flags);
2222 i915_request_get(rq[1]);
2223 err = i915_request_await_dma_fence(rq[1], &rq[0]->fence);
2224 i915_request_add(rq[1]);
2225 if (err)
2226 goto out;
2227
2228 rq[2] = spinner_create_request(&arg->b.spin,
2229 arg->a.ctx, arg->engine,
2230 MI_ARB_CHECK);
2231 if (IS_ERR(rq[2])) {
2232 err = PTR_ERR(rq[2]);
2233 goto out;
2234 }
2235
2236 i915_request_get(rq[2]);
2237 err = i915_request_await_dma_fence(rq[2], &rq[1]->fence);
2238 i915_request_add(rq[2]);
2239 if (err)
2240 goto out;
2241
2242 intel_context_set_banned(rq[2]->context);
2243 err = intel_engine_pulse(arg->engine);
2244 if (err)
2245 goto out;
2246
2247 err = wait_for_reset(arg->engine, rq[2], HZ / 2);
2248 if (err)
2249 goto out;
2250
2251 if (rq[0]->fence.error != -EIO) {
2252 pr_err("Cancelled inflight0 request did not report -EIO\n");
2253 err = -EINVAL;
2254 goto out;
2255 }
2256
2257 if (rq[1]->fence.error != 0) {
2258 pr_err("Normal inflight1 request did not complete\n");
2259 err = -EINVAL;
2260 goto out;
2261 }
2262
2263 if (rq[2]->fence.error != -EIO) {
2264 pr_err("Cancelled queued request did not report -EIO\n");
2265 err = -EINVAL;
2266 goto out;
2267 }
2268
2269 out:
2270 i915_request_put(rq[2]);
2271 i915_request_put(rq[1]);
2272 i915_request_put(rq[0]);
2273 if (igt_live_test_end(&t))
2274 err = -EIO;
2275 return err;
2276 }
2277
2278 static int __cancel_hostile(struct live_preempt_cancel *arg)
2279 {
2280 struct i915_request *rq;
2281 int err;
2282
2283 /* Preempt cancel non-preemptible spinner in ELSP0 */
2284 if (!IS_ACTIVE(CONFIG_DRM_I915_PREEMPT_TIMEOUT))
2285 return 0;
2286
2287 if (!intel_has_reset_engine(arg->engine->gt))
2288 return 0;
2289
2290 GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);
2291 rq = spinner_create_request(&arg->a.spin,
2292 arg->a.ctx, arg->engine,
2293 MI_NOOP); /* preemption disabled */
2294 if (IS_ERR(rq))
2295 return PTR_ERR(rq);
2296
2297 clear_bit(CONTEXT_BANNED, &rq->context->flags);
2298 i915_request_get(rq);
2299 i915_request_add(rq);
2300 if (!igt_wait_for_spinner(&arg->a.spin, rq)) {
2301 err = -EIO;
2302 goto out;
2303 }
2304
2305 intel_context_set_banned(rq->context);
2306 err = intel_engine_pulse(arg->engine); /* force reset */
2307 if (err)
2308 goto out;
2309
2310 err = wait_for_reset(arg->engine, rq, HZ / 2);
2311 if (err) {
2312 pr_err("Cancelled inflight0 request did not reset\n");
2313 goto out;
2314 }
2315
2316 out:
2317 i915_request_put(rq);
2318 if (igt_flush_test(arg->engine->i915))
2319 err = -EIO;
2320 return err;
2321 }
2322
2323 static int live_preempt_cancel(void *arg)
2324 {
2325 struct intel_gt *gt = arg;
2326 struct live_preempt_cancel data;
2327 enum intel_engine_id id;
2328 int err = -ENOMEM;
2329
2330 /*
2331 * To cancel an inflight context, we need to first remove it from the
2332 * GPU. That sounds like preemption! Plus a little bit of bookkeeping.
2333 */
2334
2335 if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
2336 return 0;
2337
2338 if (preempt_client_init(gt, &data.a))
2339 return -ENOMEM;
2340 if (preempt_client_init(gt, &data.b))
2341 goto err_client_a;
2342
2343 for_each_engine(data.engine, gt, id) {
2344 if (!intel_engine_has_preemption(data.engine))
2345 continue;
2346
2347 err = __cancel_active0(&data);
2348 if (err)
2349 goto err_wedged;
2350
2351 err = __cancel_active1(&data);
2352 if (err)
2353 goto err_wedged;
2354
2355 err = __cancel_queued(&data);
2356 if (err)
2357 goto err_wedged;
2358
2359 err = __cancel_hostile(&data);
2360 if (err)
2361 goto err_wedged;
2362 }
2363
2364 err = 0;
2365 err_client_b:
2366 preempt_client_fini(&data.b);
2367 err_client_a:
2368 preempt_client_fini(&data.a);
2369 return err;
2370
2371 err_wedged:
2372 GEM_TRACE_DUMP();
2373 igt_spinner_end(&data.b.spin);
2374 igt_spinner_end(&data.a.spin);
2375 intel_gt_set_wedged(gt);
2376 goto err_client_b;
2377 }
2378
2379 static int live_suppress_self_preempt(void *arg)
2380 {
2381 struct intel_gt *gt = arg;
2382 struct intel_engine_cs *engine;
2383 struct i915_sched_attr attr = {
2384 .priority = I915_USER_PRIORITY(I915_PRIORITY_MAX)
2385 };
2386 struct preempt_client a, b;
2387 enum intel_engine_id id;
2388 int err = -ENOMEM;
2389
2390 /*
2391 * Verify that if a preemption request does not cause a change in
2392 * the current execution order, the preempt-to-idle injection is
2393 * skipped and that we do not accidentally apply it after the CS
2394 * completion event.
2395 */
2396
2397 if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
2398 return 0;
2399
2400 if (intel_uc_uses_guc_submission(&gt->uc))
2401 return 0; /* presume black blox */
2402
2403 if (intel_vgpu_active(gt->i915))
2404 return 0; /* GVT forces single port & request submission */
2405
2406 if (preempt_client_init(gt, &a))
2407 return -ENOMEM;
2408 if (preempt_client_init(gt, &b))
2409 goto err_client_a;
2410
2411 for_each_engine(engine, gt, id) {
2412 struct i915_request *rq_a, *rq_b;
2413 int depth;
2414
2415 if (!intel_engine_has_preemption(engine))
2416 continue;
2417
2418 if (igt_flush_test(gt->i915))
2419 goto err_wedged;
2420
2421 st_engine_heartbeat_disable(engine);
2422 engine->execlists.preempt_hang.count = 0;
2423
2424 rq_a = spinner_create_request(&a.spin,
2425 a.ctx, engine,
2426 MI_NOOP);
2427 if (IS_ERR(rq_a)) {
2428 err = PTR_ERR(rq_a);
2429 st_engine_heartbeat_enable(engine);
2430 goto err_client_b;
2431 }
2432
2433 i915_request_add(rq_a);
2434 if (!igt_wait_for_spinner(&a.spin, rq_a)) {
2435 pr_err("First client failed to start\n");
2436 st_engine_heartbeat_enable(engine);
2437 goto err_wedged;
2438 }
2439
2440 /* Keep postponing the timer to avoid premature slicing */
2441 mod_timer(&engine->execlists.timer, jiffies + HZ);
2442 for (depth = 0; depth < 8; depth++) {
2443 rq_b = spinner_create_request(&b.spin,
2444 b.ctx, engine,
2445 MI_NOOP);
2446 if (IS_ERR(rq_b)) {
2447 err = PTR_ERR(rq_b);
2448 st_engine_heartbeat_enable(engine);
2449 goto err_client_b;
2450 }
2451 i915_request_add(rq_b);
2452
2453 GEM_BUG_ON(i915_request_completed(rq_a));
2454 engine->schedule(rq_a, &attr);
2455 igt_spinner_end(&a.spin);
2456
2457 if (!igt_wait_for_spinner(&b.spin, rq_b)) {
2458 pr_err("Second client failed to start\n");
2459 st_engine_heartbeat_enable(engine);
2460 goto err_wedged;
2461 }
2462
2463 swap(a, b);
2464 rq_a = rq_b;
2465 }
2466 igt_spinner_end(&a.spin);
2467
2468 if (engine->execlists.preempt_hang.count) {
2469 pr_err("Preemption on %s recorded x%d, depth %d; should have been suppressed!\n",
2470 engine->name,
2471 engine->execlists.preempt_hang.count,
2472 depth);
2473 st_engine_heartbeat_enable(engine);
2474 err = -EINVAL;
2475 goto err_client_b;
2476 }
2477
2478 st_engine_heartbeat_enable(engine);
2479 if (igt_flush_test(gt->i915))
2480 goto err_wedged;
2481 }
2482
2483 err = 0;
2484 err_client_b:
2485 preempt_client_fini(&b);
2486 err_client_a:
2487 preempt_client_fini(&a);
2488 return err;
2489
2490 err_wedged:
2491 igt_spinner_end(&b.spin);
2492 igt_spinner_end(&a.spin);
2493 intel_gt_set_wedged(gt);
2494 err = -EIO;
2495 goto err_client_b;
2496 }
2497
2498 static int live_chain_preempt(void *arg)
2499 {
2500 struct intel_gt *gt = arg;
2501 struct intel_engine_cs *engine;
2502 struct preempt_client hi, lo;
2503 enum intel_engine_id id;
2504 int err = -ENOMEM;
2505
2506 /*
2507 * Build a chain AB...BA between two contexts (A, B) and request
2508 * preemption of the last request. It should then complete before
2509 * the previously submitted spinner in B.
2510 */
2511
2512 if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
2513 return 0;
2514
2515 if (preempt_client_init(gt, &hi))
2516 return -ENOMEM;
2517
2518 if (preempt_client_init(gt, &lo))
2519 goto err_client_hi;
2520
2521 for_each_engine(engine, gt, id) {
2522 struct i915_sched_attr attr = {
2523 .priority = I915_USER_PRIORITY(I915_PRIORITY_MAX),
2524 };
2525 struct igt_live_test t;
2526 struct i915_request *rq;
2527 int ring_size, count, i;
2528
2529 if (!intel_engine_has_preemption(engine))
2530 continue;
2531
2532 rq = spinner_create_request(&lo.spin,
2533 lo.ctx, engine,
2534 MI_ARB_CHECK);
2535 if (IS_ERR(rq))
2536 goto err_wedged;
2537
2538 i915_request_get(rq);
2539 i915_request_add(rq);
2540
2541 ring_size = rq->wa_tail - rq->head;
2542 if (ring_size < 0)
2543 ring_size += rq->ring->size;
2544 ring_size = rq->ring->size / ring_size;
2545 pr_debug("%s(%s): Using maximum of %d requests\n",
2546 __func__, engine->name, ring_size);
2547
2548 igt_spinner_end(&lo.spin);
2549 if (i915_request_wait(rq, 0, HZ / 2) < 0) {
2550 pr_err("Timed out waiting to flush %s\n", engine->name);
2551 i915_request_put(rq);
2552 goto err_wedged;
2553 }
2554 i915_request_put(rq);
2555
2556 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
2557 err = -EIO;
2558 goto err_wedged;
2559 }
2560
2561 for_each_prime_number_from(count, 1, ring_size) {
2562 rq = spinner_create_request(&hi.spin,
2563 hi.ctx, engine,
2564 MI_ARB_CHECK);
2565 if (IS_ERR(rq))
2566 goto err_wedged;
2567 i915_request_add(rq);
2568 if (!igt_wait_for_spinner(&hi.spin, rq))
2569 goto err_wedged;
2570
2571 rq = spinner_create_request(&lo.spin,
2572 lo.ctx, engine,
2573 MI_ARB_CHECK);
2574 if (IS_ERR(rq))
2575 goto err_wedged;
2576 i915_request_add(rq);
2577
2578 for (i = 0; i < count; i++) {
2579 rq = igt_request_alloc(lo.ctx, engine);
2580 if (IS_ERR(rq))
2581 goto err_wedged;
2582 i915_request_add(rq);
2583 }
2584
2585 rq = igt_request_alloc(hi.ctx, engine);
2586 if (IS_ERR(rq))
2587 goto err_wedged;
2588
2589 i915_request_get(rq);
2590 i915_request_add(rq);
2591 engine->schedule(rq, &attr);
2592
2593 igt_spinner_end(&hi.spin);
2594 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
2595 struct drm_printer p =
2596 drm_info_printer(gt->i915->drm.dev);
2597
2598 pr_err("Failed to preempt over chain of %d\n",
2599 count);
2600 intel_engine_dump(engine, &p,
2601 "%s\n", engine->name);
2602 i915_request_put(rq);
2603 goto err_wedged;
2604 }
2605 igt_spinner_end(&lo.spin);
2606 i915_request_put(rq);
2607
2608 rq = igt_request_alloc(lo.ctx, engine);
2609 if (IS_ERR(rq))
2610 goto err_wedged;
2611
2612 i915_request_get(rq);
2613 i915_request_add(rq);
2614
2615 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
2616 struct drm_printer p =
2617 drm_info_printer(gt->i915->drm.dev);
2618
2619 pr_err("Failed to flush low priority chain of %d requests\n",
2620 count);
2621 intel_engine_dump(engine, &p,
2622 "%s\n", engine->name);
2623
2624 i915_request_put(rq);
2625 goto err_wedged;
2626 }
2627 i915_request_put(rq);
2628 }
2629
2630 if (igt_live_test_end(&t)) {
2631 err = -EIO;
2632 goto err_wedged;
2633 }
2634 }
2635
2636 err = 0;
2637 err_client_lo:
2638 preempt_client_fini(&lo);
2639 err_client_hi:
2640 preempt_client_fini(&hi);
2641 return err;
2642
2643 err_wedged:
2644 igt_spinner_end(&hi.spin);
2645 igt_spinner_end(&lo.spin);
2646 intel_gt_set_wedged(gt);
2647 err = -EIO;
2648 goto err_client_lo;
2649 }
2650
2651 static int create_gang(struct intel_engine_cs *engine,
2652 struct i915_request **prev)
2653 {
2654 struct drm_i915_gem_object *obj;
2655 struct intel_context *ce;
2656 struct i915_request *rq;
2657 struct i915_vma *vma;
2658 u32 *cs;
2659 int err;
2660
2661 ce = intel_context_create(engine);
2662 if (IS_ERR(ce))
2663 return PTR_ERR(ce);
2664
2665 obj = i915_gem_object_create_internal(engine->i915, 4096);
2666 if (IS_ERR(obj)) {
2667 err = PTR_ERR(obj);
2668 goto err_ce;
2669 }
2670
2671 vma = i915_vma_instance(obj, ce->vm, NULL);
2672 if (IS_ERR(vma)) {
2673 err = PTR_ERR(vma);
2674 goto err_obj;
2675 }
2676
2677 err = i915_vma_pin(vma, 0, 0, PIN_USER);
2678 if (err)
2679 goto err_obj;
2680
2681 cs = i915_gem_object_pin_map(obj, I915_MAP_WC);
2682 if (IS_ERR(cs))
2683 goto err_obj;
2684
2685 /* Semaphore target: spin until zero */
2686 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
2687
2688 *cs++ = MI_SEMAPHORE_WAIT |
2689 MI_SEMAPHORE_POLL |
2690 MI_SEMAPHORE_SAD_EQ_SDD;
2691 *cs++ = 0;
2692 *cs++ = lower_32_bits(vma->node.start);
2693 *cs++ = upper_32_bits(vma->node.start);
2694
2695 if (*prev) {
2696 u64 offset = (*prev)->batch->node.start;
2697
2698 /* Terminate the spinner in the next lower priority batch. */
2699 *cs++ = MI_STORE_DWORD_IMM_GEN4;
2700 *cs++ = lower_32_bits(offset);
2701 *cs++ = upper_32_bits(offset);
2702 *cs++ = 0;
2703 }
2704
2705 *cs++ = MI_BATCH_BUFFER_END;
2706 i915_gem_object_flush_map(obj);
2707 i915_gem_object_unpin_map(obj);
2708
2709 rq = intel_context_create_request(ce);
2710 if (IS_ERR(rq))
2711 goto err_obj;
2712
2713 rq->batch = i915_vma_get(vma);
2714 i915_request_get(rq);
2715
2716 i915_vma_lock(vma);
2717 err = i915_request_await_object(rq, vma->obj, false);
2718 if (!err)
2719 err = i915_vma_move_to_active(vma, rq, 0);
2720 if (!err)
2721 err = rq->engine->emit_bb_start(rq,
2722 vma->node.start,
2723 PAGE_SIZE, 0);
2724 i915_vma_unlock(vma);
2725 i915_request_add(rq);
2726 if (err)
2727 goto err_rq;
2728
2729 i915_gem_object_put(obj);
2730 intel_context_put(ce);
2731
2732 rq->mock.link.next = &(*prev)->mock.link;
2733 *prev = rq;
2734 return 0;
2735
2736 err_rq:
2737 i915_vma_put(rq->batch);
2738 i915_request_put(rq);
2739 err_obj:
2740 i915_gem_object_put(obj);
2741 err_ce:
2742 intel_context_put(ce);
2743 return err;
2744 }
2745
2746 static int __live_preempt_ring(struct intel_engine_cs *engine,
2747 struct igt_spinner *spin,
2748 int queue_sz, int ring_sz)
2749 {
2750 struct intel_context *ce[2] = {};
2751 struct i915_request *rq;
2752 struct igt_live_test t;
2753 int err = 0;
2754 int n;
2755
2756 if (igt_live_test_begin(&t, engine->i915, __func__, engine->name))
2757 return -EIO;
2758
2759 for (n = 0; n < ARRAY_SIZE(ce); n++) {
2760 struct intel_context *tmp;
2761
2762 tmp = intel_context_create(engine);
2763 if (IS_ERR(tmp)) {
2764 err = PTR_ERR(tmp);
2765 goto err_ce;
2766 }
2767
2768 tmp->ring = __intel_context_ring_size(ring_sz);
2769
2770 err = intel_context_pin(tmp);
2771 if (err) {
2772 intel_context_put(tmp);
2773 goto err_ce;
2774 }
2775
2776 memset32(tmp->ring->vaddr,
2777 0xdeadbeef, /* trigger a hang if executed */
2778 tmp->ring->vma->size / sizeof(u32));
2779
2780 ce[n] = tmp;
2781 }
2782
2783 rq = igt_spinner_create_request(spin, ce[0], MI_ARB_CHECK);
2784 if (IS_ERR(rq)) {
2785 err = PTR_ERR(rq);
2786 goto err_ce;
2787 }
2788
2789 i915_request_get(rq);
2790 rq->sched.attr.priority = I915_PRIORITY_BARRIER;
2791 i915_request_add(rq);
2792
2793 if (!igt_wait_for_spinner(spin, rq)) {
2794 intel_gt_set_wedged(engine->gt);
2795 i915_request_put(rq);
2796 err = -ETIME;
2797 goto err_ce;
2798 }
2799
2800 /* Fill the ring, until we will cause a wrap */
2801 n = 0;
2802 while (ce[0]->ring->tail - rq->wa_tail <= queue_sz) {
2803 struct i915_request *tmp;
2804
2805 tmp = intel_context_create_request(ce[0]);
2806 if (IS_ERR(tmp)) {
2807 err = PTR_ERR(tmp);
2808 i915_request_put(rq);
2809 goto err_ce;
2810 }
2811
2812 i915_request_add(tmp);
2813 intel_engine_flush_submission(engine);
2814 n++;
2815 }
2816 intel_engine_flush_submission(engine);
2817 pr_debug("%s: Filled %d with %d nop tails {size:%x, tail:%x, emit:%x, rq.tail:%x}\n",
2818 engine->name, queue_sz, n,
2819 ce[0]->ring->size,
2820 ce[0]->ring->tail,
2821 ce[0]->ring->emit,
2822 rq->tail);
2823 i915_request_put(rq);
2824
2825 /* Create a second request to preempt the first ring */
2826 rq = intel_context_create_request(ce[1]);
2827 if (IS_ERR(rq)) {
2828 err = PTR_ERR(rq);
2829 goto err_ce;
2830 }
2831
2832 rq->sched.attr.priority = I915_PRIORITY_BARRIER;
2833 i915_request_get(rq);
2834 i915_request_add(rq);
2835
2836 err = wait_for_submit(engine, rq, HZ / 2);
2837 i915_request_put(rq);
2838 if (err) {
2839 pr_err("%s: preemption request was not submited\n",
2840 engine->name);
2841 err = -ETIME;
2842 }
2843
2844 pr_debug("%s: ring[0]:{ tail:%x, emit:%x }, ring[1]:{ tail:%x, emit:%x }\n",
2845 engine->name,
2846 ce[0]->ring->tail, ce[0]->ring->emit,
2847 ce[1]->ring->tail, ce[1]->ring->emit);
2848
2849 err_ce:
2850 intel_engine_flush_submission(engine);
2851 igt_spinner_end(spin);
2852 for (n = 0; n < ARRAY_SIZE(ce); n++) {
2853 if (IS_ERR_OR_NULL(ce[n]))
2854 break;
2855
2856 intel_context_unpin(ce[n]);
2857 intel_context_put(ce[n]);
2858 }
2859 if (igt_live_test_end(&t))
2860 err = -EIO;
2861 return err;
2862 }
2863
2864 static int live_preempt_ring(void *arg)
2865 {
2866 struct intel_gt *gt = arg;
2867 struct intel_engine_cs *engine;
2868 struct igt_spinner spin;
2869 enum intel_engine_id id;
2870 int err = 0;
2871
2872 /*
2873 * Check that we rollback large chunks of a ring in order to do a
2874 * preemption event. Similar to live_unlite_ring, but looking at
2875 * ring size rather than the impact of intel_ring_direction().
2876 */
2877
2878 if (igt_spinner_init(&spin, gt))
2879 return -ENOMEM;
2880
2881 for_each_engine(engine, gt, id) {
2882 int n;
2883
2884 if (!intel_engine_has_preemption(engine))
2885 continue;
2886
2887 if (!intel_engine_can_store_dword(engine))
2888 continue;
2889
2890 st_engine_heartbeat_disable(engine);
2891
2892 for (n = 0; n <= 3; n++) {
2893 err = __live_preempt_ring(engine, &spin,
2894 n * SZ_4K / 4, SZ_4K);
2895 if (err)
2896 break;
2897 }
2898
2899 st_engine_heartbeat_enable(engine);
2900 if (err)
2901 break;
2902 }
2903
2904 igt_spinner_fini(&spin);
2905 return err;
2906 }
2907
2908 static int live_preempt_gang(void *arg)
2909 {
2910 struct intel_gt *gt = arg;
2911 struct intel_engine_cs *engine;
2912 enum intel_engine_id id;
2913
2914 if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
2915 return 0;
2916
2917 /*
2918 * Build as long a chain of preempters as we can, with each
2919 * request higher priority than the last. Once we are ready, we release
2920 * the last batch which then precolates down the chain, each releasing
2921 * the next oldest in turn. The intent is to simply push as hard as we
2922 * can with the number of preemptions, trying to exceed narrow HW
2923 * limits. At a minimum, we insist that we can sort all the user
2924 * high priority levels into execution order.
2925 */
2926
2927 for_each_engine(engine, gt, id) {
2928 struct i915_request *rq = NULL;
2929 struct igt_live_test t;
2930 IGT_TIMEOUT(end_time);
2931 int prio = 0;
2932 int err = 0;
2933 u32 *cs;
2934
2935 if (!intel_engine_has_preemption(engine))
2936 continue;
2937
2938 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name))
2939 return -EIO;
2940
2941 do {
2942 struct i915_sched_attr attr = {
2943 .priority = I915_USER_PRIORITY(prio++),
2944 };
2945
2946 err = create_gang(engine, &rq);
2947 if (err)
2948 break;
2949
2950 /* Submit each spinner at increasing priority */
2951 engine->schedule(rq, &attr);
2952 } while (prio <= I915_PRIORITY_MAX &&
2953 !__igt_timeout(end_time, NULL));
2954 pr_debug("%s: Preempt chain of %d requests\n",
2955 engine->name, prio);
2956
2957 /*
2958 * Such that the last spinner is the highest priority and
2959 * should execute first. When that spinner completes,
2960 * it will terminate the next lowest spinner until there
2961 * are no more spinners and the gang is complete.
2962 */
2963 cs = i915_gem_object_pin_map(rq->batch->obj, I915_MAP_WC);
2964 if (!IS_ERR(cs)) {
2965 *cs = 0;
2966 i915_gem_object_unpin_map(rq->batch->obj);
2967 } else {
2968 err = PTR_ERR(cs);
2969 intel_gt_set_wedged(gt);
2970 }
2971
2972 while (rq) { /* wait for each rq from highest to lowest prio */
2973 struct i915_request *n = list_next_entry(rq, mock.link);
2974
2975 if (err == 0 && i915_request_wait(rq, 0, HZ / 5) < 0) {
2976 struct drm_printer p =
2977 drm_info_printer(engine->i915->drm.dev);
2978
2979 pr_err("Failed to flush chain of %d requests, at %d\n",
2980 prio, rq_prio(rq) >> I915_USER_PRIORITY_SHIFT);
2981 intel_engine_dump(engine, &p,
2982 "%s\n", engine->name);
2983
2984 err = -ETIME;
2985 }
2986
2987 i915_vma_put(rq->batch);
2988 i915_request_put(rq);
2989 rq = n;
2990 }
2991
2992 if (igt_live_test_end(&t))
2993 err = -EIO;
2994 if (err)
2995 return err;
2996 }
2997
2998 return 0;
2999 }
3000
3001 static struct i915_vma *
3002 create_gpr_user(struct intel_engine_cs *engine,
3003 struct i915_vma *result,
3004 unsigned int offset)
3005 {
3006 struct drm_i915_gem_object *obj;
3007 struct i915_vma *vma;
3008 u32 *cs;
3009 int err;
3010 int i;
3011
3012 obj = i915_gem_object_create_internal(engine->i915, 4096);
3013 if (IS_ERR(obj))
3014 return ERR_CAST(obj);
3015
3016 vma = i915_vma_instance(obj, result->vm, NULL);
3017 if (IS_ERR(vma)) {
3018 i915_gem_object_put(obj);
3019 return vma;
3020 }
3021
3022 err = i915_vma_pin(vma, 0, 0, PIN_USER);
3023 if (err) {
3024 i915_vma_put(vma);
3025 return ERR_PTR(err);
3026 }
3027
3028 cs = i915_gem_object_pin_map(obj, I915_MAP_WC);
3029 if (IS_ERR(cs)) {
3030 i915_vma_put(vma);
3031 return ERR_CAST(cs);
3032 }
3033
3034 /* All GPR are clear for new contexts. We use GPR(0) as a constant */
3035 *cs++ = MI_LOAD_REGISTER_IMM(1);
3036 *cs++ = CS_GPR(engine, 0);
3037 *cs++ = 1;
3038
3039 for (i = 1; i < NUM_GPR; i++) {
3040 u64 addr;
3041
3042 /*
3043 * Perform: GPR[i]++
3044 *
3045 * As we read and write into the context saved GPR[i], if
3046 * we restart this batch buffer from an earlier point, we
3047 * will repeat the increment and store a value > 1.
3048 */
3049 *cs++ = MI_MATH(4);
3050 *cs++ = MI_MATH_LOAD(MI_MATH_REG_SRCA, MI_MATH_REG(i));
3051 *cs++ = MI_MATH_LOAD(MI_MATH_REG_SRCB, MI_MATH_REG(0));
3052 *cs++ = MI_MATH_ADD;
3053 *cs++ = MI_MATH_STORE(MI_MATH_REG(i), MI_MATH_REG_ACCU);
3054
3055 addr = result->node.start + offset + i * sizeof(*cs);
3056 *cs++ = MI_STORE_REGISTER_MEM_GEN8;
3057 *cs++ = CS_GPR(engine, 2 * i);
3058 *cs++ = lower_32_bits(addr);
3059 *cs++ = upper_32_bits(addr);
3060
3061 *cs++ = MI_SEMAPHORE_WAIT |
3062 MI_SEMAPHORE_POLL |
3063 MI_SEMAPHORE_SAD_GTE_SDD;
3064 *cs++ = i;
3065 *cs++ = lower_32_bits(result->node.start);
3066 *cs++ = upper_32_bits(result->node.start);
3067 }
3068
3069 *cs++ = MI_BATCH_BUFFER_END;
3070 i915_gem_object_flush_map(obj);
3071 i915_gem_object_unpin_map(obj);
3072
3073 return vma;
3074 }
3075
3076 static struct i915_vma *create_global(struct intel_gt *gt, size_t sz)
3077 {
3078 struct drm_i915_gem_object *obj;
3079 struct i915_vma *vma;
3080 int err;
3081
3082 obj = i915_gem_object_create_internal(gt->i915, sz);
3083 if (IS_ERR(obj))
3084 return ERR_CAST(obj);
3085
3086 vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);
3087 if (IS_ERR(vma)) {
3088 i915_gem_object_put(obj);
3089 return vma;
3090 }
3091
3092 err = i915_ggtt_pin(vma, NULL, 0, 0);
3093 if (err) {
3094 i915_vma_put(vma);
3095 return ERR_PTR(err);
3096 }
3097
3098 return vma;
3099 }
3100
3101 static struct i915_request *
3102 create_gpr_client(struct intel_engine_cs *engine,
3103 struct i915_vma *global,
3104 unsigned int offset)
3105 {
3106 struct i915_vma *batch, *vma;
3107 struct intel_context *ce;
3108 struct i915_request *rq;
3109 int err;
3110
3111 ce = intel_context_create(engine);
3112 if (IS_ERR(ce))
3113 return ERR_CAST(ce);
3114
3115 vma = i915_vma_instance(global->obj, ce->vm, NULL);
3116 if (IS_ERR(vma)) {
3117 err = PTR_ERR(vma);
3118 goto out_ce;
3119 }
3120
3121 err = i915_vma_pin(vma, 0, 0, PIN_USER);
3122 if (err)
3123 goto out_ce;
3124
3125 batch = create_gpr_user(engine, vma, offset);
3126 if (IS_ERR(batch)) {
3127 err = PTR_ERR(batch);
3128 goto out_vma;
3129 }
3130
3131 rq = intel_context_create_request(ce);
3132 if (IS_ERR(rq)) {
3133 err = PTR_ERR(rq);
3134 goto out_batch;
3135 }
3136
3137 i915_vma_lock(vma);
3138 err = i915_request_await_object(rq, vma->obj, false);
3139 if (!err)
3140 err = i915_vma_move_to_active(vma, rq, 0);
3141 i915_vma_unlock(vma);
3142
3143 i915_vma_lock(batch);
3144 if (!err)
3145 err = i915_request_await_object(rq, batch->obj, false);
3146 if (!err)
3147 err = i915_vma_move_to_active(batch, rq, 0);
3148 if (!err)
3149 err = rq->engine->emit_bb_start(rq,
3150 batch->node.start,
3151 PAGE_SIZE, 0);
3152 i915_vma_unlock(batch);
3153 i915_vma_unpin(batch);
3154
3155 if (!err)
3156 i915_request_get(rq);
3157 i915_request_add(rq);
3158
3159 out_batch:
3160 i915_vma_put(batch);
3161 out_vma:
3162 i915_vma_unpin(vma);
3163 out_ce:
3164 intel_context_put(ce);
3165 return err ? ERR_PTR(err) : rq;
3166 }
3167
3168 static int preempt_user(struct intel_engine_cs *engine,
3169 struct i915_vma *global,
3170 int id)
3171 {
3172 struct i915_sched_attr attr = {
3173 .priority = I915_PRIORITY_MAX
3174 };
3175 struct i915_request *rq;
3176 int err = 0;
3177 u32 *cs;
3178
3179 rq = intel_engine_create_kernel_request(engine);
3180 if (IS_ERR(rq))
3181 return PTR_ERR(rq);
3182
3183 cs = intel_ring_begin(rq, 4);
3184 if (IS_ERR(cs)) {
3185 i915_request_add(rq);
3186 return PTR_ERR(cs);
3187 }
3188
3189 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
3190 *cs++ = i915_ggtt_offset(global);
3191 *cs++ = 0;
3192 *cs++ = id;
3193
3194 intel_ring_advance(rq, cs);
3195
3196 i915_request_get(rq);
3197 i915_request_add(rq);
3198
3199 engine->schedule(rq, &attr);
3200
3201 if (i915_request_wait(rq, 0, HZ / 2) < 0)
3202 err = -ETIME;
3203 i915_request_put(rq);
3204
3205 return err;
3206 }
3207
3208 static int live_preempt_user(void *arg)
3209 {
3210 struct intel_gt *gt = arg;
3211 struct intel_engine_cs *engine;
3212 struct i915_vma *global;
3213 enum intel_engine_id id;
3214 u32 *result;
3215 int err = 0;
3216
3217 if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
3218 return 0;
3219
3220 /*
3221 * In our other tests, we look at preemption in carefully
3222 * controlled conditions in the ringbuffer. Since most of the
3223 * time is spent in user batches, most of our preemptions naturally
3224 * occur there. We want to verify that when we preempt inside a batch
3225 * we continue on from the current instruction and do not roll back
3226 * to the start, or another earlier arbitration point.
3227 *
3228 * To verify this, we create a batch which is a mixture of
3229 * MI_MATH (gpr++) MI_SRM (gpr) and preemption points. Then with
3230 * a few preempting contexts thrown into the mix, we look for any
3231 * repeated instructions (which show up as incorrect values).
3232 */
3233
3234 global = create_global(gt, 4096);
3235 if (IS_ERR(global))
3236 return PTR_ERR(global);
3237
3238 result = i915_gem_object_pin_map(global->obj, I915_MAP_WC);
3239 if (IS_ERR(result)) {
3240 i915_vma_unpin_and_release(&global, 0);
3241 return PTR_ERR(result);
3242 }
3243
3244 for_each_engine(engine, gt, id) {
3245 struct i915_request *client[3] = {};
3246 struct igt_live_test t;
3247 int i;
3248
3249 if (!intel_engine_has_preemption(engine))
3250 continue;
3251
3252 if (IS_GEN(gt->i915, 8) && engine->class != RENDER_CLASS)
3253 continue; /* we need per-context GPR */
3254
3255 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
3256 err = -EIO;
3257 break;
3258 }
3259
3260 memset(result, 0, 4096);
3261
3262 for (i = 0; i < ARRAY_SIZE(client); i++) {
3263 struct i915_request *rq;
3264
3265 rq = create_gpr_client(engine, global,
3266 NUM_GPR * i * sizeof(u32));
3267 if (IS_ERR(rq))
3268 goto end_test;
3269
3270 client[i] = rq;
3271 }
3272
3273 /* Continuously preempt the set of 3 running contexts */
3274 for (i = 1; i <= NUM_GPR; i++) {
3275 err = preempt_user(engine, global, i);
3276 if (err)
3277 goto end_test;
3278 }
3279
3280 if (READ_ONCE(result[0]) != NUM_GPR) {
3281 pr_err("%s: Failed to release semaphore\n",
3282 engine->name);
3283 err = -EIO;
3284 goto end_test;
3285 }
3286
3287 for (i = 0; i < ARRAY_SIZE(client); i++) {
3288 int gpr;
3289
3290 if (i915_request_wait(client[i], 0, HZ / 2) < 0) {
3291 err = -ETIME;
3292 goto end_test;
3293 }
3294
3295 for (gpr = 1; gpr < NUM_GPR; gpr++) {
3296 if (result[NUM_GPR * i + gpr] != 1) {
3297 pr_err("%s: Invalid result, client %d, gpr %d, result: %d\n",
3298 engine->name,
3299 i, gpr, result[NUM_GPR * i + gpr]);
3300 err = -EINVAL;
3301 goto end_test;
3302 }
3303 }
3304 }
3305
3306 end_test:
3307 for (i = 0; i < ARRAY_SIZE(client); i++) {
3308 if (!client[i])
3309 break;
3310
3311 i915_request_put(client[i]);
3312 }
3313
3314 /* Flush the semaphores on error */
3315 smp_store_mb(result[0], -1);
3316 if (igt_live_test_end(&t))
3317 err = -EIO;
3318 if (err)
3319 break;
3320 }
3321
3322 i915_vma_unpin_and_release(&global, I915_VMA_RELEASE_MAP);
3323 return err;
3324 }
3325
3326 static int live_preempt_timeout(void *arg)
3327 {
3328 struct intel_gt *gt = arg;
3329 struct i915_gem_context *ctx_hi, *ctx_lo;
3330 struct igt_spinner spin_lo;
3331 struct intel_engine_cs *engine;
3332 enum intel_engine_id id;
3333 int err = -ENOMEM;
3334
3335 /*
3336 * Check that we force preemption to occur by cancelling the previous
3337 * context if it refuses to yield the GPU.
3338 */
3339 if (!IS_ACTIVE(CONFIG_DRM_I915_PREEMPT_TIMEOUT))
3340 return 0;
3341
3342 if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
3343 return 0;
3344
3345 if (!intel_has_reset_engine(gt))
3346 return 0;
3347
3348 if (igt_spinner_init(&spin_lo, gt))
3349 return -ENOMEM;
3350
3351 ctx_hi = kernel_context(gt->i915);
3352 if (!ctx_hi)
3353 goto err_spin_lo;
3354 ctx_hi->sched.priority =
3355 I915_USER_PRIORITY(I915_CONTEXT_MAX_USER_PRIORITY);
3356
3357 ctx_lo = kernel_context(gt->i915);
3358 if (!ctx_lo)
3359 goto err_ctx_hi;
3360 ctx_lo->sched.priority =
3361 I915_USER_PRIORITY(I915_CONTEXT_MIN_USER_PRIORITY);
3362
3363 for_each_engine(engine, gt, id) {
3364 unsigned long saved_timeout;
3365 struct i915_request *rq;
3366
3367 if (!intel_engine_has_preemption(engine))
3368 continue;
3369
3370 rq = spinner_create_request(&spin_lo, ctx_lo, engine,
3371 MI_NOOP); /* preemption disabled */
3372 if (IS_ERR(rq)) {
3373 err = PTR_ERR(rq);
3374 goto err_ctx_lo;
3375 }
3376
3377 i915_request_add(rq);
3378 if (!igt_wait_for_spinner(&spin_lo, rq)) {
3379 intel_gt_set_wedged(gt);
3380 err = -EIO;
3381 goto err_ctx_lo;
3382 }
3383
3384 rq = igt_request_alloc(ctx_hi, engine);
3385 if (IS_ERR(rq)) {
3386 igt_spinner_end(&spin_lo);
3387 err = PTR_ERR(rq);
3388 goto err_ctx_lo;
3389 }
3390
3391 /* Flush the previous CS ack before changing timeouts */
3392 while (READ_ONCE(engine->execlists.pending[0]))
3393 cpu_relax();
3394
3395 saved_timeout = engine->props.preempt_timeout_ms;
3396 engine->props.preempt_timeout_ms = 1; /* in ms, -> 1 jiffie */
3397
3398 i915_request_get(rq);
3399 i915_request_add(rq);
3400
3401 intel_engine_flush_submission(engine);
3402 engine->props.preempt_timeout_ms = saved_timeout;
3403
3404 if (i915_request_wait(rq, 0, HZ / 10) < 0) {
3405 intel_gt_set_wedged(gt);
3406 i915_request_put(rq);
3407 err = -ETIME;
3408 goto err_ctx_lo;
3409 }
3410
3411 igt_spinner_end(&spin_lo);
3412 i915_request_put(rq);
3413 }
3414
3415 err = 0;
3416 err_ctx_lo:
3417 kernel_context_close(ctx_lo);
3418 err_ctx_hi:
3419 kernel_context_close(ctx_hi);
3420 err_spin_lo:
3421 igt_spinner_fini(&spin_lo);
3422 return err;
3423 }
3424
3425 static int random_range(struct rnd_state *rnd, int min, int max)
3426 {
3427 return i915_prandom_u32_max_state(max - min, rnd) + min;
3428 }
3429
3430 static int random_priority(struct rnd_state *rnd)
3431 {
3432 return random_range(rnd, I915_PRIORITY_MIN, I915_PRIORITY_MAX);
3433 }
3434
3435 struct preempt_smoke {
3436 struct intel_gt *gt;
3437 struct i915_gem_context **contexts;
3438 struct intel_engine_cs *engine;
3439 struct drm_i915_gem_object *batch;
3440 unsigned int ncontext;
3441 struct rnd_state prng;
3442 unsigned long count;
3443 };
3444
3445 static struct i915_gem_context *smoke_context(struct preempt_smoke *smoke)
3446 {
3447 return smoke->contexts[i915_prandom_u32_max_state(smoke->ncontext,
3448 &smoke->prng)];
3449 }
3450
3451 static int smoke_submit(struct preempt_smoke *smoke,
3452 struct i915_gem_context *ctx, int prio,
3453 struct drm_i915_gem_object *batch)
3454 {
3455 struct i915_request *rq;
3456 struct i915_vma *vma = NULL;
3457 int err = 0;
3458
3459 if (batch) {
3460 struct i915_address_space *vm;
3461
3462 vm = i915_gem_context_get_vm_rcu(ctx);
3463 vma = i915_vma_instance(batch, vm, NULL);
3464 i915_vm_put(vm);
3465 if (IS_ERR(vma))
3466 return PTR_ERR(vma);
3467
3468 err = i915_vma_pin(vma, 0, 0, PIN_USER);
3469 if (err)
3470 return err;
3471 }
3472
3473 ctx->sched.priority = prio;
3474
3475 rq = igt_request_alloc(ctx, smoke->engine);
3476 if (IS_ERR(rq)) {
3477 err = PTR_ERR(rq);
3478 goto unpin;
3479 }
3480
3481 if (vma) {
3482 i915_vma_lock(vma);
3483 err = i915_request_await_object(rq, vma->obj, false);
3484 if (!err)
3485 err = i915_vma_move_to_active(vma, rq, 0);
3486 if (!err)
3487 err = rq->engine->emit_bb_start(rq,
3488 vma->node.start,
3489 PAGE_SIZE, 0);
3490 i915_vma_unlock(vma);
3491 }
3492
3493 i915_request_add(rq);
3494
3495 unpin:
3496 if (vma)
3497 i915_vma_unpin(vma);
3498
3499 return err;
3500 }
3501
3502 static int smoke_crescendo_thread(void *arg)
3503 {
3504 struct preempt_smoke *smoke = arg;
3505 IGT_TIMEOUT(end_time);
3506 unsigned long count;
3507
3508 count = 0;
3509 do {
3510 struct i915_gem_context *ctx = smoke_context(smoke);
3511 int err;
3512
3513 err = smoke_submit(smoke,
3514 ctx, count % I915_PRIORITY_MAX,
3515 smoke->batch);
3516 if (err)
3517 return err;
3518
3519 count++;
3520 } while (count < smoke->ncontext && !__igt_timeout(end_time, NULL));
3521
3522 smoke->count = count;
3523 return 0;
3524 }
3525
3526 static int smoke_crescendo(struct preempt_smoke *smoke, unsigned int flags)
3527 #define BATCH BIT(0)
3528 {
3529 struct task_struct *tsk[I915_NUM_ENGINES] = {};
3530 struct preempt_smoke arg[I915_NUM_ENGINES];
3531 struct intel_engine_cs *engine;
3532 enum intel_engine_id id;
3533 unsigned long count;
3534 int err = 0;
3535
3536 for_each_engine(engine, smoke->gt, id) {
3537 arg[id] = *smoke;
3538 arg[id].engine = engine;
3539 if (!(flags & BATCH))
3540 arg[id].batch = NULL;
3541 arg[id].count = 0;
3542
3543 tsk[id] = kthread_run(smoke_crescendo_thread, &arg,
3544 "igt/smoke:%d", id);
3545 if (IS_ERR(tsk[id])) {
3546 err = PTR_ERR(tsk[id]);
3547 break;
3548 }
3549 get_task_struct(tsk[id]);
3550 }
3551
3552 yield(); /* start all threads before we kthread_stop() */
3553
3554 count = 0;
3555 for_each_engine(engine, smoke->gt, id) {
3556 int status;
3557
3558 if (IS_ERR_OR_NULL(tsk[id]))
3559 continue;
3560
3561 status = kthread_stop(tsk[id]);
3562 if (status && !err)
3563 err = status;
3564
3565 count += arg[id].count;
3566
3567 put_task_struct(tsk[id]);
3568 }
3569
3570 pr_info("Submitted %lu crescendo:%x requests across %d engines and %d contexts\n",
3571 count, flags, smoke->gt->info.num_engines, smoke->ncontext);
3572 return 0;
3573 }
3574
3575 static int smoke_random(struct preempt_smoke *smoke, unsigned int flags)
3576 {
3577 enum intel_engine_id id;
3578 IGT_TIMEOUT(end_time);
3579 unsigned long count;
3580
3581 count = 0;
3582 do {
3583 for_each_engine(smoke->engine, smoke->gt, id) {
3584 struct i915_gem_context *ctx = smoke_context(smoke);
3585 int err;
3586
3587 err = smoke_submit(smoke,
3588 ctx, random_priority(&smoke->prng),
3589 flags & BATCH ? smoke->batch : NULL);
3590 if (err)
3591 return err;
3592
3593 count++;
3594 }
3595 } while (count < smoke->ncontext && !__igt_timeout(end_time, NULL));
3596
3597 pr_info("Submitted %lu random:%x requests across %d engines and %d contexts\n",
3598 count, flags, smoke->gt->info.num_engines, smoke->ncontext);
3599 return 0;
3600 }
3601
3602 static int live_preempt_smoke(void *arg)
3603 {
3604 struct preempt_smoke smoke = {
3605 .gt = arg,
3606 .prng = I915_RND_STATE_INITIALIZER(i915_selftest.random_seed),
3607 .ncontext = 256,
3608 };
3609 const unsigned int phase[] = { 0, BATCH };
3610 struct igt_live_test t;
3611 int err = -ENOMEM;
3612 u32 *cs;
3613 int n;
3614
3615 if (!HAS_LOGICAL_RING_PREEMPTION(smoke.gt->i915))
3616 return 0;
3617
3618 smoke.contexts = kmalloc_array(smoke.ncontext,
3619 sizeof(*smoke.contexts),
3620 GFP_KERNEL);
3621 if (!smoke.contexts)
3622 return -ENOMEM;
3623
3624 smoke.batch =
3625 i915_gem_object_create_internal(smoke.gt->i915, PAGE_SIZE);
3626 if (IS_ERR(smoke.batch)) {
3627 err = PTR_ERR(smoke.batch);
3628 goto err_free;
3629 }
3630
3631 cs = i915_gem_object_pin_map(smoke.batch, I915_MAP_WB);
3632 if (IS_ERR(cs)) {
3633 err = PTR_ERR(cs);
3634 goto err_batch;
3635 }
3636 for (n = 0; n < PAGE_SIZE / sizeof(*cs) - 1; n++)
3637 cs[n] = MI_ARB_CHECK;
3638 cs[n] = MI_BATCH_BUFFER_END;
3639 i915_gem_object_flush_map(smoke.batch);
3640 i915_gem_object_unpin_map(smoke.batch);
3641
3642 if (igt_live_test_begin(&t, smoke.gt->i915, __func__, "all")) {
3643 err = -EIO;
3644 goto err_batch;
3645 }
3646
3647 for (n = 0; n < smoke.ncontext; n++) {
3648 smoke.contexts[n] = kernel_context(smoke.gt->i915);
3649 if (!smoke.contexts[n])
3650 goto err_ctx;
3651 }
3652
3653 for (n = 0; n < ARRAY_SIZE(phase); n++) {
3654 err = smoke_crescendo(&smoke, phase[n]);
3655 if (err)
3656 goto err_ctx;
3657
3658 err = smoke_random(&smoke, phase[n]);
3659 if (err)
3660 goto err_ctx;
3661 }
3662
3663 err_ctx:
3664 if (igt_live_test_end(&t))
3665 err = -EIO;
3666
3667 for (n = 0; n < smoke.ncontext; n++) {
3668 if (!smoke.contexts[n])
3669 break;
3670 kernel_context_close(smoke.contexts[n]);
3671 }
3672
3673 err_batch:
3674 i915_gem_object_put(smoke.batch);
3675 err_free:
3676 kfree(smoke.contexts);
3677
3678 return err;
3679 }
3680
3681 static int nop_virtual_engine(struct intel_gt *gt,
3682 struct intel_engine_cs **siblings,
3683 unsigned int nsibling,
3684 unsigned int nctx,
3685 unsigned int flags)
3686 #define CHAIN BIT(0)
3687 {
3688 IGT_TIMEOUT(end_time);
3689 struct i915_request *request[16] = {};
3690 struct intel_context *ve[16];
3691 unsigned long n, prime, nc;
3692 struct igt_live_test t;
3693 ktime_t times[2] = {};
3694 int err;
3695
3696 GEM_BUG_ON(!nctx || nctx > ARRAY_SIZE(ve));
3697
3698 for (n = 0; n < nctx; n++) {
3699 ve[n] = intel_execlists_create_virtual(siblings, nsibling);
3700 if (IS_ERR(ve[n])) {
3701 err = PTR_ERR(ve[n]);
3702 nctx = n;
3703 goto out;
3704 }
3705
3706 err = intel_context_pin(ve[n]);
3707 if (err) {
3708 intel_context_put(ve[n]);
3709 nctx = n;
3710 goto out;
3711 }
3712 }
3713
3714 err = igt_live_test_begin(&t, gt->i915, __func__, ve[0]->engine->name);
3715 if (err)
3716 goto out;
3717
3718 for_each_prime_number_from(prime, 1, 8192) {
3719 times[1] = ktime_get_raw();
3720
3721 if (flags & CHAIN) {
3722 for (nc = 0; nc < nctx; nc++) {
3723 for (n = 0; n < prime; n++) {
3724 struct i915_request *rq;
3725
3726 rq = i915_request_create(ve[nc]);
3727 if (IS_ERR(rq)) {
3728 err = PTR_ERR(rq);
3729 goto out;
3730 }
3731
3732 if (request[nc])
3733 i915_request_put(request[nc]);
3734 request[nc] = i915_request_get(rq);
3735 i915_request_add(rq);
3736 }
3737 }
3738 } else {
3739 for (n = 0; n < prime; n++) {
3740 for (nc = 0; nc < nctx; nc++) {
3741 struct i915_request *rq;
3742
3743 rq = i915_request_create(ve[nc]);
3744 if (IS_ERR(rq)) {
3745 err = PTR_ERR(rq);
3746 goto out;
3747 }
3748
3749 if (request[nc])
3750 i915_request_put(request[nc]);
3751 request[nc] = i915_request_get(rq);
3752 i915_request_add(rq);
3753 }
3754 }
3755 }
3756
3757 for (nc = 0; nc < nctx; nc++) {
3758 if (i915_request_wait(request[nc], 0, HZ / 10) < 0) {
3759 pr_err("%s(%s): wait for %llx:%lld timed out\n",
3760 __func__, ve[0]->engine->name,
3761 request[nc]->fence.context,
3762 request[nc]->fence.seqno);
3763
3764 GEM_TRACE("%s(%s) failed at request %llx:%lld\n",
3765 __func__, ve[0]->engine->name,
3766 request[nc]->fence.context,
3767 request[nc]->fence.seqno);
3768 GEM_TRACE_DUMP();
3769 intel_gt_set_wedged(gt);
3770 break;
3771 }
3772 }
3773
3774 times[1] = ktime_sub(ktime_get_raw(), times[1]);
3775 if (prime == 1)
3776 times[0] = times[1];
3777
3778 for (nc = 0; nc < nctx; nc++) {
3779 i915_request_put(request[nc]);
3780 request[nc] = NULL;
3781 }
3782
3783 if (__igt_timeout(end_time, NULL))
3784 break;
3785 }
3786
3787 err = igt_live_test_end(&t);
3788 if (err)
3789 goto out;
3790
3791 pr_info("Requestx%d latencies on %s: 1 = %lluns, %lu = %lluns\n",
3792 nctx, ve[0]->engine->name, ktime_to_ns(times[0]),
3793 prime, div64_u64(ktime_to_ns(times[1]), prime));
3794
3795 out:
3796 if (igt_flush_test(gt->i915))
3797 err = -EIO;
3798
3799 for (nc = 0; nc < nctx; nc++) {
3800 i915_request_put(request[nc]);
3801 intel_context_unpin(ve[nc]);
3802 intel_context_put(ve[nc]);
3803 }
3804 return err;
3805 }
3806
3807 static unsigned int
3808 __select_siblings(struct intel_gt *gt,
3809 unsigned int class,
3810 struct intel_engine_cs **siblings,
3811 bool (*filter)(const struct intel_engine_cs *))
3812 {
3813 unsigned int n = 0;
3814 unsigned int inst;
3815
3816 for (inst = 0; inst <= MAX_ENGINE_INSTANCE; inst++) {
3817 if (!gt->engine_class[class][inst])
3818 continue;
3819
3820 if (filter && !filter(gt->engine_class[class][inst]))
3821 continue;
3822
3823 siblings[n++] = gt->engine_class[class][inst];
3824 }
3825
3826 return n;
3827 }
3828
3829 static unsigned int
3830 select_siblings(struct intel_gt *gt,
3831 unsigned int class,
3832 struct intel_engine_cs **siblings)
3833 {
3834 return __select_siblings(gt, class, siblings, NULL);
3835 }
3836
3837 static int live_virtual_engine(void *arg)
3838 {
3839 struct intel_gt *gt = arg;
3840 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
3841 struct intel_engine_cs *engine;
3842 enum intel_engine_id id;
3843 unsigned int class;
3844 int err;
3845
3846 if (intel_uc_uses_guc_submission(&gt->uc))
3847 return 0;
3848
3849 for_each_engine(engine, gt, id) {
3850 err = nop_virtual_engine(gt, &engine, 1, 1, 0);
3851 if (err) {
3852 pr_err("Failed to wrap engine %s: err=%d\n",
3853 engine->name, err);
3854 return err;
3855 }
3856 }
3857
3858 for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
3859 int nsibling, n;
3860
3861 nsibling = select_siblings(gt, class, siblings);
3862 if (nsibling < 2)
3863 continue;
3864
3865 for (n = 1; n <= nsibling + 1; n++) {
3866 err = nop_virtual_engine(gt, siblings, nsibling,
3867 n, 0);
3868 if (err)
3869 return err;
3870 }
3871
3872 err = nop_virtual_engine(gt, siblings, nsibling, n, CHAIN);
3873 if (err)
3874 return err;
3875 }
3876
3877 return 0;
3878 }
3879
3880 static int mask_virtual_engine(struct intel_gt *gt,
3881 struct intel_engine_cs **siblings,
3882 unsigned int nsibling)
3883 {
3884 struct i915_request *request[MAX_ENGINE_INSTANCE + 1];
3885 struct intel_context *ve;
3886 struct igt_live_test t;
3887 unsigned int n;
3888 int err;
3889
3890 /*
3891 * Check that by setting the execution mask on a request, we can
3892 * restrict it to our desired engine within the virtual engine.
3893 */
3894
3895 ve = intel_execlists_create_virtual(siblings, nsibling);
3896 if (IS_ERR(ve)) {
3897 err = PTR_ERR(ve);
3898 goto out_close;
3899 }
3900
3901 err = intel_context_pin(ve);
3902 if (err)
3903 goto out_put;
3904
3905 err = igt_live_test_begin(&t, gt->i915, __func__, ve->engine->name);
3906 if (err)
3907 goto out_unpin;
3908
3909 for (n = 0; n < nsibling; n++) {
3910 request[n] = i915_request_create(ve);
3911 if (IS_ERR(request[n])) {
3912 err = PTR_ERR(request[n]);
3913 nsibling = n;
3914 goto out;
3915 }
3916
3917 /* Reverse order as it's more likely to be unnatural */
3918 request[n]->execution_mask = siblings[nsibling - n - 1]->mask;
3919
3920 i915_request_get(request[n]);
3921 i915_request_add(request[n]);
3922 }
3923
3924 for (n = 0; n < nsibling; n++) {
3925 if (i915_request_wait(request[n], 0, HZ / 10) < 0) {
3926 pr_err("%s(%s): wait for %llx:%lld timed out\n",
3927 __func__, ve->engine->name,
3928 request[n]->fence.context,
3929 request[n]->fence.seqno);
3930
3931 GEM_TRACE("%s(%s) failed at request %llx:%lld\n",
3932 __func__, ve->engine->name,
3933 request[n]->fence.context,
3934 request[n]->fence.seqno);
3935 GEM_TRACE_DUMP();
3936 intel_gt_set_wedged(gt);
3937 err = -EIO;
3938 goto out;
3939 }
3940
3941 if (request[n]->engine != siblings[nsibling - n - 1]) {
3942 pr_err("Executed on wrong sibling '%s', expected '%s'\n",
3943 request[n]->engine->name,
3944 siblings[nsibling - n - 1]->name);
3945 err = -EINVAL;
3946 goto out;
3947 }
3948 }
3949
3950 err = igt_live_test_end(&t);
3951 out:
3952 if (igt_flush_test(gt->i915))
3953 err = -EIO;
3954
3955 for (n = 0; n < nsibling; n++)
3956 i915_request_put(request[n]);
3957
3958 out_unpin:
3959 intel_context_unpin(ve);
3960 out_put:
3961 intel_context_put(ve);
3962 out_close:
3963 return err;
3964 }
3965
3966 static int live_virtual_mask(void *arg)
3967 {
3968 struct intel_gt *gt = arg;
3969 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
3970 unsigned int class;
3971 int err;
3972
3973 if (intel_uc_uses_guc_submission(&gt->uc))
3974 return 0;
3975
3976 for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
3977 unsigned int nsibling;
3978
3979 nsibling = select_siblings(gt, class, siblings);
3980 if (nsibling < 2)
3981 continue;
3982
3983 err = mask_virtual_engine(gt, siblings, nsibling);
3984 if (err)
3985 return err;
3986 }
3987
3988 return 0;
3989 }
3990
3991 static int slicein_virtual_engine(struct intel_gt *gt,
3992 struct intel_engine_cs **siblings,
3993 unsigned int nsibling)
3994 {
3995 const long timeout = slice_timeout(siblings[0]);
3996 struct intel_context *ce;
3997 struct i915_request *rq;
3998 struct igt_spinner spin;
3999 unsigned int n;
4000 int err = 0;
4001
4002 /*
4003 * Virtual requests must take part in timeslicing on the target engines.
4004 */
4005
4006 if (igt_spinner_init(&spin, gt))
4007 return -ENOMEM;
4008
4009 for (n = 0; n < nsibling; n++) {
4010 ce = intel_context_create(siblings[n]);
4011 if (IS_ERR(ce)) {
4012 err = PTR_ERR(ce);
4013 goto out;
4014 }
4015
4016 rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
4017 intel_context_put(ce);
4018 if (IS_ERR(rq)) {
4019 err = PTR_ERR(rq);
4020 goto out;
4021 }
4022
4023 i915_request_add(rq);
4024 }
4025
4026 ce = intel_execlists_create_virtual(siblings, nsibling);
4027 if (IS_ERR(ce)) {
4028 err = PTR_ERR(ce);
4029 goto out;
4030 }
4031
4032 rq = intel_context_create_request(ce);
4033 intel_context_put(ce);
4034 if (IS_ERR(rq)) {
4035 err = PTR_ERR(rq);
4036 goto out;
4037 }
4038
4039 i915_request_get(rq);
4040 i915_request_add(rq);
4041 if (i915_request_wait(rq, 0, timeout) < 0) {
4042 GEM_TRACE_ERR("%s(%s) failed to slice in virtual request\n",
4043 __func__, rq->engine->name);
4044 GEM_TRACE_DUMP();
4045 intel_gt_set_wedged(gt);
4046 err = -EIO;
4047 }
4048 i915_request_put(rq);
4049
4050 out:
4051 igt_spinner_end(&spin);
4052 if (igt_flush_test(gt->i915))
4053 err = -EIO;
4054 igt_spinner_fini(&spin);
4055 return err;
4056 }
4057
4058 static int sliceout_virtual_engine(struct intel_gt *gt,
4059 struct intel_engine_cs **siblings,
4060 unsigned int nsibling)
4061 {
4062 const long timeout = slice_timeout(siblings[0]);
4063 struct intel_context *ce;
4064 struct i915_request *rq;
4065 struct igt_spinner spin;
4066 unsigned int n;
4067 int err = 0;
4068
4069 /*
4070 * Virtual requests must allow others a fair timeslice.
4071 */
4072
4073 if (igt_spinner_init(&spin, gt))
4074 return -ENOMEM;
4075
4076 /* XXX We do not handle oversubscription and fairness with normal rq */
4077 for (n = 0; n < nsibling; n++) {
4078 ce = intel_execlists_create_virtual(siblings, nsibling);
4079 if (IS_ERR(ce)) {
4080 err = PTR_ERR(ce);
4081 goto out;
4082 }
4083
4084 rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
4085 intel_context_put(ce);
4086 if (IS_ERR(rq)) {
4087 err = PTR_ERR(rq);
4088 goto out;
4089 }
4090
4091 i915_request_add(rq);
4092 }
4093
4094 for (n = 0; !err && n < nsibling; n++) {
4095 ce = intel_context_create(siblings[n]);
4096 if (IS_ERR(ce)) {
4097 err = PTR_ERR(ce);
4098 goto out;
4099 }
4100
4101 rq = intel_context_create_request(ce);
4102 intel_context_put(ce);
4103 if (IS_ERR(rq)) {
4104 err = PTR_ERR(rq);
4105 goto out;
4106 }
4107
4108 i915_request_get(rq);
4109 i915_request_add(rq);
4110 if (i915_request_wait(rq, 0, timeout) < 0) {
4111 GEM_TRACE_ERR("%s(%s) failed to slice out virtual request\n",
4112 __func__, siblings[n]->name);
4113 GEM_TRACE_DUMP();
4114 intel_gt_set_wedged(gt);
4115 err = -EIO;
4116 }
4117 i915_request_put(rq);
4118 }
4119
4120 out:
4121 igt_spinner_end(&spin);
4122 if (igt_flush_test(gt->i915))
4123 err = -EIO;
4124 igt_spinner_fini(&spin);
4125 return err;
4126 }
4127
4128 static int live_virtual_slice(void *arg)
4129 {
4130 struct intel_gt *gt = arg;
4131 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
4132 unsigned int class;
4133 int err;
4134
4135 if (intel_uc_uses_guc_submission(&gt->uc))
4136 return 0;
4137
4138 for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
4139 unsigned int nsibling;
4140
4141 nsibling = __select_siblings(gt, class, siblings,
4142 intel_engine_has_timeslices);
4143 if (nsibling < 2)
4144 continue;
4145
4146 err = slicein_virtual_engine(gt, siblings, nsibling);
4147 if (err)
4148 return err;
4149
4150 err = sliceout_virtual_engine(gt, siblings, nsibling);
4151 if (err)
4152 return err;
4153 }
4154
4155 return 0;
4156 }
4157
4158 static int preserved_virtual_engine(struct intel_gt *gt,
4159 struct intel_engine_cs **siblings,
4160 unsigned int nsibling)
4161 {
4162 struct i915_request *last = NULL;
4163 struct intel_context *ve;
4164 struct i915_vma *scratch;
4165 struct igt_live_test t;
4166 unsigned int n;
4167 int err = 0;
4168 u32 *cs;
4169
4170 scratch = create_scratch(siblings[0]->gt);
4171 if (IS_ERR(scratch))
4172 return PTR_ERR(scratch);
4173
4174 err = i915_vma_sync(scratch);
4175 if (err)
4176 goto out_scratch;
4177
4178 ve = intel_execlists_create_virtual(siblings, nsibling);
4179 if (IS_ERR(ve)) {
4180 err = PTR_ERR(ve);
4181 goto out_scratch;
4182 }
4183
4184 err = intel_context_pin(ve);
4185 if (err)
4186 goto out_put;
4187
4188 err = igt_live_test_begin(&t, gt->i915, __func__, ve->engine->name);
4189 if (err)
4190 goto out_unpin;
4191
4192 for (n = 0; n < NUM_GPR_DW; n++) {
4193 struct intel_engine_cs *engine = siblings[n % nsibling];
4194 struct i915_request *rq;
4195
4196 rq = i915_request_create(ve);
4197 if (IS_ERR(rq)) {
4198 err = PTR_ERR(rq);
4199 goto out_end;
4200 }
4201
4202 i915_request_put(last);
4203 last = i915_request_get(rq);
4204
4205 cs = intel_ring_begin(rq, 8);
4206 if (IS_ERR(cs)) {
4207 i915_request_add(rq);
4208 err = PTR_ERR(cs);
4209 goto out_end;
4210 }
4211
4212 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
4213 *cs++ = CS_GPR(engine, n);
4214 *cs++ = i915_ggtt_offset(scratch) + n * sizeof(u32);
4215 *cs++ = 0;
4216
4217 *cs++ = MI_LOAD_REGISTER_IMM(1);
4218 *cs++ = CS_GPR(engine, (n + 1) % NUM_GPR_DW);
4219 *cs++ = n + 1;
4220
4221 *cs++ = MI_NOOP;
4222 intel_ring_advance(rq, cs);
4223
4224 /* Restrict this request to run on a particular engine */
4225 rq->execution_mask = engine->mask;
4226 i915_request_add(rq);
4227 }
4228
4229 if (i915_request_wait(last, 0, HZ / 5) < 0) {
4230 err = -ETIME;
4231 goto out_end;
4232 }
4233
4234 cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
4235 if (IS_ERR(cs)) {
4236 err = PTR_ERR(cs);
4237 goto out_end;
4238 }
4239
4240 for (n = 0; n < NUM_GPR_DW; n++) {
4241 if (cs[n] != n) {
4242 pr_err("Incorrect value[%d] found for GPR[%d]\n",
4243 cs[n], n);
4244 err = -EINVAL;
4245 break;
4246 }
4247 }
4248
4249 i915_gem_object_unpin_map(scratch->obj);
4250
4251 out_end:
4252 if (igt_live_test_end(&t))
4253 err = -EIO;
4254 i915_request_put(last);
4255 out_unpin:
4256 intel_context_unpin(ve);
4257 out_put:
4258 intel_context_put(ve);
4259 out_scratch:
4260 i915_vma_unpin_and_release(&scratch, 0);
4261 return err;
4262 }
4263
4264 static int live_virtual_preserved(void *arg)
4265 {
4266 struct intel_gt *gt = arg;
4267 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
4268 unsigned int class;
4269
4270 /*
4271 * Check that the context image retains non-privileged (user) registers
4272 * from one engine to the next. For this we check that the CS_GPR
4273 * are preserved.
4274 */
4275
4276 if (intel_uc_uses_guc_submission(&gt->uc))
4277 return 0;
4278
4279 /* As we use CS_GPR we cannot run before they existed on all engines. */
4280 if (INTEL_GEN(gt->i915) < 9)
4281 return 0;
4282
4283 for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
4284 int nsibling, err;
4285
4286 nsibling = select_siblings(gt, class, siblings);
4287 if (nsibling < 2)
4288 continue;
4289
4290 err = preserved_virtual_engine(gt, siblings, nsibling);
4291 if (err)
4292 return err;
4293 }
4294
4295 return 0;
4296 }
4297
4298 static int bond_virtual_engine(struct intel_gt *gt,
4299 unsigned int class,
4300 struct intel_engine_cs **siblings,
4301 unsigned int nsibling,
4302 unsigned int flags)
4303 #define BOND_SCHEDULE BIT(0)
4304 {
4305 struct intel_engine_cs *master;
4306 struct i915_request *rq[16];
4307 enum intel_engine_id id;
4308 struct igt_spinner spin;
4309 unsigned long n;
4310 int err;
4311
4312 /*
4313 * A set of bonded requests is intended to be run concurrently
4314 * across a number of engines. We use one request per-engine
4315 * and a magic fence to schedule each of the bonded requests
4316 * at the same time. A consequence of our current scheduler is that
4317 * we only move requests to the HW ready queue when the request
4318 * becomes ready, that is when all of its prerequisite fences have
4319 * been signaled. As one of those fences is the master submit fence,
4320 * there is a delay on all secondary fences as the HW may be
4321 * currently busy. Equally, as all the requests are independent,
4322 * they may have other fences that delay individual request
4323 * submission to HW. Ergo, we do not guarantee that all requests are
4324 * immediately submitted to HW at the same time, just that if the
4325 * rules are abided by, they are ready at the same time as the
4326 * first is submitted. Userspace can embed semaphores in its batch
4327 * to ensure parallel execution of its phases as it requires.
4328 * Though naturally it gets requested that perhaps the scheduler should
4329 * take care of parallel execution, even across preemption events on
4330 * different HW. (The proper answer is of course "lalalala".)
4331 *
4332 * With the submit-fence, we have identified three possible phases
4333 * of synchronisation depending on the master fence: queued (not
4334 * ready), executing, and signaled. The first two are quite simple
4335 * and checked below. However, the signaled master fence handling is
4336 * contentious. Currently we do not distinguish between a signaled
4337 * fence and an expired fence, as once signaled it does not convey
4338 * any information about the previous execution. It may even be freed
4339 * and hence checking later it may not exist at all. Ergo we currently
4340 * do not apply the bonding constraint for an already signaled fence,
4341 * as our expectation is that it should not constrain the secondaries
4342 * and is outside of the scope of the bonded request API (i.e. all
4343 * userspace requests are meant to be running in parallel). As
4344 * it imposes no constraint, and is effectively a no-op, we do not
4345 * check below as normal execution flows are checked extensively above.
4346 *
4347 * XXX Is the degenerate handling of signaled submit fences the
4348 * expected behaviour for userpace?
4349 */
4350
4351 GEM_BUG_ON(nsibling >= ARRAY_SIZE(rq) - 1);
4352
4353 if (igt_spinner_init(&spin, gt))
4354 return -ENOMEM;
4355
4356 err = 0;
4357 rq[0] = ERR_PTR(-ENOMEM);
4358 for_each_engine(master, gt, id) {
4359 struct i915_sw_fence fence = {};
4360 struct intel_context *ce;
4361
4362 if (master->class == class)
4363 continue;
4364
4365 ce = intel_context_create(master);
4366 if (IS_ERR(ce)) {
4367 err = PTR_ERR(ce);
4368 goto out;
4369 }
4370
4371 memset_p((void *)rq, ERR_PTR(-EINVAL), ARRAY_SIZE(rq));
4372
4373 rq[0] = igt_spinner_create_request(&spin, ce, MI_NOOP);
4374 intel_context_put(ce);
4375 if (IS_ERR(rq[0])) {
4376 err = PTR_ERR(rq[0]);
4377 goto out;
4378 }
4379 i915_request_get(rq[0]);
4380
4381 if (flags & BOND_SCHEDULE) {
4382 onstack_fence_init(&fence);
4383 err = i915_sw_fence_await_sw_fence_gfp(&rq[0]->submit,
4384 &fence,
4385 GFP_KERNEL);
4386 }
4387
4388 i915_request_add(rq[0]);
4389 if (err < 0)
4390 goto out;
4391
4392 if (!(flags & BOND_SCHEDULE) &&
4393 !igt_wait_for_spinner(&spin, rq[0])) {
4394 err = -EIO;
4395 goto out;
4396 }
4397
4398 for (n = 0; n < nsibling; n++) {
4399 struct intel_context *ve;
4400
4401 ve = intel_execlists_create_virtual(siblings, nsibling);
4402 if (IS_ERR(ve)) {
4403 err = PTR_ERR(ve);
4404 onstack_fence_fini(&fence);
4405 goto out;
4406 }
4407
4408 err = intel_virtual_engine_attach_bond(ve->engine,
4409 master,
4410 siblings[n]);
4411 if (err) {
4412 intel_context_put(ve);
4413 onstack_fence_fini(&fence);
4414 goto out;
4415 }
4416
4417 err = intel_context_pin(ve);
4418 intel_context_put(ve);
4419 if (err) {
4420 onstack_fence_fini(&fence);
4421 goto out;
4422 }
4423
4424 rq[n + 1] = i915_request_create(ve);
4425 intel_context_unpin(ve);
4426 if (IS_ERR(rq[n + 1])) {
4427 err = PTR_ERR(rq[n + 1]);
4428 onstack_fence_fini(&fence);
4429 goto out;
4430 }
4431 i915_request_get(rq[n + 1]);
4432
4433 err = i915_request_await_execution(rq[n + 1],
4434 &rq[0]->fence,
4435 ve->engine->bond_execute);
4436 i915_request_add(rq[n + 1]);
4437 if (err < 0) {
4438 onstack_fence_fini(&fence);
4439 goto out;
4440 }
4441 }
4442 onstack_fence_fini(&fence);
4443 intel_engine_flush_submission(master);
4444 igt_spinner_end(&spin);
4445
4446 if (i915_request_wait(rq[0], 0, HZ / 10) < 0) {
4447 pr_err("Master request did not execute (on %s)!\n",
4448 rq[0]->engine->name);
4449 err = -EIO;
4450 goto out;
4451 }
4452
4453 for (n = 0; n < nsibling; n++) {
4454 if (i915_request_wait(rq[n + 1], 0,
4455 MAX_SCHEDULE_TIMEOUT) < 0) {
4456 err = -EIO;
4457 goto out;
4458 }
4459
4460 if (rq[n + 1]->engine != siblings[n]) {
4461 pr_err("Bonded request did not execute on target engine: expected %s, used %s; master was %s\n",
4462 siblings[n]->name,
4463 rq[n + 1]->engine->name,
4464 rq[0]->engine->name);
4465 err = -EINVAL;
4466 goto out;
4467 }
4468 }
4469
4470 for (n = 0; !IS_ERR(rq[n]); n++)
4471 i915_request_put(rq[n]);
4472 rq[0] = ERR_PTR(-ENOMEM);
4473 }
4474
4475 out:
4476 for (n = 0; !IS_ERR(rq[n]); n++)
4477 i915_request_put(rq[n]);
4478 if (igt_flush_test(gt->i915))
4479 err = -EIO;
4480
4481 igt_spinner_fini(&spin);
4482 return err;
4483 }
4484
4485 static int live_virtual_bond(void *arg)
4486 {
4487 static const struct phase {
4488 const char *name;
4489 unsigned int flags;
4490 } phases[] = {
4491 { "", 0 },
4492 { "schedule", BOND_SCHEDULE },
4493 { },
4494 };
4495 struct intel_gt *gt = arg;
4496 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
4497 unsigned int class;
4498 int err;
4499
4500 if (intel_uc_uses_guc_submission(&gt->uc))
4501 return 0;
4502
4503 for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
4504 const struct phase *p;
4505 int nsibling;
4506
4507 nsibling = select_siblings(gt, class, siblings);
4508 if (nsibling < 2)
4509 continue;
4510
4511 for (p = phases; p->name; p++) {
4512 err = bond_virtual_engine(gt,
4513 class, siblings, nsibling,
4514 p->flags);
4515 if (err) {
4516 pr_err("%s(%s): failed class=%d, nsibling=%d, err=%d\n",
4517 __func__, p->name, class, nsibling, err);
4518 return err;
4519 }
4520 }
4521 }
4522
4523 return 0;
4524 }
4525
4526 static int reset_virtual_engine(struct intel_gt *gt,
4527 struct intel_engine_cs **siblings,
4528 unsigned int nsibling)
4529 {
4530 struct intel_engine_cs *engine;
4531 struct intel_context *ve;
4532 struct igt_spinner spin;
4533 struct i915_request *rq;
4534 unsigned int n;
4535 int err = 0;
4536
4537 /*
4538 * In order to support offline error capture for fast preempt reset,
4539 * we need to decouple the guilty request and ensure that it and its
4540 * descendents are not executed while the capture is in progress.
4541 */
4542
4543 if (igt_spinner_init(&spin, gt))
4544 return -ENOMEM;
4545
4546 ve = intel_execlists_create_virtual(siblings, nsibling);
4547 if (IS_ERR(ve)) {
4548 err = PTR_ERR(ve);
4549 goto out_spin;
4550 }
4551
4552 for (n = 0; n < nsibling; n++)
4553 st_engine_heartbeat_disable(siblings[n]);
4554
4555 rq = igt_spinner_create_request(&spin, ve, MI_ARB_CHECK);
4556 if (IS_ERR(rq)) {
4557 err = PTR_ERR(rq);
4558 goto out_heartbeat;
4559 }
4560 i915_request_add(rq);
4561
4562 if (!igt_wait_for_spinner(&spin, rq)) {
4563 intel_gt_set_wedged(gt);
4564 err = -ETIME;
4565 goto out_heartbeat;
4566 }
4567
4568 engine = rq->engine;
4569 GEM_BUG_ON(engine == ve->engine);
4570
4571 /* Take ownership of the reset and tasklet */
4572 if (test_and_set_bit(I915_RESET_ENGINE + engine->id,
4573 &gt->reset.flags)) {
4574 intel_gt_set_wedged(gt);
4575 err = -EBUSY;
4576 goto out_heartbeat;
4577 }
4578 tasklet_disable(&engine->execlists.tasklet);
4579
4580 engine->execlists.tasklet.func(engine->execlists.tasklet.data);
4581 GEM_BUG_ON(execlists_active(&engine->execlists) != rq);
4582
4583 /* Fake a preemption event; failed of course */
4584 spin_lock_irq(&engine->active.lock);
4585 __unwind_incomplete_requests(engine);
4586 spin_unlock_irq(&engine->active.lock);
4587 GEM_BUG_ON(rq->engine != ve->engine);
4588
4589 /* Reset the engine while keeping our active request on hold */
4590 execlists_hold(engine, rq);
4591 GEM_BUG_ON(!i915_request_on_hold(rq));
4592
4593 intel_engine_reset(engine, NULL);
4594 GEM_BUG_ON(rq->fence.error != -EIO);
4595
4596 /* Release our grasp on the engine, letting CS flow again */
4597 tasklet_enable(&engine->execlists.tasklet);
4598 clear_and_wake_up_bit(I915_RESET_ENGINE + engine->id, &gt->reset.flags);
4599
4600 /* Check that we do not resubmit the held request */
4601 i915_request_get(rq);
4602 if (!i915_request_wait(rq, 0, HZ / 5)) {
4603 pr_err("%s: on hold request completed!\n",
4604 engine->name);
4605 intel_gt_set_wedged(gt);
4606 err = -EIO;
4607 goto out_rq;
4608 }
4609 GEM_BUG_ON(!i915_request_on_hold(rq));
4610
4611 /* But is resubmitted on release */
4612 execlists_unhold(engine, rq);
4613 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
4614 pr_err("%s: held request did not complete!\n",
4615 engine->name);
4616 intel_gt_set_wedged(gt);
4617 err = -ETIME;
4618 }
4619
4620 out_rq:
4621 i915_request_put(rq);
4622 out_heartbeat:
4623 for (n = 0; n < nsibling; n++)
4624 st_engine_heartbeat_enable(siblings[n]);
4625
4626 intel_context_put(ve);
4627 out_spin:
4628 igt_spinner_fini(&spin);
4629 return err;
4630 }
4631
4632 static int live_virtual_reset(void *arg)
4633 {
4634 struct intel_gt *gt = arg;
4635 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
4636 unsigned int class;
4637
4638 /*
4639 * Check that we handle a reset event within a virtual engine.
4640 * Only the physical engine is reset, but we have to check the flow
4641 * of the virtual requests around the reset, and make sure it is not
4642 * forgotten.
4643 */
4644
4645 if (intel_uc_uses_guc_submission(&gt->uc))
4646 return 0;
4647
4648 if (!intel_has_reset_engine(gt))
4649 return 0;
4650
4651 for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
4652 int nsibling, err;
4653
4654 nsibling = select_siblings(gt, class, siblings);
4655 if (nsibling < 2)
4656 continue;
4657
4658 err = reset_virtual_engine(gt, siblings, nsibling);
4659 if (err)
4660 return err;
4661 }
4662
4663 return 0;
4664 }
4665
4666 int intel_execlists_live_selftests(struct drm_i915_private *i915)
4667 {
4668 static const struct i915_subtest tests[] = {
4669 SUBTEST(live_sanitycheck),
4670 SUBTEST(live_unlite_switch),
4671 SUBTEST(live_unlite_preempt),
4672 SUBTEST(live_unlite_ring),
4673 SUBTEST(live_pin_rewind),
4674 SUBTEST(live_hold_reset),
4675 SUBTEST(live_error_interrupt),
4676 SUBTEST(live_timeslice_preempt),
4677 SUBTEST(live_timeslice_rewind),
4678 SUBTEST(live_timeslice_queue),
4679 SUBTEST(live_timeslice_nopreempt),
4680 SUBTEST(live_busywait_preempt),
4681 SUBTEST(live_preempt),
4682 SUBTEST(live_late_preempt),
4683 SUBTEST(live_nopreempt),
4684 SUBTEST(live_preempt_cancel),
4685 SUBTEST(live_suppress_self_preempt),
4686 SUBTEST(live_chain_preempt),
4687 SUBTEST(live_preempt_ring),
4688 SUBTEST(live_preempt_gang),
4689 SUBTEST(live_preempt_timeout),
4690 SUBTEST(live_preempt_user),
4691 SUBTEST(live_preempt_smoke),
4692 SUBTEST(live_virtual_engine),
4693 SUBTEST(live_virtual_mask),
4694 SUBTEST(live_virtual_preserved),
4695 SUBTEST(live_virtual_slice),
4696 SUBTEST(live_virtual_bond),
4697 SUBTEST(live_virtual_reset),
4698 };
4699
4700 if (!HAS_EXECLISTS(i915))
4701 return 0;
4702
4703 if (intel_gt_is_wedged(&i915->gt))
4704 return 0;
4705
4706 return intel_gt_live_subtests(tests, &i915->gt);
4707 }
4708
4709 static int emit_semaphore_signal(struct intel_context *ce, void *slot)
4710 {
4711 const u32 offset =
4712 i915_ggtt_offset(ce->engine->status_page.vma) +
4713 offset_in_page(slot);
4714 struct i915_request *rq;
4715 u32 *cs;
4716
4717 rq = intel_context_create_request(ce);
4718 if (IS_ERR(rq))
4719 return PTR_ERR(rq);
4720
4721 cs = intel_ring_begin(rq, 4);
4722 if (IS_ERR(cs)) {
4723 i915_request_add(rq);
4724 return PTR_ERR(cs);
4725 }
4726
4727 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
4728 *cs++ = offset;
4729 *cs++ = 0;
4730 *cs++ = 1;
4731
4732 intel_ring_advance(rq, cs);
4733
4734 rq->sched.attr.priority = I915_PRIORITY_BARRIER;
4735 i915_request_add(rq);
4736 return 0;
4737 }
4738
4739 static int context_flush(struct intel_context *ce, long timeout)
4740 {
4741 struct i915_request *rq;
4742 struct dma_fence *fence;
4743 int err = 0;
4744
4745 rq = intel_engine_create_kernel_request(ce->engine);
4746 if (IS_ERR(rq))
4747 return PTR_ERR(rq);
4748
4749 fence = i915_active_fence_get(&ce->timeline->last_request);
4750 if (fence) {
4751 i915_request_await_dma_fence(rq, fence);
4752 dma_fence_put(fence);
4753 }
4754
4755 rq = i915_request_get(rq);
4756 i915_request_add(rq);
4757 if (i915_request_wait(rq, 0, timeout) < 0)
4758 err = -ETIME;
4759 i915_request_put(rq);
4760
4761 rmb(); /* We know the request is written, make sure all state is too! */
4762 return err;
4763 }
4764
4765 static int live_lrc_layout(void *arg)
4766 {
4767 struct intel_gt *gt = arg;
4768 struct intel_engine_cs *engine;
4769 enum intel_engine_id id;
4770 u32 *lrc;
4771 int err;
4772
4773 /*
4774 * Check the registers offsets we use to create the initial reg state
4775 * match the layout saved by HW.
4776 */
4777
4778 lrc = kmalloc(PAGE_SIZE, GFP_KERNEL);
4779 if (!lrc)
4780 return -ENOMEM;
4781
4782 err = 0;
4783 for_each_engine(engine, gt, id) {
4784 u32 *hw;
4785 int dw;
4786
4787 if (!engine->default_state)
4788 continue;
4789
4790 hw = shmem_pin_map(engine->default_state);
4791 if (IS_ERR(hw)) {
4792 err = PTR_ERR(hw);
4793 break;
4794 }
4795 hw += LRC_STATE_OFFSET / sizeof(*hw);
4796
4797 execlists_init_reg_state(memset(lrc, POISON_INUSE, PAGE_SIZE),
4798 engine->kernel_context,
4799 engine,
4800 engine->kernel_context->ring,
4801 true);
4802
4803 dw = 0;
4804 do {
4805 u32 lri = hw[dw];
4806
4807 if (lri == 0) {
4808 dw++;
4809 continue;
4810 }
4811
4812 if (lrc[dw] == 0) {
4813 pr_debug("%s: skipped instruction %x at dword %d\n",
4814 engine->name, lri, dw);
4815 dw++;
4816 continue;
4817 }
4818
4819 if ((lri & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
4820 pr_err("%s: Expected LRI command at dword %d, found %08x\n",
4821 engine->name, dw, lri);
4822 err = -EINVAL;
4823 break;
4824 }
4825
4826 if (lrc[dw] != lri) {
4827 pr_err("%s: LRI command mismatch at dword %d, expected %08x found %08x\n",
4828 engine->name, dw, lri, lrc[dw]);
4829 err = -EINVAL;
4830 break;
4831 }
4832
4833 lri &= 0x7f;
4834 lri++;
4835 dw++;
4836
4837 while (lri) {
4838 if (hw[dw] != lrc[dw]) {
4839 pr_err("%s: Different registers found at dword %d, expected %x, found %x\n",
4840 engine->name, dw, hw[dw], lrc[dw]);
4841 err = -EINVAL;
4842 break;
4843 }
4844
4845 /*
4846 * Skip over the actual register value as we
4847 * expect that to differ.
4848 */
4849 dw += 2;
4850 lri -= 2;
4851 }
4852 } while ((lrc[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
4853
4854 if (err) {
4855 pr_info("%s: HW register image:\n", engine->name);
4856 igt_hexdump(hw, PAGE_SIZE);
4857
4858 pr_info("%s: SW register image:\n", engine->name);
4859 igt_hexdump(lrc, PAGE_SIZE);
4860 }
4861
4862 shmem_unpin_map(engine->default_state, hw);
4863 if (err)
4864 break;
4865 }
4866
4867 kfree(lrc);
4868 return err;
4869 }
4870
4871 static int find_offset(const u32 *lri, u32 offset)
4872 {
4873 int i;
4874
4875 for (i = 0; i < PAGE_SIZE / sizeof(u32); i++)
4876 if (lri[i] == offset)
4877 return i;
4878
4879 return -1;
4880 }
4881
4882 static int live_lrc_fixed(void *arg)
4883 {
4884 struct intel_gt *gt = arg;
4885 struct intel_engine_cs *engine;
4886 enum intel_engine_id id;
4887 int err = 0;
4888
4889 /*
4890 * Check the assumed register offsets match the actual locations in
4891 * the context image.
4892 */
4893
4894 for_each_engine(engine, gt, id) {
4895 const struct {
4896 u32 reg;
4897 u32 offset;
4898 const char *name;
4899 } tbl[] = {
4900 {
4901 i915_mmio_reg_offset(RING_START(engine->mmio_base)),
4902 CTX_RING_START - 1,
4903 "RING_START"
4904 },
4905 {
4906 i915_mmio_reg_offset(RING_CTL(engine->mmio_base)),
4907 CTX_RING_CTL - 1,
4908 "RING_CTL"
4909 },
4910 {
4911 i915_mmio_reg_offset(RING_HEAD(engine->mmio_base)),
4912 CTX_RING_HEAD - 1,
4913 "RING_HEAD"
4914 },
4915 {
4916 i915_mmio_reg_offset(RING_TAIL(engine->mmio_base)),
4917 CTX_RING_TAIL - 1,
4918 "RING_TAIL"
4919 },
4920 {
4921 i915_mmio_reg_offset(RING_MI_MODE(engine->mmio_base)),
4922 lrc_ring_mi_mode(engine),
4923 "RING_MI_MODE"
4924 },
4925 {
4926 i915_mmio_reg_offset(RING_BBSTATE(engine->mmio_base)),
4927 CTX_BB_STATE - 1,
4928 "BB_STATE"
4929 },
4930 {
4931 i915_mmio_reg_offset(RING_BB_PER_CTX_PTR(engine->mmio_base)),
4932 lrc_ring_wa_bb_per_ctx(engine),
4933 "RING_BB_PER_CTX_PTR"
4934 },
4935 {
4936 i915_mmio_reg_offset(RING_INDIRECT_CTX(engine->mmio_base)),
4937 lrc_ring_indirect_ptr(engine),
4938 "RING_INDIRECT_CTX_PTR"
4939 },
4940 {
4941 i915_mmio_reg_offset(RING_INDIRECT_CTX_OFFSET(engine->mmio_base)),
4942 lrc_ring_indirect_offset(engine),
4943 "RING_INDIRECT_CTX_OFFSET"
4944 },
4945 {
4946 i915_mmio_reg_offset(RING_CTX_TIMESTAMP(engine->mmio_base)),
4947 CTX_TIMESTAMP - 1,
4948 "RING_CTX_TIMESTAMP"
4949 },
4950 {
4951 i915_mmio_reg_offset(GEN8_RING_CS_GPR(engine->mmio_base, 0)),
4952 lrc_ring_gpr0(engine),
4953 "RING_CS_GPR0"
4954 },
4955 {
4956 i915_mmio_reg_offset(RING_CMD_BUF_CCTL(engine->mmio_base)),
4957 lrc_ring_cmd_buf_cctl(engine),
4958 "RING_CMD_BUF_CCTL"
4959 },
4960 { },
4961 }, *t;
4962 u32 *hw;
4963
4964 if (!engine->default_state)
4965 continue;
4966
4967 hw = shmem_pin_map(engine->default_state);
4968 if (IS_ERR(hw)) {
4969 err = PTR_ERR(hw);
4970 break;
4971 }
4972 hw += LRC_STATE_OFFSET / sizeof(*hw);
4973
4974 for (t = tbl; t->name; t++) {
4975 int dw = find_offset(hw, t->reg);
4976
4977 if (dw != t->offset) {
4978 pr_err("%s: Offset for %s [0x%x] mismatch, found %x, expected %x\n",
4979 engine->name,
4980 t->name,
4981 t->reg,
4982 dw,
4983 t->offset);
4984 err = -EINVAL;
4985 }
4986 }
4987
4988 shmem_unpin_map(engine->default_state, hw);
4989 }
4990
4991 return err;
4992 }
4993
4994 static int __live_lrc_state(struct intel_engine_cs *engine,
4995 struct i915_vma *scratch)
4996 {
4997 struct intel_context *ce;
4998 struct i915_request *rq;
4999 struct i915_gem_ww_ctx ww;
5000 enum {
5001 RING_START_IDX = 0,
5002 RING_TAIL_IDX,
5003 MAX_IDX
5004 };
5005 u32 expected[MAX_IDX];
5006 u32 *cs;
5007 int err;
5008 int n;
5009
5010 ce = intel_context_create(engine);
5011 if (IS_ERR(ce))
5012 return PTR_ERR(ce);
5013
5014 i915_gem_ww_ctx_init(&ww, false);
5015 retry:
5016 err = i915_gem_object_lock(scratch->obj, &ww);
5017 if (!err)
5018 err = intel_context_pin_ww(ce, &ww);
5019 if (err)
5020 goto err_put;
5021
5022 rq = i915_request_create(ce);
5023 if (IS_ERR(rq)) {
5024 err = PTR_ERR(rq);
5025 goto err_unpin;
5026 }
5027
5028 cs = intel_ring_begin(rq, 4 * MAX_IDX);
5029 if (IS_ERR(cs)) {
5030 err = PTR_ERR(cs);
5031 i915_request_add(rq);
5032 goto err_unpin;
5033 }
5034
5035 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
5036 *cs++ = i915_mmio_reg_offset(RING_START(engine->mmio_base));
5037 *cs++ = i915_ggtt_offset(scratch) + RING_START_IDX * sizeof(u32);
5038 *cs++ = 0;
5039
5040 expected[RING_START_IDX] = i915_ggtt_offset(ce->ring->vma);
5041
5042 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
5043 *cs++ = i915_mmio_reg_offset(RING_TAIL(engine->mmio_base));
5044 *cs++ = i915_ggtt_offset(scratch) + RING_TAIL_IDX * sizeof(u32);
5045 *cs++ = 0;
5046
5047 err = i915_request_await_object(rq, scratch->obj, true);
5048 if (!err)
5049 err = i915_vma_move_to_active(scratch, rq, EXEC_OBJECT_WRITE);
5050
5051 i915_request_get(rq);
5052 i915_request_add(rq);
5053 if (err)
5054 goto err_rq;
5055
5056 intel_engine_flush_submission(engine);
5057 expected[RING_TAIL_IDX] = ce->ring->tail;
5058
5059 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
5060 err = -ETIME;
5061 goto err_rq;
5062 }
5063
5064 cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
5065 if (IS_ERR(cs)) {
5066 err = PTR_ERR(cs);
5067 goto err_rq;
5068 }
5069
5070 for (n = 0; n < MAX_IDX; n++) {
5071 if (cs[n] != expected[n]) {
5072 pr_err("%s: Stored register[%d] value[0x%x] did not match expected[0x%x]\n",
5073 engine->name, n, cs[n], expected[n]);
5074 err = -EINVAL;
5075 break;
5076 }
5077 }
5078
5079 i915_gem_object_unpin_map(scratch->obj);
5080
5081 err_rq:
5082 i915_request_put(rq);
5083 err_unpin:
5084 intel_context_unpin(ce);
5085 err_put:
5086 if (err == -EDEADLK) {
5087 err = i915_gem_ww_ctx_backoff(&ww);
5088 if (!err)
5089 goto retry;
5090 }
5091 i915_gem_ww_ctx_fini(&ww);
5092 intel_context_put(ce);
5093 return err;
5094 }
5095
5096 static int live_lrc_state(void *arg)
5097 {
5098 struct intel_gt *gt = arg;
5099 struct intel_engine_cs *engine;
5100 struct i915_vma *scratch;
5101 enum intel_engine_id id;
5102 int err = 0;
5103
5104 /*
5105 * Check the live register state matches what we expect for this
5106 * intel_context.
5107 */
5108
5109 scratch = create_scratch(gt);
5110 if (IS_ERR(scratch))
5111 return PTR_ERR(scratch);
5112
5113 for_each_engine(engine, gt, id) {
5114 err = __live_lrc_state(engine, scratch);
5115 if (err)
5116 break;
5117 }
5118
5119 if (igt_flush_test(gt->i915))
5120 err = -EIO;
5121
5122 i915_vma_unpin_and_release(&scratch, 0);
5123 return err;
5124 }
5125
5126 static int gpr_make_dirty(struct intel_context *ce)
5127 {
5128 struct i915_request *rq;
5129 u32 *cs;
5130 int n;
5131
5132 rq = intel_context_create_request(ce);
5133 if (IS_ERR(rq))
5134 return PTR_ERR(rq);
5135
5136 cs = intel_ring_begin(rq, 2 * NUM_GPR_DW + 2);
5137 if (IS_ERR(cs)) {
5138 i915_request_add(rq);
5139 return PTR_ERR(cs);
5140 }
5141
5142 *cs++ = MI_LOAD_REGISTER_IMM(NUM_GPR_DW);
5143 for (n = 0; n < NUM_GPR_DW; n++) {
5144 *cs++ = CS_GPR(ce->engine, n);
5145 *cs++ = STACK_MAGIC;
5146 }
5147 *cs++ = MI_NOOP;
5148
5149 intel_ring_advance(rq, cs);
5150
5151 rq->sched.attr.priority = I915_PRIORITY_BARRIER;
5152 i915_request_add(rq);
5153
5154 return 0;
5155 }
5156
5157 static struct i915_request *
5158 __gpr_read(struct intel_context *ce, struct i915_vma *scratch, u32 *slot)
5159 {
5160 const u32 offset =
5161 i915_ggtt_offset(ce->engine->status_page.vma) +
5162 offset_in_page(slot);
5163 struct i915_request *rq;
5164 u32 *cs;
5165 int err;
5166 int n;
5167
5168 rq = intel_context_create_request(ce);
5169 if (IS_ERR(rq))
5170 return rq;
5171
5172 cs = intel_ring_begin(rq, 6 + 4 * NUM_GPR_DW);
5173 if (IS_ERR(cs)) {
5174 i915_request_add(rq);
5175 return ERR_CAST(cs);
5176 }
5177
5178 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
5179 *cs++ = MI_NOOP;
5180
5181 *cs++ = MI_SEMAPHORE_WAIT |
5182 MI_SEMAPHORE_GLOBAL_GTT |
5183 MI_SEMAPHORE_POLL |
5184 MI_SEMAPHORE_SAD_NEQ_SDD;
5185 *cs++ = 0;
5186 *cs++ = offset;
5187 *cs++ = 0;
5188
5189 for (n = 0; n < NUM_GPR_DW; n++) {
5190 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
5191 *cs++ = CS_GPR(ce->engine, n);
5192 *cs++ = i915_ggtt_offset(scratch) + n * sizeof(u32);
5193 *cs++ = 0;
5194 }
5195
5196 i915_vma_lock(scratch);
5197 err = i915_request_await_object(rq, scratch->obj, true);
5198 if (!err)
5199 err = i915_vma_move_to_active(scratch, rq, EXEC_OBJECT_WRITE);
5200 i915_vma_unlock(scratch);
5201
5202 i915_request_get(rq);
5203 i915_request_add(rq);
5204 if (err) {
5205 i915_request_put(rq);
5206 rq = ERR_PTR(err);
5207 }
5208
5209 return rq;
5210 }
5211
5212 static int __live_lrc_gpr(struct intel_engine_cs *engine,
5213 struct i915_vma *scratch,
5214 bool preempt)
5215 {
5216 u32 *slot = memset32(engine->status_page.addr + 1000, 0, 4);
5217 struct intel_context *ce;
5218 struct i915_request *rq;
5219 u32 *cs;
5220 int err;
5221 int n;
5222
5223 if (INTEL_GEN(engine->i915) < 9 && engine->class != RENDER_CLASS)
5224 return 0; /* GPR only on rcs0 for gen8 */
5225
5226 err = gpr_make_dirty(engine->kernel_context);
5227 if (err)
5228 return err;
5229
5230 ce = intel_context_create(engine);
5231 if (IS_ERR(ce))
5232 return PTR_ERR(ce);
5233
5234 rq = __gpr_read(ce, scratch, slot);
5235 if (IS_ERR(rq)) {
5236 err = PTR_ERR(rq);
5237 goto err_put;
5238 }
5239
5240 err = wait_for_submit(engine, rq, HZ / 2);
5241 if (err)
5242 goto err_rq;
5243
5244 if (preempt) {
5245 err = gpr_make_dirty(engine->kernel_context);
5246 if (err)
5247 goto err_rq;
5248
5249 err = emit_semaphore_signal(engine->kernel_context, slot);
5250 if (err)
5251 goto err_rq;
5252 } else {
5253 slot[0] = 1;
5254 wmb();
5255 }
5256
5257 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
5258 err = -ETIME;
5259 goto err_rq;
5260 }
5261
5262 cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
5263 if (IS_ERR(cs)) {
5264 err = PTR_ERR(cs);
5265 goto err_rq;
5266 }
5267
5268 for (n = 0; n < NUM_GPR_DW; n++) {
5269 if (cs[n]) {
5270 pr_err("%s: GPR[%d].%s was not zero, found 0x%08x!\n",
5271 engine->name,
5272 n / 2, n & 1 ? "udw" : "ldw",
5273 cs[n]);
5274 err = -EINVAL;
5275 break;
5276 }
5277 }
5278
5279 i915_gem_object_unpin_map(scratch->obj);
5280
5281 err_rq:
5282 memset32(&slot[0], -1, 4);
5283 wmb();
5284 i915_request_put(rq);
5285 err_put:
5286 intel_context_put(ce);
5287 return err;
5288 }
5289
5290 static int live_lrc_gpr(void *arg)
5291 {
5292 struct intel_gt *gt = arg;
5293 struct intel_engine_cs *engine;
5294 struct i915_vma *scratch;
5295 enum intel_engine_id id;
5296 int err = 0;
5297
5298 /*
5299 * Check that GPR registers are cleared in new contexts as we need
5300 * to avoid leaking any information from previous contexts.
5301 */
5302
5303 scratch = create_scratch(gt);
5304 if (IS_ERR(scratch))
5305 return PTR_ERR(scratch);
5306
5307 for_each_engine(engine, gt, id) {
5308 st_engine_heartbeat_disable(engine);
5309
5310 err = __live_lrc_gpr(engine, scratch, false);
5311 if (err)
5312 goto err;
5313
5314 err = __live_lrc_gpr(engine, scratch, true);
5315 if (err)
5316 goto err;
5317
5318 err:
5319 st_engine_heartbeat_enable(engine);
5320 if (igt_flush_test(gt->i915))
5321 err = -EIO;
5322 if (err)
5323 break;
5324 }
5325
5326 i915_vma_unpin_and_release(&scratch, 0);
5327 return err;
5328 }
5329
5330 static struct i915_request *
5331 create_timestamp(struct intel_context *ce, void *slot, int idx)
5332 {
5333 const u32 offset =
5334 i915_ggtt_offset(ce->engine->status_page.vma) +
5335 offset_in_page(slot);
5336 struct i915_request *rq;
5337 u32 *cs;
5338 int err;
5339
5340 rq = intel_context_create_request(ce);
5341 if (IS_ERR(rq))
5342 return rq;
5343
5344 cs = intel_ring_begin(rq, 10);
5345 if (IS_ERR(cs)) {
5346 err = PTR_ERR(cs);
5347 goto err;
5348 }
5349
5350 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
5351 *cs++ = MI_NOOP;
5352
5353 *cs++ = MI_SEMAPHORE_WAIT |
5354 MI_SEMAPHORE_GLOBAL_GTT |
5355 MI_SEMAPHORE_POLL |
5356 MI_SEMAPHORE_SAD_NEQ_SDD;
5357 *cs++ = 0;
5358 *cs++ = offset;
5359 *cs++ = 0;
5360
5361 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
5362 *cs++ = i915_mmio_reg_offset(RING_CTX_TIMESTAMP(rq->engine->mmio_base));
5363 *cs++ = offset + idx * sizeof(u32);
5364 *cs++ = 0;
5365
5366 intel_ring_advance(rq, cs);
5367
5368 rq->sched.attr.priority = I915_PRIORITY_MASK;
5369 err = 0;
5370 err:
5371 i915_request_get(rq);
5372 i915_request_add(rq);
5373 if (err) {
5374 i915_request_put(rq);
5375 return ERR_PTR(err);
5376 }
5377
5378 return rq;
5379 }
5380
5381 struct lrc_timestamp {
5382 struct intel_engine_cs *engine;
5383 struct intel_context *ce[2];
5384 u32 poison;
5385 };
5386
5387 static bool timestamp_advanced(u32 start, u32 end)
5388 {
5389 return (s32)(end - start) > 0;
5390 }
5391
5392 static int __lrc_timestamp(const struct lrc_timestamp *arg, bool preempt)
5393 {
5394 u32 *slot = memset32(arg->engine->status_page.addr + 1000, 0, 4);
5395 struct i915_request *rq;
5396 u32 timestamp;
5397 int err = 0;
5398
5399 arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP] = arg->poison;
5400 rq = create_timestamp(arg->ce[0], slot, 1);
5401 if (IS_ERR(rq))
5402 return PTR_ERR(rq);
5403
5404 err = wait_for_submit(rq->engine, rq, HZ / 2);
5405 if (err)
5406 goto err;
5407
5408 if (preempt) {
5409 arg->ce[1]->lrc_reg_state[CTX_TIMESTAMP] = 0xdeadbeef;
5410 err = emit_semaphore_signal(arg->ce[1], slot);
5411 if (err)
5412 goto err;
5413 } else {
5414 slot[0] = 1;
5415 wmb();
5416 }
5417
5418 /* And wait for switch to kernel (to save our context to memory) */
5419 err = context_flush(arg->ce[0], HZ / 2);
5420 if (err)
5421 goto err;
5422
5423 if (!timestamp_advanced(arg->poison, slot[1])) {
5424 pr_err("%s(%s): invalid timestamp on restore, context:%x, request:%x\n",
5425 arg->engine->name, preempt ? "preempt" : "simple",
5426 arg->poison, slot[1]);
5427 err = -EINVAL;
5428 }
5429
5430 timestamp = READ_ONCE(arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP]);
5431 if (!timestamp_advanced(slot[1], timestamp)) {
5432 pr_err("%s(%s): invalid timestamp on save, request:%x, context:%x\n",
5433 arg->engine->name, preempt ? "preempt" : "simple",
5434 slot[1], timestamp);
5435 err = -EINVAL;
5436 }
5437
5438 err:
5439 memset32(slot, -1, 4);
5440 i915_request_put(rq);
5441 return err;
5442 }
5443
5444 static int live_lrc_timestamp(void *arg)
5445 {
5446 struct lrc_timestamp data = {};
5447 struct intel_gt *gt = arg;
5448 enum intel_engine_id id;
5449 const u32 poison[] = {
5450 0,
5451 S32_MAX,
5452 (u32)S32_MAX + 1,
5453 U32_MAX,
5454 };
5455
5456 /*
5457 * We want to verify that the timestamp is saved and restore across
5458 * context switches and is monotonic.
5459 *
5460 * So we do this with a little bit of LRC poisoning to check various
5461 * boundary conditions, and see what happens if we preempt the context
5462 * with a second request (carrying more poison into the timestamp).
5463 */
5464
5465 for_each_engine(data.engine, gt, id) {
5466 int i, err = 0;
5467
5468 st_engine_heartbeat_disable(data.engine);
5469
5470 for (i = 0; i < ARRAY_SIZE(data.ce); i++) {
5471 struct intel_context *tmp;
5472
5473 tmp = intel_context_create(data.engine);
5474 if (IS_ERR(tmp)) {
5475 err = PTR_ERR(tmp);
5476 goto err;
5477 }
5478
5479 err = intel_context_pin(tmp);
5480 if (err) {
5481 intel_context_put(tmp);
5482 goto err;
5483 }
5484
5485 data.ce[i] = tmp;
5486 }
5487
5488 for (i = 0; i < ARRAY_SIZE(poison); i++) {
5489 data.poison = poison[i];
5490
5491 err = __lrc_timestamp(&data, false);
5492 if (err)
5493 break;
5494
5495 err = __lrc_timestamp(&data, true);
5496 if (err)
5497 break;
5498 }
5499
5500 err:
5501 st_engine_heartbeat_enable(data.engine);
5502 for (i = 0; i < ARRAY_SIZE(data.ce); i++) {
5503 if (!data.ce[i])
5504 break;
5505
5506 intel_context_unpin(data.ce[i]);
5507 intel_context_put(data.ce[i]);
5508 }
5509
5510 if (igt_flush_test(gt->i915))
5511 err = -EIO;
5512 if (err)
5513 return err;
5514 }
5515
5516 return 0;
5517 }
5518
5519 static struct i915_vma *
5520 create_user_vma(struct i915_address_space *vm, unsigned long size)
5521 {
5522 struct drm_i915_gem_object *obj;
5523 struct i915_vma *vma;
5524 int err;
5525
5526 obj = i915_gem_object_create_internal(vm->i915, size);
5527 if (IS_ERR(obj))
5528 return ERR_CAST(obj);
5529
5530 vma = i915_vma_instance(obj, vm, NULL);
5531 if (IS_ERR(vma)) {
5532 i915_gem_object_put(obj);
5533 return vma;
5534 }
5535
5536 err = i915_vma_pin(vma, 0, 0, PIN_USER);
5537 if (err) {
5538 i915_gem_object_put(obj);
5539 return ERR_PTR(err);
5540 }
5541
5542 return vma;
5543 }
5544
5545 static struct i915_vma *
5546 store_context(struct intel_context *ce, struct i915_vma *scratch)
5547 {
5548 struct i915_vma *batch;
5549 u32 dw, x, *cs, *hw;
5550 u32 *defaults;
5551
5552 batch = create_user_vma(ce->vm, SZ_64K);
5553 if (IS_ERR(batch))
5554 return batch;
5555
5556 cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC);
5557 if (IS_ERR(cs)) {
5558 i915_vma_put(batch);
5559 return ERR_CAST(cs);
5560 }
5561
5562 defaults = shmem_pin_map(ce->engine->default_state);
5563 if (!defaults) {
5564 i915_gem_object_unpin_map(batch->obj);
5565 i915_vma_put(batch);
5566 return ERR_PTR(-ENOMEM);
5567 }
5568
5569 x = 0;
5570 dw = 0;
5571 hw = defaults;
5572 hw += LRC_STATE_OFFSET / sizeof(*hw);
5573 do {
5574 u32 len = hw[dw] & 0x7f;
5575
5576 if (hw[dw] == 0) {
5577 dw++;
5578 continue;
5579 }
5580
5581 if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
5582 dw += len + 2;
5583 continue;
5584 }
5585
5586 dw++;
5587 len = (len + 1) / 2;
5588 while (len--) {
5589 *cs++ = MI_STORE_REGISTER_MEM_GEN8;
5590 *cs++ = hw[dw];
5591 *cs++ = lower_32_bits(scratch->node.start + x);
5592 *cs++ = upper_32_bits(scratch->node.start + x);
5593
5594 dw += 2;
5595 x += 4;
5596 }
5597 } while (dw < PAGE_SIZE / sizeof(u32) &&
5598 (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
5599
5600 *cs++ = MI_BATCH_BUFFER_END;
5601
5602 shmem_unpin_map(ce->engine->default_state, defaults);
5603
5604 i915_gem_object_flush_map(batch->obj);
5605 i915_gem_object_unpin_map(batch->obj);
5606
5607 return batch;
5608 }
5609
5610 static int move_to_active(struct i915_request *rq,
5611 struct i915_vma *vma,
5612 unsigned int flags)
5613 {
5614 int err;
5615
5616 i915_vma_lock(vma);
5617 err = i915_request_await_object(rq, vma->obj, flags);
5618 if (!err)
5619 err = i915_vma_move_to_active(vma, rq, flags);
5620 i915_vma_unlock(vma);
5621
5622 return err;
5623 }
5624
5625 static struct i915_request *
5626 record_registers(struct intel_context *ce,
5627 struct i915_vma *before,
5628 struct i915_vma *after,
5629 u32 *sema)
5630 {
5631 struct i915_vma *b_before, *b_after;
5632 struct i915_request *rq;
5633 u32 *cs;
5634 int err;
5635
5636 b_before = store_context(ce, before);
5637 if (IS_ERR(b_before))
5638 return ERR_CAST(b_before);
5639
5640 b_after = store_context(ce, after);
5641 if (IS_ERR(b_after)) {
5642 rq = ERR_CAST(b_after);
5643 goto err_before;
5644 }
5645
5646 rq = intel_context_create_request(ce);
5647 if (IS_ERR(rq))
5648 goto err_after;
5649
5650 err = move_to_active(rq, before, EXEC_OBJECT_WRITE);
5651 if (err)
5652 goto err_rq;
5653
5654 err = move_to_active(rq, b_before, 0);
5655 if (err)
5656 goto err_rq;
5657
5658 err = move_to_active(rq, after, EXEC_OBJECT_WRITE);
5659 if (err)
5660 goto err_rq;
5661
5662 err = move_to_active(rq, b_after, 0);
5663 if (err)
5664 goto err_rq;
5665
5666 cs = intel_ring_begin(rq, 14);
5667 if (IS_ERR(cs)) {
5668 err = PTR_ERR(cs);
5669 goto err_rq;
5670 }
5671
5672 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
5673 *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
5674 *cs++ = lower_32_bits(b_before->node.start);
5675 *cs++ = upper_32_bits(b_before->node.start);
5676
5677 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
5678 *cs++ = MI_SEMAPHORE_WAIT |
5679 MI_SEMAPHORE_GLOBAL_GTT |
5680 MI_SEMAPHORE_POLL |
5681 MI_SEMAPHORE_SAD_NEQ_SDD;
5682 *cs++ = 0;
5683 *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) +
5684 offset_in_page(sema);
5685 *cs++ = 0;
5686 *cs++ = MI_NOOP;
5687
5688 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
5689 *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
5690 *cs++ = lower_32_bits(b_after->node.start);
5691 *cs++ = upper_32_bits(b_after->node.start);
5692
5693 intel_ring_advance(rq, cs);
5694
5695 WRITE_ONCE(*sema, 0);
5696 i915_request_get(rq);
5697 i915_request_add(rq);
5698 err_after:
5699 i915_vma_put(b_after);
5700 err_before:
5701 i915_vma_put(b_before);
5702 return rq;
5703
5704 err_rq:
5705 i915_request_add(rq);
5706 rq = ERR_PTR(err);
5707 goto err_after;
5708 }
5709
5710 static struct i915_vma *load_context(struct intel_context *ce, u32 poison)
5711 {
5712 struct i915_vma *batch;
5713 u32 dw, *cs, *hw;
5714 u32 *defaults;
5715
5716 batch = create_user_vma(ce->vm, SZ_64K);
5717 if (IS_ERR(batch))
5718 return batch;
5719
5720 cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC);
5721 if (IS_ERR(cs)) {
5722 i915_vma_put(batch);
5723 return ERR_CAST(cs);
5724 }
5725
5726 defaults = shmem_pin_map(ce->engine->default_state);
5727 if (!defaults) {
5728 i915_gem_object_unpin_map(batch->obj);
5729 i915_vma_put(batch);
5730 return ERR_PTR(-ENOMEM);
5731 }
5732
5733 dw = 0;
5734 hw = defaults;
5735 hw += LRC_STATE_OFFSET / sizeof(*hw);
5736 do {
5737 u32 len = hw[dw] & 0x7f;
5738
5739 if (hw[dw] == 0) {
5740 dw++;
5741 continue;
5742 }
5743
5744 if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
5745 dw += len + 2;
5746 continue;
5747 }
5748
5749 dw++;
5750 len = (len + 1) / 2;
5751 *cs++ = MI_LOAD_REGISTER_IMM(len);
5752 while (len--) {
5753 *cs++ = hw[dw];
5754 *cs++ = poison;
5755 dw += 2;
5756 }
5757 } while (dw < PAGE_SIZE / sizeof(u32) &&
5758 (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
5759
5760 *cs++ = MI_BATCH_BUFFER_END;
5761
5762 shmem_unpin_map(ce->engine->default_state, defaults);
5763
5764 i915_gem_object_flush_map(batch->obj);
5765 i915_gem_object_unpin_map(batch->obj);
5766
5767 return batch;
5768 }
5769
5770 static int poison_registers(struct intel_context *ce, u32 poison, u32 *sema)
5771 {
5772 struct i915_request *rq;
5773 struct i915_vma *batch;
5774 u32 *cs;
5775 int err;
5776
5777 batch = load_context(ce, poison);
5778 if (IS_ERR(batch))
5779 return PTR_ERR(batch);
5780
5781 rq = intel_context_create_request(ce);
5782 if (IS_ERR(rq)) {
5783 err = PTR_ERR(rq);
5784 goto err_batch;
5785 }
5786
5787 err = move_to_active(rq, batch, 0);
5788 if (err)
5789 goto err_rq;
5790
5791 cs = intel_ring_begin(rq, 8);
5792 if (IS_ERR(cs)) {
5793 err = PTR_ERR(cs);
5794 goto err_rq;
5795 }
5796
5797 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
5798 *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
5799 *cs++ = lower_32_bits(batch->node.start);
5800 *cs++ = upper_32_bits(batch->node.start);
5801
5802 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
5803 *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) +
5804 offset_in_page(sema);
5805 *cs++ = 0;
5806 *cs++ = 1;
5807
5808 intel_ring_advance(rq, cs);
5809
5810 rq->sched.attr.priority = I915_PRIORITY_BARRIER;
5811 err_rq:
5812 i915_request_add(rq);
5813 err_batch:
5814 i915_vma_put(batch);
5815 return err;
5816 }
5817
5818 static bool is_moving(u32 a, u32 b)
5819 {
5820 return a != b;
5821 }
5822
5823 static int compare_isolation(struct intel_engine_cs *engine,
5824 struct i915_vma *ref[2],
5825 struct i915_vma *result[2],
5826 struct intel_context *ce,
5827 u32 poison)
5828 {
5829 u32 x, dw, *hw, *lrc;
5830 u32 *A[2], *B[2];
5831 u32 *defaults;
5832 int err = 0;
5833
5834 A[0] = i915_gem_object_pin_map(ref[0]->obj, I915_MAP_WC);
5835 if (IS_ERR(A[0]))
5836 return PTR_ERR(A[0]);
5837
5838 A[1] = i915_gem_object_pin_map(ref[1]->obj, I915_MAP_WC);
5839 if (IS_ERR(A[1])) {
5840 err = PTR_ERR(A[1]);
5841 goto err_A0;
5842 }
5843
5844 B[0] = i915_gem_object_pin_map(result[0]->obj, I915_MAP_WC);
5845 if (IS_ERR(B[0])) {
5846 err = PTR_ERR(B[0]);
5847 goto err_A1;
5848 }
5849
5850 B[1] = i915_gem_object_pin_map(result[1]->obj, I915_MAP_WC);
5851 if (IS_ERR(B[1])) {
5852 err = PTR_ERR(B[1]);
5853 goto err_B0;
5854 }
5855
5856 lrc = i915_gem_object_pin_map(ce->state->obj,
5857 i915_coherent_map_type(engine->i915));
5858 if (IS_ERR(lrc)) {
5859 err = PTR_ERR(lrc);
5860 goto err_B1;
5861 }
5862 lrc += LRC_STATE_OFFSET / sizeof(*hw);
5863
5864 defaults = shmem_pin_map(ce->engine->default_state);
5865 if (!defaults) {
5866 err = -ENOMEM;
5867 goto err_lrc;
5868 }
5869
5870 x = 0;
5871 dw = 0;
5872 hw = defaults;
5873 hw += LRC_STATE_OFFSET / sizeof(*hw);
5874 do {
5875 u32 len = hw[dw] & 0x7f;
5876
5877 if (hw[dw] == 0) {
5878 dw++;
5879 continue;
5880 }
5881
5882 if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
5883 dw += len + 2;
5884 continue;
5885 }
5886
5887 dw++;
5888 len = (len + 1) / 2;
5889 while (len--) {
5890 if (!is_moving(A[0][x], A[1][x]) &&
5891 (A[0][x] != B[0][x] || A[1][x] != B[1][x])) {
5892 switch (hw[dw] & 4095) {
5893 case 0x30: /* RING_HEAD */
5894 case 0x34: /* RING_TAIL */
5895 break;
5896
5897 default:
5898 pr_err("%s[%d]: Mismatch for register %4x, default %08x, reference %08x, result (%08x, %08x), poison %08x, context %08x\n",
5899 engine->name, dw,
5900 hw[dw], hw[dw + 1],
5901 A[0][x], B[0][x], B[1][x],
5902 poison, lrc[dw + 1]);
5903 err = -EINVAL;
5904 }
5905 }
5906 dw += 2;
5907 x++;
5908 }
5909 } while (dw < PAGE_SIZE / sizeof(u32) &&
5910 (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
5911
5912 shmem_unpin_map(ce->engine->default_state, defaults);
5913 err_lrc:
5914 i915_gem_object_unpin_map(ce->state->obj);
5915 err_B1:
5916 i915_gem_object_unpin_map(result[1]->obj);
5917 err_B0:
5918 i915_gem_object_unpin_map(result[0]->obj);
5919 err_A1:
5920 i915_gem_object_unpin_map(ref[1]->obj);
5921 err_A0:
5922 i915_gem_object_unpin_map(ref[0]->obj);
5923 return err;
5924 }
5925
5926 static int __lrc_isolation(struct intel_engine_cs *engine, u32 poison)
5927 {
5928 u32 *sema = memset32(engine->status_page.addr + 1000, 0, 1);
5929 struct i915_vma *ref[2], *result[2];
5930 struct intel_context *A, *B;
5931 struct i915_request *rq;
5932 int err;
5933
5934 A = intel_context_create(engine);
5935 if (IS_ERR(A))
5936 return PTR_ERR(A);
5937
5938 B = intel_context_create(engine);
5939 if (IS_ERR(B)) {
5940 err = PTR_ERR(B);
5941 goto err_A;
5942 }
5943
5944 ref[0] = create_user_vma(A->vm, SZ_64K);
5945 if (IS_ERR(ref[0])) {
5946 err = PTR_ERR(ref[0]);
5947 goto err_B;
5948 }
5949
5950 ref[1] = create_user_vma(A->vm, SZ_64K);
5951 if (IS_ERR(ref[1])) {
5952 err = PTR_ERR(ref[1]);
5953 goto err_ref0;
5954 }
5955
5956 rq = record_registers(A, ref[0], ref[1], sema);
5957 if (IS_ERR(rq)) {
5958 err = PTR_ERR(rq);
5959 goto err_ref1;
5960 }
5961
5962 WRITE_ONCE(*sema, 1);
5963 wmb();
5964
5965 if (i915_request_wait(rq, 0, HZ / 2) < 0) {
5966 i915_request_put(rq);
5967 err = -ETIME;
5968 goto err_ref1;
5969 }
5970 i915_request_put(rq);
5971
5972 result[0] = create_user_vma(A->vm, SZ_64K);
5973 if (IS_ERR(result[0])) {
5974 err = PTR_ERR(result[0]);
5975 goto err_ref1;
5976 }
5977
5978 result[1] = create_user_vma(A->vm, SZ_64K);
5979 if (IS_ERR(result[1])) {
5980 err = PTR_ERR(result[1]);
5981 goto err_result0;
5982 }
5983
5984 rq = record_registers(A, result[0], result[1], sema);
5985 if (IS_ERR(rq)) {
5986 err = PTR_ERR(rq);
5987 goto err_result1;
5988 }
5989
5990 err = poison_registers(B, poison, sema);
5991 if (err) {
5992 WRITE_ONCE(*sema, -1);
5993 i915_request_put(rq);
5994 goto err_result1;
5995 }
5996
5997 if (i915_request_wait(rq, 0, HZ / 2) < 0) {
5998 i915_request_put(rq);
5999 err = -ETIME;
6000 goto err_result1;
6001 }
6002 i915_request_put(rq);
6003
6004 err = compare_isolation(engine, ref, result, A, poison);
6005
6006 err_result1:
6007 i915_vma_put(result[1]);
6008 err_result0:
6009 i915_vma_put(result[0]);
6010 err_ref1:
6011 i915_vma_put(ref[1]);
6012 err_ref0:
6013 i915_vma_put(ref[0]);
6014 err_B:
6015 intel_context_put(B);
6016 err_A:
6017 intel_context_put(A);
6018 return err;
6019 }
6020
6021 static bool skip_isolation(const struct intel_engine_cs *engine)
6022 {
6023 if (engine->class == COPY_ENGINE_CLASS && INTEL_GEN(engine->i915) == 9)
6024 return true;
6025
6026 if (engine->class == RENDER_CLASS && INTEL_GEN(engine->i915) == 11)
6027 return true;
6028
6029 return false;
6030 }
6031
6032 static int live_lrc_isolation(void *arg)
6033 {
6034 struct intel_gt *gt = arg;
6035 struct intel_engine_cs *engine;
6036 enum intel_engine_id id;
6037 const u32 poison[] = {
6038 STACK_MAGIC,
6039 0x3a3a3a3a,
6040 0x5c5c5c5c,
6041 0xffffffff,
6042 0xffff0000,
6043 };
6044 int err = 0;
6045
6046 /*
6047 * Our goal is try and verify that per-context state cannot be
6048 * tampered with by another non-privileged client.
6049 *
6050 * We take the list of context registers from the LRI in the default
6051 * context image and attempt to modify that list from a remote context.
6052 */
6053
6054 for_each_engine(engine, gt, id) {
6055 int i;
6056
6057 /* Just don't even ask */
6058 if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN) &&
6059 skip_isolation(engine))
6060 continue;
6061
6062 intel_engine_pm_get(engine);
6063 for (i = 0; i < ARRAY_SIZE(poison); i++) {
6064 int result;
6065
6066 result = __lrc_isolation(engine, poison[i]);
6067 if (result && !err)
6068 err = result;
6069
6070 result = __lrc_isolation(engine, ~poison[i]);
6071 if (result && !err)
6072 err = result;
6073 }
6074 intel_engine_pm_put(engine);
6075 if (igt_flush_test(gt->i915)) {
6076 err = -EIO;
6077 break;
6078 }
6079 }
6080
6081 return err;
6082 }
6083
6084 static int indirect_ctx_submit_req(struct intel_context *ce)
6085 {
6086 struct i915_request *rq;
6087 int err = 0;
6088
6089 rq = intel_context_create_request(ce);
6090 if (IS_ERR(rq))
6091 return PTR_ERR(rq);
6092
6093 i915_request_get(rq);
6094 i915_request_add(rq);
6095
6096 if (i915_request_wait(rq, 0, HZ / 5) < 0)
6097 err = -ETIME;
6098
6099 i915_request_put(rq);
6100
6101 return err;
6102 }
6103
6104 #define CTX_BB_CANARY_OFFSET (3 * 1024)
6105 #define CTX_BB_CANARY_INDEX (CTX_BB_CANARY_OFFSET / sizeof(u32))
6106
6107 static u32 *
6108 emit_indirect_ctx_bb_canary(const struct intel_context *ce, u32 *cs)
6109 {
6110 *cs++ = MI_STORE_REGISTER_MEM_GEN8 |
6111 MI_SRM_LRM_GLOBAL_GTT |
6112 MI_LRI_LRM_CS_MMIO;
6113 *cs++ = i915_mmio_reg_offset(RING_START(0));
6114 *cs++ = i915_ggtt_offset(ce->state) +
6115 context_wa_bb_offset(ce) +
6116 CTX_BB_CANARY_OFFSET;
6117 *cs++ = 0;
6118
6119 return cs;
6120 }
6121
6122 static void
6123 indirect_ctx_bb_setup(struct intel_context *ce)
6124 {
6125 u32 *cs = context_indirect_bb(ce);
6126
6127 cs[CTX_BB_CANARY_INDEX] = 0xdeadf00d;
6128
6129 setup_indirect_ctx_bb(ce, ce->engine, emit_indirect_ctx_bb_canary);
6130 }
6131
6132 static bool check_ring_start(struct intel_context *ce)
6133 {
6134 const u32 * const ctx_bb = (void *)(ce->lrc_reg_state) -
6135 LRC_STATE_OFFSET + context_wa_bb_offset(ce);
6136
6137 if (ctx_bb[CTX_BB_CANARY_INDEX] == ce->lrc_reg_state[CTX_RING_START])
6138 return true;
6139
6140 pr_err("ring start mismatch: canary 0x%08x vs state 0x%08x\n",
6141 ctx_bb[CTX_BB_CANARY_INDEX],
6142 ce->lrc_reg_state[CTX_RING_START]);
6143
6144 return false;
6145 }
6146
6147 static int indirect_ctx_bb_check(struct intel_context *ce)
6148 {
6149 int err;
6150
6151 err = indirect_ctx_submit_req(ce);
6152 if (err)
6153 return err;
6154
6155 if (!check_ring_start(ce))
6156 return -EINVAL;
6157
6158 return 0;
6159 }
6160
6161 static int __live_lrc_indirect_ctx_bb(struct intel_engine_cs *engine)
6162 {
6163 struct intel_context *a, *b;
6164 int err;
6165
6166 a = intel_context_create(engine);
6167 if (IS_ERR(a))
6168 return PTR_ERR(a);
6169 err = intel_context_pin(a);
6170 if (err)
6171 goto put_a;
6172
6173 b = intel_context_create(engine);
6174 if (IS_ERR(b)) {
6175 err = PTR_ERR(b);
6176 goto unpin_a;
6177 }
6178 err = intel_context_pin(b);
6179 if (err)
6180 goto put_b;
6181
6182 /* We use the already reserved extra page in context state */
6183 if (!a->wa_bb_page) {
6184 GEM_BUG_ON(b->wa_bb_page);
6185 GEM_BUG_ON(INTEL_GEN(engine->i915) == 12);
6186 goto unpin_b;
6187 }
6188
6189 /*
6190 * In order to test that our per context bb is truly per context,
6191 * and executes at the intended spot on context restoring process,
6192 * make the batch store the ring start value to memory.
6193 * As ring start is restored apriori of starting the indirect ctx bb and
6194 * as it will be different for each context, it fits to this purpose.
6195 */
6196 indirect_ctx_bb_setup(a);
6197 indirect_ctx_bb_setup(b);
6198
6199 err = indirect_ctx_bb_check(a);
6200 if (err)
6201 goto unpin_b;
6202
6203 err = indirect_ctx_bb_check(b);
6204
6205 unpin_b:
6206 intel_context_unpin(b);
6207 put_b:
6208 intel_context_put(b);
6209 unpin_a:
6210 intel_context_unpin(a);
6211 put_a:
6212 intel_context_put(a);
6213
6214 return err;
6215 }
6216
6217 static int live_lrc_indirect_ctx_bb(void *arg)
6218 {
6219 struct intel_gt *gt = arg;
6220 struct intel_engine_cs *engine;
6221 enum intel_engine_id id;
6222 int err = 0;
6223
6224 for_each_engine(engine, gt, id) {
6225 intel_engine_pm_get(engine);
6226 err = __live_lrc_indirect_ctx_bb(engine);
6227 intel_engine_pm_put(engine);
6228
6229 if (igt_flush_test(gt->i915))
6230 err = -EIO;
6231
6232 if (err)
6233 break;
6234 }
6235
6236 return err;
6237 }
6238
6239 static void garbage_reset(struct intel_engine_cs *engine,
6240 struct i915_request *rq)
6241 {
6242 const unsigned int bit = I915_RESET_ENGINE + engine->id;
6243 unsigned long *lock = &engine->gt->reset.flags;
6244
6245 if (test_and_set_bit(bit, lock))
6246 return;
6247
6248 tasklet_disable(&engine->execlists.tasklet);
6249
6250 if (!rq->fence.error)
6251 intel_engine_reset(engine, NULL);
6252
6253 tasklet_enable(&engine->execlists.tasklet);
6254 clear_and_wake_up_bit(bit, lock);
6255 }
6256
6257 static struct i915_request *garbage(struct intel_context *ce,
6258 struct rnd_state *prng)
6259 {
6260 struct i915_request *rq;
6261 int err;
6262
6263 err = intel_context_pin(ce);
6264 if (err)
6265 return ERR_PTR(err);
6266
6267 prandom_bytes_state(prng,
6268 ce->lrc_reg_state,
6269 ce->engine->context_size -
6270 LRC_STATE_OFFSET);
6271
6272 rq = intel_context_create_request(ce);
6273 if (IS_ERR(rq)) {
6274 err = PTR_ERR(rq);
6275 goto err_unpin;
6276 }
6277
6278 i915_request_get(rq);
6279 i915_request_add(rq);
6280 return rq;
6281
6282 err_unpin:
6283 intel_context_unpin(ce);
6284 return ERR_PTR(err);
6285 }
6286
6287 static int __lrc_garbage(struct intel_engine_cs *engine, struct rnd_state *prng)
6288 {
6289 struct intel_context *ce;
6290 struct i915_request *hang;
6291 int err = 0;
6292
6293 ce = intel_context_create(engine);
6294 if (IS_ERR(ce))
6295 return PTR_ERR(ce);
6296
6297 hang = garbage(ce, prng);
6298 if (IS_ERR(hang)) {
6299 err = PTR_ERR(hang);
6300 goto err_ce;
6301 }
6302
6303 if (wait_for_submit(engine, hang, HZ / 2)) {
6304 i915_request_put(hang);
6305 err = -ETIME;
6306 goto err_ce;
6307 }
6308
6309 intel_context_set_banned(ce);
6310 garbage_reset(engine, hang);
6311
6312 intel_engine_flush_submission(engine);
6313 if (!hang->fence.error) {
6314 i915_request_put(hang);
6315 pr_err("%s: corrupted context was not reset\n",
6316 engine->name);
6317 err = -EINVAL;
6318 goto err_ce;
6319 }
6320
6321 if (i915_request_wait(hang, 0, HZ / 2) < 0) {
6322 pr_err("%s: corrupted context did not recover\n",
6323 engine->name);
6324 i915_request_put(hang);
6325 err = -EIO;
6326 goto err_ce;
6327 }
6328 i915_request_put(hang);
6329
6330 err_ce:
6331 intel_context_put(ce);
6332 return err;
6333 }
6334
6335 static int live_lrc_garbage(void *arg)
6336 {
6337 struct intel_gt *gt = arg;
6338 struct intel_engine_cs *engine;
6339 enum intel_engine_id id;
6340
6341 /*
6342 * Verify that we can recover if one context state is completely
6343 * corrupted.
6344 */
6345
6346 if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN))
6347 return 0;
6348
6349 for_each_engine(engine, gt, id) {
6350 I915_RND_STATE(prng);
6351 int err = 0, i;
6352
6353 if (!intel_has_reset_engine(engine->gt))
6354 continue;
6355
6356 intel_engine_pm_get(engine);
6357 for (i = 0; i < 3; i++) {
6358 err = __lrc_garbage(engine, &prng);
6359 if (err)
6360 break;
6361 }
6362 intel_engine_pm_put(engine);
6363
6364 if (igt_flush_test(gt->i915))
6365 err = -EIO;
6366 if (err)
6367 return err;
6368 }
6369
6370 return 0;
6371 }
6372
6373 static int __live_pphwsp_runtime(struct intel_engine_cs *engine)
6374 {
6375 struct intel_context *ce;
6376 struct i915_request *rq;
6377 IGT_TIMEOUT(end_time);
6378 int err;
6379
6380 ce = intel_context_create(engine);
6381 if (IS_ERR(ce))
6382 return PTR_ERR(ce);
6383
6384 ce->runtime.num_underflow = 0;
6385 ce->runtime.max_underflow = 0;
6386
6387 do {
6388 unsigned int loop = 1024;
6389
6390 while (loop) {
6391 rq = intel_context_create_request(ce);
6392 if (IS_ERR(rq)) {
6393 err = PTR_ERR(rq);
6394 goto err_rq;
6395 }
6396
6397 if (--loop == 0)
6398 i915_request_get(rq);
6399
6400 i915_request_add(rq);
6401 }
6402
6403 if (__igt_timeout(end_time, NULL))
6404 break;
6405
6406 i915_request_put(rq);
6407 } while (1);
6408
6409 err = i915_request_wait(rq, 0, HZ / 5);
6410 if (err < 0) {
6411 pr_err("%s: request not completed!\n", engine->name);
6412 goto err_wait;
6413 }
6414
6415 igt_flush_test(engine->i915);
6416
6417 pr_info("%s: pphwsp runtime %lluns, average %lluns\n",
6418 engine->name,
6419 intel_context_get_total_runtime_ns(ce),
6420 intel_context_get_avg_runtime_ns(ce));
6421
6422 err = 0;
6423 if (ce->runtime.num_underflow) {
6424 pr_err("%s: pphwsp underflow %u time(s), max %u cycles!\n",
6425 engine->name,
6426 ce->runtime.num_underflow,
6427 ce->runtime.max_underflow);
6428 GEM_TRACE_DUMP();
6429 err = -EOVERFLOW;
6430 }
6431
6432 err_wait:
6433 i915_request_put(rq);
6434 err_rq:
6435 intel_context_put(ce);
6436 return err;
6437 }
6438
6439 static int live_pphwsp_runtime(void *arg)
6440 {
6441 struct intel_gt *gt = arg;
6442 struct intel_engine_cs *engine;
6443 enum intel_engine_id id;
6444 int err = 0;
6445
6446 /*
6447 * Check that cumulative context runtime as stored in the pphwsp[16]
6448 * is monotonic.
6449 */
6450
6451 for_each_engine(engine, gt, id) {
6452 err = __live_pphwsp_runtime(engine);
6453 if (err)
6454 break;
6455 }
6456
6457 if (igt_flush_test(gt->i915))
6458 err = -EIO;
6459
6460 return err;
6461 }
6462
6463 int intel_lrc_live_selftests(struct drm_i915_private *i915)
6464 {
6465 static const struct i915_subtest tests[] = {
6466 SUBTEST(live_lrc_layout),
6467 SUBTEST(live_lrc_fixed),
6468 SUBTEST(live_lrc_state),
6469 SUBTEST(live_lrc_gpr),
6470 SUBTEST(live_lrc_isolation),
6471 SUBTEST(live_lrc_timestamp),
6472 SUBTEST(live_lrc_garbage),
6473 SUBTEST(live_pphwsp_runtime),
6474 SUBTEST(live_lrc_indirect_ctx_bb),
6475 };
6476
6477 if (!HAS_LOGICAL_RING_CONTEXTS(i915))
6478 return 0;
6479
6480 return intel_gt_live_subtests(tests, &i915->gt);
6481 }
Linux with added FPC-III support