?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'io_uring-5.11-2021-01-16' of git://git.kernel.dk/linux-block
[linux/fpc-iii.git] / drivers / gpu / drm / i915 / gem / selftests / i915_gem_context.c
blobd3f87dc4eda3822dd5fda9816534dd74008c6cf8
1 /*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright © 2017 Intel Corporation
5 */
6
7 #include <linux/prime_numbers.h>
8
9 #include "gem/i915_gem_pm.h"
10 #include "gt/intel_engine_pm.h"
11 #include "gt/intel_gt.h"
12 #include "gt/intel_gt_requests.h"
13 #include "gt/intel_reset.h"
14 #include "i915_selftest.h"
15
16 #include "gem/selftests/igt_gem_utils.h"
17 #include "selftests/i915_random.h"
18 #include "selftests/igt_flush_test.h"
19 #include "selftests/igt_live_test.h"
20 #include "selftests/igt_reset.h"
21 #include "selftests/igt_spinner.h"
22 #include "selftests/mock_drm.h"
23 #include "selftests/mock_gem_device.h"
24
25 #include "huge_gem_object.h"
26 #include "igt_gem_utils.h"
27
28 #define DW_PER_PAGE (PAGE_SIZE / sizeof(u32))
29
30 static inline struct i915_address_space *ctx_vm(struct i915_gem_context *ctx)
31 {
32 /* single threaded, private ctx */
33 return rcu_dereference_protected(ctx->vm, true);
34 }
35
36 static int live_nop_switch(void *arg)
37 {
38 const unsigned int nctx = 1024;
39 struct drm_i915_private *i915 = arg;
40 struct intel_engine_cs *engine;
41 struct i915_gem_context **ctx;
42 struct igt_live_test t;
43 struct file *file;
44 unsigned long n;
45 int err = -ENODEV;
46
47 /*
48 * Create as many contexts as we can feasibly get away with
49 * and check we can switch between them rapidly.
50 *
51 * Serves as very simple stress test for submission and HW switching
52 * between contexts.
53 */
54
55 if (!DRIVER_CAPS(i915)->has_logical_contexts)
56 return 0;
57
58 file = mock_file(i915);
59 if (IS_ERR(file))
60 return PTR_ERR(file);
61
62 ctx = kcalloc(nctx, sizeof(*ctx), GFP_KERNEL);
63 if (!ctx) {
64 err = -ENOMEM;
65 goto out_file;
66 }
67
68 for (n = 0; n < nctx; n++) {
69 ctx[n] = live_context(i915, file);
70 if (IS_ERR(ctx[n])) {
71 err = PTR_ERR(ctx[n]);
72 goto out_file;
73 }
74 }
75
76 for_each_uabi_engine(engine, i915) {
77 struct i915_request *rq = NULL;
78 unsigned long end_time, prime;
79 ktime_t times[2] = {};
80
81 times[0] = ktime_get_raw();
82 for (n = 0; n < nctx; n++) {
83 struct i915_request *this;
84
85 this = igt_request_alloc(ctx[n], engine);
86 if (IS_ERR(this)) {
87 err = PTR_ERR(this);
88 goto out_file;
89 }
90 if (rq) {
91 i915_request_await_dma_fence(this, &rq->fence);
92 i915_request_put(rq);
93 }
94 rq = i915_request_get(this);
95 i915_request_add(this);
96 }
97 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
98 pr_err("Failed to populated %d contexts\n", nctx);
99 intel_gt_set_wedged(&i915->gt);
100 i915_request_put(rq);
101 err = -EIO;
102 goto out_file;
103 }
104 i915_request_put(rq);
105
106 times[1] = ktime_get_raw();
107
108 pr_info("Populated %d contexts on %s in %lluns\n",
109 nctx, engine->name, ktime_to_ns(times[1] - times[0]));
110
111 err = igt_live_test_begin(&t, i915, __func__, engine->name);
112 if (err)
113 goto out_file;
114
115 end_time = jiffies + i915_selftest.timeout_jiffies;
116 for_each_prime_number_from(prime, 2, 8192) {
117 times[1] = ktime_get_raw();
118
119 rq = NULL;
120 for (n = 0; n < prime; n++) {
121 struct i915_request *this;
122
123 this = igt_request_alloc(ctx[n % nctx], engine);
124 if (IS_ERR(this)) {
125 err = PTR_ERR(this);
126 goto out_file;
127 }
128
129 if (rq) { /* Force submission order */
130 i915_request_await_dma_fence(this, &rq->fence);
131 i915_request_put(rq);
132 }
133
134 /*
135 * This space is left intentionally blank.
136 *
137 * We do not actually want to perform any
138 * action with this request, we just want
139 * to measure the latency in allocation
140 * and submission of our breadcrumbs -
141 * ensuring that the bare request is sufficient
142 * for the system to work (i.e. proper HEAD
143 * tracking of the rings, interrupt handling,
144 * etc). It also gives us the lowest bounds
145 * for latency.
146 */
147
148 rq = i915_request_get(this);
149 i915_request_add(this);
150 }
151 GEM_BUG_ON(!rq);
152 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
153 pr_err("Switching between %ld contexts timed out\n",
154 prime);
155 intel_gt_set_wedged(&i915->gt);
156 i915_request_put(rq);
157 break;
158 }
159 i915_request_put(rq);
160
161 times[1] = ktime_sub(ktime_get_raw(), times[1]);
162 if (prime == 2)
163 times[0] = times[1];
164
165 if (__igt_timeout(end_time, NULL))
166 break;
167 }
168
169 err = igt_live_test_end(&t);
170 if (err)
171 goto out_file;
172
173 pr_info("Switch latencies on %s: 1 = %lluns, %lu = %lluns\n",
174 engine->name,
175 ktime_to_ns(times[0]),
176 prime - 1, div64_u64(ktime_to_ns(times[1]), prime - 1));
177 }
178
179 out_file:
180 fput(file);
181 return err;
182 }
183
184 struct parallel_switch {
185 struct task_struct *tsk;
186 struct intel_context *ce[2];
187 };
188
189 static int __live_parallel_switch1(void *data)
190 {
191 struct parallel_switch *arg = data;
192 IGT_TIMEOUT(end_time);
193 unsigned long count;
194
195 count = 0;
196 do {
197 struct i915_request *rq = NULL;
198 int err, n;
199
200 err = 0;
201 for (n = 0; !err && n < ARRAY_SIZE(arg->ce); n++) {
202 struct i915_request *prev = rq;
203
204 rq = i915_request_create(arg->ce[n]);
205 if (IS_ERR(rq)) {
206 i915_request_put(prev);
207 return PTR_ERR(rq);
208 }
209
210 i915_request_get(rq);
211 if (prev) {
212 err = i915_request_await_dma_fence(rq, &prev->fence);
213 i915_request_put(prev);
214 }
215
216 i915_request_add(rq);
217 }
218 if (i915_request_wait(rq, 0, HZ / 5) < 0)
219 err = -ETIME;
220 i915_request_put(rq);
221 if (err)
222 return err;
223
224 count++;
225 } while (!__igt_timeout(end_time, NULL));
226
227 pr_info("%s: %lu switches (sync)\n", arg->ce[0]->engine->name, count);
228 return 0;
229 }
230
231 static int __live_parallel_switchN(void *data)
232 {
233 struct parallel_switch *arg = data;
234 struct i915_request *rq = NULL;
235 IGT_TIMEOUT(end_time);
236 unsigned long count;
237 int n;
238
239 count = 0;
240 do {
241 for (n = 0; n < ARRAY_SIZE(arg->ce); n++) {
242 struct i915_request *prev = rq;
243 int err = 0;
244
245 rq = i915_request_create(arg->ce[n]);
246 if (IS_ERR(rq)) {
247 i915_request_put(prev);
248 return PTR_ERR(rq);
249 }
250
251 i915_request_get(rq);
252 if (prev) {
253 err = i915_request_await_dma_fence(rq, &prev->fence);
254 i915_request_put(prev);
255 }
256
257 i915_request_add(rq);
258 if (err) {
259 i915_request_put(rq);
260 return err;
261 }
262 }
263
264 count++;
265 } while (!__igt_timeout(end_time, NULL));
266 i915_request_put(rq);
267
268 pr_info("%s: %lu switches (many)\n", arg->ce[0]->engine->name, count);
269 return 0;
270 }
271
272 static int live_parallel_switch(void *arg)
273 {
274 struct drm_i915_private *i915 = arg;
275 static int (* const func[])(void *arg) = {
276 __live_parallel_switch1,
277 __live_parallel_switchN,
278 NULL,
279 };
280 struct parallel_switch *data = NULL;
281 struct i915_gem_engines *engines;
282 struct i915_gem_engines_iter it;
283 int (* const *fn)(void *arg);
284 struct i915_gem_context *ctx;
285 struct intel_context *ce;
286 struct file *file;
287 int n, m, count;
288 int err = 0;
289
290 /*
291 * Check we can process switches on all engines simultaneously.
292 */
293
294 if (!DRIVER_CAPS(i915)->has_logical_contexts)
295 return 0;
296
297 file = mock_file(i915);
298 if (IS_ERR(file))
299 return PTR_ERR(file);
300
301 ctx = live_context(i915, file);
302 if (IS_ERR(ctx)) {
303 err = PTR_ERR(ctx);
304 goto out_file;
305 }
306
307 engines = i915_gem_context_lock_engines(ctx);
308 count = engines->num_engines;
309
310 data = kcalloc(count, sizeof(*data), GFP_KERNEL);
311 if (!data) {
312 i915_gem_context_unlock_engines(ctx);
313 err = -ENOMEM;
314 goto out_file;
315 }
316
317 m = 0; /* Use the first context as our template for the engines */
318 for_each_gem_engine(ce, engines, it) {
319 err = intel_context_pin(ce);
320 if (err) {
321 i915_gem_context_unlock_engines(ctx);
322 goto out;
323 }
324 data[m++].ce[0] = intel_context_get(ce);
325 }
326 i915_gem_context_unlock_engines(ctx);
327
328 /* Clone the same set of engines into the other contexts */
329 for (n = 1; n < ARRAY_SIZE(data->ce); n++) {
330 ctx = live_context(i915, file);
331 if (IS_ERR(ctx)) {
332 err = PTR_ERR(ctx);
333 goto out;
334 }
335
336 for (m = 0; m < count; m++) {
337 if (!data[m].ce[0])
338 continue;
339
340 ce = intel_context_create(data[m].ce[0]->engine);
341 if (IS_ERR(ce))
342 goto out;
343
344 err = intel_context_pin(ce);
345 if (err) {
346 intel_context_put(ce);
347 goto out;
348 }
349
350 data[m].ce[n] = ce;
351 }
352 }
353
354 for (fn = func; !err && *fn; fn++) {
355 struct igt_live_test t;
356 int n;
357
358 err = igt_live_test_begin(&t, i915, __func__, "");
359 if (err)
360 break;
361
362 for (n = 0; n < count; n++) {
363 if (!data[n].ce[0])
364 continue;
365
366 data[n].tsk = kthread_run(*fn, &data[n],
367 "igt/parallel:%s",
368 data[n].ce[0]->engine->name);
369 if (IS_ERR(data[n].tsk)) {
370 err = PTR_ERR(data[n].tsk);
371 break;
372 }
373 get_task_struct(data[n].tsk);
374 }
375
376 yield(); /* start all threads before we kthread_stop() */
377
378 for (n = 0; n < count; n++) {
379 int status;
380
381 if (IS_ERR_OR_NULL(data[n].tsk))
382 continue;
383
384 status = kthread_stop(data[n].tsk);
385 if (status && !err)
386 err = status;
387
388 put_task_struct(data[n].tsk);
389 data[n].tsk = NULL;
390 }
391
392 if (igt_live_test_end(&t))
393 err = -EIO;
394 }
395
396 out:
397 for (n = 0; n < count; n++) {
398 for (m = 0; m < ARRAY_SIZE(data->ce); m++) {
399 if (!data[n].ce[m])
400 continue;
401
402 intel_context_unpin(data[n].ce[m]);
403 intel_context_put(data[n].ce[m]);
404 }
405 }
406 kfree(data);
407 out_file:
408 fput(file);
409 return err;
410 }
411
412 static unsigned long real_page_count(struct drm_i915_gem_object *obj)
413 {
414 return huge_gem_object_phys_size(obj) >> PAGE_SHIFT;
415 }
416
417 static unsigned long fake_page_count(struct drm_i915_gem_object *obj)
418 {
419 return huge_gem_object_dma_size(obj) >> PAGE_SHIFT;
420 }
421
422 static int gpu_fill(struct intel_context *ce,
423 struct drm_i915_gem_object *obj,
424 unsigned int dw)
425 {
426 struct i915_vma *vma;
427 int err;
428
429 GEM_BUG_ON(obj->base.size > ce->vm->total);
430 GEM_BUG_ON(!intel_engine_can_store_dword(ce->engine));
431
432 vma = i915_vma_instance(obj, ce->vm, NULL);
433 if (IS_ERR(vma))
434 return PTR_ERR(vma);
435
436 err = i915_vma_pin(vma, 0, 0, PIN_HIGH | PIN_USER);
437 if (err)
438 return err;
439
440 /*
441 * Within the GTT the huge objects maps every page onto
442 * its 1024 real pages (using phys_pfn = dma_pfn % 1024).
443 * We set the nth dword within the page using the nth
444 * mapping via the GTT - this should exercise the GTT mapping
445 * whilst checking that each context provides a unique view
446 * into the object.
447 */
448 err = igt_gpu_fill_dw(ce, vma,
449 (dw * real_page_count(obj)) << PAGE_SHIFT |
450 (dw * sizeof(u32)),
451 real_page_count(obj),
452 dw);
453 i915_vma_unpin(vma);
454
455 return err;
456 }
457
458 static int cpu_fill(struct drm_i915_gem_object *obj, u32 value)
459 {
460 const bool has_llc = HAS_LLC(to_i915(obj->base.dev));
461 unsigned int n, m, need_flush;
462 int err;
463
464 i915_gem_object_lock(obj, NULL);
465 err = i915_gem_object_prepare_write(obj, &need_flush);
466 if (err)
467 goto out;
468
469 for (n = 0; n < real_page_count(obj); n++) {
470 u32 *map;
471
472 map = kmap_atomic(i915_gem_object_get_page(obj, n));
473 for (m = 0; m < DW_PER_PAGE; m++)
474 map[m] = value;
475 if (!has_llc)
476 drm_clflush_virt_range(map, PAGE_SIZE);
477 kunmap_atomic(map);
478 }
479
480 i915_gem_object_finish_access(obj);
481 obj->read_domains = I915_GEM_DOMAIN_GTT | I915_GEM_DOMAIN_CPU;
482 obj->write_domain = 0;
483 out:
484 i915_gem_object_unlock(obj);
485 return err;
486 }
487
488 static noinline int cpu_check(struct drm_i915_gem_object *obj,
489 unsigned int idx, unsigned int max)
490 {
491 unsigned int n, m, needs_flush;
492 int err;
493
494 i915_gem_object_lock(obj, NULL);
495 err = i915_gem_object_prepare_read(obj, &needs_flush);
496 if (err)
497 goto out_unlock;
498
499 for (n = 0; n < real_page_count(obj); n++) {
500 u32 *map;
501
502 map = kmap_atomic(i915_gem_object_get_page(obj, n));
503 if (needs_flush & CLFLUSH_BEFORE)
504 drm_clflush_virt_range(map, PAGE_SIZE);
505
506 for (m = 0; m < max; m++) {
507 if (map[m] != m) {
508 pr_err("%pS: Invalid value at object %d page %d/%ld, offset %d/%d: found %x expected %x\n",
509 __builtin_return_address(0), idx,
510 n, real_page_count(obj), m, max,
511 map[m], m);
512 err = -EINVAL;
513 goto out_unmap;
514 }
515 }
516
517 for (; m < DW_PER_PAGE; m++) {
518 if (map[m] != STACK_MAGIC) {
519 pr_err("%pS: Invalid value at object %d page %d, offset %d: found %x expected %x (uninitialised)\n",
520 __builtin_return_address(0), idx, n, m,
521 map[m], STACK_MAGIC);
522 err = -EINVAL;
523 goto out_unmap;
524 }
525 }
526
527 out_unmap:
528 kunmap_atomic(map);
529 if (err)
530 break;
531 }
532
533 i915_gem_object_finish_access(obj);
534 out_unlock:
535 i915_gem_object_unlock(obj);
536 return err;
537 }
538
539 static int file_add_object(struct file *file, struct drm_i915_gem_object *obj)
540 {
541 int err;
542
543 GEM_BUG_ON(obj->base.handle_count);
544
545 /* tie the object to the drm_file for easy reaping */
546 err = idr_alloc(&to_drm_file(file)->object_idr,
547 &obj->base, 1, 0, GFP_KERNEL);
548 if (err < 0)
549 return err;
550
551 i915_gem_object_get(obj);
552 obj->base.handle_count++;
553 return 0;
554 }
555
556 static struct drm_i915_gem_object *
557 create_test_object(struct i915_address_space *vm,
558 struct file *file,
559 struct list_head *objects)
560 {
561 struct drm_i915_gem_object *obj;
562 u64 size;
563 int err;
564
565 /* Keep in GEM's good graces */
566 intel_gt_retire_requests(vm->gt);
567
568 size = min(vm->total / 2, 1024ull * DW_PER_PAGE * PAGE_SIZE);
569 size = round_down(size, DW_PER_PAGE * PAGE_SIZE);
570
571 obj = huge_gem_object(vm->i915, DW_PER_PAGE * PAGE_SIZE, size);
572 if (IS_ERR(obj))
573 return obj;
574
575 err = file_add_object(file, obj);
576 i915_gem_object_put(obj);
577 if (err)
578 return ERR_PTR(err);
579
580 err = cpu_fill(obj, STACK_MAGIC);
581 if (err) {
582 pr_err("Failed to fill object with cpu, err=%d\n",
583 err);
584 return ERR_PTR(err);
585 }
586
587 list_add_tail(&obj->st_link, objects);
588 return obj;
589 }
590
591 static unsigned long max_dwords(struct drm_i915_gem_object *obj)
592 {
593 unsigned long npages = fake_page_count(obj);
594
595 GEM_BUG_ON(!IS_ALIGNED(npages, DW_PER_PAGE));
596 return npages / DW_PER_PAGE;
597 }
598
599 static void throttle_release(struct i915_request **q, int count)
600 {
601 int i;
602
603 for (i = 0; i < count; i++) {
604 if (IS_ERR_OR_NULL(q[i]))
605 continue;
606
607 i915_request_put(fetch_and_zero(&q[i]));
608 }
609 }
610
611 static int throttle(struct intel_context *ce,
612 struct i915_request **q, int count)
613 {
614 int i;
615
616 if (!IS_ERR_OR_NULL(q[0])) {
617 if (i915_request_wait(q[0],
618 I915_WAIT_INTERRUPTIBLE,
619 MAX_SCHEDULE_TIMEOUT) < 0)
620 return -EINTR;
621
622 i915_request_put(q[0]);
623 }
624
625 for (i = 0; i < count - 1; i++)
626 q[i] = q[i + 1];
627
628 q[i] = intel_context_create_request(ce);
629 if (IS_ERR(q[i]))
630 return PTR_ERR(q[i]);
631
632 i915_request_get(q[i]);
633 i915_request_add(q[i]);
634
635 return 0;
636 }
637
638 static int igt_ctx_exec(void *arg)
639 {
640 struct drm_i915_private *i915 = arg;
641 struct intel_engine_cs *engine;
642 int err = -ENODEV;
643
644 /*
645 * Create a few different contexts (with different mm) and write
646 * through each ctx/mm using the GPU making sure those writes end
647 * up in the expected pages of our obj.
648 */
649
650 if (!DRIVER_CAPS(i915)->has_logical_contexts)
651 return 0;
652
653 for_each_uabi_engine(engine, i915) {
654 struct drm_i915_gem_object *obj = NULL;
655 unsigned long ncontexts, ndwords, dw;
656 struct i915_request *tq[5] = {};
657 struct igt_live_test t;
658 IGT_TIMEOUT(end_time);
659 LIST_HEAD(objects);
660 struct file *file;
661
662 if (!intel_engine_can_store_dword(engine))
663 continue;
664
665 if (!engine->context_size)
666 continue; /* No logical context support in HW */
667
668 file = mock_file(i915);
669 if (IS_ERR(file))
670 return PTR_ERR(file);
671
672 err = igt_live_test_begin(&t, i915, __func__, engine->name);
673 if (err)
674 goto out_file;
675
676 ncontexts = 0;
677 ndwords = 0;
678 dw = 0;
679 while (!time_after(jiffies, end_time)) {
680 struct i915_gem_context *ctx;
681 struct intel_context *ce;
682
683 ctx = kernel_context(i915);
684 if (IS_ERR(ctx)) {
685 err = PTR_ERR(ctx);
686 goto out_file;
687 }
688
689 ce = i915_gem_context_get_engine(ctx, engine->legacy_idx);
690 GEM_BUG_ON(IS_ERR(ce));
691
692 if (!obj) {
693 obj = create_test_object(ce->vm, file, &objects);
694 if (IS_ERR(obj)) {
695 err = PTR_ERR(obj);
696 intel_context_put(ce);
697 kernel_context_close(ctx);
698 goto out_file;
699 }
700 }
701
702 err = gpu_fill(ce, obj, dw);
703 if (err) {
704 pr_err("Failed to fill dword %lu [%lu/%lu] with gpu (%s) [full-ppgtt? %s], err=%d\n",
705 ndwords, dw, max_dwords(obj),
706 engine->name,
707 yesno(!!rcu_access_pointer(ctx->vm)),
708 err);
709 intel_context_put(ce);
710 kernel_context_close(ctx);
711 goto out_file;
712 }
713
714 err = throttle(ce, tq, ARRAY_SIZE(tq));
715 if (err) {
716 intel_context_put(ce);
717 kernel_context_close(ctx);
718 goto out_file;
719 }
720
721 if (++dw == max_dwords(obj)) {
722 obj = NULL;
723 dw = 0;
724 }
725
726 ndwords++;
727 ncontexts++;
728
729 intel_context_put(ce);
730 kernel_context_close(ctx);
731 }
732
733 pr_info("Submitted %lu contexts to %s, filling %lu dwords\n",
734 ncontexts, engine->name, ndwords);
735
736 ncontexts = dw = 0;
737 list_for_each_entry(obj, &objects, st_link) {
738 unsigned int rem =
739 min_t(unsigned int, ndwords - dw, max_dwords(obj));
740
741 err = cpu_check(obj, ncontexts++, rem);
742 if (err)
743 break;
744
745 dw += rem;
746 }
747
748 out_file:
749 throttle_release(tq, ARRAY_SIZE(tq));
750 if (igt_live_test_end(&t))
751 err = -EIO;
752
753 fput(file);
754 if (err)
755 return err;
756
757 i915_gem_drain_freed_objects(i915);
758 }
759
760 return 0;
761 }
762
763 static int igt_shared_ctx_exec(void *arg)
764 {
765 struct drm_i915_private *i915 = arg;
766 struct i915_request *tq[5] = {};
767 struct i915_gem_context *parent;
768 struct intel_engine_cs *engine;
769 struct igt_live_test t;
770 struct file *file;
771 int err = 0;
772
773 /*
774 * Create a few different contexts with the same mm and write
775 * through each ctx using the GPU making sure those writes end
776 * up in the expected pages of our obj.
777 */
778 if (!DRIVER_CAPS(i915)->has_logical_contexts)
779 return 0;
780
781 file = mock_file(i915);
782 if (IS_ERR(file))
783 return PTR_ERR(file);
784
785 parent = live_context(i915, file);
786 if (IS_ERR(parent)) {
787 err = PTR_ERR(parent);
788 goto out_file;
789 }
790
791 if (!parent->vm) { /* not full-ppgtt; nothing to share */
792 err = 0;
793 goto out_file;
794 }
795
796 err = igt_live_test_begin(&t, i915, __func__, "");
797 if (err)
798 goto out_file;
799
800 for_each_uabi_engine(engine, i915) {
801 unsigned long ncontexts, ndwords, dw;
802 struct drm_i915_gem_object *obj = NULL;
803 IGT_TIMEOUT(end_time);
804 LIST_HEAD(objects);
805
806 if (!intel_engine_can_store_dword(engine))
807 continue;
808
809 dw = 0;
810 ndwords = 0;
811 ncontexts = 0;
812 while (!time_after(jiffies, end_time)) {
813 struct i915_gem_context *ctx;
814 struct intel_context *ce;
815
816 ctx = kernel_context(i915);
817 if (IS_ERR(ctx)) {
818 err = PTR_ERR(ctx);
819 goto out_test;
820 }
821
822 mutex_lock(&ctx->mutex);
823 __assign_ppgtt(ctx, ctx_vm(parent));
824 mutex_unlock(&ctx->mutex);
825
826 ce = i915_gem_context_get_engine(ctx, engine->legacy_idx);
827 GEM_BUG_ON(IS_ERR(ce));
828
829 if (!obj) {
830 obj = create_test_object(ctx_vm(parent),
831 file, &objects);
832 if (IS_ERR(obj)) {
833 err = PTR_ERR(obj);
834 intel_context_put(ce);
835 kernel_context_close(ctx);
836 goto out_test;
837 }
838 }
839
840 err = gpu_fill(ce, obj, dw);
841 if (err) {
842 pr_err("Failed to fill dword %lu [%lu/%lu] with gpu (%s) [full-ppgtt? %s], err=%d\n",
843 ndwords, dw, max_dwords(obj),
844 engine->name,
845 yesno(!!rcu_access_pointer(ctx->vm)),
846 err);
847 intel_context_put(ce);
848 kernel_context_close(ctx);
849 goto out_test;
850 }
851
852 err = throttle(ce, tq, ARRAY_SIZE(tq));
853 if (err) {
854 intel_context_put(ce);
855 kernel_context_close(ctx);
856 goto out_test;
857 }
858
859 if (++dw == max_dwords(obj)) {
860 obj = NULL;
861 dw = 0;
862 }
863
864 ndwords++;
865 ncontexts++;
866
867 intel_context_put(ce);
868 kernel_context_close(ctx);
869 }
870 pr_info("Submitted %lu contexts to %s, filling %lu dwords\n",
871 ncontexts, engine->name, ndwords);
872
873 ncontexts = dw = 0;
874 list_for_each_entry(obj, &objects, st_link) {
875 unsigned int rem =
876 min_t(unsigned int, ndwords - dw, max_dwords(obj));
877
878 err = cpu_check(obj, ncontexts++, rem);
879 if (err)
880 goto out_test;
881
882 dw += rem;
883 }
884
885 i915_gem_drain_freed_objects(i915);
886 }
887 out_test:
888 throttle_release(tq, ARRAY_SIZE(tq));
889 if (igt_live_test_end(&t))
890 err = -EIO;
891 out_file:
892 fput(file);
893 return err;
894 }
895
896 static int rpcs_query_batch(struct drm_i915_gem_object *rpcs, struct i915_vma *vma)
897 {
898 u32 *cmd;
899
900 GEM_BUG_ON(INTEL_GEN(vma->vm->i915) < 8);
901
902 cmd = i915_gem_object_pin_map(rpcs, I915_MAP_WB);
903 if (IS_ERR(cmd))
904 return PTR_ERR(cmd);
905
906 *cmd++ = MI_STORE_REGISTER_MEM_GEN8;
907 *cmd++ = i915_mmio_reg_offset(GEN8_R_PWR_CLK_STATE);
908 *cmd++ = lower_32_bits(vma->node.start);
909 *cmd++ = upper_32_bits(vma->node.start);
910 *cmd = MI_BATCH_BUFFER_END;
911
912 __i915_gem_object_flush_map(rpcs, 0, 64);
913 i915_gem_object_unpin_map(rpcs);
914
915 intel_gt_chipset_flush(vma->vm->gt);
916
917 return 0;
918 }
919
920 static int
921 emit_rpcs_query(struct drm_i915_gem_object *obj,
922 struct intel_context *ce,
923 struct i915_request **rq_out)
924 {
925 struct drm_i915_private *i915 = to_i915(obj->base.dev);
926 struct i915_request *rq;
927 struct i915_gem_ww_ctx ww;
928 struct i915_vma *batch;
929 struct i915_vma *vma;
930 struct drm_i915_gem_object *rpcs;
931 int err;
932
933 GEM_BUG_ON(!intel_engine_can_store_dword(ce->engine));
934
935 if (INTEL_GEN(i915) < 8)
936 return -EINVAL;
937
938 vma = i915_vma_instance(obj, ce->vm, NULL);
939 if (IS_ERR(vma))
940 return PTR_ERR(vma);
941
942 rpcs = i915_gem_object_create_internal(i915, PAGE_SIZE);
943 if (IS_ERR(rpcs))
944 return PTR_ERR(rpcs);
945
946 batch = i915_vma_instance(rpcs, ce->vm, NULL);
947 if (IS_ERR(batch)) {
948 err = PTR_ERR(batch);
949 goto err_put;
950 }
951
952 i915_gem_ww_ctx_init(&ww, false);
953 retry:
954 err = i915_gem_object_lock(obj, &ww);
955 if (!err)
956 err = i915_gem_object_lock(rpcs, &ww);
957 if (!err)
958 err = i915_gem_object_set_to_gtt_domain(obj, false);
959 if (!err)
960 err = i915_vma_pin_ww(vma, &ww, 0, 0, PIN_USER);
961 if (err)
962 goto err_put;
963
964 err = i915_vma_pin_ww(batch, &ww, 0, 0, PIN_USER);
965 if (err)
966 goto err_vma;
967
968 err = rpcs_query_batch(rpcs, vma);
969 if (err)
970 goto err_batch;
971
972 rq = i915_request_create(ce);
973 if (IS_ERR(rq)) {
974 err = PTR_ERR(rq);
975 goto err_batch;
976 }
977
978 err = i915_request_await_object(rq, batch->obj, false);
979 if (err == 0)
980 err = i915_vma_move_to_active(batch, rq, 0);
981 if (err)
982 goto skip_request;
983
984 err = i915_request_await_object(rq, vma->obj, true);
985 if (err == 0)
986 err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
987 if (err)
988 goto skip_request;
989
990 if (rq->engine->emit_init_breadcrumb) {
991 err = rq->engine->emit_init_breadcrumb(rq);
992 if (err)
993 goto skip_request;
994 }
995
996 err = rq->engine->emit_bb_start(rq,
997 batch->node.start, batch->node.size,
998 0);
999 if (err)
1000 goto skip_request;
1001
1002 *rq_out = i915_request_get(rq);
1003
1004 skip_request:
1005 if (err)
1006 i915_request_set_error_once(rq, err);
1007 i915_request_add(rq);
1008 err_batch:
1009 i915_vma_unpin(batch);
1010 err_vma:
1011 i915_vma_unpin(vma);
1012 err_put:
1013 if (err == -EDEADLK) {
1014 err = i915_gem_ww_ctx_backoff(&ww);
1015 if (!err)
1016 goto retry;
1017 }
1018 i915_gem_ww_ctx_fini(&ww);
1019 i915_gem_object_put(rpcs);
1020 return err;
1021 }
1022
1023 #define TEST_IDLE BIT(0)
1024 #define TEST_BUSY BIT(1)
1025 #define TEST_RESET BIT(2)
1026
1027 static int
1028 __sseu_prepare(const char *name,
1029 unsigned int flags,
1030 struct intel_context *ce,
1031 struct igt_spinner **spin)
1032 {
1033 struct i915_request *rq;
1034 int ret;
1035
1036 *spin = NULL;
1037 if (!(flags & (TEST_BUSY | TEST_RESET)))
1038 return 0;
1039
1040 *spin = kzalloc(sizeof(**spin), GFP_KERNEL);
1041 if (!*spin)
1042 return -ENOMEM;
1043
1044 ret = igt_spinner_init(*spin, ce->engine->gt);
1045 if (ret)
1046 goto err_free;
1047
1048 rq = igt_spinner_create_request(*spin, ce, MI_NOOP);
1049 if (IS_ERR(rq)) {
1050 ret = PTR_ERR(rq);
1051 goto err_fini;
1052 }
1053
1054 i915_request_add(rq);
1055
1056 if (!igt_wait_for_spinner(*spin, rq)) {
1057 pr_err("%s: Spinner failed to start!\n", name);
1058 ret = -ETIMEDOUT;
1059 goto err_end;
1060 }
1061
1062 return 0;
1063
1064 err_end:
1065 igt_spinner_end(*spin);
1066 err_fini:
1067 igt_spinner_fini(*spin);
1068 err_free:
1069 kfree(fetch_and_zero(spin));
1070 return ret;
1071 }
1072
1073 static int
1074 __read_slice_count(struct intel_context *ce,
1075 struct drm_i915_gem_object *obj,
1076 struct igt_spinner *spin,
1077 u32 *rpcs)
1078 {
1079 struct i915_request *rq = NULL;
1080 u32 s_mask, s_shift;
1081 unsigned int cnt;
1082 u32 *buf, val;
1083 long ret;
1084
1085 ret = emit_rpcs_query(obj, ce, &rq);
1086 if (ret)
1087 return ret;
1088
1089 if (spin)
1090 igt_spinner_end(spin);
1091
1092 ret = i915_request_wait(rq, 0, MAX_SCHEDULE_TIMEOUT);
1093 i915_request_put(rq);
1094 if (ret < 0)
1095 return ret;
1096
1097 buf = i915_gem_object_pin_map(obj, I915_MAP_WB);
1098 if (IS_ERR(buf)) {
1099 ret = PTR_ERR(buf);
1100 return ret;
1101 }
1102
1103 if (INTEL_GEN(ce->engine->i915) >= 11) {
1104 s_mask = GEN11_RPCS_S_CNT_MASK;
1105 s_shift = GEN11_RPCS_S_CNT_SHIFT;
1106 } else {
1107 s_mask = GEN8_RPCS_S_CNT_MASK;
1108 s_shift = GEN8_RPCS_S_CNT_SHIFT;
1109 }
1110
1111 val = *buf;
1112 cnt = (val & s_mask) >> s_shift;
1113 *rpcs = val;
1114
1115 i915_gem_object_unpin_map(obj);
1116
1117 return cnt;
1118 }
1119
1120 static int
1121 __check_rpcs(const char *name, u32 rpcs, int slices, unsigned int expected,
1122 const char *prefix, const char *suffix)
1123 {
1124 if (slices == expected)
1125 return 0;
1126
1127 if (slices < 0) {
1128 pr_err("%s: %s read slice count failed with %d%s\n",
1129 name, prefix, slices, suffix);
1130 return slices;
1131 }
1132
1133 pr_err("%s: %s slice count %d is not %u%s\n",
1134 name, prefix, slices, expected, suffix);
1135
1136 pr_info("RPCS=0x%x; %u%sx%u%s\n",
1137 rpcs, slices,
1138 (rpcs & GEN8_RPCS_S_CNT_ENABLE) ? "*" : "",
1139 (rpcs & GEN8_RPCS_SS_CNT_MASK) >> GEN8_RPCS_SS_CNT_SHIFT,
1140 (rpcs & GEN8_RPCS_SS_CNT_ENABLE) ? "*" : "");
1141
1142 return -EINVAL;
1143 }
1144
1145 static int
1146 __sseu_finish(const char *name,
1147 unsigned int flags,
1148 struct intel_context *ce,
1149 struct drm_i915_gem_object *obj,
1150 unsigned int expected,
1151 struct igt_spinner *spin)
1152 {
1153 unsigned int slices = hweight32(ce->engine->sseu.slice_mask);
1154 u32 rpcs = 0;
1155 int ret = 0;
1156
1157 if (flags & TEST_RESET) {
1158 ret = intel_engine_reset(ce->engine, "sseu");
1159 if (ret)
1160 goto out;
1161 }
1162
1163 ret = __read_slice_count(ce, obj,
1164 flags & TEST_RESET ? NULL : spin, &rpcs);
1165 ret = __check_rpcs(name, rpcs, ret, expected, "Context", "!");
1166 if (ret)
1167 goto out;
1168
1169 ret = __read_slice_count(ce->engine->kernel_context, obj, NULL, &rpcs);
1170 ret = __check_rpcs(name, rpcs, ret, slices, "Kernel context", "!");
1171
1172 out:
1173 if (spin)
1174 igt_spinner_end(spin);
1175
1176 if ((flags & TEST_IDLE) && ret == 0) {
1177 ret = igt_flush_test(ce->engine->i915);
1178 if (ret)
1179 return ret;
1180
1181 ret = __read_slice_count(ce, obj, NULL, &rpcs);
1182 ret = __check_rpcs(name, rpcs, ret, expected,
1183 "Context", " after idle!");
1184 }
1185
1186 return ret;
1187 }
1188
1189 static int
1190 __sseu_test(const char *name,
1191 unsigned int flags,
1192 struct intel_context *ce,
1193 struct drm_i915_gem_object *obj,
1194 struct intel_sseu sseu)
1195 {
1196 struct igt_spinner *spin = NULL;
1197 int ret;
1198
1199 intel_engine_pm_get(ce->engine);
1200
1201 ret = __sseu_prepare(name, flags, ce, &spin);
1202 if (ret)
1203 goto out_pm;
1204
1205 ret = intel_context_reconfigure_sseu(ce, sseu);
1206 if (ret)
1207 goto out_spin;
1208
1209 ret = __sseu_finish(name, flags, ce, obj,
1210 hweight32(sseu.slice_mask), spin);
1211
1212 out_spin:
1213 if (spin) {
1214 igt_spinner_end(spin);
1215 igt_spinner_fini(spin);
1216 kfree(spin);
1217 }
1218 out_pm:
1219 intel_engine_pm_put(ce->engine);
1220 return ret;
1221 }
1222
1223 static int
1224 __igt_ctx_sseu(struct drm_i915_private *i915,
1225 const char *name,
1226 unsigned int flags)
1227 {
1228 struct drm_i915_gem_object *obj;
1229 int inst = 0;
1230 int ret = 0;
1231
1232 if (INTEL_GEN(i915) < 9)
1233 return 0;
1234
1235 if (flags & TEST_RESET)
1236 igt_global_reset_lock(&i915->gt);
1237
1238 obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
1239 if (IS_ERR(obj)) {
1240 ret = PTR_ERR(obj);
1241 goto out_unlock;
1242 }
1243
1244 do {
1245 struct intel_engine_cs *engine;
1246 struct intel_context *ce;
1247 struct intel_sseu pg_sseu;
1248
1249 engine = intel_engine_lookup_user(i915,
1250 I915_ENGINE_CLASS_RENDER,
1251 inst++);
1252 if (!engine)
1253 break;
1254
1255 if (hweight32(engine->sseu.slice_mask) < 2)
1256 continue;
1257
1258 if (!engine->gt->info.sseu.has_slice_pg)
1259 continue;
1260
1261 /*
1262 * Gen11 VME friendly power-gated configuration with
1263 * half enabled sub-slices.
1264 */
1265 pg_sseu = engine->sseu;
1266 pg_sseu.slice_mask = 1;
1267 pg_sseu.subslice_mask =
1268 ~(~0 << (hweight32(engine->sseu.subslice_mask) / 2));
1269
1270 pr_info("%s: SSEU subtest '%s', flags=%x, def_slices=%u, pg_slices=%u\n",
1271 engine->name, name, flags,
1272 hweight32(engine->sseu.slice_mask),
1273 hweight32(pg_sseu.slice_mask));
1274
1275 ce = intel_context_create(engine);
1276 if (IS_ERR(ce)) {
1277 ret = PTR_ERR(ce);
1278 goto out_put;
1279 }
1280
1281 ret = intel_context_pin(ce);
1282 if (ret)
1283 goto out_ce;
1284
1285 /* First set the default mask. */
1286 ret = __sseu_test(name, flags, ce, obj, engine->sseu);
1287 if (ret)
1288 goto out_unpin;
1289
1290 /* Then set a power-gated configuration. */
1291 ret = __sseu_test(name, flags, ce, obj, pg_sseu);
1292 if (ret)
1293 goto out_unpin;
1294
1295 /* Back to defaults. */
1296 ret = __sseu_test(name, flags, ce, obj, engine->sseu);
1297 if (ret)
1298 goto out_unpin;
1299
1300 /* One last power-gated configuration for the road. */
1301 ret = __sseu_test(name, flags, ce, obj, pg_sseu);
1302 if (ret)
1303 goto out_unpin;
1304
1305 out_unpin:
1306 intel_context_unpin(ce);
1307 out_ce:
1308 intel_context_put(ce);
1309 } while (!ret);
1310
1311 if (igt_flush_test(i915))
1312 ret = -EIO;
1313
1314 out_put:
1315 i915_gem_object_put(obj);
1316
1317 out_unlock:
1318 if (flags & TEST_RESET)
1319 igt_global_reset_unlock(&i915->gt);
1320
1321 if (ret)
1322 pr_err("%s: Failed with %d!\n", name, ret);
1323
1324 return ret;
1325 }
1326
1327 static int igt_ctx_sseu(void *arg)
1328 {
1329 struct {
1330 const char *name;
1331 unsigned int flags;
1332 } *phase, phases[] = {
1333 { .name = "basic", .flags = 0 },
1334 { .name = "idle", .flags = TEST_IDLE },
1335 { .name = "busy", .flags = TEST_BUSY },
1336 { .name = "busy-reset", .flags = TEST_BUSY | TEST_RESET },
1337 { .name = "busy-idle", .flags = TEST_BUSY | TEST_IDLE },
1338 { .name = "reset-idle", .flags = TEST_RESET | TEST_IDLE },
1339 };
1340 unsigned int i;
1341 int ret = 0;
1342
1343 for (i = 0, phase = phases; ret == 0 && i < ARRAY_SIZE(phases);
1344 i++, phase++)
1345 ret = __igt_ctx_sseu(arg, phase->name, phase->flags);
1346
1347 return ret;
1348 }
1349
1350 static int igt_ctx_readonly(void *arg)
1351 {
1352 struct drm_i915_private *i915 = arg;
1353 unsigned long idx, ndwords, dw, num_engines;
1354 struct drm_i915_gem_object *obj = NULL;
1355 struct i915_request *tq[5] = {};
1356 struct i915_gem_engines_iter it;
1357 struct i915_address_space *vm;
1358 struct i915_gem_context *ctx;
1359 struct intel_context *ce;
1360 struct igt_live_test t;
1361 I915_RND_STATE(prng);
1362 IGT_TIMEOUT(end_time);
1363 LIST_HEAD(objects);
1364 struct file *file;
1365 int err = -ENODEV;
1366
1367 /*
1368 * Create a few read-only objects (with the occasional writable object)
1369 * and try to write into these object checking that the GPU discards
1370 * any write to a read-only object.
1371 */
1372
1373 file = mock_file(i915);
1374 if (IS_ERR(file))
1375 return PTR_ERR(file);
1376
1377 err = igt_live_test_begin(&t, i915, __func__, "");
1378 if (err)
1379 goto out_file;
1380
1381 ctx = live_context(i915, file);
1382 if (IS_ERR(ctx)) {
1383 err = PTR_ERR(ctx);
1384 goto out_file;
1385 }
1386
1387 vm = ctx_vm(ctx) ?: &i915->ggtt.alias->vm;
1388 if (!vm || !vm->has_read_only) {
1389 err = 0;
1390 goto out_file;
1391 }
1392
1393 num_engines = 0;
1394 for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it)
1395 if (intel_engine_can_store_dword(ce->engine))
1396 num_engines++;
1397 i915_gem_context_unlock_engines(ctx);
1398
1399 ndwords = 0;
1400 dw = 0;
1401 while (!time_after(jiffies, end_time)) {
1402 for_each_gem_engine(ce,
1403 i915_gem_context_lock_engines(ctx), it) {
1404 if (!intel_engine_can_store_dword(ce->engine))
1405 continue;
1406
1407 if (!obj) {
1408 obj = create_test_object(ce->vm, file, &objects);
1409 if (IS_ERR(obj)) {
1410 err = PTR_ERR(obj);
1411 i915_gem_context_unlock_engines(ctx);
1412 goto out_file;
1413 }
1414
1415 if (prandom_u32_state(&prng) & 1)
1416 i915_gem_object_set_readonly(obj);
1417 }
1418
1419 err = gpu_fill(ce, obj, dw);
1420 if (err) {
1421 pr_err("Failed to fill dword %lu [%lu/%lu] with gpu (%s) [full-ppgtt? %s], err=%d\n",
1422 ndwords, dw, max_dwords(obj),
1423 ce->engine->name,
1424 yesno(!!ctx_vm(ctx)),
1425 err);
1426 i915_gem_context_unlock_engines(ctx);
1427 goto out_file;
1428 }
1429
1430 err = throttle(ce, tq, ARRAY_SIZE(tq));
1431 if (err) {
1432 i915_gem_context_unlock_engines(ctx);
1433 goto out_file;
1434 }
1435
1436 if (++dw == max_dwords(obj)) {
1437 obj = NULL;
1438 dw = 0;
1439 }
1440 ndwords++;
1441 }
1442 i915_gem_context_unlock_engines(ctx);
1443 }
1444 pr_info("Submitted %lu dwords (across %lu engines)\n",
1445 ndwords, num_engines);
1446
1447 dw = 0;
1448 idx = 0;
1449 list_for_each_entry(obj, &objects, st_link) {
1450 unsigned int rem =
1451 min_t(unsigned int, ndwords - dw, max_dwords(obj));
1452 unsigned int num_writes;
1453
1454 num_writes = rem;
1455 if (i915_gem_object_is_readonly(obj))
1456 num_writes = 0;
1457
1458 err = cpu_check(obj, idx++, num_writes);
1459 if (err)
1460 break;
1461
1462 dw += rem;
1463 }
1464
1465 out_file:
1466 throttle_release(tq, ARRAY_SIZE(tq));
1467 if (igt_live_test_end(&t))
1468 err = -EIO;
1469
1470 fput(file);
1471 return err;
1472 }
1473
1474 static int check_scratch(struct i915_address_space *vm, u64 offset)
1475 {
1476 struct drm_mm_node *node;
1477
1478 mutex_lock(&vm->mutex);
1479 node = __drm_mm_interval_first(&vm->mm,
1480 offset, offset + sizeof(u32) - 1);
1481 mutex_unlock(&vm->mutex);
1482 if (!node || node->start > offset)
1483 return 0;
1484
1485 GEM_BUG_ON(offset >= node->start + node->size);
1486
1487 pr_err("Target offset 0x%08x_%08x overlaps with a node in the mm!\n",
1488 upper_32_bits(offset), lower_32_bits(offset));
1489 return -EINVAL;
1490 }
1491
1492 static int write_to_scratch(struct i915_gem_context *ctx,
1493 struct intel_engine_cs *engine,
1494 u64 offset, u32 value)
1495 {
1496 struct drm_i915_private *i915 = ctx->i915;
1497 struct drm_i915_gem_object *obj;
1498 struct i915_address_space *vm;
1499 struct i915_request *rq;
1500 struct i915_vma *vma;
1501 u32 *cmd;
1502 int err;
1503
1504 GEM_BUG_ON(offset < I915_GTT_PAGE_SIZE);
1505
1506 err = check_scratch(ctx_vm(ctx), offset);
1507 if (err)
1508 return err;
1509
1510 obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
1511 if (IS_ERR(obj))
1512 return PTR_ERR(obj);
1513
1514 cmd = i915_gem_object_pin_map(obj, I915_MAP_WB);
1515 if (IS_ERR(cmd)) {
1516 err = PTR_ERR(cmd);
1517 goto out;
1518 }
1519
1520 *cmd++ = MI_STORE_DWORD_IMM_GEN4;
1521 if (INTEL_GEN(i915) >= 8) {
1522 *cmd++ = lower_32_bits(offset);
1523 *cmd++ = upper_32_bits(offset);
1524 } else {
1525 *cmd++ = 0;
1526 *cmd++ = offset;
1527 }
1528 *cmd++ = value;
1529 *cmd = MI_BATCH_BUFFER_END;
1530 __i915_gem_object_flush_map(obj, 0, 64);
1531 i915_gem_object_unpin_map(obj);
1532
1533 intel_gt_chipset_flush(engine->gt);
1534
1535 vm = i915_gem_context_get_vm_rcu(ctx);
1536 vma = i915_vma_instance(obj, vm, NULL);
1537 if (IS_ERR(vma)) {
1538 err = PTR_ERR(vma);
1539 goto out_vm;
1540 }
1541
1542 err = i915_vma_pin(vma, 0, 0, PIN_USER | PIN_OFFSET_FIXED);
1543 if (err)
1544 goto out_vm;
1545
1546 rq = igt_request_alloc(ctx, engine);
1547 if (IS_ERR(rq)) {
1548 err = PTR_ERR(rq);
1549 goto err_unpin;
1550 }
1551
1552 i915_vma_lock(vma);
1553 err = i915_request_await_object(rq, vma->obj, false);
1554 if (err == 0)
1555 err = i915_vma_move_to_active(vma, rq, 0);
1556 i915_vma_unlock(vma);
1557 if (err)
1558 goto skip_request;
1559
1560 if (rq->engine->emit_init_breadcrumb) {
1561 err = rq->engine->emit_init_breadcrumb(rq);
1562 if (err)
1563 goto skip_request;
1564 }
1565
1566 err = engine->emit_bb_start(rq, vma->node.start, vma->node.size, 0);
1567 if (err)
1568 goto skip_request;
1569
1570 i915_vma_unpin(vma);
1571
1572 i915_request_add(rq);
1573
1574 goto out_vm;
1575 skip_request:
1576 i915_request_set_error_once(rq, err);
1577 i915_request_add(rq);
1578 err_unpin:
1579 i915_vma_unpin(vma);
1580 out_vm:
1581 i915_vm_put(vm);
1582 out:
1583 i915_gem_object_put(obj);
1584 return err;
1585 }
1586
1587 static int read_from_scratch(struct i915_gem_context *ctx,
1588 struct intel_engine_cs *engine,
1589 u64 offset, u32 *value)
1590 {
1591 struct drm_i915_private *i915 = ctx->i915;
1592 struct drm_i915_gem_object *obj;
1593 struct i915_address_space *vm;
1594 const u32 result = 0x100;
1595 struct i915_request *rq;
1596 struct i915_vma *vma;
1597 unsigned int flags;
1598 u32 *cmd;
1599 int err;
1600
1601 GEM_BUG_ON(offset < I915_GTT_PAGE_SIZE);
1602
1603 err = check_scratch(ctx_vm(ctx), offset);
1604 if (err)
1605 return err;
1606
1607 obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
1608 if (IS_ERR(obj))
1609 return PTR_ERR(obj);
1610
1611 if (INTEL_GEN(i915) >= 8) {
1612 const u32 GPR0 = engine->mmio_base + 0x600;
1613
1614 vm = i915_gem_context_get_vm_rcu(ctx);
1615 vma = i915_vma_instance(obj, vm, NULL);
1616 if (IS_ERR(vma)) {
1617 err = PTR_ERR(vma);
1618 goto out_vm;
1619 }
1620
1621 err = i915_vma_pin(vma, 0, 0, PIN_USER | PIN_OFFSET_FIXED);
1622 if (err)
1623 goto out_vm;
1624
1625 cmd = i915_gem_object_pin_map(obj, I915_MAP_WB);
1626 if (IS_ERR(cmd)) {
1627 err = PTR_ERR(cmd);
1628 goto out;
1629 }
1630
1631 memset(cmd, POISON_INUSE, PAGE_SIZE);
1632 *cmd++ = MI_LOAD_REGISTER_MEM_GEN8;
1633 *cmd++ = GPR0;
1634 *cmd++ = lower_32_bits(offset);
1635 *cmd++ = upper_32_bits(offset);
1636 *cmd++ = MI_STORE_REGISTER_MEM_GEN8;
1637 *cmd++ = GPR0;
1638 *cmd++ = result;
1639 *cmd++ = 0;
1640 *cmd = MI_BATCH_BUFFER_END;
1641
1642 i915_gem_object_flush_map(obj);
1643 i915_gem_object_unpin_map(obj);
1644
1645 flags = 0;
1646 } else {
1647 const u32 reg = engine->mmio_base + 0x420;
1648
1649 /* hsw: register access even to 3DPRIM! is protected */
1650 vm = i915_vm_get(&engine->gt->ggtt->vm);
1651 vma = i915_vma_instance(obj, vm, NULL);
1652 if (IS_ERR(vma)) {
1653 err = PTR_ERR(vma);
1654 goto out_vm;
1655 }
1656
1657 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
1658 if (err)
1659 goto out_vm;
1660
1661 cmd = i915_gem_object_pin_map(obj, I915_MAP_WB);
1662 if (IS_ERR(cmd)) {
1663 err = PTR_ERR(cmd);
1664 goto out;
1665 }
1666
1667 memset(cmd, POISON_INUSE, PAGE_SIZE);
1668 *cmd++ = MI_LOAD_REGISTER_MEM;
1669 *cmd++ = reg;
1670 *cmd++ = offset;
1671 *cmd++ = MI_STORE_REGISTER_MEM | MI_USE_GGTT;
1672 *cmd++ = reg;
1673 *cmd++ = vma->node.start + result;
1674 *cmd = MI_BATCH_BUFFER_END;
1675
1676 i915_gem_object_flush_map(obj);
1677 i915_gem_object_unpin_map(obj);
1678
1679 flags = I915_DISPATCH_SECURE;
1680 }
1681
1682 intel_gt_chipset_flush(engine->gt);
1683
1684 rq = igt_request_alloc(ctx, engine);
1685 if (IS_ERR(rq)) {
1686 err = PTR_ERR(rq);
1687 goto err_unpin;
1688 }
1689
1690 i915_vma_lock(vma);
1691 err = i915_request_await_object(rq, vma->obj, true);
1692 if (err == 0)
1693 err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
1694 i915_vma_unlock(vma);
1695 if (err)
1696 goto skip_request;
1697
1698 if (rq->engine->emit_init_breadcrumb) {
1699 err = rq->engine->emit_init_breadcrumb(rq);
1700 if (err)
1701 goto skip_request;
1702 }
1703
1704 err = engine->emit_bb_start(rq, vma->node.start, vma->node.size, flags);
1705 if (err)
1706 goto skip_request;
1707
1708 i915_vma_unpin(vma);
1709
1710 i915_request_add(rq);
1711
1712 i915_gem_object_lock(obj, NULL);
1713 err = i915_gem_object_set_to_cpu_domain(obj, false);
1714 i915_gem_object_unlock(obj);
1715 if (err)
1716 goto out_vm;
1717
1718 cmd = i915_gem_object_pin_map(obj, I915_MAP_WB);
1719 if (IS_ERR(cmd)) {
1720 err = PTR_ERR(cmd);
1721 goto out_vm;
1722 }
1723
1724 *value = cmd[result / sizeof(*cmd)];
1725 i915_gem_object_unpin_map(obj);
1726
1727 goto out_vm;
1728 skip_request:
1729 i915_request_set_error_once(rq, err);
1730 i915_request_add(rq);
1731 err_unpin:
1732 i915_vma_unpin(vma);
1733 out_vm:
1734 i915_vm_put(vm);
1735 out:
1736 i915_gem_object_put(obj);
1737 return err;
1738 }
1739
1740 static int check_scratch_page(struct i915_gem_context *ctx, u32 *out)
1741 {
1742 struct i915_address_space *vm;
1743 struct page *page;
1744 u32 *vaddr;
1745 int err = 0;
1746
1747 vm = ctx_vm(ctx);
1748 if (!vm)
1749 return -ENODEV;
1750
1751 page = __px_page(vm->scratch[0]);
1752 if (!page) {
1753 pr_err("No scratch page!\n");
1754 return -EINVAL;
1755 }
1756
1757 vaddr = kmap(page);
1758 if (!vaddr) {
1759 pr_err("No (mappable) scratch page!\n");
1760 return -EINVAL;
1761 }
1762
1763 memcpy(out, vaddr, sizeof(*out));
1764 if (memchr_inv(vaddr, *out, PAGE_SIZE)) {
1765 pr_err("Inconsistent initial state of scratch page!\n");
1766 err = -EINVAL;
1767 }
1768 kunmap(page);
1769
1770 return err;
1771 }
1772
1773 static int igt_vm_isolation(void *arg)
1774 {
1775 struct drm_i915_private *i915 = arg;
1776 struct i915_gem_context *ctx_a, *ctx_b;
1777 unsigned long num_engines, count;
1778 struct intel_engine_cs *engine;
1779 struct igt_live_test t;
1780 I915_RND_STATE(prng);
1781 struct file *file;
1782 u64 vm_total;
1783 u32 expected;
1784 int err;
1785
1786 if (INTEL_GEN(i915) < 7)
1787 return 0;
1788
1789 /*
1790 * The simple goal here is that a write into one context is not
1791 * observed in a second (separate page tables and scratch).
1792 */
1793
1794 file = mock_file(i915);
1795 if (IS_ERR(file))
1796 return PTR_ERR(file);
1797
1798 err = igt_live_test_begin(&t, i915, __func__, "");
1799 if (err)
1800 goto out_file;
1801
1802 ctx_a = live_context(i915, file);
1803 if (IS_ERR(ctx_a)) {
1804 err = PTR_ERR(ctx_a);
1805 goto out_file;
1806 }
1807
1808 ctx_b = live_context(i915, file);
1809 if (IS_ERR(ctx_b)) {
1810 err = PTR_ERR(ctx_b);
1811 goto out_file;
1812 }
1813
1814 /* We can only test vm isolation, if the vm are distinct */
1815 if (ctx_vm(ctx_a) == ctx_vm(ctx_b))
1816 goto out_file;
1817
1818 /* Read the initial state of the scratch page */
1819 err = check_scratch_page(ctx_a, &expected);
1820 if (err)
1821 goto out_file;
1822
1823 err = check_scratch_page(ctx_b, &expected);
1824 if (err)
1825 goto out_file;
1826
1827 vm_total = ctx_vm(ctx_a)->total;
1828 GEM_BUG_ON(ctx_vm(ctx_b)->total != vm_total);
1829
1830 count = 0;
1831 num_engines = 0;
1832 for_each_uabi_engine(engine, i915) {
1833 IGT_TIMEOUT(end_time);
1834 unsigned long this = 0;
1835
1836 if (!intel_engine_can_store_dword(engine))
1837 continue;
1838
1839 /* Not all engines have their own GPR! */
1840 if (INTEL_GEN(i915) < 8 && engine->class != RENDER_CLASS)
1841 continue;
1842
1843 while (!__igt_timeout(end_time, NULL)) {
1844 u32 value = 0xc5c5c5c5;
1845 u64 offset;
1846
1847 /* Leave enough space at offset 0 for the batch */
1848 offset = igt_random_offset(&prng,
1849 I915_GTT_PAGE_SIZE, vm_total,
1850 sizeof(u32), alignof_dword);
1851
1852 err = write_to_scratch(ctx_a, engine,
1853 offset, 0xdeadbeef);
1854 if (err == 0)
1855 err = read_from_scratch(ctx_b, engine,
1856 offset, &value);
1857 if (err)
1858 goto out_file;
1859
1860 if (value != expected) {
1861 pr_err("%s: Read %08x from scratch (offset 0x%08x_%08x), after %lu reads!\n",
1862 engine->name, value,
1863 upper_32_bits(offset),
1864 lower_32_bits(offset),
1865 this);
1866 err = -EINVAL;
1867 goto out_file;
1868 }
1869
1870 this++;
1871 }
1872 count += this;
1873 num_engines++;
1874 }
1875 pr_info("Checked %lu scratch offsets across %lu engines\n",
1876 count, num_engines);
1877
1878 out_file:
1879 if (igt_live_test_end(&t))
1880 err = -EIO;
1881 fput(file);
1882 return err;
1883 }
1884
1885 static bool skip_unused_engines(struct intel_context *ce, void *data)
1886 {
1887 return !ce->state;
1888 }
1889
1890 static void mock_barrier_task(void *data)
1891 {
1892 unsigned int *counter = data;
1893
1894 ++*counter;
1895 }
1896
1897 static int mock_context_barrier(void *arg)
1898 {
1899 #undef pr_fmt
1900 #define pr_fmt(x) "context_barrier_task():" # x
1901 struct drm_i915_private *i915 = arg;
1902 struct i915_gem_context *ctx;
1903 struct i915_request *rq;
1904 unsigned int counter;
1905 int err;
1906
1907 /*
1908 * The context barrier provides us with a callback after it emits
1909 * a request; useful for retiring old state after loading new.
1910 */
1911
1912 ctx = mock_context(i915, "mock");
1913 if (!ctx)
1914 return -ENOMEM;
1915
1916 counter = 0;
1917 err = context_barrier_task(ctx, 0, NULL, NULL, NULL,
1918 mock_barrier_task, &counter);
1919 if (err) {
1920 pr_err("Failed at line %d, err=%d\n", __LINE__, err);
1921 goto out;
1922 }
1923 if (counter == 0) {
1924 pr_err("Did not retire immediately with 0 engines\n");
1925 err = -EINVAL;
1926 goto out;
1927 }
1928
1929 counter = 0;
1930 err = context_barrier_task(ctx, ALL_ENGINES, skip_unused_engines,
1931 NULL, NULL, mock_barrier_task, &counter);
1932 if (err) {
1933 pr_err("Failed at line %d, err=%d\n", __LINE__, err);
1934 goto out;
1935 }
1936 if (counter == 0) {
1937 pr_err("Did not retire immediately for all unused engines\n");
1938 err = -EINVAL;
1939 goto out;
1940 }
1941
1942 rq = igt_request_alloc(ctx, i915->gt.engine[RCS0]);
1943 if (IS_ERR(rq)) {
1944 pr_err("Request allocation failed!\n");
1945 goto out;
1946 }
1947 i915_request_add(rq);
1948
1949 counter = 0;
1950 context_barrier_inject_fault = BIT(RCS0);
1951 err = context_barrier_task(ctx, ALL_ENGINES, NULL, NULL, NULL,
1952 mock_barrier_task, &counter);
1953 context_barrier_inject_fault = 0;
1954 if (err == -ENXIO)
1955 err = 0;
1956 else
1957 pr_err("Did not hit fault injection!\n");
1958 if (counter != 0) {
1959 pr_err("Invoked callback on error!\n");
1960 err = -EIO;
1961 }
1962 if (err)
1963 goto out;
1964
1965 counter = 0;
1966 err = context_barrier_task(ctx, ALL_ENGINES, skip_unused_engines,
1967 NULL, NULL, mock_barrier_task, &counter);
1968 if (err) {
1969 pr_err("Failed at line %d, err=%d\n", __LINE__, err);
1970 goto out;
1971 }
1972 mock_device_flush(i915);
1973 if (counter == 0) {
1974 pr_err("Did not retire on each active engines\n");
1975 err = -EINVAL;
1976 goto out;
1977 }
1978
1979 out:
1980 mock_context_close(ctx);
1981 return err;
1982 #undef pr_fmt
1983 #define pr_fmt(x) x
1984 }
1985
1986 int i915_gem_context_mock_selftests(void)
1987 {
1988 static const struct i915_subtest tests[] = {
1989 SUBTEST(mock_context_barrier),
1990 };
1991 struct drm_i915_private *i915;
1992 int err;
1993
1994 i915 = mock_gem_device();
1995 if (!i915)
1996 return -ENOMEM;
1997
1998 err = i915_subtests(tests, i915);
1999
2000 mock_destroy_device(i915);
2001 return err;
2002 }
2003
2004 int i915_gem_context_live_selftests(struct drm_i915_private *i915)
2005 {
2006 static const struct i915_subtest tests[] = {
2007 SUBTEST(live_nop_switch),
2008 SUBTEST(live_parallel_switch),
2009 SUBTEST(igt_ctx_exec),
2010 SUBTEST(igt_ctx_readonly),
2011 SUBTEST(igt_ctx_sseu),
2012 SUBTEST(igt_shared_ctx_exec),
2013 SUBTEST(igt_vm_isolation),
2014 };
2015
2016 if (intel_gt_is_wedged(&i915->gt))
2017 return 0;
2018
2019 return i915_live_subtests(tests, i915);
2020 }
Linux with added FPC-III support