?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
treewide: remove redundant IS_ERR() before error code check
[linux/fpc-iii.git] / drivers / gpu / drm / i915 / selftests / i915_gem_gtt.c
blobb342bef5e7c95613397d618ae87145375fcb7ec2
1 /*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
25 #include <linux/list_sort.h>
26 #include <linux/prime_numbers.h>
27
28 #include "gem/i915_gem_context.h"
29 #include "gem/selftests/mock_context.h"
30 #include "gt/intel_context.h"
31
32 #include "i915_random.h"
33 #include "i915_selftest.h"
34
35 #include "mock_drm.h"
36 #include "mock_gem_device.h"
37 #include "mock_gtt.h"
38 #include "igt_flush_test.h"
39
40 static void cleanup_freed_objects(struct drm_i915_private *i915)
41 {
42 i915_gem_drain_freed_objects(i915);
43 }
44
45 static void fake_free_pages(struct drm_i915_gem_object *obj,
46 struct sg_table *pages)
47 {
48 sg_free_table(pages);
49 kfree(pages);
50 }
51
52 static int fake_get_pages(struct drm_i915_gem_object *obj)
53 {
54 #define GFP (GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY)
55 #define PFN_BIAS 0x1000
56 struct sg_table *pages;
57 struct scatterlist *sg;
58 unsigned int sg_page_sizes;
59 typeof(obj->base.size) rem;
60
61 pages = kmalloc(sizeof(*pages), GFP);
62 if (!pages)
63 return -ENOMEM;
64
65 rem = round_up(obj->base.size, BIT(31)) >> 31;
66 if (sg_alloc_table(pages, rem, GFP)) {
67 kfree(pages);
68 return -ENOMEM;
69 }
70
71 sg_page_sizes = 0;
72 rem = obj->base.size;
73 for (sg = pages->sgl; sg; sg = sg_next(sg)) {
74 unsigned long len = min_t(typeof(rem), rem, BIT(31));
75
76 GEM_BUG_ON(!len);
77 sg_set_page(sg, pfn_to_page(PFN_BIAS), len, 0);
78 sg_dma_address(sg) = page_to_phys(sg_page(sg));
79 sg_dma_len(sg) = len;
80 sg_page_sizes |= len;
81
82 rem -= len;
83 }
84 GEM_BUG_ON(rem);
85
86 __i915_gem_object_set_pages(obj, pages, sg_page_sizes);
87
88 return 0;
89 #undef GFP
90 }
91
92 static void fake_put_pages(struct drm_i915_gem_object *obj,
93 struct sg_table *pages)
94 {
95 fake_free_pages(obj, pages);
96 obj->mm.dirty = false;
97 }
98
99 static const struct drm_i915_gem_object_ops fake_ops = {
100 .flags = I915_GEM_OBJECT_IS_SHRINKABLE,
101 .get_pages = fake_get_pages,
102 .put_pages = fake_put_pages,
103 };
104
105 static struct drm_i915_gem_object *
106 fake_dma_object(struct drm_i915_private *i915, u64 size)
107 {
108 static struct lock_class_key lock_class;
109 struct drm_i915_gem_object *obj;
110
111 GEM_BUG_ON(!size);
112 GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
113
114 if (overflows_type(size, obj->base.size))
115 return ERR_PTR(-E2BIG);
116
117 obj = i915_gem_object_alloc();
118 if (!obj)
119 goto err;
120
121 drm_gem_private_object_init(&i915->drm, &obj->base, size);
122 i915_gem_object_init(obj, &fake_ops, &lock_class);
123
124 i915_gem_object_set_volatile(obj);
125
126 obj->write_domain = I915_GEM_DOMAIN_CPU;
127 obj->read_domains = I915_GEM_DOMAIN_CPU;
128 obj->cache_level = I915_CACHE_NONE;
129
130 /* Preallocate the "backing storage" */
131 if (i915_gem_object_pin_pages(obj))
132 goto err_obj;
133
134 i915_gem_object_unpin_pages(obj);
135 return obj;
136
137 err_obj:
138 i915_gem_object_put(obj);
139 err:
140 return ERR_PTR(-ENOMEM);
141 }
142
143 static int igt_ppgtt_alloc(void *arg)
144 {
145 struct drm_i915_private *dev_priv = arg;
146 struct i915_ppgtt *ppgtt;
147 u64 size, last, limit;
148 int err = 0;
149
150 /* Allocate a ppggt and try to fill the entire range */
151
152 if (!HAS_PPGTT(dev_priv))
153 return 0;
154
155 ppgtt = i915_ppgtt_create(&dev_priv->gt);
156 if (IS_ERR(ppgtt))
157 return PTR_ERR(ppgtt);
158
159 if (!ppgtt->vm.allocate_va_range)
160 goto err_ppgtt_cleanup;
161
162 /*
163 * While we only allocate the page tables here and so we could
164 * address a much larger GTT than we could actually fit into
165 * RAM, a practical limit is the amount of physical pages in the system.
166 * This should ensure that we do not run into the oomkiller during
167 * the test and take down the machine wilfully.
168 */
169 limit = totalram_pages() << PAGE_SHIFT;
170 limit = min(ppgtt->vm.total, limit);
171
172 /* Check we can allocate the entire range */
173 for (size = 4096; size <= limit; size <<= 2) {
174 err = ppgtt->vm.allocate_va_range(&ppgtt->vm, 0, size);
175 if (err) {
176 if (err == -ENOMEM) {
177 pr_info("[1] Ran out of memory for va_range [0 + %llx] [bit %d]\n",
178 size, ilog2(size));
179 err = 0; /* virtual space too large! */
180 }
181 goto err_ppgtt_cleanup;
182 }
183
184 cond_resched();
185
186 ppgtt->vm.clear_range(&ppgtt->vm, 0, size);
187 }
188
189 /* Check we can incrementally allocate the entire range */
190 for (last = 0, size = 4096; size <= limit; last = size, size <<= 2) {
191 err = ppgtt->vm.allocate_va_range(&ppgtt->vm,
192 last, size - last);
193 if (err) {
194 if (err == -ENOMEM) {
195 pr_info("[2] Ran out of memory for va_range [%llx + %llx] [bit %d]\n",
196 last, size - last, ilog2(size));
197 err = 0; /* virtual space too large! */
198 }
199 goto err_ppgtt_cleanup;
200 }
201
202 cond_resched();
203 }
204
205 err_ppgtt_cleanup:
206 i915_vm_put(&ppgtt->vm);
207 return err;
208 }
209
210 static int lowlevel_hole(struct i915_address_space *vm,
211 u64 hole_start, u64 hole_end,
212 unsigned long end_time)
213 {
214 I915_RND_STATE(seed_prng);
215 struct i915_vma *mock_vma;
216 unsigned int size;
217
218 mock_vma = kzalloc(sizeof(*mock_vma), GFP_KERNEL);
219 if (!mock_vma)
220 return -ENOMEM;
221
222 /* Keep creating larger objects until one cannot fit into the hole */
223 for (size = 12; (hole_end - hole_start) >> size; size++) {
224 I915_RND_SUBSTATE(prng, seed_prng);
225 struct drm_i915_gem_object *obj;
226 unsigned int *order, count, n;
227 u64 hole_size;
228
229 hole_size = (hole_end - hole_start) >> size;
230 if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32))
231 hole_size = KMALLOC_MAX_SIZE / sizeof(u32);
232 count = hole_size >> 1;
233 if (!count) {
234 pr_debug("%s: hole is too small [%llx - %llx] >> %d: %lld\n",
235 __func__, hole_start, hole_end, size, hole_size);
236 break;
237 }
238
239 do {
240 order = i915_random_order(count, &prng);
241 if (order)
242 break;
243 } while (count >>= 1);
244 if (!count) {
245 kfree(mock_vma);
246 return -ENOMEM;
247 }
248 GEM_BUG_ON(!order);
249
250 GEM_BUG_ON(count * BIT_ULL(size) > vm->total);
251 GEM_BUG_ON(hole_start + count * BIT_ULL(size) > hole_end);
252
253 /* Ignore allocation failures (i.e. don't report them as
254 * a test failure) as we are purposefully allocating very
255 * large objects without checking that we have sufficient
256 * memory. We expect to hit -ENOMEM.
257 */
258
259 obj = fake_dma_object(vm->i915, BIT_ULL(size));
260 if (IS_ERR(obj)) {
261 kfree(order);
262 break;
263 }
264
265 GEM_BUG_ON(obj->base.size != BIT_ULL(size));
266
267 if (i915_gem_object_pin_pages(obj)) {
268 i915_gem_object_put(obj);
269 kfree(order);
270 break;
271 }
272
273 for (n = 0; n < count; n++) {
274 u64 addr = hole_start + order[n] * BIT_ULL(size);
275 intel_wakeref_t wakeref;
276
277 GEM_BUG_ON(addr + BIT_ULL(size) > vm->total);
278
279 if (igt_timeout(end_time,
280 "%s timed out before %d/%d\n",
281 __func__, n, count)) {
282 hole_end = hole_start; /* quit */
283 break;
284 }
285
286 if (vm->allocate_va_range &&
287 vm->allocate_va_range(vm, addr, BIT_ULL(size)))
288 break;
289
290 mock_vma->pages = obj->mm.pages;
291 mock_vma->node.size = BIT_ULL(size);
292 mock_vma->node.start = addr;
293
294 with_intel_runtime_pm(vm->gt->uncore->rpm, wakeref)
295 vm->insert_entries(vm, mock_vma,
296 I915_CACHE_NONE, 0);
297 }
298 count = n;
299
300 i915_random_reorder(order, count, &prng);
301 for (n = 0; n < count; n++) {
302 u64 addr = hole_start + order[n] * BIT_ULL(size);
303 intel_wakeref_t wakeref;
304
305 GEM_BUG_ON(addr + BIT_ULL(size) > vm->total);
306 with_intel_runtime_pm(vm->gt->uncore->rpm, wakeref)
307 vm->clear_range(vm, addr, BIT_ULL(size));
308 }
309
310 i915_gem_object_unpin_pages(obj);
311 i915_gem_object_put(obj);
312
313 kfree(order);
314
315 cleanup_freed_objects(vm->i915);
316 }
317
318 kfree(mock_vma);
319 return 0;
320 }
321
322 static void close_object_list(struct list_head *objects,
323 struct i915_address_space *vm)
324 {
325 struct drm_i915_gem_object *obj, *on;
326 int ignored;
327
328 list_for_each_entry_safe(obj, on, objects, st_link) {
329 struct i915_vma *vma;
330
331 vma = i915_vma_instance(obj, vm, NULL);
332 if (!IS_ERR(vma))
333 ignored = i915_vma_unbind(vma);
334 /* Only ppgtt vma may be closed before the object is freed */
335 if (!IS_ERR(vma) && !i915_vma_is_ggtt(vma))
336 i915_vma_close(vma);
337
338 list_del(&obj->st_link);
339 i915_gem_object_put(obj);
340 }
341 }
342
343 static int fill_hole(struct i915_address_space *vm,
344 u64 hole_start, u64 hole_end,
345 unsigned long end_time)
346 {
347 const u64 hole_size = hole_end - hole_start;
348 struct drm_i915_gem_object *obj;
349 const unsigned long max_pages =
350 min_t(u64, ULONG_MAX - 1, hole_size/2 >> PAGE_SHIFT);
351 const unsigned long max_step = max(int_sqrt(max_pages), 2UL);
352 unsigned long npages, prime, flags;
353 struct i915_vma *vma;
354 LIST_HEAD(objects);
355 int err;
356
357 /* Try binding many VMA working inwards from either edge */
358
359 flags = PIN_OFFSET_FIXED | PIN_USER;
360 if (i915_is_ggtt(vm))
361 flags |= PIN_GLOBAL;
362
363 for_each_prime_number_from(prime, 2, max_step) {
364 for (npages = 1; npages <= max_pages; npages *= prime) {
365 const u64 full_size = npages << PAGE_SHIFT;
366 const struct {
367 const char *name;
368 u64 offset;
369 int step;
370 } phases[] = {
371 { "top-down", hole_end, -1, },
372 { "bottom-up", hole_start, 1, },
373 { }
374 }, *p;
375
376 obj = fake_dma_object(vm->i915, full_size);
377 if (IS_ERR(obj))
378 break;
379
380 list_add(&obj->st_link, &objects);
381
382 /* Align differing sized objects against the edges, and
383 * check we don't walk off into the void when binding
384 * them into the GTT.
385 */
386 for (p = phases; p->name; p++) {
387 u64 offset;
388
389 offset = p->offset;
390 list_for_each_entry(obj, &objects, st_link) {
391 vma = i915_vma_instance(obj, vm, NULL);
392 if (IS_ERR(vma))
393 continue;
394
395 if (p->step < 0) {
396 if (offset < hole_start + obj->base.size)
397 break;
398 offset -= obj->base.size;
399 }
400
401 err = i915_vma_pin(vma, 0, 0, offset | flags);
402 if (err) {
403 pr_err("%s(%s) pin (forward) failed with err=%d on size=%lu pages (prime=%lu), offset=%llx\n",
404 __func__, p->name, err, npages, prime, offset);
405 goto err;
406 }
407
408 if (!drm_mm_node_allocated(&vma->node) ||
409 i915_vma_misplaced(vma, 0, 0, offset | flags)) {
410 pr_err("%s(%s) (forward) insert failed: vma.node=%llx + %llx [allocated? %d], expected offset %llx\n",
411 __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node),
412 offset);
413 err = -EINVAL;
414 goto err;
415 }
416
417 i915_vma_unpin(vma);
418
419 if (p->step > 0) {
420 if (offset + obj->base.size > hole_end)
421 break;
422 offset += obj->base.size;
423 }
424 }
425
426 offset = p->offset;
427 list_for_each_entry(obj, &objects, st_link) {
428 vma = i915_vma_instance(obj, vm, NULL);
429 if (IS_ERR(vma))
430 continue;
431
432 if (p->step < 0) {
433 if (offset < hole_start + obj->base.size)
434 break;
435 offset -= obj->base.size;
436 }
437
438 if (!drm_mm_node_allocated(&vma->node) ||
439 i915_vma_misplaced(vma, 0, 0, offset | flags)) {
440 pr_err("%s(%s) (forward) moved vma.node=%llx + %llx, expected offset %llx\n",
441 __func__, p->name, vma->node.start, vma->node.size,
442 offset);
443 err = -EINVAL;
444 goto err;
445 }
446
447 err = i915_vma_unbind(vma);
448 if (err) {
449 pr_err("%s(%s) (forward) unbind of vma.node=%llx + %llx failed with err=%d\n",
450 __func__, p->name, vma->node.start, vma->node.size,
451 err);
452 goto err;
453 }
454
455 if (p->step > 0) {
456 if (offset + obj->base.size > hole_end)
457 break;
458 offset += obj->base.size;
459 }
460 }
461
462 offset = p->offset;
463 list_for_each_entry_reverse(obj, &objects, st_link) {
464 vma = i915_vma_instance(obj, vm, NULL);
465 if (IS_ERR(vma))
466 continue;
467
468 if (p->step < 0) {
469 if (offset < hole_start + obj->base.size)
470 break;
471 offset -= obj->base.size;
472 }
473
474 err = i915_vma_pin(vma, 0, 0, offset | flags);
475 if (err) {
476 pr_err("%s(%s) pin (backward) failed with err=%d on size=%lu pages (prime=%lu), offset=%llx\n",
477 __func__, p->name, err, npages, prime, offset);
478 goto err;
479 }
480
481 if (!drm_mm_node_allocated(&vma->node) ||
482 i915_vma_misplaced(vma, 0, 0, offset | flags)) {
483 pr_err("%s(%s) (backward) insert failed: vma.node=%llx + %llx [allocated? %d], expected offset %llx\n",
484 __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node),
485 offset);
486 err = -EINVAL;
487 goto err;
488 }
489
490 i915_vma_unpin(vma);
491
492 if (p->step > 0) {
493 if (offset + obj->base.size > hole_end)
494 break;
495 offset += obj->base.size;
496 }
497 }
498
499 offset = p->offset;
500 list_for_each_entry_reverse(obj, &objects, st_link) {
501 vma = i915_vma_instance(obj, vm, NULL);
502 if (IS_ERR(vma))
503 continue;
504
505 if (p->step < 0) {
506 if (offset < hole_start + obj->base.size)
507 break;
508 offset -= obj->base.size;
509 }
510
511 if (!drm_mm_node_allocated(&vma->node) ||
512 i915_vma_misplaced(vma, 0, 0, offset | flags)) {
513 pr_err("%s(%s) (backward) moved vma.node=%llx + %llx [allocated? %d], expected offset %llx\n",
514 __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node),
515 offset);
516 err = -EINVAL;
517 goto err;
518 }
519
520 err = i915_vma_unbind(vma);
521 if (err) {
522 pr_err("%s(%s) (backward) unbind of vma.node=%llx + %llx failed with err=%d\n",
523 __func__, p->name, vma->node.start, vma->node.size,
524 err);
525 goto err;
526 }
527
528 if (p->step > 0) {
529 if (offset + obj->base.size > hole_end)
530 break;
531 offset += obj->base.size;
532 }
533 }
534 }
535
536 if (igt_timeout(end_time, "%s timed out (npages=%lu, prime=%lu)\n",
537 __func__, npages, prime)) {
538 err = -EINTR;
539 goto err;
540 }
541 }
542
543 close_object_list(&objects, vm);
544 cleanup_freed_objects(vm->i915);
545 }
546
547 return 0;
548
549 err:
550 close_object_list(&objects, vm);
551 return err;
552 }
553
554 static int walk_hole(struct i915_address_space *vm,
555 u64 hole_start, u64 hole_end,
556 unsigned long end_time)
557 {
558 const u64 hole_size = hole_end - hole_start;
559 const unsigned long max_pages =
560 min_t(u64, ULONG_MAX - 1, hole_size >> PAGE_SHIFT);
561 unsigned long flags;
562 u64 size;
563
564 /* Try binding a single VMA in different positions within the hole */
565
566 flags = PIN_OFFSET_FIXED | PIN_USER;
567 if (i915_is_ggtt(vm))
568 flags |= PIN_GLOBAL;
569
570 for_each_prime_number_from(size, 1, max_pages) {
571 struct drm_i915_gem_object *obj;
572 struct i915_vma *vma;
573 u64 addr;
574 int err = 0;
575
576 obj = fake_dma_object(vm->i915, size << PAGE_SHIFT);
577 if (IS_ERR(obj))
578 break;
579
580 vma = i915_vma_instance(obj, vm, NULL);
581 if (IS_ERR(vma)) {
582 err = PTR_ERR(vma);
583 goto err_put;
584 }
585
586 for (addr = hole_start;
587 addr + obj->base.size < hole_end;
588 addr += obj->base.size) {
589 err = i915_vma_pin(vma, 0, 0, addr | flags);
590 if (err) {
591 pr_err("%s bind failed at %llx + %llx [hole %llx- %llx] with err=%d\n",
592 __func__, addr, vma->size,
593 hole_start, hole_end, err);
594 goto err_close;
595 }
596 i915_vma_unpin(vma);
597
598 if (!drm_mm_node_allocated(&vma->node) ||
599 i915_vma_misplaced(vma, 0, 0, addr | flags)) {
600 pr_err("%s incorrect at %llx + %llx\n",
601 __func__, addr, vma->size);
602 err = -EINVAL;
603 goto err_close;
604 }
605
606 err = i915_vma_unbind(vma);
607 if (err) {
608 pr_err("%s unbind failed at %llx + %llx with err=%d\n",
609 __func__, addr, vma->size, err);
610 goto err_close;
611 }
612
613 GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
614
615 if (igt_timeout(end_time,
616 "%s timed out at %llx\n",
617 __func__, addr)) {
618 err = -EINTR;
619 goto err_close;
620 }
621 }
622
623 err_close:
624 if (!i915_vma_is_ggtt(vma))
625 i915_vma_close(vma);
626 err_put:
627 i915_gem_object_put(obj);
628 if (err)
629 return err;
630
631 cleanup_freed_objects(vm->i915);
632 }
633
634 return 0;
635 }
636
637 static int pot_hole(struct i915_address_space *vm,
638 u64 hole_start, u64 hole_end,
639 unsigned long end_time)
640 {
641 struct drm_i915_gem_object *obj;
642 struct i915_vma *vma;
643 unsigned long flags;
644 unsigned int pot;
645 int err = 0;
646
647 flags = PIN_OFFSET_FIXED | PIN_USER;
648 if (i915_is_ggtt(vm))
649 flags |= PIN_GLOBAL;
650
651 obj = i915_gem_object_create_internal(vm->i915, 2 * I915_GTT_PAGE_SIZE);
652 if (IS_ERR(obj))
653 return PTR_ERR(obj);
654
655 vma = i915_vma_instance(obj, vm, NULL);
656 if (IS_ERR(vma)) {
657 err = PTR_ERR(vma);
658 goto err_obj;
659 }
660
661 /* Insert a pair of pages across every pot boundary within the hole */
662 for (pot = fls64(hole_end - 1) - 1;
663 pot > ilog2(2 * I915_GTT_PAGE_SIZE);
664 pot--) {
665 u64 step = BIT_ULL(pot);
666 u64 addr;
667
668 for (addr = round_up(hole_start + I915_GTT_PAGE_SIZE, step) - I915_GTT_PAGE_SIZE;
669 addr <= round_down(hole_end - 2*I915_GTT_PAGE_SIZE, step) - I915_GTT_PAGE_SIZE;
670 addr += step) {
671 err = i915_vma_pin(vma, 0, 0, addr | flags);
672 if (err) {
673 pr_err("%s failed to pin object at %llx in hole [%llx - %llx], with err=%d\n",
674 __func__,
675 addr,
676 hole_start, hole_end,
677 err);
678 goto err;
679 }
680
681 if (!drm_mm_node_allocated(&vma->node) ||
682 i915_vma_misplaced(vma, 0, 0, addr | flags)) {
683 pr_err("%s incorrect at %llx + %llx\n",
684 __func__, addr, vma->size);
685 i915_vma_unpin(vma);
686 err = i915_vma_unbind(vma);
687 err = -EINVAL;
688 goto err;
689 }
690
691 i915_vma_unpin(vma);
692 err = i915_vma_unbind(vma);
693 GEM_BUG_ON(err);
694 }
695
696 if (igt_timeout(end_time,
697 "%s timed out after %d/%d\n",
698 __func__, pot, fls64(hole_end - 1) - 1)) {
699 err = -EINTR;
700 goto err;
701 }
702 }
703
704 err:
705 if (!i915_vma_is_ggtt(vma))
706 i915_vma_close(vma);
707 err_obj:
708 i915_gem_object_put(obj);
709 return err;
710 }
711
712 static int drunk_hole(struct i915_address_space *vm,
713 u64 hole_start, u64 hole_end,
714 unsigned long end_time)
715 {
716 I915_RND_STATE(prng);
717 unsigned int size;
718 unsigned long flags;
719
720 flags = PIN_OFFSET_FIXED | PIN_USER;
721 if (i915_is_ggtt(vm))
722 flags |= PIN_GLOBAL;
723
724 /* Keep creating larger objects until one cannot fit into the hole */
725 for (size = 12; (hole_end - hole_start) >> size; size++) {
726 struct drm_i915_gem_object *obj;
727 unsigned int *order, count, n;
728 struct i915_vma *vma;
729 u64 hole_size;
730 int err = -ENODEV;
731
732 hole_size = (hole_end - hole_start) >> size;
733 if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32))
734 hole_size = KMALLOC_MAX_SIZE / sizeof(u32);
735 count = hole_size >> 1;
736 if (!count) {
737 pr_debug("%s: hole is too small [%llx - %llx] >> %d: %lld\n",
738 __func__, hole_start, hole_end, size, hole_size);
739 break;
740 }
741
742 do {
743 order = i915_random_order(count, &prng);
744 if (order)
745 break;
746 } while (count >>= 1);
747 if (!count)
748 return -ENOMEM;
749 GEM_BUG_ON(!order);
750
751 /* Ignore allocation failures (i.e. don't report them as
752 * a test failure) as we are purposefully allocating very
753 * large objects without checking that we have sufficient
754 * memory. We expect to hit -ENOMEM.
755 */
756
757 obj = fake_dma_object(vm->i915, BIT_ULL(size));
758 if (IS_ERR(obj)) {
759 kfree(order);
760 break;
761 }
762
763 vma = i915_vma_instance(obj, vm, NULL);
764 if (IS_ERR(vma)) {
765 err = PTR_ERR(vma);
766 goto err_obj;
767 }
768
769 GEM_BUG_ON(vma->size != BIT_ULL(size));
770
771 for (n = 0; n < count; n++) {
772 u64 addr = hole_start + order[n] * BIT_ULL(size);
773
774 err = i915_vma_pin(vma, 0, 0, addr | flags);
775 if (err) {
776 pr_err("%s failed to pin object at %llx + %llx in hole [%llx - %llx], with err=%d\n",
777 __func__,
778 addr, BIT_ULL(size),
779 hole_start, hole_end,
780 err);
781 goto err;
782 }
783
784 if (!drm_mm_node_allocated(&vma->node) ||
785 i915_vma_misplaced(vma, 0, 0, addr | flags)) {
786 pr_err("%s incorrect at %llx + %llx\n",
787 __func__, addr, BIT_ULL(size));
788 i915_vma_unpin(vma);
789 err = i915_vma_unbind(vma);
790 err = -EINVAL;
791 goto err;
792 }
793
794 i915_vma_unpin(vma);
795 err = i915_vma_unbind(vma);
796 GEM_BUG_ON(err);
797
798 if (igt_timeout(end_time,
799 "%s timed out after %d/%d\n",
800 __func__, n, count)) {
801 err = -EINTR;
802 goto err;
803 }
804 }
805
806 err:
807 if (!i915_vma_is_ggtt(vma))
808 i915_vma_close(vma);
809 err_obj:
810 i915_gem_object_put(obj);
811 kfree(order);
812 if (err)
813 return err;
814
815 cleanup_freed_objects(vm->i915);
816 }
817
818 return 0;
819 }
820
821 static int __shrink_hole(struct i915_address_space *vm,
822 u64 hole_start, u64 hole_end,
823 unsigned long end_time)
824 {
825 struct drm_i915_gem_object *obj;
826 unsigned long flags = PIN_OFFSET_FIXED | PIN_USER;
827 unsigned int order = 12;
828 LIST_HEAD(objects);
829 int err = 0;
830 u64 addr;
831
832 /* Keep creating larger objects until one cannot fit into the hole */
833 for (addr = hole_start; addr < hole_end; ) {
834 struct i915_vma *vma;
835 u64 size = BIT_ULL(order++);
836
837 size = min(size, hole_end - addr);
838 obj = fake_dma_object(vm->i915, size);
839 if (IS_ERR(obj)) {
840 err = PTR_ERR(obj);
841 break;
842 }
843
844 list_add(&obj->st_link, &objects);
845
846 vma = i915_vma_instance(obj, vm, NULL);
847 if (IS_ERR(vma)) {
848 err = PTR_ERR(vma);
849 break;
850 }
851
852 GEM_BUG_ON(vma->size != size);
853
854 err = i915_vma_pin(vma, 0, 0, addr | flags);
855 if (err) {
856 pr_err("%s failed to pin object at %llx + %llx in hole [%llx - %llx], with err=%d\n",
857 __func__, addr, size, hole_start, hole_end, err);
858 break;
859 }
860
861 if (!drm_mm_node_allocated(&vma->node) ||
862 i915_vma_misplaced(vma, 0, 0, addr | flags)) {
863 pr_err("%s incorrect at %llx + %llx\n",
864 __func__, addr, size);
865 i915_vma_unpin(vma);
866 err = i915_vma_unbind(vma);
867 err = -EINVAL;
868 break;
869 }
870
871 i915_vma_unpin(vma);
872 addr += size;
873
874 /*
875 * Since we are injecting allocation faults at random intervals,
876 * wait for this allocation to complete before we change the
877 * faultinjection.
878 */
879 err = i915_vma_sync(vma);
880 if (err)
881 break;
882
883 if (igt_timeout(end_time,
884 "%s timed out at ofset %llx [%llx - %llx]\n",
885 __func__, addr, hole_start, hole_end)) {
886 err = -EINTR;
887 break;
888 }
889 }
890
891 close_object_list(&objects, vm);
892 cleanup_freed_objects(vm->i915);
893 return err;
894 }
895
896 static int shrink_hole(struct i915_address_space *vm,
897 u64 hole_start, u64 hole_end,
898 unsigned long end_time)
899 {
900 unsigned long prime;
901 int err;
902
903 vm->fault_attr.probability = 999;
904 atomic_set(&vm->fault_attr.times, -1);
905
906 for_each_prime_number_from(prime, 0, ULONG_MAX - 1) {
907 vm->fault_attr.interval = prime;
908 err = __shrink_hole(vm, hole_start, hole_end, end_time);
909 if (err)
910 break;
911 }
912
913 memset(&vm->fault_attr, 0, sizeof(vm->fault_attr));
914
915 return err;
916 }
917
918 static int shrink_boom(struct i915_address_space *vm,
919 u64 hole_start, u64 hole_end,
920 unsigned long end_time)
921 {
922 unsigned int sizes[] = { SZ_2M, SZ_1G };
923 struct drm_i915_gem_object *purge;
924 struct drm_i915_gem_object *explode;
925 int err;
926 int i;
927
928 /*
929 * Catch the case which shrink_hole seems to miss. The setup here
930 * requires invoking the shrinker as we do the alloc_pt/alloc_pd, while
931 * ensuring that all vma assiocated with the respective pd/pdp are
932 * unpinned at the time.
933 */
934
935 for (i = 0; i < ARRAY_SIZE(sizes); ++i) {
936 unsigned int flags = PIN_USER | PIN_OFFSET_FIXED;
937 unsigned int size = sizes[i];
938 struct i915_vma *vma;
939
940 purge = fake_dma_object(vm->i915, size);
941 if (IS_ERR(purge))
942 return PTR_ERR(purge);
943
944 vma = i915_vma_instance(purge, vm, NULL);
945 if (IS_ERR(vma)) {
946 err = PTR_ERR(vma);
947 goto err_purge;
948 }
949
950 err = i915_vma_pin(vma, 0, 0, flags);
951 if (err)
952 goto err_purge;
953
954 /* Should now be ripe for purging */
955 i915_vma_unpin(vma);
956
957 explode = fake_dma_object(vm->i915, size);
958 if (IS_ERR(explode)) {
959 err = PTR_ERR(explode);
960 goto err_purge;
961 }
962
963 vm->fault_attr.probability = 100;
964 vm->fault_attr.interval = 1;
965 atomic_set(&vm->fault_attr.times, -1);
966
967 vma = i915_vma_instance(explode, vm, NULL);
968 if (IS_ERR(vma)) {
969 err = PTR_ERR(vma);
970 goto err_explode;
971 }
972
973 err = i915_vma_pin(vma, 0, 0, flags | size);
974 if (err)
975 goto err_explode;
976
977 i915_vma_unpin(vma);
978
979 i915_gem_object_put(purge);
980 i915_gem_object_put(explode);
981
982 memset(&vm->fault_attr, 0, sizeof(vm->fault_attr));
983 cleanup_freed_objects(vm->i915);
984 }
985
986 return 0;
987
988 err_explode:
989 i915_gem_object_put(explode);
990 err_purge:
991 i915_gem_object_put(purge);
992 memset(&vm->fault_attr, 0, sizeof(vm->fault_attr));
993 return err;
994 }
995
996 static int exercise_ppgtt(struct drm_i915_private *dev_priv,
997 int (*func)(struct i915_address_space *vm,
998 u64 hole_start, u64 hole_end,
999 unsigned long end_time))
1000 {
1001 struct i915_ppgtt *ppgtt;
1002 IGT_TIMEOUT(end_time);
1003 struct file *file;
1004 int err;
1005
1006 if (!HAS_FULL_PPGTT(dev_priv))
1007 return 0;
1008
1009 file = mock_file(dev_priv);
1010 if (IS_ERR(file))
1011 return PTR_ERR(file);
1012
1013 ppgtt = i915_ppgtt_create(&dev_priv->gt);
1014 if (IS_ERR(ppgtt)) {
1015 err = PTR_ERR(ppgtt);
1016 goto out_free;
1017 }
1018 GEM_BUG_ON(offset_in_page(ppgtt->vm.total));
1019 GEM_BUG_ON(!atomic_read(&ppgtt->vm.open));
1020
1021 err = func(&ppgtt->vm, 0, ppgtt->vm.total, end_time);
1022
1023 i915_vm_put(&ppgtt->vm);
1024
1025 out_free:
1026 fput(file);
1027 return err;
1028 }
1029
1030 static int igt_ppgtt_fill(void *arg)
1031 {
1032 return exercise_ppgtt(arg, fill_hole);
1033 }
1034
1035 static int igt_ppgtt_walk(void *arg)
1036 {
1037 return exercise_ppgtt(arg, walk_hole);
1038 }
1039
1040 static int igt_ppgtt_pot(void *arg)
1041 {
1042 return exercise_ppgtt(arg, pot_hole);
1043 }
1044
1045 static int igt_ppgtt_drunk(void *arg)
1046 {
1047 return exercise_ppgtt(arg, drunk_hole);
1048 }
1049
1050 static int igt_ppgtt_lowlevel(void *arg)
1051 {
1052 return exercise_ppgtt(arg, lowlevel_hole);
1053 }
1054
1055 static int igt_ppgtt_shrink(void *arg)
1056 {
1057 return exercise_ppgtt(arg, shrink_hole);
1058 }
1059
1060 static int igt_ppgtt_shrink_boom(void *arg)
1061 {
1062 return exercise_ppgtt(arg, shrink_boom);
1063 }
1064
1065 static int sort_holes(void *priv, struct list_head *A, struct list_head *B)
1066 {
1067 struct drm_mm_node *a = list_entry(A, typeof(*a), hole_stack);
1068 struct drm_mm_node *b = list_entry(B, typeof(*b), hole_stack);
1069
1070 if (a->start < b->start)
1071 return -1;
1072 else
1073 return 1;
1074 }
1075
1076 static int exercise_ggtt(struct drm_i915_private *i915,
1077 int (*func)(struct i915_address_space *vm,
1078 u64 hole_start, u64 hole_end,
1079 unsigned long end_time))
1080 {
1081 struct i915_ggtt *ggtt = &i915->ggtt;
1082 u64 hole_start, hole_end, last = 0;
1083 struct drm_mm_node *node;
1084 IGT_TIMEOUT(end_time);
1085 int err = 0;
1086
1087 restart:
1088 list_sort(NULL, &ggtt->vm.mm.hole_stack, sort_holes);
1089 drm_mm_for_each_hole(node, &ggtt->vm.mm, hole_start, hole_end) {
1090 if (hole_start < last)
1091 continue;
1092
1093 if (ggtt->vm.mm.color_adjust)
1094 ggtt->vm.mm.color_adjust(node, 0,
1095 &hole_start, &hole_end);
1096 if (hole_start >= hole_end)
1097 continue;
1098
1099 err = func(&ggtt->vm, hole_start, hole_end, end_time);
1100 if (err)
1101 break;
1102
1103 /* As we have manipulated the drm_mm, the list may be corrupt */
1104 last = hole_end;
1105 goto restart;
1106 }
1107
1108 return err;
1109 }
1110
1111 static int igt_ggtt_fill(void *arg)
1112 {
1113 return exercise_ggtt(arg, fill_hole);
1114 }
1115
1116 static int igt_ggtt_walk(void *arg)
1117 {
1118 return exercise_ggtt(arg, walk_hole);
1119 }
1120
1121 static int igt_ggtt_pot(void *arg)
1122 {
1123 return exercise_ggtt(arg, pot_hole);
1124 }
1125
1126 static int igt_ggtt_drunk(void *arg)
1127 {
1128 return exercise_ggtt(arg, drunk_hole);
1129 }
1130
1131 static int igt_ggtt_lowlevel(void *arg)
1132 {
1133 return exercise_ggtt(arg, lowlevel_hole);
1134 }
1135
1136 static int igt_ggtt_page(void *arg)
1137 {
1138 const unsigned int count = PAGE_SIZE/sizeof(u32);
1139 I915_RND_STATE(prng);
1140 struct drm_i915_private *i915 = arg;
1141 struct i915_ggtt *ggtt = &i915->ggtt;
1142 struct drm_i915_gem_object *obj;
1143 intel_wakeref_t wakeref;
1144 struct drm_mm_node tmp;
1145 unsigned int *order, n;
1146 int err;
1147
1148 if (!i915_ggtt_has_aperture(ggtt))
1149 return 0;
1150
1151 obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
1152 if (IS_ERR(obj))
1153 return PTR_ERR(obj);
1154
1155 err = i915_gem_object_pin_pages(obj);
1156 if (err)
1157 goto out_free;
1158
1159 memset(&tmp, 0, sizeof(tmp));
1160 mutex_lock(&ggtt->vm.mutex);
1161 err = drm_mm_insert_node_in_range(&ggtt->vm.mm, &tmp,
1162 count * PAGE_SIZE, 0,
1163 I915_COLOR_UNEVICTABLE,
1164 0, ggtt->mappable_end,
1165 DRM_MM_INSERT_LOW);
1166 mutex_unlock(&ggtt->vm.mutex);
1167 if (err)
1168 goto out_unpin;
1169
1170 wakeref = intel_runtime_pm_get(&i915->runtime_pm);
1171
1172 for (n = 0; n < count; n++) {
1173 u64 offset = tmp.start + n * PAGE_SIZE;
1174
1175 ggtt->vm.insert_page(&ggtt->vm,
1176 i915_gem_object_get_dma_address(obj, 0),
1177 offset, I915_CACHE_NONE, 0);
1178 }
1179
1180 order = i915_random_order(count, &prng);
1181 if (!order) {
1182 err = -ENOMEM;
1183 goto out_remove;
1184 }
1185
1186 for (n = 0; n < count; n++) {
1187 u64 offset = tmp.start + order[n] * PAGE_SIZE;
1188 u32 __iomem *vaddr;
1189
1190 vaddr = io_mapping_map_atomic_wc(&ggtt->iomap, offset);
1191 iowrite32(n, vaddr + n);
1192 io_mapping_unmap_atomic(vaddr);
1193 }
1194 intel_gt_flush_ggtt_writes(ggtt->vm.gt);
1195
1196 i915_random_reorder(order, count, &prng);
1197 for (n = 0; n < count; n++) {
1198 u64 offset = tmp.start + order[n] * PAGE_SIZE;
1199 u32 __iomem *vaddr;
1200 u32 val;
1201
1202 vaddr = io_mapping_map_atomic_wc(&ggtt->iomap, offset);
1203 val = ioread32(vaddr + n);
1204 io_mapping_unmap_atomic(vaddr);
1205
1206 if (val != n) {
1207 pr_err("insert page failed: found %d, expected %d\n",
1208 val, n);
1209 err = -EINVAL;
1210 break;
1211 }
1212 }
1213
1214 kfree(order);
1215 out_remove:
1216 ggtt->vm.clear_range(&ggtt->vm, tmp.start, tmp.size);
1217 intel_runtime_pm_put(&i915->runtime_pm, wakeref);
1218 mutex_lock(&ggtt->vm.mutex);
1219 drm_mm_remove_node(&tmp);
1220 mutex_unlock(&ggtt->vm.mutex);
1221 out_unpin:
1222 i915_gem_object_unpin_pages(obj);
1223 out_free:
1224 i915_gem_object_put(obj);
1225 return err;
1226 }
1227
1228 static void track_vma_bind(struct i915_vma *vma)
1229 {
1230 struct drm_i915_gem_object *obj = vma->obj;
1231
1232 atomic_inc(&obj->bind_count); /* track for eviction later */
1233 __i915_gem_object_pin_pages(obj);
1234
1235 GEM_BUG_ON(vma->pages);
1236 atomic_set(&vma->pages_count, I915_VMA_PAGES_ACTIVE);
1237 __i915_gem_object_pin_pages(obj);
1238 vma->pages = obj->mm.pages;
1239
1240 mutex_lock(&vma->vm->mutex);
1241 list_add_tail(&vma->vm_link, &vma->vm->bound_list);
1242 mutex_unlock(&vma->vm->mutex);
1243 }
1244
1245 static int exercise_mock(struct drm_i915_private *i915,
1246 int (*func)(struct i915_address_space *vm,
1247 u64 hole_start, u64 hole_end,
1248 unsigned long end_time))
1249 {
1250 const u64 limit = totalram_pages() << PAGE_SHIFT;
1251 struct i915_address_space *vm;
1252 struct i915_gem_context *ctx;
1253 IGT_TIMEOUT(end_time);
1254 int err;
1255
1256 ctx = mock_context(i915, "mock");
1257 if (!ctx)
1258 return -ENOMEM;
1259
1260 vm = i915_gem_context_get_vm_rcu(ctx);
1261 err = func(vm, 0, min(vm->total, limit), end_time);
1262 i915_vm_put(vm);
1263
1264 mock_context_close(ctx);
1265 return err;
1266 }
1267
1268 static int igt_mock_fill(void *arg)
1269 {
1270 struct i915_ggtt *ggtt = arg;
1271
1272 return exercise_mock(ggtt->vm.i915, fill_hole);
1273 }
1274
1275 static int igt_mock_walk(void *arg)
1276 {
1277 struct i915_ggtt *ggtt = arg;
1278
1279 return exercise_mock(ggtt->vm.i915, walk_hole);
1280 }
1281
1282 static int igt_mock_pot(void *arg)
1283 {
1284 struct i915_ggtt *ggtt = arg;
1285
1286 return exercise_mock(ggtt->vm.i915, pot_hole);
1287 }
1288
1289 static int igt_mock_drunk(void *arg)
1290 {
1291 struct i915_ggtt *ggtt = arg;
1292
1293 return exercise_mock(ggtt->vm.i915, drunk_hole);
1294 }
1295
1296 static int igt_gtt_reserve(void *arg)
1297 {
1298 struct i915_ggtt *ggtt = arg;
1299 struct drm_i915_gem_object *obj, *on;
1300 I915_RND_STATE(prng);
1301 LIST_HEAD(objects);
1302 u64 total;
1303 int err = -ENODEV;
1304
1305 /* i915_gem_gtt_reserve() tries to reserve the precise range
1306 * for the node, and evicts if it has to. So our test checks that
1307 * it can give us the requsted space and prevent overlaps.
1308 */
1309
1310 /* Start by filling the GGTT */
1311 for (total = 0;
1312 total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total;
1313 total += 2 * I915_GTT_PAGE_SIZE) {
1314 struct i915_vma *vma;
1315
1316 obj = i915_gem_object_create_internal(ggtt->vm.i915,
1317 2 * PAGE_SIZE);
1318 if (IS_ERR(obj)) {
1319 err = PTR_ERR(obj);
1320 goto out;
1321 }
1322
1323 err = i915_gem_object_pin_pages(obj);
1324 if (err) {
1325 i915_gem_object_put(obj);
1326 goto out;
1327 }
1328
1329 list_add(&obj->st_link, &objects);
1330
1331 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1332 if (IS_ERR(vma)) {
1333 err = PTR_ERR(vma);
1334 goto out;
1335 }
1336
1337 mutex_lock(&ggtt->vm.mutex);
1338 err = i915_gem_gtt_reserve(&ggtt->vm, &vma->node,
1339 obj->base.size,
1340 total,
1341 obj->cache_level,
1342 0);
1343 mutex_unlock(&ggtt->vm.mutex);
1344 if (err) {
1345 pr_err("i915_gem_gtt_reserve (pass 1) failed at %llu/%llu with err=%d\n",
1346 total, ggtt->vm.total, err);
1347 goto out;
1348 }
1349 track_vma_bind(vma);
1350
1351 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1352 if (vma->node.start != total ||
1353 vma->node.size != 2*I915_GTT_PAGE_SIZE) {
1354 pr_err("i915_gem_gtt_reserve (pass 1) placement failed, found (%llx + %llx), expected (%llx + %llx)\n",
1355 vma->node.start, vma->node.size,
1356 total, 2*I915_GTT_PAGE_SIZE);
1357 err = -EINVAL;
1358 goto out;
1359 }
1360 }
1361
1362 /* Now we start forcing evictions */
1363 for (total = I915_GTT_PAGE_SIZE;
1364 total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total;
1365 total += 2 * I915_GTT_PAGE_SIZE) {
1366 struct i915_vma *vma;
1367
1368 obj = i915_gem_object_create_internal(ggtt->vm.i915,
1369 2 * PAGE_SIZE);
1370 if (IS_ERR(obj)) {
1371 err = PTR_ERR(obj);
1372 goto out;
1373 }
1374
1375 err = i915_gem_object_pin_pages(obj);
1376 if (err) {
1377 i915_gem_object_put(obj);
1378 goto out;
1379 }
1380
1381 list_add(&obj->st_link, &objects);
1382
1383 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1384 if (IS_ERR(vma)) {
1385 err = PTR_ERR(vma);
1386 goto out;
1387 }
1388
1389 mutex_lock(&ggtt->vm.mutex);
1390 err = i915_gem_gtt_reserve(&ggtt->vm, &vma->node,
1391 obj->base.size,
1392 total,
1393 obj->cache_level,
1394 0);
1395 mutex_unlock(&ggtt->vm.mutex);
1396 if (err) {
1397 pr_err("i915_gem_gtt_reserve (pass 2) failed at %llu/%llu with err=%d\n",
1398 total, ggtt->vm.total, err);
1399 goto out;
1400 }
1401 track_vma_bind(vma);
1402
1403 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1404 if (vma->node.start != total ||
1405 vma->node.size != 2*I915_GTT_PAGE_SIZE) {
1406 pr_err("i915_gem_gtt_reserve (pass 2) placement failed, found (%llx + %llx), expected (%llx + %llx)\n",
1407 vma->node.start, vma->node.size,
1408 total, 2*I915_GTT_PAGE_SIZE);
1409 err = -EINVAL;
1410 goto out;
1411 }
1412 }
1413
1414 /* And then try at random */
1415 list_for_each_entry_safe(obj, on, &objects, st_link) {
1416 struct i915_vma *vma;
1417 u64 offset;
1418
1419 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1420 if (IS_ERR(vma)) {
1421 err = PTR_ERR(vma);
1422 goto out;
1423 }
1424
1425 err = i915_vma_unbind(vma);
1426 if (err) {
1427 pr_err("i915_vma_unbind failed with err=%d!\n", err);
1428 goto out;
1429 }
1430
1431 offset = igt_random_offset(&prng,
1432 0, ggtt->vm.total,
1433 2 * I915_GTT_PAGE_SIZE,
1434 I915_GTT_MIN_ALIGNMENT);
1435
1436 mutex_lock(&ggtt->vm.mutex);
1437 err = i915_gem_gtt_reserve(&ggtt->vm, &vma->node,
1438 obj->base.size,
1439 offset,
1440 obj->cache_level,
1441 0);
1442 mutex_unlock(&ggtt->vm.mutex);
1443 if (err) {
1444 pr_err("i915_gem_gtt_reserve (pass 3) failed at %llu/%llu with err=%d\n",
1445 total, ggtt->vm.total, err);
1446 goto out;
1447 }
1448 track_vma_bind(vma);
1449
1450 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1451 if (vma->node.start != offset ||
1452 vma->node.size != 2*I915_GTT_PAGE_SIZE) {
1453 pr_err("i915_gem_gtt_reserve (pass 3) placement failed, found (%llx + %llx), expected (%llx + %llx)\n",
1454 vma->node.start, vma->node.size,
1455 offset, 2*I915_GTT_PAGE_SIZE);
1456 err = -EINVAL;
1457 goto out;
1458 }
1459 }
1460
1461 out:
1462 list_for_each_entry_safe(obj, on, &objects, st_link) {
1463 i915_gem_object_unpin_pages(obj);
1464 i915_gem_object_put(obj);
1465 }
1466 return err;
1467 }
1468
1469 static int igt_gtt_insert(void *arg)
1470 {
1471 struct i915_ggtt *ggtt = arg;
1472 struct drm_i915_gem_object *obj, *on;
1473 struct drm_mm_node tmp = {};
1474 const struct invalid_insert {
1475 u64 size;
1476 u64 alignment;
1477 u64 start, end;
1478 } invalid_insert[] = {
1479 {
1480 ggtt->vm.total + I915_GTT_PAGE_SIZE, 0,
1481 0, ggtt->vm.total,
1482 },
1483 {
1484 2*I915_GTT_PAGE_SIZE, 0,
1485 0, I915_GTT_PAGE_SIZE,
1486 },
1487 {
1488 -(u64)I915_GTT_PAGE_SIZE, 0,
1489 0, 4*I915_GTT_PAGE_SIZE,
1490 },
1491 {
1492 -(u64)2*I915_GTT_PAGE_SIZE, 2*I915_GTT_PAGE_SIZE,
1493 0, 4*I915_GTT_PAGE_SIZE,
1494 },
1495 {
1496 I915_GTT_PAGE_SIZE, I915_GTT_MIN_ALIGNMENT << 1,
1497 I915_GTT_MIN_ALIGNMENT, I915_GTT_MIN_ALIGNMENT << 1,
1498 },
1499 {}
1500 }, *ii;
1501 LIST_HEAD(objects);
1502 u64 total;
1503 int err = -ENODEV;
1504
1505 /* i915_gem_gtt_insert() tries to allocate some free space in the GTT
1506 * to the node, evicting if required.
1507 */
1508
1509 /* Check a couple of obviously invalid requests */
1510 for (ii = invalid_insert; ii->size; ii++) {
1511 mutex_lock(&ggtt->vm.mutex);
1512 err = i915_gem_gtt_insert(&ggtt->vm, &tmp,
1513 ii->size, ii->alignment,
1514 I915_COLOR_UNEVICTABLE,
1515 ii->start, ii->end,
1516 0);
1517 mutex_unlock(&ggtt->vm.mutex);
1518 if (err != -ENOSPC) {
1519 pr_err("Invalid i915_gem_gtt_insert(.size=%llx, .alignment=%llx, .start=%llx, .end=%llx) succeeded (err=%d)\n",
1520 ii->size, ii->alignment, ii->start, ii->end,
1521 err);
1522 return -EINVAL;
1523 }
1524 }
1525
1526 /* Start by filling the GGTT */
1527 for (total = 0;
1528 total + I915_GTT_PAGE_SIZE <= ggtt->vm.total;
1529 total += I915_GTT_PAGE_SIZE) {
1530 struct i915_vma *vma;
1531
1532 obj = i915_gem_object_create_internal(ggtt->vm.i915,
1533 I915_GTT_PAGE_SIZE);
1534 if (IS_ERR(obj)) {
1535 err = PTR_ERR(obj);
1536 goto out;
1537 }
1538
1539 err = i915_gem_object_pin_pages(obj);
1540 if (err) {
1541 i915_gem_object_put(obj);
1542 goto out;
1543 }
1544
1545 list_add(&obj->st_link, &objects);
1546
1547 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1548 if (IS_ERR(vma)) {
1549 err = PTR_ERR(vma);
1550 goto out;
1551 }
1552
1553 mutex_lock(&ggtt->vm.mutex);
1554 err = i915_gem_gtt_insert(&ggtt->vm, &vma->node,
1555 obj->base.size, 0, obj->cache_level,
1556 0, ggtt->vm.total,
1557 0);
1558 mutex_unlock(&ggtt->vm.mutex);
1559 if (err == -ENOSPC) {
1560 /* maxed out the GGTT space */
1561 i915_gem_object_put(obj);
1562 break;
1563 }
1564 if (err) {
1565 pr_err("i915_gem_gtt_insert (pass 1) failed at %llu/%llu with err=%d\n",
1566 total, ggtt->vm.total, err);
1567 goto out;
1568 }
1569 track_vma_bind(vma);
1570 __i915_vma_pin(vma);
1571
1572 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1573 }
1574
1575 list_for_each_entry(obj, &objects, st_link) {
1576 struct i915_vma *vma;
1577
1578 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1579 if (IS_ERR(vma)) {
1580 err = PTR_ERR(vma);
1581 goto out;
1582 }
1583
1584 if (!drm_mm_node_allocated(&vma->node)) {
1585 pr_err("VMA was unexpectedly evicted!\n");
1586 err = -EINVAL;
1587 goto out;
1588 }
1589
1590 __i915_vma_unpin(vma);
1591 }
1592
1593 /* If we then reinsert, we should find the same hole */
1594 list_for_each_entry_safe(obj, on, &objects, st_link) {
1595 struct i915_vma *vma;
1596 u64 offset;
1597
1598 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1599 if (IS_ERR(vma)) {
1600 err = PTR_ERR(vma);
1601 goto out;
1602 }
1603
1604 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1605 offset = vma->node.start;
1606
1607 err = i915_vma_unbind(vma);
1608 if (err) {
1609 pr_err("i915_vma_unbind failed with err=%d!\n", err);
1610 goto out;
1611 }
1612
1613 mutex_lock(&ggtt->vm.mutex);
1614 err = i915_gem_gtt_insert(&ggtt->vm, &vma->node,
1615 obj->base.size, 0, obj->cache_level,
1616 0, ggtt->vm.total,
1617 0);
1618 mutex_unlock(&ggtt->vm.mutex);
1619 if (err) {
1620 pr_err("i915_gem_gtt_insert (pass 2) failed at %llu/%llu with err=%d\n",
1621 total, ggtt->vm.total, err);
1622 goto out;
1623 }
1624 track_vma_bind(vma);
1625
1626 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1627 if (vma->node.start != offset) {
1628 pr_err("i915_gem_gtt_insert did not return node to its previous location (the only hole), expected address %llx, found %llx\n",
1629 offset, vma->node.start);
1630 err = -EINVAL;
1631 goto out;
1632 }
1633 }
1634
1635 /* And then force evictions */
1636 for (total = 0;
1637 total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total;
1638 total += 2 * I915_GTT_PAGE_SIZE) {
1639 struct i915_vma *vma;
1640
1641 obj = i915_gem_object_create_internal(ggtt->vm.i915,
1642 2 * I915_GTT_PAGE_SIZE);
1643 if (IS_ERR(obj)) {
1644 err = PTR_ERR(obj);
1645 goto out;
1646 }
1647
1648 err = i915_gem_object_pin_pages(obj);
1649 if (err) {
1650 i915_gem_object_put(obj);
1651 goto out;
1652 }
1653
1654 list_add(&obj->st_link, &objects);
1655
1656 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1657 if (IS_ERR(vma)) {
1658 err = PTR_ERR(vma);
1659 goto out;
1660 }
1661
1662 mutex_lock(&ggtt->vm.mutex);
1663 err = i915_gem_gtt_insert(&ggtt->vm, &vma->node,
1664 obj->base.size, 0, obj->cache_level,
1665 0, ggtt->vm.total,
1666 0);
1667 mutex_unlock(&ggtt->vm.mutex);
1668 if (err) {
1669 pr_err("i915_gem_gtt_insert (pass 3) failed at %llu/%llu with err=%d\n",
1670 total, ggtt->vm.total, err);
1671 goto out;
1672 }
1673 track_vma_bind(vma);
1674
1675 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1676 }
1677
1678 out:
1679 list_for_each_entry_safe(obj, on, &objects, st_link) {
1680 i915_gem_object_unpin_pages(obj);
1681 i915_gem_object_put(obj);
1682 }
1683 return err;
1684 }
1685
1686 int i915_gem_gtt_mock_selftests(void)
1687 {
1688 static const struct i915_subtest tests[] = {
1689 SUBTEST(igt_mock_drunk),
1690 SUBTEST(igt_mock_walk),
1691 SUBTEST(igt_mock_pot),
1692 SUBTEST(igt_mock_fill),
1693 SUBTEST(igt_gtt_reserve),
1694 SUBTEST(igt_gtt_insert),
1695 };
1696 struct drm_i915_private *i915;
1697 struct i915_ggtt *ggtt;
1698 int err;
1699
1700 i915 = mock_gem_device();
1701 if (!i915)
1702 return -ENOMEM;
1703
1704 ggtt = kmalloc(sizeof(*ggtt), GFP_KERNEL);
1705 if (!ggtt) {
1706 err = -ENOMEM;
1707 goto out_put;
1708 }
1709 mock_init_ggtt(i915, ggtt);
1710
1711 err = i915_subtests(tests, ggtt);
1712
1713 mock_device_flush(i915);
1714 i915_gem_drain_freed_objects(i915);
1715 mock_fini_ggtt(ggtt);
1716 kfree(ggtt);
1717 out_put:
1718 drm_dev_put(&i915->drm);
1719 return err;
1720 }
1721
1722 static int context_sync(struct intel_context *ce)
1723 {
1724 struct i915_request *rq;
1725 long timeout;
1726
1727 rq = intel_context_create_request(ce);
1728 if (IS_ERR(rq))
1729 return PTR_ERR(rq);
1730
1731 i915_request_get(rq);
1732 i915_request_add(rq);
1733
1734 timeout = i915_request_wait(rq, 0, HZ / 5);
1735 i915_request_put(rq);
1736
1737 return timeout < 0 ? -EIO : 0;
1738 }
1739
1740 static struct i915_request *
1741 submit_batch(struct intel_context *ce, u64 addr)
1742 {
1743 struct i915_request *rq;
1744 int err;
1745
1746 rq = intel_context_create_request(ce);
1747 if (IS_ERR(rq))
1748 return rq;
1749
1750 err = 0;
1751 if (rq->engine->emit_init_breadcrumb) /* detect a hang */
1752 err = rq->engine->emit_init_breadcrumb(rq);
1753 if (err == 0)
1754 err = rq->engine->emit_bb_start(rq, addr, 0, 0);
1755
1756 if (err == 0)
1757 i915_request_get(rq);
1758 i915_request_add(rq);
1759
1760 return err ? ERR_PTR(err) : rq;
1761 }
1762
1763 static u32 *spinner(u32 *batch, int i)
1764 {
1765 return batch + i * 64 / sizeof(*batch) + 4;
1766 }
1767
1768 static void end_spin(u32 *batch, int i)
1769 {
1770 *spinner(batch, i) = MI_BATCH_BUFFER_END;
1771 wmb();
1772 }
1773
1774 static int igt_cs_tlb(void *arg)
1775 {
1776 const unsigned int count = PAGE_SIZE / 64;
1777 const unsigned int chunk_size = count * PAGE_SIZE;
1778 struct drm_i915_private *i915 = arg;
1779 struct drm_i915_gem_object *bbe, *act, *out;
1780 struct i915_gem_engines_iter it;
1781 struct i915_address_space *vm;
1782 struct i915_gem_context *ctx;
1783 struct intel_context *ce;
1784 struct i915_vma *vma;
1785 I915_RND_STATE(prng);
1786 struct file *file;
1787 unsigned int i;
1788 u32 *result;
1789 u32 *batch;
1790 int err = 0;
1791
1792 /*
1793 * Our mission here is to fool the hardware to execute something
1794 * from scratch as it has not seen the batch move (due to missing
1795 * the TLB invalidate).
1796 */
1797
1798 file = mock_file(i915);
1799 if (IS_ERR(file))
1800 return PTR_ERR(file);
1801
1802 ctx = live_context(i915, file);
1803 if (IS_ERR(ctx)) {
1804 err = PTR_ERR(ctx);
1805 goto out_unlock;
1806 }
1807
1808 vm = i915_gem_context_get_vm_rcu(ctx);
1809 if (i915_is_ggtt(vm))
1810 goto out_vm;
1811
1812 /* Create two pages; dummy we prefill the TLB, and intended */
1813 bbe = i915_gem_object_create_internal(i915, PAGE_SIZE);
1814 if (IS_ERR(bbe)) {
1815 err = PTR_ERR(bbe);
1816 goto out_vm;
1817 }
1818
1819 batch = i915_gem_object_pin_map(bbe, I915_MAP_WC);
1820 if (IS_ERR(batch)) {
1821 err = PTR_ERR(batch);
1822 goto out_put_bbe;
1823 }
1824 memset32(batch, MI_BATCH_BUFFER_END, PAGE_SIZE / sizeof(u32));
1825 i915_gem_object_flush_map(bbe);
1826 i915_gem_object_unpin_map(bbe);
1827
1828 act = i915_gem_object_create_internal(i915, PAGE_SIZE);
1829 if (IS_ERR(act)) {
1830 err = PTR_ERR(act);
1831 goto out_put_bbe;
1832 }
1833
1834 /* Track the execution of each request by writing into different slot */
1835 batch = i915_gem_object_pin_map(act, I915_MAP_WC);
1836 if (IS_ERR(batch)) {
1837 err = PTR_ERR(batch);
1838 goto out_put_act;
1839 }
1840 for (i = 0; i < count; i++) {
1841 u32 *cs = batch + i * 64 / sizeof(*cs);
1842 u64 addr = (vm->total - PAGE_SIZE) + i * sizeof(u32);
1843
1844 GEM_BUG_ON(INTEL_GEN(i915) < 6);
1845 cs[0] = MI_STORE_DWORD_IMM_GEN4;
1846 if (INTEL_GEN(i915) >= 8) {
1847 cs[1] = lower_32_bits(addr);
1848 cs[2] = upper_32_bits(addr);
1849 cs[3] = i;
1850 cs[4] = MI_NOOP;
1851 cs[5] = MI_BATCH_BUFFER_START_GEN8;
1852 } else {
1853 cs[1] = 0;
1854 cs[2] = lower_32_bits(addr);
1855 cs[3] = i;
1856 cs[4] = MI_NOOP;
1857 cs[5] = MI_BATCH_BUFFER_START;
1858 }
1859 }
1860
1861 out = i915_gem_object_create_internal(i915, PAGE_SIZE);
1862 if (IS_ERR(out)) {
1863 err = PTR_ERR(out);
1864 goto out_put_batch;
1865 }
1866 i915_gem_object_set_cache_coherency(out, I915_CACHING_CACHED);
1867
1868 vma = i915_vma_instance(out, vm, NULL);
1869 if (IS_ERR(vma)) {
1870 err = PTR_ERR(vma);
1871 goto out_put_batch;
1872 }
1873
1874 err = i915_vma_pin(vma, 0, 0,
1875 PIN_USER |
1876 PIN_OFFSET_FIXED |
1877 (vm->total - PAGE_SIZE));
1878 if (err)
1879 goto out_put_out;
1880 GEM_BUG_ON(vma->node.start != vm->total - PAGE_SIZE);
1881
1882 result = i915_gem_object_pin_map(out, I915_MAP_WB);
1883 if (IS_ERR(result)) {
1884 err = PTR_ERR(result);
1885 goto out_put_out;
1886 }
1887
1888 for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) {
1889 IGT_TIMEOUT(end_time);
1890 unsigned long pass = 0;
1891
1892 if (!intel_engine_can_store_dword(ce->engine))
1893 continue;
1894
1895 while (!__igt_timeout(end_time, NULL)) {
1896 struct i915_request *rq;
1897 u64 offset;
1898
1899 offset = igt_random_offset(&prng,
1900 0, vm->total - PAGE_SIZE,
1901 chunk_size, PAGE_SIZE);
1902
1903 err = vm->allocate_va_range(vm, offset, chunk_size);
1904 if (err)
1905 goto end;
1906
1907 memset32(result, STACK_MAGIC, PAGE_SIZE / sizeof(u32));
1908
1909 vma = i915_vma_instance(bbe, vm, NULL);
1910 if (IS_ERR(vma)) {
1911 err = PTR_ERR(vma);
1912 goto end;
1913 }
1914
1915 err = vma->ops->set_pages(vma);
1916 if (err)
1917 goto end;
1918
1919 /* Prime the TLB with the dummy pages */
1920 for (i = 0; i < count; i++) {
1921 vma->node.start = offset + i * PAGE_SIZE;
1922 vm->insert_entries(vm, vma, I915_CACHE_NONE, 0);
1923
1924 rq = submit_batch(ce, vma->node.start);
1925 if (IS_ERR(rq)) {
1926 err = PTR_ERR(rq);
1927 goto end;
1928 }
1929 i915_request_put(rq);
1930 }
1931
1932 vma->ops->clear_pages(vma);
1933
1934 err = context_sync(ce);
1935 if (err) {
1936 pr_err("%s: dummy setup timed out\n",
1937 ce->engine->name);
1938 goto end;
1939 }
1940
1941 vma = i915_vma_instance(act, vm, NULL);
1942 if (IS_ERR(vma)) {
1943 err = PTR_ERR(vma);
1944 goto end;
1945 }
1946
1947 err = vma->ops->set_pages(vma);
1948 if (err)
1949 goto end;
1950
1951 /* Replace the TLB with target batches */
1952 for (i = 0; i < count; i++) {
1953 struct i915_request *rq;
1954 u32 *cs = batch + i * 64 / sizeof(*cs);
1955 u64 addr;
1956
1957 vma->node.start = offset + i * PAGE_SIZE;
1958 vm->insert_entries(vm, vma, I915_CACHE_NONE, 0);
1959
1960 addr = vma->node.start + i * 64;
1961 cs[4] = MI_NOOP;
1962 cs[6] = lower_32_bits(addr);
1963 cs[7] = upper_32_bits(addr);
1964 wmb();
1965
1966 rq = submit_batch(ce, addr);
1967 if (IS_ERR(rq)) {
1968 err = PTR_ERR(rq);
1969 goto end;
1970 }
1971
1972 /* Wait until the context chain has started */
1973 if (i == 0) {
1974 while (READ_ONCE(result[i]) &&
1975 !i915_request_completed(rq))
1976 cond_resched();
1977 } else {
1978 end_spin(batch, i - 1);
1979 }
1980
1981 i915_request_put(rq);
1982 }
1983 end_spin(batch, count - 1);
1984
1985 vma->ops->clear_pages(vma);
1986
1987 err = context_sync(ce);
1988 if (err) {
1989 pr_err("%s: writes timed out\n",
1990 ce->engine->name);
1991 goto end;
1992 }
1993
1994 for (i = 0; i < count; i++) {
1995 if (result[i] != i) {
1996 pr_err("%s: Write lost on pass %lu, at offset %llx, index %d, found %x, expected %x\n",
1997 ce->engine->name, pass,
1998 offset, i, result[i], i);
1999 err = -EINVAL;
2000 goto end;
2001 }
2002 }
2003
2004 vm->clear_range(vm, offset, chunk_size);
2005 pass++;
2006 }
2007 }
2008 end:
2009 if (igt_flush_test(i915))
2010 err = -EIO;
2011 i915_gem_context_unlock_engines(ctx);
2012 i915_gem_object_unpin_map(out);
2013 out_put_out:
2014 i915_gem_object_put(out);
2015 out_put_batch:
2016 i915_gem_object_unpin_map(act);
2017 out_put_act:
2018 i915_gem_object_put(act);
2019 out_put_bbe:
2020 i915_gem_object_put(bbe);
2021 out_vm:
2022 i915_vm_put(vm);
2023 out_unlock:
2024 fput(file);
2025 return err;
2026 }
2027
2028 int i915_gem_gtt_live_selftests(struct drm_i915_private *i915)
2029 {
2030 static const struct i915_subtest tests[] = {
2031 SUBTEST(igt_ppgtt_alloc),
2032 SUBTEST(igt_ppgtt_lowlevel),
2033 SUBTEST(igt_ppgtt_drunk),
2034 SUBTEST(igt_ppgtt_walk),
2035 SUBTEST(igt_ppgtt_pot),
2036 SUBTEST(igt_ppgtt_fill),
2037 SUBTEST(igt_ppgtt_shrink),
2038 SUBTEST(igt_ppgtt_shrink_boom),
2039 SUBTEST(igt_ggtt_lowlevel),
2040 SUBTEST(igt_ggtt_drunk),
2041 SUBTEST(igt_ggtt_walk),
2042 SUBTEST(igt_ggtt_pot),
2043 SUBTEST(igt_ggtt_fill),
2044 SUBTEST(igt_ggtt_page),
2045 SUBTEST(igt_cs_tlb),
2046 };
2047
2048 GEM_BUG_ON(offset_in_page(i915->ggtt.vm.total));
2049
2050 return i915_subtests(tests, i915);
2051 }
Linux with added FPC-III support