?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
dt-bindings: mtd: ingenic: Use standard ecc-engine property
[linux/fpc-iii.git] / drivers / gpu / drm / nouveau / nouveau_dmem.c
blobaa9fec80492d167f720a07ee58f8e0196d858c3a
1 /*
2 * Copyright 2018 Red Hat Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22 #include "nouveau_dmem.h"
23 #include "nouveau_drv.h"
24 #include "nouveau_chan.h"
25 #include "nouveau_dma.h"
26 #include "nouveau_mem.h"
27 #include "nouveau_bo.h"
28
29 #include <nvif/class.h>
30 #include <nvif/object.h>
31 #include <nvif/if500b.h>
32 #include <nvif/if900b.h>
33
34 #include <linux/sched/mm.h>
35 #include <linux/hmm.h>
36
37 /*
38 * FIXME: this is ugly right now we are using TTM to allocate vram and we pin
39 * it in vram while in use. We likely want to overhaul memory management for
40 * nouveau to be more page like (not necessarily with system page size but a
41 * bigger page size) at lowest level and have some shim layer on top that would
42 * provide the same functionality as TTM.
43 */
44 #define DMEM_CHUNK_SIZE (2UL << 20)
45 #define DMEM_CHUNK_NPAGES (DMEM_CHUNK_SIZE >> PAGE_SHIFT)
46
47 struct nouveau_migrate;
48
49 enum nouveau_aper {
50 NOUVEAU_APER_VIRT,
51 NOUVEAU_APER_VRAM,
52 NOUVEAU_APER_HOST,
53 };
54
55 typedef int (*nouveau_migrate_copy_t)(struct nouveau_drm *drm, u64 npages,
56 enum nouveau_aper, u64 dst_addr,
57 enum nouveau_aper, u64 src_addr);
58
59 struct nouveau_dmem_chunk {
60 struct list_head list;
61 struct nouveau_bo *bo;
62 struct nouveau_drm *drm;
63 unsigned long pfn_first;
64 unsigned long callocated;
65 unsigned long bitmap[BITS_TO_LONGS(DMEM_CHUNK_NPAGES)];
66 spinlock_t lock;
67 };
68
69 struct nouveau_dmem_migrate {
70 nouveau_migrate_copy_t copy_func;
71 struct nouveau_channel *chan;
72 };
73
74 struct nouveau_dmem {
75 struct hmm_devmem *devmem;
76 struct nouveau_dmem_migrate migrate;
77 struct list_head chunk_free;
78 struct list_head chunk_full;
79 struct list_head chunk_empty;
80 struct mutex mutex;
81 };
82
83 struct nouveau_dmem_fault {
84 struct nouveau_drm *drm;
85 struct nouveau_fence *fence;
86 dma_addr_t *dma;
87 unsigned long npages;
88 };
89
90 struct nouveau_migrate {
91 struct vm_area_struct *vma;
92 struct nouveau_drm *drm;
93 struct nouveau_fence *fence;
94 unsigned long npages;
95 dma_addr_t *dma;
96 unsigned long dma_nr;
97 };
98
99 static void
100 nouveau_dmem_free(struct hmm_devmem *devmem, struct page *page)
101 {
102 struct nouveau_dmem_chunk *chunk;
103 struct nouveau_drm *drm;
104 unsigned long idx;
105
106 chunk = (void *)hmm_devmem_page_get_drvdata(page);
107 idx = page_to_pfn(page) - chunk->pfn_first;
108 drm = chunk->drm;
109
110 /*
111 * FIXME:
112 *
113 * This is really a bad example, we need to overhaul nouveau memory
114 * management to be more page focus and allow lighter locking scheme
115 * to be use in the process.
116 */
117 spin_lock(&chunk->lock);
118 clear_bit(idx, chunk->bitmap);
119 WARN_ON(!chunk->callocated);
120 chunk->callocated--;
121 /*
122 * FIXME when chunk->callocated reach 0 we should add the chunk to
123 * a reclaim list so that it can be freed in case of memory pressure.
124 */
125 spin_unlock(&chunk->lock);
126 }
127
128 static void
129 nouveau_dmem_fault_alloc_and_copy(struct vm_area_struct *vma,
130 const unsigned long *src_pfns,
131 unsigned long *dst_pfns,
132 unsigned long start,
133 unsigned long end,
134 void *private)
135 {
136 struct nouveau_dmem_fault *fault = private;
137 struct nouveau_drm *drm = fault->drm;
138 struct device *dev = drm->dev->dev;
139 unsigned long addr, i, npages = 0;
140 nouveau_migrate_copy_t copy;
141 int ret;
142
143
144 /* First allocate new memory */
145 for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) {
146 struct page *dpage, *spage;
147
148 dst_pfns[i] = 0;
149 spage = migrate_pfn_to_page(src_pfns[i]);
150 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE))
151 continue;
152
153 dpage = hmm_vma_alloc_locked_page(vma, addr);
154 if (!dpage) {
155 dst_pfns[i] = MIGRATE_PFN_ERROR;
156 continue;
157 }
158
159 dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)) |
160 MIGRATE_PFN_LOCKED;
161 npages++;
162 }
163
164 /* Allocate storage for DMA addresses, so we can unmap later. */
165 fault->dma = kmalloc(sizeof(*fault->dma) * npages, GFP_KERNEL);
166 if (!fault->dma)
167 goto error;
168
169 /* Copy things over */
170 copy = drm->dmem->migrate.copy_func;
171 for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) {
172 struct nouveau_dmem_chunk *chunk;
173 struct page *spage, *dpage;
174 u64 src_addr, dst_addr;
175
176 dpage = migrate_pfn_to_page(dst_pfns[i]);
177 if (!dpage || dst_pfns[i] == MIGRATE_PFN_ERROR)
178 continue;
179
180 spage = migrate_pfn_to_page(src_pfns[i]);
181 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) {
182 dst_pfns[i] = MIGRATE_PFN_ERROR;
183 __free_page(dpage);
184 continue;
185 }
186
187 fault->dma[fault->npages] =
188 dma_map_page_attrs(dev, dpage, 0, PAGE_SIZE,
189 PCI_DMA_BIDIRECTIONAL,
190 DMA_ATTR_SKIP_CPU_SYNC);
191 if (dma_mapping_error(dev, fault->dma[fault->npages])) {
192 dst_pfns[i] = MIGRATE_PFN_ERROR;
193 __free_page(dpage);
194 continue;
195 }
196
197 dst_addr = fault->dma[fault->npages++];
198
199 chunk = (void *)hmm_devmem_page_get_drvdata(spage);
200 src_addr = page_to_pfn(spage) - chunk->pfn_first;
201 src_addr = (src_addr << PAGE_SHIFT) + chunk->bo->bo.offset;
202
203 ret = copy(drm, 1, NOUVEAU_APER_HOST, dst_addr,
204 NOUVEAU_APER_VRAM, src_addr);
205 if (ret) {
206 dst_pfns[i] = MIGRATE_PFN_ERROR;
207 __free_page(dpage);
208 continue;
209 }
210 }
211
212 nouveau_fence_new(drm->dmem->migrate.chan, false, &fault->fence);
213
214 return;
215
216 error:
217 for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, ++i) {
218 struct page *page;
219
220 if (!dst_pfns[i] || dst_pfns[i] == MIGRATE_PFN_ERROR)
221 continue;
222
223 page = migrate_pfn_to_page(dst_pfns[i]);
224 dst_pfns[i] = MIGRATE_PFN_ERROR;
225 if (page == NULL)
226 continue;
227
228 __free_page(page);
229 }
230 }
231
232 void nouveau_dmem_fault_finalize_and_map(struct vm_area_struct *vma,
233 const unsigned long *src_pfns,
234 const unsigned long *dst_pfns,
235 unsigned long start,
236 unsigned long end,
237 void *private)
238 {
239 struct nouveau_dmem_fault *fault = private;
240 struct nouveau_drm *drm = fault->drm;
241
242 if (fault->fence) {
243 nouveau_fence_wait(fault->fence, true, false);
244 nouveau_fence_unref(&fault->fence);
245 } else {
246 /*
247 * FIXME wait for channel to be IDLE before calling finalizing
248 * the hmem object below (nouveau_migrate_hmem_fini()).
249 */
250 }
251
252 while (fault->npages--) {
253 dma_unmap_page(drm->dev->dev, fault->dma[fault->npages],
254 PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
255 }
256 kfree(fault->dma);
257 }
258
259 static const struct migrate_vma_ops nouveau_dmem_fault_migrate_ops = {
260 .alloc_and_copy = nouveau_dmem_fault_alloc_and_copy,
261 .finalize_and_map = nouveau_dmem_fault_finalize_and_map,
262 };
263
264 static vm_fault_t
265 nouveau_dmem_fault(struct hmm_devmem *devmem,
266 struct vm_area_struct *vma,
267 unsigned long addr,
268 const struct page *page,
269 unsigned int flags,
270 pmd_t *pmdp)
271 {
272 struct drm_device *drm_dev = dev_get_drvdata(devmem->device);
273 unsigned long src[1] = {0}, dst[1] = {0};
274 struct nouveau_dmem_fault fault = {0};
275 int ret;
276
277
278
279 /*
280 * FIXME what we really want is to find some heuristic to migrate more
281 * than just one page on CPU fault. When such fault happens it is very
282 * likely that more surrounding page will CPU fault too.
283 */
284 fault.drm = nouveau_drm(drm_dev);
285 ret = migrate_vma(&nouveau_dmem_fault_migrate_ops, vma, addr,
286 addr + PAGE_SIZE, src, dst, &fault);
287 if (ret)
288 return VM_FAULT_SIGBUS;
289
290 if (dst[0] == MIGRATE_PFN_ERROR)
291 return VM_FAULT_SIGBUS;
292
293 return 0;
294 }
295
296 static const struct hmm_devmem_ops
297 nouveau_dmem_devmem_ops = {
298 .free = nouveau_dmem_free,
299 .fault = nouveau_dmem_fault,
300 };
301
302 static int
303 nouveau_dmem_chunk_alloc(struct nouveau_drm *drm)
304 {
305 struct nouveau_dmem_chunk *chunk;
306 int ret;
307
308 if (drm->dmem == NULL)
309 return -EINVAL;
310
311 mutex_lock(&drm->dmem->mutex);
312 chunk = list_first_entry_or_null(&drm->dmem->chunk_empty,
313 struct nouveau_dmem_chunk,
314 list);
315 if (chunk == NULL) {
316 mutex_unlock(&drm->dmem->mutex);
317 return -ENOMEM;
318 }
319
320 list_del(&chunk->list);
321 mutex_unlock(&drm->dmem->mutex);
322
323 ret = nouveau_bo_new(&drm->client, DMEM_CHUNK_SIZE, 0,
324 TTM_PL_FLAG_VRAM, 0, 0, NULL, NULL,
325 &chunk->bo);
326 if (ret)
327 goto out;
328
329 ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false);
330 if (ret) {
331 nouveau_bo_ref(NULL, &chunk->bo);
332 goto out;
333 }
334
335 bitmap_zero(chunk->bitmap, DMEM_CHUNK_NPAGES);
336 spin_lock_init(&chunk->lock);
337
338 out:
339 mutex_lock(&drm->dmem->mutex);
340 if (chunk->bo)
341 list_add(&chunk->list, &drm->dmem->chunk_empty);
342 else
343 list_add_tail(&chunk->list, &drm->dmem->chunk_empty);
344 mutex_unlock(&drm->dmem->mutex);
345
346 return ret;
347 }
348
349 static struct nouveau_dmem_chunk *
350 nouveau_dmem_chunk_first_free_locked(struct nouveau_drm *drm)
351 {
352 struct nouveau_dmem_chunk *chunk;
353
354 chunk = list_first_entry_or_null(&drm->dmem->chunk_free,
355 struct nouveau_dmem_chunk,
356 list);
357 if (chunk)
358 return chunk;
359
360 chunk = list_first_entry_or_null(&drm->dmem->chunk_empty,
361 struct nouveau_dmem_chunk,
362 list);
363 if (chunk->bo)
364 return chunk;
365
366 return NULL;
367 }
368
369 static int
370 nouveau_dmem_pages_alloc(struct nouveau_drm *drm,
371 unsigned long npages,
372 unsigned long *pages)
373 {
374 struct nouveau_dmem_chunk *chunk;
375 unsigned long c;
376 int ret;
377
378 memset(pages, 0xff, npages * sizeof(*pages));
379
380 mutex_lock(&drm->dmem->mutex);
381 for (c = 0; c < npages;) {
382 unsigned long i;
383
384 chunk = nouveau_dmem_chunk_first_free_locked(drm);
385 if (chunk == NULL) {
386 mutex_unlock(&drm->dmem->mutex);
387 ret = nouveau_dmem_chunk_alloc(drm);
388 if (ret) {
389 if (c)
390 break;
391 return ret;
392 }
393 continue;
394 }
395
396 spin_lock(&chunk->lock);
397 i = find_first_zero_bit(chunk->bitmap, DMEM_CHUNK_NPAGES);
398 while (i < DMEM_CHUNK_NPAGES && c < npages) {
399 pages[c] = chunk->pfn_first + i;
400 set_bit(i, chunk->bitmap);
401 chunk->callocated++;
402 c++;
403
404 i = find_next_zero_bit(chunk->bitmap,
405 DMEM_CHUNK_NPAGES, i);
406 }
407 spin_unlock(&chunk->lock);
408 }
409 mutex_unlock(&drm->dmem->mutex);
410
411 return 0;
412 }
413
414 static struct page *
415 nouveau_dmem_page_alloc_locked(struct nouveau_drm *drm)
416 {
417 unsigned long pfns[1];
418 struct page *page;
419 int ret;
420
421 /* FIXME stop all the miss-match API ... */
422 ret = nouveau_dmem_pages_alloc(drm, 1, pfns);
423 if (ret)
424 return NULL;
425
426 page = pfn_to_page(pfns[0]);
427 get_page(page);
428 lock_page(page);
429 return page;
430 }
431
432 static void
433 nouveau_dmem_page_free_locked(struct nouveau_drm *drm, struct page *page)
434 {
435 unlock_page(page);
436 put_page(page);
437 }
438
439 void
440 nouveau_dmem_resume(struct nouveau_drm *drm)
441 {
442 struct nouveau_dmem_chunk *chunk;
443 int ret;
444
445 if (drm->dmem == NULL)
446 return;
447
448 mutex_lock(&drm->dmem->mutex);
449 list_for_each_entry (chunk, &drm->dmem->chunk_free, list) {
450 ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false);
451 /* FIXME handle pin failure */
452 WARN_ON(ret);
453 }
454 list_for_each_entry (chunk, &drm->dmem->chunk_full, list) {
455 ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false);
456 /* FIXME handle pin failure */
457 WARN_ON(ret);
458 }
459 list_for_each_entry (chunk, &drm->dmem->chunk_empty, list) {
460 ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false);
461 /* FIXME handle pin failure */
462 WARN_ON(ret);
463 }
464 mutex_unlock(&drm->dmem->mutex);
465 }
466
467 void
468 nouveau_dmem_suspend(struct nouveau_drm *drm)
469 {
470 struct nouveau_dmem_chunk *chunk;
471
472 if (drm->dmem == NULL)
473 return;
474
475 mutex_lock(&drm->dmem->mutex);
476 list_for_each_entry (chunk, &drm->dmem->chunk_free, list) {
477 nouveau_bo_unpin(chunk->bo);
478 }
479 list_for_each_entry (chunk, &drm->dmem->chunk_full, list) {
480 nouveau_bo_unpin(chunk->bo);
481 }
482 list_for_each_entry (chunk, &drm->dmem->chunk_empty, list) {
483 nouveau_bo_unpin(chunk->bo);
484 }
485 mutex_unlock(&drm->dmem->mutex);
486 }
487
488 void
489 nouveau_dmem_fini(struct nouveau_drm *drm)
490 {
491 struct nouveau_dmem_chunk *chunk, *tmp;
492
493 if (drm->dmem == NULL)
494 return;
495
496 mutex_lock(&drm->dmem->mutex);
497
498 WARN_ON(!list_empty(&drm->dmem->chunk_free));
499 WARN_ON(!list_empty(&drm->dmem->chunk_full));
500
501 list_for_each_entry_safe (chunk, tmp, &drm->dmem->chunk_empty, list) {
502 if (chunk->bo) {
503 nouveau_bo_unpin(chunk->bo);
504 nouveau_bo_ref(NULL, &chunk->bo);
505 }
506 list_del(&chunk->list);
507 kfree(chunk);
508 }
509
510 mutex_unlock(&drm->dmem->mutex);
511 }
512
513 static int
514 nvc0b5_migrate_copy(struct nouveau_drm *drm, u64 npages,
515 enum nouveau_aper dst_aper, u64 dst_addr,
516 enum nouveau_aper src_aper, u64 src_addr)
517 {
518 struct nouveau_channel *chan = drm->dmem->migrate.chan;
519 u32 launch_dma = (1 << 9) /* MULTI_LINE_ENABLE. */ |
520 (1 << 8) /* DST_MEMORY_LAYOUT_PITCH. */ |
521 (1 << 7) /* SRC_MEMORY_LAYOUT_PITCH. */ |
522 (1 << 2) /* FLUSH_ENABLE_TRUE. */ |
523 (2 << 0) /* DATA_TRANSFER_TYPE_NON_PIPELINED. */;
524 int ret;
525
526 ret = RING_SPACE(chan, 13);
527 if (ret)
528 return ret;
529
530 if (src_aper != NOUVEAU_APER_VIRT) {
531 switch (src_aper) {
532 case NOUVEAU_APER_VRAM:
533 BEGIN_IMC0(chan, NvSubCopy, 0x0260, 0);
534 break;
535 case NOUVEAU_APER_HOST:
536 BEGIN_IMC0(chan, NvSubCopy, 0x0260, 1);
537 break;
538 default:
539 return -EINVAL;
540 }
541 launch_dma |= 0x00001000; /* SRC_TYPE_PHYSICAL. */
542 }
543
544 if (dst_aper != NOUVEAU_APER_VIRT) {
545 switch (dst_aper) {
546 case NOUVEAU_APER_VRAM:
547 BEGIN_IMC0(chan, NvSubCopy, 0x0264, 0);
548 break;
549 case NOUVEAU_APER_HOST:
550 BEGIN_IMC0(chan, NvSubCopy, 0x0264, 1);
551 break;
552 default:
553 return -EINVAL;
554 }
555 launch_dma |= 0x00002000; /* DST_TYPE_PHYSICAL. */
556 }
557
558 BEGIN_NVC0(chan, NvSubCopy, 0x0400, 8);
559 OUT_RING (chan, upper_32_bits(src_addr));
560 OUT_RING (chan, lower_32_bits(src_addr));
561 OUT_RING (chan, upper_32_bits(dst_addr));
562 OUT_RING (chan, lower_32_bits(dst_addr));
563 OUT_RING (chan, PAGE_SIZE);
564 OUT_RING (chan, PAGE_SIZE);
565 OUT_RING (chan, PAGE_SIZE);
566 OUT_RING (chan, npages);
567 BEGIN_NVC0(chan, NvSubCopy, 0x0300, 1);
568 OUT_RING (chan, launch_dma);
569 return 0;
570 }
571
572 static int
573 nouveau_dmem_migrate_init(struct nouveau_drm *drm)
574 {
575 switch (drm->ttm.copy.oclass) {
576 case PASCAL_DMA_COPY_A:
577 case PASCAL_DMA_COPY_B:
578 case VOLTA_DMA_COPY_A:
579 case TURING_DMA_COPY_A:
580 drm->dmem->migrate.copy_func = nvc0b5_migrate_copy;
581 drm->dmem->migrate.chan = drm->ttm.chan;
582 return 0;
583 default:
584 break;
585 }
586 return -ENODEV;
587 }
588
589 void
590 nouveau_dmem_init(struct nouveau_drm *drm)
591 {
592 struct device *device = drm->dev->dev;
593 unsigned long i, size;
594 int ret;
595
596 /* This only make sense on PASCAL or newer */
597 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_PASCAL)
598 return;
599
600 if (!(drm->dmem = kzalloc(sizeof(*drm->dmem), GFP_KERNEL)))
601 return;
602
603 mutex_init(&drm->dmem->mutex);
604 INIT_LIST_HEAD(&drm->dmem->chunk_free);
605 INIT_LIST_HEAD(&drm->dmem->chunk_full);
606 INIT_LIST_HEAD(&drm->dmem->chunk_empty);
607
608 size = ALIGN(drm->client.device.info.ram_user, DMEM_CHUNK_SIZE);
609
610 /* Initialize migration dma helpers before registering memory */
611 ret = nouveau_dmem_migrate_init(drm);
612 if (ret) {
613 kfree(drm->dmem);
614 drm->dmem = NULL;
615 return;
616 }
617
618 /*
619 * FIXME we need some kind of policy to decide how much VRAM we
620 * want to register with HMM. For now just register everything
621 * and latter if we want to do thing like over commit then we
622 * could revisit this.
623 */
624 drm->dmem->devmem = hmm_devmem_add(&nouveau_dmem_devmem_ops,
625 device, size);
626 if (drm->dmem->devmem == NULL) {
627 kfree(drm->dmem);
628 drm->dmem = NULL;
629 return;
630 }
631
632 for (i = 0; i < (size / DMEM_CHUNK_SIZE); ++i) {
633 struct nouveau_dmem_chunk *chunk;
634 struct page *page;
635 unsigned long j;
636
637 chunk = kzalloc(sizeof(*chunk), GFP_KERNEL);
638 if (chunk == NULL) {
639 nouveau_dmem_fini(drm);
640 return;
641 }
642
643 chunk->drm = drm;
644 chunk->pfn_first = drm->dmem->devmem->pfn_first;
645 chunk->pfn_first += (i * DMEM_CHUNK_NPAGES);
646 list_add_tail(&chunk->list, &drm->dmem->chunk_empty);
647
648 page = pfn_to_page(chunk->pfn_first);
649 for (j = 0; j < DMEM_CHUNK_NPAGES; ++j, ++page) {
650 hmm_devmem_page_set_drvdata(page, (long)chunk);
651 }
652 }
653
654 NV_INFO(drm, "DMEM: registered %ldMB of device memory\n", size >> 20);
655 }
656
657 static void
658 nouveau_dmem_migrate_alloc_and_copy(struct vm_area_struct *vma,
659 const unsigned long *src_pfns,
660 unsigned long *dst_pfns,
661 unsigned long start,
662 unsigned long end,
663 void *private)
664 {
665 struct nouveau_migrate *migrate = private;
666 struct nouveau_drm *drm = migrate->drm;
667 struct device *dev = drm->dev->dev;
668 unsigned long addr, i, npages = 0;
669 nouveau_migrate_copy_t copy;
670 int ret;
671
672 /* First allocate new memory */
673 for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) {
674 struct page *dpage, *spage;
675
676 dst_pfns[i] = 0;
677 spage = migrate_pfn_to_page(src_pfns[i]);
678 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE))
679 continue;
680
681 dpage = nouveau_dmem_page_alloc_locked(drm);
682 if (!dpage)
683 continue;
684
685 dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)) |
686 MIGRATE_PFN_LOCKED |
687 MIGRATE_PFN_DEVICE;
688 npages++;
689 }
690
691 if (!npages)
692 return;
693
694 /* Allocate storage for DMA addresses, so we can unmap later. */
695 migrate->dma = kmalloc(sizeof(*migrate->dma) * npages, GFP_KERNEL);
696 if (!migrate->dma)
697 goto error;
698
699 /* Copy things over */
700 copy = drm->dmem->migrate.copy_func;
701 for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) {
702 struct nouveau_dmem_chunk *chunk;
703 struct page *spage, *dpage;
704 u64 src_addr, dst_addr;
705
706 dpage = migrate_pfn_to_page(dst_pfns[i]);
707 if (!dpage || dst_pfns[i] == MIGRATE_PFN_ERROR)
708 continue;
709
710 chunk = (void *)hmm_devmem_page_get_drvdata(dpage);
711 dst_addr = page_to_pfn(dpage) - chunk->pfn_first;
712 dst_addr = (dst_addr << PAGE_SHIFT) + chunk->bo->bo.offset;
713
714 spage = migrate_pfn_to_page(src_pfns[i]);
715 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) {
716 nouveau_dmem_page_free_locked(drm, dpage);
717 dst_pfns[i] = 0;
718 continue;
719 }
720
721 migrate->dma[migrate->dma_nr] =
722 dma_map_page_attrs(dev, spage, 0, PAGE_SIZE,
723 PCI_DMA_BIDIRECTIONAL,
724 DMA_ATTR_SKIP_CPU_SYNC);
725 if (dma_mapping_error(dev, migrate->dma[migrate->dma_nr])) {
726 nouveau_dmem_page_free_locked(drm, dpage);
727 dst_pfns[i] = 0;
728 continue;
729 }
730
731 src_addr = migrate->dma[migrate->dma_nr++];
732
733 ret = copy(drm, 1, NOUVEAU_APER_VRAM, dst_addr,
734 NOUVEAU_APER_HOST, src_addr);
735 if (ret) {
736 nouveau_dmem_page_free_locked(drm, dpage);
737 dst_pfns[i] = 0;
738 continue;
739 }
740 }
741
742 nouveau_fence_new(drm->dmem->migrate.chan, false, &migrate->fence);
743
744 return;
745
746 error:
747 for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, ++i) {
748 struct page *page;
749
750 if (!dst_pfns[i] || dst_pfns[i] == MIGRATE_PFN_ERROR)
751 continue;
752
753 page = migrate_pfn_to_page(dst_pfns[i]);
754 dst_pfns[i] = MIGRATE_PFN_ERROR;
755 if (page == NULL)
756 continue;
757
758 __free_page(page);
759 }
760 }
761
762 void nouveau_dmem_migrate_finalize_and_map(struct vm_area_struct *vma,
763 const unsigned long *src_pfns,
764 const unsigned long *dst_pfns,
765 unsigned long start,
766 unsigned long end,
767 void *private)
768 {
769 struct nouveau_migrate *migrate = private;
770 struct nouveau_drm *drm = migrate->drm;
771
772 if (migrate->fence) {
773 nouveau_fence_wait(migrate->fence, true, false);
774 nouveau_fence_unref(&migrate->fence);
775 } else {
776 /*
777 * FIXME wait for channel to be IDLE before finalizing
778 * the hmem object below (nouveau_migrate_hmem_fini()) ?
779 */
780 }
781
782 while (migrate->dma_nr--) {
783 dma_unmap_page(drm->dev->dev, migrate->dma[migrate->dma_nr],
784 PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
785 }
786 kfree(migrate->dma);
787
788 /*
789 * FIXME optimization: update GPU page table to point to newly
790 * migrated memory.
791 */
792 }
793
794 static const struct migrate_vma_ops nouveau_dmem_migrate_ops = {
795 .alloc_and_copy = nouveau_dmem_migrate_alloc_and_copy,
796 .finalize_and_map = nouveau_dmem_migrate_finalize_and_map,
797 };
798
799 int
800 nouveau_dmem_migrate_vma(struct nouveau_drm *drm,
801 struct vm_area_struct *vma,
802 unsigned long start,
803 unsigned long end)
804 {
805 unsigned long *src_pfns, *dst_pfns, npages;
806 struct nouveau_migrate migrate = {0};
807 unsigned long i, c, max;
808 int ret = 0;
809
810 npages = (end - start) >> PAGE_SHIFT;
811 max = min(SG_MAX_SINGLE_ALLOC, npages);
812 src_pfns = kzalloc(sizeof(long) * max, GFP_KERNEL);
813 if (src_pfns == NULL)
814 return -ENOMEM;
815 dst_pfns = kzalloc(sizeof(long) * max, GFP_KERNEL);
816 if (dst_pfns == NULL) {
817 kfree(src_pfns);
818 return -ENOMEM;
819 }
820
821 migrate.drm = drm;
822 migrate.vma = vma;
823 migrate.npages = npages;
824 for (i = 0; i < npages; i += c) {
825 unsigned long next;
826
827 c = min(SG_MAX_SINGLE_ALLOC, npages);
828 next = start + (c << PAGE_SHIFT);
829 ret = migrate_vma(&nouveau_dmem_migrate_ops, vma, start,
830 next, src_pfns, dst_pfns, &migrate);
831 if (ret)
832 goto out;
833 start = next;
834 }
835
836 out:
837 kfree(dst_pfns);
838 kfree(src_pfns);
839 return ret;
840 }
841
842 static inline bool
843 nouveau_dmem_page(struct nouveau_drm *drm, struct page *page)
844 {
845 if (!is_device_private_page(page))
846 return false;
847
848 if (drm->dmem->devmem != page->pgmap->data)
849 return false;
850
851 return true;
852 }
853
854 void
855 nouveau_dmem_convert_pfn(struct nouveau_drm *drm,
856 struct hmm_range *range)
857 {
858 unsigned long i, npages;
859
860 npages = (range->end - range->start) >> PAGE_SHIFT;
861 for (i = 0; i < npages; ++i) {
862 struct nouveau_dmem_chunk *chunk;
863 struct page *page;
864 uint64_t addr;
865
866 page = hmm_pfn_to_page(range, range->pfns[i]);
867 if (page == NULL)
868 continue;
869
870 if (!(range->pfns[i] & range->flags[HMM_PFN_DEVICE_PRIVATE])) {
871 continue;
872 }
873
874 if (!nouveau_dmem_page(drm, page)) {
875 WARN(1, "Some unknown device memory !\n");
876 range->pfns[i] = 0;
877 continue;
878 }
879
880 chunk = (void *)hmm_devmem_page_get_drvdata(page);
881 addr = page_to_pfn(page) - chunk->pfn_first;
882 addr = (addr + chunk->bo->bo.mem.start) << PAGE_SHIFT;
883
884 range->pfns[i] &= ((1UL << range->pfn_shift) - 1);
885 range->pfns[i] |= (addr >> PAGE_SHIFT) << range->pfn_shift;
886 }
887 }
Linux with added FPC-III support