?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
dt-bindings: mtd: ingenic: Use standard ecc-engine property
[linux/fpc-iii.git] / drivers / gpu / drm / amd / amdgpu / amdgpu_ttm.c
blob73e71e61dc996aca4fd3d0c387077721efeddf3d
1 /*
2 * Copyright 2009 Jerome Glisse.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sub license, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19 * USE OR OTHER DEALINGS IN THE SOFTWARE.
20 *
21 * The above copyright notice and this permission notice (including the
22 * next paragraph) shall be included in all copies or substantial portions
23 * of the Software.
24 *
25 */
26 /*
27 * Authors:
28 * Jerome Glisse <glisse@freedesktop.org>
29 * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
30 * Dave Airlie
31 */
32 #include <drm/ttm/ttm_bo_api.h>
33 #include <drm/ttm/ttm_bo_driver.h>
34 #include <drm/ttm/ttm_placement.h>
35 #include <drm/ttm/ttm_module.h>
36 #include <drm/ttm/ttm_page_alloc.h>
37 #include <drm/drmP.h>
38 #include <drm/amdgpu_drm.h>
39 #include <linux/seq_file.h>
40 #include <linux/slab.h>
41 #include <linux/swiotlb.h>
42 #include <linux/swap.h>
43 #include <linux/pagemap.h>
44 #include <linux/debugfs.h>
45 #include <linux/iommu.h>
46 #include "amdgpu.h"
47 #include "amdgpu_object.h"
48 #include "amdgpu_trace.h"
49 #include "amdgpu_amdkfd.h"
50 #include "amdgpu_sdma.h"
51 #include "bif/bif_4_1_d.h"
52
53 #define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT)
54
55 static int amdgpu_map_buffer(struct ttm_buffer_object *bo,
56 struct ttm_mem_reg *mem, unsigned num_pages,
57 uint64_t offset, unsigned window,
58 struct amdgpu_ring *ring,
59 uint64_t *addr);
60
61 static int amdgpu_ttm_debugfs_init(struct amdgpu_device *adev);
62 static void amdgpu_ttm_debugfs_fini(struct amdgpu_device *adev);
63
64 static int amdgpu_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags)
65 {
66 return 0;
67 }
68
69 /**
70 * amdgpu_init_mem_type - Initialize a memory manager for a specific type of
71 * memory request.
72 *
73 * @bdev: The TTM BO device object (contains a reference to amdgpu_device)
74 * @type: The type of memory requested
75 * @man: The memory type manager for each domain
76 *
77 * This is called by ttm_bo_init_mm() when a buffer object is being
78 * initialized.
79 */
80 static int amdgpu_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
81 struct ttm_mem_type_manager *man)
82 {
83 struct amdgpu_device *adev;
84
85 adev = amdgpu_ttm_adev(bdev);
86
87 switch (type) {
88 case TTM_PL_SYSTEM:
89 /* System memory */
90 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
91 man->available_caching = TTM_PL_MASK_CACHING;
92 man->default_caching = TTM_PL_FLAG_CACHED;
93 break;
94 case TTM_PL_TT:
95 /* GTT memory */
96 man->func = &amdgpu_gtt_mgr_func;
97 man->gpu_offset = adev->gmc.gart_start;
98 man->available_caching = TTM_PL_MASK_CACHING;
99 man->default_caching = TTM_PL_FLAG_CACHED;
100 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE | TTM_MEMTYPE_FLAG_CMA;
101 break;
102 case TTM_PL_VRAM:
103 /* "On-card" video ram */
104 man->func = &amdgpu_vram_mgr_func;
105 man->gpu_offset = adev->gmc.vram_start;
106 man->flags = TTM_MEMTYPE_FLAG_FIXED |
107 TTM_MEMTYPE_FLAG_MAPPABLE;
108 man->available_caching = TTM_PL_FLAG_UNCACHED | TTM_PL_FLAG_WC;
109 man->default_caching = TTM_PL_FLAG_WC;
110 break;
111 case AMDGPU_PL_GDS:
112 case AMDGPU_PL_GWS:
113 case AMDGPU_PL_OA:
114 /* On-chip GDS memory*/
115 man->func = &ttm_bo_manager_func;
116 man->gpu_offset = 0;
117 man->flags = TTM_MEMTYPE_FLAG_FIXED | TTM_MEMTYPE_FLAG_CMA;
118 man->available_caching = TTM_PL_FLAG_UNCACHED;
119 man->default_caching = TTM_PL_FLAG_UNCACHED;
120 break;
121 default:
122 DRM_ERROR("Unsupported memory type %u\n", (unsigned)type);
123 return -EINVAL;
124 }
125 return 0;
126 }
127
128 /**
129 * amdgpu_evict_flags - Compute placement flags
130 *
131 * @bo: The buffer object to evict
132 * @placement: Possible destination(s) for evicted BO
133 *
134 * Fill in placement data when ttm_bo_evict() is called
135 */
136 static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
137 struct ttm_placement *placement)
138 {
139 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
140 struct amdgpu_bo *abo;
141 static const struct ttm_place placements = {
142 .fpfn = 0,
143 .lpfn = 0,
144 .flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM
145 };
146
147 /* Don't handle scatter gather BOs */
148 if (bo->type == ttm_bo_type_sg) {
149 placement->num_placement = 0;
150 placement->num_busy_placement = 0;
151 return;
152 }
153
154 /* Object isn't an AMDGPU object so ignore */
155 if (!amdgpu_bo_is_amdgpu_bo(bo)) {
156 placement->placement = &placements;
157 placement->busy_placement = &placements;
158 placement->num_placement = 1;
159 placement->num_busy_placement = 1;
160 return;
161 }
162
163 abo = ttm_to_amdgpu_bo(bo);
164 switch (bo->mem.mem_type) {
165 case AMDGPU_PL_GDS:
166 case AMDGPU_PL_GWS:
167 case AMDGPU_PL_OA:
168 placement->num_placement = 0;
169 placement->num_busy_placement = 0;
170 return;
171
172 case TTM_PL_VRAM:
173 if (!adev->mman.buffer_funcs_enabled) {
174 /* Move to system memory */
175 amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU);
176 } else if (!amdgpu_gmc_vram_full_visible(&adev->gmc) &&
177 !(abo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) &&
178 amdgpu_bo_in_cpu_visible_vram(abo)) {
179
180 /* Try evicting to the CPU inaccessible part of VRAM
181 * first, but only set GTT as busy placement, so this
182 * BO will be evicted to GTT rather than causing other
183 * BOs to be evicted from VRAM
184 */
185 amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM |
186 AMDGPU_GEM_DOMAIN_GTT);
187 abo->placements[0].fpfn = adev->gmc.visible_vram_size >> PAGE_SHIFT;
188 abo->placements[0].lpfn = 0;
189 abo->placement.busy_placement = &abo->placements[1];
190 abo->placement.num_busy_placement = 1;
191 } else {
192 /* Move to GTT memory */
193 amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_GTT);
194 }
195 break;
196 case TTM_PL_TT:
197 default:
198 amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU);
199 break;
200 }
201 *placement = abo->placement;
202 }
203
204 /**
205 * amdgpu_verify_access - Verify access for a mmap call
206 *
207 * @bo: The buffer object to map
208 * @filp: The file pointer from the process performing the mmap
209 *
210 * This is called by ttm_bo_mmap() to verify whether a process
211 * has the right to mmap a BO to their process space.
212 */
213 static int amdgpu_verify_access(struct ttm_buffer_object *bo, struct file *filp)
214 {
215 struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
216
217 /*
218 * Don't verify access for KFD BOs. They don't have a GEM
219 * object associated with them.
220 */
221 if (abo->kfd_bo)
222 return 0;
223
224 if (amdgpu_ttm_tt_get_usermm(bo->ttm))
225 return -EPERM;
226 return drm_vma_node_verify_access(&abo->gem_base.vma_node,
227 filp->private_data);
228 }
229
230 /**
231 * amdgpu_move_null - Register memory for a buffer object
232 *
233 * @bo: The bo to assign the memory to
234 * @new_mem: The memory to be assigned.
235 *
236 * Assign the memory from new_mem to the memory of the buffer object bo.
237 */
238 static void amdgpu_move_null(struct ttm_buffer_object *bo,
239 struct ttm_mem_reg *new_mem)
240 {
241 struct ttm_mem_reg *old_mem = &bo->mem;
242
243 BUG_ON(old_mem->mm_node != NULL);
244 *old_mem = *new_mem;
245 new_mem->mm_node = NULL;
246 }
247
248 /**
249 * amdgpu_mm_node_addr - Compute the GPU relative offset of a GTT buffer.
250 *
251 * @bo: The bo to assign the memory to.
252 * @mm_node: Memory manager node for drm allocator.
253 * @mem: The region where the bo resides.
254 *
255 */
256 static uint64_t amdgpu_mm_node_addr(struct ttm_buffer_object *bo,
257 struct drm_mm_node *mm_node,
258 struct ttm_mem_reg *mem)
259 {
260 uint64_t addr = 0;
261
262 if (mm_node->start != AMDGPU_BO_INVALID_OFFSET) {
263 addr = mm_node->start << PAGE_SHIFT;
264 addr += bo->bdev->man[mem->mem_type].gpu_offset;
265 }
266 return addr;
267 }
268
269 /**
270 * amdgpu_find_mm_node - Helper function finds the drm_mm_node corresponding to
271 * @offset. It also modifies the offset to be within the drm_mm_node returned
272 *
273 * @mem: The region where the bo resides.
274 * @offset: The offset that drm_mm_node is used for finding.
275 *
276 */
277 static struct drm_mm_node *amdgpu_find_mm_node(struct ttm_mem_reg *mem,
278 unsigned long *offset)
279 {
280 struct drm_mm_node *mm_node = mem->mm_node;
281
282 while (*offset >= (mm_node->size << PAGE_SHIFT)) {
283 *offset -= (mm_node->size << PAGE_SHIFT);
284 ++mm_node;
285 }
286 return mm_node;
287 }
288
289 /**
290 * amdgpu_copy_ttm_mem_to_mem - Helper function for copy
291 *
292 * The function copies @size bytes from {src->mem + src->offset} to
293 * {dst->mem + dst->offset}. src->bo and dst->bo could be same BO for a
294 * move and different for a BO to BO copy.
295 *
296 * @f: Returns the last fence if multiple jobs are submitted.
297 */
298 int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev,
299 struct amdgpu_copy_mem *src,
300 struct amdgpu_copy_mem *dst,
301 uint64_t size,
302 struct reservation_object *resv,
303 struct dma_fence **f)
304 {
305 struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
306 struct drm_mm_node *src_mm, *dst_mm;
307 uint64_t src_node_start, dst_node_start, src_node_size,
308 dst_node_size, src_page_offset, dst_page_offset;
309 struct dma_fence *fence = NULL;
310 int r = 0;
311 const uint64_t GTT_MAX_BYTES = (AMDGPU_GTT_MAX_TRANSFER_SIZE *
312 AMDGPU_GPU_PAGE_SIZE);
313
314 if (!adev->mman.buffer_funcs_enabled) {
315 DRM_ERROR("Trying to move memory with ring turned off.\n");
316 return -EINVAL;
317 }
318
319 src_mm = amdgpu_find_mm_node(src->mem, &src->offset);
320 src_node_start = amdgpu_mm_node_addr(src->bo, src_mm, src->mem) +
321 src->offset;
322 src_node_size = (src_mm->size << PAGE_SHIFT) - src->offset;
323 src_page_offset = src_node_start & (PAGE_SIZE - 1);
324
325 dst_mm = amdgpu_find_mm_node(dst->mem, &dst->offset);
326 dst_node_start = amdgpu_mm_node_addr(dst->bo, dst_mm, dst->mem) +
327 dst->offset;
328 dst_node_size = (dst_mm->size << PAGE_SHIFT) - dst->offset;
329 dst_page_offset = dst_node_start & (PAGE_SIZE - 1);
330
331 mutex_lock(&adev->mman.gtt_window_lock);
332
333 while (size) {
334 unsigned long cur_size;
335 uint64_t from = src_node_start, to = dst_node_start;
336 struct dma_fence *next;
337
338 /* Copy size cannot exceed GTT_MAX_BYTES. So if src or dst
339 * begins at an offset, then adjust the size accordingly
340 */
341 cur_size = min3(min(src_node_size, dst_node_size), size,
342 GTT_MAX_BYTES);
343 if (cur_size + src_page_offset > GTT_MAX_BYTES ||
344 cur_size + dst_page_offset > GTT_MAX_BYTES)
345 cur_size -= max(src_page_offset, dst_page_offset);
346
347 /* Map only what needs to be accessed. Map src to window 0 and
348 * dst to window 1
349 */
350 if (src->mem->start == AMDGPU_BO_INVALID_OFFSET) {
351 r = amdgpu_map_buffer(src->bo, src->mem,
352 PFN_UP(cur_size + src_page_offset),
353 src_node_start, 0, ring,
354 &from);
355 if (r)
356 goto error;
357 /* Adjust the offset because amdgpu_map_buffer returns
358 * start of mapped page
359 */
360 from += src_page_offset;
361 }
362
363 if (dst->mem->start == AMDGPU_BO_INVALID_OFFSET) {
364 r = amdgpu_map_buffer(dst->bo, dst->mem,
365 PFN_UP(cur_size + dst_page_offset),
366 dst_node_start, 1, ring,
367 &to);
368 if (r)
369 goto error;
370 to += dst_page_offset;
371 }
372
373 r = amdgpu_copy_buffer(ring, from, to, cur_size,
374 resv, &next, false, true);
375 if (r)
376 goto error;
377
378 dma_fence_put(fence);
379 fence = next;
380
381 size -= cur_size;
382 if (!size)
383 break;
384
385 src_node_size -= cur_size;
386 if (!src_node_size) {
387 src_node_start = amdgpu_mm_node_addr(src->bo, ++src_mm,
388 src->mem);
389 src_node_size = (src_mm->size << PAGE_SHIFT);
390 } else {
391 src_node_start += cur_size;
392 src_page_offset = src_node_start & (PAGE_SIZE - 1);
393 }
394 dst_node_size -= cur_size;
395 if (!dst_node_size) {
396 dst_node_start = amdgpu_mm_node_addr(dst->bo, ++dst_mm,
397 dst->mem);
398 dst_node_size = (dst_mm->size << PAGE_SHIFT);
399 } else {
400 dst_node_start += cur_size;
401 dst_page_offset = dst_node_start & (PAGE_SIZE - 1);
402 }
403 }
404 error:
405 mutex_unlock(&adev->mman.gtt_window_lock);
406 if (f)
407 *f = dma_fence_get(fence);
408 dma_fence_put(fence);
409 return r;
410 }
411
412 /**
413 * amdgpu_move_blit - Copy an entire buffer to another buffer
414 *
415 * This is a helper called by amdgpu_bo_move() and amdgpu_move_vram_ram() to
416 * help move buffers to and from VRAM.
417 */
418 static int amdgpu_move_blit(struct ttm_buffer_object *bo,
419 bool evict, bool no_wait_gpu,
420 struct ttm_mem_reg *new_mem,
421 struct ttm_mem_reg *old_mem)
422 {
423 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
424 struct amdgpu_copy_mem src, dst;
425 struct dma_fence *fence = NULL;
426 int r;
427
428 src.bo = bo;
429 dst.bo = bo;
430 src.mem = old_mem;
431 dst.mem = new_mem;
432 src.offset = 0;
433 dst.offset = 0;
434
435 r = amdgpu_ttm_copy_mem_to_mem(adev, &src, &dst,
436 new_mem->num_pages << PAGE_SHIFT,
437 bo->resv, &fence);
438 if (r)
439 goto error;
440
441 /* Always block for VM page tables before committing the new location */
442 if (bo->type == ttm_bo_type_kernel)
443 r = ttm_bo_move_accel_cleanup(bo, fence, true, new_mem);
444 else
445 r = ttm_bo_pipeline_move(bo, fence, evict, new_mem);
446 dma_fence_put(fence);
447 return r;
448
449 error:
450 if (fence)
451 dma_fence_wait(fence, false);
452 dma_fence_put(fence);
453 return r;
454 }
455
456 /**
457 * amdgpu_move_vram_ram - Copy VRAM buffer to RAM buffer
458 *
459 * Called by amdgpu_bo_move().
460 */
461 static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo, bool evict,
462 struct ttm_operation_ctx *ctx,
463 struct ttm_mem_reg *new_mem)
464 {
465 struct amdgpu_device *adev;
466 struct ttm_mem_reg *old_mem = &bo->mem;
467 struct ttm_mem_reg tmp_mem;
468 struct ttm_place placements;
469 struct ttm_placement placement;
470 int r;
471
472 adev = amdgpu_ttm_adev(bo->bdev);
473
474 /* create space/pages for new_mem in GTT space */
475 tmp_mem = *new_mem;
476 tmp_mem.mm_node = NULL;
477 placement.num_placement = 1;
478 placement.placement = &placements;
479 placement.num_busy_placement = 1;
480 placement.busy_placement = &placements;
481 placements.fpfn = 0;
482 placements.lpfn = 0;
483 placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
484 r = ttm_bo_mem_space(bo, &placement, &tmp_mem, ctx);
485 if (unlikely(r)) {
486 return r;
487 }
488
489 /* set caching flags */
490 r = ttm_tt_set_placement_caching(bo->ttm, tmp_mem.placement);
491 if (unlikely(r)) {
492 goto out_cleanup;
493 }
494
495 /* Bind the memory to the GTT space */
496 r = ttm_tt_bind(bo->ttm, &tmp_mem, ctx);
497 if (unlikely(r)) {
498 goto out_cleanup;
499 }
500
501 /* blit VRAM to GTT */
502 r = amdgpu_move_blit(bo, evict, ctx->no_wait_gpu, &tmp_mem, old_mem);
503 if (unlikely(r)) {
504 goto out_cleanup;
505 }
506
507 /* move BO (in tmp_mem) to new_mem */
508 r = ttm_bo_move_ttm(bo, ctx, new_mem);
509 out_cleanup:
510 ttm_bo_mem_put(bo, &tmp_mem);
511 return r;
512 }
513
514 /**
515 * amdgpu_move_ram_vram - Copy buffer from RAM to VRAM
516 *
517 * Called by amdgpu_bo_move().
518 */
519 static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo, bool evict,
520 struct ttm_operation_ctx *ctx,
521 struct ttm_mem_reg *new_mem)
522 {
523 struct amdgpu_device *adev;
524 struct ttm_mem_reg *old_mem = &bo->mem;
525 struct ttm_mem_reg tmp_mem;
526 struct ttm_placement placement;
527 struct ttm_place placements;
528 int r;
529
530 adev = amdgpu_ttm_adev(bo->bdev);
531
532 /* make space in GTT for old_mem buffer */
533 tmp_mem = *new_mem;
534 tmp_mem.mm_node = NULL;
535 placement.num_placement = 1;
536 placement.placement = &placements;
537 placement.num_busy_placement = 1;
538 placement.busy_placement = &placements;
539 placements.fpfn = 0;
540 placements.lpfn = 0;
541 placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
542 r = ttm_bo_mem_space(bo, &placement, &tmp_mem, ctx);
543 if (unlikely(r)) {
544 return r;
545 }
546
547 /* move/bind old memory to GTT space */
548 r = ttm_bo_move_ttm(bo, ctx, &tmp_mem);
549 if (unlikely(r)) {
550 goto out_cleanup;
551 }
552
553 /* copy to VRAM */
554 r = amdgpu_move_blit(bo, evict, ctx->no_wait_gpu, new_mem, old_mem);
555 if (unlikely(r)) {
556 goto out_cleanup;
557 }
558 out_cleanup:
559 ttm_bo_mem_put(bo, &tmp_mem);
560 return r;
561 }
562
563 /**
564 * amdgpu_bo_move - Move a buffer object to a new memory location
565 *
566 * Called by ttm_bo_handle_move_mem()
567 */
568 static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict,
569 struct ttm_operation_ctx *ctx,
570 struct ttm_mem_reg *new_mem)
571 {
572 struct amdgpu_device *adev;
573 struct amdgpu_bo *abo;
574 struct ttm_mem_reg *old_mem = &bo->mem;
575 int r;
576
577 /* Can't move a pinned BO */
578 abo = ttm_to_amdgpu_bo(bo);
579 if (WARN_ON_ONCE(abo->pin_count > 0))
580 return -EINVAL;
581
582 adev = amdgpu_ttm_adev(bo->bdev);
583
584 if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) {
585 amdgpu_move_null(bo, new_mem);
586 return 0;
587 }
588 if ((old_mem->mem_type == TTM_PL_TT &&
589 new_mem->mem_type == TTM_PL_SYSTEM) ||
590 (old_mem->mem_type == TTM_PL_SYSTEM &&
591 new_mem->mem_type == TTM_PL_TT)) {
592 /* bind is enough */
593 amdgpu_move_null(bo, new_mem);
594 return 0;
595 }
596 if (old_mem->mem_type == AMDGPU_PL_GDS ||
597 old_mem->mem_type == AMDGPU_PL_GWS ||
598 old_mem->mem_type == AMDGPU_PL_OA ||
599 new_mem->mem_type == AMDGPU_PL_GDS ||
600 new_mem->mem_type == AMDGPU_PL_GWS ||
601 new_mem->mem_type == AMDGPU_PL_OA) {
602 /* Nothing to save here */
603 amdgpu_move_null(bo, new_mem);
604 return 0;
605 }
606
607 if (!adev->mman.buffer_funcs_enabled)
608 goto memcpy;
609
610 if (old_mem->mem_type == TTM_PL_VRAM &&
611 new_mem->mem_type == TTM_PL_SYSTEM) {
612 r = amdgpu_move_vram_ram(bo, evict, ctx, new_mem);
613 } else if (old_mem->mem_type == TTM_PL_SYSTEM &&
614 new_mem->mem_type == TTM_PL_VRAM) {
615 r = amdgpu_move_ram_vram(bo, evict, ctx, new_mem);
616 } else {
617 r = amdgpu_move_blit(bo, evict, ctx->no_wait_gpu,
618 new_mem, old_mem);
619 }
620
621 if (r) {
622 memcpy:
623 r = ttm_bo_move_memcpy(bo, ctx, new_mem);
624 if (r) {
625 return r;
626 }
627 }
628
629 if (bo->type == ttm_bo_type_device &&
630 new_mem->mem_type == TTM_PL_VRAM &&
631 old_mem->mem_type != TTM_PL_VRAM) {
632 /* amdgpu_bo_fault_reserve_notify will re-set this if the CPU
633 * accesses the BO after it's moved.
634 */
635 abo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
636 }
637
638 /* update statistics */
639 atomic64_add((u64)bo->num_pages << PAGE_SHIFT, &adev->num_bytes_moved);
640 return 0;
641 }
642
643 /**
644 * amdgpu_ttm_io_mem_reserve - Reserve a block of memory during a fault
645 *
646 * Called by ttm_mem_io_reserve() ultimately via ttm_bo_vm_fault()
647 */
648 static int amdgpu_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
649 {
650 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
651 struct amdgpu_device *adev = amdgpu_ttm_adev(bdev);
652 struct drm_mm_node *mm_node = mem->mm_node;
653
654 mem->bus.addr = NULL;
655 mem->bus.offset = 0;
656 mem->bus.size = mem->num_pages << PAGE_SHIFT;
657 mem->bus.base = 0;
658 mem->bus.is_iomem = false;
659 if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE))
660 return -EINVAL;
661 switch (mem->mem_type) {
662 case TTM_PL_SYSTEM:
663 /* system memory */
664 return 0;
665 case TTM_PL_TT:
666 break;
667 case TTM_PL_VRAM:
668 mem->bus.offset = mem->start << PAGE_SHIFT;
669 /* check if it's visible */
670 if ((mem->bus.offset + mem->bus.size) > adev->gmc.visible_vram_size)
671 return -EINVAL;
672 /* Only physically contiguous buffers apply. In a contiguous
673 * buffer, size of the first mm_node would match the number of
674 * pages in ttm_mem_reg.
675 */
676 if (adev->mman.aper_base_kaddr &&
677 (mm_node->size == mem->num_pages))
678 mem->bus.addr = (u8 *)adev->mman.aper_base_kaddr +
679 mem->bus.offset;
680
681 mem->bus.base = adev->gmc.aper_base;
682 mem->bus.is_iomem = true;
683 break;
684 default:
685 return -EINVAL;
686 }
687 return 0;
688 }
689
690 static void amdgpu_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
691 {
692 }
693
694 static unsigned long amdgpu_ttm_io_mem_pfn(struct ttm_buffer_object *bo,
695 unsigned long page_offset)
696 {
697 struct drm_mm_node *mm;
698 unsigned long offset = (page_offset << PAGE_SHIFT);
699
700 mm = amdgpu_find_mm_node(&bo->mem, &offset);
701 return (bo->mem.bus.base >> PAGE_SHIFT) + mm->start +
702 (offset >> PAGE_SHIFT);
703 }
704
705 /*
706 * TTM backend functions.
707 */
708 struct amdgpu_ttm_gup_task_list {
709 struct list_head list;
710 struct task_struct *task;
711 };
712
713 struct amdgpu_ttm_tt {
714 struct ttm_dma_tt ttm;
715 u64 offset;
716 uint64_t userptr;
717 struct task_struct *usertask;
718 uint32_t userflags;
719 spinlock_t guptasklock;
720 struct list_head guptasks;
721 atomic_t mmu_invalidations;
722 uint32_t last_set_pages;
723 };
724
725 /**
726 * amdgpu_ttm_tt_get_user_pages - Pin pages of memory pointed to by a USERPTR
727 * pointer to memory
728 *
729 * Called by amdgpu_gem_userptr_ioctl() and amdgpu_cs_parser_bos().
730 * This provides a wrapper around the get_user_pages() call to provide
731 * device accessible pages that back user memory.
732 */
733 int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages)
734 {
735 struct amdgpu_ttm_tt *gtt = (void *)ttm;
736 struct mm_struct *mm = gtt->usertask->mm;
737 unsigned int flags = 0;
738 unsigned pinned = 0;
739 int r;
740
741 if (!mm) /* Happens during process shutdown */
742 return -ESRCH;
743
744 if (!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY))
745 flags |= FOLL_WRITE;
746
747 down_read(&mm->mmap_sem);
748
749 if (gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) {
750 /*
751 * check that we only use anonymous memory to prevent problems
752 * with writeback
753 */
754 unsigned long end = gtt->userptr + ttm->num_pages * PAGE_SIZE;
755 struct vm_area_struct *vma;
756
757 vma = find_vma(mm, gtt->userptr);
758 if (!vma || vma->vm_file || vma->vm_end < end) {
759 up_read(&mm->mmap_sem);
760 return -EPERM;
761 }
762 }
763
764 /* loop enough times using contiguous pages of memory */
765 do {
766 unsigned num_pages = ttm->num_pages - pinned;
767 uint64_t userptr = gtt->userptr + pinned * PAGE_SIZE;
768 struct page **p = pages + pinned;
769 struct amdgpu_ttm_gup_task_list guptask;
770
771 guptask.task = current;
772 spin_lock(&gtt->guptasklock);
773 list_add(&guptask.list, &gtt->guptasks);
774 spin_unlock(&gtt->guptasklock);
775
776 if (mm == current->mm)
777 r = get_user_pages(userptr, num_pages, flags, p, NULL);
778 else
779 r = get_user_pages_remote(gtt->usertask,
780 mm, userptr, num_pages,
781 flags, p, NULL, NULL);
782
783 spin_lock(&gtt->guptasklock);
784 list_del(&guptask.list);
785 spin_unlock(&gtt->guptasklock);
786
787 if (r < 0)
788 goto release_pages;
789
790 pinned += r;
791
792 } while (pinned < ttm->num_pages);
793
794 up_read(&mm->mmap_sem);
795 return 0;
796
797 release_pages:
798 release_pages(pages, pinned);
799 up_read(&mm->mmap_sem);
800 return r;
801 }
802
803 /**
804 * amdgpu_ttm_tt_set_user_pages - Copy pages in, putting old pages as necessary.
805 *
806 * Called by amdgpu_cs_list_validate(). This creates the page list
807 * that backs user memory and will ultimately be mapped into the device
808 * address space.
809 */
810 void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct page **pages)
811 {
812 struct amdgpu_ttm_tt *gtt = (void *)ttm;
813 unsigned i;
814
815 gtt->last_set_pages = atomic_read(&gtt->mmu_invalidations);
816 for (i = 0; i < ttm->num_pages; ++i) {
817 if (ttm->pages[i])
818 put_page(ttm->pages[i]);
819
820 ttm->pages[i] = pages ? pages[i] : NULL;
821 }
822 }
823
824 /**
825 * amdgpu_ttm_tt_mark_user_page - Mark pages as dirty
826 *
827 * Called while unpinning userptr pages
828 */
829 void amdgpu_ttm_tt_mark_user_pages(struct ttm_tt *ttm)
830 {
831 struct amdgpu_ttm_tt *gtt = (void *)ttm;
832 unsigned i;
833
834 for (i = 0; i < ttm->num_pages; ++i) {
835 struct page *page = ttm->pages[i];
836
837 if (!page)
838 continue;
839
840 if (!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY))
841 set_page_dirty(page);
842
843 mark_page_accessed(page);
844 }
845 }
846
847 /**
848 * amdgpu_ttm_tt_pin_userptr - prepare the sg table with the user pages
849 *
850 * Called by amdgpu_ttm_backend_bind()
851 **/
852 static int amdgpu_ttm_tt_pin_userptr(struct ttm_tt *ttm)
853 {
854 struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
855 struct amdgpu_ttm_tt *gtt = (void *)ttm;
856 unsigned nents;
857 int r;
858
859 int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
860 enum dma_data_direction direction = write ?
861 DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
862
863 /* Allocate an SG array and squash pages into it */
864 r = sg_alloc_table_from_pages(ttm->sg, ttm->pages, ttm->num_pages, 0,
865 ttm->num_pages << PAGE_SHIFT,
866 GFP_KERNEL);
867 if (r)
868 goto release_sg;
869
870 /* Map SG to device */
871 r = -ENOMEM;
872 nents = dma_map_sg(adev->dev, ttm->sg->sgl, ttm->sg->nents, direction);
873 if (nents != ttm->sg->nents)
874 goto release_sg;
875
876 /* convert SG to linear array of pages and dma addresses */
877 drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,
878 gtt->ttm.dma_address, ttm->num_pages);
879
880 return 0;
881
882 release_sg:
883 kfree(ttm->sg);
884 return r;
885 }
886
887 /**
888 * amdgpu_ttm_tt_unpin_userptr - Unpin and unmap userptr pages
889 */
890 static void amdgpu_ttm_tt_unpin_userptr(struct ttm_tt *ttm)
891 {
892 struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
893 struct amdgpu_ttm_tt *gtt = (void *)ttm;
894
895 int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
896 enum dma_data_direction direction = write ?
897 DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
898
899 /* double check that we don't free the table twice */
900 if (!ttm->sg->sgl)
901 return;
902
903 /* unmap the pages mapped to the device */
904 dma_unmap_sg(adev->dev, ttm->sg->sgl, ttm->sg->nents, direction);
905
906 /* mark the pages as dirty */
907 amdgpu_ttm_tt_mark_user_pages(ttm);
908
909 sg_free_table(ttm->sg);
910 }
911
912 int amdgpu_ttm_gart_bind(struct amdgpu_device *adev,
913 struct ttm_buffer_object *tbo,
914 uint64_t flags)
915 {
916 struct amdgpu_bo *abo = ttm_to_amdgpu_bo(tbo);
917 struct ttm_tt *ttm = tbo->ttm;
918 struct amdgpu_ttm_tt *gtt = (void *)ttm;
919 int r;
920
921 if (abo->flags & AMDGPU_GEM_CREATE_MQD_GFX9) {
922 uint64_t page_idx = 1;
923
924 r = amdgpu_gart_bind(adev, gtt->offset, page_idx,
925 ttm->pages, gtt->ttm.dma_address, flags);
926 if (r)
927 goto gart_bind_fail;
928
929 /* Patch mtype of the second part BO */
930 flags &= ~AMDGPU_PTE_MTYPE_MASK;
931 flags |= AMDGPU_PTE_MTYPE(AMDGPU_MTYPE_NC);
932
933 r = amdgpu_gart_bind(adev,
934 gtt->offset + (page_idx << PAGE_SHIFT),
935 ttm->num_pages - page_idx,
936 &ttm->pages[page_idx],
937 &(gtt->ttm.dma_address[page_idx]), flags);
938 } else {
939 r = amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages,
940 ttm->pages, gtt->ttm.dma_address, flags);
941 }
942
943 gart_bind_fail:
944 if (r)
945 DRM_ERROR("failed to bind %lu pages at 0x%08llX\n",
946 ttm->num_pages, gtt->offset);
947
948 return r;
949 }
950
951 /**
952 * amdgpu_ttm_backend_bind - Bind GTT memory
953 *
954 * Called by ttm_tt_bind() on behalf of ttm_bo_handle_move_mem().
955 * This handles binding GTT memory to the device address space.
956 */
957 static int amdgpu_ttm_backend_bind(struct ttm_tt *ttm,
958 struct ttm_mem_reg *bo_mem)
959 {
960 struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
961 struct amdgpu_ttm_tt *gtt = (void*)ttm;
962 uint64_t flags;
963 int r = 0;
964
965 if (gtt->userptr) {
966 r = amdgpu_ttm_tt_pin_userptr(ttm);
967 if (r) {
968 DRM_ERROR("failed to pin userptr\n");
969 return r;
970 }
971 }
972 if (!ttm->num_pages) {
973 WARN(1, "nothing to bind %lu pages for mreg %p back %p!\n",
974 ttm->num_pages, bo_mem, ttm);
975 }
976
977 if (bo_mem->mem_type == AMDGPU_PL_GDS ||
978 bo_mem->mem_type == AMDGPU_PL_GWS ||
979 bo_mem->mem_type == AMDGPU_PL_OA)
980 return -EINVAL;
981
982 if (!amdgpu_gtt_mgr_has_gart_addr(bo_mem)) {
983 gtt->offset = AMDGPU_BO_INVALID_OFFSET;
984 return 0;
985 }
986
987 /* compute PTE flags relevant to this BO memory */
988 flags = amdgpu_ttm_tt_pte_flags(adev, ttm, bo_mem);
989
990 /* bind pages into GART page tables */
991 gtt->offset = (u64)bo_mem->start << PAGE_SHIFT;
992 r = amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages,
993 ttm->pages, gtt->ttm.dma_address, flags);
994
995 if (r)
996 DRM_ERROR("failed to bind %lu pages at 0x%08llX\n",
997 ttm->num_pages, gtt->offset);
998 return r;
999 }
1000
1001 /**
1002 * amdgpu_ttm_alloc_gart - Allocate GART memory for buffer object
1003 */
1004 int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo)
1005 {
1006 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
1007 struct ttm_operation_ctx ctx = { false, false };
1008 struct amdgpu_ttm_tt *gtt = (void*)bo->ttm;
1009 struct ttm_mem_reg tmp;
1010 struct ttm_placement placement;
1011 struct ttm_place placements;
1012 uint64_t addr, flags;
1013 int r;
1014
1015 if (bo->mem.start != AMDGPU_BO_INVALID_OFFSET)
1016 return 0;
1017
1018 addr = amdgpu_gmc_agp_addr(bo);
1019 if (addr != AMDGPU_BO_INVALID_OFFSET) {
1020 bo->mem.start = addr >> PAGE_SHIFT;
1021 } else {
1022
1023 /* allocate GART space */
1024 tmp = bo->mem;
1025 tmp.mm_node = NULL;
1026 placement.num_placement = 1;
1027 placement.placement = &placements;
1028 placement.num_busy_placement = 1;
1029 placement.busy_placement = &placements;
1030 placements.fpfn = 0;
1031 placements.lpfn = adev->gmc.gart_size >> PAGE_SHIFT;
1032 placements.flags = (bo->mem.placement & ~TTM_PL_MASK_MEM) |
1033 TTM_PL_FLAG_TT;
1034
1035 r = ttm_bo_mem_space(bo, &placement, &tmp, &ctx);
1036 if (unlikely(r))
1037 return r;
1038
1039 /* compute PTE flags for this buffer object */
1040 flags = amdgpu_ttm_tt_pte_flags(adev, bo->ttm, &tmp);
1041
1042 /* Bind pages */
1043 gtt->offset = (u64)tmp.start << PAGE_SHIFT;
1044 r = amdgpu_ttm_gart_bind(adev, bo, flags);
1045 if (unlikely(r)) {
1046 ttm_bo_mem_put(bo, &tmp);
1047 return r;
1048 }
1049
1050 ttm_bo_mem_put(bo, &bo->mem);
1051 bo->mem = tmp;
1052 }
1053
1054 bo->offset = (bo->mem.start << PAGE_SHIFT) +
1055 bo->bdev->man[bo->mem.mem_type].gpu_offset;
1056
1057 return 0;
1058 }
1059
1060 /**
1061 * amdgpu_ttm_recover_gart - Rebind GTT pages
1062 *
1063 * Called by amdgpu_gtt_mgr_recover() from amdgpu_device_reset() to
1064 * rebind GTT pages during a GPU reset.
1065 */
1066 int amdgpu_ttm_recover_gart(struct ttm_buffer_object *tbo)
1067 {
1068 struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev);
1069 uint64_t flags;
1070 int r;
1071
1072 if (!tbo->ttm)
1073 return 0;
1074
1075 flags = amdgpu_ttm_tt_pte_flags(adev, tbo->ttm, &tbo->mem);
1076 r = amdgpu_ttm_gart_bind(adev, tbo, flags);
1077
1078 return r;
1079 }
1080
1081 /**
1082 * amdgpu_ttm_backend_unbind - Unbind GTT mapped pages
1083 *
1084 * Called by ttm_tt_unbind() on behalf of ttm_bo_move_ttm() and
1085 * ttm_tt_destroy().
1086 */
1087 static int amdgpu_ttm_backend_unbind(struct ttm_tt *ttm)
1088 {
1089 struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
1090 struct amdgpu_ttm_tt *gtt = (void *)ttm;
1091 int r;
1092
1093 /* if the pages have userptr pinning then clear that first */
1094 if (gtt->userptr)
1095 amdgpu_ttm_tt_unpin_userptr(ttm);
1096
1097 if (gtt->offset == AMDGPU_BO_INVALID_OFFSET)
1098 return 0;
1099
1100 /* unbind shouldn't be done for GDS/GWS/OA in ttm_bo_clean_mm */
1101 r = amdgpu_gart_unbind(adev, gtt->offset, ttm->num_pages);
1102 if (r)
1103 DRM_ERROR("failed to unbind %lu pages at 0x%08llX\n",
1104 gtt->ttm.ttm.num_pages, gtt->offset);
1105 return r;
1106 }
1107
1108 static void amdgpu_ttm_backend_destroy(struct ttm_tt *ttm)
1109 {
1110 struct amdgpu_ttm_tt *gtt = (void *)ttm;
1111
1112 if (gtt->usertask)
1113 put_task_struct(gtt->usertask);
1114
1115 ttm_dma_tt_fini(&gtt->ttm);
1116 kfree(gtt);
1117 }
1118
1119 static struct ttm_backend_func amdgpu_backend_func = {
1120 .bind = &amdgpu_ttm_backend_bind,
1121 .unbind = &amdgpu_ttm_backend_unbind,
1122 .destroy = &amdgpu_ttm_backend_destroy,
1123 };
1124
1125 /**
1126 * amdgpu_ttm_tt_create - Create a ttm_tt object for a given BO
1127 *
1128 * @bo: The buffer object to create a GTT ttm_tt object around
1129 *
1130 * Called by ttm_tt_create().
1131 */
1132 static struct ttm_tt *amdgpu_ttm_tt_create(struct ttm_buffer_object *bo,
1133 uint32_t page_flags)
1134 {
1135 struct amdgpu_device *adev;
1136 struct amdgpu_ttm_tt *gtt;
1137
1138 adev = amdgpu_ttm_adev(bo->bdev);
1139
1140 gtt = kzalloc(sizeof(struct amdgpu_ttm_tt), GFP_KERNEL);
1141 if (gtt == NULL) {
1142 return NULL;
1143 }
1144 gtt->ttm.ttm.func = &amdgpu_backend_func;
1145
1146 /* allocate space for the uninitialized page entries */
1147 if (ttm_sg_tt_init(&gtt->ttm, bo, page_flags)) {
1148 kfree(gtt);
1149 return NULL;
1150 }
1151 return &gtt->ttm.ttm;
1152 }
1153
1154 /**
1155 * amdgpu_ttm_tt_populate - Map GTT pages visible to the device
1156 *
1157 * Map the pages of a ttm_tt object to an address space visible
1158 * to the underlying device.
1159 */
1160 static int amdgpu_ttm_tt_populate(struct ttm_tt *ttm,
1161 struct ttm_operation_ctx *ctx)
1162 {
1163 struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
1164 struct amdgpu_ttm_tt *gtt = (void *)ttm;
1165 bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
1166
1167 /* user pages are bound by amdgpu_ttm_tt_pin_userptr() */
1168 if (gtt && gtt->userptr) {
1169 ttm->sg = kzalloc(sizeof(struct sg_table), GFP_KERNEL);
1170 if (!ttm->sg)
1171 return -ENOMEM;
1172
1173 ttm->page_flags |= TTM_PAGE_FLAG_SG;
1174 ttm->state = tt_unbound;
1175 return 0;
1176 }
1177
1178 if (slave && ttm->sg) {
1179 drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,
1180 gtt->ttm.dma_address,
1181 ttm->num_pages);
1182 ttm->state = tt_unbound;
1183 return 0;
1184 }
1185
1186 #ifdef CONFIG_SWIOTLB
1187 if (adev->need_swiotlb && swiotlb_nr_tbl()) {
1188 return ttm_dma_populate(&gtt->ttm, adev->dev, ctx);
1189 }
1190 #endif
1191
1192 /* fall back to generic helper to populate the page array
1193 * and map them to the device */
1194 return ttm_populate_and_map_pages(adev->dev, &gtt->ttm, ctx);
1195 }
1196
1197 /**
1198 * amdgpu_ttm_tt_unpopulate - unmap GTT pages and unpopulate page arrays
1199 *
1200 * Unmaps pages of a ttm_tt object from the device address space and
1201 * unpopulates the page array backing it.
1202 */
1203 static void amdgpu_ttm_tt_unpopulate(struct ttm_tt *ttm)
1204 {
1205 struct amdgpu_device *adev;
1206 struct amdgpu_ttm_tt *gtt = (void *)ttm;
1207 bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
1208
1209 if (gtt && gtt->userptr) {
1210 amdgpu_ttm_tt_set_user_pages(ttm, NULL);
1211 kfree(ttm->sg);
1212 ttm->page_flags &= ~TTM_PAGE_FLAG_SG;
1213 return;
1214 }
1215
1216 if (slave)
1217 return;
1218
1219 adev = amdgpu_ttm_adev(ttm->bdev);
1220
1221 #ifdef CONFIG_SWIOTLB
1222 if (adev->need_swiotlb && swiotlb_nr_tbl()) {
1223 ttm_dma_unpopulate(&gtt->ttm, adev->dev);
1224 return;
1225 }
1226 #endif
1227
1228 /* fall back to generic helper to unmap and unpopulate array */
1229 ttm_unmap_and_unpopulate_pages(adev->dev, &gtt->ttm);
1230 }
1231
1232 /**
1233 * amdgpu_ttm_tt_set_userptr - Initialize userptr GTT ttm_tt for the current
1234 * task
1235 *
1236 * @ttm: The ttm_tt object to bind this userptr object to
1237 * @addr: The address in the current tasks VM space to use
1238 * @flags: Requirements of userptr object.
1239 *
1240 * Called by amdgpu_gem_userptr_ioctl() to bind userptr pages
1241 * to current task
1242 */
1243 int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr,
1244 uint32_t flags)
1245 {
1246 struct amdgpu_ttm_tt *gtt = (void *)ttm;
1247
1248 if (gtt == NULL)
1249 return -EINVAL;
1250
1251 gtt->userptr = addr;
1252 gtt->userflags = flags;
1253
1254 if (gtt->usertask)
1255 put_task_struct(gtt->usertask);
1256 gtt->usertask = current->group_leader;
1257 get_task_struct(gtt->usertask);
1258
1259 spin_lock_init(&gtt->guptasklock);
1260 INIT_LIST_HEAD(&gtt->guptasks);
1261 atomic_set(&gtt->mmu_invalidations, 0);
1262 gtt->last_set_pages = 0;
1263
1264 return 0;
1265 }
1266
1267 /**
1268 * amdgpu_ttm_tt_get_usermm - Return memory manager for ttm_tt object
1269 */
1270 struct mm_struct *amdgpu_ttm_tt_get_usermm(struct ttm_tt *ttm)
1271 {
1272 struct amdgpu_ttm_tt *gtt = (void *)ttm;
1273
1274 if (gtt == NULL)
1275 return NULL;
1276
1277 if (gtt->usertask == NULL)
1278 return NULL;
1279
1280 return gtt->usertask->mm;
1281 }
1282
1283 /**
1284 * amdgpu_ttm_tt_affect_userptr - Determine if a ttm_tt object lays inside an
1285 * address range for the current task.
1286 *
1287 */
1288 bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,
1289 unsigned long end)
1290 {
1291 struct amdgpu_ttm_tt *gtt = (void *)ttm;
1292 struct amdgpu_ttm_gup_task_list *entry;
1293 unsigned long size;
1294
1295 if (gtt == NULL || !gtt->userptr)
1296 return false;
1297
1298 /* Return false if no part of the ttm_tt object lies within
1299 * the range
1300 */
1301 size = (unsigned long)gtt->ttm.ttm.num_pages * PAGE_SIZE;
1302 if (gtt->userptr > end || gtt->userptr + size <= start)
1303 return false;
1304
1305 /* Search the lists of tasks that hold this mapping and see
1306 * if current is one of them. If it is return false.
1307 */
1308 spin_lock(&gtt->guptasklock);
1309 list_for_each_entry(entry, &gtt->guptasks, list) {
1310 if (entry->task == current) {
1311 spin_unlock(&gtt->guptasklock);
1312 return false;
1313 }
1314 }
1315 spin_unlock(&gtt->guptasklock);
1316
1317 atomic_inc(&gtt->mmu_invalidations);
1318
1319 return true;
1320 }
1321
1322 /**
1323 * amdgpu_ttm_tt_userptr_invalidated - Has the ttm_tt object been invalidated?
1324 */
1325 bool amdgpu_ttm_tt_userptr_invalidated(struct ttm_tt *ttm,
1326 int *last_invalidated)
1327 {
1328 struct amdgpu_ttm_tt *gtt = (void *)ttm;
1329 int prev_invalidated = *last_invalidated;
1330
1331 *last_invalidated = atomic_read(&gtt->mmu_invalidations);
1332 return prev_invalidated != *last_invalidated;
1333 }
1334
1335 /**
1336 * amdgpu_ttm_tt_userptr_needs_pages - Have the pages backing this ttm_tt object
1337 * been invalidated since the last time they've been set?
1338 */
1339 bool amdgpu_ttm_tt_userptr_needs_pages(struct ttm_tt *ttm)
1340 {
1341 struct amdgpu_ttm_tt *gtt = (void *)ttm;
1342
1343 if (gtt == NULL || !gtt->userptr)
1344 return false;
1345
1346 return atomic_read(&gtt->mmu_invalidations) != gtt->last_set_pages;
1347 }
1348
1349 /**
1350 * amdgpu_ttm_tt_is_readonly - Is the ttm_tt object read only?
1351 */
1352 bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm)
1353 {
1354 struct amdgpu_ttm_tt *gtt = (void *)ttm;
1355
1356 if (gtt == NULL)
1357 return false;
1358
1359 return !!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
1360 }
1361
1362 /**
1363 * amdgpu_ttm_tt_pde_flags - Compute PDE flags for ttm_tt object
1364 *
1365 * @ttm: The ttm_tt object to compute the flags for
1366 * @mem: The memory registry backing this ttm_tt object
1367 *
1368 * Figure out the flags to use for a VM PDE (Page Directory Entry).
1369 */
1370 uint64_t amdgpu_ttm_tt_pde_flags(struct ttm_tt *ttm, struct ttm_mem_reg *mem)
1371 {
1372 uint64_t flags = 0;
1373
1374 if (mem && mem->mem_type != TTM_PL_SYSTEM)
1375 flags |= AMDGPU_PTE_VALID;
1376
1377 if (mem && mem->mem_type == TTM_PL_TT) {
1378 flags |= AMDGPU_PTE_SYSTEM;
1379
1380 if (ttm->caching_state == tt_cached)
1381 flags |= AMDGPU_PTE_SNOOPED;
1382 }
1383
1384 return flags;
1385 }
1386
1387 /**
1388 * amdgpu_ttm_tt_pte_flags - Compute PTE flags for ttm_tt object
1389 *
1390 * @ttm: The ttm_tt object to compute the flags for
1391 * @mem: The memory registry backing this ttm_tt object
1392
1393 * Figure out the flags to use for a VM PTE (Page Table Entry).
1394 */
1395 uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,
1396 struct ttm_mem_reg *mem)
1397 {
1398 uint64_t flags = amdgpu_ttm_tt_pde_flags(ttm, mem);
1399
1400 flags |= adev->gart.gart_pte_flags;
1401 flags |= AMDGPU_PTE_READABLE;
1402
1403 if (!amdgpu_ttm_tt_is_readonly(ttm))
1404 flags |= AMDGPU_PTE_WRITEABLE;
1405
1406 return flags;
1407 }
1408
1409 /**
1410 * amdgpu_ttm_bo_eviction_valuable - Check to see if we can evict a buffer
1411 * object.
1412 *
1413 * Return true if eviction is sensible. Called by ttm_mem_evict_first() on
1414 * behalf of ttm_bo_mem_force_space() which tries to evict buffer objects until
1415 * it can find space for a new object and by ttm_bo_force_list_clean() which is
1416 * used to clean out a memory space.
1417 */
1418 static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
1419 const struct ttm_place *place)
1420 {
1421 unsigned long num_pages = bo->mem.num_pages;
1422 struct drm_mm_node *node = bo->mem.mm_node;
1423 struct reservation_object_list *flist;
1424 struct dma_fence *f;
1425 int i;
1426
1427 /* If bo is a KFD BO, check if the bo belongs to the current process.
1428 * If true, then return false as any KFD process needs all its BOs to
1429 * be resident to run successfully
1430 */
1431 flist = reservation_object_get_list(bo->resv);
1432 if (flist) {
1433 for (i = 0; i < flist->shared_count; ++i) {
1434 f = rcu_dereference_protected(flist->shared[i],
1435 reservation_object_held(bo->resv));
1436 if (amdkfd_fence_check_mm(f, current->mm))
1437 return false;
1438 }
1439 }
1440
1441 switch (bo->mem.mem_type) {
1442 case TTM_PL_TT:
1443 return true;
1444
1445 case TTM_PL_VRAM:
1446 /* Check each drm MM node individually */
1447 while (num_pages) {
1448 if (place->fpfn < (node->start + node->size) &&
1449 !(place->lpfn && place->lpfn <= node->start))
1450 return true;
1451
1452 num_pages -= node->size;
1453 ++node;
1454 }
1455 return false;
1456
1457 default:
1458 break;
1459 }
1460
1461 return ttm_bo_eviction_valuable(bo, place);
1462 }
1463
1464 /**
1465 * amdgpu_ttm_access_memory - Read or Write memory that backs a buffer object.
1466 *
1467 * @bo: The buffer object to read/write
1468 * @offset: Offset into buffer object
1469 * @buf: Secondary buffer to write/read from
1470 * @len: Length in bytes of access
1471 * @write: true if writing
1472 *
1473 * This is used to access VRAM that backs a buffer object via MMIO
1474 * access for debugging purposes.
1475 */
1476 static int amdgpu_ttm_access_memory(struct ttm_buffer_object *bo,
1477 unsigned long offset,
1478 void *buf, int len, int write)
1479 {
1480 struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
1481 struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
1482 struct drm_mm_node *nodes;
1483 uint32_t value = 0;
1484 int ret = 0;
1485 uint64_t pos;
1486 unsigned long flags;
1487
1488 if (bo->mem.mem_type != TTM_PL_VRAM)
1489 return -EIO;
1490
1491 nodes = amdgpu_find_mm_node(&abo->tbo.mem, &offset);
1492 pos = (nodes->start << PAGE_SHIFT) + offset;
1493
1494 while (len && pos < adev->gmc.mc_vram_size) {
1495 uint64_t aligned_pos = pos & ~(uint64_t)3;
1496 uint32_t bytes = 4 - (pos & 3);
1497 uint32_t shift = (pos & 3) * 8;
1498 uint32_t mask = 0xffffffff << shift;
1499
1500 if (len < bytes) {
1501 mask &= 0xffffffff >> (bytes - len) * 8;
1502 bytes = len;
1503 }
1504
1505 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
1506 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)aligned_pos) | 0x80000000);
1507 WREG32_NO_KIQ(mmMM_INDEX_HI, aligned_pos >> 31);
1508 if (!write || mask != 0xffffffff)
1509 value = RREG32_NO_KIQ(mmMM_DATA);
1510 if (write) {
1511 value &= ~mask;
1512 value |= (*(uint32_t *)buf << shift) & mask;
1513 WREG32_NO_KIQ(mmMM_DATA, value);
1514 }
1515 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
1516 if (!write) {
1517 value = (value & mask) >> shift;
1518 memcpy(buf, &value, bytes);
1519 }
1520
1521 ret += bytes;
1522 buf = (uint8_t *)buf + bytes;
1523 pos += bytes;
1524 len -= bytes;
1525 if (pos >= (nodes->start + nodes->size) << PAGE_SHIFT) {
1526 ++nodes;
1527 pos = (nodes->start << PAGE_SHIFT);
1528 }
1529 }
1530
1531 return ret;
1532 }
1533
1534 static struct ttm_bo_driver amdgpu_bo_driver = {
1535 .ttm_tt_create = &amdgpu_ttm_tt_create,
1536 .ttm_tt_populate = &amdgpu_ttm_tt_populate,
1537 .ttm_tt_unpopulate = &amdgpu_ttm_tt_unpopulate,
1538 .invalidate_caches = &amdgpu_invalidate_caches,
1539 .init_mem_type = &amdgpu_init_mem_type,
1540 .eviction_valuable = amdgpu_ttm_bo_eviction_valuable,
1541 .evict_flags = &amdgpu_evict_flags,
1542 .move = &amdgpu_bo_move,
1543 .verify_access = &amdgpu_verify_access,
1544 .move_notify = &amdgpu_bo_move_notify,
1545 .fault_reserve_notify = &amdgpu_bo_fault_reserve_notify,
1546 .io_mem_reserve = &amdgpu_ttm_io_mem_reserve,
1547 .io_mem_free = &amdgpu_ttm_io_mem_free,
1548 .io_mem_pfn = amdgpu_ttm_io_mem_pfn,
1549 .access_memory = &amdgpu_ttm_access_memory,
1550 .del_from_lru_notify = &amdgpu_vm_del_from_lru_notify
1551 };
1552
1553 /*
1554 * Firmware Reservation functions
1555 */
1556 /**
1557 * amdgpu_ttm_fw_reserve_vram_fini - free fw reserved vram
1558 *
1559 * @adev: amdgpu_device pointer
1560 *
1561 * free fw reserved vram if it has been reserved.
1562 */
1563 static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev)
1564 {
1565 amdgpu_bo_free_kernel(&adev->fw_vram_usage.reserved_bo,
1566 NULL, &adev->fw_vram_usage.va);
1567 }
1568
1569 /**
1570 * amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
1571 *
1572 * @adev: amdgpu_device pointer
1573 *
1574 * create bo vram reservation from fw.
1575 */
1576 static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
1577 {
1578 struct ttm_operation_ctx ctx = { false, false };
1579 struct amdgpu_bo_param bp;
1580 int r = 0;
1581 int i;
1582 u64 vram_size = adev->gmc.visible_vram_size;
1583 u64 offset = adev->fw_vram_usage.start_offset;
1584 u64 size = adev->fw_vram_usage.size;
1585 struct amdgpu_bo *bo;
1586
1587 memset(&bp, 0, sizeof(bp));
1588 bp.size = adev->fw_vram_usage.size;
1589 bp.byte_align = PAGE_SIZE;
1590 bp.domain = AMDGPU_GEM_DOMAIN_VRAM;
1591 bp.flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
1592 AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
1593 bp.type = ttm_bo_type_kernel;
1594 bp.resv = NULL;
1595 adev->fw_vram_usage.va = NULL;
1596 adev->fw_vram_usage.reserved_bo = NULL;
1597
1598 if (adev->fw_vram_usage.size > 0 &&
1599 adev->fw_vram_usage.size <= vram_size) {
1600
1601 r = amdgpu_bo_create(adev, &bp,
1602 &adev->fw_vram_usage.reserved_bo);
1603 if (r)
1604 goto error_create;
1605
1606 r = amdgpu_bo_reserve(adev->fw_vram_usage.reserved_bo, false);
1607 if (r)
1608 goto error_reserve;
1609
1610 /* remove the original mem node and create a new one at the
1611 * request position
1612 */
1613 bo = adev->fw_vram_usage.reserved_bo;
1614 offset = ALIGN(offset, PAGE_SIZE);
1615 for (i = 0; i < bo->placement.num_placement; ++i) {
1616 bo->placements[i].fpfn = offset >> PAGE_SHIFT;
1617 bo->placements[i].lpfn = (offset + size) >> PAGE_SHIFT;
1618 }
1619
1620 ttm_bo_mem_put(&bo->tbo, &bo->tbo.mem);
1621 r = ttm_bo_mem_space(&bo->tbo, &bo->placement,
1622 &bo->tbo.mem, &ctx);
1623 if (r)
1624 goto error_pin;
1625
1626 r = amdgpu_bo_pin_restricted(adev->fw_vram_usage.reserved_bo,
1627 AMDGPU_GEM_DOMAIN_VRAM,
1628 adev->fw_vram_usage.start_offset,
1629 (adev->fw_vram_usage.start_offset +
1630 adev->fw_vram_usage.size));
1631 if (r)
1632 goto error_pin;
1633 r = amdgpu_bo_kmap(adev->fw_vram_usage.reserved_bo,
1634 &adev->fw_vram_usage.va);
1635 if (r)
1636 goto error_kmap;
1637
1638 amdgpu_bo_unreserve(adev->fw_vram_usage.reserved_bo);
1639 }
1640 return r;
1641
1642 error_kmap:
1643 amdgpu_bo_unpin(adev->fw_vram_usage.reserved_bo);
1644 error_pin:
1645 amdgpu_bo_unreserve(adev->fw_vram_usage.reserved_bo);
1646 error_reserve:
1647 amdgpu_bo_unref(&adev->fw_vram_usage.reserved_bo);
1648 error_create:
1649 adev->fw_vram_usage.va = NULL;
1650 adev->fw_vram_usage.reserved_bo = NULL;
1651 return r;
1652 }
1653 /**
1654 * amdgpu_ttm_init - Init the memory management (ttm) as well as various
1655 * gtt/vram related fields.
1656 *
1657 * This initializes all of the memory space pools that the TTM layer
1658 * will need such as the GTT space (system memory mapped to the device),
1659 * VRAM (on-board memory), and on-chip memories (GDS, GWS, OA) which
1660 * can be mapped per VMID.
1661 */
1662 int amdgpu_ttm_init(struct amdgpu_device *adev)
1663 {
1664 uint64_t gtt_size;
1665 int r;
1666 u64 vis_vram_limit;
1667
1668 mutex_init(&adev->mman.gtt_window_lock);
1669
1670 /* No others user of address space so set it to 0 */
1671 r = ttm_bo_device_init(&adev->mman.bdev,
1672 &amdgpu_bo_driver,
1673 adev->ddev->anon_inode->i_mapping,
1674 DRM_FILE_PAGE_OFFSET,
1675 adev->need_dma32);
1676 if (r) {
1677 DRM_ERROR("failed initializing buffer object driver(%d).\n", r);
1678 return r;
1679 }
1680 adev->mman.initialized = true;
1681
1682 /* We opt to avoid OOM on system pages allocations */
1683 adev->mman.bdev.no_retry = true;
1684
1685 /* Initialize VRAM pool with all of VRAM divided into pages */
1686 r = ttm_bo_init_mm(&adev->mman.bdev, TTM_PL_VRAM,
1687 adev->gmc.real_vram_size >> PAGE_SHIFT);
1688 if (r) {
1689 DRM_ERROR("Failed initializing VRAM heap.\n");
1690 return r;
1691 }
1692
1693 /* Reduce size of CPU-visible VRAM if requested */
1694 vis_vram_limit = (u64)amdgpu_vis_vram_limit * 1024 * 1024;
1695 if (amdgpu_vis_vram_limit > 0 &&
1696 vis_vram_limit <= adev->gmc.visible_vram_size)
1697 adev->gmc.visible_vram_size = vis_vram_limit;
1698
1699 /* Change the size here instead of the init above so only lpfn is affected */
1700 amdgpu_ttm_set_buffer_funcs_status(adev, false);
1701 #ifdef CONFIG_64BIT
1702 adev->mman.aper_base_kaddr = ioremap_wc(adev->gmc.aper_base,
1703 adev->gmc.visible_vram_size);
1704 #endif
1705
1706 /*
1707 *The reserved vram for firmware must be pinned to the specified
1708 *place on the VRAM, so reserve it early.
1709 */
1710 r = amdgpu_ttm_fw_reserve_vram_init(adev);
1711 if (r) {
1712 return r;
1713 }
1714
1715 /* allocate memory as required for VGA
1716 * This is used for VGA emulation and pre-OS scanout buffers to
1717 * avoid display artifacts while transitioning between pre-OS
1718 * and driver. */
1719 r = amdgpu_bo_create_kernel(adev, adev->gmc.stolen_size, PAGE_SIZE,
1720 AMDGPU_GEM_DOMAIN_VRAM,
1721 &adev->stolen_vga_memory,
1722 NULL, NULL);
1723 if (r)
1724 return r;
1725 DRM_INFO("amdgpu: %uM of VRAM memory ready\n",
1726 (unsigned) (adev->gmc.real_vram_size / (1024 * 1024)));
1727
1728 /* Compute GTT size, either bsaed on 3/4th the size of RAM size
1729 * or whatever the user passed on module init */
1730 if (amdgpu_gtt_size == -1) {
1731 struct sysinfo si;
1732
1733 si_meminfo(&si);
1734 gtt_size = min(max((AMDGPU_DEFAULT_GTT_SIZE_MB << 20),
1735 adev->gmc.mc_vram_size),
1736 ((uint64_t)si.totalram * si.mem_unit * 3/4));
1737 }
1738 else
1739 gtt_size = (uint64_t)amdgpu_gtt_size << 20;
1740
1741 /* Initialize GTT memory pool */
1742 r = ttm_bo_init_mm(&adev->mman.bdev, TTM_PL_TT, gtt_size >> PAGE_SHIFT);
1743 if (r) {
1744 DRM_ERROR("Failed initializing GTT heap.\n");
1745 return r;
1746 }
1747 DRM_INFO("amdgpu: %uM of GTT memory ready.\n",
1748 (unsigned)(gtt_size / (1024 * 1024)));
1749
1750 /* Initialize various on-chip memory pools */
1751 r = ttm_bo_init_mm(&adev->mman.bdev, AMDGPU_PL_GDS,
1752 adev->gds.mem.total_size);
1753 if (r) {
1754 DRM_ERROR("Failed initializing GDS heap.\n");
1755 return r;
1756 }
1757
1758 r = amdgpu_bo_create_kernel(adev, adev->gds.mem.gfx_partition_size,
1759 4, AMDGPU_GEM_DOMAIN_GDS,
1760 &adev->gds.gds_gfx_bo, NULL, NULL);
1761 if (r)
1762 return r;
1763
1764 r = ttm_bo_init_mm(&adev->mman.bdev, AMDGPU_PL_GWS,
1765 adev->gds.gws.total_size);
1766 if (r) {
1767 DRM_ERROR("Failed initializing gws heap.\n");
1768 return r;
1769 }
1770
1771 r = amdgpu_bo_create_kernel(adev, adev->gds.gws.gfx_partition_size,
1772 1, AMDGPU_GEM_DOMAIN_GWS,
1773 &adev->gds.gws_gfx_bo, NULL, NULL);
1774 if (r)
1775 return r;
1776
1777 r = ttm_bo_init_mm(&adev->mman.bdev, AMDGPU_PL_OA,
1778 adev->gds.oa.total_size);
1779 if (r) {
1780 DRM_ERROR("Failed initializing oa heap.\n");
1781 return r;
1782 }
1783
1784 r = amdgpu_bo_create_kernel(adev, adev->gds.oa.gfx_partition_size,
1785 1, AMDGPU_GEM_DOMAIN_OA,
1786 &adev->gds.oa_gfx_bo, NULL, NULL);
1787 if (r)
1788 return r;
1789
1790 /* Register debugfs entries for amdgpu_ttm */
1791 r = amdgpu_ttm_debugfs_init(adev);
1792 if (r) {
1793 DRM_ERROR("Failed to init debugfs\n");
1794 return r;
1795 }
1796 return 0;
1797 }
1798
1799 /**
1800 * amdgpu_ttm_late_init - Handle any late initialization for amdgpu_ttm
1801 */
1802 void amdgpu_ttm_late_init(struct amdgpu_device *adev)
1803 {
1804 /* return the VGA stolen memory (if any) back to VRAM */
1805 amdgpu_bo_free_kernel(&adev->stolen_vga_memory, NULL, NULL);
1806 }
1807
1808 /**
1809 * amdgpu_ttm_fini - De-initialize the TTM memory pools
1810 */
1811 void amdgpu_ttm_fini(struct amdgpu_device *adev)
1812 {
1813 if (!adev->mman.initialized)
1814 return;
1815
1816 amdgpu_ttm_debugfs_fini(adev);
1817 amdgpu_ttm_fw_reserve_vram_fini(adev);
1818 if (adev->mman.aper_base_kaddr)
1819 iounmap(adev->mman.aper_base_kaddr);
1820 adev->mman.aper_base_kaddr = NULL;
1821
1822 ttm_bo_clean_mm(&adev->mman.bdev, TTM_PL_VRAM);
1823 ttm_bo_clean_mm(&adev->mman.bdev, TTM_PL_TT);
1824 ttm_bo_clean_mm(&adev->mman.bdev, AMDGPU_PL_GDS);
1825 ttm_bo_clean_mm(&adev->mman.bdev, AMDGPU_PL_GWS);
1826 ttm_bo_clean_mm(&adev->mman.bdev, AMDGPU_PL_OA);
1827 ttm_bo_device_release(&adev->mman.bdev);
1828 adev->mman.initialized = false;
1829 DRM_INFO("amdgpu: ttm finalized\n");
1830 }
1831
1832 /**
1833 * amdgpu_ttm_set_buffer_funcs_status - enable/disable use of buffer functions
1834 *
1835 * @adev: amdgpu_device pointer
1836 * @enable: true when we can use buffer functions.
1837 *
1838 * Enable/disable use of buffer functions during suspend/resume. This should
1839 * only be called at bootup or when userspace isn't running.
1840 */
1841 void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device *adev, bool enable)
1842 {
1843 struct ttm_mem_type_manager *man = &adev->mman.bdev.man[TTM_PL_VRAM];
1844 uint64_t size;
1845 int r;
1846
1847 if (!adev->mman.initialized || adev->in_gpu_reset ||
1848 adev->mman.buffer_funcs_enabled == enable)
1849 return;
1850
1851 if (enable) {
1852 struct amdgpu_ring *ring;
1853 struct drm_sched_rq *rq;
1854
1855 ring = adev->mman.buffer_funcs_ring;
1856 rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_KERNEL];
1857 r = drm_sched_entity_init(&adev->mman.entity, &rq, 1, NULL);
1858 if (r) {
1859 DRM_ERROR("Failed setting up TTM BO move entity (%d)\n",
1860 r);
1861 return;
1862 }
1863 } else {
1864 drm_sched_entity_destroy(&adev->mman.entity);
1865 dma_fence_put(man->move);
1866 man->move = NULL;
1867 }
1868
1869 /* this just adjusts TTM size idea, which sets lpfn to the correct value */
1870 if (enable)
1871 size = adev->gmc.real_vram_size;
1872 else
1873 size = adev->gmc.visible_vram_size;
1874 man->size = size >> PAGE_SHIFT;
1875 adev->mman.buffer_funcs_enabled = enable;
1876 }
1877
1878 int amdgpu_mmap(struct file *filp, struct vm_area_struct *vma)
1879 {
1880 struct drm_file *file_priv;
1881 struct amdgpu_device *adev;
1882
1883 if (unlikely(vma->vm_pgoff < DRM_FILE_PAGE_OFFSET))
1884 return -EINVAL;
1885
1886 file_priv = filp->private_data;
1887 adev = file_priv->minor->dev->dev_private;
1888 if (adev == NULL)
1889 return -EINVAL;
1890
1891 return ttm_bo_mmap(filp, vma, &adev->mman.bdev);
1892 }
1893
1894 static int amdgpu_map_buffer(struct ttm_buffer_object *bo,
1895 struct ttm_mem_reg *mem, unsigned num_pages,
1896 uint64_t offset, unsigned window,
1897 struct amdgpu_ring *ring,
1898 uint64_t *addr)
1899 {
1900 struct amdgpu_ttm_tt *gtt = (void *)bo->ttm;
1901 struct amdgpu_device *adev = ring->adev;
1902 struct ttm_tt *ttm = bo->ttm;
1903 struct amdgpu_job *job;
1904 unsigned num_dw, num_bytes;
1905 dma_addr_t *dma_address;
1906 struct dma_fence *fence;
1907 uint64_t src_addr, dst_addr;
1908 uint64_t flags;
1909 int r;
1910
1911 BUG_ON(adev->mman.buffer_funcs->copy_max_bytes <
1912 AMDGPU_GTT_MAX_TRANSFER_SIZE * 8);
1913
1914 *addr = adev->gmc.gart_start;
1915 *addr += (u64)window * AMDGPU_GTT_MAX_TRANSFER_SIZE *
1916 AMDGPU_GPU_PAGE_SIZE;
1917
1918 num_dw = adev->mman.buffer_funcs->copy_num_dw;
1919 while (num_dw & 0x7)
1920 num_dw++;
1921
1922 num_bytes = num_pages * 8;
1923
1924 r = amdgpu_job_alloc_with_ib(adev, num_dw * 4 + num_bytes, &job);
1925 if (r)
1926 return r;
1927
1928 src_addr = num_dw * 4;
1929 src_addr += job->ibs[0].gpu_addr;
1930
1931 dst_addr = amdgpu_bo_gpu_offset(adev->gart.bo);
1932 dst_addr += window * AMDGPU_GTT_MAX_TRANSFER_SIZE * 8;
1933 amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_addr,
1934 dst_addr, num_bytes);
1935
1936 amdgpu_ring_pad_ib(ring, &job->ibs[0]);
1937 WARN_ON(job->ibs[0].length_dw > num_dw);
1938
1939 dma_address = &gtt->ttm.dma_address[offset >> PAGE_SHIFT];
1940 flags = amdgpu_ttm_tt_pte_flags(adev, ttm, mem);
1941 r = amdgpu_gart_map(adev, 0, num_pages, dma_address, flags,
1942 &job->ibs[0].ptr[num_dw]);
1943 if (r)
1944 goto error_free;
1945
1946 r = amdgpu_job_submit(job, &adev->mman.entity,
1947 AMDGPU_FENCE_OWNER_UNDEFINED, &fence);
1948 if (r)
1949 goto error_free;
1950
1951 dma_fence_put(fence);
1952
1953 return r;
1954
1955 error_free:
1956 amdgpu_job_free(job);
1957 return r;
1958 }
1959
1960 int amdgpu_copy_buffer(struct amdgpu_ring *ring, uint64_t src_offset,
1961 uint64_t dst_offset, uint32_t byte_count,
1962 struct reservation_object *resv,
1963 struct dma_fence **fence, bool direct_submit,
1964 bool vm_needs_flush)
1965 {
1966 struct amdgpu_device *adev = ring->adev;
1967 struct amdgpu_job *job;
1968
1969 uint32_t max_bytes;
1970 unsigned num_loops, num_dw;
1971 unsigned i;
1972 int r;
1973
1974 if (direct_submit && !ring->sched.ready) {
1975 DRM_ERROR("Trying to move memory with ring turned off.\n");
1976 return -EINVAL;
1977 }
1978
1979 max_bytes = adev->mman.buffer_funcs->copy_max_bytes;
1980 num_loops = DIV_ROUND_UP(byte_count, max_bytes);
1981 num_dw = num_loops * adev->mman.buffer_funcs->copy_num_dw;
1982
1983 /* for IB padding */
1984 while (num_dw & 0x7)
1985 num_dw++;
1986
1987 r = amdgpu_job_alloc_with_ib(adev, num_dw * 4, &job);
1988 if (r)
1989 return r;
1990
1991 if (vm_needs_flush) {
1992 job->vm_pd_addr = amdgpu_gmc_pd_addr(adev->gart.bo);
1993 job->vm_needs_flush = true;
1994 }
1995 if (resv) {
1996 r = amdgpu_sync_resv(adev, &job->sync, resv,
1997 AMDGPU_FENCE_OWNER_UNDEFINED,
1998 false);
1999 if (r) {
2000 DRM_ERROR("sync failed (%d).\n", r);
2001 goto error_free;
2002 }
2003 }
2004
2005 for (i = 0; i < num_loops; i++) {
2006 uint32_t cur_size_in_bytes = min(byte_count, max_bytes);
2007
2008 amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_offset,
2009 dst_offset, cur_size_in_bytes);
2010
2011 src_offset += cur_size_in_bytes;
2012 dst_offset += cur_size_in_bytes;
2013 byte_count -= cur_size_in_bytes;
2014 }
2015
2016 amdgpu_ring_pad_ib(ring, &job->ibs[0]);
2017 WARN_ON(job->ibs[0].length_dw > num_dw);
2018 if (direct_submit)
2019 r = amdgpu_job_submit_direct(job, ring, fence);
2020 else
2021 r = amdgpu_job_submit(job, &adev->mman.entity,
2022 AMDGPU_FENCE_OWNER_UNDEFINED, fence);
2023 if (r)
2024 goto error_free;
2025
2026 return r;
2027
2028 error_free:
2029 amdgpu_job_free(job);
2030 DRM_ERROR("Error scheduling IBs (%d)\n", r);
2031 return r;
2032 }
2033
2034 int amdgpu_fill_buffer(struct amdgpu_bo *bo,
2035 uint32_t src_data,
2036 struct reservation_object *resv,
2037 struct dma_fence **fence)
2038 {
2039 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
2040 uint32_t max_bytes = adev->mman.buffer_funcs->fill_max_bytes;
2041 struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
2042
2043 struct drm_mm_node *mm_node;
2044 unsigned long num_pages;
2045 unsigned int num_loops, num_dw;
2046
2047 struct amdgpu_job *job;
2048 int r;
2049
2050 if (!adev->mman.buffer_funcs_enabled) {
2051 DRM_ERROR("Trying to clear memory with ring turned off.\n");
2052 return -EINVAL;
2053 }
2054
2055 if (bo->tbo.mem.mem_type == TTM_PL_TT) {
2056 r = amdgpu_ttm_alloc_gart(&bo->tbo);
2057 if (r)
2058 return r;
2059 }
2060
2061 num_pages = bo->tbo.num_pages;
2062 mm_node = bo->tbo.mem.mm_node;
2063 num_loops = 0;
2064 while (num_pages) {
2065 uint32_t byte_count = mm_node->size << PAGE_SHIFT;
2066
2067 num_loops += DIV_ROUND_UP(byte_count, max_bytes);
2068 num_pages -= mm_node->size;
2069 ++mm_node;
2070 }
2071 num_dw = num_loops * adev->mman.buffer_funcs->fill_num_dw;
2072
2073 /* for IB padding */
2074 num_dw += 64;
2075
2076 r = amdgpu_job_alloc_with_ib(adev, num_dw * 4, &job);
2077 if (r)
2078 return r;
2079
2080 if (resv) {
2081 r = amdgpu_sync_resv(adev, &job->sync, resv,
2082 AMDGPU_FENCE_OWNER_UNDEFINED, false);
2083 if (r) {
2084 DRM_ERROR("sync failed (%d).\n", r);
2085 goto error_free;
2086 }
2087 }
2088
2089 num_pages = bo->tbo.num_pages;
2090 mm_node = bo->tbo.mem.mm_node;
2091
2092 while (num_pages) {
2093 uint32_t byte_count = mm_node->size << PAGE_SHIFT;
2094 uint64_t dst_addr;
2095
2096 dst_addr = amdgpu_mm_node_addr(&bo->tbo, mm_node, &bo->tbo.mem);
2097 while (byte_count) {
2098 uint32_t cur_size_in_bytes = min(byte_count, max_bytes);
2099
2100 amdgpu_emit_fill_buffer(adev, &job->ibs[0], src_data,
2101 dst_addr, cur_size_in_bytes);
2102
2103 dst_addr += cur_size_in_bytes;
2104 byte_count -= cur_size_in_bytes;
2105 }
2106
2107 num_pages -= mm_node->size;
2108 ++mm_node;
2109 }
2110
2111 amdgpu_ring_pad_ib(ring, &job->ibs[0]);
2112 WARN_ON(job->ibs[0].length_dw > num_dw);
2113 r = amdgpu_job_submit(job, &adev->mman.entity,
2114 AMDGPU_FENCE_OWNER_UNDEFINED, fence);
2115 if (r)
2116 goto error_free;
2117
2118 return 0;
2119
2120 error_free:
2121 amdgpu_job_free(job);
2122 return r;
2123 }
2124
2125 #if defined(CONFIG_DEBUG_FS)
2126
2127 static int amdgpu_mm_dump_table(struct seq_file *m, void *data)
2128 {
2129 struct drm_info_node *node = (struct drm_info_node *)m->private;
2130 unsigned ttm_pl = (uintptr_t)node->info_ent->data;
2131 struct drm_device *dev = node->minor->dev;
2132 struct amdgpu_device *adev = dev->dev_private;
2133 struct ttm_mem_type_manager *man = &adev->mman.bdev.man[ttm_pl];
2134 struct drm_printer p = drm_seq_file_printer(m);
2135
2136 man->func->debug(man, &p);
2137 return 0;
2138 }
2139
2140 static const struct drm_info_list amdgpu_ttm_debugfs_list[] = {
2141 {"amdgpu_vram_mm", amdgpu_mm_dump_table, 0, (void *)TTM_PL_VRAM},
2142 {"amdgpu_gtt_mm", amdgpu_mm_dump_table, 0, (void *)TTM_PL_TT},
2143 {"amdgpu_gds_mm", amdgpu_mm_dump_table, 0, (void *)AMDGPU_PL_GDS},
2144 {"amdgpu_gws_mm", amdgpu_mm_dump_table, 0, (void *)AMDGPU_PL_GWS},
2145 {"amdgpu_oa_mm", amdgpu_mm_dump_table, 0, (void *)AMDGPU_PL_OA},
2146 {"ttm_page_pool", ttm_page_alloc_debugfs, 0, NULL},
2147 #ifdef CONFIG_SWIOTLB
2148 {"ttm_dma_page_pool", ttm_dma_page_alloc_debugfs, 0, NULL}
2149 #endif
2150 };
2151
2152 /**
2153 * amdgpu_ttm_vram_read - Linear read access to VRAM
2154 *
2155 * Accesses VRAM via MMIO for debugging purposes.
2156 */
2157 static ssize_t amdgpu_ttm_vram_read(struct file *f, char __user *buf,
2158 size_t size, loff_t *pos)
2159 {
2160 struct amdgpu_device *adev = file_inode(f)->i_private;
2161 ssize_t result = 0;
2162 int r;
2163
2164 if (size & 0x3 || *pos & 0x3)
2165 return -EINVAL;
2166
2167 if (*pos >= adev->gmc.mc_vram_size)
2168 return -ENXIO;
2169
2170 while (size) {
2171 unsigned long flags;
2172 uint32_t value;
2173
2174 if (*pos >= adev->gmc.mc_vram_size)
2175 return result;
2176
2177 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
2178 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)*pos) | 0x80000000);
2179 WREG32_NO_KIQ(mmMM_INDEX_HI, *pos >> 31);
2180 value = RREG32_NO_KIQ(mmMM_DATA);
2181 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
2182
2183 r = put_user(value, (uint32_t *)buf);
2184 if (r)
2185 return r;
2186
2187 result += 4;
2188 buf += 4;
2189 *pos += 4;
2190 size -= 4;
2191 }
2192
2193 return result;
2194 }
2195
2196 /**
2197 * amdgpu_ttm_vram_write - Linear write access to VRAM
2198 *
2199 * Accesses VRAM via MMIO for debugging purposes.
2200 */
2201 static ssize_t amdgpu_ttm_vram_write(struct file *f, const char __user *buf,
2202 size_t size, loff_t *pos)
2203 {
2204 struct amdgpu_device *adev = file_inode(f)->i_private;
2205 ssize_t result = 0;
2206 int r;
2207
2208 if (size & 0x3 || *pos & 0x3)
2209 return -EINVAL;
2210
2211 if (*pos >= adev->gmc.mc_vram_size)
2212 return -ENXIO;
2213
2214 while (size) {
2215 unsigned long flags;
2216 uint32_t value;
2217
2218 if (*pos >= adev->gmc.mc_vram_size)
2219 return result;
2220
2221 r = get_user(value, (uint32_t *)buf);
2222 if (r)
2223 return r;
2224
2225 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
2226 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)*pos) | 0x80000000);
2227 WREG32_NO_KIQ(mmMM_INDEX_HI, *pos >> 31);
2228 WREG32_NO_KIQ(mmMM_DATA, value);
2229 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
2230
2231 result += 4;
2232 buf += 4;
2233 *pos += 4;
2234 size -= 4;
2235 }
2236
2237 return result;
2238 }
2239
2240 static const struct file_operations amdgpu_ttm_vram_fops = {
2241 .owner = THIS_MODULE,
2242 .read = amdgpu_ttm_vram_read,
2243 .write = amdgpu_ttm_vram_write,
2244 .llseek = default_llseek,
2245 };
2246
2247 #ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS
2248
2249 /**
2250 * amdgpu_ttm_gtt_read - Linear read access to GTT memory
2251 */
2252 static ssize_t amdgpu_ttm_gtt_read(struct file *f, char __user *buf,
2253 size_t size, loff_t *pos)
2254 {
2255 struct amdgpu_device *adev = file_inode(f)->i_private;
2256 ssize_t result = 0;
2257 int r;
2258
2259 while (size) {
2260 loff_t p = *pos / PAGE_SIZE;
2261 unsigned off = *pos & ~PAGE_MASK;
2262 size_t cur_size = min_t(size_t, size, PAGE_SIZE - off);
2263 struct page *page;
2264 void *ptr;
2265
2266 if (p >= adev->gart.num_cpu_pages)
2267 return result;
2268
2269 page = adev->gart.pages[p];
2270 if (page) {
2271 ptr = kmap(page);
2272 ptr += off;
2273
2274 r = copy_to_user(buf, ptr, cur_size);
2275 kunmap(adev->gart.pages[p]);
2276 } else
2277 r = clear_user(buf, cur_size);
2278
2279 if (r)
2280 return -EFAULT;
2281
2282 result += cur_size;
2283 buf += cur_size;
2284 *pos += cur_size;
2285 size -= cur_size;
2286 }
2287
2288 return result;
2289 }
2290
2291 static const struct file_operations amdgpu_ttm_gtt_fops = {
2292 .owner = THIS_MODULE,
2293 .read = amdgpu_ttm_gtt_read,
2294 .llseek = default_llseek
2295 };
2296
2297 #endif
2298
2299 /**
2300 * amdgpu_iomem_read - Virtual read access to GPU mapped memory
2301 *
2302 * This function is used to read memory that has been mapped to the
2303 * GPU and the known addresses are not physical addresses but instead
2304 * bus addresses (e.g., what you'd put in an IB or ring buffer).
2305 */
2306 static ssize_t amdgpu_iomem_read(struct file *f, char __user *buf,
2307 size_t size, loff_t *pos)
2308 {
2309 struct amdgpu_device *adev = file_inode(f)->i_private;
2310 struct iommu_domain *dom;
2311 ssize_t result = 0;
2312 int r;
2313
2314 /* retrieve the IOMMU domain if any for this device */
2315 dom = iommu_get_domain_for_dev(adev->dev);
2316
2317 while (size) {
2318 phys_addr_t addr = *pos & PAGE_MASK;
2319 loff_t off = *pos & ~PAGE_MASK;
2320 size_t bytes = PAGE_SIZE - off;
2321 unsigned long pfn;
2322 struct page *p;
2323 void *ptr;
2324
2325 bytes = bytes < size ? bytes : size;
2326
2327 /* Translate the bus address to a physical address. If
2328 * the domain is NULL it means there is no IOMMU active
2329 * and the address translation is the identity
2330 */
2331 addr = dom ? iommu_iova_to_phys(dom, addr) : addr;
2332
2333 pfn = addr >> PAGE_SHIFT;
2334 if (!pfn_valid(pfn))
2335 return -EPERM;
2336
2337 p = pfn_to_page(pfn);
2338 if (p->mapping != adev->mman.bdev.dev_mapping)
2339 return -EPERM;
2340
2341 ptr = kmap(p);
2342 r = copy_to_user(buf, ptr + off, bytes);
2343 kunmap(p);
2344 if (r)
2345 return -EFAULT;
2346
2347 size -= bytes;
2348 *pos += bytes;
2349 result += bytes;
2350 }
2351
2352 return result;
2353 }
2354
2355 /**
2356 * amdgpu_iomem_write - Virtual write access to GPU mapped memory
2357 *
2358 * This function is used to write memory that has been mapped to the
2359 * GPU and the known addresses are not physical addresses but instead
2360 * bus addresses (e.g., what you'd put in an IB or ring buffer).
2361 */
2362 static ssize_t amdgpu_iomem_write(struct file *f, const char __user *buf,
2363 size_t size, loff_t *pos)
2364 {
2365 struct amdgpu_device *adev = file_inode(f)->i_private;
2366 struct iommu_domain *dom;
2367 ssize_t result = 0;
2368 int r;
2369
2370 dom = iommu_get_domain_for_dev(adev->dev);
2371
2372 while (size) {
2373 phys_addr_t addr = *pos & PAGE_MASK;
2374 loff_t off = *pos & ~PAGE_MASK;
2375 size_t bytes = PAGE_SIZE - off;
2376 unsigned long pfn;
2377 struct page *p;
2378 void *ptr;
2379
2380 bytes = bytes < size ? bytes : size;
2381
2382 addr = dom ? iommu_iova_to_phys(dom, addr) : addr;
2383
2384 pfn = addr >> PAGE_SHIFT;
2385 if (!pfn_valid(pfn))
2386 return -EPERM;
2387
2388 p = pfn_to_page(pfn);
2389 if (p->mapping != adev->mman.bdev.dev_mapping)
2390 return -EPERM;
2391
2392 ptr = kmap(p);
2393 r = copy_from_user(ptr + off, buf, bytes);
2394 kunmap(p);
2395 if (r)
2396 return -EFAULT;
2397
2398 size -= bytes;
2399 *pos += bytes;
2400 result += bytes;
2401 }
2402
2403 return result;
2404 }
2405
2406 static const struct file_operations amdgpu_ttm_iomem_fops = {
2407 .owner = THIS_MODULE,
2408 .read = amdgpu_iomem_read,
2409 .write = amdgpu_iomem_write,
2410 .llseek = default_llseek
2411 };
2412
2413 static const struct {
2414 char *name;
2415 const struct file_operations *fops;
2416 int domain;
2417 } ttm_debugfs_entries[] = {
2418 { "amdgpu_vram", &amdgpu_ttm_vram_fops, TTM_PL_VRAM },
2419 #ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS
2420 { "amdgpu_gtt", &amdgpu_ttm_gtt_fops, TTM_PL_TT },
2421 #endif
2422 { "amdgpu_iomem", &amdgpu_ttm_iomem_fops, TTM_PL_SYSTEM },
2423 };
2424
2425 #endif
2426
2427 static int amdgpu_ttm_debugfs_init(struct amdgpu_device *adev)
2428 {
2429 #if defined(CONFIG_DEBUG_FS)
2430 unsigned count;
2431
2432 struct drm_minor *minor = adev->ddev->primary;
2433 struct dentry *ent, *root = minor->debugfs_root;
2434
2435 for (count = 0; count < ARRAY_SIZE(ttm_debugfs_entries); count++) {
2436 ent = debugfs_create_file(
2437 ttm_debugfs_entries[count].name,
2438 S_IFREG | S_IRUGO, root,
2439 adev,
2440 ttm_debugfs_entries[count].fops);
2441 if (IS_ERR(ent))
2442 return PTR_ERR(ent);
2443 if (ttm_debugfs_entries[count].domain == TTM_PL_VRAM)
2444 i_size_write(ent->d_inode, adev->gmc.mc_vram_size);
2445 else if (ttm_debugfs_entries[count].domain == TTM_PL_TT)
2446 i_size_write(ent->d_inode, adev->gmc.gart_size);
2447 adev->mman.debugfs_entries[count] = ent;
2448 }
2449
2450 count = ARRAY_SIZE(amdgpu_ttm_debugfs_list);
2451
2452 #ifdef CONFIG_SWIOTLB
2453 if (!(adev->need_swiotlb && swiotlb_nr_tbl()))
2454 --count;
2455 #endif
2456
2457 return amdgpu_debugfs_add_files(adev, amdgpu_ttm_debugfs_list, count);
2458 #else
2459 return 0;
2460 #endif
2461 }
2462
2463 static void amdgpu_ttm_debugfs_fini(struct amdgpu_device *adev)
2464 {
2465 #if defined(CONFIG_DEBUG_FS)
2466 unsigned i;
2467
2468 for (i = 0; i < ARRAY_SIZE(ttm_debugfs_entries); i++)
2469 debugfs_remove(adev->mman.debugfs_entries[i]);
2470 #endif
2471 }
Linux with added FPC-III support