?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
treewide: remove redundant IS_ERR() before error code check
[linux/fpc-iii.git] / drivers / gpu / drm / amd / amdgpu / amdgpu_ttm.c
blobdee446278417c01653ff6fb8a981ba9835a6d7c3
1 /*
2 * Copyright 2009 Jerome Glisse.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sub license, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19 * USE OR OTHER DEALINGS IN THE SOFTWARE.
20 *
21 * The above copyright notice and this permission notice (including the
22 * next paragraph) shall be included in all copies or substantial portions
23 * of the Software.
24 *
25 */
26 /*
27 * Authors:
28 * Jerome Glisse <glisse@freedesktop.org>
29 * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
30 * Dave Airlie
31 */
32
33 #include <linux/dma-mapping.h>
34 #include <linux/iommu.h>
35 #include <linux/hmm.h>
36 #include <linux/pagemap.h>
37 #include <linux/sched/task.h>
38 #include <linux/sched/mm.h>
39 #include <linux/seq_file.h>
40 #include <linux/slab.h>
41 #include <linux/swap.h>
42 #include <linux/swiotlb.h>
43 #include <linux/dma-buf.h>
44 #include <linux/sizes.h>
45
46 #include <drm/ttm/ttm_bo_api.h>
47 #include <drm/ttm/ttm_bo_driver.h>
48 #include <drm/ttm/ttm_placement.h>
49 #include <drm/ttm/ttm_module.h>
50 #include <drm/ttm/ttm_page_alloc.h>
51
52 #include <drm/drm_debugfs.h>
53 #include <drm/amdgpu_drm.h>
54
55 #include "amdgpu.h"
56 #include "amdgpu_object.h"
57 #include "amdgpu_trace.h"
58 #include "amdgpu_amdkfd.h"
59 #include "amdgpu_sdma.h"
60 #include "amdgpu_ras.h"
61 #include "bif/bif_4_1_d.h"
62
63 static int amdgpu_map_buffer(struct ttm_buffer_object *bo,
64 struct ttm_mem_reg *mem, unsigned num_pages,
65 uint64_t offset, unsigned window,
66 struct amdgpu_ring *ring,
67 uint64_t *addr);
68
69 static int amdgpu_ttm_debugfs_init(struct amdgpu_device *adev);
70 static void amdgpu_ttm_debugfs_fini(struct amdgpu_device *adev);
71
72 static int amdgpu_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags)
73 {
74 return 0;
75 }
76
77 /**
78 * amdgpu_init_mem_type - Initialize a memory manager for a specific type of
79 * memory request.
80 *
81 * @bdev: The TTM BO device object (contains a reference to amdgpu_device)
82 * @type: The type of memory requested
83 * @man: The memory type manager for each domain
84 *
85 * This is called by ttm_bo_init_mm() when a buffer object is being
86 * initialized.
87 */
88 static int amdgpu_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
89 struct ttm_mem_type_manager *man)
90 {
91 struct amdgpu_device *adev;
92
93 adev = amdgpu_ttm_adev(bdev);
94
95 switch (type) {
96 case TTM_PL_SYSTEM:
97 /* System memory */
98 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
99 man->available_caching = TTM_PL_MASK_CACHING;
100 man->default_caching = TTM_PL_FLAG_CACHED;
101 break;
102 case TTM_PL_TT:
103 /* GTT memory */
104 man->func = &amdgpu_gtt_mgr_func;
105 man->gpu_offset = adev->gmc.gart_start;
106 man->available_caching = TTM_PL_MASK_CACHING;
107 man->default_caching = TTM_PL_FLAG_CACHED;
108 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE | TTM_MEMTYPE_FLAG_CMA;
109 break;
110 case TTM_PL_VRAM:
111 /* "On-card" video ram */
112 man->func = &amdgpu_vram_mgr_func;
113 man->gpu_offset = adev->gmc.vram_start;
114 man->flags = TTM_MEMTYPE_FLAG_FIXED |
115 TTM_MEMTYPE_FLAG_MAPPABLE;
116 man->available_caching = TTM_PL_FLAG_UNCACHED | TTM_PL_FLAG_WC;
117 man->default_caching = TTM_PL_FLAG_WC;
118 break;
119 case AMDGPU_PL_GDS:
120 case AMDGPU_PL_GWS:
121 case AMDGPU_PL_OA:
122 /* On-chip GDS memory*/
123 man->func = &ttm_bo_manager_func;
124 man->gpu_offset = 0;
125 man->flags = TTM_MEMTYPE_FLAG_FIXED | TTM_MEMTYPE_FLAG_CMA;
126 man->available_caching = TTM_PL_FLAG_UNCACHED;
127 man->default_caching = TTM_PL_FLAG_UNCACHED;
128 break;
129 default:
130 DRM_ERROR("Unsupported memory type %u\n", (unsigned)type);
131 return -EINVAL;
132 }
133 return 0;
134 }
135
136 /**
137 * amdgpu_evict_flags - Compute placement flags
138 *
139 * @bo: The buffer object to evict
140 * @placement: Possible destination(s) for evicted BO
141 *
142 * Fill in placement data when ttm_bo_evict() is called
143 */
144 static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
145 struct ttm_placement *placement)
146 {
147 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
148 struct amdgpu_bo *abo;
149 static const struct ttm_place placements = {
150 .fpfn = 0,
151 .lpfn = 0,
152 .flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM
153 };
154
155 /* Don't handle scatter gather BOs */
156 if (bo->type == ttm_bo_type_sg) {
157 placement->num_placement = 0;
158 placement->num_busy_placement = 0;
159 return;
160 }
161
162 /* Object isn't an AMDGPU object so ignore */
163 if (!amdgpu_bo_is_amdgpu_bo(bo)) {
164 placement->placement = &placements;
165 placement->busy_placement = &placements;
166 placement->num_placement = 1;
167 placement->num_busy_placement = 1;
168 return;
169 }
170
171 abo = ttm_to_amdgpu_bo(bo);
172 switch (bo->mem.mem_type) {
173 case AMDGPU_PL_GDS:
174 case AMDGPU_PL_GWS:
175 case AMDGPU_PL_OA:
176 placement->num_placement = 0;
177 placement->num_busy_placement = 0;
178 return;
179
180 case TTM_PL_VRAM:
181 if (!adev->mman.buffer_funcs_enabled) {
182 /* Move to system memory */
183 amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU);
184 } else if (!amdgpu_gmc_vram_full_visible(&adev->gmc) &&
185 !(abo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) &&
186 amdgpu_bo_in_cpu_visible_vram(abo)) {
187
188 /* Try evicting to the CPU inaccessible part of VRAM
189 * first, but only set GTT as busy placement, so this
190 * BO will be evicted to GTT rather than causing other
191 * BOs to be evicted from VRAM
192 */
193 amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM |
194 AMDGPU_GEM_DOMAIN_GTT);
195 abo->placements[0].fpfn = adev->gmc.visible_vram_size >> PAGE_SHIFT;
196 abo->placements[0].lpfn = 0;
197 abo->placement.busy_placement = &abo->placements[1];
198 abo->placement.num_busy_placement = 1;
199 } else {
200 /* Move to GTT memory */
201 amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_GTT);
202 }
203 break;
204 case TTM_PL_TT:
205 default:
206 amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU);
207 break;
208 }
209 *placement = abo->placement;
210 }
211
212 /**
213 * amdgpu_verify_access - Verify access for a mmap call
214 *
215 * @bo: The buffer object to map
216 * @filp: The file pointer from the process performing the mmap
217 *
218 * This is called by ttm_bo_mmap() to verify whether a process
219 * has the right to mmap a BO to their process space.
220 */
221 static int amdgpu_verify_access(struct ttm_buffer_object *bo, struct file *filp)
222 {
223 struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
224
225 /*
226 * Don't verify access for KFD BOs. They don't have a GEM
227 * object associated with them.
228 */
229 if (abo->kfd_bo)
230 return 0;
231
232 if (amdgpu_ttm_tt_get_usermm(bo->ttm))
233 return -EPERM;
234 return drm_vma_node_verify_access(&abo->tbo.base.vma_node,
235 filp->private_data);
236 }
237
238 /**
239 * amdgpu_move_null - Register memory for a buffer object
240 *
241 * @bo: The bo to assign the memory to
242 * @new_mem: The memory to be assigned.
243 *
244 * Assign the memory from new_mem to the memory of the buffer object bo.
245 */
246 static void amdgpu_move_null(struct ttm_buffer_object *bo,
247 struct ttm_mem_reg *new_mem)
248 {
249 struct ttm_mem_reg *old_mem = &bo->mem;
250
251 BUG_ON(old_mem->mm_node != NULL);
252 *old_mem = *new_mem;
253 new_mem->mm_node = NULL;
254 }
255
256 /**
257 * amdgpu_mm_node_addr - Compute the GPU relative offset of a GTT buffer.
258 *
259 * @bo: The bo to assign the memory to.
260 * @mm_node: Memory manager node for drm allocator.
261 * @mem: The region where the bo resides.
262 *
263 */
264 static uint64_t amdgpu_mm_node_addr(struct ttm_buffer_object *bo,
265 struct drm_mm_node *mm_node,
266 struct ttm_mem_reg *mem)
267 {
268 uint64_t addr = 0;
269
270 if (mm_node->start != AMDGPU_BO_INVALID_OFFSET) {
271 addr = mm_node->start << PAGE_SHIFT;
272 addr += bo->bdev->man[mem->mem_type].gpu_offset;
273 }
274 return addr;
275 }
276
277 /**
278 * amdgpu_find_mm_node - Helper function finds the drm_mm_node corresponding to
279 * @offset. It also modifies the offset to be within the drm_mm_node returned
280 *
281 * @mem: The region where the bo resides.
282 * @offset: The offset that drm_mm_node is used for finding.
283 *
284 */
285 static struct drm_mm_node *amdgpu_find_mm_node(struct ttm_mem_reg *mem,
286 unsigned long *offset)
287 {
288 struct drm_mm_node *mm_node = mem->mm_node;
289
290 while (*offset >= (mm_node->size << PAGE_SHIFT)) {
291 *offset -= (mm_node->size << PAGE_SHIFT);
292 ++mm_node;
293 }
294 return mm_node;
295 }
296
297 /**
298 * amdgpu_copy_ttm_mem_to_mem - Helper function for copy
299 *
300 * The function copies @size bytes from {src->mem + src->offset} to
301 * {dst->mem + dst->offset}. src->bo and dst->bo could be same BO for a
302 * move and different for a BO to BO copy.
303 *
304 * @f: Returns the last fence if multiple jobs are submitted.
305 */
306 int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev,
307 struct amdgpu_copy_mem *src,
308 struct amdgpu_copy_mem *dst,
309 uint64_t size,
310 struct dma_resv *resv,
311 struct dma_fence **f)
312 {
313 struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
314 struct drm_mm_node *src_mm, *dst_mm;
315 uint64_t src_node_start, dst_node_start, src_node_size,
316 dst_node_size, src_page_offset, dst_page_offset;
317 struct dma_fence *fence = NULL;
318 int r = 0;
319 const uint64_t GTT_MAX_BYTES = (AMDGPU_GTT_MAX_TRANSFER_SIZE *
320 AMDGPU_GPU_PAGE_SIZE);
321
322 if (!adev->mman.buffer_funcs_enabled) {
323 DRM_ERROR("Trying to move memory with ring turned off.\n");
324 return -EINVAL;
325 }
326
327 src_mm = amdgpu_find_mm_node(src->mem, &src->offset);
328 src_node_start = amdgpu_mm_node_addr(src->bo, src_mm, src->mem) +
329 src->offset;
330 src_node_size = (src_mm->size << PAGE_SHIFT) - src->offset;
331 src_page_offset = src_node_start & (PAGE_SIZE - 1);
332
333 dst_mm = amdgpu_find_mm_node(dst->mem, &dst->offset);
334 dst_node_start = amdgpu_mm_node_addr(dst->bo, dst_mm, dst->mem) +
335 dst->offset;
336 dst_node_size = (dst_mm->size << PAGE_SHIFT) - dst->offset;
337 dst_page_offset = dst_node_start & (PAGE_SIZE - 1);
338
339 mutex_lock(&adev->mman.gtt_window_lock);
340
341 while (size) {
342 unsigned long cur_size;
343 uint64_t from = src_node_start, to = dst_node_start;
344 struct dma_fence *next;
345
346 /* Copy size cannot exceed GTT_MAX_BYTES. So if src or dst
347 * begins at an offset, then adjust the size accordingly
348 */
349 cur_size = min3(min(src_node_size, dst_node_size), size,
350 GTT_MAX_BYTES);
351 if (cur_size + src_page_offset > GTT_MAX_BYTES ||
352 cur_size + dst_page_offset > GTT_MAX_BYTES)
353 cur_size -= max(src_page_offset, dst_page_offset);
354
355 /* Map only what needs to be accessed. Map src to window 0 and
356 * dst to window 1
357 */
358 if (src->mem->start == AMDGPU_BO_INVALID_OFFSET) {
359 r = amdgpu_map_buffer(src->bo, src->mem,
360 PFN_UP(cur_size + src_page_offset),
361 src_node_start, 0, ring,
362 &from);
363 if (r)
364 goto error;
365 /* Adjust the offset because amdgpu_map_buffer returns
366 * start of mapped page
367 */
368 from += src_page_offset;
369 }
370
371 if (dst->mem->start == AMDGPU_BO_INVALID_OFFSET) {
372 r = amdgpu_map_buffer(dst->bo, dst->mem,
373 PFN_UP(cur_size + dst_page_offset),
374 dst_node_start, 1, ring,
375 &to);
376 if (r)
377 goto error;
378 to += dst_page_offset;
379 }
380
381 r = amdgpu_copy_buffer(ring, from, to, cur_size,
382 resv, &next, false, true);
383 if (r)
384 goto error;
385
386 dma_fence_put(fence);
387 fence = next;
388
389 size -= cur_size;
390 if (!size)
391 break;
392
393 src_node_size -= cur_size;
394 if (!src_node_size) {
395 src_node_start = amdgpu_mm_node_addr(src->bo, ++src_mm,
396 src->mem);
397 src_node_size = (src_mm->size << PAGE_SHIFT);
398 src_page_offset = 0;
399 } else {
400 src_node_start += cur_size;
401 src_page_offset = src_node_start & (PAGE_SIZE - 1);
402 }
403 dst_node_size -= cur_size;
404 if (!dst_node_size) {
405 dst_node_start = amdgpu_mm_node_addr(dst->bo, ++dst_mm,
406 dst->mem);
407 dst_node_size = (dst_mm->size << PAGE_SHIFT);
408 dst_page_offset = 0;
409 } else {
410 dst_node_start += cur_size;
411 dst_page_offset = dst_node_start & (PAGE_SIZE - 1);
412 }
413 }
414 error:
415 mutex_unlock(&adev->mman.gtt_window_lock);
416 if (f)
417 *f = dma_fence_get(fence);
418 dma_fence_put(fence);
419 return r;
420 }
421
422 /**
423 * amdgpu_move_blit - Copy an entire buffer to another buffer
424 *
425 * This is a helper called by amdgpu_bo_move() and amdgpu_move_vram_ram() to
426 * help move buffers to and from VRAM.
427 */
428 static int amdgpu_move_blit(struct ttm_buffer_object *bo,
429 bool evict, bool no_wait_gpu,
430 struct ttm_mem_reg *new_mem,
431 struct ttm_mem_reg *old_mem)
432 {
433 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
434 struct amdgpu_copy_mem src, dst;
435 struct dma_fence *fence = NULL;
436 int r;
437
438 src.bo = bo;
439 dst.bo = bo;
440 src.mem = old_mem;
441 dst.mem = new_mem;
442 src.offset = 0;
443 dst.offset = 0;
444
445 r = amdgpu_ttm_copy_mem_to_mem(adev, &src, &dst,
446 new_mem->num_pages << PAGE_SHIFT,
447 bo->base.resv, &fence);
448 if (r)
449 goto error;
450
451 /* clear the space being freed */
452 if (old_mem->mem_type == TTM_PL_VRAM &&
453 (ttm_to_amdgpu_bo(bo)->flags &
454 AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE)) {
455 struct dma_fence *wipe_fence = NULL;
456
457 r = amdgpu_fill_buffer(ttm_to_amdgpu_bo(bo), AMDGPU_POISON,
458 NULL, &wipe_fence);
459 if (r) {
460 goto error;
461 } else if (wipe_fence) {
462 dma_fence_put(fence);
463 fence = wipe_fence;
464 }
465 }
466
467 /* Always block for VM page tables before committing the new location */
468 if (bo->type == ttm_bo_type_kernel)
469 r = ttm_bo_move_accel_cleanup(bo, fence, true, new_mem);
470 else
471 r = ttm_bo_pipeline_move(bo, fence, evict, new_mem);
472 dma_fence_put(fence);
473 return r;
474
475 error:
476 if (fence)
477 dma_fence_wait(fence, false);
478 dma_fence_put(fence);
479 return r;
480 }
481
482 /**
483 * amdgpu_move_vram_ram - Copy VRAM buffer to RAM buffer
484 *
485 * Called by amdgpu_bo_move().
486 */
487 static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo, bool evict,
488 struct ttm_operation_ctx *ctx,
489 struct ttm_mem_reg *new_mem)
490 {
491 struct ttm_mem_reg *old_mem = &bo->mem;
492 struct ttm_mem_reg tmp_mem;
493 struct ttm_place placements;
494 struct ttm_placement placement;
495 int r;
496
497 /* create space/pages for new_mem in GTT space */
498 tmp_mem = *new_mem;
499 tmp_mem.mm_node = NULL;
500 placement.num_placement = 1;
501 placement.placement = &placements;
502 placement.num_busy_placement = 1;
503 placement.busy_placement = &placements;
504 placements.fpfn = 0;
505 placements.lpfn = 0;
506 placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
507 r = ttm_bo_mem_space(bo, &placement, &tmp_mem, ctx);
508 if (unlikely(r)) {
509 pr_err("Failed to find GTT space for blit from VRAM\n");
510 return r;
511 }
512
513 /* set caching flags */
514 r = ttm_tt_set_placement_caching(bo->ttm, tmp_mem.placement);
515 if (unlikely(r)) {
516 goto out_cleanup;
517 }
518
519 /* Bind the memory to the GTT space */
520 r = ttm_tt_bind(bo->ttm, &tmp_mem, ctx);
521 if (unlikely(r)) {
522 goto out_cleanup;
523 }
524
525 /* blit VRAM to GTT */
526 r = amdgpu_move_blit(bo, evict, ctx->no_wait_gpu, &tmp_mem, old_mem);
527 if (unlikely(r)) {
528 goto out_cleanup;
529 }
530
531 /* move BO (in tmp_mem) to new_mem */
532 r = ttm_bo_move_ttm(bo, ctx, new_mem);
533 out_cleanup:
534 ttm_bo_mem_put(bo, &tmp_mem);
535 return r;
536 }
537
538 /**
539 * amdgpu_move_ram_vram - Copy buffer from RAM to VRAM
540 *
541 * Called by amdgpu_bo_move().
542 */
543 static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo, bool evict,
544 struct ttm_operation_ctx *ctx,
545 struct ttm_mem_reg *new_mem)
546 {
547 struct ttm_mem_reg *old_mem = &bo->mem;
548 struct ttm_mem_reg tmp_mem;
549 struct ttm_placement placement;
550 struct ttm_place placements;
551 int r;
552
553 /* make space in GTT for old_mem buffer */
554 tmp_mem = *new_mem;
555 tmp_mem.mm_node = NULL;
556 placement.num_placement = 1;
557 placement.placement = &placements;
558 placement.num_busy_placement = 1;
559 placement.busy_placement = &placements;
560 placements.fpfn = 0;
561 placements.lpfn = 0;
562 placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
563 r = ttm_bo_mem_space(bo, &placement, &tmp_mem, ctx);
564 if (unlikely(r)) {
565 pr_err("Failed to find GTT space for blit to VRAM\n");
566 return r;
567 }
568
569 /* move/bind old memory to GTT space */
570 r = ttm_bo_move_ttm(bo, ctx, &tmp_mem);
571 if (unlikely(r)) {
572 goto out_cleanup;
573 }
574
575 /* copy to VRAM */
576 r = amdgpu_move_blit(bo, evict, ctx->no_wait_gpu, new_mem, old_mem);
577 if (unlikely(r)) {
578 goto out_cleanup;
579 }
580 out_cleanup:
581 ttm_bo_mem_put(bo, &tmp_mem);
582 return r;
583 }
584
585 /**
586 * amdgpu_mem_visible - Check that memory can be accessed by ttm_bo_move_memcpy
587 *
588 * Called by amdgpu_bo_move()
589 */
590 static bool amdgpu_mem_visible(struct amdgpu_device *adev,
591 struct ttm_mem_reg *mem)
592 {
593 struct drm_mm_node *nodes = mem->mm_node;
594
595 if (mem->mem_type == TTM_PL_SYSTEM ||
596 mem->mem_type == TTM_PL_TT)
597 return true;
598 if (mem->mem_type != TTM_PL_VRAM)
599 return false;
600
601 /* ttm_mem_reg_ioremap only supports contiguous memory */
602 if (nodes->size != mem->num_pages)
603 return false;
604
605 return ((nodes->start + nodes->size) << PAGE_SHIFT)
606 <= adev->gmc.visible_vram_size;
607 }
608
609 /**
610 * amdgpu_bo_move - Move a buffer object to a new memory location
611 *
612 * Called by ttm_bo_handle_move_mem()
613 */
614 static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict,
615 struct ttm_operation_ctx *ctx,
616 struct ttm_mem_reg *new_mem)
617 {
618 struct amdgpu_device *adev;
619 struct amdgpu_bo *abo;
620 struct ttm_mem_reg *old_mem = &bo->mem;
621 int r;
622
623 /* Can't move a pinned BO */
624 abo = ttm_to_amdgpu_bo(bo);
625 if (WARN_ON_ONCE(abo->pin_count > 0))
626 return -EINVAL;
627
628 adev = amdgpu_ttm_adev(bo->bdev);
629
630 if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) {
631 amdgpu_move_null(bo, new_mem);
632 return 0;
633 }
634 if ((old_mem->mem_type == TTM_PL_TT &&
635 new_mem->mem_type == TTM_PL_SYSTEM) ||
636 (old_mem->mem_type == TTM_PL_SYSTEM &&
637 new_mem->mem_type == TTM_PL_TT)) {
638 /* bind is enough */
639 amdgpu_move_null(bo, new_mem);
640 return 0;
641 }
642 if (old_mem->mem_type == AMDGPU_PL_GDS ||
643 old_mem->mem_type == AMDGPU_PL_GWS ||
644 old_mem->mem_type == AMDGPU_PL_OA ||
645 new_mem->mem_type == AMDGPU_PL_GDS ||
646 new_mem->mem_type == AMDGPU_PL_GWS ||
647 new_mem->mem_type == AMDGPU_PL_OA) {
648 /* Nothing to save here */
649 amdgpu_move_null(bo, new_mem);
650 return 0;
651 }
652
653 if (!adev->mman.buffer_funcs_enabled) {
654 r = -ENODEV;
655 goto memcpy;
656 }
657
658 if (old_mem->mem_type == TTM_PL_VRAM &&
659 new_mem->mem_type == TTM_PL_SYSTEM) {
660 r = amdgpu_move_vram_ram(bo, evict, ctx, new_mem);
661 } else if (old_mem->mem_type == TTM_PL_SYSTEM &&
662 new_mem->mem_type == TTM_PL_VRAM) {
663 r = amdgpu_move_ram_vram(bo, evict, ctx, new_mem);
664 } else {
665 r = amdgpu_move_blit(bo, evict, ctx->no_wait_gpu,
666 new_mem, old_mem);
667 }
668
669 if (r) {
670 memcpy:
671 /* Check that all memory is CPU accessible */
672 if (!amdgpu_mem_visible(adev, old_mem) ||
673 !amdgpu_mem_visible(adev, new_mem)) {
674 pr_err("Move buffer fallback to memcpy unavailable\n");
675 return r;
676 }
677
678 r = ttm_bo_move_memcpy(bo, ctx, new_mem);
679 if (r)
680 return r;
681 }
682
683 if (bo->type == ttm_bo_type_device &&
684 new_mem->mem_type == TTM_PL_VRAM &&
685 old_mem->mem_type != TTM_PL_VRAM) {
686 /* amdgpu_bo_fault_reserve_notify will re-set this if the CPU
687 * accesses the BO after it's moved.
688 */
689 abo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
690 }
691
692 /* update statistics */
693 atomic64_add((u64)bo->num_pages << PAGE_SHIFT, &adev->num_bytes_moved);
694 return 0;
695 }
696
697 /**
698 * amdgpu_ttm_io_mem_reserve - Reserve a block of memory during a fault
699 *
700 * Called by ttm_mem_io_reserve() ultimately via ttm_bo_vm_fault()
701 */
702 static int amdgpu_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
703 {
704 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
705 struct amdgpu_device *adev = amdgpu_ttm_adev(bdev);
706 struct drm_mm_node *mm_node = mem->mm_node;
707
708 mem->bus.addr = NULL;
709 mem->bus.offset = 0;
710 mem->bus.size = mem->num_pages << PAGE_SHIFT;
711 mem->bus.base = 0;
712 mem->bus.is_iomem = false;
713 if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE))
714 return -EINVAL;
715 switch (mem->mem_type) {
716 case TTM_PL_SYSTEM:
717 /* system memory */
718 return 0;
719 case TTM_PL_TT:
720 break;
721 case TTM_PL_VRAM:
722 mem->bus.offset = mem->start << PAGE_SHIFT;
723 /* check if it's visible */
724 if ((mem->bus.offset + mem->bus.size) > adev->gmc.visible_vram_size)
725 return -EINVAL;
726 /* Only physically contiguous buffers apply. In a contiguous
727 * buffer, size of the first mm_node would match the number of
728 * pages in ttm_mem_reg.
729 */
730 if (adev->mman.aper_base_kaddr &&
731 (mm_node->size == mem->num_pages))
732 mem->bus.addr = (u8 *)adev->mman.aper_base_kaddr +
733 mem->bus.offset;
734
735 mem->bus.base = adev->gmc.aper_base;
736 mem->bus.is_iomem = true;
737 break;
738 default:
739 return -EINVAL;
740 }
741 return 0;
742 }
743
744 static void amdgpu_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
745 {
746 }
747
748 static unsigned long amdgpu_ttm_io_mem_pfn(struct ttm_buffer_object *bo,
749 unsigned long page_offset)
750 {
751 struct drm_mm_node *mm;
752 unsigned long offset = (page_offset << PAGE_SHIFT);
753
754 mm = amdgpu_find_mm_node(&bo->mem, &offset);
755 return (bo->mem.bus.base >> PAGE_SHIFT) + mm->start +
756 (offset >> PAGE_SHIFT);
757 }
758
759 /*
760 * TTM backend functions.
761 */
762 struct amdgpu_ttm_tt {
763 struct ttm_dma_tt ttm;
764 struct drm_gem_object *gobj;
765 u64 offset;
766 uint64_t userptr;
767 struct task_struct *usertask;
768 uint32_t userflags;
769 #if IS_ENABLED(CONFIG_DRM_AMDGPU_USERPTR)
770 struct hmm_range *range;
771 #endif
772 };
773
774 #ifdef CONFIG_DRM_AMDGPU_USERPTR
775 /* flags used by HMM internal, not related to CPU/GPU PTE flags */
776 static const uint64_t hmm_range_flags[HMM_PFN_FLAG_MAX] = {
777 (1 << 0), /* HMM_PFN_VALID */
778 (1 << 1), /* HMM_PFN_WRITE */
779 0 /* HMM_PFN_DEVICE_PRIVATE */
780 };
781
782 static const uint64_t hmm_range_values[HMM_PFN_VALUE_MAX] = {
783 0xfffffffffffffffeUL, /* HMM_PFN_ERROR */
784 0, /* HMM_PFN_NONE */
785 0xfffffffffffffffcUL /* HMM_PFN_SPECIAL */
786 };
787
788 /**
789 * amdgpu_ttm_tt_get_user_pages - get device accessible pages that back user
790 * memory and start HMM tracking CPU page table update
791 *
792 * Calling function must call amdgpu_ttm_tt_userptr_range_done() once and only
793 * once afterwards to stop HMM tracking
794 */
795 int amdgpu_ttm_tt_get_user_pages(struct amdgpu_bo *bo, struct page **pages)
796 {
797 struct ttm_tt *ttm = bo->tbo.ttm;
798 struct amdgpu_ttm_tt *gtt = (void *)ttm;
799 unsigned long start = gtt->userptr;
800 struct vm_area_struct *vma;
801 struct hmm_range *range;
802 unsigned long timeout;
803 struct mm_struct *mm;
804 unsigned long i;
805 int r = 0;
806
807 mm = bo->notifier.mm;
808 if (unlikely(!mm)) {
809 DRM_DEBUG_DRIVER("BO is not registered?\n");
810 return -EFAULT;
811 }
812
813 /* Another get_user_pages is running at the same time?? */
814 if (WARN_ON(gtt->range))
815 return -EFAULT;
816
817 if (!mmget_not_zero(mm)) /* Happens during process shutdown */
818 return -ESRCH;
819
820 range = kzalloc(sizeof(*range), GFP_KERNEL);
821 if (unlikely(!range)) {
822 r = -ENOMEM;
823 goto out;
824 }
825 range->notifier = &bo->notifier;
826 range->flags = hmm_range_flags;
827 range->values = hmm_range_values;
828 range->pfn_shift = PAGE_SHIFT;
829 range->start = bo->notifier.interval_tree.start;
830 range->end = bo->notifier.interval_tree.last + 1;
831 range->default_flags = hmm_range_flags[HMM_PFN_VALID];
832 if (!amdgpu_ttm_tt_is_readonly(ttm))
833 range->default_flags |= range->flags[HMM_PFN_WRITE];
834
835 range->pfns = kvmalloc_array(ttm->num_pages, sizeof(*range->pfns),
836 GFP_KERNEL);
837 if (unlikely(!range->pfns)) {
838 r = -ENOMEM;
839 goto out_free_ranges;
840 }
841
842 down_read(&mm->mmap_sem);
843 vma = find_vma(mm, start);
844 if (unlikely(!vma || start < vma->vm_start)) {
845 r = -EFAULT;
846 goto out_unlock;
847 }
848 if (unlikely((gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) &&
849 vma->vm_file)) {
850 r = -EPERM;
851 goto out_unlock;
852 }
853 up_read(&mm->mmap_sem);
854 timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
855
856 retry:
857 range->notifier_seq = mmu_interval_read_begin(&bo->notifier);
858
859 down_read(&mm->mmap_sem);
860 r = hmm_range_fault(range, 0);
861 up_read(&mm->mmap_sem);
862 if (unlikely(r <= 0)) {
863 /*
864 * FIXME: This timeout should encompass the retry from
865 * mmu_interval_read_retry() as well.
866 */
867 if ((r == 0 || r == -EBUSY) && !time_after(jiffies, timeout))
868 goto retry;
869 goto out_free_pfns;
870 }
871
872 for (i = 0; i < ttm->num_pages; i++) {
873 /* FIXME: The pages cannot be touched outside the notifier_lock */
874 pages[i] = hmm_device_entry_to_page(range, range->pfns[i]);
875 if (unlikely(!pages[i])) {
876 pr_err("Page fault failed for pfn[%lu] = 0x%llx\n",
877 i, range->pfns[i]);
878 r = -ENOMEM;
879
880 goto out_free_pfns;
881 }
882 }
883
884 gtt->range = range;
885 mmput(mm);
886
887 return 0;
888
889 out_unlock:
890 up_read(&mm->mmap_sem);
891 out_free_pfns:
892 kvfree(range->pfns);
893 out_free_ranges:
894 kfree(range);
895 out:
896 mmput(mm);
897 return r;
898 }
899
900 /**
901 * amdgpu_ttm_tt_userptr_range_done - stop HMM track the CPU page table change
902 * Check if the pages backing this ttm range have been invalidated
903 *
904 * Returns: true if pages are still valid
905 */
906 bool amdgpu_ttm_tt_get_user_pages_done(struct ttm_tt *ttm)
907 {
908 struct amdgpu_ttm_tt *gtt = (void *)ttm;
909 bool r = false;
910
911 if (!gtt || !gtt->userptr)
912 return false;
913
914 DRM_DEBUG_DRIVER("user_pages_done 0x%llx pages 0x%lx\n",
915 gtt->userptr, ttm->num_pages);
916
917 WARN_ONCE(!gtt->range || !gtt->range->pfns,
918 "No user pages to check\n");
919
920 if (gtt->range) {
921 /*
922 * FIXME: Must always hold notifier_lock for this, and must
923 * not ignore the return code.
924 */
925 r = mmu_interval_read_retry(gtt->range->notifier,
926 gtt->range->notifier_seq);
927 kvfree(gtt->range->pfns);
928 kfree(gtt->range);
929 gtt->range = NULL;
930 }
931
932 return !r;
933 }
934 #endif
935
936 /**
937 * amdgpu_ttm_tt_set_user_pages - Copy pages in, putting old pages as necessary.
938 *
939 * Called by amdgpu_cs_list_validate(). This creates the page list
940 * that backs user memory and will ultimately be mapped into the device
941 * address space.
942 */
943 void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct page **pages)
944 {
945 unsigned long i;
946
947 for (i = 0; i < ttm->num_pages; ++i)
948 ttm->pages[i] = pages ? pages[i] : NULL;
949 }
950
951 /**
952 * amdgpu_ttm_tt_pin_userptr - prepare the sg table with the user pages
953 *
954 * Called by amdgpu_ttm_backend_bind()
955 **/
956 static int amdgpu_ttm_tt_pin_userptr(struct ttm_tt *ttm)
957 {
958 struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
959 struct amdgpu_ttm_tt *gtt = (void *)ttm;
960 unsigned nents;
961 int r;
962
963 int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
964 enum dma_data_direction direction = write ?
965 DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
966
967 /* Allocate an SG array and squash pages into it */
968 r = sg_alloc_table_from_pages(ttm->sg, ttm->pages, ttm->num_pages, 0,
969 ttm->num_pages << PAGE_SHIFT,
970 GFP_KERNEL);
971 if (r)
972 goto release_sg;
973
974 /* Map SG to device */
975 r = -ENOMEM;
976 nents = dma_map_sg(adev->dev, ttm->sg->sgl, ttm->sg->nents, direction);
977 if (nents != ttm->sg->nents)
978 goto release_sg;
979
980 /* convert SG to linear array of pages and dma addresses */
981 drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,
982 gtt->ttm.dma_address, ttm->num_pages);
983
984 return 0;
985
986 release_sg:
987 kfree(ttm->sg);
988 return r;
989 }
990
991 /**
992 * amdgpu_ttm_tt_unpin_userptr - Unpin and unmap userptr pages
993 */
994 static void amdgpu_ttm_tt_unpin_userptr(struct ttm_tt *ttm)
995 {
996 struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
997 struct amdgpu_ttm_tt *gtt = (void *)ttm;
998
999 int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
1000 enum dma_data_direction direction = write ?
1001 DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
1002
1003 /* double check that we don't free the table twice */
1004 if (!ttm->sg->sgl)
1005 return;
1006
1007 /* unmap the pages mapped to the device */
1008 dma_unmap_sg(adev->dev, ttm->sg->sgl, ttm->sg->nents, direction);
1009
1010 sg_free_table(ttm->sg);
1011
1012 #if IS_ENABLED(CONFIG_DRM_AMDGPU_USERPTR)
1013 if (gtt->range) {
1014 unsigned long i;
1015
1016 for (i = 0; i < ttm->num_pages; i++) {
1017 if (ttm->pages[i] !=
1018 hmm_device_entry_to_page(gtt->range,
1019 gtt->range->pfns[i]))
1020 break;
1021 }
1022
1023 WARN((i == ttm->num_pages), "Missing get_user_page_done\n");
1024 }
1025 #endif
1026 }
1027
1028 int amdgpu_ttm_gart_bind(struct amdgpu_device *adev,
1029 struct ttm_buffer_object *tbo,
1030 uint64_t flags)
1031 {
1032 struct amdgpu_bo *abo = ttm_to_amdgpu_bo(tbo);
1033 struct ttm_tt *ttm = tbo->ttm;
1034 struct amdgpu_ttm_tt *gtt = (void *)ttm;
1035 int r;
1036
1037 if (abo->flags & AMDGPU_GEM_CREATE_MQD_GFX9) {
1038 uint64_t page_idx = 1;
1039
1040 r = amdgpu_gart_bind(adev, gtt->offset, page_idx,
1041 ttm->pages, gtt->ttm.dma_address, flags);
1042 if (r)
1043 goto gart_bind_fail;
1044
1045 /* Patch mtype of the second part BO */
1046 flags &= ~AMDGPU_PTE_MTYPE_VG10_MASK;
1047 flags |= AMDGPU_PTE_MTYPE_VG10(AMDGPU_MTYPE_NC);
1048
1049 r = amdgpu_gart_bind(adev,
1050 gtt->offset + (page_idx << PAGE_SHIFT),
1051 ttm->num_pages - page_idx,
1052 &ttm->pages[page_idx],
1053 &(gtt->ttm.dma_address[page_idx]), flags);
1054 } else {
1055 r = amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages,
1056 ttm->pages, gtt->ttm.dma_address, flags);
1057 }
1058
1059 gart_bind_fail:
1060 if (r)
1061 DRM_ERROR("failed to bind %lu pages at 0x%08llX\n",
1062 ttm->num_pages, gtt->offset);
1063
1064 return r;
1065 }
1066
1067 /**
1068 * amdgpu_ttm_backend_bind - Bind GTT memory
1069 *
1070 * Called by ttm_tt_bind() on behalf of ttm_bo_handle_move_mem().
1071 * This handles binding GTT memory to the device address space.
1072 */
1073 static int amdgpu_ttm_backend_bind(struct ttm_tt *ttm,
1074 struct ttm_mem_reg *bo_mem)
1075 {
1076 struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
1077 struct amdgpu_ttm_tt *gtt = (void*)ttm;
1078 uint64_t flags;
1079 int r = 0;
1080
1081 if (gtt->userptr) {
1082 r = amdgpu_ttm_tt_pin_userptr(ttm);
1083 if (r) {
1084 DRM_ERROR("failed to pin userptr\n");
1085 return r;
1086 }
1087 }
1088 if (!ttm->num_pages) {
1089 WARN(1, "nothing to bind %lu pages for mreg %p back %p!\n",
1090 ttm->num_pages, bo_mem, ttm);
1091 }
1092
1093 if (bo_mem->mem_type == AMDGPU_PL_GDS ||
1094 bo_mem->mem_type == AMDGPU_PL_GWS ||
1095 bo_mem->mem_type == AMDGPU_PL_OA)
1096 return -EINVAL;
1097
1098 if (!amdgpu_gtt_mgr_has_gart_addr(bo_mem)) {
1099 gtt->offset = AMDGPU_BO_INVALID_OFFSET;
1100 return 0;
1101 }
1102
1103 /* compute PTE flags relevant to this BO memory */
1104 flags = amdgpu_ttm_tt_pte_flags(adev, ttm, bo_mem);
1105
1106 /* bind pages into GART page tables */
1107 gtt->offset = (u64)bo_mem->start << PAGE_SHIFT;
1108 r = amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages,
1109 ttm->pages, gtt->ttm.dma_address, flags);
1110
1111 if (r)
1112 DRM_ERROR("failed to bind %lu pages at 0x%08llX\n",
1113 ttm->num_pages, gtt->offset);
1114 return r;
1115 }
1116
1117 /**
1118 * amdgpu_ttm_alloc_gart - Allocate GART memory for buffer object
1119 */
1120 int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo)
1121 {
1122 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
1123 struct ttm_operation_ctx ctx = { false, false };
1124 struct amdgpu_ttm_tt *gtt = (void*)bo->ttm;
1125 struct ttm_mem_reg tmp;
1126 struct ttm_placement placement;
1127 struct ttm_place placements;
1128 uint64_t addr, flags;
1129 int r;
1130
1131 if (bo->mem.start != AMDGPU_BO_INVALID_OFFSET)
1132 return 0;
1133
1134 addr = amdgpu_gmc_agp_addr(bo);
1135 if (addr != AMDGPU_BO_INVALID_OFFSET) {
1136 bo->mem.start = addr >> PAGE_SHIFT;
1137 } else {
1138
1139 /* allocate GART space */
1140 tmp = bo->mem;
1141 tmp.mm_node = NULL;
1142 placement.num_placement = 1;
1143 placement.placement = &placements;
1144 placement.num_busy_placement = 1;
1145 placement.busy_placement = &placements;
1146 placements.fpfn = 0;
1147 placements.lpfn = adev->gmc.gart_size >> PAGE_SHIFT;
1148 placements.flags = (bo->mem.placement & ~TTM_PL_MASK_MEM) |
1149 TTM_PL_FLAG_TT;
1150
1151 r = ttm_bo_mem_space(bo, &placement, &tmp, &ctx);
1152 if (unlikely(r))
1153 return r;
1154
1155 /* compute PTE flags for this buffer object */
1156 flags = amdgpu_ttm_tt_pte_flags(adev, bo->ttm, &tmp);
1157
1158 /* Bind pages */
1159 gtt->offset = (u64)tmp.start << PAGE_SHIFT;
1160 r = amdgpu_ttm_gart_bind(adev, bo, flags);
1161 if (unlikely(r)) {
1162 ttm_bo_mem_put(bo, &tmp);
1163 return r;
1164 }
1165
1166 ttm_bo_mem_put(bo, &bo->mem);
1167 bo->mem = tmp;
1168 }
1169
1170 bo->offset = (bo->mem.start << PAGE_SHIFT) +
1171 bo->bdev->man[bo->mem.mem_type].gpu_offset;
1172
1173 return 0;
1174 }
1175
1176 /**
1177 * amdgpu_ttm_recover_gart - Rebind GTT pages
1178 *
1179 * Called by amdgpu_gtt_mgr_recover() from amdgpu_device_reset() to
1180 * rebind GTT pages during a GPU reset.
1181 */
1182 int amdgpu_ttm_recover_gart(struct ttm_buffer_object *tbo)
1183 {
1184 struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev);
1185 uint64_t flags;
1186 int r;
1187
1188 if (!tbo->ttm)
1189 return 0;
1190
1191 flags = amdgpu_ttm_tt_pte_flags(adev, tbo->ttm, &tbo->mem);
1192 r = amdgpu_ttm_gart_bind(adev, tbo, flags);
1193
1194 return r;
1195 }
1196
1197 /**
1198 * amdgpu_ttm_backend_unbind - Unbind GTT mapped pages
1199 *
1200 * Called by ttm_tt_unbind() on behalf of ttm_bo_move_ttm() and
1201 * ttm_tt_destroy().
1202 */
1203 static int amdgpu_ttm_backend_unbind(struct ttm_tt *ttm)
1204 {
1205 struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
1206 struct amdgpu_ttm_tt *gtt = (void *)ttm;
1207 int r;
1208
1209 /* if the pages have userptr pinning then clear that first */
1210 if (gtt->userptr)
1211 amdgpu_ttm_tt_unpin_userptr(ttm);
1212
1213 if (gtt->offset == AMDGPU_BO_INVALID_OFFSET)
1214 return 0;
1215
1216 /* unbind shouldn't be done for GDS/GWS/OA in ttm_bo_clean_mm */
1217 r = amdgpu_gart_unbind(adev, gtt->offset, ttm->num_pages);
1218 if (r)
1219 DRM_ERROR("failed to unbind %lu pages at 0x%08llX\n",
1220 gtt->ttm.ttm.num_pages, gtt->offset);
1221 return r;
1222 }
1223
1224 static void amdgpu_ttm_backend_destroy(struct ttm_tt *ttm)
1225 {
1226 struct amdgpu_ttm_tt *gtt = (void *)ttm;
1227
1228 if (gtt->usertask)
1229 put_task_struct(gtt->usertask);
1230
1231 ttm_dma_tt_fini(&gtt->ttm);
1232 kfree(gtt);
1233 }
1234
1235 static struct ttm_backend_func amdgpu_backend_func = {
1236 .bind = &amdgpu_ttm_backend_bind,
1237 .unbind = &amdgpu_ttm_backend_unbind,
1238 .destroy = &amdgpu_ttm_backend_destroy,
1239 };
1240
1241 /**
1242 * amdgpu_ttm_tt_create - Create a ttm_tt object for a given BO
1243 *
1244 * @bo: The buffer object to create a GTT ttm_tt object around
1245 *
1246 * Called by ttm_tt_create().
1247 */
1248 static struct ttm_tt *amdgpu_ttm_tt_create(struct ttm_buffer_object *bo,
1249 uint32_t page_flags)
1250 {
1251 struct amdgpu_ttm_tt *gtt;
1252
1253 gtt = kzalloc(sizeof(struct amdgpu_ttm_tt), GFP_KERNEL);
1254 if (gtt == NULL) {
1255 return NULL;
1256 }
1257 gtt->ttm.ttm.func = &amdgpu_backend_func;
1258 gtt->gobj = &bo->base;
1259
1260 /* allocate space for the uninitialized page entries */
1261 if (ttm_sg_tt_init(&gtt->ttm, bo, page_flags)) {
1262 kfree(gtt);
1263 return NULL;
1264 }
1265 return &gtt->ttm.ttm;
1266 }
1267
1268 /**
1269 * amdgpu_ttm_tt_populate - Map GTT pages visible to the device
1270 *
1271 * Map the pages of a ttm_tt object to an address space visible
1272 * to the underlying device.
1273 */
1274 static int amdgpu_ttm_tt_populate(struct ttm_tt *ttm,
1275 struct ttm_operation_ctx *ctx)
1276 {
1277 struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
1278 struct amdgpu_ttm_tt *gtt = (void *)ttm;
1279
1280 /* user pages are bound by amdgpu_ttm_tt_pin_userptr() */
1281 if (gtt && gtt->userptr) {
1282 ttm->sg = kzalloc(sizeof(struct sg_table), GFP_KERNEL);
1283 if (!ttm->sg)
1284 return -ENOMEM;
1285
1286 ttm->page_flags |= TTM_PAGE_FLAG_SG;
1287 ttm->state = tt_unbound;
1288 return 0;
1289 }
1290
1291 if (ttm->page_flags & TTM_PAGE_FLAG_SG) {
1292 if (!ttm->sg) {
1293 struct dma_buf_attachment *attach;
1294 struct sg_table *sgt;
1295
1296 attach = gtt->gobj->import_attach;
1297 sgt = dma_buf_map_attachment(attach, DMA_BIDIRECTIONAL);
1298 if (IS_ERR(sgt))
1299 return PTR_ERR(sgt);
1300
1301 ttm->sg = sgt;
1302 }
1303
1304 drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,
1305 gtt->ttm.dma_address,
1306 ttm->num_pages);
1307 ttm->state = tt_unbound;
1308 return 0;
1309 }
1310
1311 #ifdef CONFIG_SWIOTLB
1312 if (adev->need_swiotlb && swiotlb_nr_tbl()) {
1313 return ttm_dma_populate(&gtt->ttm, adev->dev, ctx);
1314 }
1315 #endif
1316
1317 /* fall back to generic helper to populate the page array
1318 * and map them to the device */
1319 return ttm_populate_and_map_pages(adev->dev, &gtt->ttm, ctx);
1320 }
1321
1322 /**
1323 * amdgpu_ttm_tt_unpopulate - unmap GTT pages and unpopulate page arrays
1324 *
1325 * Unmaps pages of a ttm_tt object from the device address space and
1326 * unpopulates the page array backing it.
1327 */
1328 static void amdgpu_ttm_tt_unpopulate(struct ttm_tt *ttm)
1329 {
1330 struct amdgpu_ttm_tt *gtt = (void *)ttm;
1331 struct amdgpu_device *adev;
1332
1333 if (gtt && gtt->userptr) {
1334 amdgpu_ttm_tt_set_user_pages(ttm, NULL);
1335 kfree(ttm->sg);
1336 ttm->page_flags &= ~TTM_PAGE_FLAG_SG;
1337 return;
1338 }
1339
1340 if (ttm->sg && gtt->gobj->import_attach) {
1341 struct dma_buf_attachment *attach;
1342
1343 attach = gtt->gobj->import_attach;
1344 dma_buf_unmap_attachment(attach, ttm->sg, DMA_BIDIRECTIONAL);
1345 ttm->sg = NULL;
1346 return;
1347 }
1348
1349 if (ttm->page_flags & TTM_PAGE_FLAG_SG)
1350 return;
1351
1352 adev = amdgpu_ttm_adev(ttm->bdev);
1353
1354 #ifdef CONFIG_SWIOTLB
1355 if (adev->need_swiotlb && swiotlb_nr_tbl()) {
1356 ttm_dma_unpopulate(&gtt->ttm, adev->dev);
1357 return;
1358 }
1359 #endif
1360
1361 /* fall back to generic helper to unmap and unpopulate array */
1362 ttm_unmap_and_unpopulate_pages(adev->dev, &gtt->ttm);
1363 }
1364
1365 /**
1366 * amdgpu_ttm_tt_set_userptr - Initialize userptr GTT ttm_tt for the current
1367 * task
1368 *
1369 * @ttm: The ttm_tt object to bind this userptr object to
1370 * @addr: The address in the current tasks VM space to use
1371 * @flags: Requirements of userptr object.
1372 *
1373 * Called by amdgpu_gem_userptr_ioctl() to bind userptr pages
1374 * to current task
1375 */
1376 int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr,
1377 uint32_t flags)
1378 {
1379 struct amdgpu_ttm_tt *gtt = (void *)ttm;
1380
1381 if (gtt == NULL)
1382 return -EINVAL;
1383
1384 gtt->userptr = addr;
1385 gtt->userflags = flags;
1386
1387 if (gtt->usertask)
1388 put_task_struct(gtt->usertask);
1389 gtt->usertask = current->group_leader;
1390 get_task_struct(gtt->usertask);
1391
1392 return 0;
1393 }
1394
1395 /**
1396 * amdgpu_ttm_tt_get_usermm - Return memory manager for ttm_tt object
1397 */
1398 struct mm_struct *amdgpu_ttm_tt_get_usermm(struct ttm_tt *ttm)
1399 {
1400 struct amdgpu_ttm_tt *gtt = (void *)ttm;
1401
1402 if (gtt == NULL)
1403 return NULL;
1404
1405 if (gtt->usertask == NULL)
1406 return NULL;
1407
1408 return gtt->usertask->mm;
1409 }
1410
1411 /**
1412 * amdgpu_ttm_tt_affect_userptr - Determine if a ttm_tt object lays inside an
1413 * address range for the current task.
1414 *
1415 */
1416 bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,
1417 unsigned long end)
1418 {
1419 struct amdgpu_ttm_tt *gtt = (void *)ttm;
1420 unsigned long size;
1421
1422 if (gtt == NULL || !gtt->userptr)
1423 return false;
1424
1425 /* Return false if no part of the ttm_tt object lies within
1426 * the range
1427 */
1428 size = (unsigned long)gtt->ttm.ttm.num_pages * PAGE_SIZE;
1429 if (gtt->userptr > end || gtt->userptr + size <= start)
1430 return false;
1431
1432 return true;
1433 }
1434
1435 /**
1436 * amdgpu_ttm_tt_is_userptr - Have the pages backing by userptr?
1437 */
1438 bool amdgpu_ttm_tt_is_userptr(struct ttm_tt *ttm)
1439 {
1440 struct amdgpu_ttm_tt *gtt = (void *)ttm;
1441
1442 if (gtt == NULL || !gtt->userptr)
1443 return false;
1444
1445 return true;
1446 }
1447
1448 /**
1449 * amdgpu_ttm_tt_is_readonly - Is the ttm_tt object read only?
1450 */
1451 bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm)
1452 {
1453 struct amdgpu_ttm_tt *gtt = (void *)ttm;
1454
1455 if (gtt == NULL)
1456 return false;
1457
1458 return !!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
1459 }
1460
1461 /**
1462 * amdgpu_ttm_tt_pde_flags - Compute PDE flags for ttm_tt object
1463 *
1464 * @ttm: The ttm_tt object to compute the flags for
1465 * @mem: The memory registry backing this ttm_tt object
1466 *
1467 * Figure out the flags to use for a VM PDE (Page Directory Entry).
1468 */
1469 uint64_t amdgpu_ttm_tt_pde_flags(struct ttm_tt *ttm, struct ttm_mem_reg *mem)
1470 {
1471 uint64_t flags = 0;
1472
1473 if (mem && mem->mem_type != TTM_PL_SYSTEM)
1474 flags |= AMDGPU_PTE_VALID;
1475
1476 if (mem && mem->mem_type == TTM_PL_TT) {
1477 flags |= AMDGPU_PTE_SYSTEM;
1478
1479 if (ttm->caching_state == tt_cached)
1480 flags |= AMDGPU_PTE_SNOOPED;
1481 }
1482
1483 return flags;
1484 }
1485
1486 /**
1487 * amdgpu_ttm_tt_pte_flags - Compute PTE flags for ttm_tt object
1488 *
1489 * @ttm: The ttm_tt object to compute the flags for
1490 * @mem: The memory registry backing this ttm_tt object
1491
1492 * Figure out the flags to use for a VM PTE (Page Table Entry).
1493 */
1494 uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,
1495 struct ttm_mem_reg *mem)
1496 {
1497 uint64_t flags = amdgpu_ttm_tt_pde_flags(ttm, mem);
1498
1499 flags |= adev->gart.gart_pte_flags;
1500 flags |= AMDGPU_PTE_READABLE;
1501
1502 if (!amdgpu_ttm_tt_is_readonly(ttm))
1503 flags |= AMDGPU_PTE_WRITEABLE;
1504
1505 return flags;
1506 }
1507
1508 /**
1509 * amdgpu_ttm_bo_eviction_valuable - Check to see if we can evict a buffer
1510 * object.
1511 *
1512 * Return true if eviction is sensible. Called by ttm_mem_evict_first() on
1513 * behalf of ttm_bo_mem_force_space() which tries to evict buffer objects until
1514 * it can find space for a new object and by ttm_bo_force_list_clean() which is
1515 * used to clean out a memory space.
1516 */
1517 static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
1518 const struct ttm_place *place)
1519 {
1520 unsigned long num_pages = bo->mem.num_pages;
1521 struct drm_mm_node *node = bo->mem.mm_node;
1522 struct dma_resv_list *flist;
1523 struct dma_fence *f;
1524 int i;
1525
1526 if (bo->type == ttm_bo_type_kernel &&
1527 !amdgpu_vm_evictable(ttm_to_amdgpu_bo(bo)))
1528 return false;
1529
1530 /* If bo is a KFD BO, check if the bo belongs to the current process.
1531 * If true, then return false as any KFD process needs all its BOs to
1532 * be resident to run successfully
1533 */
1534 flist = dma_resv_get_list(bo->base.resv);
1535 if (flist) {
1536 for (i = 0; i < flist->shared_count; ++i) {
1537 f = rcu_dereference_protected(flist->shared[i],
1538 dma_resv_held(bo->base.resv));
1539 if (amdkfd_fence_check_mm(f, current->mm))
1540 return false;
1541 }
1542 }
1543
1544 switch (bo->mem.mem_type) {
1545 case TTM_PL_TT:
1546 return true;
1547
1548 case TTM_PL_VRAM:
1549 /* Check each drm MM node individually */
1550 while (num_pages) {
1551 if (place->fpfn < (node->start + node->size) &&
1552 !(place->lpfn && place->lpfn <= node->start))
1553 return true;
1554
1555 num_pages -= node->size;
1556 ++node;
1557 }
1558 return false;
1559
1560 default:
1561 break;
1562 }
1563
1564 return ttm_bo_eviction_valuable(bo, place);
1565 }
1566
1567 /**
1568 * amdgpu_ttm_access_memory - Read or Write memory that backs a buffer object.
1569 *
1570 * @bo: The buffer object to read/write
1571 * @offset: Offset into buffer object
1572 * @buf: Secondary buffer to write/read from
1573 * @len: Length in bytes of access
1574 * @write: true if writing
1575 *
1576 * This is used to access VRAM that backs a buffer object via MMIO
1577 * access for debugging purposes.
1578 */
1579 static int amdgpu_ttm_access_memory(struct ttm_buffer_object *bo,
1580 unsigned long offset,
1581 void *buf, int len, int write)
1582 {
1583 struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
1584 struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
1585 struct drm_mm_node *nodes;
1586 uint32_t value = 0;
1587 int ret = 0;
1588 uint64_t pos;
1589 unsigned long flags;
1590
1591 if (bo->mem.mem_type != TTM_PL_VRAM)
1592 return -EIO;
1593
1594 nodes = amdgpu_find_mm_node(&abo->tbo.mem, &offset);
1595 pos = (nodes->start << PAGE_SHIFT) + offset;
1596
1597 while (len && pos < adev->gmc.mc_vram_size) {
1598 uint64_t aligned_pos = pos & ~(uint64_t)3;
1599 uint32_t bytes = 4 - (pos & 3);
1600 uint32_t shift = (pos & 3) * 8;
1601 uint32_t mask = 0xffffffff << shift;
1602
1603 if (len < bytes) {
1604 mask &= 0xffffffff >> (bytes - len) * 8;
1605 bytes = len;
1606 }
1607
1608 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
1609 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)aligned_pos) | 0x80000000);
1610 WREG32_NO_KIQ(mmMM_INDEX_HI, aligned_pos >> 31);
1611 if (!write || mask != 0xffffffff)
1612 value = RREG32_NO_KIQ(mmMM_DATA);
1613 if (write) {
1614 value &= ~mask;
1615 value |= (*(uint32_t *)buf << shift) & mask;
1616 WREG32_NO_KIQ(mmMM_DATA, value);
1617 }
1618 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
1619 if (!write) {
1620 value = (value & mask) >> shift;
1621 memcpy(buf, &value, bytes);
1622 }
1623
1624 ret += bytes;
1625 buf = (uint8_t *)buf + bytes;
1626 pos += bytes;
1627 len -= bytes;
1628 if (pos >= (nodes->start + nodes->size) << PAGE_SHIFT) {
1629 ++nodes;
1630 pos = (nodes->start << PAGE_SHIFT);
1631 }
1632 }
1633
1634 return ret;
1635 }
1636
1637 static struct ttm_bo_driver amdgpu_bo_driver = {
1638 .ttm_tt_create = &amdgpu_ttm_tt_create,
1639 .ttm_tt_populate = &amdgpu_ttm_tt_populate,
1640 .ttm_tt_unpopulate = &amdgpu_ttm_tt_unpopulate,
1641 .invalidate_caches = &amdgpu_invalidate_caches,
1642 .init_mem_type = &amdgpu_init_mem_type,
1643 .eviction_valuable = amdgpu_ttm_bo_eviction_valuable,
1644 .evict_flags = &amdgpu_evict_flags,
1645 .move = &amdgpu_bo_move,
1646 .verify_access = &amdgpu_verify_access,
1647 .move_notify = &amdgpu_bo_move_notify,
1648 .release_notify = &amdgpu_bo_release_notify,
1649 .fault_reserve_notify = &amdgpu_bo_fault_reserve_notify,
1650 .io_mem_reserve = &amdgpu_ttm_io_mem_reserve,
1651 .io_mem_free = &amdgpu_ttm_io_mem_free,
1652 .io_mem_pfn = amdgpu_ttm_io_mem_pfn,
1653 .access_memory = &amdgpu_ttm_access_memory,
1654 .del_from_lru_notify = &amdgpu_vm_del_from_lru_notify
1655 };
1656
1657 /*
1658 * Firmware Reservation functions
1659 */
1660 /**
1661 * amdgpu_ttm_fw_reserve_vram_fini - free fw reserved vram
1662 *
1663 * @adev: amdgpu_device pointer
1664 *
1665 * free fw reserved vram if it has been reserved.
1666 */
1667 static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev)
1668 {
1669 amdgpu_bo_free_kernel(&adev->fw_vram_usage.reserved_bo,
1670 NULL, &adev->fw_vram_usage.va);
1671 }
1672
1673 /**
1674 * amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
1675 *
1676 * @adev: amdgpu_device pointer
1677 *
1678 * create bo vram reservation from fw.
1679 */
1680 static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
1681 {
1682 uint64_t vram_size = adev->gmc.visible_vram_size;
1683
1684 adev->fw_vram_usage.va = NULL;
1685 adev->fw_vram_usage.reserved_bo = NULL;
1686
1687 if (adev->fw_vram_usage.size == 0 ||
1688 adev->fw_vram_usage.size > vram_size)
1689 return 0;
1690
1691 return amdgpu_bo_create_kernel_at(adev,
1692 adev->fw_vram_usage.start_offset,
1693 adev->fw_vram_usage.size,
1694 AMDGPU_GEM_DOMAIN_VRAM,
1695 &adev->fw_vram_usage.reserved_bo,
1696 &adev->fw_vram_usage.va);
1697 }
1698
1699 /*
1700 * Memoy training reservation functions
1701 */
1702
1703 /**
1704 * amdgpu_ttm_training_reserve_vram_fini - free memory training reserved vram
1705 *
1706 * @adev: amdgpu_device pointer
1707 *
1708 * free memory training reserved vram if it has been reserved.
1709 */
1710 static int amdgpu_ttm_training_reserve_vram_fini(struct amdgpu_device *adev)
1711 {
1712 struct psp_memory_training_context *ctx = &adev->psp.mem_train_ctx;
1713
1714 ctx->init = PSP_MEM_TRAIN_NOT_SUPPORT;
1715 amdgpu_bo_free_kernel(&ctx->c2p_bo, NULL, NULL);
1716 ctx->c2p_bo = NULL;
1717
1718 return 0;
1719 }
1720
1721 static u64 amdgpu_ttm_training_get_c2p_offset(u64 vram_size)
1722 {
1723 if ((vram_size & (SZ_1M - 1)) < (SZ_4K + 1) )
1724 vram_size -= SZ_1M;
1725
1726 return ALIGN(vram_size, SZ_1M);
1727 }
1728
1729 /**
1730 * amdgpu_ttm_training_reserve_vram_init - create bo vram reservation from memory training
1731 *
1732 * @adev: amdgpu_device pointer
1733 *
1734 * create bo vram reservation from memory training.
1735 */
1736 static int amdgpu_ttm_training_reserve_vram_init(struct amdgpu_device *adev)
1737 {
1738 int ret;
1739 struct psp_memory_training_context *ctx = &adev->psp.mem_train_ctx;
1740
1741 memset(ctx, 0, sizeof(*ctx));
1742 if (!adev->fw_vram_usage.mem_train_support) {
1743 DRM_DEBUG("memory training does not support!\n");
1744 return 0;
1745 }
1746
1747 ctx->c2p_train_data_offset = amdgpu_ttm_training_get_c2p_offset(adev->gmc.mc_vram_size);
1748 ctx->p2c_train_data_offset = (adev->gmc.mc_vram_size - GDDR6_MEM_TRAINING_OFFSET);
1749 ctx->train_data_size = GDDR6_MEM_TRAINING_DATA_SIZE_IN_BYTES;
1750
1751 DRM_DEBUG("train_data_size:%llx,p2c_train_data_offset:%llx,c2p_train_data_offset:%llx.\n",
1752 ctx->train_data_size,
1753 ctx->p2c_train_data_offset,
1754 ctx->c2p_train_data_offset);
1755
1756 ret = amdgpu_bo_create_kernel_at(adev,
1757 ctx->c2p_train_data_offset,
1758 ctx->train_data_size,
1759 AMDGPU_GEM_DOMAIN_VRAM,
1760 &ctx->c2p_bo,
1761 NULL);
1762 if (ret) {
1763 DRM_ERROR("alloc c2p_bo failed(%d)!\n", ret);
1764 amdgpu_ttm_training_reserve_vram_fini(adev);
1765 return ret;
1766 }
1767
1768 ctx->init = PSP_MEM_TRAIN_RESERVE_SUCCESS;
1769 return 0;
1770 }
1771
1772 /**
1773 * amdgpu_ttm_init - Init the memory management (ttm) as well as various
1774 * gtt/vram related fields.
1775 *
1776 * This initializes all of the memory space pools that the TTM layer
1777 * will need such as the GTT space (system memory mapped to the device),
1778 * VRAM (on-board memory), and on-chip memories (GDS, GWS, OA) which
1779 * can be mapped per VMID.
1780 */
1781 int amdgpu_ttm_init(struct amdgpu_device *adev)
1782 {
1783 uint64_t gtt_size;
1784 int r;
1785 u64 vis_vram_limit;
1786 void *stolen_vga_buf;
1787
1788 mutex_init(&adev->mman.gtt_window_lock);
1789
1790 /* No others user of address space so set it to 0 */
1791 r = ttm_bo_device_init(&adev->mman.bdev,
1792 &amdgpu_bo_driver,
1793 adev->ddev->anon_inode->i_mapping,
1794 adev->ddev->vma_offset_manager,
1795 dma_addressing_limited(adev->dev));
1796 if (r) {
1797 DRM_ERROR("failed initializing buffer object driver(%d).\n", r);
1798 return r;
1799 }
1800 adev->mman.initialized = true;
1801
1802 /* We opt to avoid OOM on system pages allocations */
1803 adev->mman.bdev.no_retry = true;
1804
1805 /* Initialize VRAM pool with all of VRAM divided into pages */
1806 r = ttm_bo_init_mm(&adev->mman.bdev, TTM_PL_VRAM,
1807 adev->gmc.real_vram_size >> PAGE_SHIFT);
1808 if (r) {
1809 DRM_ERROR("Failed initializing VRAM heap.\n");
1810 return r;
1811 }
1812
1813 /* Reduce size of CPU-visible VRAM if requested */
1814 vis_vram_limit = (u64)amdgpu_vis_vram_limit * 1024 * 1024;
1815 if (amdgpu_vis_vram_limit > 0 &&
1816 vis_vram_limit <= adev->gmc.visible_vram_size)
1817 adev->gmc.visible_vram_size = vis_vram_limit;
1818
1819 /* Change the size here instead of the init above so only lpfn is affected */
1820 amdgpu_ttm_set_buffer_funcs_status(adev, false);
1821 #ifdef CONFIG_64BIT
1822 adev->mman.aper_base_kaddr = ioremap_wc(adev->gmc.aper_base,
1823 adev->gmc.visible_vram_size);
1824 #endif
1825
1826 /*
1827 *The reserved vram for firmware must be pinned to the specified
1828 *place on the VRAM, so reserve it early.
1829 */
1830 r = amdgpu_ttm_fw_reserve_vram_init(adev);
1831 if (r) {
1832 return r;
1833 }
1834
1835 /*
1836 *The reserved vram for memory training must be pinned to the specified
1837 *place on the VRAM, so reserve it early.
1838 */
1839 r = amdgpu_ttm_training_reserve_vram_init(adev);
1840 if (r)
1841 return r;
1842
1843 /* allocate memory as required for VGA
1844 * This is used for VGA emulation and pre-OS scanout buffers to
1845 * avoid display artifacts while transitioning between pre-OS
1846 * and driver. */
1847 r = amdgpu_bo_create_kernel(adev, adev->gmc.stolen_size, PAGE_SIZE,
1848 AMDGPU_GEM_DOMAIN_VRAM,
1849 &adev->stolen_vga_memory,
1850 NULL, &stolen_vga_buf);
1851 if (r)
1852 return r;
1853
1854 /*
1855 * reserve one TMR (64K) memory at the top of VRAM which holds
1856 * IP Discovery data and is protected by PSP.
1857 */
1858 r = amdgpu_bo_create_kernel_at(adev,
1859 adev->gmc.real_vram_size - DISCOVERY_TMR_SIZE,
1860 DISCOVERY_TMR_SIZE,
1861 AMDGPU_GEM_DOMAIN_VRAM,
1862 &adev->discovery_memory,
1863 NULL);
1864 if (r)
1865 return r;
1866
1867 DRM_INFO("amdgpu: %uM of VRAM memory ready\n",
1868 (unsigned) (adev->gmc.real_vram_size / (1024 * 1024)));
1869
1870 /* Compute GTT size, either bsaed on 3/4th the size of RAM size
1871 * or whatever the user passed on module init */
1872 if (amdgpu_gtt_size == -1) {
1873 struct sysinfo si;
1874
1875 si_meminfo(&si);
1876 gtt_size = min(max((AMDGPU_DEFAULT_GTT_SIZE_MB << 20),
1877 adev->gmc.mc_vram_size),
1878 ((uint64_t)si.totalram * si.mem_unit * 3/4));
1879 }
1880 else
1881 gtt_size = (uint64_t)amdgpu_gtt_size << 20;
1882
1883 /* Initialize GTT memory pool */
1884 r = ttm_bo_init_mm(&adev->mman.bdev, TTM_PL_TT, gtt_size >> PAGE_SHIFT);
1885 if (r) {
1886 DRM_ERROR("Failed initializing GTT heap.\n");
1887 return r;
1888 }
1889 DRM_INFO("amdgpu: %uM of GTT memory ready.\n",
1890 (unsigned)(gtt_size / (1024 * 1024)));
1891
1892 /* Initialize various on-chip memory pools */
1893 r = ttm_bo_init_mm(&adev->mman.bdev, AMDGPU_PL_GDS,
1894 adev->gds.gds_size);
1895 if (r) {
1896 DRM_ERROR("Failed initializing GDS heap.\n");
1897 return r;
1898 }
1899
1900 r = ttm_bo_init_mm(&adev->mman.bdev, AMDGPU_PL_GWS,
1901 adev->gds.gws_size);
1902 if (r) {
1903 DRM_ERROR("Failed initializing gws heap.\n");
1904 return r;
1905 }
1906
1907 r = ttm_bo_init_mm(&adev->mman.bdev, AMDGPU_PL_OA,
1908 adev->gds.oa_size);
1909 if (r) {
1910 DRM_ERROR("Failed initializing oa heap.\n");
1911 return r;
1912 }
1913
1914 /* Register debugfs entries for amdgpu_ttm */
1915 r = amdgpu_ttm_debugfs_init(adev);
1916 if (r) {
1917 DRM_ERROR("Failed to init debugfs\n");
1918 return r;
1919 }
1920 return 0;
1921 }
1922
1923 /**
1924 * amdgpu_ttm_late_init - Handle any late initialization for amdgpu_ttm
1925 */
1926 void amdgpu_ttm_late_init(struct amdgpu_device *adev)
1927 {
1928 void *stolen_vga_buf;
1929 /* return the VGA stolen memory (if any) back to VRAM */
1930 amdgpu_bo_free_kernel(&adev->stolen_vga_memory, NULL, &stolen_vga_buf);
1931 }
1932
1933 /**
1934 * amdgpu_ttm_fini - De-initialize the TTM memory pools
1935 */
1936 void amdgpu_ttm_fini(struct amdgpu_device *adev)
1937 {
1938 if (!adev->mman.initialized)
1939 return;
1940
1941 amdgpu_ttm_debugfs_fini(adev);
1942 amdgpu_ttm_training_reserve_vram_fini(adev);
1943 /* return the IP Discovery TMR memory back to VRAM */
1944 amdgpu_bo_free_kernel(&adev->discovery_memory, NULL, NULL);
1945 amdgpu_ttm_fw_reserve_vram_fini(adev);
1946
1947 if (adev->mman.aper_base_kaddr)
1948 iounmap(adev->mman.aper_base_kaddr);
1949 adev->mman.aper_base_kaddr = NULL;
1950
1951 ttm_bo_clean_mm(&adev->mman.bdev, TTM_PL_VRAM);
1952 ttm_bo_clean_mm(&adev->mman.bdev, TTM_PL_TT);
1953 ttm_bo_clean_mm(&adev->mman.bdev, AMDGPU_PL_GDS);
1954 ttm_bo_clean_mm(&adev->mman.bdev, AMDGPU_PL_GWS);
1955 ttm_bo_clean_mm(&adev->mman.bdev, AMDGPU_PL_OA);
1956 ttm_bo_device_release(&adev->mman.bdev);
1957 adev->mman.initialized = false;
1958 DRM_INFO("amdgpu: ttm finalized\n");
1959 }
1960
1961 /**
1962 * amdgpu_ttm_set_buffer_funcs_status - enable/disable use of buffer functions
1963 *
1964 * @adev: amdgpu_device pointer
1965 * @enable: true when we can use buffer functions.
1966 *
1967 * Enable/disable use of buffer functions during suspend/resume. This should
1968 * only be called at bootup or when userspace isn't running.
1969 */
1970 void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device *adev, bool enable)
1971 {
1972 struct ttm_mem_type_manager *man = &adev->mman.bdev.man[TTM_PL_VRAM];
1973 uint64_t size;
1974 int r;
1975
1976 if (!adev->mman.initialized || adev->in_gpu_reset ||
1977 adev->mman.buffer_funcs_enabled == enable)
1978 return;
1979
1980 if (enable) {
1981 struct amdgpu_ring *ring;
1982 struct drm_gpu_scheduler *sched;
1983
1984 ring = adev->mman.buffer_funcs_ring;
1985 sched = &ring->sched;
1986 r = drm_sched_entity_init(&adev->mman.entity,
1987 DRM_SCHED_PRIORITY_KERNEL, &sched,
1988 1, NULL);
1989 if (r) {
1990 DRM_ERROR("Failed setting up TTM BO move entity (%d)\n",
1991 r);
1992 return;
1993 }
1994 } else {
1995 drm_sched_entity_destroy(&adev->mman.entity);
1996 dma_fence_put(man->move);
1997 man->move = NULL;
1998 }
1999
2000 /* this just adjusts TTM size idea, which sets lpfn to the correct value */
2001 if (enable)
2002 size = adev->gmc.real_vram_size;
2003 else
2004 size = adev->gmc.visible_vram_size;
2005 man->size = size >> PAGE_SHIFT;
2006 adev->mman.buffer_funcs_enabled = enable;
2007 }
2008
2009 int amdgpu_mmap(struct file *filp, struct vm_area_struct *vma)
2010 {
2011 struct drm_file *file_priv = filp->private_data;
2012 struct amdgpu_device *adev = file_priv->minor->dev->dev_private;
2013
2014 if (adev == NULL)
2015 return -EINVAL;
2016
2017 return ttm_bo_mmap(filp, vma, &adev->mman.bdev);
2018 }
2019
2020 static int amdgpu_map_buffer(struct ttm_buffer_object *bo,
2021 struct ttm_mem_reg *mem, unsigned num_pages,
2022 uint64_t offset, unsigned window,
2023 struct amdgpu_ring *ring,
2024 uint64_t *addr)
2025 {
2026 struct amdgpu_ttm_tt *gtt = (void *)bo->ttm;
2027 struct amdgpu_device *adev = ring->adev;
2028 struct ttm_tt *ttm = bo->ttm;
2029 struct amdgpu_job *job;
2030 unsigned num_dw, num_bytes;
2031 dma_addr_t *dma_address;
2032 struct dma_fence *fence;
2033 uint64_t src_addr, dst_addr;
2034 uint64_t flags;
2035 int r;
2036
2037 BUG_ON(adev->mman.buffer_funcs->copy_max_bytes <
2038 AMDGPU_GTT_MAX_TRANSFER_SIZE * 8);
2039
2040 *addr = adev->gmc.gart_start;
2041 *addr += (u64)window * AMDGPU_GTT_MAX_TRANSFER_SIZE *
2042 AMDGPU_GPU_PAGE_SIZE;
2043
2044 num_dw = ALIGN(adev->mman.buffer_funcs->copy_num_dw, 8);
2045 num_bytes = num_pages * 8;
2046
2047 r = amdgpu_job_alloc_with_ib(adev, num_dw * 4 + num_bytes, &job);
2048 if (r)
2049 return r;
2050
2051 src_addr = num_dw * 4;
2052 src_addr += job->ibs[0].gpu_addr;
2053
2054 dst_addr = amdgpu_bo_gpu_offset(adev->gart.bo);
2055 dst_addr += window * AMDGPU_GTT_MAX_TRANSFER_SIZE * 8;
2056 amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_addr,
2057 dst_addr, num_bytes);
2058
2059 amdgpu_ring_pad_ib(ring, &job->ibs[0]);
2060 WARN_ON(job->ibs[0].length_dw > num_dw);
2061
2062 dma_address = &gtt->ttm.dma_address[offset >> PAGE_SHIFT];
2063 flags = amdgpu_ttm_tt_pte_flags(adev, ttm, mem);
2064 r = amdgpu_gart_map(adev, 0, num_pages, dma_address, flags,
2065 &job->ibs[0].ptr[num_dw]);
2066 if (r)
2067 goto error_free;
2068
2069 r = amdgpu_job_submit(job, &adev->mman.entity,
2070 AMDGPU_FENCE_OWNER_UNDEFINED, &fence);
2071 if (r)
2072 goto error_free;
2073
2074 dma_fence_put(fence);
2075
2076 return r;
2077
2078 error_free:
2079 amdgpu_job_free(job);
2080 return r;
2081 }
2082
2083 int amdgpu_copy_buffer(struct amdgpu_ring *ring, uint64_t src_offset,
2084 uint64_t dst_offset, uint32_t byte_count,
2085 struct dma_resv *resv,
2086 struct dma_fence **fence, bool direct_submit,
2087 bool vm_needs_flush)
2088 {
2089 struct amdgpu_device *adev = ring->adev;
2090 struct amdgpu_job *job;
2091
2092 uint32_t max_bytes;
2093 unsigned num_loops, num_dw;
2094 unsigned i;
2095 int r;
2096
2097 if (direct_submit && !ring->sched.ready) {
2098 DRM_ERROR("Trying to move memory with ring turned off.\n");
2099 return -EINVAL;
2100 }
2101
2102 max_bytes = adev->mman.buffer_funcs->copy_max_bytes;
2103 num_loops = DIV_ROUND_UP(byte_count, max_bytes);
2104 num_dw = ALIGN(num_loops * adev->mman.buffer_funcs->copy_num_dw, 8);
2105
2106 r = amdgpu_job_alloc_with_ib(adev, num_dw * 4, &job);
2107 if (r)
2108 return r;
2109
2110 if (vm_needs_flush) {
2111 job->vm_pd_addr = amdgpu_gmc_pd_addr(adev->gart.bo);
2112 job->vm_needs_flush = true;
2113 }
2114 if (resv) {
2115 r = amdgpu_sync_resv(adev, &job->sync, resv,
2116 AMDGPU_FENCE_OWNER_UNDEFINED,
2117 false);
2118 if (r) {
2119 DRM_ERROR("sync failed (%d).\n", r);
2120 goto error_free;
2121 }
2122 }
2123
2124 for (i = 0; i < num_loops; i++) {
2125 uint32_t cur_size_in_bytes = min(byte_count, max_bytes);
2126
2127 amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_offset,
2128 dst_offset, cur_size_in_bytes);
2129
2130 src_offset += cur_size_in_bytes;
2131 dst_offset += cur_size_in_bytes;
2132 byte_count -= cur_size_in_bytes;
2133 }
2134
2135 amdgpu_ring_pad_ib(ring, &job->ibs[0]);
2136 WARN_ON(job->ibs[0].length_dw > num_dw);
2137 if (direct_submit)
2138 r = amdgpu_job_submit_direct(job, ring, fence);
2139 else
2140 r = amdgpu_job_submit(job, &adev->mman.entity,
2141 AMDGPU_FENCE_OWNER_UNDEFINED, fence);
2142 if (r)
2143 goto error_free;
2144
2145 return r;
2146
2147 error_free:
2148 amdgpu_job_free(job);
2149 DRM_ERROR("Error scheduling IBs (%d)\n", r);
2150 return r;
2151 }
2152
2153 int amdgpu_fill_buffer(struct amdgpu_bo *bo,
2154 uint32_t src_data,
2155 struct dma_resv *resv,
2156 struct dma_fence **fence)
2157 {
2158 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
2159 uint32_t max_bytes = adev->mman.buffer_funcs->fill_max_bytes;
2160 struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
2161
2162 struct drm_mm_node *mm_node;
2163 unsigned long num_pages;
2164 unsigned int num_loops, num_dw;
2165
2166 struct amdgpu_job *job;
2167 int r;
2168
2169 if (!adev->mman.buffer_funcs_enabled) {
2170 DRM_ERROR("Trying to clear memory with ring turned off.\n");
2171 return -EINVAL;
2172 }
2173
2174 if (bo->tbo.mem.mem_type == TTM_PL_TT) {
2175 r = amdgpu_ttm_alloc_gart(&bo->tbo);
2176 if (r)
2177 return r;
2178 }
2179
2180 num_pages = bo->tbo.num_pages;
2181 mm_node = bo->tbo.mem.mm_node;
2182 num_loops = 0;
2183 while (num_pages) {
2184 uint64_t byte_count = mm_node->size << PAGE_SHIFT;
2185
2186 num_loops += DIV_ROUND_UP_ULL(byte_count, max_bytes);
2187 num_pages -= mm_node->size;
2188 ++mm_node;
2189 }
2190 num_dw = num_loops * adev->mman.buffer_funcs->fill_num_dw;
2191
2192 /* for IB padding */
2193 num_dw += 64;
2194
2195 r = amdgpu_job_alloc_with_ib(adev, num_dw * 4, &job);
2196 if (r)
2197 return r;
2198
2199 if (resv) {
2200 r = amdgpu_sync_resv(adev, &job->sync, resv,
2201 AMDGPU_FENCE_OWNER_UNDEFINED, false);
2202 if (r) {
2203 DRM_ERROR("sync failed (%d).\n", r);
2204 goto error_free;
2205 }
2206 }
2207
2208 num_pages = bo->tbo.num_pages;
2209 mm_node = bo->tbo.mem.mm_node;
2210
2211 while (num_pages) {
2212 uint64_t byte_count = mm_node->size << PAGE_SHIFT;
2213 uint64_t dst_addr;
2214
2215 dst_addr = amdgpu_mm_node_addr(&bo->tbo, mm_node, &bo->tbo.mem);
2216 while (byte_count) {
2217 uint32_t cur_size_in_bytes = min_t(uint64_t, byte_count,
2218 max_bytes);
2219
2220 amdgpu_emit_fill_buffer(adev, &job->ibs[0], src_data,
2221 dst_addr, cur_size_in_bytes);
2222
2223 dst_addr += cur_size_in_bytes;
2224 byte_count -= cur_size_in_bytes;
2225 }
2226
2227 num_pages -= mm_node->size;
2228 ++mm_node;
2229 }
2230
2231 amdgpu_ring_pad_ib(ring, &job->ibs[0]);
2232 WARN_ON(job->ibs[0].length_dw > num_dw);
2233 r = amdgpu_job_submit(job, &adev->mman.entity,
2234 AMDGPU_FENCE_OWNER_UNDEFINED, fence);
2235 if (r)
2236 goto error_free;
2237
2238 return 0;
2239
2240 error_free:
2241 amdgpu_job_free(job);
2242 return r;
2243 }
2244
2245 #if defined(CONFIG_DEBUG_FS)
2246
2247 static int amdgpu_mm_dump_table(struct seq_file *m, void *data)
2248 {
2249 struct drm_info_node *node = (struct drm_info_node *)m->private;
2250 unsigned ttm_pl = (uintptr_t)node->info_ent->data;
2251 struct drm_device *dev = node->minor->dev;
2252 struct amdgpu_device *adev = dev->dev_private;
2253 struct ttm_mem_type_manager *man = &adev->mman.bdev.man[ttm_pl];
2254 struct drm_printer p = drm_seq_file_printer(m);
2255
2256 man->func->debug(man, &p);
2257 return 0;
2258 }
2259
2260 static const struct drm_info_list amdgpu_ttm_debugfs_list[] = {
2261 {"amdgpu_vram_mm", amdgpu_mm_dump_table, 0, (void *)TTM_PL_VRAM},
2262 {"amdgpu_gtt_mm", amdgpu_mm_dump_table, 0, (void *)TTM_PL_TT},
2263 {"amdgpu_gds_mm", amdgpu_mm_dump_table, 0, (void *)AMDGPU_PL_GDS},
2264 {"amdgpu_gws_mm", amdgpu_mm_dump_table, 0, (void *)AMDGPU_PL_GWS},
2265 {"amdgpu_oa_mm", amdgpu_mm_dump_table, 0, (void *)AMDGPU_PL_OA},
2266 {"ttm_page_pool", ttm_page_alloc_debugfs, 0, NULL},
2267 #ifdef CONFIG_SWIOTLB
2268 {"ttm_dma_page_pool", ttm_dma_page_alloc_debugfs, 0, NULL}
2269 #endif
2270 };
2271
2272 /**
2273 * amdgpu_ttm_vram_read - Linear read access to VRAM
2274 *
2275 * Accesses VRAM via MMIO for debugging purposes.
2276 */
2277 static ssize_t amdgpu_ttm_vram_read(struct file *f, char __user *buf,
2278 size_t size, loff_t *pos)
2279 {
2280 struct amdgpu_device *adev = file_inode(f)->i_private;
2281 ssize_t result = 0;
2282 int r;
2283
2284 if (size & 0x3 || *pos & 0x3)
2285 return -EINVAL;
2286
2287 if (*pos >= adev->gmc.mc_vram_size)
2288 return -ENXIO;
2289
2290 while (size) {
2291 unsigned long flags;
2292 uint32_t value;
2293
2294 if (*pos >= adev->gmc.mc_vram_size)
2295 return result;
2296
2297 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
2298 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)*pos) | 0x80000000);
2299 WREG32_NO_KIQ(mmMM_INDEX_HI, *pos >> 31);
2300 value = RREG32_NO_KIQ(mmMM_DATA);
2301 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
2302
2303 r = put_user(value, (uint32_t *)buf);
2304 if (r)
2305 return r;
2306
2307 result += 4;
2308 buf += 4;
2309 *pos += 4;
2310 size -= 4;
2311 }
2312
2313 return result;
2314 }
2315
2316 /**
2317 * amdgpu_ttm_vram_write - Linear write access to VRAM
2318 *
2319 * Accesses VRAM via MMIO for debugging purposes.
2320 */
2321 static ssize_t amdgpu_ttm_vram_write(struct file *f, const char __user *buf,
2322 size_t size, loff_t *pos)
2323 {
2324 struct amdgpu_device *adev = file_inode(f)->i_private;
2325 ssize_t result = 0;
2326 int r;
2327
2328 if (size & 0x3 || *pos & 0x3)
2329 return -EINVAL;
2330
2331 if (*pos >= adev->gmc.mc_vram_size)
2332 return -ENXIO;
2333
2334 while (size) {
2335 unsigned long flags;
2336 uint32_t value;
2337
2338 if (*pos >= adev->gmc.mc_vram_size)
2339 return result;
2340
2341 r = get_user(value, (uint32_t *)buf);
2342 if (r)
2343 return r;
2344
2345 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
2346 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)*pos) | 0x80000000);
2347 WREG32_NO_KIQ(mmMM_INDEX_HI, *pos >> 31);
2348 WREG32_NO_KIQ(mmMM_DATA, value);
2349 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
2350
2351 result += 4;
2352 buf += 4;
2353 *pos += 4;
2354 size -= 4;
2355 }
2356
2357 return result;
2358 }
2359
2360 static const struct file_operations amdgpu_ttm_vram_fops = {
2361 .owner = THIS_MODULE,
2362 .read = amdgpu_ttm_vram_read,
2363 .write = amdgpu_ttm_vram_write,
2364 .llseek = default_llseek,
2365 };
2366
2367 #ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS
2368
2369 /**
2370 * amdgpu_ttm_gtt_read - Linear read access to GTT memory
2371 */
2372 static ssize_t amdgpu_ttm_gtt_read(struct file *f, char __user *buf,
2373 size_t size, loff_t *pos)
2374 {
2375 struct amdgpu_device *adev = file_inode(f)->i_private;
2376 ssize_t result = 0;
2377 int r;
2378
2379 while (size) {
2380 loff_t p = *pos / PAGE_SIZE;
2381 unsigned off = *pos & ~PAGE_MASK;
2382 size_t cur_size = min_t(size_t, size, PAGE_SIZE - off);
2383 struct page *page;
2384 void *ptr;
2385
2386 if (p >= adev->gart.num_cpu_pages)
2387 return result;
2388
2389 page = adev->gart.pages[p];
2390 if (page) {
2391 ptr = kmap(page);
2392 ptr += off;
2393
2394 r = copy_to_user(buf, ptr, cur_size);
2395 kunmap(adev->gart.pages[p]);
2396 } else
2397 r = clear_user(buf, cur_size);
2398
2399 if (r)
2400 return -EFAULT;
2401
2402 result += cur_size;
2403 buf += cur_size;
2404 *pos += cur_size;
2405 size -= cur_size;
2406 }
2407
2408 return result;
2409 }
2410
2411 static const struct file_operations amdgpu_ttm_gtt_fops = {
2412 .owner = THIS_MODULE,
2413 .read = amdgpu_ttm_gtt_read,
2414 .llseek = default_llseek
2415 };
2416
2417 #endif
2418
2419 /**
2420 * amdgpu_iomem_read - Virtual read access to GPU mapped memory
2421 *
2422 * This function is used to read memory that has been mapped to the
2423 * GPU and the known addresses are not physical addresses but instead
2424 * bus addresses (e.g., what you'd put in an IB or ring buffer).
2425 */
2426 static ssize_t amdgpu_iomem_read(struct file *f, char __user *buf,
2427 size_t size, loff_t *pos)
2428 {
2429 struct amdgpu_device *adev = file_inode(f)->i_private;
2430 struct iommu_domain *dom;
2431 ssize_t result = 0;
2432 int r;
2433
2434 /* retrieve the IOMMU domain if any for this device */
2435 dom = iommu_get_domain_for_dev(adev->dev);
2436
2437 while (size) {
2438 phys_addr_t addr = *pos & PAGE_MASK;
2439 loff_t off = *pos & ~PAGE_MASK;
2440 size_t bytes = PAGE_SIZE - off;
2441 unsigned long pfn;
2442 struct page *p;
2443 void *ptr;
2444
2445 bytes = bytes < size ? bytes : size;
2446
2447 /* Translate the bus address to a physical address. If
2448 * the domain is NULL it means there is no IOMMU active
2449 * and the address translation is the identity
2450 */
2451 addr = dom ? iommu_iova_to_phys(dom, addr) : addr;
2452
2453 pfn = addr >> PAGE_SHIFT;
2454 if (!pfn_valid(pfn))
2455 return -EPERM;
2456
2457 p = pfn_to_page(pfn);
2458 if (p->mapping != adev->mman.bdev.dev_mapping)
2459 return -EPERM;
2460
2461 ptr = kmap(p);
2462 r = copy_to_user(buf, ptr + off, bytes);
2463 kunmap(p);
2464 if (r)
2465 return -EFAULT;
2466
2467 size -= bytes;
2468 *pos += bytes;
2469 result += bytes;
2470 }
2471
2472 return result;
2473 }
2474
2475 /**
2476 * amdgpu_iomem_write - Virtual write access to GPU mapped memory
2477 *
2478 * This function is used to write memory that has been mapped to the
2479 * GPU and the known addresses are not physical addresses but instead
2480 * bus addresses (e.g., what you'd put in an IB or ring buffer).
2481 */
2482 static ssize_t amdgpu_iomem_write(struct file *f, const char __user *buf,
2483 size_t size, loff_t *pos)
2484 {
2485 struct amdgpu_device *adev = file_inode(f)->i_private;
2486 struct iommu_domain *dom;
2487 ssize_t result = 0;
2488 int r;
2489
2490 dom = iommu_get_domain_for_dev(adev->dev);
2491
2492 while (size) {
2493 phys_addr_t addr = *pos & PAGE_MASK;
2494 loff_t off = *pos & ~PAGE_MASK;
2495 size_t bytes = PAGE_SIZE - off;
2496 unsigned long pfn;
2497 struct page *p;
2498 void *ptr;
2499
2500 bytes = bytes < size ? bytes : size;
2501
2502 addr = dom ? iommu_iova_to_phys(dom, addr) : addr;
2503
2504 pfn = addr >> PAGE_SHIFT;
2505 if (!pfn_valid(pfn))
2506 return -EPERM;
2507
2508 p = pfn_to_page(pfn);
2509 if (p->mapping != adev->mman.bdev.dev_mapping)
2510 return -EPERM;
2511
2512 ptr = kmap(p);
2513 r = copy_from_user(ptr + off, buf, bytes);
2514 kunmap(p);
2515 if (r)
2516 return -EFAULT;
2517
2518 size -= bytes;
2519 *pos += bytes;
2520 result += bytes;
2521 }
2522
2523 return result;
2524 }
2525
2526 static const struct file_operations amdgpu_ttm_iomem_fops = {
2527 .owner = THIS_MODULE,
2528 .read = amdgpu_iomem_read,
2529 .write = amdgpu_iomem_write,
2530 .llseek = default_llseek
2531 };
2532
2533 static const struct {
2534 char *name;
2535 const struct file_operations *fops;
2536 int domain;
2537 } ttm_debugfs_entries[] = {
2538 { "amdgpu_vram", &amdgpu_ttm_vram_fops, TTM_PL_VRAM },
2539 #ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS
2540 { "amdgpu_gtt", &amdgpu_ttm_gtt_fops, TTM_PL_TT },
2541 #endif
2542 { "amdgpu_iomem", &amdgpu_ttm_iomem_fops, TTM_PL_SYSTEM },
2543 };
2544
2545 #endif
2546
2547 static int amdgpu_ttm_debugfs_init(struct amdgpu_device *adev)
2548 {
2549 #if defined(CONFIG_DEBUG_FS)
2550 unsigned count;
2551
2552 struct drm_minor *minor = adev->ddev->primary;
2553 struct dentry *ent, *root = minor->debugfs_root;
2554
2555 for (count = 0; count < ARRAY_SIZE(ttm_debugfs_entries); count++) {
2556 ent = debugfs_create_file(
2557 ttm_debugfs_entries[count].name,
2558 S_IFREG | S_IRUGO, root,
2559 adev,
2560 ttm_debugfs_entries[count].fops);
2561 if (IS_ERR(ent))
2562 return PTR_ERR(ent);
2563 if (ttm_debugfs_entries[count].domain == TTM_PL_VRAM)
2564 i_size_write(ent->d_inode, adev->gmc.mc_vram_size);
2565 else if (ttm_debugfs_entries[count].domain == TTM_PL_TT)
2566 i_size_write(ent->d_inode, adev->gmc.gart_size);
2567 adev->mman.debugfs_entries[count] = ent;
2568 }
2569
2570 count = ARRAY_SIZE(amdgpu_ttm_debugfs_list);
2571
2572 #ifdef CONFIG_SWIOTLB
2573 if (!(adev->need_swiotlb && swiotlb_nr_tbl()))
2574 --count;
2575 #endif
2576
2577 return amdgpu_debugfs_add_files(adev, amdgpu_ttm_debugfs_list, count);
2578 #else
2579 return 0;
2580 #endif
2581 }
2582
2583 static void amdgpu_ttm_debugfs_fini(struct amdgpu_device *adev)
2584 {
2585 #if defined(CONFIG_DEBUG_FS)
2586 unsigned i;
2587
2588 for (i = 0; i < ARRAY_SIZE(ttm_debugfs_entries); i++)
2589 debugfs_remove(adev->mman.debugfs_entries[i]);
2590 #endif
2591 }
Linux with added FPC-III support