1 /**************************************************************************
3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **************************************************************************/
28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
31 #define pr_fmt(fmt) "[TTM] " fmt
33 #include <linux/sched.h>
34 #include <linux/highmem.h>
35 #include <linux/pagemap.h>
36 #include <linux/shmem_fs.h>
37 #include <linux/file.h>
38 #include <linux/swap.h>
39 #include <linux/slab.h>
40 #include <linux/export.h>
41 #include <drm/drm_cache.h>
42 #include <drm/drm_mem_util.h>
43 #include <drm/ttm/ttm_module.h>
44 #include <drm/ttm/ttm_bo_driver.h>
45 #include <drm/ttm/ttm_placement.h>
46 #include <drm/ttm/ttm_page_alloc.h>
49 * Allocates storage for pointers to the pages that back the ttm.
51 static void ttm_tt_alloc_page_directory(struct ttm_tt
*ttm
)
53 ttm
->pages
= drm_calloc_large(ttm
->num_pages
, sizeof(void*));
56 static void ttm_dma_tt_alloc_page_directory(struct ttm_dma_tt
*ttm
)
58 ttm
->ttm
.pages
= drm_calloc_large(ttm
->ttm
.num_pages
,
59 sizeof(*ttm
->ttm
.pages
) +
60 sizeof(*ttm
->dma_address
) +
61 sizeof(*ttm
->cpu_address
));
62 ttm
->cpu_address
= (void *) (ttm
->ttm
.pages
+ ttm
->ttm
.num_pages
);
63 ttm
->dma_address
= (void *) (ttm
->cpu_address
+ ttm
->ttm
.num_pages
);
67 static inline int ttm_tt_set_page_caching(struct page
*p
,
68 enum ttm_caching_state c_old
,
69 enum ttm_caching_state c_new
)
76 if (c_old
!= tt_cached
) {
77 /* p isn't in the default caching state, set it to
78 * writeback first to free its current memtype. */
80 ret
= set_pages_wb(p
, 1);
86 ret
= set_memory_wc((unsigned long) page_address(p
), 1);
87 else if (c_new
== tt_uncached
)
88 ret
= set_pages_uc(p
, 1);
92 #else /* CONFIG_X86 */
93 static inline int ttm_tt_set_page_caching(struct page
*p
,
94 enum ttm_caching_state c_old
,
95 enum ttm_caching_state c_new
)
99 #endif /* CONFIG_X86 */
102 * Change caching policy for the linear kernel map
103 * for range of pages in a ttm.
106 static int ttm_tt_set_caching(struct ttm_tt
*ttm
,
107 enum ttm_caching_state c_state
)
110 struct page
*cur_page
;
113 if (ttm
->caching_state
== c_state
)
116 if (ttm
->state
== tt_unpopulated
) {
117 /* Change caching but don't populate */
118 ttm
->caching_state
= c_state
;
122 if (ttm
->caching_state
== tt_cached
)
123 drm_clflush_pages(ttm
->pages
, ttm
->num_pages
);
125 for (i
= 0; i
< ttm
->num_pages
; ++i
) {
126 cur_page
= ttm
->pages
[i
];
127 if (likely(cur_page
!= NULL
)) {
128 ret
= ttm_tt_set_page_caching(cur_page
,
131 if (unlikely(ret
!= 0))
136 ttm
->caching_state
= c_state
;
141 for (j
= 0; j
< i
; ++j
) {
142 cur_page
= ttm
->pages
[j
];
143 if (likely(cur_page
!= NULL
)) {
144 (void)ttm_tt_set_page_caching(cur_page
, c_state
,
152 int ttm_tt_set_placement_caching(struct ttm_tt
*ttm
, uint32_t placement
)
154 enum ttm_caching_state state
;
156 if (placement
& TTM_PL_FLAG_WC
)
158 else if (placement
& TTM_PL_FLAG_UNCACHED
)
163 return ttm_tt_set_caching(ttm
, state
);
165 EXPORT_SYMBOL(ttm_tt_set_placement_caching
);
167 void ttm_tt_destroy(struct ttm_tt
*ttm
)
169 if (unlikely(ttm
== NULL
))
172 if (ttm
->state
== tt_bound
) {
176 if (ttm
->state
== tt_unbound
)
177 ttm_tt_unpopulate(ttm
);
179 if (!(ttm
->page_flags
& TTM_PAGE_FLAG_PERSISTENT_SWAP
) &&
181 fput(ttm
->swap_storage
);
183 ttm
->swap_storage
= NULL
;
184 ttm
->func
->destroy(ttm
);
187 int ttm_tt_init(struct ttm_tt
*ttm
, struct ttm_bo_device
*bdev
,
188 unsigned long size
, uint32_t page_flags
,
189 struct page
*dummy_read_page
)
192 ttm
->glob
= bdev
->glob
;
193 ttm
->num_pages
= (size
+ PAGE_SIZE
- 1) >> PAGE_SHIFT
;
194 ttm
->caching_state
= tt_cached
;
195 ttm
->page_flags
= page_flags
;
196 ttm
->dummy_read_page
= dummy_read_page
;
197 ttm
->state
= tt_unpopulated
;
198 ttm
->swap_storage
= NULL
;
200 ttm_tt_alloc_page_directory(ttm
);
203 pr_err("Failed allocating page table\n");
208 EXPORT_SYMBOL(ttm_tt_init
);
210 void ttm_tt_fini(struct ttm_tt
*ttm
)
212 drm_free_large(ttm
->pages
);
215 EXPORT_SYMBOL(ttm_tt_fini
);
217 int ttm_dma_tt_init(struct ttm_dma_tt
*ttm_dma
, struct ttm_bo_device
*bdev
,
218 unsigned long size
, uint32_t page_flags
,
219 struct page
*dummy_read_page
)
221 struct ttm_tt
*ttm
= &ttm_dma
->ttm
;
224 ttm
->glob
= bdev
->glob
;
225 ttm
->num_pages
= (size
+ PAGE_SIZE
- 1) >> PAGE_SHIFT
;
226 ttm
->caching_state
= tt_cached
;
227 ttm
->page_flags
= page_flags
;
228 ttm
->dummy_read_page
= dummy_read_page
;
229 ttm
->state
= tt_unpopulated
;
230 ttm
->swap_storage
= NULL
;
232 INIT_LIST_HEAD(&ttm_dma
->pages_list
);
233 ttm_dma_tt_alloc_page_directory(ttm_dma
);
236 pr_err("Failed allocating page table\n");
241 EXPORT_SYMBOL(ttm_dma_tt_init
);
243 void ttm_dma_tt_fini(struct ttm_dma_tt
*ttm_dma
)
245 struct ttm_tt
*ttm
= &ttm_dma
->ttm
;
247 drm_free_large(ttm
->pages
);
249 ttm_dma
->cpu_address
= NULL
;
250 ttm_dma
->dma_address
= NULL
;
252 EXPORT_SYMBOL(ttm_dma_tt_fini
);
254 void ttm_tt_unbind(struct ttm_tt
*ttm
)
258 if (ttm
->state
== tt_bound
) {
259 ret
= ttm
->func
->unbind(ttm
);
261 ttm
->state
= tt_unbound
;
265 int ttm_tt_bind(struct ttm_tt
*ttm
, struct ttm_mem_reg
*bo_mem
)
272 if (ttm
->state
== tt_bound
)
275 ret
= ttm
->bdev
->driver
->ttm_tt_populate(ttm
);
279 ret
= ttm
->func
->bind(ttm
, bo_mem
);
280 if (unlikely(ret
!= 0))
283 ttm
->state
= tt_bound
;
287 EXPORT_SYMBOL(ttm_tt_bind
);
289 int ttm_tt_swapin(struct ttm_tt
*ttm
)
291 struct address_space
*swap_space
;
292 struct file
*swap_storage
;
293 struct page
*from_page
;
294 struct page
*to_page
;
298 swap_storage
= ttm
->swap_storage
;
299 BUG_ON(swap_storage
== NULL
);
301 swap_space
= file_inode(swap_storage
)->i_mapping
;
303 for (i
= 0; i
< ttm
->num_pages
; ++i
) {
304 from_page
= shmem_read_mapping_page(swap_space
, i
);
305 if (IS_ERR(from_page
)) {
306 ret
= PTR_ERR(from_page
);
309 to_page
= ttm
->pages
[i
];
310 if (unlikely(to_page
== NULL
))
313 copy_highpage(to_page
, from_page
);
314 page_cache_release(from_page
);
317 if (!(ttm
->page_flags
& TTM_PAGE_FLAG_PERSISTENT_SWAP
))
319 ttm
->swap_storage
= NULL
;
320 ttm
->page_flags
&= ~TTM_PAGE_FLAG_SWAPPED
;
327 int ttm_tt_swapout(struct ttm_tt
*ttm
, struct file
*persistent_swap_storage
)
329 struct address_space
*swap_space
;
330 struct file
*swap_storage
;
331 struct page
*from_page
;
332 struct page
*to_page
;
336 BUG_ON(ttm
->state
!= tt_unbound
&& ttm
->state
!= tt_unpopulated
);
337 BUG_ON(ttm
->caching_state
!= tt_cached
);
339 if (!persistent_swap_storage
) {
340 swap_storage
= shmem_file_setup("ttm swap",
341 ttm
->num_pages
<< PAGE_SHIFT
,
343 if (unlikely(IS_ERR(swap_storage
))) {
344 pr_err("Failed allocating swap storage\n");
345 return PTR_ERR(swap_storage
);
348 swap_storage
= persistent_swap_storage
;
350 swap_space
= file_inode(swap_storage
)->i_mapping
;
352 for (i
= 0; i
< ttm
->num_pages
; ++i
) {
353 from_page
= ttm
->pages
[i
];
354 if (unlikely(from_page
== NULL
))
356 to_page
= shmem_read_mapping_page(swap_space
, i
);
357 if (unlikely(IS_ERR(to_page
))) {
358 ret
= PTR_ERR(to_page
);
361 copy_highpage(to_page
, from_page
);
362 set_page_dirty(to_page
);
363 mark_page_accessed(to_page
);
364 page_cache_release(to_page
);
367 ttm_tt_unpopulate(ttm
);
368 ttm
->swap_storage
= swap_storage
;
369 ttm
->page_flags
|= TTM_PAGE_FLAG_SWAPPED
;
370 if (persistent_swap_storage
)
371 ttm
->page_flags
|= TTM_PAGE_FLAG_PERSISTENT_SWAP
;
375 if (!persistent_swap_storage
)
381 static void ttm_tt_clear_mapping(struct ttm_tt
*ttm
)
384 struct page
**page
= ttm
->pages
;
386 if (ttm
->page_flags
& TTM_PAGE_FLAG_SG
)
389 for (i
= 0; i
< ttm
->num_pages
; ++i
) {
390 (*page
)->mapping
= NULL
;
391 (*page
++)->index
= 0;
395 void ttm_tt_unpopulate(struct ttm_tt
*ttm
)
397 if (ttm
->state
== tt_unpopulated
)
400 ttm_tt_clear_mapping(ttm
);
401 ttm
->bdev
->driver
->ttm_tt_unpopulate(ttm
);