1 /**************************************************************************
3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **************************************************************************/
28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
31 #include <ttm/ttm_module.h>
32 #include <ttm/ttm_bo_driver.h>
33 #include <ttm/ttm_placement.h>
35 #include <linux/version.h>
36 #include <linux/rbtree.h>
37 #include <linux/module.h>
38 #include <linux/uaccess.h>
40 #define TTM_BO_VM_NUM_PREFAULT 16
42 static struct ttm_buffer_object
*ttm_bo_vm_lookup_rb(struct ttm_bo_device
*bdev
,
43 unsigned long page_start
,
44 unsigned long num_pages
)
46 struct rb_node
*cur
= bdev
->addr_space_rb
.rb_node
;
47 unsigned long cur_offset
;
48 struct ttm_buffer_object
*bo
;
49 struct ttm_buffer_object
*best_bo
= NULL
;
51 while (likely(cur
!= NULL
)) {
52 bo
= rb_entry(cur
, struct ttm_buffer_object
, vm_rb
);
53 cur_offset
= bo
->vm_node
->start
;
54 if (page_start
>= cur_offset
) {
57 if (page_start
== cur_offset
)
63 if (unlikely(best_bo
== NULL
))
66 if (unlikely((best_bo
->vm_node
->start
+ best_bo
->num_pages
) <
67 (page_start
+ num_pages
)))
73 static int ttm_bo_vm_fault(struct vm_area_struct
*vma
, struct vm_fault
*vmf
)
75 struct ttm_buffer_object
*bo
= (struct ttm_buffer_object
*)
77 struct ttm_bo_device
*bdev
= bo
->bdev
;
78 unsigned long bus_base
;
79 unsigned long bus_offset
;
80 unsigned long bus_size
;
81 unsigned long page_offset
;
82 unsigned long page_last
;
84 struct ttm_tt
*ttm
= NULL
;
89 unsigned long address
= (unsigned long)vmf
->virtual_address
;
90 int retval
= VM_FAULT_NOPAGE
;
93 * Work around locking order reversal in fault / nopfn
94 * between mmap_sem and bo_reserve: Perform a trylock operation
95 * for reserve, and if it fails, retry the fault after scheduling.
98 ret
= ttm_bo_reserve(bo
, true, true, false, 0);
99 if (unlikely(ret
!= 0)) {
102 return VM_FAULT_NOPAGE
;
106 * Wait for buffer data in transit, due to a pipelined
110 spin_lock(&bo
->lock
);
111 if (test_bit(TTM_BO_PRIV_FLAG_MOVING
, &bo
->priv_flags
)) {
112 ret
= ttm_bo_wait(bo
, false, true, false);
113 spin_unlock(&bo
->lock
);
114 if (unlikely(ret
!= 0)) {
115 retval
= (ret
!= -ERESTART
) ?
116 VM_FAULT_SIGBUS
: VM_FAULT_NOPAGE
;
120 spin_unlock(&bo
->lock
);
123 ret
= ttm_bo_pci_offset(bdev
, &bo
->mem
, &bus_base
, &bus_offset
,
125 if (unlikely(ret
!= 0)) {
126 retval
= VM_FAULT_SIGBUS
;
130 is_iomem
= (bus_size
!= 0);
132 page_offset
= ((address
- vma
->vm_start
) >> PAGE_SHIFT
) +
133 bo
->vm_node
->start
- vma
->vm_pgoff
;
134 page_last
= ((vma
->vm_end
- vma
->vm_start
) >> PAGE_SHIFT
) +
135 bo
->vm_node
->start
- vma
->vm_pgoff
;
137 if (unlikely(page_offset
>= bo
->num_pages
)) {
138 retval
= VM_FAULT_SIGBUS
;
143 * Strictly, we're not allowed to modify vma->vm_page_prot here,
144 * since the mmap_sem is only held in read mode. However, we
145 * modify only the caching bits of vma->vm_page_prot and
146 * consider those bits protected by
147 * the bo->mutex, as we should be the only writers.
148 * There shouldn't really be any readers of these bits except
149 * within vm_insert_mixed()? fork?
151 * TODO: Add a list of vmas to the bo, and change the
152 * vma->vm_page_prot when the object changes caching policy, with
153 * the correct locks held.
157 vma
->vm_page_prot
= ttm_io_prot(bo
->mem
.placement
,
161 vma
->vm_page_prot
= (bo
->mem
.placement
& TTM_PL_FLAG_CACHED
) ?
162 vm_get_page_prot(vma
->vm_flags
) :
163 ttm_io_prot(bo
->mem
.placement
, vma
->vm_page_prot
);
167 * Speculatively prefault a number of pages. Only error on
171 for (i
= 0; i
< TTM_BO_VM_NUM_PREFAULT
; ++i
) {
174 pfn
= ((bus_base
+ bus_offset
) >> PAGE_SHIFT
) +
177 page
= ttm_tt_get_page(ttm
, page_offset
);
178 if (unlikely(!page
&& i
== 0)) {
179 retval
= VM_FAULT_OOM
;
181 } else if (unlikely(!page
)) {
184 pfn
= page_to_pfn(page
);
187 ret
= vm_insert_mixed(vma
, address
, pfn
);
189 * Somebody beat us to this PTE or prefaulting to
190 * an already populated PTE, or prefaulting error.
193 if (unlikely((ret
== -EBUSY
) || (ret
!= 0 && i
> 0)))
195 else if (unlikely(ret
!= 0)) {
197 (ret
== -ENOMEM
) ? VM_FAULT_OOM
: VM_FAULT_SIGBUS
;
202 address
+= PAGE_SIZE
;
203 if (unlikely(++page_offset
>= page_last
))
208 ttm_bo_unreserve(bo
);
212 static void ttm_bo_vm_open(struct vm_area_struct
*vma
)
214 struct ttm_buffer_object
*bo
=
215 (struct ttm_buffer_object
*)vma
->vm_private_data
;
217 (void)ttm_bo_reference(bo
);
220 static void ttm_bo_vm_close(struct vm_area_struct
*vma
)
222 struct ttm_buffer_object
*bo
=
223 (struct ttm_buffer_object
*)vma
->vm_private_data
;
226 vma
->vm_private_data
= NULL
;
229 static struct vm_operations_struct ttm_bo_vm_ops
= {
230 .fault
= ttm_bo_vm_fault
,
231 .open
= ttm_bo_vm_open
,
232 .close
= ttm_bo_vm_close
235 int ttm_bo_mmap(struct file
*filp
, struct vm_area_struct
*vma
,
236 struct ttm_bo_device
*bdev
)
238 struct ttm_bo_driver
*driver
;
239 struct ttm_buffer_object
*bo
;
242 read_lock(&bdev
->vm_lock
);
243 bo
= ttm_bo_vm_lookup_rb(bdev
, vma
->vm_pgoff
,
244 (vma
->vm_end
- vma
->vm_start
) >> PAGE_SHIFT
);
245 if (likely(bo
!= NULL
))
246 ttm_bo_reference(bo
);
247 read_unlock(&bdev
->vm_lock
);
249 if (unlikely(bo
== NULL
)) {
250 printk(KERN_ERR TTM_PFX
251 "Could not find buffer object to map.\n");
255 driver
= bo
->bdev
->driver
;
256 if (unlikely(!driver
->verify_access
)) {
260 ret
= driver
->verify_access(bo
, filp
);
261 if (unlikely(ret
!= 0))
264 vma
->vm_ops
= &ttm_bo_vm_ops
;
267 * Note: We're transferring the bo reference to
268 * vma->vm_private_data here.
271 vma
->vm_private_data
= bo
;
272 vma
->vm_flags
|= VM_RESERVED
| VM_IO
| VM_MIXEDMAP
| VM_DONTEXPAND
;
278 EXPORT_SYMBOL(ttm_bo_mmap
);
280 int ttm_fbdev_mmap(struct vm_area_struct
*vma
, struct ttm_buffer_object
*bo
)
282 if (vma
->vm_pgoff
!= 0)
285 vma
->vm_ops
= &ttm_bo_vm_ops
;
286 vma
->vm_private_data
= ttm_bo_reference(bo
);
287 vma
->vm_flags
|= VM_RESERVED
| VM_IO
| VM_MIXEDMAP
| VM_DONTEXPAND
;
290 EXPORT_SYMBOL(ttm_fbdev_mmap
);
293 ssize_t
ttm_bo_io(struct ttm_bo_device
*bdev
, struct file
*filp
,
294 const char __user
*wbuf
, char __user
*rbuf
, size_t count
,
295 loff_t
*f_pos
, bool write
)
297 struct ttm_buffer_object
*bo
;
298 struct ttm_bo_driver
*driver
;
299 struct ttm_bo_kmap_obj map
;
300 unsigned long dev_offset
= (*f_pos
>> PAGE_SHIFT
);
301 unsigned long kmap_offset
;
302 unsigned long kmap_end
;
303 unsigned long kmap_num
;
305 unsigned int page_offset
;
308 bool no_wait
= false;
311 read_lock(&bdev
->vm_lock
);
312 bo
= ttm_bo_vm_lookup_rb(bdev
, dev_offset
, 1);
313 if (likely(bo
!= NULL
))
314 ttm_bo_reference(bo
);
315 read_unlock(&bdev
->vm_lock
);
317 if (unlikely(bo
== NULL
))
320 driver
= bo
->bdev
->driver
;
321 if (unlikely(driver
->verify_access
)) {
326 ret
= driver
->verify_access(bo
, filp
);
327 if (unlikely(ret
!= 0))
330 kmap_offset
= dev_offset
- bo
->vm_node
->start
;
331 if (unlikely(kmap_offset
) >= bo
->num_pages
) {
336 page_offset
= *f_pos
& ~PAGE_MASK
;
337 io_size
= bo
->num_pages
- kmap_offset
;
338 io_size
= (io_size
<< PAGE_SHIFT
) - page_offset
;
342 kmap_end
= (*f_pos
+ count
- 1) >> PAGE_SHIFT
;
343 kmap_num
= kmap_end
- kmap_offset
+ 1;
345 ret
= ttm_bo_reserve(bo
, true, no_wait
, false, 0);
360 ret
= ttm_bo_kmap(bo
, kmap_offset
, kmap_num
, &map
);
361 if (unlikely(ret
!= 0)) {
362 ttm_bo_unreserve(bo
);
366 virtual = ttm_kmap_obj_virtual(&map
, &dummy
);
367 virtual += page_offset
;
370 ret
= copy_from_user(virtual, wbuf
, io_size
);
372 ret
= copy_to_user(rbuf
, virtual, io_size
);
375 ttm_bo_unreserve(bo
);
378 if (unlikely(ret
!= 0))
389 ssize_t
ttm_bo_fbdev_io(struct ttm_buffer_object
*bo
, const char __user
*wbuf
,
390 char __user
*rbuf
, size_t count
, loff_t
*f_pos
,
393 struct ttm_bo_kmap_obj map
;
394 unsigned long kmap_offset
;
395 unsigned long kmap_end
;
396 unsigned long kmap_num
;
398 unsigned int page_offset
;
401 bool no_wait
= false;
404 kmap_offset
= (*f_pos
>> PAGE_SHIFT
);
405 if (unlikely(kmap_offset
) >= bo
->num_pages
)
408 page_offset
= *f_pos
& ~PAGE_MASK
;
409 io_size
= bo
->num_pages
- kmap_offset
;
410 io_size
= (io_size
<< PAGE_SHIFT
) - page_offset
;
414 kmap_end
= (*f_pos
+ count
- 1) >> PAGE_SHIFT
;
415 kmap_num
= kmap_end
- kmap_offset
+ 1;
417 ret
= ttm_bo_reserve(bo
, true, no_wait
, false, 0);
430 ret
= ttm_bo_kmap(bo
, kmap_offset
, kmap_num
, &map
);
431 if (unlikely(ret
!= 0)) {
432 ttm_bo_unreserve(bo
);
436 virtual = ttm_kmap_obj_virtual(&map
, &dummy
);
437 virtual += page_offset
;
440 ret
= copy_from_user(virtual, wbuf
, io_size
);
442 ret
= copy_to_user(rbuf
, virtual, io_size
);
445 ttm_bo_unreserve(bo
);
448 if (unlikely(ret
!= 0))