2 * Copyright 2010 Red Hat Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
26 #include "nouveau_drv.h"
27 #include "nouveau_mm.h"
28 #include "nouveau_vm.h"
31 nouveau_vm_map_at(struct nouveau_vma
*vma
, u64 delta
, struct nouveau_mem
*node
)
33 struct nouveau_vm
*vm
= vma
->vm
;
34 struct nouveau_mm_node
*r
;
35 int big
= vma
->node
->type
!= vm
->spg_shift
;
36 u32 offset
= vma
->node
->offset
+ (delta
>> 12);
37 u32 bits
= vma
->node
->type
- 12;
38 u32 pde
= (offset
>> vm
->pgt_bits
) - vm
->fpde
;
39 u32 pte
= (offset
& ((1 << vm
->pgt_bits
) - 1)) >> bits
;
40 u32 max
= 1 << (vm
->pgt_bits
- bits
);
44 list_for_each_entry(r
, &node
->regions
, rl_entry
) {
45 u64 phys
= (u64
)r
->offset
<< 12;
46 u32 num
= r
->length
>> bits
;
49 struct nouveau_gpuobj
*pgt
= vm
->pgt
[pde
].obj
[big
];
52 if (unlikely(end
>= max
))
56 vm
->map(vma
, pgt
, node
, pte
, len
, phys
, delta
);
60 if (unlikely(end
>= max
)) {
61 phys
+= len
<< (bits
+ 12);
66 delta
+= (u64
)len
<< vma
->node
->type
;
74 nouveau_vm_map(struct nouveau_vma
*vma
, struct nouveau_mem
*node
)
76 nouveau_vm_map_at(vma
, 0, node
);
80 nouveau_vm_map_sg(struct nouveau_vma
*vma
, u64 delta
, u64 length
,
81 struct nouveau_mem
*mem
, dma_addr_t
*list
)
83 struct nouveau_vm
*vm
= vma
->vm
;
84 int big
= vma
->node
->type
!= vm
->spg_shift
;
85 u32 offset
= vma
->node
->offset
+ (delta
>> 12);
86 u32 bits
= vma
->node
->type
- 12;
87 u32 num
= length
>> vma
->node
->type
;
88 u32 pde
= (offset
>> vm
->pgt_bits
) - vm
->fpde
;
89 u32 pte
= (offset
& ((1 << vm
->pgt_bits
) - 1)) >> bits
;
90 u32 max
= 1 << (vm
->pgt_bits
- bits
);
94 struct nouveau_gpuobj
*pgt
= vm
->pgt
[pde
].obj
[big
];
97 if (unlikely(end
>= max
))
101 vm
->map_sg(vma
, pgt
, mem
, pte
, len
, list
);
106 if (unlikely(end
>= max
)) {
116 nouveau_vm_unmap_at(struct nouveau_vma
*vma
, u64 delta
, u64 length
)
118 struct nouveau_vm
*vm
= vma
->vm
;
119 int big
= vma
->node
->type
!= vm
->spg_shift
;
120 u32 offset
= vma
->node
->offset
+ (delta
>> 12);
121 u32 bits
= vma
->node
->type
- 12;
122 u32 num
= length
>> vma
->node
->type
;
123 u32 pde
= (offset
>> vm
->pgt_bits
) - vm
->fpde
;
124 u32 pte
= (offset
& ((1 << vm
->pgt_bits
) - 1)) >> bits
;
125 u32 max
= 1 << (vm
->pgt_bits
- bits
);
129 struct nouveau_gpuobj
*pgt
= vm
->pgt
[pde
].obj
[big
];
132 if (unlikely(end
>= max
))
136 vm
->unmap(pgt
, pte
, len
);
140 if (unlikely(end
>= max
)) {
150 nouveau_vm_unmap(struct nouveau_vma
*vma
)
152 nouveau_vm_unmap_at(vma
, 0, (u64
)vma
->node
->length
<< 12);
156 nouveau_vm_unmap_pgt(struct nouveau_vm
*vm
, int big
, u32 fpde
, u32 lpde
)
158 struct nouveau_vm_pgd
*vpgd
;
159 struct nouveau_vm_pgt
*vpgt
;
160 struct nouveau_gpuobj
*pgt
;
163 for (pde
= fpde
; pde
<= lpde
; pde
++) {
164 vpgt
= &vm
->pgt
[pde
- vm
->fpde
];
165 if (--vpgt
->refcount
[big
])
168 pgt
= vpgt
->obj
[big
];
169 vpgt
->obj
[big
] = NULL
;
171 list_for_each_entry(vpgd
, &vm
->pgd_list
, head
) {
172 vm
->map_pgt(vpgd
->obj
, pde
, vpgt
->obj
);
175 mutex_unlock(&vm
->mm
->mutex
);
176 nouveau_gpuobj_ref(NULL
, &pgt
);
177 mutex_lock(&vm
->mm
->mutex
);
182 nouveau_vm_map_pgt(struct nouveau_vm
*vm
, u32 pde
, u32 type
)
184 struct nouveau_vm_pgt
*vpgt
= &vm
->pgt
[pde
- vm
->fpde
];
185 struct nouveau_vm_pgd
*vpgd
;
186 struct nouveau_gpuobj
*pgt
;
187 int big
= (type
!= vm
->spg_shift
);
191 pgt_size
= (1 << (vm
->pgt_bits
+ 12)) >> type
;
194 mutex_unlock(&vm
->mm
->mutex
);
195 ret
= nouveau_gpuobj_new(vm
->dev
, NULL
, pgt_size
, 0x1000,
196 NVOBJ_FLAG_ZERO_ALLOC
, &pgt
);
197 mutex_lock(&vm
->mm
->mutex
);
201 /* someone beat us to filling the PDE while we didn't have the lock */
202 if (unlikely(vpgt
->refcount
[big
]++)) {
203 mutex_unlock(&vm
->mm
->mutex
);
204 nouveau_gpuobj_ref(NULL
, &pgt
);
205 mutex_lock(&vm
->mm
->mutex
);
209 vpgt
->obj
[big
] = pgt
;
210 list_for_each_entry(vpgd
, &vm
->pgd_list
, head
) {
211 vm
->map_pgt(vpgd
->obj
, pde
, vpgt
->obj
);
218 nouveau_vm_get(struct nouveau_vm
*vm
, u64 size
, u32 page_shift
,
219 u32 access
, struct nouveau_vma
*vma
)
221 u32 align
= (1 << page_shift
) >> 12;
222 u32 msize
= size
>> 12;
226 mutex_lock(&vm
->mm
->mutex
);
227 ret
= nouveau_mm_get(vm
->mm
, page_shift
, msize
, 0, align
, &vma
->node
);
228 if (unlikely(ret
!= 0)) {
229 mutex_unlock(&vm
->mm
->mutex
);
233 fpde
= (vma
->node
->offset
>> vm
->pgt_bits
);
234 lpde
= (vma
->node
->offset
+ vma
->node
->length
- 1) >> vm
->pgt_bits
;
235 for (pde
= fpde
; pde
<= lpde
; pde
++) {
236 struct nouveau_vm_pgt
*vpgt
= &vm
->pgt
[pde
- vm
->fpde
];
237 int big
= (vma
->node
->type
!= vm
->spg_shift
);
239 if (likely(vpgt
->refcount
[big
])) {
240 vpgt
->refcount
[big
]++;
244 ret
= nouveau_vm_map_pgt(vm
, pde
, vma
->node
->type
);
247 nouveau_vm_unmap_pgt(vm
, big
, fpde
, pde
- 1);
248 nouveau_mm_put(vm
->mm
, vma
->node
);
249 mutex_unlock(&vm
->mm
->mutex
);
254 mutex_unlock(&vm
->mm
->mutex
);
257 vma
->offset
= (u64
)vma
->node
->offset
<< 12;
258 vma
->access
= access
;
263 nouveau_vm_put(struct nouveau_vma
*vma
)
265 struct nouveau_vm
*vm
= vma
->vm
;
268 if (unlikely(vma
->node
== NULL
))
270 fpde
= (vma
->node
->offset
>> vm
->pgt_bits
);
271 lpde
= (vma
->node
->offset
+ vma
->node
->length
- 1) >> vm
->pgt_bits
;
273 mutex_lock(&vm
->mm
->mutex
);
274 nouveau_vm_unmap_pgt(vm
, vma
->node
->type
!= vm
->spg_shift
, fpde
, lpde
);
275 nouveau_mm_put(vm
->mm
, vma
->node
);
277 mutex_unlock(&vm
->mm
->mutex
);
281 nouveau_vm_new(struct drm_device
*dev
, u64 offset
, u64 length
, u64 mm_offset
,
282 struct nouveau_vm
**pvm
)
284 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
285 struct nouveau_vm
*vm
;
286 u64 mm_length
= (offset
+ length
) - mm_offset
;
290 vm
= kzalloc(sizeof(*vm
), GFP_KERNEL
);
294 if (dev_priv
->card_type
== NV_50
) {
295 vm
->map_pgt
= nv50_vm_map_pgt
;
296 vm
->map
= nv50_vm_map
;
297 vm
->map_sg
= nv50_vm_map_sg
;
298 vm
->unmap
= nv50_vm_unmap
;
299 vm
->flush
= nv50_vm_flush
;
304 block
= (1 << pgt_bits
);
309 if (dev_priv
->card_type
== NV_C0
) {
310 vm
->map_pgt
= nvc0_vm_map_pgt
;
311 vm
->map
= nvc0_vm_map
;
312 vm
->map_sg
= nvc0_vm_map_sg
;
313 vm
->unmap
= nvc0_vm_unmap
;
314 vm
->flush
= nvc0_vm_flush
;
324 vm
->fpde
= offset
>> pgt_bits
;
325 vm
->lpde
= (offset
+ length
- 1) >> pgt_bits
;
326 vm
->pgt
= kcalloc(vm
->lpde
- vm
->fpde
+ 1, sizeof(*vm
->pgt
), GFP_KERNEL
);
332 INIT_LIST_HEAD(&vm
->pgd_list
);
335 vm
->pgt_bits
= pgt_bits
- 12;
337 ret
= nouveau_mm_init(&vm
->mm
, mm_offset
>> 12, mm_length
>> 12,
349 nouveau_vm_link(struct nouveau_vm
*vm
, struct nouveau_gpuobj
*pgd
)
351 struct nouveau_vm_pgd
*vpgd
;
357 vpgd
= kzalloc(sizeof(*vpgd
), GFP_KERNEL
);
361 nouveau_gpuobj_ref(pgd
, &vpgd
->obj
);
363 mutex_lock(&vm
->mm
->mutex
);
364 for (i
= vm
->fpde
; i
<= vm
->lpde
; i
++)
365 vm
->map_pgt(pgd
, i
, vm
->pgt
[i
- vm
->fpde
].obj
);
366 list_add(&vpgd
->head
, &vm
->pgd_list
);
367 mutex_unlock(&vm
->mm
->mutex
);
372 nouveau_vm_unlink(struct nouveau_vm
*vm
, struct nouveau_gpuobj
*mpgd
)
374 struct nouveau_vm_pgd
*vpgd
, *tmp
;
375 struct nouveau_gpuobj
*pgd
= NULL
;
380 mutex_lock(&vm
->mm
->mutex
);
381 list_for_each_entry_safe(vpgd
, tmp
, &vm
->pgd_list
, head
) {
382 if (vpgd
->obj
== mpgd
) {
384 list_del(&vpgd
->head
);
389 mutex_unlock(&vm
->mm
->mutex
);
391 nouveau_gpuobj_ref(NULL
, &pgd
);
395 nouveau_vm_del(struct nouveau_vm
*vm
)
397 struct nouveau_vm_pgd
*vpgd
, *tmp
;
399 list_for_each_entry_safe(vpgd
, tmp
, &vm
->pgd_list
, head
) {
400 nouveau_vm_unlink(vm
, vpgd
->obj
);
403 nouveau_mm_fini(&vm
->mm
);
409 nouveau_vm_ref(struct nouveau_vm
*ref
, struct nouveau_vm
**ptr
,
410 struct nouveau_gpuobj
*pgd
)
412 struct nouveau_vm
*vm
;
417 ret
= nouveau_vm_link(vm
, pgd
);
428 nouveau_vm_unlink(vm
, pgd
);
430 if (--vm
->refcount
== 0)