?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
dt-bindings: mtd: ingenic: Use standard ecc-engine property
[linux/fpc-iii.git] / drivers / gpu / drm / nouveau / nouveau_gem.c
blobb4bda716564dac1ff56298ac8cbb186b9471fb39
1 /*
2 * Copyright (C) 2008 Ben Skeggs.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining
6 * a copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sublicense, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
12 *
13 * The above copyright notice and this permission notice (including the
14 * next paragraph) shall be included in all copies or substantial
15 * portions of the Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
20 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
24 *
25 */
26
27 #include "nouveau_drv.h"
28 #include "nouveau_dma.h"
29 #include "nouveau_fence.h"
30 #include "nouveau_abi16.h"
31
32 #include "nouveau_ttm.h"
33 #include "nouveau_gem.h"
34 #include "nouveau_mem.h"
35 #include "nouveau_vmm.h"
36
37 #include <nvif/class.h>
38
39 void
40 nouveau_gem_object_del(struct drm_gem_object *gem)
41 {
42 struct nouveau_bo *nvbo = nouveau_gem_object(gem);
43 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
44 struct device *dev = drm->dev->dev;
45 int ret;
46
47 ret = pm_runtime_get_sync(dev);
48 if (WARN_ON(ret < 0 && ret != -EACCES))
49 return;
50
51 if (gem->import_attach)
52 drm_prime_gem_destroy(gem, nvbo->bo.sg);
53
54 drm_gem_object_release(gem);
55
56 /* reset filp so nouveau_bo_del_ttm() can test for it */
57 gem->filp = NULL;
58 ttm_bo_put(&nvbo->bo);
59
60 pm_runtime_mark_last_busy(dev);
61 pm_runtime_put_autosuspend(dev);
62 }
63
64 int
65 nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv)
66 {
67 struct nouveau_cli *cli = nouveau_cli(file_priv);
68 struct nouveau_bo *nvbo = nouveau_gem_object(gem);
69 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
70 struct device *dev = drm->dev->dev;
71 struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : &cli->vmm;
72 struct nouveau_vma *vma;
73 int ret;
74
75 if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50)
76 return 0;
77
78 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL);
79 if (ret)
80 return ret;
81
82 ret = pm_runtime_get_sync(dev);
83 if (ret < 0 && ret != -EACCES)
84 goto out;
85
86 ret = nouveau_vma_new(nvbo, vmm, &vma);
87 pm_runtime_mark_last_busy(dev);
88 pm_runtime_put_autosuspend(dev);
89 out:
90 ttm_bo_unreserve(&nvbo->bo);
91 return ret;
92 }
93
94 struct nouveau_gem_object_unmap {
95 struct nouveau_cli_work work;
96 struct nouveau_vma *vma;
97 };
98
99 static void
100 nouveau_gem_object_delete(struct nouveau_vma *vma)
101 {
102 nouveau_fence_unref(&vma->fence);
103 nouveau_vma_del(&vma);
104 }
105
106 static void
107 nouveau_gem_object_delete_work(struct nouveau_cli_work *w)
108 {
109 struct nouveau_gem_object_unmap *work =
110 container_of(w, typeof(*work), work);
111 nouveau_gem_object_delete(work->vma);
112 kfree(work);
113 }
114
115 static void
116 nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nouveau_vma *vma)
117 {
118 struct dma_fence *fence = vma->fence ? &vma->fence->base : NULL;
119 struct nouveau_gem_object_unmap *work;
120
121 list_del_init(&vma->head);
122
123 if (!fence) {
124 nouveau_gem_object_delete(vma);
125 return;
126 }
127
128 if (!(work = kmalloc(sizeof(*work), GFP_KERNEL))) {
129 WARN_ON(dma_fence_wait_timeout(fence, false, 2 * HZ) <= 0);
130 nouveau_gem_object_delete(vma);
131 return;
132 }
133
134 work->work.func = nouveau_gem_object_delete_work;
135 work->vma = vma;
136 nouveau_cli_work_queue(vma->vmm->cli, fence, &work->work);
137 }
138
139 void
140 nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv)
141 {
142 struct nouveau_cli *cli = nouveau_cli(file_priv);
143 struct nouveau_bo *nvbo = nouveau_gem_object(gem);
144 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
145 struct device *dev = drm->dev->dev;
146 struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : & cli->vmm;
147 struct nouveau_vma *vma;
148 int ret;
149
150 if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50)
151 return;
152
153 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL);
154 if (ret)
155 return;
156
157 vma = nouveau_vma_find(nvbo, vmm);
158 if (vma) {
159 if (--vma->refs == 0) {
160 ret = pm_runtime_get_sync(dev);
161 if (!WARN_ON(ret < 0 && ret != -EACCES)) {
162 nouveau_gem_object_unmap(nvbo, vma);
163 pm_runtime_mark_last_busy(dev);
164 pm_runtime_put_autosuspend(dev);
165 }
166 }
167 }
168 ttm_bo_unreserve(&nvbo->bo);
169 }
170
171 int
172 nouveau_gem_new(struct nouveau_cli *cli, u64 size, int align, uint32_t domain,
173 uint32_t tile_mode, uint32_t tile_flags,
174 struct nouveau_bo **pnvbo)
175 {
176 struct nouveau_drm *drm = cli->drm;
177 struct nouveau_bo *nvbo;
178 u32 flags = 0;
179 int ret;
180
181 if (domain & NOUVEAU_GEM_DOMAIN_VRAM)
182 flags |= TTM_PL_FLAG_VRAM;
183 if (domain & NOUVEAU_GEM_DOMAIN_GART)
184 flags |= TTM_PL_FLAG_TT;
185 if (!flags || domain & NOUVEAU_GEM_DOMAIN_CPU)
186 flags |= TTM_PL_FLAG_SYSTEM;
187
188 if (domain & NOUVEAU_GEM_DOMAIN_COHERENT)
189 flags |= TTM_PL_FLAG_UNCACHED;
190
191 ret = nouveau_bo_new(cli, size, align, flags, tile_mode,
192 tile_flags, NULL, NULL, pnvbo);
193 if (ret)
194 return ret;
195 nvbo = *pnvbo;
196
197 /* we restrict allowed domains on nv50+ to only the types
198 * that were requested at creation time. not possibly on
199 * earlier chips without busting the ABI.
200 */
201 nvbo->valid_domains = NOUVEAU_GEM_DOMAIN_VRAM |
202 NOUVEAU_GEM_DOMAIN_GART;
203 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA)
204 nvbo->valid_domains &= domain;
205
206 /* Initialize the embedded gem-object. We return a single gem-reference
207 * to the caller, instead of a normal nouveau_bo ttm reference. */
208 ret = drm_gem_object_init(drm->dev, &nvbo->gem, nvbo->bo.mem.size);
209 if (ret) {
210 nouveau_bo_ref(NULL, pnvbo);
211 return -ENOMEM;
212 }
213
214 nvbo->bo.persistent_swap_storage = nvbo->gem.filp;
215 return 0;
216 }
217
218 static int
219 nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem,
220 struct drm_nouveau_gem_info *rep)
221 {
222 struct nouveau_cli *cli = nouveau_cli(file_priv);
223 struct nouveau_bo *nvbo = nouveau_gem_object(gem);
224 struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : &cli->vmm;
225 struct nouveau_vma *vma;
226
227 if (is_power_of_2(nvbo->valid_domains))
228 rep->domain = nvbo->valid_domains;
229 else if (nvbo->bo.mem.mem_type == TTM_PL_TT)
230 rep->domain = NOUVEAU_GEM_DOMAIN_GART;
231 else
232 rep->domain = NOUVEAU_GEM_DOMAIN_VRAM;
233 rep->offset = nvbo->bo.offset;
234 if (vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) {
235 vma = nouveau_vma_find(nvbo, vmm);
236 if (!vma)
237 return -EINVAL;
238
239 rep->offset = vma->addr;
240 }
241
242 rep->size = nvbo->bo.mem.num_pages << PAGE_SHIFT;
243 rep->map_handle = drm_vma_node_offset_addr(&nvbo->bo.vma_node);
244 rep->tile_mode = nvbo->mode;
245 rep->tile_flags = nvbo->contig ? 0 : NOUVEAU_GEM_TILE_NONCONTIG;
246 if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI)
247 rep->tile_flags |= nvbo->kind << 8;
248 else
249 if (cli->device.info.family >= NV_DEVICE_INFO_V0_TESLA)
250 rep->tile_flags |= nvbo->kind << 8 | nvbo->comp << 16;
251 else
252 rep->tile_flags |= nvbo->zeta;
253 return 0;
254 }
255
256 int
257 nouveau_gem_ioctl_new(struct drm_device *dev, void *data,
258 struct drm_file *file_priv)
259 {
260 struct nouveau_cli *cli = nouveau_cli(file_priv);
261 struct drm_nouveau_gem_new *req = data;
262 struct nouveau_bo *nvbo = NULL;
263 int ret = 0;
264
265 ret = nouveau_gem_new(cli, req->info.size, req->align,
266 req->info.domain, req->info.tile_mode,
267 req->info.tile_flags, &nvbo);
268 if (ret)
269 return ret;
270
271 ret = drm_gem_handle_create(file_priv, &nvbo->gem, &req->info.handle);
272 if (ret == 0) {
273 ret = nouveau_gem_info(file_priv, &nvbo->gem, &req->info);
274 if (ret)
275 drm_gem_handle_delete(file_priv, req->info.handle);
276 }
277
278 /* drop reference from allocate - handle holds it now */
279 drm_gem_object_put_unlocked(&nvbo->gem);
280 return ret;
281 }
282
283 static int
284 nouveau_gem_set_domain(struct drm_gem_object *gem, uint32_t read_domains,
285 uint32_t write_domains, uint32_t valid_domains)
286 {
287 struct nouveau_bo *nvbo = nouveau_gem_object(gem);
288 struct ttm_buffer_object *bo = &nvbo->bo;
289 uint32_t domains = valid_domains & nvbo->valid_domains &
290 (write_domains ? write_domains : read_domains);
291 uint32_t pref_flags = 0, valid_flags = 0;
292
293 if (!domains)
294 return -EINVAL;
295
296 if (valid_domains & NOUVEAU_GEM_DOMAIN_VRAM)
297 valid_flags |= TTM_PL_FLAG_VRAM;
298
299 if (valid_domains & NOUVEAU_GEM_DOMAIN_GART)
300 valid_flags |= TTM_PL_FLAG_TT;
301
302 if ((domains & NOUVEAU_GEM_DOMAIN_VRAM) &&
303 bo->mem.mem_type == TTM_PL_VRAM)
304 pref_flags |= TTM_PL_FLAG_VRAM;
305
306 else if ((domains & NOUVEAU_GEM_DOMAIN_GART) &&
307 bo->mem.mem_type == TTM_PL_TT)
308 pref_flags |= TTM_PL_FLAG_TT;
309
310 else if (domains & NOUVEAU_GEM_DOMAIN_VRAM)
311 pref_flags |= TTM_PL_FLAG_VRAM;
312
313 else
314 pref_flags |= TTM_PL_FLAG_TT;
315
316 nouveau_bo_placement_set(nvbo, pref_flags, valid_flags);
317
318 return 0;
319 }
320
321 struct validate_op {
322 struct list_head list;
323 struct ww_acquire_ctx ticket;
324 };
325
326 static void
327 validate_fini_no_ticket(struct validate_op *op, struct nouveau_channel *chan,
328 struct nouveau_fence *fence,
329 struct drm_nouveau_gem_pushbuf_bo *pbbo)
330 {
331 struct nouveau_bo *nvbo;
332 struct drm_nouveau_gem_pushbuf_bo *b;
333
334 while (!list_empty(&op->list)) {
335 nvbo = list_entry(op->list.next, struct nouveau_bo, entry);
336 b = &pbbo[nvbo->pbbo_index];
337
338 if (likely(fence)) {
339 nouveau_bo_fence(nvbo, fence, !!b->write_domains);
340
341 if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) {
342 struct nouveau_vma *vma =
343 (void *)(unsigned long)b->user_priv;
344 nouveau_fence_unref(&vma->fence);
345 dma_fence_get(&fence->base);
346 vma->fence = fence;
347 }
348 }
349
350 if (unlikely(nvbo->validate_mapped)) {
351 ttm_bo_kunmap(&nvbo->kmap);
352 nvbo->validate_mapped = false;
353 }
354
355 list_del(&nvbo->entry);
356 nvbo->reserved_by = NULL;
357 ttm_bo_unreserve(&nvbo->bo);
358 drm_gem_object_put_unlocked(&nvbo->gem);
359 }
360 }
361
362 static void
363 validate_fini(struct validate_op *op, struct nouveau_channel *chan,
364 struct nouveau_fence *fence,
365 struct drm_nouveau_gem_pushbuf_bo *pbbo)
366 {
367 validate_fini_no_ticket(op, chan, fence, pbbo);
368 ww_acquire_fini(&op->ticket);
369 }
370
371 static int
372 validate_init(struct nouveau_channel *chan, struct drm_file *file_priv,
373 struct drm_nouveau_gem_pushbuf_bo *pbbo,
374 int nr_buffers, struct validate_op *op)
375 {
376 struct nouveau_cli *cli = nouveau_cli(file_priv);
377 int trycnt = 0;
378 int ret = -EINVAL, i;
379 struct nouveau_bo *res_bo = NULL;
380 LIST_HEAD(gart_list);
381 LIST_HEAD(vram_list);
382 LIST_HEAD(both_list);
383
384 ww_acquire_init(&op->ticket, &reservation_ww_class);
385 retry:
386 if (++trycnt > 100000) {
387 NV_PRINTK(err, cli, "%s failed and gave up.\n", __func__);
388 return -EINVAL;
389 }
390
391 for (i = 0; i < nr_buffers; i++) {
392 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[i];
393 struct drm_gem_object *gem;
394 struct nouveau_bo *nvbo;
395
396 gem = drm_gem_object_lookup(file_priv, b->handle);
397 if (!gem) {
398 NV_PRINTK(err, cli, "Unknown handle 0x%08x\n", b->handle);
399 ret = -ENOENT;
400 break;
401 }
402 nvbo = nouveau_gem_object(gem);
403 if (nvbo == res_bo) {
404 res_bo = NULL;
405 drm_gem_object_put_unlocked(gem);
406 continue;
407 }
408
409 if (nvbo->reserved_by && nvbo->reserved_by == file_priv) {
410 NV_PRINTK(err, cli, "multiple instances of buffer %d on "
411 "validation list\n", b->handle);
412 drm_gem_object_put_unlocked(gem);
413 ret = -EINVAL;
414 break;
415 }
416
417 ret = ttm_bo_reserve(&nvbo->bo, true, false, &op->ticket);
418 if (ret) {
419 list_splice_tail_init(&vram_list, &op->list);
420 list_splice_tail_init(&gart_list, &op->list);
421 list_splice_tail_init(&both_list, &op->list);
422 validate_fini_no_ticket(op, chan, NULL, NULL);
423 if (unlikely(ret == -EDEADLK)) {
424 ret = ttm_bo_reserve_slowpath(&nvbo->bo, true,
425 &op->ticket);
426 if (!ret)
427 res_bo = nvbo;
428 }
429 if (unlikely(ret)) {
430 if (ret != -ERESTARTSYS)
431 NV_PRINTK(err, cli, "fail reserve\n");
432 break;
433 }
434 }
435
436 if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) {
437 struct nouveau_vmm *vmm = chan->vmm;
438 struct nouveau_vma *vma = nouveau_vma_find(nvbo, vmm);
439 if (!vma) {
440 NV_PRINTK(err, cli, "vma not found!\n");
441 ret = -EINVAL;
442 break;
443 }
444
445 b->user_priv = (uint64_t)(unsigned long)vma;
446 } else {
447 b->user_priv = (uint64_t)(unsigned long)nvbo;
448 }
449
450 nvbo->reserved_by = file_priv;
451 nvbo->pbbo_index = i;
452 if ((b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) &&
453 (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART))
454 list_add_tail(&nvbo->entry, &both_list);
455 else
456 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM)
457 list_add_tail(&nvbo->entry, &vram_list);
458 else
459 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART)
460 list_add_tail(&nvbo->entry, &gart_list);
461 else {
462 NV_PRINTK(err, cli, "invalid valid domains: 0x%08x\n",
463 b->valid_domains);
464 list_add_tail(&nvbo->entry, &both_list);
465 ret = -EINVAL;
466 break;
467 }
468 if (nvbo == res_bo)
469 goto retry;
470 }
471
472 ww_acquire_done(&op->ticket);
473 list_splice_tail(&vram_list, &op->list);
474 list_splice_tail(&gart_list, &op->list);
475 list_splice_tail(&both_list, &op->list);
476 if (ret)
477 validate_fini(op, chan, NULL, NULL);
478 return ret;
479
480 }
481
482 static int
483 validate_list(struct nouveau_channel *chan, struct nouveau_cli *cli,
484 struct list_head *list, struct drm_nouveau_gem_pushbuf_bo *pbbo,
485 uint64_t user_pbbo_ptr)
486 {
487 struct nouveau_drm *drm = chan->drm;
488 struct drm_nouveau_gem_pushbuf_bo __user *upbbo =
489 (void __force __user *)(uintptr_t)user_pbbo_ptr;
490 struct nouveau_bo *nvbo;
491 int ret, relocs = 0;
492
493 list_for_each_entry(nvbo, list, entry) {
494 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[nvbo->pbbo_index];
495
496 ret = nouveau_gem_set_domain(&nvbo->gem, b->read_domains,
497 b->write_domains,
498 b->valid_domains);
499 if (unlikely(ret)) {
500 NV_PRINTK(err, cli, "fail set_domain\n");
501 return ret;
502 }
503
504 ret = nouveau_bo_validate(nvbo, true, false);
505 if (unlikely(ret)) {
506 if (ret != -ERESTARTSYS)
507 NV_PRINTK(err, cli, "fail ttm_validate\n");
508 return ret;
509 }
510
511 ret = nouveau_fence_sync(nvbo, chan, !!b->write_domains, true);
512 if (unlikely(ret)) {
513 if (ret != -ERESTARTSYS)
514 NV_PRINTK(err, cli, "fail post-validate sync\n");
515 return ret;
516 }
517
518 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) {
519 if (nvbo->bo.offset == b->presumed.offset &&
520 ((nvbo->bo.mem.mem_type == TTM_PL_VRAM &&
521 b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) ||
522 (nvbo->bo.mem.mem_type == TTM_PL_TT &&
523 b->presumed.domain & NOUVEAU_GEM_DOMAIN_GART)))
524 continue;
525
526 if (nvbo->bo.mem.mem_type == TTM_PL_TT)
527 b->presumed.domain = NOUVEAU_GEM_DOMAIN_GART;
528 else
529 b->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM;
530 b->presumed.offset = nvbo->bo.offset;
531 b->presumed.valid = 0;
532 relocs++;
533
534 if (copy_to_user(&upbbo[nvbo->pbbo_index].presumed,
535 &b->presumed, sizeof(b->presumed)))
536 return -EFAULT;
537 }
538 }
539
540 return relocs;
541 }
542
543 static int
544 nouveau_gem_pushbuf_validate(struct nouveau_channel *chan,
545 struct drm_file *file_priv,
546 struct drm_nouveau_gem_pushbuf_bo *pbbo,
547 uint64_t user_buffers, int nr_buffers,
548 struct validate_op *op, int *apply_relocs)
549 {
550 struct nouveau_cli *cli = nouveau_cli(file_priv);
551 int ret;
552
553 INIT_LIST_HEAD(&op->list);
554
555 if (nr_buffers == 0)
556 return 0;
557
558 ret = validate_init(chan, file_priv, pbbo, nr_buffers, op);
559 if (unlikely(ret)) {
560 if (ret != -ERESTARTSYS)
561 NV_PRINTK(err, cli, "validate_init\n");
562 return ret;
563 }
564
565 ret = validate_list(chan, cli, &op->list, pbbo, user_buffers);
566 if (unlikely(ret < 0)) {
567 if (ret != -ERESTARTSYS)
568 NV_PRINTK(err, cli, "validating bo list\n");
569 validate_fini(op, chan, NULL, NULL);
570 return ret;
571 }
572 *apply_relocs = ret;
573 return 0;
574 }
575
576 static inline void
577 u_free(void *addr)
578 {
579 kvfree(addr);
580 }
581
582 static inline void *
583 u_memcpya(uint64_t user, unsigned nmemb, unsigned size)
584 {
585 void *mem;
586 void __user *userptr = (void __force __user *)(uintptr_t)user;
587
588 size *= nmemb;
589
590 mem = kvmalloc(size, GFP_KERNEL);
591 if (!mem)
592 return ERR_PTR(-ENOMEM);
593
594 if (copy_from_user(mem, userptr, size)) {
595 u_free(mem);
596 return ERR_PTR(-EFAULT);
597 }
598
599 return mem;
600 }
601
602 static int
603 nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli,
604 struct drm_nouveau_gem_pushbuf *req,
605 struct drm_nouveau_gem_pushbuf_bo *bo)
606 {
607 struct drm_nouveau_gem_pushbuf_reloc *reloc = NULL;
608 int ret = 0;
609 unsigned i;
610
611 reloc = u_memcpya(req->relocs, req->nr_relocs, sizeof(*reloc));
612 if (IS_ERR(reloc))
613 return PTR_ERR(reloc);
614
615 for (i = 0; i < req->nr_relocs; i++) {
616 struct drm_nouveau_gem_pushbuf_reloc *r = &reloc[i];
617 struct drm_nouveau_gem_pushbuf_bo *b;
618 struct nouveau_bo *nvbo;
619 uint32_t data;
620
621 if (unlikely(r->bo_index >= req->nr_buffers)) {
622 NV_PRINTK(err, cli, "reloc bo index invalid\n");
623 ret = -EINVAL;
624 break;
625 }
626
627 b = &bo[r->bo_index];
628 if (b->presumed.valid)
629 continue;
630
631 if (unlikely(r->reloc_bo_index >= req->nr_buffers)) {
632 NV_PRINTK(err, cli, "reloc container bo index invalid\n");
633 ret = -EINVAL;
634 break;
635 }
636 nvbo = (void *)(unsigned long)bo[r->reloc_bo_index].user_priv;
637
638 if (unlikely(r->reloc_bo_offset + 4 >
639 nvbo->bo.mem.num_pages << PAGE_SHIFT)) {
640 NV_PRINTK(err, cli, "reloc outside of bo\n");
641 ret = -EINVAL;
642 break;
643 }
644
645 if (!nvbo->kmap.virtual) {
646 ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages,
647 &nvbo->kmap);
648 if (ret) {
649 NV_PRINTK(err, cli, "failed kmap for reloc\n");
650 break;
651 }
652 nvbo->validate_mapped = true;
653 }
654
655 if (r->flags & NOUVEAU_GEM_RELOC_LOW)
656 data = b->presumed.offset + r->data;
657 else
658 if (r->flags & NOUVEAU_GEM_RELOC_HIGH)
659 data = (b->presumed.offset + r->data) >> 32;
660 else
661 data = r->data;
662
663 if (r->flags & NOUVEAU_GEM_RELOC_OR) {
664 if (b->presumed.domain == NOUVEAU_GEM_DOMAIN_GART)
665 data |= r->tor;
666 else
667 data |= r->vor;
668 }
669
670 ret = ttm_bo_wait(&nvbo->bo, false, false);
671 if (ret) {
672 NV_PRINTK(err, cli, "reloc wait_idle failed: %d\n", ret);
673 break;
674 }
675
676 nouveau_bo_wr32(nvbo, r->reloc_bo_offset >> 2, data);
677 }
678
679 u_free(reloc);
680 return ret;
681 }
682
683 int
684 nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data,
685 struct drm_file *file_priv)
686 {
687 struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv);
688 struct nouveau_cli *cli = nouveau_cli(file_priv);
689 struct nouveau_abi16_chan *temp;
690 struct nouveau_drm *drm = nouveau_drm(dev);
691 struct drm_nouveau_gem_pushbuf *req = data;
692 struct drm_nouveau_gem_pushbuf_push *push;
693 struct drm_nouveau_gem_pushbuf_bo *bo;
694 struct nouveau_channel *chan = NULL;
695 struct validate_op op;
696 struct nouveau_fence *fence = NULL;
697 int i, j, ret = 0, do_reloc = 0;
698
699 if (unlikely(!abi16))
700 return -ENOMEM;
701
702 list_for_each_entry(temp, &abi16->channels, head) {
703 if (temp->chan->chid == req->channel) {
704 chan = temp->chan;
705 break;
706 }
707 }
708
709 if (!chan)
710 return nouveau_abi16_put(abi16, -ENOENT);
711
712 req->vram_available = drm->gem.vram_available;
713 req->gart_available = drm->gem.gart_available;
714 if (unlikely(req->nr_push == 0))
715 goto out_next;
716
717 if (unlikely(req->nr_push > NOUVEAU_GEM_MAX_PUSH)) {
718 NV_PRINTK(err, cli, "pushbuf push count exceeds limit: %d max %d\n",
719 req->nr_push, NOUVEAU_GEM_MAX_PUSH);
720 return nouveau_abi16_put(abi16, -EINVAL);
721 }
722
723 if (unlikely(req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS)) {
724 NV_PRINTK(err, cli, "pushbuf bo count exceeds limit: %d max %d\n",
725 req->nr_buffers, NOUVEAU_GEM_MAX_BUFFERS);
726 return nouveau_abi16_put(abi16, -EINVAL);
727 }
728
729 if (unlikely(req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS)) {
730 NV_PRINTK(err, cli, "pushbuf reloc count exceeds limit: %d max %d\n",
731 req->nr_relocs, NOUVEAU_GEM_MAX_RELOCS);
732 return nouveau_abi16_put(abi16, -EINVAL);
733 }
734
735 push = u_memcpya(req->push, req->nr_push, sizeof(*push));
736 if (IS_ERR(push))
737 return nouveau_abi16_put(abi16, PTR_ERR(push));
738
739 bo = u_memcpya(req->buffers, req->nr_buffers, sizeof(*bo));
740 if (IS_ERR(bo)) {
741 u_free(push);
742 return nouveau_abi16_put(abi16, PTR_ERR(bo));
743 }
744
745 /* Ensure all push buffers are on validate list */
746 for (i = 0; i < req->nr_push; i++) {
747 if (push[i].bo_index >= req->nr_buffers) {
748 NV_PRINTK(err, cli, "push %d buffer not in list\n", i);
749 ret = -EINVAL;
750 goto out_prevalid;
751 }
752 }
753
754 /* Validate buffer list */
755 ret = nouveau_gem_pushbuf_validate(chan, file_priv, bo, req->buffers,
756 req->nr_buffers, &op, &do_reloc);
757 if (ret) {
758 if (ret != -ERESTARTSYS)
759 NV_PRINTK(err, cli, "validate: %d\n", ret);
760 goto out_prevalid;
761 }
762
763 /* Apply any relocations that are required */
764 if (do_reloc) {
765 ret = nouveau_gem_pushbuf_reloc_apply(cli, req, bo);
766 if (ret) {
767 NV_PRINTK(err, cli, "reloc apply: %d\n", ret);
768 goto out;
769 }
770 }
771
772 if (chan->dma.ib_max) {
773 ret = nouveau_dma_wait(chan, req->nr_push + 1, 16);
774 if (ret) {
775 NV_PRINTK(err, cli, "nv50cal_space: %d\n", ret);
776 goto out;
777 }
778
779 for (i = 0; i < req->nr_push; i++) {
780 struct nouveau_vma *vma = (void *)(unsigned long)
781 bo[push[i].bo_index].user_priv;
782
783 nv50_dma_push(chan, vma->addr + push[i].offset,
784 push[i].length);
785 }
786 } else
787 if (drm->client.device.info.chipset >= 0x25) {
788 ret = RING_SPACE(chan, req->nr_push * 2);
789 if (ret) {
790 NV_PRINTK(err, cli, "cal_space: %d\n", ret);
791 goto out;
792 }
793
794 for (i = 0; i < req->nr_push; i++) {
795 struct nouveau_bo *nvbo = (void *)(unsigned long)
796 bo[push[i].bo_index].user_priv;
797
798 OUT_RING(chan, (nvbo->bo.offset + push[i].offset) | 2);
799 OUT_RING(chan, 0);
800 }
801 } else {
802 ret = RING_SPACE(chan, req->nr_push * (2 + NOUVEAU_DMA_SKIPS));
803 if (ret) {
804 NV_PRINTK(err, cli, "jmp_space: %d\n", ret);
805 goto out;
806 }
807
808 for (i = 0; i < req->nr_push; i++) {
809 struct nouveau_bo *nvbo = (void *)(unsigned long)
810 bo[push[i].bo_index].user_priv;
811 uint32_t cmd;
812
813 cmd = chan->push.addr + ((chan->dma.cur + 2) << 2);
814 cmd |= 0x20000000;
815 if (unlikely(cmd != req->suffix0)) {
816 if (!nvbo->kmap.virtual) {
817 ret = ttm_bo_kmap(&nvbo->bo, 0,
818 nvbo->bo.mem.
819 num_pages,
820 &nvbo->kmap);
821 if (ret) {
822 WIND_RING(chan);
823 goto out;
824 }
825 nvbo->validate_mapped = true;
826 }
827
828 nouveau_bo_wr32(nvbo, (push[i].offset +
829 push[i].length - 8) / 4, cmd);
830 }
831
832 OUT_RING(chan, 0x20000000 |
833 (nvbo->bo.offset + push[i].offset));
834 OUT_RING(chan, 0);
835 for (j = 0; j < NOUVEAU_DMA_SKIPS; j++)
836 OUT_RING(chan, 0);
837 }
838 }
839
840 ret = nouveau_fence_new(chan, false, &fence);
841 if (ret) {
842 NV_PRINTK(err, cli, "error fencing pushbuf: %d\n", ret);
843 WIND_RING(chan);
844 goto out;
845 }
846
847 out:
848 validate_fini(&op, chan, fence, bo);
849 nouveau_fence_unref(&fence);
850
851 out_prevalid:
852 u_free(bo);
853 u_free(push);
854
855 out_next:
856 if (chan->dma.ib_max) {
857 req->suffix0 = 0x00000000;
858 req->suffix1 = 0x00000000;
859 } else
860 if (drm->client.device.info.chipset >= 0x25) {
861 req->suffix0 = 0x00020000;
862 req->suffix1 = 0x00000000;
863 } else {
864 req->suffix0 = 0x20000000 |
865 (chan->push.addr + ((chan->dma.cur + 2) << 2));
866 req->suffix1 = 0x00000000;
867 }
868
869 return nouveau_abi16_put(abi16, ret);
870 }
871
872 int
873 nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data,
874 struct drm_file *file_priv)
875 {
876 struct drm_nouveau_gem_cpu_prep *req = data;
877 struct drm_gem_object *gem;
878 struct nouveau_bo *nvbo;
879 bool no_wait = !!(req->flags & NOUVEAU_GEM_CPU_PREP_NOWAIT);
880 bool write = !!(req->flags & NOUVEAU_GEM_CPU_PREP_WRITE);
881 long lret;
882 int ret;
883
884 gem = drm_gem_object_lookup(file_priv, req->handle);
885 if (!gem)
886 return -ENOENT;
887 nvbo = nouveau_gem_object(gem);
888
889 lret = reservation_object_wait_timeout_rcu(nvbo->bo.resv, write, true,
890 no_wait ? 0 : 30 * HZ);
891 if (!lret)
892 ret = -EBUSY;
893 else if (lret > 0)
894 ret = 0;
895 else
896 ret = lret;
897
898 nouveau_bo_sync_for_cpu(nvbo);
899 drm_gem_object_put_unlocked(gem);
900
901 return ret;
902 }
903
904 int
905 nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data,
906 struct drm_file *file_priv)
907 {
908 struct drm_nouveau_gem_cpu_fini *req = data;
909 struct drm_gem_object *gem;
910 struct nouveau_bo *nvbo;
911
912 gem = drm_gem_object_lookup(file_priv, req->handle);
913 if (!gem)
914 return -ENOENT;
915 nvbo = nouveau_gem_object(gem);
916
917 nouveau_bo_sync_for_device(nvbo);
918 drm_gem_object_put_unlocked(gem);
919 return 0;
920 }
921
922 int
923 nouveau_gem_ioctl_info(struct drm_device *dev, void *data,
924 struct drm_file *file_priv)
925 {
926 struct drm_nouveau_gem_info *req = data;
927 struct drm_gem_object *gem;
928 int ret;
929
930 gem = drm_gem_object_lookup(file_priv, req->handle);
931 if (!gem)
932 return -ENOENT;
933
934 ret = nouveau_gem_info(file_priv, gem, req);
935 drm_gem_object_put_unlocked(gem);
936 return ret;
937 }
938
Linux with added FPC-III support