drivers/gpu/drm/tegra/gem.c

   1 /*
   2  * NVIDIA Tegra DRM GEM helper functions
   3  *
   4  * Copyright (C) 2012 Sascha Hauer, Pengutronix
   5  * Copyright (C) 2013 NVIDIA CORPORATION, All rights reserved.
   6  *
   7  * Based on the GEM/CMA helpers
   8  *
   9  * Copyright (c) 2011 Samsung Electronics Co., Ltd.
  10  *
  11  * This program is free software; you can redistribute it and/or modify
  12  * it under the terms of the GNU General Public License version 2 as
  13  * published by the Free Software Foundation.
  14  */
  15
  16 #include <linux/dma-buf.h>
  17 #include <linux/iommu.h>
  18 #include <drm/tegra_drm.h>
  19
  20 #include "drm.h"
  21 #include "gem.h"
  22
  23 static inline struct tegra_bo *host1x_to_tegra_bo(struct host1x_bo *bo)
  24 {
  25         return container_of(bo, struct tegra_bo, base);
  26 }
  27
  28 static void tegra_bo_put(struct host1x_bo *bo)
  29 {
  30         struct tegra_bo *obj = host1x_to_tegra_bo(bo);
  31         struct drm_device *drm = obj->gem.dev;
  32
  33         mutex_lock(&drm->struct_mutex);
  34         drm_gem_object_unreference(&obj->gem);
  35         mutex_unlock(&drm->struct_mutex);
  36 }
  37
  38 static dma_addr_t tegra_bo_pin(struct host1x_bo *bo, struct sg_table **sgt)
  39 {
  40         struct tegra_bo *obj = host1x_to_tegra_bo(bo);
  41
  42         return obj->paddr;
  43 }
  44
  45 static void tegra_bo_unpin(struct host1x_bo *bo, struct sg_table *sgt)
  46 {
  47 }
  48
  49 static void *tegra_bo_mmap(struct host1x_bo *bo)
  50 {
  51         struct tegra_bo *obj = host1x_to_tegra_bo(bo);
  52
  53         return obj->vaddr;
  54 }
  55
  56 static void tegra_bo_munmap(struct host1x_bo *bo, void *addr)
  57 {
  58 }
  59
  60 static void *tegra_bo_kmap(struct host1x_bo *bo, unsigned int page)
  61 {
  62         struct tegra_bo *obj = host1x_to_tegra_bo(bo);
  63
  64         return obj->vaddr + page * PAGE_SIZE;
  65 }
  66
  67 static void tegra_bo_kunmap(struct host1x_bo *bo, unsigned int page,
  68                             void *addr)
  69 {
  70 }
  71
  72 static struct host1x_bo *tegra_bo_get(struct host1x_bo *bo)
  73 {
  74         struct tegra_bo *obj = host1x_to_tegra_bo(bo);
  75         struct drm_device *drm = obj->gem.dev;
  76
  77         mutex_lock(&drm->struct_mutex);
  78         drm_gem_object_reference(&obj->gem);
  79         mutex_unlock(&drm->struct_mutex);
  80
  81         return bo;
  82 }
  83
  84 static const struct host1x_bo_ops tegra_bo_ops = {
  85         .get = tegra_bo_get,
  86         .put = tegra_bo_put,
  87         .pin = tegra_bo_pin,
  88         .unpin = tegra_bo_unpin,
  89         .mmap = tegra_bo_mmap,
  90         .munmap = tegra_bo_munmap,
  91         .kmap = tegra_bo_kmap,
  92         .kunmap = tegra_bo_kunmap,
  93 };
  94
  95 static int tegra_bo_iommu_map(struct tegra_drm *tegra, struct tegra_bo *bo)
  96 {
  97         int prot = IOMMU_READ | IOMMU_WRITE;
  98         ssize_t err;
  99
 100         if (bo->mm)
 101                 return -EBUSY;
 102
 103         bo->mm = kzalloc(sizeof(*bo->mm), GFP_KERNEL);
 104         if (!bo->mm)
 105                 return -ENOMEM;
 106
 107         err = drm_mm_insert_node_generic(&tegra->mm, bo->mm, bo->gem.size,
 108                                          PAGE_SIZE, 0, 0, 0);
 109         if (err < 0) {
 110                 dev_err(tegra->drm->dev, "out of I/O virtual memory: %zd\n",
 111                         err);
 112                 goto free;
 113         }
 114
 115         bo->paddr = bo->mm->start;
 116
 117         err = iommu_map_sg(tegra->domain, bo->paddr, bo->sgt->sgl,
 118                            bo->sgt->nents, prot);
 119         if (err < 0) {
 120                 dev_err(tegra->drm->dev, "failed to map buffer: %zd\n", err);
 121                 goto remove;
 122         }
 123
 124         bo->size = err;
 125
 126         return 0;
 127
 128 remove:
 129         drm_mm_remove_node(bo->mm);
 130 free:
 131         kfree(bo->mm);
 132         return err;
 133 }
 134
 135 static int tegra_bo_iommu_unmap(struct tegra_drm *tegra, struct tegra_bo *bo)
 136 {
 137         if (!bo->mm)
 138                 return 0;
 139
 140         iommu_unmap(tegra->domain, bo->paddr, bo->size);
 141         drm_mm_remove_node(bo->mm);
 142         kfree(bo->mm);
 143
 144         return 0;
 145 }
 146
 147 static struct tegra_bo *tegra_bo_alloc_object(struct drm_device *drm,
 148                                               size_t size)
 149 {
 150         struct tegra_bo *bo;
 151         int err;
 152
 153         bo = kzalloc(sizeof(*bo), GFP_KERNEL);
 154         if (!bo)
 155                 return ERR_PTR(-ENOMEM);
 156
 157         host1x_bo_init(&bo->base, &tegra_bo_ops);
 158         size = round_up(size, PAGE_SIZE);
 159
 160         err = drm_gem_object_init(drm, &bo->gem, size);
 161         if (err < 0)
 162                 goto free;
 163
 164         err = drm_gem_create_mmap_offset(&bo->gem);
 165         if (err < 0)
 166                 goto release;
 167
 168         return bo;
 169
 170 release:
 171         drm_gem_object_release(&bo->gem);
 172 free:
 173         kfree(bo);
 174         return ERR_PTR(err);
 175 }
 176
 177 static void tegra_bo_free(struct drm_device *drm, struct tegra_bo *bo)
 178 {
 179         if (bo->pages) {
 180                 drm_gem_put_pages(&bo->gem, bo->pages, true, true);
 181                 sg_free_table(bo->sgt);
 182                 kfree(bo->sgt);
 183         } else if (bo->vaddr) {
 184                 dma_free_writecombine(drm->dev, bo->gem.size, bo->vaddr,
 185                                       bo->paddr);
 186         }
 187 }
 188
 189 static int tegra_bo_get_pages(struct drm_device *drm, struct tegra_bo *bo)
 190 {
 191         struct scatterlist *s;
 192         unsigned int i;
 193
 194         bo->pages = drm_gem_get_pages(&bo->gem);
 195         if (IS_ERR(bo->pages))
 196                 return PTR_ERR(bo->pages);
 197
 198         bo->num_pages = bo->gem.size >> PAGE_SHIFT;
 199
 200         bo->sgt = drm_prime_pages_to_sg(bo->pages, bo->num_pages);
 201         if (IS_ERR(bo->sgt))
 202                 goto put_pages;
 203
 204         /*
 205          * Fake up the SG table so that dma_sync_sg_for_device() can be used
 206          * to flush the pages associated with it.
 207          *
 208          * TODO: Replace this by drm_clflash_sg() once it can be implemented
 209          * without relying on symbols that are not exported.
 210          */
 211         for_each_sg(bo->sgt->sgl, s, bo->sgt->nents, i)
 212                 sg_dma_address(s) = sg_phys(s);
 213
 214         dma_sync_sg_for_device(drm->dev, bo->sgt->sgl, bo->sgt->nents,
 215                                DMA_TO_DEVICE);
 216
 217         return 0;
 218
 219 put_pages:
 220         drm_gem_put_pages(&bo->gem, bo->pages, false, false);
 221         return PTR_ERR(bo->sgt);
 222 }
 223
 224 static int tegra_bo_alloc(struct drm_device *drm, struct tegra_bo *bo)
 225 {
 226         struct tegra_drm *tegra = drm->dev_private;
 227         int err;
 228
 229         if (tegra->domain) {
 230                 err = tegra_bo_get_pages(drm, bo);
 231                 if (err < 0)
 232                         return err;
 233
 234                 err = tegra_bo_iommu_map(tegra, bo);
 235                 if (err < 0) {
 236                         tegra_bo_free(drm, bo);
 237                         return err;
 238                 }
 239         } else {
 240                 size_t size = bo->gem.size;
 241
 242                 bo->vaddr = dma_alloc_writecombine(drm->dev, size, &bo->paddr,
 243                                                    GFP_KERNEL | __GFP_NOWARN);
 244                 if (!bo->vaddr) {
 245                         dev_err(drm->dev,
 246                                 "failed to allocate buffer of size %zu\n",
 247                                 size);
 248                         return -ENOMEM;
 249                 }
 250         }
 251
 252         return 0;
 253 }
 254
 255 struct tegra_bo *tegra_bo_create(struct drm_device *drm, size_t size,
 256                                  unsigned long flags)
 257 {
 258         struct tegra_bo *bo;
 259         int err;
 260
 261         bo = tegra_bo_alloc_object(drm, size);
 262         if (IS_ERR(bo))
 263                 return bo;
 264
 265         err = tegra_bo_alloc(drm, bo);
 266         if (err < 0)
 267                 goto release;
 268
 269         if (flags & DRM_TEGRA_GEM_CREATE_TILED)
 270                 bo->tiling.mode = TEGRA_BO_TILING_MODE_TILED;
 271
 272         if (flags & DRM_TEGRA_GEM_CREATE_BOTTOM_UP)
 273                 bo->flags |= TEGRA_BO_BOTTOM_UP;
 274
 275         return bo;
 276
 277 release:
 278         drm_gem_object_release(&bo->gem);
 279         kfree(bo);
 280         return ERR_PTR(err);
 281 }
 282
 283 struct tegra_bo *tegra_bo_create_with_handle(struct drm_file *file,
 284                                              struct drm_device *drm,
 285                                              size_t size,
 286                                              unsigned long flags,
 287                                              u32 *handle)
 288 {
 289         struct tegra_bo *bo;
 290         int err;
 291
 292         bo = tegra_bo_create(drm, size, flags);
 293         if (IS_ERR(bo))
 294                 return bo;
 295
 296         err = drm_gem_handle_create(file, &bo->gem, handle);
 297         if (err) {
 298                 tegra_bo_free_object(&bo->gem);
 299                 return ERR_PTR(err);
 300         }
 301
 302         drm_gem_object_unreference_unlocked(&bo->gem);
 303
 304         return bo;
 305 }
 306
 307 static struct tegra_bo *tegra_bo_import(struct drm_device *drm,
 308                                         struct dma_buf *buf)
 309 {
 310         struct tegra_drm *tegra = drm->dev_private;
 311         struct dma_buf_attachment *attach;
 312         struct tegra_bo *bo;
 313         int err;
 314
 315         bo = tegra_bo_alloc_object(drm, buf->size);
 316         if (IS_ERR(bo))
 317                 return bo;
 318
 319         attach = dma_buf_attach(buf, drm->dev);
 320         if (IS_ERR(attach)) {
 321                 err = PTR_ERR(attach);
 322                 goto free;
 323         }
 324
 325         get_dma_buf(buf);
 326
 327         bo->sgt = dma_buf_map_attachment(attach, DMA_TO_DEVICE);
 328         if (!bo->sgt) {
 329                 err = -ENOMEM;
 330                 goto detach;
 331         }
 332
 333         if (IS_ERR(bo->sgt)) {
 334                 err = PTR_ERR(bo->sgt);
 335                 goto detach;
 336         }
 337
 338         if (tegra->domain) {
 339                 err = tegra_bo_iommu_map(tegra, bo);
 340                 if (err < 0)
 341                         goto detach;
 342         } else {
 343                 if (bo->sgt->nents > 1) {
 344                         err = -EINVAL;
 345                         goto detach;
 346                 }
 347
 348                 bo->paddr = sg_dma_address(bo->sgt->sgl);
 349         }
 350
 351         bo->gem.import_attach = attach;
 352
 353         return bo;
 354
 355 detach:
 356         if (!IS_ERR_OR_NULL(bo->sgt))
 357                 dma_buf_unmap_attachment(attach, bo->sgt, DMA_TO_DEVICE);
 358
 359         dma_buf_detach(buf, attach);
 360         dma_buf_put(buf);
 361 free:
 362         drm_gem_object_release(&bo->gem);
 363         kfree(bo);
 364         return ERR_PTR(err);
 365 }
 366
 367 void tegra_bo_free_object(struct drm_gem_object *gem)
 368 {
 369         struct tegra_drm *tegra = gem->dev->dev_private;
 370         struct tegra_bo *bo = to_tegra_bo(gem);
 371
 372         if (tegra->domain)
 373                 tegra_bo_iommu_unmap(tegra, bo);
 374
 375         if (gem->import_attach) {
 376                 dma_buf_unmap_attachment(gem->import_attach, bo->sgt,
 377                                          DMA_TO_DEVICE);
 378                 drm_prime_gem_destroy(gem, NULL);
 379         } else {
 380                 tegra_bo_free(gem->dev, bo);
 381         }
 382
 383         drm_gem_object_release(gem);
 384         kfree(bo);
 385 }
 386
 387 int tegra_bo_dumb_create(struct drm_file *file, struct drm_device *drm,
 388                          struct drm_mode_create_dumb *args)
 389 {
 390         unsigned int min_pitch = DIV_ROUND_UP(args->width * args->bpp, 8);
 391         struct tegra_drm *tegra = drm->dev_private;
 392         struct tegra_bo *bo;
 393
 394         args->pitch = round_up(min_pitch, tegra->pitch_align);
 395         args->size = args->pitch * args->height;
 396
 397         bo = tegra_bo_create_with_handle(file, drm, args->size, 0,
 398                                          &args->handle);
 399         if (IS_ERR(bo))
 400                 return PTR_ERR(bo);
 401
 402         return 0;
 403 }
 404
 405 int tegra_bo_dumb_map_offset(struct drm_file *file, struct drm_device *drm,
 406                              u32 handle, u64 *offset)
 407 {
 408         struct drm_gem_object *gem;
 409         struct tegra_bo *bo;
 410
 411         mutex_lock(&drm->struct_mutex);
 412
 413         gem = drm_gem_object_lookup(drm, file, handle);
 414         if (!gem) {
 415                 dev_err(drm->dev, "failed to lookup GEM object\n");
 416                 mutex_unlock(&drm->struct_mutex);
 417                 return -EINVAL;
 418         }
 419
 420         bo = to_tegra_bo(gem);
 421
 422         *offset = drm_vma_node_offset_addr(&bo->gem.vma_node);
 423
 424         drm_gem_object_unreference(gem);
 425
 426         mutex_unlock(&drm->struct_mutex);
 427
 428         return 0;
 429 }
 430
 431 static int tegra_bo_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 432 {
 433         struct drm_gem_object *gem = vma->vm_private_data;
 434         struct tegra_bo *bo = to_tegra_bo(gem);
 435         struct page *page;
 436         pgoff_t offset;
 437         int err;
 438
 439         if (!bo->pages)
 440                 return VM_FAULT_SIGBUS;
 441
 442         offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >> PAGE_SHIFT;
 443         page = bo->pages[offset];
 444
 445         err = vm_insert_page(vma, (unsigned long)vmf->virtual_address, page);
 446         switch (err) {
 447         case -EAGAIN:
 448         case 0:
 449         case -ERESTARTSYS:
 450         case -EINTR:
 451         case -EBUSY:
 452                 return VM_FAULT_NOPAGE;
 453
 454         case -ENOMEM:
 455                 return VM_FAULT_OOM;
 456         }
 457
 458         return VM_FAULT_SIGBUS;
 459 }
 460
 461 const struct vm_operations_struct tegra_bo_vm_ops = {
 462         .fault = tegra_bo_fault,
 463         .open = drm_gem_vm_open,
 464         .close = drm_gem_vm_close,
 465 };
 466
 467 int tegra_drm_mmap(struct file *file, struct vm_area_struct *vma)
 468 {
 469         struct drm_gem_object *gem;
 470         struct tegra_bo *bo;
 471         int ret;
 472
 473         ret = drm_gem_mmap(file, vma);
 474         if (ret)
 475                 return ret;
 476
 477         gem = vma->vm_private_data;
 478         bo = to_tegra_bo(gem);
 479
 480         if (!bo->pages) {
 481                 unsigned long vm_pgoff = vma->vm_pgoff;
 482
 483                 vma->vm_flags &= ~VM_PFNMAP;
 484                 vma->vm_pgoff = 0;
 485
 486                 ret = dma_mmap_writecombine(gem->dev->dev, vma, bo->vaddr,
 487                                             bo->paddr, gem->size);
 488                 if (ret) {
 489                         drm_gem_vm_close(vma);
 490                         return ret;
 491                 }
 492
 493                 vma->vm_pgoff = vm_pgoff;
 494         } else {
 495                 pgprot_t prot = vm_get_page_prot(vma->vm_flags);
 496
 497                 vma->vm_flags |= VM_MIXEDMAP;
 498                 vma->vm_flags &= ~VM_PFNMAP;
 499
 500                 vma->vm_page_prot = pgprot_writecombine(prot);
 501         }
 502
 503         return 0;
 504 }
 505
 506 static struct sg_table *
 507 tegra_gem_prime_map_dma_buf(struct dma_buf_attachment *attach,
 508                             enum dma_data_direction dir)
 509 {
 510         struct drm_gem_object *gem = attach->dmabuf->priv;
 511         struct tegra_bo *bo = to_tegra_bo(gem);
 512         struct sg_table *sgt;
 513
 514         sgt = kmalloc(sizeof(*sgt), GFP_KERNEL);
 515         if (!sgt)
 516                 return NULL;
 517
 518         if (bo->pages) {
 519                 struct scatterlist *sg;
 520                 unsigned int i;
 521
 522                 if (sg_alloc_table(sgt, bo->num_pages, GFP_KERNEL))
 523                         goto free;
 524
 525                 for_each_sg(sgt->sgl, sg, bo->num_pages, i)
 526                         sg_set_page(sg, bo->pages[i], PAGE_SIZE, 0);
 527
 528                 if (dma_map_sg(attach->dev, sgt->sgl, sgt->nents, dir) == 0)
 529                         goto free;
 530         } else {
 531                 if (sg_alloc_table(sgt, 1, GFP_KERNEL))
 532                         goto free;
 533
 534                 sg_dma_address(sgt->sgl) = bo->paddr;
 535                 sg_dma_len(sgt->sgl) = gem->size;
 536         }
 537
 538         return sgt;
 539
 540 free:
 541         sg_free_table(sgt);
 542         kfree(sgt);
 543         return NULL;
 544 }
 545
 546 static void tegra_gem_prime_unmap_dma_buf(struct dma_buf_attachment *attach,
 547                                           struct sg_table *sgt,
 548                                           enum dma_data_direction dir)
 549 {
 550         struct drm_gem_object *gem = attach->dmabuf->priv;
 551         struct tegra_bo *bo = to_tegra_bo(gem);
 552
 553         if (bo->pages)
 554                 dma_unmap_sg(attach->dev, sgt->sgl, sgt->nents, dir);
 555
 556         sg_free_table(sgt);
 557         kfree(sgt);
 558 }
 559
 560 static void tegra_gem_prime_release(struct dma_buf *buf)
 561 {
 562         drm_gem_dmabuf_release(buf);
 563 }
 564
 565 static void *tegra_gem_prime_kmap_atomic(struct dma_buf *buf,
 566                                          unsigned long page)
 567 {
 568         return NULL;
 569 }
 570
 571 static void tegra_gem_prime_kunmap_atomic(struct dma_buf *buf,
 572                                           unsigned long page,
 573                                           void *addr)
 574 {
 575 }
 576
 577 static void *tegra_gem_prime_kmap(struct dma_buf *buf, unsigned long page)
 578 {
 579         return NULL;
 580 }
 581
 582 static void tegra_gem_prime_kunmap(struct dma_buf *buf, unsigned long page,
 583                                    void *addr)
 584 {
 585 }
 586
 587 static int tegra_gem_prime_mmap(struct dma_buf *buf, struct vm_area_struct *vma)
 588 {
 589         return -EINVAL;
 590 }
 591
 592 static void *tegra_gem_prime_vmap(struct dma_buf *buf)
 593 {
 594         struct drm_gem_object *gem = buf->priv;
 595         struct tegra_bo *bo = to_tegra_bo(gem);
 596
 597         return bo->vaddr;
 598 }
 599
 600 static void tegra_gem_prime_vunmap(struct dma_buf *buf, void *vaddr)
 601 {
 602 }
 603
 604 static const struct dma_buf_ops tegra_gem_prime_dmabuf_ops = {
 605         .map_dma_buf = tegra_gem_prime_map_dma_buf,
 606         .unmap_dma_buf = tegra_gem_prime_unmap_dma_buf,
 607         .release = tegra_gem_prime_release,
 608         .kmap_atomic = tegra_gem_prime_kmap_atomic,
 609         .kunmap_atomic = tegra_gem_prime_kunmap_atomic,
 610         .kmap = tegra_gem_prime_kmap,
 611         .kunmap = tegra_gem_prime_kunmap,
 612         .mmap = tegra_gem_prime_mmap,
 613         .vmap = tegra_gem_prime_vmap,
 614         .vunmap = tegra_gem_prime_vunmap,
 615 };
 616
 617 struct dma_buf *tegra_gem_prime_export(struct drm_device *drm,
 618                                        struct drm_gem_object *gem,
 619                                        int flags)
 620 {
 621         DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
 622
 623         exp_info.ops = &tegra_gem_prime_dmabuf_ops;
 624         exp_info.size = gem->size;
 625         exp_info.flags = flags;
 626         exp_info.priv = gem;
 627
 628         return dma_buf_export(&exp_info);
 629 }
 630
 631 struct drm_gem_object *tegra_gem_prime_import(struct drm_device *drm,
 632                                               struct dma_buf *buf)
 633 {
 634         struct tegra_bo *bo;
 635
 636         if (buf->ops == &tegra_gem_prime_dmabuf_ops) {
 637                 struct drm_gem_object *gem = buf->priv;
 638
 639                 if (gem->dev == drm) {
 640                         drm_gem_object_reference(gem);
 641                         return gem;
 642                 }
 643         }
 644
 645         bo = tegra_bo_import(drm, buf);
 646         if (IS_ERR(bo))
 647                 return ERR_CAST(bo);
 648
 649         return &bo->gem;
 650 }