drivers/accel/ivpu/ivpu_gem.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2020-2023 Intel Corporation
   4  */
   5
   6 #include <linux/dma-buf.h>
   7 #include <linux/highmem.h>
   8 #include <linux/module.h>
   9 #include <linux/set_memory.h>
  10 #include <linux/xarray.h>
  11
  12 #include <drm/drm_cache.h>
  13 #include <drm/drm_debugfs.h>
  14 #include <drm/drm_file.h>
  15 #include <drm/drm_utils.h>
  16
  17 #include "ivpu_drv.h"
  18 #include "ivpu_gem.h"
  19 #include "ivpu_hw.h"
  20 #include "ivpu_mmu.h"
  21 #include "ivpu_mmu_context.h"
  22
  23 static const struct drm_gem_object_funcs ivpu_gem_funcs;
  24
  25 static inline void ivpu_dbg_bo(struct ivpu_device *vdev, struct ivpu_bo *bo, const char *action)
  26 {
  27         ivpu_dbg(vdev, BO,
  28                  "%6s: bo %8p vpu_addr %9llx size %8zu ctx %d has_pages %d dma_mapped %d mmu_mapped %d wc %d imported %d\n",
  29                  action, bo, bo->vpu_addr, ivpu_bo_size(bo), bo->ctx ? bo->ctx->id : 0,
  30                  (bool)bo->base.pages, (bool)bo->base.sgt, bo->mmu_mapped, bo->base.map_wc,
  31                  (bool)bo->base.base.import_attach);
  32 }
  33
  34 /*
  35  * ivpu_bo_pin() - pin the backing physical pages and map them to VPU.
  36  *
  37  * This function pins physical memory pages, then maps the physical pages
  38  * to IOMMU address space and finally updates the VPU MMU page tables
  39  * to allow the VPU to translate VPU address to IOMMU address.
  40  */
  41 int __must_check ivpu_bo_pin(struct ivpu_bo *bo)
  42 {
  43         struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
  44         int ret = 0;
  45
  46         mutex_lock(&bo->lock);
  47
  48         ivpu_dbg_bo(vdev, bo, "pin");
  49         drm_WARN_ON(&vdev->drm, !bo->ctx);
  50
  51         if (!bo->mmu_mapped) {
  52                 struct sg_table *sgt = drm_gem_shmem_get_pages_sgt(&bo->base);
  53
  54                 if (IS_ERR(sgt)) {
  55                         ret = PTR_ERR(sgt);
  56                         ivpu_err(vdev, "Failed to map BO in IOMMU: %d\n", ret);
  57                         goto unlock;
  58                 }
  59
  60                 ret = ivpu_mmu_context_map_sgt(vdev, bo->ctx, bo->vpu_addr, sgt,
  61                                                ivpu_bo_is_snooped(bo));
  62                 if (ret) {
  63                         ivpu_err(vdev, "Failed to map BO in MMU: %d\n", ret);
  64                         goto unlock;
  65                 }
  66                 bo->mmu_mapped = true;
  67         }
  68
  69 unlock:
  70         mutex_unlock(&bo->lock);
  71
  72         return ret;
  73 }
  74
  75 static int
  76 ivpu_bo_alloc_vpu_addr(struct ivpu_bo *bo, struct ivpu_mmu_context *ctx,
  77                        const struct ivpu_addr_range *range)
  78 {
  79         struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
  80         int idx, ret;
  81
  82         if (!drm_dev_enter(&vdev->drm, &idx))
  83                 return -ENODEV;
  84
  85         mutex_lock(&bo->lock);
  86
  87         ret = ivpu_mmu_context_insert_node(ctx, range, ivpu_bo_size(bo), &bo->mm_node);
  88         if (!ret) {
  89                 bo->ctx = ctx;
  90                 bo->vpu_addr = bo->mm_node.start;
  91         } else {
  92                 ivpu_err(vdev, "Failed to add BO to context %u: %d\n", ctx->id, ret);
  93         }
  94
  95         ivpu_dbg_bo(vdev, bo, "alloc");
  96
  97         mutex_unlock(&bo->lock);
  98
  99         drm_dev_exit(idx);
 100
 101         return ret;
 102 }
 103
 104 static void ivpu_bo_unbind_locked(struct ivpu_bo *bo)
 105 {
 106         struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
 107
 108         lockdep_assert(lockdep_is_held(&bo->lock) || !kref_read(&bo->base.base.refcount));
 109
 110         if (bo->mmu_mapped) {
 111                 drm_WARN_ON(&vdev->drm, !bo->ctx);
 112                 drm_WARN_ON(&vdev->drm, !bo->vpu_addr);
 113                 drm_WARN_ON(&vdev->drm, !bo->base.sgt);
 114                 ivpu_mmu_context_unmap_sgt(vdev, bo->ctx, bo->vpu_addr, bo->base.sgt);
 115                 bo->mmu_mapped = false;
 116         }
 117
 118         if (bo->ctx) {
 119                 ivpu_mmu_context_remove_node(bo->ctx, &bo->mm_node);
 120                 bo->ctx = NULL;
 121         }
 122
 123         if (bo->base.base.import_attach)
 124                 return;
 125
 126         dma_resv_lock(bo->base.base.resv, NULL);
 127         if (bo->base.sgt) {
 128                 dma_unmap_sgtable(vdev->drm.dev, bo->base.sgt, DMA_BIDIRECTIONAL, 0);
 129                 sg_free_table(bo->base.sgt);
 130                 kfree(bo->base.sgt);
 131                 bo->base.sgt = NULL;
 132         }
 133         dma_resv_unlock(bo->base.base.resv);
 134 }
 135
 136 void ivpu_bo_unbind_all_bos_from_context(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx)
 137 {
 138         struct ivpu_bo *bo;
 139
 140         if (drm_WARN_ON(&vdev->drm, !ctx))
 141                 return;
 142
 143         mutex_lock(&vdev->bo_list_lock);
 144         list_for_each_entry(bo, &vdev->bo_list, bo_list_node) {
 145                 mutex_lock(&bo->lock);
 146                 if (bo->ctx == ctx) {
 147                         ivpu_dbg_bo(vdev, bo, "unbind");
 148                         ivpu_bo_unbind_locked(bo);
 149                 }
 150                 mutex_unlock(&bo->lock);
 151         }
 152         mutex_unlock(&vdev->bo_list_lock);
 153 }
 154
 155 struct drm_gem_object *ivpu_gem_create_object(struct drm_device *dev, size_t size)
 156 {
 157         struct ivpu_bo *bo;
 158
 159         if (size == 0 || !PAGE_ALIGNED(size))
 160                 return ERR_PTR(-EINVAL);
 161
 162         bo = kzalloc(sizeof(*bo), GFP_KERNEL);
 163         if (!bo)
 164                 return ERR_PTR(-ENOMEM);
 165
 166         bo->base.base.funcs = &ivpu_gem_funcs;
 167         bo->base.pages_mark_dirty_on_put = true; /* VPU can dirty a BO anytime */
 168
 169         INIT_LIST_HEAD(&bo->bo_list_node);
 170         mutex_init(&bo->lock);
 171
 172         return &bo->base.base;
 173 }
 174
 175 static struct ivpu_bo *ivpu_bo_alloc(struct ivpu_device *vdev, u64 size, u32 flags)
 176 {
 177         struct drm_gem_shmem_object *shmem;
 178         struct ivpu_bo *bo;
 179
 180         switch (flags & DRM_IVPU_BO_CACHE_MASK) {
 181         case DRM_IVPU_BO_CACHED:
 182         case DRM_IVPU_BO_WC:
 183                 break;
 184         default:
 185                 return ERR_PTR(-EINVAL);
 186         }
 187
 188         shmem = drm_gem_shmem_create(&vdev->drm, size);
 189         if (IS_ERR(shmem))
 190                 return ERR_CAST(shmem);
 191
 192         bo = to_ivpu_bo(&shmem->base);
 193         bo->base.map_wc = flags & DRM_IVPU_BO_WC;
 194         bo->flags = flags;
 195
 196         mutex_lock(&vdev->bo_list_lock);
 197         list_add_tail(&bo->bo_list_node, &vdev->bo_list);
 198         mutex_unlock(&vdev->bo_list_lock);
 199
 200         return bo;
 201 }
 202
 203 static int ivpu_gem_bo_open(struct drm_gem_object *obj, struct drm_file *file)
 204 {
 205         struct ivpu_file_priv *file_priv = file->driver_priv;
 206         struct ivpu_device *vdev = file_priv->vdev;
 207         struct ivpu_bo *bo = to_ivpu_bo(obj);
 208         struct ivpu_addr_range *range;
 209
 210         if (bo->ctx) {
 211                 ivpu_warn(vdev, "Can't add BO to ctx %u: already in ctx %u\n",
 212                           file_priv->ctx.id, bo->ctx->id);
 213                 return -EALREADY;
 214         }
 215
 216         if (bo->flags & DRM_IVPU_BO_SHAVE_MEM)
 217                 range = &vdev->hw->ranges.shave;
 218         else if (bo->flags & DRM_IVPU_BO_DMA_MEM)
 219                 range = &vdev->hw->ranges.dma;
 220         else
 221                 range = &vdev->hw->ranges.user;
 222
 223         return ivpu_bo_alloc_vpu_addr(bo, &file_priv->ctx, range);
 224 }
 225
 226 static void ivpu_gem_bo_free(struct drm_gem_object *obj)
 227 {
 228         struct ivpu_device *vdev = to_ivpu_device(obj->dev);
 229         struct ivpu_bo *bo = to_ivpu_bo(obj);
 230
 231         ivpu_dbg_bo(vdev, bo, "free");
 232
 233         mutex_lock(&vdev->bo_list_lock);
 234         list_del(&bo->bo_list_node);
 235         mutex_unlock(&vdev->bo_list_lock);
 236
 237         drm_WARN_ON(&vdev->drm, !dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_READ));
 238
 239         ivpu_bo_unbind_locked(bo);
 240         mutex_destroy(&bo->lock);
 241
 242         drm_WARN_ON(obj->dev, bo->base.pages_use_count > 1);
 243         drm_gem_shmem_free(&bo->base);
 244 }
 245
 246 static const struct drm_gem_object_funcs ivpu_gem_funcs = {
 247         .free = ivpu_gem_bo_free,
 248         .open = ivpu_gem_bo_open,
 249         .print_info = drm_gem_shmem_object_print_info,
 250         .pin = drm_gem_shmem_object_pin,
 251         .unpin = drm_gem_shmem_object_unpin,
 252         .get_sg_table = drm_gem_shmem_object_get_sg_table,
 253         .vmap = drm_gem_shmem_object_vmap,
 254         .vunmap = drm_gem_shmem_object_vunmap,
 255         .mmap = drm_gem_shmem_object_mmap,
 256         .vm_ops = &drm_gem_shmem_vm_ops,
 257 };
 258
 259 int ivpu_bo_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 260 {
 261         struct ivpu_file_priv *file_priv = file->driver_priv;
 262         struct ivpu_device *vdev = file_priv->vdev;
 263         struct drm_ivpu_bo_create *args = data;
 264         u64 size = PAGE_ALIGN(args->size);
 265         struct ivpu_bo *bo;
 266         int ret;
 267
 268         if (args->flags & ~DRM_IVPU_BO_FLAGS)
 269                 return -EINVAL;
 270
 271         if (size == 0)
 272                 return -EINVAL;
 273
 274         bo = ivpu_bo_alloc(vdev, size, args->flags);
 275         if (IS_ERR(bo)) {
 276                 ivpu_err(vdev, "Failed to allocate BO: %pe (ctx %u size %llu flags 0x%x)",
 277                          bo, file_priv->ctx.id, args->size, args->flags);
 278                 return PTR_ERR(bo);
 279         }
 280
 281         ret = drm_gem_handle_create(file, &bo->base.base, &args->handle);
 282         if (!ret)
 283                 args->vpu_addr = bo->vpu_addr;
 284
 285         drm_gem_object_put(&bo->base.base);
 286
 287         return ret;
 288 }
 289
 290 struct ivpu_bo *
 291 ivpu_bo_create(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
 292                struct ivpu_addr_range *range, u64 size, u32 flags)
 293 {
 294         struct iosys_map map;
 295         struct ivpu_bo *bo;
 296         int ret;
 297
 298         if (drm_WARN_ON(&vdev->drm, !range))
 299                 return NULL;
 300
 301         drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(range->start));
 302         drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(range->end));
 303         drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(size));
 304
 305         bo = ivpu_bo_alloc(vdev, size, flags);
 306         if (IS_ERR(bo)) {
 307                 ivpu_err(vdev, "Failed to allocate BO: %pe (vpu_addr 0x%llx size %llu flags 0x%x)",
 308                          bo, range->start, size, flags);
 309                 return NULL;
 310         }
 311
 312         ret = ivpu_bo_alloc_vpu_addr(bo, ctx, range);
 313         if (ret)
 314                 goto err_put;
 315
 316         ret = ivpu_bo_pin(bo);
 317         if (ret)
 318                 goto err_put;
 319
 320         if (flags & DRM_IVPU_BO_MAPPABLE) {
 321                 dma_resv_lock(bo->base.base.resv, NULL);
 322                 ret = drm_gem_shmem_vmap(&bo->base, &map);
 323                 dma_resv_unlock(bo->base.base.resv);
 324
 325                 if (ret)
 326                         goto err_put;
 327         }
 328
 329         return bo;
 330
 331 err_put:
 332         drm_gem_object_put(&bo->base.base);
 333         return NULL;
 334 }
 335
 336 struct ivpu_bo *ivpu_bo_create_global(struct ivpu_device *vdev, u64 size, u32 flags)
 337 {
 338         return ivpu_bo_create(vdev, &vdev->gctx, &vdev->hw->ranges.global, size, flags);
 339 }
 340
 341 void ivpu_bo_free(struct ivpu_bo *bo)
 342 {
 343         struct iosys_map map = IOSYS_MAP_INIT_VADDR(bo->base.vaddr);
 344
 345         if (bo->flags & DRM_IVPU_BO_MAPPABLE) {
 346                 dma_resv_lock(bo->base.base.resv, NULL);
 347                 drm_gem_shmem_vunmap(&bo->base, &map);
 348                 dma_resv_unlock(bo->base.base.resv);
 349         }
 350
 351         drm_gem_object_put(&bo->base.base);
 352 }
 353
 354 int ivpu_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 355 {
 356         struct drm_ivpu_bo_info *args = data;
 357         struct drm_gem_object *obj;
 358         struct ivpu_bo *bo;
 359         int ret = 0;
 360
 361         obj = drm_gem_object_lookup(file, args->handle);
 362         if (!obj)
 363                 return -ENOENT;
 364
 365         bo = to_ivpu_bo(obj);
 366
 367         mutex_lock(&bo->lock);
 368         args->flags = bo->flags;
 369         args->mmap_offset = drm_vma_node_offset_addr(&obj->vma_node);
 370         args->vpu_addr = bo->vpu_addr;
 371         args->size = obj->size;
 372         mutex_unlock(&bo->lock);
 373
 374         drm_gem_object_put(obj);
 375         return ret;
 376 }
 377
 378 int ivpu_bo_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 379 {
 380         struct drm_ivpu_bo_wait *args = data;
 381         struct drm_gem_object *obj;
 382         unsigned long timeout;
 383         long ret;
 384
 385         timeout = drm_timeout_abs_to_jiffies(args->timeout_ns);
 386
 387         /* Add 1 jiffy to ensure the wait function never times out before intended timeout_ns */
 388         timeout += 1;
 389
 390         obj = drm_gem_object_lookup(file, args->handle);
 391         if (!obj)
 392                 return -EINVAL;
 393
 394         ret = dma_resv_wait_timeout(obj->resv, DMA_RESV_USAGE_READ, true, timeout);
 395         if (ret == 0) {
 396                 ret = -ETIMEDOUT;
 397         } else if (ret > 0) {
 398                 ret = 0;
 399                 args->job_status = to_ivpu_bo(obj)->job_status;
 400         }
 401
 402         drm_gem_object_put(obj);
 403
 404         return ret;
 405 }
 406
 407 static void ivpu_bo_print_info(struct ivpu_bo *bo, struct drm_printer *p)
 408 {
 409         mutex_lock(&bo->lock);
 410
 411         drm_printf(p, "%-9p %-3u 0x%-12llx %-10lu 0x%-8x %-4u",
 412                    bo, bo->ctx->id, bo->vpu_addr, bo->base.base.size,
 413                    bo->flags, kref_read(&bo->base.base.refcount));
 414
 415         if (bo->base.pages)
 416                 drm_printf(p, " has_pages");
 417
 418         if (bo->mmu_mapped)
 419                 drm_printf(p, " mmu_mapped");
 420
 421         if (bo->base.base.import_attach)
 422                 drm_printf(p, " imported");
 423
 424         drm_printf(p, "\n");
 425
 426         mutex_unlock(&bo->lock);
 427 }
 428
 429 void ivpu_bo_list(struct drm_device *dev, struct drm_printer *p)
 430 {
 431         struct ivpu_device *vdev = to_ivpu_device(dev);
 432         struct ivpu_bo *bo;
 433
 434         drm_printf(p, "%-9s %-3s %-14s %-10s %-10s %-4s %s\n",
 435                    "bo", "ctx", "vpu_addr", "size", "flags", "refs", "attribs");
 436
 437         mutex_lock(&vdev->bo_list_lock);
 438         list_for_each_entry(bo, &vdev->bo_list, bo_list_node)
 439                 ivpu_bo_print_info(bo, p);
 440         mutex_unlock(&vdev->bo_list_lock);
 441 }
 442
 443 void ivpu_bo_list_print(struct drm_device *dev)
 444 {
 445         struct drm_printer p = drm_info_printer(dev->dev);
 446
 447         ivpu_bo_list(dev, &p);
 448 }