drivers/misc/habanalabs/command_buffer.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /*
   4  * Copyright 2016-2019 HabanaLabs, Ltd.
   5  * All Rights Reserved.
   6  */
   7
   8 #include <uapi/misc/habanalabs.h>
   9 #include "habanalabs.h"
  10
  11 #include <linux/mm.h>
  12 #include <linux/slab.h>
  13
  14 static void cb_fini(struct hl_device *hdev, struct hl_cb *cb)
  15 {
  16         hdev->asic_funcs->asic_dma_free_coherent(hdev, cb->size,
  17                         (void *) (uintptr_t) cb->kernel_address,
  18                         cb->bus_address);
  19         kfree(cb);
  20 }
  21
  22 static void cb_do_release(struct hl_device *hdev, struct hl_cb *cb)
  23 {
  24         if (cb->is_pool) {
  25                 spin_lock(&hdev->cb_pool_lock);
  26                 list_add(&cb->pool_list, &hdev->cb_pool);
  27                 spin_unlock(&hdev->cb_pool_lock);
  28         } else {
  29                 cb_fini(hdev, cb);
  30         }
  31 }
  32
  33 static void cb_release(struct kref *ref)
  34 {
  35         struct hl_device *hdev;
  36         struct hl_cb *cb;
  37
  38         cb = container_of(ref, struct hl_cb, refcount);
  39         hdev = cb->hdev;
  40
  41         hl_debugfs_remove_cb(cb);
  42
  43         cb_do_release(hdev, cb);
  44 }
  45
  46 static struct hl_cb *hl_cb_alloc(struct hl_device *hdev, u32 cb_size,
  47                                         int ctx_id)
  48 {
  49         struct hl_cb *cb;
  50         void *p;
  51
  52         /*
  53          * We use of GFP_ATOMIC here because this function can be called from
  54          * the latency-sensitive code path for command submission. Due to H/W
  55          * limitations in some of the ASICs, the kernel must copy the user CB
  56          * that is designated for an external queue and actually enqueue
  57          * the kernel's copy. Hence, we must never sleep in this code section
  58          * and must use GFP_ATOMIC for all memory allocations.
  59          */
  60         if (ctx_id == HL_KERNEL_ASID_ID)
  61                 cb = kzalloc(sizeof(*cb), GFP_ATOMIC);
  62         else
  63                 cb = kzalloc(sizeof(*cb), GFP_KERNEL);
  64
  65         if (!cb)
  66                 return NULL;
  67
  68         if (ctx_id == HL_KERNEL_ASID_ID)
  69                 p = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, cb_size,
  70                                                 &cb->bus_address, GFP_ATOMIC);
  71         else
  72                 p = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, cb_size,
  73                                                 &cb->bus_address,
  74                                                 GFP_USER | __GFP_ZERO);
  75         if (!p) {
  76                 dev_err(hdev->dev,
  77                         "failed to allocate %d of dma memory for CB\n",
  78                         cb_size);
  79                 kfree(cb);
  80                 return NULL;
  81         }
  82
  83         cb->kernel_address = (u64) (uintptr_t) p;
  84         cb->size = cb_size;
  85
  86         return cb;
  87 }
  88
  89 int hl_cb_create(struct hl_device *hdev, struct hl_cb_mgr *mgr,
  90                         u32 cb_size, u64 *handle, int ctx_id)
  91 {
  92         struct hl_cb *cb;
  93         bool alloc_new_cb = true;
  94         int rc;
  95
  96         /*
  97          * Can't use generic function to check this because of special case
  98          * where we create a CB as part of the reset process
  99          */
 100         if ((hdev->disabled) || ((atomic_read(&hdev->in_reset)) &&
 101                                         (ctx_id != HL_KERNEL_ASID_ID))) {
 102                 dev_warn_ratelimited(hdev->dev,
 103                         "Device is disabled or in reset. Can't create new CBs\n");
 104                 rc = -EBUSY;
 105                 goto out_err;
 106         }
 107
 108         if (cb_size > HL_MAX_CB_SIZE) {
 109                 dev_err(hdev->dev,
 110                         "CB size %d must be less then %d\n",
 111                         cb_size, HL_MAX_CB_SIZE);
 112                 rc = -EINVAL;
 113                 goto out_err;
 114         }
 115
 116         /* Minimum allocation must be PAGE SIZE */
 117         if (cb_size < PAGE_SIZE)
 118                 cb_size = PAGE_SIZE;
 119
 120         if (ctx_id == HL_KERNEL_ASID_ID &&
 121                         cb_size <= hdev->asic_prop.cb_pool_cb_size) {
 122
 123                 spin_lock(&hdev->cb_pool_lock);
 124                 if (!list_empty(&hdev->cb_pool)) {
 125                         cb = list_first_entry(&hdev->cb_pool, typeof(*cb),
 126                                         pool_list);
 127                         list_del(&cb->pool_list);
 128                         spin_unlock(&hdev->cb_pool_lock);
 129                         alloc_new_cb = false;
 130                 } else {
 131                         spin_unlock(&hdev->cb_pool_lock);
 132                         dev_dbg(hdev->dev, "CB pool is empty\n");
 133                 }
 134         }
 135
 136         if (alloc_new_cb) {
 137                 cb = hl_cb_alloc(hdev, cb_size, ctx_id);
 138                 if (!cb) {
 139                         rc = -ENOMEM;
 140                         goto out_err;
 141                 }
 142         }
 143
 144         cb->hdev = hdev;
 145         cb->ctx_id = ctx_id;
 146
 147         spin_lock(&mgr->cb_lock);
 148         rc = idr_alloc(&mgr->cb_handles, cb, 1, 0, GFP_ATOMIC);
 149         spin_unlock(&mgr->cb_lock);
 150
 151         if (rc < 0) {
 152                 dev_err(hdev->dev, "Failed to allocate IDR for a new CB\n");
 153                 goto release_cb;
 154         }
 155
 156         cb->id = rc;
 157
 158         kref_init(&cb->refcount);
 159         spin_lock_init(&cb->lock);
 160
 161         /*
 162          * idr is 32-bit so we can safely OR it with a mask that is above
 163          * 32 bit
 164          */
 165         *handle = cb->id | HL_MMAP_CB_MASK;
 166         *handle <<= PAGE_SHIFT;
 167
 168         hl_debugfs_add_cb(cb);
 169
 170         return 0;
 171
 172 release_cb:
 173         cb_do_release(hdev, cb);
 174 out_err:
 175         *handle = 0;
 176
 177         return rc;
 178 }
 179
 180 int hl_cb_destroy(struct hl_device *hdev, struct hl_cb_mgr *mgr, u64 cb_handle)
 181 {
 182         struct hl_cb *cb;
 183         u32 handle;
 184         int rc = 0;
 185
 186         /*
 187          * handle was given to user to do mmap, I need to shift it back to
 188          * how the idr module gave it to me
 189          */
 190         cb_handle >>= PAGE_SHIFT;
 191         handle = (u32) cb_handle;
 192
 193         spin_lock(&mgr->cb_lock);
 194
 195         cb = idr_find(&mgr->cb_handles, handle);
 196         if (cb) {
 197                 idr_remove(&mgr->cb_handles, handle);
 198                 spin_unlock(&mgr->cb_lock);
 199                 kref_put(&cb->refcount, cb_release);
 200         } else {
 201                 spin_unlock(&mgr->cb_lock);
 202                 dev_err(hdev->dev,
 203                         "CB destroy failed, no match to handle 0x%x\n", handle);
 204                 rc = -EINVAL;
 205         }
 206
 207         return rc;
 208 }
 209
 210 int hl_cb_ioctl(struct hl_fpriv *hpriv, void *data)
 211 {
 212         union hl_cb_args *args = data;
 213         struct hl_device *hdev = hpriv->hdev;
 214         u64 handle;
 215         int rc;
 216
 217         if (hl_device_disabled_or_in_reset(hdev)) {
 218                 dev_warn_ratelimited(hdev->dev,
 219                         "Device is %s. Can't execute CB IOCTL\n",
 220                         atomic_read(&hdev->in_reset) ? "in_reset" : "disabled");
 221                 return -EBUSY;
 222         }
 223
 224         switch (args->in.op) {
 225         case HL_CB_OP_CREATE:
 226                 rc = hl_cb_create(hdev, &hpriv->cb_mgr, args->in.cb_size,
 227                                         &handle, hpriv->ctx->asid);
 228                 memset(args, 0, sizeof(*args));
 229                 args->out.cb_handle = handle;
 230                 break;
 231         case HL_CB_OP_DESTROY:
 232                 rc = hl_cb_destroy(hdev, &hpriv->cb_mgr,
 233                                         args->in.cb_handle);
 234                 break;
 235         default:
 236                 rc = -ENOTTY;
 237                 break;
 238         }
 239
 240         return rc;
 241 }
 242
 243 static void cb_vm_close(struct vm_area_struct *vma)
 244 {
 245         struct hl_cb *cb = (struct hl_cb *) vma->vm_private_data;
 246         long new_mmap_size;
 247
 248         new_mmap_size = cb->mmap_size - (vma->vm_end - vma->vm_start);
 249
 250         if (new_mmap_size > 0) {
 251                 cb->mmap_size = new_mmap_size;
 252                 return;
 253         }
 254
 255         spin_lock(&cb->lock);
 256         cb->mmap = false;
 257         spin_unlock(&cb->lock);
 258
 259         hl_cb_put(cb);
 260         vma->vm_private_data = NULL;
 261 }
 262
 263 static const struct vm_operations_struct cb_vm_ops = {
 264         .close = cb_vm_close
 265 };
 266
 267 int hl_cb_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma)
 268 {
 269         struct hl_device *hdev = hpriv->hdev;
 270         struct hl_cb *cb;
 271         phys_addr_t address;
 272         u32 handle;
 273         int rc;
 274
 275         handle = vma->vm_pgoff;
 276
 277         /* reference was taken here */
 278         cb = hl_cb_get(hdev, &hpriv->cb_mgr, handle);
 279         if (!cb) {
 280                 dev_err(hdev->dev,
 281                         "CB mmap failed, no match to handle %d\n", handle);
 282                 return -EINVAL;
 283         }
 284
 285         /* Validation check */
 286         if ((vma->vm_end - vma->vm_start) != ALIGN(cb->size, PAGE_SIZE)) {
 287                 dev_err(hdev->dev,
 288                         "CB mmap failed, mmap size 0x%lx != 0x%x cb size\n",
 289                         vma->vm_end - vma->vm_start, cb->size);
 290                 rc = -EINVAL;
 291                 goto put_cb;
 292         }
 293
 294         spin_lock(&cb->lock);
 295
 296         if (cb->mmap) {
 297                 dev_err(hdev->dev,
 298                         "CB mmap failed, CB already mmaped to user\n");
 299                 rc = -EINVAL;
 300                 goto release_lock;
 301         }
 302
 303         cb->mmap = true;
 304
 305         spin_unlock(&cb->lock);
 306
 307         vma->vm_ops = &cb_vm_ops;
 308
 309         /*
 310          * Note: We're transferring the cb reference to
 311          * vma->vm_private_data here.
 312          */
 313
 314         vma->vm_private_data = cb;
 315
 316         /* Calculate address for CB */
 317         address = virt_to_phys((void *) (uintptr_t) cb->kernel_address);
 318
 319         rc = hdev->asic_funcs->cb_mmap(hdev, vma, cb->kernel_address,
 320                                         address, cb->size);
 321
 322         if (rc) {
 323                 spin_lock(&cb->lock);
 324                 cb->mmap = false;
 325                 goto release_lock;
 326         }
 327
 328         cb->mmap_size = cb->size;
 329
 330         return 0;
 331
 332 release_lock:
 333         spin_unlock(&cb->lock);
 334 put_cb:
 335         hl_cb_put(cb);
 336         return rc;
 337 }
 338
 339 struct hl_cb *hl_cb_get(struct hl_device *hdev, struct hl_cb_mgr *mgr,
 340                         u32 handle)
 341 {
 342         struct hl_cb *cb;
 343
 344         spin_lock(&mgr->cb_lock);
 345         cb = idr_find(&mgr->cb_handles, handle);
 346
 347         if (!cb) {
 348                 spin_unlock(&mgr->cb_lock);
 349                 dev_warn(hdev->dev,
 350                         "CB get failed, no match to handle %d\n", handle);
 351                 return NULL;
 352         }
 353
 354         kref_get(&cb->refcount);
 355
 356         spin_unlock(&mgr->cb_lock);
 357
 358         return cb;
 359
 360 }
 361
 362 void hl_cb_put(struct hl_cb *cb)
 363 {
 364         kref_put(&cb->refcount, cb_release);
 365 }
 366
 367 void hl_cb_mgr_init(struct hl_cb_mgr *mgr)
 368 {
 369         spin_lock_init(&mgr->cb_lock);
 370         idr_init(&mgr->cb_handles);
 371 }
 372
 373 void hl_cb_mgr_fini(struct hl_device *hdev, struct hl_cb_mgr *mgr)
 374 {
 375         struct hl_cb *cb;
 376         struct idr *idp;
 377         u32 id;
 378
 379         idp = &mgr->cb_handles;
 380
 381         idr_for_each_entry(idp, cb, id) {
 382                 if (kref_put(&cb->refcount, cb_release) != 1)
 383                         dev_err(hdev->dev,
 384                                 "CB %d for CTX ID %d is still alive\n",
 385                                 id, cb->ctx_id);
 386         }
 387
 388         idr_destroy(&mgr->cb_handles);
 389 }
 390
 391 struct hl_cb *hl_cb_kernel_create(struct hl_device *hdev, u32 cb_size)
 392 {
 393         u64 cb_handle;
 394         struct hl_cb *cb;
 395         int rc;
 396
 397         rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr, cb_size, &cb_handle,
 398                         HL_KERNEL_ASID_ID);
 399         if (rc) {
 400                 dev_err(hdev->dev,
 401                         "Failed to allocate CB for the kernel driver %d\n", rc);
 402                 return NULL;
 403         }
 404
 405         cb_handle >>= PAGE_SHIFT;
 406         cb = hl_cb_get(hdev, &hdev->kernel_cb_mgr, (u32) cb_handle);
 407         /* hl_cb_get should never fail here so use kernel WARN */
 408         WARN(!cb, "Kernel CB handle invalid 0x%x\n", (u32) cb_handle);
 409         if (!cb)
 410                 goto destroy_cb;
 411
 412         return cb;
 413
 414 destroy_cb:
 415         hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, cb_handle << PAGE_SHIFT);
 416
 417         return NULL;
 418 }
 419
 420 int hl_cb_pool_init(struct hl_device *hdev)
 421 {
 422         struct hl_cb *cb;
 423         int i;
 424
 425         INIT_LIST_HEAD(&hdev->cb_pool);
 426         spin_lock_init(&hdev->cb_pool_lock);
 427
 428         for (i = 0 ; i < hdev->asic_prop.cb_pool_cb_cnt ; i++) {
 429                 cb = hl_cb_alloc(hdev, hdev->asic_prop.cb_pool_cb_size,
 430                                 HL_KERNEL_ASID_ID);
 431                 if (cb) {
 432                         cb->is_pool = true;
 433                         list_add(&cb->pool_list, &hdev->cb_pool);
 434                 } else {
 435                         hl_cb_pool_fini(hdev);
 436                         return -ENOMEM;
 437                 }
 438         }
 439
 440         return 0;
 441 }
 442
 443 int hl_cb_pool_fini(struct hl_device *hdev)
 444 {
 445         struct hl_cb *cb, *tmp;
 446
 447         list_for_each_entry_safe(cb, tmp, &hdev->cb_pool, pool_list) {
 448                 list_del(&cb->pool_list);
 449                 cb_fini(hdev, cb);
 450         }
 451
 452         return 0;
 453 }