drivers/dma/idxd/cdev.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
   3 #include <linux/init.h>
   4 #include <linux/kernel.h>
   5 #include <linux/module.h>
   6 #include <linux/pci.h>
   7 #include <linux/device.h>
   8 #include <linux/sched/task.h>
   9 #include <linux/intel-svm.h>
  10 #include <linux/io-64-nonatomic-lo-hi.h>
  11 #include <linux/cdev.h>
  12 #include <linux/fs.h>
  13 #include <linux/poll.h>
  14 #include <linux/iommu.h>
  15 #include <uapi/linux/idxd.h>
  16 #include "registers.h"
  17 #include "idxd.h"
  18
  19 struct idxd_cdev_context {
  20         const char *name;
  21         dev_t devt;
  22         struct ida minor_ida;
  23 };
  24
  25 /*
  26  * ictx is an array based off of accelerator types. enum idxd_type
  27  * is used as index
  28  */
  29 static struct idxd_cdev_context ictx[IDXD_TYPE_MAX] = {
  30         { .name = "dsa" },
  31         { .name = "iax" }
  32 };
  33
  34 struct idxd_user_context {
  35         struct idxd_wq *wq;
  36         struct task_struct *task;
  37         unsigned int pasid;
  38         unsigned int flags;
  39         struct iommu_sva *sva;
  40 };
  41
  42 enum idxd_cdev_cleanup {
  43         CDEV_NORMAL = 0,
  44         CDEV_FAILED,
  45 };
  46
  47 static void idxd_cdev_dev_release(struct device *dev)
  48 {
  49         dev_dbg(dev, "releasing cdev device\n");
  50         kfree(dev);
  51 }
  52
  53 static struct device_type idxd_cdev_device_type = {
  54         .name = "idxd_cdev",
  55         .release = idxd_cdev_dev_release,
  56 };
  57
  58 static inline struct idxd_cdev *inode_idxd_cdev(struct inode *inode)
  59 {
  60         struct cdev *cdev = inode->i_cdev;
  61
  62         return container_of(cdev, struct idxd_cdev, cdev);
  63 }
  64
  65 static inline struct idxd_wq *idxd_cdev_wq(struct idxd_cdev *idxd_cdev)
  66 {
  67         return container_of(idxd_cdev, struct idxd_wq, idxd_cdev);
  68 }
  69
  70 static inline struct idxd_wq *inode_wq(struct inode *inode)
  71 {
  72         return idxd_cdev_wq(inode_idxd_cdev(inode));
  73 }
  74
  75 static int idxd_cdev_open(struct inode *inode, struct file *filp)
  76 {
  77         struct idxd_user_context *ctx;
  78         struct idxd_device *idxd;
  79         struct idxd_wq *wq;
  80         struct device *dev;
  81         int rc = 0;
  82         struct iommu_sva *sva;
  83         unsigned int pasid;
  84
  85         wq = inode_wq(inode);
  86         idxd = wq->idxd;
  87         dev = &idxd->pdev->dev;
  88
  89         dev_dbg(dev, "%s called: %d\n", __func__, idxd_wq_refcount(wq));
  90
  91         ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
  92         if (!ctx)
  93                 return -ENOMEM;
  94
  95         mutex_lock(&wq->wq_lock);
  96
  97         if (idxd_wq_refcount(wq) > 0 && wq_dedicated(wq)) {
  98                 rc = -EBUSY;
  99                 goto failed;
 100         }
 101
 102         ctx->wq = wq;
 103         filp->private_data = ctx;
 104
 105         if (device_pasid_enabled(idxd)) {
 106                 sva = iommu_sva_bind_device(dev, current->mm, NULL);
 107                 if (IS_ERR(sva)) {
 108                         rc = PTR_ERR(sva);
 109                         dev_err(dev, "pasid allocation failed: %d\n", rc);
 110                         goto failed;
 111                 }
 112
 113                 pasid = iommu_sva_get_pasid(sva);
 114                 if (pasid == IOMMU_PASID_INVALID) {
 115                         iommu_sva_unbind_device(sva);
 116                         goto failed;
 117                 }
 118
 119                 ctx->sva = sva;
 120                 ctx->pasid = pasid;
 121
 122                 if (wq_dedicated(wq)) {
 123                         rc = idxd_wq_set_pasid(wq, pasid);
 124                         if (rc < 0) {
 125                                 iommu_sva_unbind_device(sva);
 126                                 dev_err(dev, "wq set pasid failed: %d\n", rc);
 127                                 goto failed;
 128                         }
 129                 }
 130         }
 131
 132         idxd_wq_get(wq);
 133         mutex_unlock(&wq->wq_lock);
 134         return 0;
 135
 136  failed:
 137         mutex_unlock(&wq->wq_lock);
 138         kfree(ctx);
 139         return rc;
 140 }
 141
 142 static int idxd_cdev_release(struct inode *node, struct file *filep)
 143 {
 144         struct idxd_user_context *ctx = filep->private_data;
 145         struct idxd_wq *wq = ctx->wq;
 146         struct idxd_device *idxd = wq->idxd;
 147         struct device *dev = &idxd->pdev->dev;
 148         int rc;
 149
 150         dev_dbg(dev, "%s called\n", __func__);
 151         filep->private_data = NULL;
 152
 153         /* Wait for in-flight operations to complete. */
 154         if (wq_shared(wq)) {
 155                 idxd_device_drain_pasid(idxd, ctx->pasid);
 156         } else {
 157                 if (device_pasid_enabled(idxd)) {
 158                         /* The wq disable in the disable pasid function will drain the wq */
 159                         rc = idxd_wq_disable_pasid(wq);
 160                         if (rc < 0)
 161                                 dev_err(dev, "wq disable pasid failed.\n");
 162                 } else {
 163                         idxd_wq_drain(wq);
 164                 }
 165         }
 166
 167         if (ctx->sva)
 168                 iommu_sva_unbind_device(ctx->sva);
 169         kfree(ctx);
 170         mutex_lock(&wq->wq_lock);
 171         idxd_wq_put(wq);
 172         mutex_unlock(&wq->wq_lock);
 173         return 0;
 174 }
 175
 176 static int check_vma(struct idxd_wq *wq, struct vm_area_struct *vma,
 177                      const char *func)
 178 {
 179         struct device *dev = &wq->idxd->pdev->dev;
 180
 181         if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) {
 182                 dev_info_ratelimited(dev,
 183                                      "%s: %s: mapping too large: %lu\n",
 184                                      current->comm, func,
 185                                      vma->vm_end - vma->vm_start);
 186                 return -EINVAL;
 187         }
 188
 189         return 0;
 190 }
 191
 192 static int idxd_cdev_mmap(struct file *filp, struct vm_area_struct *vma)
 193 {
 194         struct idxd_user_context *ctx = filp->private_data;
 195         struct idxd_wq *wq = ctx->wq;
 196         struct idxd_device *idxd = wq->idxd;
 197         struct pci_dev *pdev = idxd->pdev;
 198         phys_addr_t base = pci_resource_start(pdev, IDXD_WQ_BAR);
 199         unsigned long pfn;
 200         int rc;
 201
 202         dev_dbg(&pdev->dev, "%s called\n", __func__);
 203         rc = check_vma(wq, vma, __func__);
 204         if (rc < 0)
 205                 return rc;
 206
 207         vma->vm_flags |= VM_DONTCOPY;
 208         pfn = (base + idxd_get_wq_portal_full_offset(wq->id,
 209                                 IDXD_PORTAL_LIMITED)) >> PAGE_SHIFT;
 210         vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 211         vma->vm_private_data = ctx;
 212
 213         return io_remap_pfn_range(vma, vma->vm_start, pfn, PAGE_SIZE,
 214                         vma->vm_page_prot);
 215 }
 216
 217 static __poll_t idxd_cdev_poll(struct file *filp,
 218                                struct poll_table_struct *wait)
 219 {
 220         struct idxd_user_context *ctx = filp->private_data;
 221         struct idxd_wq *wq = ctx->wq;
 222         struct idxd_device *idxd = wq->idxd;
 223         struct idxd_cdev *idxd_cdev = &wq->idxd_cdev;
 224         unsigned long flags;
 225         __poll_t out = 0;
 226
 227         poll_wait(filp, &idxd_cdev->err_queue, wait);
 228         spin_lock_irqsave(&idxd->dev_lock, flags);
 229         if (idxd->sw_err.valid)
 230                 out = EPOLLIN | EPOLLRDNORM;
 231         spin_unlock_irqrestore(&idxd->dev_lock, flags);
 232
 233         return out;
 234 }
 235
 236 static const struct file_operations idxd_cdev_fops = {
 237         .owner = THIS_MODULE,
 238         .open = idxd_cdev_open,
 239         .release = idxd_cdev_release,
 240         .mmap = idxd_cdev_mmap,
 241         .poll = idxd_cdev_poll,
 242 };
 243
 244 int idxd_cdev_get_major(struct idxd_device *idxd)
 245 {
 246         return MAJOR(ictx[idxd->type].devt);
 247 }
 248
 249 static int idxd_wq_cdev_dev_setup(struct idxd_wq *wq)
 250 {
 251         struct idxd_device *idxd = wq->idxd;
 252         struct idxd_cdev *idxd_cdev = &wq->idxd_cdev;
 253         struct idxd_cdev_context *cdev_ctx;
 254         struct device *dev;
 255         int minor, rc;
 256
 257         idxd_cdev->dev = kzalloc(sizeof(*idxd_cdev->dev), GFP_KERNEL);
 258         if (!idxd_cdev->dev)
 259                 return -ENOMEM;
 260
 261         dev = idxd_cdev->dev;
 262         dev->parent = &idxd->pdev->dev;
 263         dev_set_name(dev, "%s/wq%u.%u", idxd_get_dev_name(idxd),
 264                      idxd->id, wq->id);
 265         dev->bus = idxd_get_bus_type(idxd);
 266
 267         cdev_ctx = &ictx[wq->idxd->type];
 268         minor = ida_simple_get(&cdev_ctx->minor_ida, 0, MINORMASK, GFP_KERNEL);
 269         if (minor < 0) {
 270                 rc = minor;
 271                 kfree(dev);
 272                 goto ida_err;
 273         }
 274
 275         dev->devt = MKDEV(MAJOR(cdev_ctx->devt), minor);
 276         dev->type = &idxd_cdev_device_type;
 277         rc = device_register(dev);
 278         if (rc < 0) {
 279                 dev_err(&idxd->pdev->dev, "device register failed\n");
 280                 goto dev_reg_err;
 281         }
 282         idxd_cdev->minor = minor;
 283
 284         return 0;
 285
 286  dev_reg_err:
 287         ida_simple_remove(&cdev_ctx->minor_ida, MINOR(dev->devt));
 288         put_device(dev);
 289  ida_err:
 290         idxd_cdev->dev = NULL;
 291         return rc;
 292 }
 293
 294 static void idxd_wq_cdev_cleanup(struct idxd_wq *wq,
 295                                  enum idxd_cdev_cleanup cdev_state)
 296 {
 297         struct idxd_cdev *idxd_cdev = &wq->idxd_cdev;
 298         struct idxd_cdev_context *cdev_ctx;
 299
 300         cdev_ctx = &ictx[wq->idxd->type];
 301         if (cdev_state == CDEV_NORMAL)
 302                 cdev_del(&idxd_cdev->cdev);
 303         device_unregister(idxd_cdev->dev);
 304         /*
 305          * The device_type->release() will be called on the device and free
 306          * the allocated struct device. We can just forget it.
 307          */
 308         ida_simple_remove(&cdev_ctx->minor_ida, idxd_cdev->minor);
 309         idxd_cdev->dev = NULL;
 310         idxd_cdev->minor = -1;
 311 }
 312
 313 int idxd_wq_add_cdev(struct idxd_wq *wq)
 314 {
 315         struct idxd_cdev *idxd_cdev = &wq->idxd_cdev;
 316         struct cdev *cdev = &idxd_cdev->cdev;
 317         struct device *dev;
 318         int rc;
 319
 320         rc = idxd_wq_cdev_dev_setup(wq);
 321         if (rc < 0)
 322                 return rc;
 323
 324         dev = idxd_cdev->dev;
 325         cdev_init(cdev, &idxd_cdev_fops);
 326         cdev_set_parent(cdev, &dev->kobj);
 327         rc = cdev_add(cdev, dev->devt, 1);
 328         if (rc) {
 329                 dev_dbg(&wq->idxd->pdev->dev, "cdev_add failed: %d\n", rc);
 330                 idxd_wq_cdev_cleanup(wq, CDEV_FAILED);
 331                 return rc;
 332         }
 333
 334         init_waitqueue_head(&idxd_cdev->err_queue);
 335         return 0;
 336 }
 337
 338 void idxd_wq_del_cdev(struct idxd_wq *wq)
 339 {
 340         idxd_wq_cdev_cleanup(wq, CDEV_NORMAL);
 341 }
 342
 343 int idxd_cdev_register(void)
 344 {
 345         int rc, i;
 346
 347         for (i = 0; i < IDXD_TYPE_MAX; i++) {
 348                 ida_init(&ictx[i].minor_ida);
 349                 rc = alloc_chrdev_region(&ictx[i].devt, 0, MINORMASK,
 350                                          ictx[i].name);
 351                 if (rc)
 352                         return rc;
 353         }
 354
 355         return 0;
 356 }
 357
 358 void idxd_cdev_remove(void)
 359 {
 360         int i;
 361
 362         for (i = 0; i < IDXD_TYPE_MAX; i++) {
 363                 unregister_chrdev_region(ictx[i].devt, MINORMASK);
 364                 ida_destroy(&ictx[i].minor_ida);
 365         }
 366 }