drivers/vfio/virqfd.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * VFIO generic eventfd code for IRQFD support.
   4  * Derived from drivers/vfio/pci/vfio_pci_intrs.c
   5  *
   6  * Copyright (C) 2012 Red Hat, Inc.  All rights reserved.
   7  *     Author: Alex Williamson <alex.williamson@redhat.com>
   8  */
   9
  10 #include <linux/vfio.h>
  11 #include <linux/eventfd.h>
  12 #include <linux/file.h>
  13 #include <linux/module.h>
  14 #include <linux/slab.h>
  15
  16 #define DRIVER_VERSION  "0.1"
  17 #define DRIVER_AUTHOR   "Alex Williamson <alex.williamson@redhat.com>"
  18 #define DRIVER_DESC     "IRQFD support for VFIO bus drivers"
  19
  20 static struct workqueue_struct *vfio_irqfd_cleanup_wq;
  21 static DEFINE_SPINLOCK(virqfd_lock);
  22
  23 static int __init vfio_virqfd_init(void)
  24 {
  25         vfio_irqfd_cleanup_wq =
  26                 create_singlethread_workqueue("vfio-irqfd-cleanup");
  27         if (!vfio_irqfd_cleanup_wq)
  28                 return -ENOMEM;
  29
  30         return 0;
  31 }
  32
  33 static void __exit vfio_virqfd_exit(void)
  34 {
  35         destroy_workqueue(vfio_irqfd_cleanup_wq);
  36 }
  37
  38 static void virqfd_deactivate(struct virqfd *virqfd)
  39 {
  40         queue_work(vfio_irqfd_cleanup_wq, &virqfd->shutdown);
  41 }
  42
  43 static int virqfd_wakeup(wait_queue_entry_t *wait, unsigned mode, int sync, void *key)
  44 {
  45         struct virqfd *virqfd = container_of(wait, struct virqfd, wait);
  46         __poll_t flags = key_to_poll(key);
  47
  48         if (flags & EPOLLIN) {
  49                 u64 cnt;
  50                 eventfd_ctx_do_read(virqfd->eventfd, &cnt);
  51
  52                 /* An event has been signaled, call function */
  53                 if ((!virqfd->handler ||
  54                      virqfd->handler(virqfd->opaque, virqfd->data)) &&
  55                     virqfd->thread)
  56                         schedule_work(&virqfd->inject);
  57         }
  58
  59         if (flags & EPOLLHUP) {
  60                 unsigned long flags;
  61                 spin_lock_irqsave(&virqfd_lock, flags);
  62
  63                 /*
  64                  * The eventfd is closing, if the virqfd has not yet been
  65                  * queued for release, as determined by testing whether the
  66                  * virqfd pointer to it is still valid, queue it now.  As
  67                  * with kvm irqfds, we know we won't race against the virqfd
  68                  * going away because we hold the lock to get here.
  69                  */
  70                 if (*(virqfd->pvirqfd) == virqfd) {
  71                         *(virqfd->pvirqfd) = NULL;
  72                         virqfd_deactivate(virqfd);
  73                 }
  74
  75                 spin_unlock_irqrestore(&virqfd_lock, flags);
  76         }
  77
  78         return 0;
  79 }
  80
  81 static void virqfd_ptable_queue_proc(struct file *file,
  82                                      wait_queue_head_t *wqh, poll_table *pt)
  83 {
  84         struct virqfd *virqfd = container_of(pt, struct virqfd, pt);
  85         add_wait_queue(wqh, &virqfd->wait);
  86 }
  87
  88 static void virqfd_shutdown(struct work_struct *work)
  89 {
  90         struct virqfd *virqfd = container_of(work, struct virqfd, shutdown);
  91         u64 cnt;
  92
  93         eventfd_ctx_remove_wait_queue(virqfd->eventfd, &virqfd->wait, &cnt);
  94         flush_work(&virqfd->inject);
  95         eventfd_ctx_put(virqfd->eventfd);
  96
  97         kfree(virqfd);
  98 }
  99
 100 static void virqfd_inject(struct work_struct *work)
 101 {
 102         struct virqfd *virqfd = container_of(work, struct virqfd, inject);
 103         if (virqfd->thread)
 104                 virqfd->thread(virqfd->opaque, virqfd->data);
 105 }
 106
 107 int vfio_virqfd_enable(void *opaque,
 108                        int (*handler)(void *, void *),
 109                        void (*thread)(void *, void *),
 110                        void *data, struct virqfd **pvirqfd, int fd)
 111 {
 112         struct fd irqfd;
 113         struct eventfd_ctx *ctx;
 114         struct virqfd *virqfd;
 115         int ret = 0;
 116         __poll_t events;
 117
 118         virqfd = kzalloc(sizeof(*virqfd), GFP_KERNEL);
 119         if (!virqfd)
 120                 return -ENOMEM;
 121
 122         virqfd->pvirqfd = pvirqfd;
 123         virqfd->opaque = opaque;
 124         virqfd->handler = handler;
 125         virqfd->thread = thread;
 126         virqfd->data = data;
 127
 128         INIT_WORK(&virqfd->shutdown, virqfd_shutdown);
 129         INIT_WORK(&virqfd->inject, virqfd_inject);
 130
 131         irqfd = fdget(fd);
 132         if (!irqfd.file) {
 133                 ret = -EBADF;
 134                 goto err_fd;
 135         }
 136
 137         ctx = eventfd_ctx_fileget(irqfd.file);
 138         if (IS_ERR(ctx)) {
 139                 ret = PTR_ERR(ctx);
 140                 goto err_ctx;
 141         }
 142
 143         virqfd->eventfd = ctx;
 144
 145         /*
 146          * virqfds can be released by closing the eventfd or directly
 147          * through ioctl.  These are both done through a workqueue, so
 148          * we update the pointer to the virqfd under lock to avoid
 149          * pushing multiple jobs to release the same virqfd.
 150          */
 151         spin_lock_irq(&virqfd_lock);
 152
 153         if (*pvirqfd) {
 154                 spin_unlock_irq(&virqfd_lock);
 155                 ret = -EBUSY;
 156                 goto err_busy;
 157         }
 158         *pvirqfd = virqfd;
 159
 160         spin_unlock_irq(&virqfd_lock);
 161
 162         /*
 163          * Install our own custom wake-up handling so we are notified via
 164          * a callback whenever someone signals the underlying eventfd.
 165          */
 166         init_waitqueue_func_entry(&virqfd->wait, virqfd_wakeup);
 167         init_poll_funcptr(&virqfd->pt, virqfd_ptable_queue_proc);
 168
 169         events = vfs_poll(irqfd.file, &virqfd->pt);
 170
 171         /*
 172          * Check if there was an event already pending on the eventfd
 173          * before we registered and trigger it as if we didn't miss it.
 174          */
 175         if (events & EPOLLIN) {
 176                 if ((!handler || handler(opaque, data)) && thread)
 177                         schedule_work(&virqfd->inject);
 178         }
 179
 180         /*
 181          * Do not drop the file until the irqfd is fully initialized,
 182          * otherwise we might race against the EPOLLHUP.
 183          */
 184         fdput(irqfd);
 185
 186         return 0;
 187 err_busy:
 188         eventfd_ctx_put(ctx);
 189 err_ctx:
 190         fdput(irqfd);
 191 err_fd:
 192         kfree(virqfd);
 193
 194         return ret;
 195 }
 196 EXPORT_SYMBOL_GPL(vfio_virqfd_enable);
 197
 198 void vfio_virqfd_disable(struct virqfd **pvirqfd)
 199 {
 200         unsigned long flags;
 201
 202         spin_lock_irqsave(&virqfd_lock, flags);
 203
 204         if (*pvirqfd) {
 205                 virqfd_deactivate(*pvirqfd);
 206                 *pvirqfd = NULL;
 207         }
 208
 209         spin_unlock_irqrestore(&virqfd_lock, flags);
 210
 211         /*
 212          * Block until we know all outstanding shutdown jobs have completed.
 213          * Even if we don't queue the job, flush the wq to be sure it's
 214          * been released.
 215          */
 216         flush_workqueue(vfio_irqfd_cleanup_wq);
 217 }
 218 EXPORT_SYMBOL_GPL(vfio_virqfd_disable);
 219
 220 module_init(vfio_virqfd_init);
 221 module_exit(vfio_virqfd_exit);
 222
 223 MODULE_VERSION(DRIVER_VERSION);
 224 MODULE_LICENSE("GPL v2");
 225 MODULE_AUTHOR(DRIVER_AUTHOR);
 226 MODULE_DESCRIPTION(DRIVER_DESC);