drivers/vfio/virqfd.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * VFIO generic eventfd code for IRQFD support.
   4  * Derived from drivers/vfio/pci/vfio_pci_intrs.c
   5  *
   6  * Copyright (C) 2012 Red Hat, Inc.  All rights reserved.
   7  *     Author: Alex Williamson <alex.williamson@redhat.com>
   8  */
   9
  10 #include <linux/vfio.h>
  11 #include <linux/eventfd.h>
  12 #include <linux/file.h>
  13 #include <linux/module.h>
  14 #include <linux/slab.h>
  15 #include "vfio.h"
  16
  17 static struct workqueue_struct *vfio_irqfd_cleanup_wq;
  18 static DEFINE_SPINLOCK(virqfd_lock);
  19
  20 int __init vfio_virqfd_init(void)
  21 {
  22         vfio_irqfd_cleanup_wq =
  23                 create_singlethread_workqueue("vfio-irqfd-cleanup");
  24         if (!vfio_irqfd_cleanup_wq)
  25                 return -ENOMEM;
  26
  27         return 0;
  28 }
  29
  30 void vfio_virqfd_exit(void)
  31 {
  32         destroy_workqueue(vfio_irqfd_cleanup_wq);
  33 }
  34
  35 static void virqfd_deactivate(struct virqfd *virqfd)
  36 {
  37         queue_work(vfio_irqfd_cleanup_wq, &virqfd->shutdown);
  38 }
  39
  40 static int virqfd_wakeup(wait_queue_entry_t *wait, unsigned mode, int sync, void *key)
  41 {
  42         struct virqfd *virqfd = container_of(wait, struct virqfd, wait);
  43         __poll_t flags = key_to_poll(key);
  44
  45         if (flags & EPOLLIN) {
  46                 u64 cnt;
  47                 eventfd_ctx_do_read(virqfd->eventfd, &cnt);
  48
  49                 /* An event has been signaled, call function */
  50                 if ((!virqfd->handler ||
  51                      virqfd->handler(virqfd->opaque, virqfd->data)) &&
  52                     virqfd->thread)
  53                         schedule_work(&virqfd->inject);
  54         }
  55
  56         if (flags & EPOLLHUP) {
  57                 unsigned long flags;
  58                 spin_lock_irqsave(&virqfd_lock, flags);
  59
  60                 /*
  61                  * The eventfd is closing, if the virqfd has not yet been
  62                  * queued for release, as determined by testing whether the
  63                  * virqfd pointer to it is still valid, queue it now.  As
  64                  * with kvm irqfds, we know we won't race against the virqfd
  65                  * going away because we hold the lock to get here.
  66                  */
  67                 if (*(virqfd->pvirqfd) == virqfd) {
  68                         *(virqfd->pvirqfd) = NULL;
  69                         virqfd_deactivate(virqfd);
  70                 }
  71
  72                 spin_unlock_irqrestore(&virqfd_lock, flags);
  73         }
  74
  75         return 0;
  76 }
  77
  78 static void virqfd_ptable_queue_proc(struct file *file,
  79                                      wait_queue_head_t *wqh, poll_table *pt)
  80 {
  81         struct virqfd *virqfd = container_of(pt, struct virqfd, pt);
  82         add_wait_queue(wqh, &virqfd->wait);
  83 }
  84
  85 static void virqfd_shutdown(struct work_struct *work)
  86 {
  87         struct virqfd *virqfd = container_of(work, struct virqfd, shutdown);
  88         u64 cnt;
  89
  90         eventfd_ctx_remove_wait_queue(virqfd->eventfd, &virqfd->wait, &cnt);
  91         flush_work(&virqfd->inject);
  92         eventfd_ctx_put(virqfd->eventfd);
  93
  94         kfree(virqfd);
  95 }
  96
  97 static void virqfd_inject(struct work_struct *work)
  98 {
  99         struct virqfd *virqfd = container_of(work, struct virqfd, inject);
 100         if (virqfd->thread)
 101                 virqfd->thread(virqfd->opaque, virqfd->data);
 102 }
 103
 104 static void virqfd_flush_inject(struct work_struct *work)
 105 {
 106         struct virqfd *virqfd = container_of(work, struct virqfd, flush_inject);
 107
 108         flush_work(&virqfd->inject);
 109 }
 110
 111 int vfio_virqfd_enable(void *opaque,
 112                        int (*handler)(void *, void *),
 113                        void (*thread)(void *, void *),
 114                        void *data, struct virqfd **pvirqfd, int fd)
 115 {
 116         struct eventfd_ctx *ctx;
 117         struct virqfd *virqfd;
 118         int ret = 0;
 119         __poll_t events;
 120
 121         virqfd = kzalloc(sizeof(*virqfd), GFP_KERNEL_ACCOUNT);
 122         if (!virqfd)
 123                 return -ENOMEM;
 124
 125         virqfd->pvirqfd = pvirqfd;
 126         virqfd->opaque = opaque;
 127         virqfd->handler = handler;
 128         virqfd->thread = thread;
 129         virqfd->data = data;
 130
 131         INIT_WORK(&virqfd->shutdown, virqfd_shutdown);
 132         INIT_WORK(&virqfd->inject, virqfd_inject);
 133         INIT_WORK(&virqfd->flush_inject, virqfd_flush_inject);
 134
 135         CLASS(fd, irqfd)(fd);
 136         if (fd_empty(irqfd)) {
 137                 ret = -EBADF;
 138                 goto err_fd;
 139         }
 140
 141         ctx = eventfd_ctx_fileget(fd_file(irqfd));
 142         if (IS_ERR(ctx)) {
 143                 ret = PTR_ERR(ctx);
 144                 goto err_fd;
 145         }
 146
 147         virqfd->eventfd = ctx;
 148
 149         /*
 150          * virqfds can be released by closing the eventfd or directly
 151          * through ioctl.  These are both done through a workqueue, so
 152          * we update the pointer to the virqfd under lock to avoid
 153          * pushing multiple jobs to release the same virqfd.
 154          */
 155         spin_lock_irq(&virqfd_lock);
 156
 157         if (*pvirqfd) {
 158                 spin_unlock_irq(&virqfd_lock);
 159                 ret = -EBUSY;
 160                 goto err_busy;
 161         }
 162         *pvirqfd = virqfd;
 163
 164         spin_unlock_irq(&virqfd_lock);
 165
 166         /*
 167          * Install our own custom wake-up handling so we are notified via
 168          * a callback whenever someone signals the underlying eventfd.
 169          */
 170         init_waitqueue_func_entry(&virqfd->wait, virqfd_wakeup);
 171         init_poll_funcptr(&virqfd->pt, virqfd_ptable_queue_proc);
 172
 173         events = vfs_poll(fd_file(irqfd), &virqfd->pt);
 174
 175         /*
 176          * Check if there was an event already pending on the eventfd
 177          * before we registered and trigger it as if we didn't miss it.
 178          */
 179         if (events & EPOLLIN) {
 180                 if ((!handler || handler(opaque, data)) && thread)
 181                         schedule_work(&virqfd->inject);
 182         }
 183         return 0;
 184 err_busy:
 185         eventfd_ctx_put(ctx);
 186 err_fd:
 187         kfree(virqfd);
 188
 189         return ret;
 190 }
 191 EXPORT_SYMBOL_GPL(vfio_virqfd_enable);
 192
 193 void vfio_virqfd_disable(struct virqfd **pvirqfd)
 194 {
 195         unsigned long flags;
 196
 197         spin_lock_irqsave(&virqfd_lock, flags);
 198
 199         if (*pvirqfd) {
 200                 virqfd_deactivate(*pvirqfd);
 201                 *pvirqfd = NULL;
 202         }
 203
 204         spin_unlock_irqrestore(&virqfd_lock, flags);
 205
 206         /*
 207          * Block until we know all outstanding shutdown jobs have completed.
 208          * Even if we don't queue the job, flush the wq to be sure it's
 209          * been released.
 210          */
 211         flush_workqueue(vfio_irqfd_cleanup_wq);
 212 }
 213 EXPORT_SYMBOL_GPL(vfio_virqfd_disable);
 214
 215 void vfio_virqfd_flush_thread(struct virqfd **pvirqfd)
 216 {
 217         unsigned long flags;
 218
 219         spin_lock_irqsave(&virqfd_lock, flags);
 220         if (*pvirqfd && (*pvirqfd)->thread)
 221                 queue_work(vfio_irqfd_cleanup_wq, &(*pvirqfd)->flush_inject);
 222         spin_unlock_irqrestore(&virqfd_lock, flags);
 223
 224         flush_workqueue(vfio_irqfd_cleanup_wq);
 225 }
 226 EXPORT_SYMBOL_GPL(vfio_virqfd_flush_thread);