Linux 4.19.133

[linux/fpc-iii.git] / drivers / staging / android / vsoc.c

// SPDX-License-Identifier: GPL-2.0
/*
 * drivers/android/staging/vsoc.c
 *
 * Android Virtual System on a Chip (VSoC) driver
 *
 * Copyright (C) 2017 Google, Inc.
 *
 * Author: ghartman@google.com
 *
 * Based on drivers/char/kvm_ivshmem.c - driver for KVM Inter-VM shared memory
 *         Copyright 2009 Cam Macdonell <cam@cs.ualberta.ca>
 *
 * Based on cirrusfb.c and 8139cp.c:
 *   Copyright 1999-2001 Jeff Garzik
 *   Copyright 2001-2004 Jeff Garzik
 */

#include <linux/dma-mapping.h>
#include <linux/freezer.h>
#include <linux/futex.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/pci.h>
#include <linux/proc_fs.h>
#include <linux/sched.h>
#include <linux/syscalls.h>
#include <linux/uaccess.h>
#include <linux/interrupt.h>
#include <linux/mutex.h>
#include <linux/cdev.h>
#include <linux/file.h>
#include "uapi/vsoc_shm.h"

#define VSOC_DEV_NAME "vsoc"

/*
 * Description of the ivshmem-doorbell PCI device used by QEmu. These
 * constants follow docs/specs/ivshmem-spec.txt, which can be found in
 * the QEmu repository. This was last reconciled with the version that
 * came out with 2.8
 */

/*
 * These constants are determined KVM Inter-VM shared memory device
 * register offsets
 */
enum {
        INTR_MASK = 0x00,       /* Interrupt Mask */
        INTR_STATUS = 0x04,     /* Interrupt Status */
        IV_POSITION = 0x08,     /* VM ID */
        DOORBELL = 0x0c,        /* Doorbell */
};

static const int REGISTER_BAR;  /* Equal to 0 */
static const int MAX_REGISTER_BAR_LEN = 0x100;
/*
 * The MSI-x BAR is not used directly.
 *
 * static const int MSI_X_BAR = 1;
 */
static const int SHARED_MEMORY_BAR = 2;

struct vsoc_region_data {
        char name[VSOC_DEVICE_NAME_SZ + 1];
        wait_queue_head_t interrupt_wait_queue;
        /* TODO(b/73664181): Use multiple futex wait queues */
        wait_queue_head_t futex_wait_queue;
        /* Flag indicating that an interrupt has been signalled by the host. */
        atomic_t *incoming_signalled;
        /* Flag indicating the guest has signalled the host. */
        atomic_t *outgoing_signalled;
        bool irq_requested;
        bool device_created;
};

struct vsoc_device {
        /* Kernel virtual address of REGISTER_BAR. */
        void __iomem *regs;
        /* Physical address of SHARED_MEMORY_BAR. */
        phys_addr_t shm_phys_start;
        /* Kernel virtual address of SHARED_MEMORY_BAR. */
        void __iomem *kernel_mapped_shm;
        /* Size of the entire shared memory window in bytes. */
        size_t shm_size;
        /*
         * Pointer to the virtual address of the shared memory layout structure.
         * This is probably identical to kernel_mapped_shm, but saving this
         * here saves a lot of annoying casts.
         */
        struct vsoc_shm_layout_descriptor *layout;
        /*
         * Points to a table of region descriptors in the kernel's virtual
         * address space. Calculated from
         * vsoc_shm_layout_descriptor.vsoc_region_desc_offset
         */
        struct vsoc_device_region *regions;
        /* Head of a list of permissions that have been granted. */
        struct list_head permissions;
        struct pci_dev *dev;
        /* Per-region (and therefore per-interrupt) information. */
        struct vsoc_region_data *regions_data;
        /*
         * Table of msi-x entries. This has to be separated from struct
         * vsoc_region_data because the kernel deals with them as an array.
         */
        struct msix_entry *msix_entries;
        /* Mutex that protectes the permission list */
        struct mutex mtx;
        /* Major number assigned by the kernel */
        int major;
        /* Character device assigned by the kernel */
        struct cdev cdev;
        /* Device class assigned by the kernel */
        struct class *class;
        /*
         * Flags that indicate what we've initialized. These are used to do an
         * orderly cleanup of the device.
         */
        bool enabled_device;
        bool requested_regions;
        bool cdev_added;
        bool class_added;
        bool msix_enabled;
};

static struct vsoc_device vsoc_dev;

/*
 * TODO(ghartman): Add a /sys filesystem entry that summarizes the permissions.
 */

struct fd_scoped_permission_node {
        struct fd_scoped_permission permission;
        struct list_head list;
};

struct vsoc_private_data {
        struct fd_scoped_permission_node *fd_scoped_permission_node;
};

static long vsoc_ioctl(struct file *, unsigned int, unsigned long);
static int vsoc_mmap(struct file *, struct vm_area_struct *);
static int vsoc_open(struct inode *, struct file *);
static int vsoc_release(struct inode *, struct file *);
static ssize_t vsoc_read(struct file *, char __user *, size_t, loff_t *);
static ssize_t vsoc_write(struct file *, const char __user *, size_t, loff_t *);
static loff_t vsoc_lseek(struct file *filp, loff_t offset, int origin);
static int
do_create_fd_scoped_permission(struct vsoc_device_region *region_p,
                               struct fd_scoped_permission_node *np,
                               struct fd_scoped_permission_arg __user *arg);
static void
do_destroy_fd_scoped_permission(struct vsoc_device_region *owner_region_p,
                                struct fd_scoped_permission *perm);
static long do_vsoc_describe_region(struct file *,
                                    struct vsoc_device_region __user *);
static ssize_t vsoc_get_area(struct file *filp, __u32 *perm_off);

/**
 * Validate arguments on entry points to the driver.
 */
inline int vsoc_validate_inode(struct inode *inode)
{
        if (iminor(inode) >= vsoc_dev.layout->region_count) {
                dev_err(&vsoc_dev.dev->dev,
                        "describe_region: invalid region %d\n", iminor(inode));
                return -ENODEV;
        }
        return 0;
}

inline int vsoc_validate_filep(struct file *filp)
{
        int ret = vsoc_validate_inode(file_inode(filp));

        if (ret)
                return ret;
        if (!filp->private_data) {
                dev_err(&vsoc_dev.dev->dev,
                        "No private data on fd, region %d\n",
                        iminor(file_inode(filp)));
                return -EBADFD;
        }
        return 0;
}

/* Converts from shared memory offset to virtual address */
static inline void *shm_off_to_virtual_addr(__u32 offset)
{
        return (void __force *)vsoc_dev.kernel_mapped_shm + offset;
}

/* Converts from shared memory offset to physical address */
static inline phys_addr_t shm_off_to_phys_addr(__u32 offset)
{
        return vsoc_dev.shm_phys_start + offset;
}

/**
 * Convenience functions to obtain the region from the inode or file.
 * Dangerous to call before validating the inode/file.
 */
static
inline struct vsoc_device_region *vsoc_region_from_inode(struct inode *inode)
{
        return &vsoc_dev.regions[iminor(inode)];
}

static
inline struct vsoc_device_region *vsoc_region_from_filep(struct file *inode)
{
        return vsoc_region_from_inode(file_inode(inode));
}

static inline uint32_t vsoc_device_region_size(struct vsoc_device_region *r)
{
        return r->region_end_offset - r->region_begin_offset;
}

static const struct file_operations vsoc_ops = {
        .owner = THIS_MODULE,
        .open = vsoc_open,
        .mmap = vsoc_mmap,
        .read = vsoc_read,
        .unlocked_ioctl = vsoc_ioctl,
        .compat_ioctl = vsoc_ioctl,
        .write = vsoc_write,
        .llseek = vsoc_lseek,
        .release = vsoc_release,
};

static struct pci_device_id vsoc_id_table[] = {
        {0x1af4, 0x1110, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
        {0},
};

MODULE_DEVICE_TABLE(pci, vsoc_id_table);

static void vsoc_remove_device(struct pci_dev *pdev);
static int vsoc_probe_device(struct pci_dev *pdev,
                             const struct pci_device_id *ent);

static struct pci_driver vsoc_pci_driver = {
        .name = "vsoc",
        .id_table = vsoc_id_table,
        .probe = vsoc_probe_device,
        .remove = vsoc_remove_device,
};

static int
do_create_fd_scoped_permission(struct vsoc_device_region *region_p,
                               struct fd_scoped_permission_node *np,
                               struct fd_scoped_permission_arg __user *arg)
{
        struct file *managed_filp;
        s32 managed_fd;
        atomic_t *owner_ptr = NULL;
        struct vsoc_device_region *managed_region_p;

        if (copy_from_user(&np->permission,
                           &arg->perm, sizeof(np->permission)) ||
            copy_from_user(&managed_fd,
                           &arg->managed_region_fd, sizeof(managed_fd))) {
                return -EFAULT;
        }
        managed_filp = fdget(managed_fd).file;
        /* Check that it's a valid fd, */
        if (!managed_filp || vsoc_validate_filep(managed_filp))
                return -EPERM;
        /* EEXIST if the given fd already has a permission. */
        if (((struct vsoc_private_data *)managed_filp->private_data)->
            fd_scoped_permission_node)
                return -EEXIST;
        managed_region_p = vsoc_region_from_filep(managed_filp);
        /* Check that the provided region is managed by this one */
        if (&vsoc_dev.regions[managed_region_p->managed_by] != region_p)
                return -EPERM;
        /* The area must be well formed and have non-zero size */
        if (np->permission.begin_offset >= np->permission.end_offset)
                return -EINVAL;
        /* The area must fit in the memory window */
        if (np->permission.end_offset >
            vsoc_device_region_size(managed_region_p))
                return -ERANGE;
        /* The area must be in the region data section */
        if (np->permission.begin_offset <
            managed_region_p->offset_of_region_data)
                return -ERANGE;
        /* The area must be page aligned */
        if (!PAGE_ALIGNED(np->permission.begin_offset) ||
            !PAGE_ALIGNED(np->permission.end_offset))
                return -EINVAL;
        /* Owner offset must be naturally aligned in the window */
        if (np->permission.owner_offset &
            (sizeof(np->permission.owner_offset) - 1))
                return -EINVAL;
        /* The owner flag must reside in the owner memory */
        if (np->permission.owner_offset + sizeof(np->permission.owner_offset) >
            vsoc_device_region_size(region_p))
                return -ERANGE;
        /* The owner flag must reside in the data section */
        if (np->permission.owner_offset < region_p->offset_of_region_data)
                return -EINVAL;
        /* The owner value must change to claim the memory */
        if (np->permission.owned_value == VSOC_REGION_FREE)
                return -EINVAL;
        owner_ptr =
            (atomic_t *)shm_off_to_virtual_addr(region_p->region_begin_offset +
                                                np->permission.owner_offset);
        /* We've already verified that this is in the shared memory window, so
         * it should be safe to write to this address.
         */
        if (atomic_cmpxchg(owner_ptr,
                           VSOC_REGION_FREE,
                           np->permission.owned_value) != VSOC_REGION_FREE) {
                return -EBUSY;
        }
        ((struct vsoc_private_data *)managed_filp->private_data)->
            fd_scoped_permission_node = np;
        /* The file offset needs to be adjusted if the calling
         * process did any read/write operations on the fd
         * before creating the permission.
         */
        if (managed_filp->f_pos) {
                if (managed_filp->f_pos > np->permission.end_offset) {
                        /* If the offset is beyond the permission end, set it
                         * to the end.
                         */
                        managed_filp->f_pos = np->permission.end_offset;
                } else {
                        /* If the offset is within the permission interval
                         * keep it there otherwise reset it to zero.
                         */
                        if (managed_filp->f_pos < np->permission.begin_offset) {
                                managed_filp->f_pos = 0;
                        } else {
                                managed_filp->f_pos -=
                                    np->permission.begin_offset;
                        }
                }
        }
        return 0;
}

static void
do_destroy_fd_scoped_permission_node(struct vsoc_device_region *owner_region_p,
                                     struct fd_scoped_permission_node *node)
{
        if (node) {
                do_destroy_fd_scoped_permission(owner_region_p,
                                                &node->permission);
                mutex_lock(&vsoc_dev.mtx);
                list_del(&node->list);
                mutex_unlock(&vsoc_dev.mtx);
                kfree(node);
        }
}

static void
do_destroy_fd_scoped_permission(struct vsoc_device_region *owner_region_p,
                                struct fd_scoped_permission *perm)
{
        atomic_t *owner_ptr = NULL;
        int prev = 0;

        if (!perm)
                return;
        owner_ptr = (atomic_t *)shm_off_to_virtual_addr
                (owner_region_p->region_begin_offset + perm->owner_offset);
        prev = atomic_xchg(owner_ptr, VSOC_REGION_FREE);
        if (prev != perm->owned_value)
                dev_err(&vsoc_dev.dev->dev,
                        "%x-%x: owner (%s) %x: expected to be %x was %x",
                        perm->begin_offset, perm->end_offset,
                        owner_region_p->device_name, perm->owner_offset,
                        perm->owned_value, prev);
}

static long do_vsoc_describe_region(struct file *filp,
                                    struct vsoc_device_region __user *dest)
{
        struct vsoc_device_region *region_p;
        int retval = vsoc_validate_filep(filp);

        if (retval)
                return retval;
        region_p = vsoc_region_from_filep(filp);
        if (copy_to_user(dest, region_p, sizeof(*region_p)))
                return -EFAULT;
        return 0;
}

/**
 * Implements the inner logic of cond_wait. Copies to and from userspace are
 * done in the helper function below.
 */
static int handle_vsoc_cond_wait(struct file *filp, struct vsoc_cond_wait *arg)
{
        DEFINE_WAIT(wait);
        u32 region_number = iminor(file_inode(filp));
        struct vsoc_region_data *data = vsoc_dev.regions_data + region_number;
        struct hrtimer_sleeper timeout, *to = NULL;
        int ret = 0;
        struct vsoc_device_region *region_p = vsoc_region_from_filep(filp);
        atomic_t *address = NULL;
        ktime_t wake_time;

        /* Ensure that the offset is aligned */
        if (arg->offset & (sizeof(uint32_t) - 1))
                return -EADDRNOTAVAIL;
        /* Ensure that the offset is within shared memory */
        if (((uint64_t)arg->offset) + region_p->region_begin_offset +
            sizeof(uint32_t) > region_p->region_end_offset)
                return -E2BIG;
        address = shm_off_to_virtual_addr(region_p->region_begin_offset +
                                          arg->offset);

        /* Ensure that the type of wait is valid */
        switch (arg->wait_type) {
        case VSOC_WAIT_IF_EQUAL:
                break;
        case VSOC_WAIT_IF_EQUAL_TIMEOUT:
                to = &timeout;
                break;
        default:
                return -EINVAL;
        }

        if (to) {
                /* Copy the user-supplied timesec into the kernel structure.
                 * We do things this way to flatten differences between 32 bit
                 * and 64 bit timespecs.
                 */
                if (arg->wake_time_nsec >= NSEC_PER_SEC)
                        return -EINVAL;
                wake_time = ktime_set(arg->wake_time_sec, arg->wake_time_nsec);

                hrtimer_init_on_stack(&to->timer, CLOCK_MONOTONIC,
                                      HRTIMER_MODE_ABS);
                hrtimer_set_expires_range_ns(&to->timer, wake_time,
                                             current->timer_slack_ns);

                hrtimer_init_sleeper(to, current);
        }

        while (1) {
                prepare_to_wait(&data->futex_wait_queue, &wait,
                                TASK_INTERRUPTIBLE);
                /*
                 * Check the sentinel value after prepare_to_wait. If the value
                 * changes after this check the writer will call signal,
                 * changing the task state from INTERRUPTIBLE to RUNNING. That
                 * will ensure that schedule() will eventually schedule this
                 * task.
                 */
                if (atomic_read(address) != arg->value) {
                        ret = 0;
                        break;
                }
                if (to) {
                        hrtimer_start_expires(&to->timer, HRTIMER_MODE_ABS);
                        if (likely(to->task))
                                freezable_schedule();
                        hrtimer_cancel(&to->timer);
                        if (!to->task) {
                                ret = -ETIMEDOUT;
                                break;
                        }
                } else {
                        freezable_schedule();
                }
                /* Count the number of times that we woke up. This is useful
                 * for unit testing.
                 */
                ++arg->wakes;
                if (signal_pending(current)) {
                        ret = -EINTR;
                        break;
                }
        }
        finish_wait(&data->futex_wait_queue, &wait);
        if (to)
                destroy_hrtimer_on_stack(&to->timer);
        return ret;
}

/**
 * Handles the details of copying from/to userspace to ensure that the copies
 * happen on all of the return paths of cond_wait.
 */
static int do_vsoc_cond_wait(struct file *filp,
                             struct vsoc_cond_wait __user *untrusted_in)
{
        struct vsoc_cond_wait arg;
        int rval = 0;

        if (copy_from_user(&arg, untrusted_in, sizeof(arg)))
                return -EFAULT;
        /* wakes is an out parameter. Initialize it to something sensible. */
        arg.wakes = 0;
        rval = handle_vsoc_cond_wait(filp, &arg);
        if (copy_to_user(untrusted_in, &arg, sizeof(arg)))
                return -EFAULT;
        return rval;
}

static int do_vsoc_cond_wake(struct file *filp, uint32_t offset)
{
        struct vsoc_device_region *region_p = vsoc_region_from_filep(filp);
        u32 region_number = iminor(file_inode(filp));
        struct vsoc_region_data *data = vsoc_dev.regions_data + region_number;
        /* Ensure that the offset is aligned */
        if (offset & (sizeof(uint32_t) - 1))
                return -EADDRNOTAVAIL;
        /* Ensure that the offset is within shared memory */
        if (((uint64_t)offset) + region_p->region_begin_offset +
            sizeof(uint32_t) > region_p->region_end_offset)
                return -E2BIG;
        /*
         * TODO(b/73664181): Use multiple futex wait queues.
         * We need to wake every sleeper when the condition changes. Typically
         * only a single thread will be waiting on the condition, but there
         * are exceptions. The worst case is about 10 threads.
         */
        wake_up_interruptible_all(&data->futex_wait_queue);
        return 0;
}

static long vsoc_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
{
        int rv = 0;
        struct vsoc_device_region *region_p;
        u32 reg_num;
        struct vsoc_region_data *reg_data;
        int retval = vsoc_validate_filep(filp);

        if (retval)
                return retval;
        region_p = vsoc_region_from_filep(filp);
        reg_num = iminor(file_inode(filp));
        reg_data = vsoc_dev.regions_data + reg_num;
        switch (cmd) {
        case VSOC_CREATE_FD_SCOPED_PERMISSION:
                {
                        struct fd_scoped_permission_node *node = NULL;

                        node = kzalloc(sizeof(*node), GFP_KERNEL);
                        /* We can't allocate memory for the permission */
                        if (!node)
                                return -ENOMEM;
                        INIT_LIST_HEAD(&node->list);
                        rv = do_create_fd_scoped_permission
                                (region_p,
                                 node,
                                 (struct fd_scoped_permission_arg __user *)arg);
                        if (!rv) {
                                mutex_lock(&vsoc_dev.mtx);
                                list_add(&node->list, &vsoc_dev.permissions);
                                mutex_unlock(&vsoc_dev.mtx);
                        } else {
                                kfree(node);
                                return rv;
                        }
                }
                break;

        case VSOC_GET_FD_SCOPED_PERMISSION:
                {
                        struct fd_scoped_permission_node *node =
                            ((struct vsoc_private_data *)filp->private_data)->
                            fd_scoped_permission_node;
                        if (!node)
                                return -ENOENT;
                        if (copy_to_user
                            ((struct fd_scoped_permission __user *)arg,
                             &node->permission, sizeof(node->permission)))
                                return -EFAULT;
                }
                break;

        case VSOC_MAYBE_SEND_INTERRUPT_TO_HOST:
                if (!atomic_xchg(reg_data->outgoing_signalled, 1)) {
                        writel(reg_num, vsoc_dev.regs + DOORBELL);
                        return 0;
                } else {
                        return -EBUSY;
                }
                break;

        case VSOC_SEND_INTERRUPT_TO_HOST:
                writel(reg_num, vsoc_dev.regs + DOORBELL);
                return 0;
        case VSOC_WAIT_FOR_INCOMING_INTERRUPT:
                wait_event_interruptible
                        (reg_data->interrupt_wait_queue,
                         (atomic_read(reg_data->incoming_signalled) != 0));
                break;

        case VSOC_DESCRIBE_REGION:
                return do_vsoc_describe_region
                        (filp,
                         (struct vsoc_device_region __user *)arg);

        case VSOC_SELF_INTERRUPT:
                atomic_set(reg_data->incoming_signalled, 1);
                wake_up_interruptible(&reg_data->interrupt_wait_queue);
                break;

        case VSOC_COND_WAIT:
                return do_vsoc_cond_wait(filp,
                                         (struct vsoc_cond_wait __user *)arg);
        case VSOC_COND_WAKE:
                return do_vsoc_cond_wake(filp, arg);

        default:
                return -EINVAL;
        }
        return 0;
}

static ssize_t vsoc_read(struct file *filp, char __user *buffer, size_t len,
                         loff_t *poffset)
{
        __u32 area_off;
        const void *area_p;
        ssize_t area_len;
        int retval = vsoc_validate_filep(filp);

        if (retval)
                return retval;
        area_len = vsoc_get_area(filp, &area_off);
        area_p = shm_off_to_virtual_addr(area_off);
        area_p += *poffset;
        area_len -= *poffset;
        if (area_len <= 0)
                return 0;
        if (area_len < len)
                len = area_len;
        if (copy_to_user(buffer, area_p, len))
                return -EFAULT;
        *poffset += len;
        return len;
}

static loff_t vsoc_lseek(struct file *filp, loff_t offset, int origin)
{
        ssize_t area_len = 0;
        int retval = vsoc_validate_filep(filp);

        if (retval)
                return retval;
        area_len = vsoc_get_area(filp, NULL);
        switch (origin) {
        case SEEK_SET:
                break;

        case SEEK_CUR:
                if (offset > 0 && offset + filp->f_pos < 0)
                        return -EOVERFLOW;
                offset += filp->f_pos;
                break;

        case SEEK_END:
                if (offset > 0 && offset + area_len < 0)
                        return -EOVERFLOW;
                offset += area_len;
                break;

        case SEEK_DATA:
                if (offset >= area_len)
                        return -EINVAL;
                if (offset < 0)
                        offset = 0;
                break;

        case SEEK_HOLE:
                /* Next hole is always the end of the region, unless offset is
                 * beyond that
                 */
                if (offset < area_len)
                        offset = area_len;
                break;

        default:
                return -EINVAL;
        }

        if (offset < 0 || offset > area_len)
                return -EINVAL;
        filp->f_pos = offset;

        return offset;
}

static ssize_t vsoc_write(struct file *filp, const char __user *buffer,
                          size_t len, loff_t *poffset)
{
        __u32 area_off;
        void *area_p;
        ssize_t area_len;
        int retval = vsoc_validate_filep(filp);

        if (retval)
                return retval;
        area_len = vsoc_get_area(filp, &area_off);
        area_p = shm_off_to_virtual_addr(area_off);
        area_p += *poffset;
        area_len -= *poffset;
        if (area_len <= 0)
                return 0;
        if (area_len < len)
                len = area_len;
        if (copy_from_user(area_p, buffer, len))
                return -EFAULT;
        *poffset += len;
        return len;
}

static irqreturn_t vsoc_interrupt(int irq, void *region_data_v)
{
        struct vsoc_region_data *region_data =
            (struct vsoc_region_data *)region_data_v;
        int reg_num = region_data - vsoc_dev.regions_data;

        if (unlikely(!region_data))
                return IRQ_NONE;

        if (unlikely(reg_num < 0 ||
                     reg_num >= vsoc_dev.layout->region_count)) {
                dev_err(&vsoc_dev.dev->dev,
                        "invalid irq @%p reg_num=0x%04x\n",
                        region_data, reg_num);
                return IRQ_NONE;
        }
        if (unlikely(vsoc_dev.regions_data + reg_num != region_data)) {
                dev_err(&vsoc_dev.dev->dev,
                        "irq not aligned @%p reg_num=0x%04x\n",
                        region_data, reg_num);
                return IRQ_NONE;
        }
        wake_up_interruptible(&region_data->interrupt_wait_queue);
        return IRQ_HANDLED;
}

static int vsoc_probe_device(struct pci_dev *pdev,
                             const struct pci_device_id *ent)
{
        int result;
        int i;
        resource_size_t reg_size;
        dev_t devt;

        vsoc_dev.dev = pdev;
        result = pci_enable_device(pdev);
        if (result) {
                dev_err(&pdev->dev,
                        "pci_enable_device failed %s: error %d\n",
                        pci_name(pdev), result);
                return result;
        }
        vsoc_dev.enabled_device = true;
        result = pci_request_regions(pdev, "vsoc");
        if (result < 0) {
                dev_err(&pdev->dev, "pci_request_regions failed\n");
                vsoc_remove_device(pdev);
                return -EBUSY;
        }
        vsoc_dev.requested_regions = true;
        /* Set up the control registers in BAR 0 */
        reg_size = pci_resource_len(pdev, REGISTER_BAR);
        if (reg_size > MAX_REGISTER_BAR_LEN)
                vsoc_dev.regs =
                    pci_iomap(pdev, REGISTER_BAR, MAX_REGISTER_BAR_LEN);
        else
                vsoc_dev.regs = pci_iomap(pdev, REGISTER_BAR, reg_size);

        if (!vsoc_dev.regs) {
                dev_err(&pdev->dev,
                        "cannot map registers of size %zu\n",
                       (size_t)reg_size);
                vsoc_remove_device(pdev);
                return -EBUSY;
        }

        /* Map the shared memory in BAR 2 */
        vsoc_dev.shm_phys_start = pci_resource_start(pdev, SHARED_MEMORY_BAR);
        vsoc_dev.shm_size = pci_resource_len(pdev, SHARED_MEMORY_BAR);

        dev_info(&pdev->dev, "shared memory @ DMA %pa size=0x%zx\n",
                 &vsoc_dev.shm_phys_start, vsoc_dev.shm_size);
        vsoc_dev.kernel_mapped_shm = pci_iomap_wc(pdev, SHARED_MEMORY_BAR, 0);
        if (!vsoc_dev.kernel_mapped_shm) {
                dev_err(&vsoc_dev.dev->dev, "cannot iomap region\n");
                vsoc_remove_device(pdev);
                return -EBUSY;
        }

        vsoc_dev.layout = (struct vsoc_shm_layout_descriptor __force *)
                                vsoc_dev.kernel_mapped_shm;
        dev_info(&pdev->dev, "major_version: %d\n",
                 vsoc_dev.layout->major_version);
        dev_info(&pdev->dev, "minor_version: %d\n",
                 vsoc_dev.layout->minor_version);
        dev_info(&pdev->dev, "size: 0x%x\n", vsoc_dev.layout->size);
        dev_info(&pdev->dev, "regions: %d\n", vsoc_dev.layout->region_count);
        if (vsoc_dev.layout->major_version !=
            CURRENT_VSOC_LAYOUT_MAJOR_VERSION) {
                dev_err(&vsoc_dev.dev->dev,
                        "driver supports only major_version %d\n",
                        CURRENT_VSOC_LAYOUT_MAJOR_VERSION);
                vsoc_remove_device(pdev);
                return -EBUSY;
        }
        result = alloc_chrdev_region(&devt, 0, vsoc_dev.layout->region_count,
                                     VSOC_DEV_NAME);
        if (result) {
                dev_err(&vsoc_dev.dev->dev, "alloc_chrdev_region failed\n");
                vsoc_remove_device(pdev);
                return -EBUSY;
        }
        vsoc_dev.major = MAJOR(devt);
        cdev_init(&vsoc_dev.cdev, &vsoc_ops);
        vsoc_dev.cdev.owner = THIS_MODULE;
        result = cdev_add(&vsoc_dev.cdev, devt, vsoc_dev.layout->region_count);
        if (result) {
                dev_err(&vsoc_dev.dev->dev, "cdev_add error\n");
                vsoc_remove_device(pdev);
                return -EBUSY;
        }
        vsoc_dev.cdev_added = true;
        vsoc_dev.class = class_create(THIS_MODULE, VSOC_DEV_NAME);
        if (IS_ERR(vsoc_dev.class)) {
                dev_err(&vsoc_dev.dev->dev, "class_create failed\n");
                vsoc_remove_device(pdev);
                return PTR_ERR(vsoc_dev.class);
        }
        vsoc_dev.class_added = true;
        vsoc_dev.regions = (struct vsoc_device_region __force *)
                ((void *)vsoc_dev.layout +
                 vsoc_dev.layout->vsoc_region_desc_offset);
        vsoc_dev.msix_entries =
                kcalloc(vsoc_dev.layout->region_count,
                        sizeof(vsoc_dev.msix_entries[0]), GFP_KERNEL);
        if (!vsoc_dev.msix_entries) {
                dev_err(&vsoc_dev.dev->dev,
                        "unable to allocate msix_entries\n");
                vsoc_remove_device(pdev);
                return -ENOSPC;
        }
        vsoc_dev.regions_data =
                kcalloc(vsoc_dev.layout->region_count,
                        sizeof(vsoc_dev.regions_data[0]), GFP_KERNEL);
        if (!vsoc_dev.regions_data) {
                dev_err(&vsoc_dev.dev->dev,
                        "unable to allocate regions' data\n");
                vsoc_remove_device(pdev);
                return -ENOSPC;
        }
        for (i = 0; i < vsoc_dev.layout->region_count; ++i)
                vsoc_dev.msix_entries[i].entry = i;

        result = pci_enable_msix_exact(vsoc_dev.dev, vsoc_dev.msix_entries,
                                       vsoc_dev.layout->region_count);
        if (result) {
                dev_info(&pdev->dev, "pci_enable_msix failed: %d\n", result);
                vsoc_remove_device(pdev);
                return -ENOSPC;
        }
        /* Check that all regions are well formed */
        for (i = 0; i < vsoc_dev.layout->region_count; ++i) {
                const struct vsoc_device_region *region = vsoc_dev.regions + i;

                if (!PAGE_ALIGNED(region->region_begin_offset) ||
                    !PAGE_ALIGNED(region->region_end_offset)) {
                        dev_err(&vsoc_dev.dev->dev,
                                "region %d not aligned (%x:%x)", i,
                                region->region_begin_offset,
                                region->region_end_offset);
                        vsoc_remove_device(pdev);
                        return -EFAULT;
                }
                if (region->region_begin_offset >= region->region_end_offset ||
                    region->region_end_offset > vsoc_dev.shm_size) {
                        dev_err(&vsoc_dev.dev->dev,
                                "region %d offsets are wrong: %x %x %zx",
                                i, region->region_begin_offset,
                                region->region_end_offset, vsoc_dev.shm_size);
                        vsoc_remove_device(pdev);
                        return -EFAULT;
                }
                if (region->managed_by >= vsoc_dev.layout->region_count) {
                        dev_err(&vsoc_dev.dev->dev,
                                "region %d has invalid owner: %u",
                                i, region->managed_by);
                        vsoc_remove_device(pdev);
                        return -EFAULT;
                }
        }
        vsoc_dev.msix_enabled = true;
        for (i = 0; i < vsoc_dev.layout->region_count; ++i) {
                const struct vsoc_device_region *region = vsoc_dev.regions + i;
                size_t name_sz = sizeof(vsoc_dev.regions_data[i].name) - 1;
                const struct vsoc_signal_table_layout *h_to_g_signal_table =
                        &region->host_to_guest_signal_table;
                const struct vsoc_signal_table_layout *g_to_h_signal_table =
                        &region->guest_to_host_signal_table;

                vsoc_dev.regions_data[i].name[name_sz] = '\0';
                memcpy(vsoc_dev.regions_data[i].name, region->device_name,
                       name_sz);
                dev_info(&pdev->dev, "region %d name=%s\n",
                         i, vsoc_dev.regions_data[i].name);
                init_waitqueue_head
                        (&vsoc_dev.regions_data[i].interrupt_wait_queue);
                init_waitqueue_head(&vsoc_dev.regions_data[i].futex_wait_queue);
                vsoc_dev.regions_data[i].incoming_signalled =
                        shm_off_to_virtual_addr(region->region_begin_offset) +
                        h_to_g_signal_table->interrupt_signalled_offset;
                vsoc_dev.regions_data[i].outgoing_signalled =
                        shm_off_to_virtual_addr(region->region_begin_offset) +
                        g_to_h_signal_table->interrupt_signalled_offset;
                result = request_irq(vsoc_dev.msix_entries[i].vector,
                                     vsoc_interrupt, 0,
                                     vsoc_dev.regions_data[i].name,
                                     vsoc_dev.regions_data + i);
                if (result) {
                        dev_info(&pdev->dev,
                                 "request_irq failed irq=%d vector=%d\n",
                                i, vsoc_dev.msix_entries[i].vector);
                        vsoc_remove_device(pdev);
                        return -ENOSPC;
                }
                vsoc_dev.regions_data[i].irq_requested = true;
                if (!device_create(vsoc_dev.class, NULL,
                                   MKDEV(vsoc_dev.major, i),
                                   NULL, vsoc_dev.regions_data[i].name)) {
                        dev_err(&vsoc_dev.dev->dev, "device_create failed\n");
                        vsoc_remove_device(pdev);
                        return -EBUSY;
                }
                vsoc_dev.regions_data[i].device_created = true;
        }
        return 0;
}

/*
 * This should undo all of the allocations in the probe function in reverse
 * order.
 *
 * Notes:
 *
 *   The device may have been partially initialized, so double check
 *   that the allocations happened.
 *
 *   This function may be called multiple times, so mark resources as freed
 *   as they are deallocated.
 */
static void vsoc_remove_device(struct pci_dev *pdev)
{
        int i;
        /*
         * pdev is the first thing to be set on probe and the last thing
         * to be cleared here. If it's NULL then there is no cleanup.
         */
        if (!pdev || !vsoc_dev.dev)
                return;
        dev_info(&pdev->dev, "remove_device\n");
        if (vsoc_dev.regions_data) {
                for (i = 0; i < vsoc_dev.layout->region_count; ++i) {
                        if (vsoc_dev.regions_data[i].device_created) {
                                device_destroy(vsoc_dev.class,
                                               MKDEV(vsoc_dev.major, i));
                                vsoc_dev.regions_data[i].device_created = false;
                        }
                        if (vsoc_dev.regions_data[i].irq_requested)
                                free_irq(vsoc_dev.msix_entries[i].vector, NULL);
                        vsoc_dev.regions_data[i].irq_requested = false;
                }
                kfree(vsoc_dev.regions_data);
                vsoc_dev.regions_data = NULL;
        }
        if (vsoc_dev.msix_enabled) {
                pci_disable_msix(pdev);
                vsoc_dev.msix_enabled = false;
        }
        kfree(vsoc_dev.msix_entries);
        vsoc_dev.msix_entries = NULL;
        vsoc_dev.regions = NULL;
        if (vsoc_dev.class_added) {
                class_destroy(vsoc_dev.class);
                vsoc_dev.class_added = false;
        }
        if (vsoc_dev.cdev_added) {
                cdev_del(&vsoc_dev.cdev);
                vsoc_dev.cdev_added = false;
        }
        if (vsoc_dev.major && vsoc_dev.layout) {
                unregister_chrdev_region(MKDEV(vsoc_dev.major, 0),
                                         vsoc_dev.layout->region_count);
                vsoc_dev.major = 0;
        }
        vsoc_dev.layout = NULL;
        if (vsoc_dev.kernel_mapped_shm) {
                pci_iounmap(pdev, vsoc_dev.kernel_mapped_shm);
                vsoc_dev.kernel_mapped_shm = NULL;
        }
        if (vsoc_dev.regs) {
                pci_iounmap(pdev, vsoc_dev.regs);
                vsoc_dev.regs = NULL;
        }
        if (vsoc_dev.requested_regions) {
                pci_release_regions(pdev);
                vsoc_dev.requested_regions = false;
        }
        if (vsoc_dev.enabled_device) {
                pci_disable_device(pdev);
                vsoc_dev.enabled_device = false;
        }
        /* Do this last: it indicates that the device is not initialized. */
        vsoc_dev.dev = NULL;
}

static void __exit vsoc_cleanup_module(void)
{
        vsoc_remove_device(vsoc_dev.dev);
        pci_unregister_driver(&vsoc_pci_driver);
}

static int __init vsoc_init_module(void)
{
        int err = -ENOMEM;

        INIT_LIST_HEAD(&vsoc_dev.permissions);
        mutex_init(&vsoc_dev.mtx);

        err = pci_register_driver(&vsoc_pci_driver);
        if (err < 0)
                return err;
        return 0;
}

static int vsoc_open(struct inode *inode, struct file *filp)
{
        /* Can't use vsoc_validate_filep because filp is still incomplete */
        int ret = vsoc_validate_inode(inode);

        if (ret)
                return ret;
        filp->private_data =
                kzalloc(sizeof(struct vsoc_private_data), GFP_KERNEL);
        if (!filp->private_data)
                return -ENOMEM;
        return 0;
}

static int vsoc_release(struct inode *inode, struct file *filp)
{
        struct vsoc_private_data *private_data = NULL;
        struct fd_scoped_permission_node *node = NULL;
        struct vsoc_device_region *owner_region_p = NULL;
        int retval = vsoc_validate_filep(filp);

        if (retval)
                return retval;
        private_data = (struct vsoc_private_data *)filp->private_data;
        if (!private_data)
                return 0;

        node = private_data->fd_scoped_permission_node;
        if (node) {
                owner_region_p = vsoc_region_from_inode(inode);
                if (owner_region_p->managed_by != VSOC_REGION_WHOLE) {
                        owner_region_p =
                            &vsoc_dev.regions[owner_region_p->managed_by];
                }
                do_destroy_fd_scoped_permission_node(owner_region_p, node);
                private_data->fd_scoped_permission_node = NULL;
        }
        kfree(private_data);
        filp->private_data = NULL;

        return 0;
}

/*
 * Returns the device relative offset and length of the area specified by the
 * fd scoped permission. If there is no fd scoped permission set, a default
 * permission covering the entire region is assumed, unless the region is owned
 * by another one, in which case the default is a permission with zero size.
 */
static ssize_t vsoc_get_area(struct file *filp, __u32 *area_offset)
{
        __u32 off = 0;
        ssize_t length = 0;
        struct vsoc_device_region *region_p;
        struct fd_scoped_permission *perm;

        region_p = vsoc_region_from_filep(filp);
        off = region_p->region_begin_offset;
        perm = &((struct vsoc_private_data *)filp->private_data)->
                fd_scoped_permission_node->permission;
        if (perm) {
                off += perm->begin_offset;
                length = perm->end_offset - perm->begin_offset;
        } else if (region_p->managed_by == VSOC_REGION_WHOLE) {
                /* No permission set and the regions is not owned by another,
                 * default to full region access.
                 */
                length = vsoc_device_region_size(region_p);
        } else {
                /* return zero length, access is denied. */
                length = 0;
        }
        if (area_offset)
                *area_offset = off;
        return length;
}

static int vsoc_mmap(struct file *filp, struct vm_area_struct *vma)
{
        unsigned long len = vma->vm_end - vma->vm_start;
        __u32 area_off;
        phys_addr_t mem_off;
        ssize_t area_len;
        int retval = vsoc_validate_filep(filp);

        if (retval)
                return retval;
        area_len = vsoc_get_area(filp, &area_off);
        /* Add the requested offset */
        area_off += (vma->vm_pgoff << PAGE_SHIFT);
        area_len -= (vma->vm_pgoff << PAGE_SHIFT);
        if (area_len < len)
                return -EINVAL;
        vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
        mem_off = shm_off_to_phys_addr(area_off);
        if (io_remap_pfn_range(vma, vma->vm_start, mem_off >> PAGE_SHIFT,
                               len, vma->vm_page_prot))
                return -EAGAIN;
        return 0;
}

module_init(vsoc_init_module);
module_exit(vsoc_cleanup_module);

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Greg Hartman <ghartman@google.com>");
MODULE_DESCRIPTION("VSoC interpretation of QEmu's ivshmem device");
MODULE_VERSION("1.0");