Merge tag 'trace-printf-v6.13' of git://git.kernel.org/pub/scm/linux/kernel/git/trace...

[drm/drm-misc.git] / drivers / net / ethernet / fungible / funcore / fun_dev.c

// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)

#include <linux/bitmap.h>
#include <linux/delay.h>
#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/io-64-nonatomic-lo-hi.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/nvme.h>
#include <linux/pci.h>
#include <linux/wait.h>
#include <linux/sched/signal.h>

#include "fun_queue.h"
#include "fun_dev.h"

#define FUN_ADMIN_CMD_TO_MS 3000

enum {
        AQA_ASQS_SHIFT = 0,
        AQA_ACQS_SHIFT = 16,
        AQA_MIN_QUEUE_SIZE = 2,
        AQA_MAX_QUEUE_SIZE = 4096
};

/* context for admin commands */
struct fun_cmd_ctx {
        fun_admin_callback_t cb;  /* callback to invoke on completion */
        void *cb_data;            /* user data provided to callback */
        int cpu;                  /* CPU where the cmd's tag was allocated */
};

/* Context for synchronous admin commands. */
struct fun_sync_cmd_ctx {
        struct completion compl;
        u8 *rsp_buf;              /* caller provided response buffer */
        unsigned int rsp_len;     /* response buffer size */
        u8 rsp_status;            /* command response status */
};

/* Wait for the CSTS.RDY bit to match @enabled. */
static int fun_wait_ready(struct fun_dev *fdev, bool enabled)
{
        unsigned int cap_to = NVME_CAP_TIMEOUT(fdev->cap_reg);
        u32 bit = enabled ? NVME_CSTS_RDY : 0;
        unsigned long deadline;

        deadline = ((cap_to + 1) * HZ / 2) + jiffies; /* CAP.TO is in 500ms */

        for (;;) {
                u32 csts = readl(fdev->bar + NVME_REG_CSTS);

                if (csts == ~0) {
                        dev_err(fdev->dev, "CSTS register read %#x\n", csts);
                        return -EIO;
                }

                if ((csts & NVME_CSTS_RDY) == bit)
                        return 0;

                if (time_is_before_jiffies(deadline))
                        break;

                msleep(100);
        }

        dev_err(fdev->dev,
                "Timed out waiting for device to indicate RDY %u; aborting %s\n",
                enabled, enabled ? "initialization" : "reset");
        return -ETIMEDOUT;
}

/* Check CSTS and return an error if it is unreadable or has unexpected
 * RDY value.
 */
static int fun_check_csts_rdy(struct fun_dev *fdev, unsigned int expected_rdy)
{
        u32 csts = readl(fdev->bar + NVME_REG_CSTS);
        u32 actual_rdy = csts & NVME_CSTS_RDY;

        if (csts == ~0) {
                dev_err(fdev->dev, "CSTS register read %#x\n", csts);
                return -EIO;
        }
        if (actual_rdy != expected_rdy) {
                dev_err(fdev->dev, "Unexpected CSTS RDY %u\n", actual_rdy);
                return -EINVAL;
        }
        return 0;
}

/* Check that CSTS RDY has the expected value. Then write a new value to the CC
 * register and wait for CSTS RDY to match the new CC ENABLE state.
 */
static int fun_update_cc_enable(struct fun_dev *fdev, unsigned int initial_rdy)
{
        int rc = fun_check_csts_rdy(fdev, initial_rdy);

        if (rc)
                return rc;
        writel(fdev->cc_reg, fdev->bar + NVME_REG_CC);
        return fun_wait_ready(fdev, !!(fdev->cc_reg & NVME_CC_ENABLE));
}

static int fun_disable_ctrl(struct fun_dev *fdev)
{
        fdev->cc_reg &= ~(NVME_CC_SHN_MASK | NVME_CC_ENABLE);
        return fun_update_cc_enable(fdev, 1);
}

static int fun_enable_ctrl(struct fun_dev *fdev, u32 admin_cqesz_log2,
                           u32 admin_sqesz_log2)
{
        fdev->cc_reg = (admin_cqesz_log2 << NVME_CC_IOCQES_SHIFT) |
                       (admin_sqesz_log2 << NVME_CC_IOSQES_SHIFT) |
                       ((PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT) |
                       NVME_CC_ENABLE;

        return fun_update_cc_enable(fdev, 0);
}

static int fun_map_bars(struct fun_dev *fdev, const char *name)
{
        struct pci_dev *pdev = to_pci_dev(fdev->dev);
        int err;

        err = pci_request_mem_regions(pdev, name);
        if (err) {
                dev_err(&pdev->dev,
                        "Couldn't get PCI memory resources, err %d\n", err);
                return err;
        }

        fdev->bar = pci_ioremap_bar(pdev, 0);
        if (!fdev->bar) {
                dev_err(&pdev->dev, "Couldn't map BAR 0\n");
                pci_release_mem_regions(pdev);
                return -ENOMEM;
        }

        return 0;
}

static void fun_unmap_bars(struct fun_dev *fdev)
{
        struct pci_dev *pdev = to_pci_dev(fdev->dev);

        if (fdev->bar) {
                iounmap(fdev->bar);
                fdev->bar = NULL;
                pci_release_mem_regions(pdev);
        }
}

static int fun_set_dma_masks(struct device *dev)
{
        int err;

        err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
        if (err)
                dev_err(dev, "DMA mask configuration failed, err %d\n", err);
        return err;
}

static irqreturn_t fun_admin_irq(int irq, void *data)
{
        struct fun_queue *funq = data;

        return fun_process_cq(funq, 0) ? IRQ_HANDLED : IRQ_NONE;
}

static void fun_complete_admin_cmd(struct fun_queue *funq, void *data,
                                   void *entry, const struct fun_cqe_info *info)
{
        const struct fun_admin_rsp_common *rsp_common = entry;
        struct fun_dev *fdev = funq->fdev;
        struct fun_cmd_ctx *cmd_ctx;
        int cpu;
        u16 cid;

        if (info->sqhd == cpu_to_be16(0xffff)) {
                dev_dbg(fdev->dev, "adminq event");
                if (fdev->adminq_cb)
                        fdev->adminq_cb(fdev, entry);
                return;
        }

        cid = be16_to_cpu(rsp_common->cid);
        dev_dbg(fdev->dev, "admin CQE cid %u, op %u, ret %u\n", cid,
                rsp_common->op, rsp_common->ret);

        cmd_ctx = &fdev->cmd_ctx[cid];
        if (cmd_ctx->cpu < 0) {
                dev_err(fdev->dev,
                        "admin CQE with CID=%u, op=%u does not match a pending command\n",
                        cid, rsp_common->op);
                return;
        }

        if (cmd_ctx->cb)
                cmd_ctx->cb(fdev, entry, xchg(&cmd_ctx->cb_data, NULL));

        cpu = cmd_ctx->cpu;
        cmd_ctx->cpu = -1;
        sbitmap_queue_clear(&fdev->admin_sbq, cid, cpu);
}

static int fun_init_cmd_ctx(struct fun_dev *fdev, unsigned int ntags)
{
        unsigned int i;

        fdev->cmd_ctx = kvcalloc(ntags, sizeof(*fdev->cmd_ctx), GFP_KERNEL);
        if (!fdev->cmd_ctx)
                return -ENOMEM;

        for (i = 0; i < ntags; i++)
                fdev->cmd_ctx[i].cpu = -1;

        return 0;
}

/* Allocate and enable an admin queue and assign it the first IRQ vector. */
static int fun_enable_admin_queue(struct fun_dev *fdev,
                                  const struct fun_dev_params *areq)
{
        struct fun_queue_alloc_req qreq = {
                .cqe_size_log2 = areq->cqe_size_log2,
                .sqe_size_log2 = areq->sqe_size_log2,
                .cq_depth = areq->cq_depth,
                .sq_depth = areq->sq_depth,
                .rq_depth = areq->rq_depth,
        };
        unsigned int ntags = areq->sq_depth - 1;
        struct fun_queue *funq;
        int rc;

        if (fdev->admin_q)
                return -EEXIST;

        if (areq->sq_depth < AQA_MIN_QUEUE_SIZE ||
            areq->sq_depth > AQA_MAX_QUEUE_SIZE ||
            areq->cq_depth < AQA_MIN_QUEUE_SIZE ||
            areq->cq_depth > AQA_MAX_QUEUE_SIZE)
                return -EINVAL;

        fdev->admin_q = fun_alloc_queue(fdev, 0, &qreq);
        if (!fdev->admin_q)
                return -ENOMEM;

        rc = fun_init_cmd_ctx(fdev, ntags);
        if (rc)
                goto free_q;

        rc = sbitmap_queue_init_node(&fdev->admin_sbq, ntags, -1, false,
                                     GFP_KERNEL, dev_to_node(fdev->dev));
        if (rc)
                goto free_cmd_ctx;

        funq = fdev->admin_q;
        funq->cq_vector = 0;
        rc = fun_request_irq(funq, dev_name(fdev->dev), fun_admin_irq, funq);
        if (rc)
                goto free_sbq;

        fun_set_cq_callback(funq, fun_complete_admin_cmd, NULL);
        fdev->adminq_cb = areq->event_cb;

        writel((funq->sq_depth - 1) << AQA_ASQS_SHIFT |
               (funq->cq_depth - 1) << AQA_ACQS_SHIFT,
               fdev->bar + NVME_REG_AQA);

        writeq(funq->sq_dma_addr, fdev->bar + NVME_REG_ASQ);
        writeq(funq->cq_dma_addr, fdev->bar + NVME_REG_ACQ);

        rc = fun_enable_ctrl(fdev, areq->cqe_size_log2, areq->sqe_size_log2);
        if (rc)
                goto free_irq;

        if (areq->rq_depth) {
                rc = fun_create_rq(funq);
                if (rc)
                        goto disable_ctrl;

                funq_rq_post(funq);
        }

        return 0;

disable_ctrl:
        fun_disable_ctrl(fdev);
free_irq:
        fun_free_irq(funq);
free_sbq:
        sbitmap_queue_free(&fdev->admin_sbq);
free_cmd_ctx:
        kvfree(fdev->cmd_ctx);
        fdev->cmd_ctx = NULL;
free_q:
        fun_free_queue(fdev->admin_q);
        fdev->admin_q = NULL;
        return rc;
}

static void fun_disable_admin_queue(struct fun_dev *fdev)
{
        struct fun_queue *admq = fdev->admin_q;

        if (!admq)
                return;

        fun_disable_ctrl(fdev);

        fun_free_irq(admq);
        __fun_process_cq(admq, 0);

        sbitmap_queue_free(&fdev->admin_sbq);

        kvfree(fdev->cmd_ctx);
        fdev->cmd_ctx = NULL;

        fun_free_queue(admq);
        fdev->admin_q = NULL;
}

/* Return %true if the admin queue has stopped servicing commands as can be
 * detected through registers. This isn't exhaustive and may provide false
 * negatives.
 */
static bool fun_adminq_stopped(struct fun_dev *fdev)
{
        u32 csts = readl(fdev->bar + NVME_REG_CSTS);

        return (csts & (NVME_CSTS_CFS | NVME_CSTS_RDY)) != NVME_CSTS_RDY;
}

static int fun_wait_for_tag(struct fun_dev *fdev, int *cpup)
{
        struct sbitmap_queue *sbq = &fdev->admin_sbq;
        struct sbq_wait_state *ws = &sbq->ws[0];
        DEFINE_SBQ_WAIT(wait);
        int tag;

        for (;;) {
                sbitmap_prepare_to_wait(sbq, ws, &wait, TASK_UNINTERRUPTIBLE);
                if (fdev->suppress_cmds) {
                        tag = -ESHUTDOWN;
                        break;
                }
                tag = sbitmap_queue_get(sbq, cpup);
                if (tag >= 0)
                        break;
                schedule();
        }

        sbitmap_finish_wait(sbq, ws, &wait);
        return tag;
}

/* Submit an asynchronous admin command. Caller is responsible for implementing
 * any waiting or timeout. Upon command completion the callback @cb is called.
 */
int fun_submit_admin_cmd(struct fun_dev *fdev, struct fun_admin_req_common *cmd,
                         fun_admin_callback_t cb, void *cb_data, bool wait_ok)
{
        struct fun_queue *funq = fdev->admin_q;
        unsigned int cmdsize = cmd->len8 * 8;
        struct fun_cmd_ctx *cmd_ctx;
        int tag, cpu, rc = 0;

        if (WARN_ON(cmdsize > (1 << funq->sqe_size_log2)))
                return -EMSGSIZE;

        tag = sbitmap_queue_get(&fdev->admin_sbq, &cpu);
        if (tag < 0) {
                if (!wait_ok)
                        return -EAGAIN;
                tag = fun_wait_for_tag(fdev, &cpu);
                if (tag < 0)
                        return tag;
        }

        cmd->cid = cpu_to_be16(tag);

        cmd_ctx = &fdev->cmd_ctx[tag];
        cmd_ctx->cb = cb;
        cmd_ctx->cb_data = cb_data;

        spin_lock(&funq->sq_lock);

        if (unlikely(fdev->suppress_cmds)) {
                rc = -ESHUTDOWN;
                sbitmap_queue_clear(&fdev->admin_sbq, tag, cpu);
        } else {
                cmd_ctx->cpu = cpu;
                memcpy(fun_sqe_at(funq, funq->sq_tail), cmd, cmdsize);

                dev_dbg(fdev->dev, "admin cmd @ %u: %8ph\n", funq->sq_tail,
                        cmd);

                if (++funq->sq_tail == funq->sq_depth)
                        funq->sq_tail = 0;
                writel(funq->sq_tail, funq->sq_db);
        }
        spin_unlock(&funq->sq_lock);
        return rc;
}

/* Abandon a pending admin command by clearing the issuer's callback data.
 * Failure indicates that the command either has already completed or its
 * completion is racing with this call.
 */
static bool fun_abandon_admin_cmd(struct fun_dev *fd,
                                  const struct fun_admin_req_common *cmd,
                                  void *cb_data)
{
        u16 cid = be16_to_cpu(cmd->cid);
        struct fun_cmd_ctx *cmd_ctx = &fd->cmd_ctx[cid];

        return cmpxchg(&cmd_ctx->cb_data, cb_data, NULL) == cb_data;
}

/* Stop submission of new admin commands and wake up any processes waiting for
 * tags. Already submitted commands are left to complete or time out.
 */
static void fun_admin_stop(struct fun_dev *fdev)
{
        spin_lock(&fdev->admin_q->sq_lock);
        fdev->suppress_cmds = true;
        spin_unlock(&fdev->admin_q->sq_lock);
        sbitmap_queue_wake_all(&fdev->admin_sbq);
}

/* The callback for synchronous execution of admin commands. It copies the
 * command response to the caller's buffer and signals completion.
 */
static void fun_admin_cmd_sync_cb(struct fun_dev *fd, void *rsp, void *cb_data)
{
        const struct fun_admin_rsp_common *rsp_common = rsp;
        struct fun_sync_cmd_ctx *ctx = cb_data;

        if (!ctx)
                return;         /* command issuer timed out and left */
        if (ctx->rsp_buf) {
                unsigned int rsp_len = rsp_common->len8 * 8;

                if (unlikely(rsp_len > ctx->rsp_len)) {
                        dev_err(fd->dev,
                                "response for op %u is %uB > response buffer %uB\n",
                                rsp_common->op, rsp_len, ctx->rsp_len);
                        rsp_len = ctx->rsp_len;
                }
                memcpy(ctx->rsp_buf, rsp, rsp_len);
        }
        ctx->rsp_status = rsp_common->ret;
        complete(&ctx->compl);
}

/* Submit a synchronous admin command. */
int fun_submit_admin_sync_cmd(struct fun_dev *fdev,
                              struct fun_admin_req_common *cmd, void *rsp,
                              size_t rspsize, unsigned int timeout)
{
        struct fun_sync_cmd_ctx ctx = {
                .compl = COMPLETION_INITIALIZER_ONSTACK(ctx.compl),
                .rsp_buf = rsp,
                .rsp_len = rspsize,
        };
        unsigned int cmdlen = cmd->len8 * 8;
        unsigned long jiffies_left;
        int ret;

        ret = fun_submit_admin_cmd(fdev, cmd, fun_admin_cmd_sync_cb, &ctx,
                                   true);
        if (ret)
                return ret;

        if (!timeout)
                timeout = FUN_ADMIN_CMD_TO_MS;

        jiffies_left = wait_for_completion_timeout(&ctx.compl,
                                                   msecs_to_jiffies(timeout));
        if (!jiffies_left) {
                /* The command timed out. Attempt to cancel it so we can return.
                 * But if the command is in the process of completing we'll
                 * wait for it.
                 */
                if (fun_abandon_admin_cmd(fdev, cmd, &ctx)) {
                        dev_err(fdev->dev, "admin command timed out: %*ph\n",
                                cmdlen, cmd);
                        fun_admin_stop(fdev);
                        /* see if the timeout was due to a queue failure */
                        if (fun_adminq_stopped(fdev))
                                dev_err(fdev->dev,
                                        "device does not accept admin commands\n");

                        return -ETIMEDOUT;
                }
                wait_for_completion(&ctx.compl);
        }

        if (ctx.rsp_status) {
                dev_err(fdev->dev, "admin command failed, err %d: %*ph\n",
                        ctx.rsp_status, cmdlen, cmd);
        }

        return -ctx.rsp_status;
}
EXPORT_SYMBOL_GPL(fun_submit_admin_sync_cmd);

/* Return the number of device resources of the requested type. */
int fun_get_res_count(struct fun_dev *fdev, enum fun_admin_op res)
{
        union {
                struct fun_admin_res_count_req req;
                struct fun_admin_res_count_rsp rsp;
        } cmd;
        int rc;

        cmd.req.common = FUN_ADMIN_REQ_COMMON_INIT2(res, sizeof(cmd.req));
        cmd.req.count = FUN_ADMIN_SIMPLE_SUBOP_INIT(FUN_ADMIN_SUBOP_RES_COUNT,
                                                    0, 0);

        rc = fun_submit_admin_sync_cmd(fdev, &cmd.req.common, &cmd.rsp,
                                       sizeof(cmd), 0);
        return rc ? rc : be32_to_cpu(cmd.rsp.count.data);
}
EXPORT_SYMBOL_GPL(fun_get_res_count);

/* Request that the instance of resource @res with the given id be deleted. */
int fun_res_destroy(struct fun_dev *fdev, enum fun_admin_op res,
                    unsigned int flags, u32 id)
{
        struct fun_admin_generic_destroy_req req = {
                .common = FUN_ADMIN_REQ_COMMON_INIT2(res, sizeof(req)),
                .destroy = FUN_ADMIN_SIMPLE_SUBOP_INIT(FUN_ADMIN_SUBOP_DESTROY,
                                                       flags, id)
        };

        return fun_submit_admin_sync_cmd(fdev, &req.common, NULL, 0, 0);
}
EXPORT_SYMBOL_GPL(fun_res_destroy);

/* Bind two entities of the given types and IDs. */
int fun_bind(struct fun_dev *fdev, enum fun_admin_bind_type type0,
             unsigned int id0, enum fun_admin_bind_type type1,
             unsigned int id1)
{
        DEFINE_RAW_FLEX(struct fun_admin_bind_req, cmd, entry, 2);

        cmd->common = FUN_ADMIN_REQ_COMMON_INIT2(FUN_ADMIN_OP_BIND,
                                                 __struct_size(cmd));
        cmd->entry[0] = FUN_ADMIN_BIND_ENTRY_INIT(type0, id0);
        cmd->entry[1] = FUN_ADMIN_BIND_ENTRY_INIT(type1, id1);

        return fun_submit_admin_sync_cmd(fdev, &cmd->common, NULL, 0, 0);
}
EXPORT_SYMBOL_GPL(fun_bind);

static int fun_get_dev_limits(struct fun_dev *fdev)
{
        struct pci_dev *pdev = to_pci_dev(fdev->dev);
        unsigned int cq_count, sq_count, num_dbs;
        int rc;

        rc = fun_get_res_count(fdev, FUN_ADMIN_OP_EPCQ);
        if (rc < 0)
                return rc;
        cq_count = rc;

        rc = fun_get_res_count(fdev, FUN_ADMIN_OP_EPSQ);
        if (rc < 0)
                return rc;
        sq_count = rc;

        /* The admin queue consumes 1 CQ and at least 1 SQ. To be usable the
         * device must provide additional queues.
         */
        if (cq_count < 2 || sq_count < 2 + !!fdev->admin_q->rq_depth)
                return -EINVAL;

        /* Calculate the max QID based on SQ/CQ/doorbell counts.
         * SQ/CQ doorbells alternate.
         */
        num_dbs = (pci_resource_len(pdev, 0) - NVME_REG_DBS) >>
                  (2 + NVME_CAP_STRIDE(fdev->cap_reg));
        fdev->max_qid = min3(cq_count, sq_count, num_dbs / 2) - 1;
        fdev->kern_end_qid = fdev->max_qid + 1;
        return 0;
}

/* Allocate all MSI-X vectors available on a function and at least @min_vecs. */
static int fun_alloc_irqs(struct pci_dev *pdev, unsigned int min_vecs)
{
        int vecs, num_msix = pci_msix_vec_count(pdev);

        if (num_msix < 0)
                return num_msix;
        if (min_vecs > num_msix)
                return -ERANGE;

        vecs = pci_alloc_irq_vectors(pdev, min_vecs, num_msix, PCI_IRQ_MSIX);
        if (vecs > 0) {
                dev_info(&pdev->dev,
                         "Allocated %d IRQ vectors of %d requested\n",
                         vecs, num_msix);
        } else {
                dev_err(&pdev->dev,
                        "Unable to allocate at least %u IRQ vectors\n",
                        min_vecs);
        }
        return vecs;
}

/* Allocate and initialize the IRQ manager state. */
static int fun_alloc_irq_mgr(struct fun_dev *fdev)
{
        fdev->irq_map = bitmap_zalloc(fdev->num_irqs, GFP_KERNEL);
        if (!fdev->irq_map)
                return -ENOMEM;

        spin_lock_init(&fdev->irqmgr_lock);
        /* mark IRQ 0 allocated, it is used by the admin queue */
        __set_bit(0, fdev->irq_map);
        fdev->irqs_avail = fdev->num_irqs - 1;
        return 0;
}

/* Reserve @nirqs of the currently available IRQs and return their indices. */
int fun_reserve_irqs(struct fun_dev *fdev, unsigned int nirqs, u16 *irq_indices)
{
        unsigned int b, n = 0;
        int err = -ENOSPC;

        if (!nirqs)
                return 0;

        spin_lock(&fdev->irqmgr_lock);
        if (nirqs > fdev->irqs_avail)
                goto unlock;

        for_each_clear_bit(b, fdev->irq_map, fdev->num_irqs) {
                __set_bit(b, fdev->irq_map);
                irq_indices[n++] = b;
                if (n >= nirqs)
                        break;
        }

        WARN_ON(n < nirqs);
        fdev->irqs_avail -= n;
        err = n;
unlock:
        spin_unlock(&fdev->irqmgr_lock);
        return err;
}
EXPORT_SYMBOL(fun_reserve_irqs);

/* Release @nirqs previously allocated IRQS with the supplied indices. */
void fun_release_irqs(struct fun_dev *fdev, unsigned int nirqs,
                      u16 *irq_indices)
{
        unsigned int i;

        spin_lock(&fdev->irqmgr_lock);
        for (i = 0; i < nirqs; i++)
                __clear_bit(irq_indices[i], fdev->irq_map);
        fdev->irqs_avail += nirqs;
        spin_unlock(&fdev->irqmgr_lock);
}
EXPORT_SYMBOL(fun_release_irqs);

static void fun_serv_handler(struct work_struct *work)
{
        struct fun_dev *fd = container_of(work, struct fun_dev, service_task);

        if (test_bit(FUN_SERV_DISABLED, &fd->service_flags))
                return;
        if (fd->serv_cb)
                fd->serv_cb(fd);
}

void fun_serv_stop(struct fun_dev *fd)
{
        set_bit(FUN_SERV_DISABLED, &fd->service_flags);
        cancel_work_sync(&fd->service_task);
}
EXPORT_SYMBOL_GPL(fun_serv_stop);

void fun_serv_restart(struct fun_dev *fd)
{
        clear_bit(FUN_SERV_DISABLED, &fd->service_flags);
        if (fd->service_flags)
                schedule_work(&fd->service_task);
}
EXPORT_SYMBOL_GPL(fun_serv_restart);

void fun_serv_sched(struct fun_dev *fd)
{
        if (!test_bit(FUN_SERV_DISABLED, &fd->service_flags))
                schedule_work(&fd->service_task);
}
EXPORT_SYMBOL_GPL(fun_serv_sched);

/* Check and try to get the device into a proper state for initialization,
 * i.e., CSTS.RDY = CC.EN = 0.
 */
static int sanitize_dev(struct fun_dev *fdev)
{
        int rc;

        fdev->cap_reg = readq(fdev->bar + NVME_REG_CAP);
        fdev->cc_reg = readl(fdev->bar + NVME_REG_CC);

        /* First get RDY to agree with the current EN. Give RDY the opportunity
         * to complete a potential recent EN change.
         */
        rc = fun_wait_ready(fdev, fdev->cc_reg & NVME_CC_ENABLE);
        if (rc)
                return rc;

        /* Next, reset the device if EN is currently 1. */
        if (fdev->cc_reg & NVME_CC_ENABLE)
                rc = fun_disable_ctrl(fdev);

        return rc;
}

/* Undo the device initialization of fun_dev_enable(). */
void fun_dev_disable(struct fun_dev *fdev)
{
        struct pci_dev *pdev = to_pci_dev(fdev->dev);

        pci_set_drvdata(pdev, NULL);

        if (fdev->fw_handle != FUN_HCI_ID_INVALID) {
                fun_res_destroy(fdev, FUN_ADMIN_OP_SWUPGRADE, 0,
                                fdev->fw_handle);
                fdev->fw_handle = FUN_HCI_ID_INVALID;
        }

        fun_disable_admin_queue(fdev);

        bitmap_free(fdev->irq_map);
        pci_free_irq_vectors(pdev);

        pci_disable_device(pdev);

        fun_unmap_bars(fdev);
}
EXPORT_SYMBOL(fun_dev_disable);

/* Perform basic initialization of a device, including
 * - PCI config space setup and BAR0 mapping
 * - interrupt management initialization
 * - 1 admin queue setup
 * - determination of some device limits, such as number of queues.
 */
int fun_dev_enable(struct fun_dev *fdev, struct pci_dev *pdev,
                   const struct fun_dev_params *areq, const char *name)
{
        int rc;

        fdev->dev = &pdev->dev;
        rc = fun_map_bars(fdev, name);
        if (rc)
                return rc;

        rc = fun_set_dma_masks(fdev->dev);
        if (rc)
                goto unmap;

        rc = pci_enable_device_mem(pdev);
        if (rc) {
                dev_err(&pdev->dev, "Couldn't enable device, err %d\n", rc);
                goto unmap;
        }

        rc = sanitize_dev(fdev);
        if (rc)
                goto disable_dev;

        fdev->fw_handle = FUN_HCI_ID_INVALID;
        fdev->q_depth = NVME_CAP_MQES(fdev->cap_reg) + 1;
        fdev->db_stride = 1 << NVME_CAP_STRIDE(fdev->cap_reg);
        fdev->dbs = fdev->bar + NVME_REG_DBS;

        INIT_WORK(&fdev->service_task, fun_serv_handler);
        fdev->service_flags = FUN_SERV_DISABLED;
        fdev->serv_cb = areq->serv_cb;

        rc = fun_alloc_irqs(pdev, areq->min_msix + 1); /* +1 for admin CQ */
        if (rc < 0)
                goto disable_dev;
        fdev->num_irqs = rc;

        rc = fun_alloc_irq_mgr(fdev);
        if (rc)
                goto free_irqs;

        pci_set_master(pdev);
        rc = fun_enable_admin_queue(fdev, areq);
        if (rc)
                goto free_irq_mgr;

        rc = fun_get_dev_limits(fdev);
        if (rc < 0)
                goto disable_admin;

        pci_save_state(pdev);
        pci_set_drvdata(pdev, fdev);
        pcie_print_link_status(pdev);
        dev_dbg(fdev->dev, "q_depth %u, db_stride %u, max qid %d kern_end_qid %d\n",
                fdev->q_depth, fdev->db_stride, fdev->max_qid,
                fdev->kern_end_qid);
        return 0;

disable_admin:
        fun_disable_admin_queue(fdev);
free_irq_mgr:
        bitmap_free(fdev->irq_map);
free_irqs:
        pci_free_irq_vectors(pdev);
disable_dev:
        pci_disable_device(pdev);
unmap:
        fun_unmap_bars(fdev);
        return rc;
}
EXPORT_SYMBOL(fun_dev_enable);

MODULE_AUTHOR("Dimitris Michailidis <dmichail@fungible.com>");
MODULE_DESCRIPTION("Core services driver for Fungible devices");
MODULE_LICENSE("Dual BSD/GPL");