8354 sync regcomp(3C) with upstream (fix make catalog)

[unleashed/tickless.git] / usr / src / uts / intel / io / vmxnet3s / vmxnet3_main.c

/*
 * Copyright (C) 2007-2014 VMware, Inc. All rights reserved.
 *
 * The contents of this file are subject to the terms of the Common
 * Development and Distribution License (the "License") version 1.0
 * and no later version.  You may not use this file except in
 * compliance with the License.
 *
 * You can obtain a copy of the License at
 *         http://www.opensource.org/licenses/cddl1.php
 *
 * See the License for the specific language governing permissions
 * and limitations under the License.
 */

/*
 * Copyright (c) 2012, 2016 by Delphix. All rights reserved.
 */

#include <vmxnet3.h>

/*
 * This driver is based on VMware's version 3227872, and contains additional
 * enhancements (see README.txt).
 */
#define BUILD_NUMBER_NUMERIC    3227872

/*
 * If we run out of rxPool buffers, only allocate if the MTU is <= PAGESIZE
 * so that we don't have to incur the cost of allocating multiple contiguous
 * pages (very slow) in interrupt context.
 */
#define VMXNET3_ALLOC_OK(dp)    ((dp)->cur_mtu <= PAGESIZE)

/*
 * TODO:
 *    - Tx data ring
 *    - MAC_CAPAB_POLL support
 *    - Dynamic RX pool
 */

static int vmxnet3_getstat(void *, uint_t, uint64_t *);
static int vmxnet3_start(void *);
static void vmxnet3_stop(void *);
static int vmxnet3_setpromisc(void *, boolean_t);
static void vmxnet3_ioctl(void *arg, queue_t *wq, mblk_t *mp);
static int vmxnet3_multicst(void *, boolean_t, const uint8_t *);
static int vmxnet3_unicst(void *, const uint8_t *);
static boolean_t vmxnet3_getcapab(void *, mac_capab_t, void *);
static int vmxnet3_get_prop(void *, const char *, mac_prop_id_t, uint_t,
    void *);
static int vmxnet3_set_prop(void *, const char *, mac_prop_id_t, uint_t,
    const void *);
static void vmxnet3_prop_info(void *, const char *, mac_prop_id_t,
    mac_prop_info_handle_t);

int vmxnet3s_debug = 0;

/* MAC callbacks */
static mac_callbacks_t vmxnet3_mac_callbacks = {
        .mc_callbacks = MC_GETCAPAB | MC_IOCTL | MC_SETPROP | MC_PROPINFO,
        .mc_getstat =   vmxnet3_getstat,
        .mc_start =     vmxnet3_start,
        .mc_stop =      vmxnet3_stop,
        .mc_setpromisc = vmxnet3_setpromisc,
        .mc_multicst =  vmxnet3_multicst,
        .mc_unicst =    vmxnet3_unicst,
        .mc_tx =        vmxnet3_tx,
        .mc_ioctl =     vmxnet3_ioctl,
        .mc_getcapab =  vmxnet3_getcapab,
        .mc_getprop =   vmxnet3_get_prop,
        .mc_setprop =   vmxnet3_set_prop,
        .mc_propinfo =  vmxnet3_prop_info
};

/* Tx DMA engine description */
static ddi_dma_attr_t vmxnet3_dma_attrs_tx = {
        .dma_attr_version =     DMA_ATTR_V0,
        .dma_attr_addr_lo =     0x0000000000000000ull,
        .dma_attr_addr_hi =     0xFFFFFFFFFFFFFFFFull,
        .dma_attr_count_max =   0xFFFFFFFFFFFFFFFFull,
        .dma_attr_align =       0x0000000000000001ull,
        .dma_attr_burstsizes =  0x0000000000000001ull,
        .dma_attr_minxfer =     0x00000001,
        .dma_attr_maxxfer =     0x000000000000FFFFull,
        .dma_attr_seg =         0xFFFFFFFFFFFFFFFFull,
        .dma_attr_sgllen =      -1,
        .dma_attr_granular =    0x00000001,
        .dma_attr_flags =       0
};

/* --- */

/*
 * Fetch the statistics of a vmxnet3 device.
 *
 * Returns:
 *      0 on success, non-zero on failure.
 */
static int
vmxnet3_getstat(void *data, uint_t stat, uint64_t *val)
{
        vmxnet3_softc_t *dp = data;
        UPT1_TxStats *txStats;
        UPT1_RxStats *rxStats;

        VMXNET3_DEBUG(dp, 3, "getstat(%u)\n", stat);

        if (!dp->devEnabled) {
                return (EBUSY);
        }

        txStats = &VMXNET3_TQDESC(dp)->stats;
        rxStats = &VMXNET3_RQDESC(dp)->stats;

        /*
         * First touch the related register
         */
        switch (stat) {
        case MAC_STAT_MULTIRCV:
        case MAC_STAT_BRDCSTRCV:
        case MAC_STAT_MULTIXMT:
        case MAC_STAT_BRDCSTXMT:
        case MAC_STAT_NORCVBUF:
        case MAC_STAT_IERRORS:
        case MAC_STAT_NOXMTBUF:
        case MAC_STAT_OERRORS:
        case MAC_STAT_RBYTES:
        case MAC_STAT_IPACKETS:
        case MAC_STAT_OBYTES:
        case MAC_STAT_OPACKETS:
                VMXNET3_BAR1_PUT32(dp, VMXNET3_REG_CMD, VMXNET3_CMD_GET_STATS);
                break;
        case MAC_STAT_IFSPEED:
        case MAC_STAT_COLLISIONS:
        case ETHER_STAT_LINK_DUPLEX:
                /* nothing */
                break;
        default:
                return (ENOTSUP);
        }

        /*
         * Then fetch the corresponding stat
         */
        switch (stat) {
        case MAC_STAT_IFSPEED:
                *val = dp->linkSpeed;
                break;
        case MAC_STAT_MULTIRCV:
                *val = rxStats->mcastPktsRxOK;
                break;
        case MAC_STAT_BRDCSTRCV:
                *val = rxStats->bcastPktsRxOK;
                break;
        case MAC_STAT_MULTIXMT:
                *val = txStats->mcastPktsTxOK;
                break;
        case MAC_STAT_BRDCSTXMT:
                *val = txStats->bcastPktsTxOK;
                break;
        case MAC_STAT_NORCVBUF:
                *val = rxStats->pktsRxOutOfBuf + dp->rx_alloc_failed;
                break;
        case MAC_STAT_IERRORS:
                *val = rxStats->pktsRxError;
                break;
        case MAC_STAT_NOXMTBUF:
                *val = txStats->pktsTxDiscard + dp->tx_pullup_failed;
                break;
        case MAC_STAT_OERRORS:
                *val = txStats->pktsTxError + dp->tx_error;
                break;
        case MAC_STAT_COLLISIONS:
                *val = 0;
                break;
        case MAC_STAT_RBYTES:
                *val = rxStats->ucastBytesRxOK + rxStats->mcastBytesRxOK +
                    rxStats->bcastBytesRxOK;
                break;
        case MAC_STAT_IPACKETS:
                *val = rxStats->ucastPktsRxOK + rxStats->mcastPktsRxOK +
                    rxStats->bcastPktsRxOK;
                break;
        case MAC_STAT_OBYTES:
                *val = txStats->ucastBytesTxOK + txStats->mcastBytesTxOK +
                    txStats->bcastBytesTxOK;
                break;
        case MAC_STAT_OPACKETS:
                *val = txStats->ucastPktsTxOK + txStats->mcastPktsTxOK +
                    txStats->bcastPktsTxOK;
                break;
        case ETHER_STAT_LINK_DUPLEX:
                *val = LINK_DUPLEX_FULL;
                break;
        default:
                ASSERT(B_FALSE);
        }

        return (0);
}

/*
 * Allocate and initialize the shared data structures of a vmxnet3 device.
 *
 * Returns:
 *      0 on sucess, non-zero on failure.
 */
static int
vmxnet3_prepare_drivershared(vmxnet3_softc_t *dp)
{
        Vmxnet3_DriverShared *ds;
        size_t allocSize = sizeof (Vmxnet3_DriverShared);
        int err;

        if ((err = vmxnet3_alloc_dma_mem_1(dp, &dp->sharedData, allocSize,
            B_TRUE)) != 0) {
                return (err);
        }
        ds = VMXNET3_DS(dp);
        (void) memset(ds, 0, allocSize);

        allocSize = sizeof (Vmxnet3_TxQueueDesc) + sizeof (Vmxnet3_RxQueueDesc);
        if ((err = vmxnet3_alloc_dma_mem_128(dp, &dp->queueDescs, allocSize,
            B_TRUE)) != 0) {
                vmxnet3_free_dma_mem(&dp->sharedData);
                return (err);
        }
        (void) memset(dp->queueDescs.buf, 0, allocSize);

        ds->magic = VMXNET3_REV1_MAGIC;

        /* Take care of most of devRead */
        ds->devRead.misc.driverInfo.version = BUILD_NUMBER_NUMERIC;
#ifdef _LP64
        ds->devRead.misc.driverInfo.gos.gosBits = VMXNET3_GOS_BITS_64;
#else
        ds->devRead.misc.driverInfo.gos.gosBits = VMXNET3_GOS_BITS_32;
#endif
        ds->devRead.misc.driverInfo.gos.gosType = VMXNET3_GOS_TYPE_SOLARIS;
        ds->devRead.misc.driverInfo.gos.gosVer = 10;
        ds->devRead.misc.driverInfo.vmxnet3RevSpt = 1;
        ds->devRead.misc.driverInfo.uptVerSpt = 1;

        ds->devRead.misc.uptFeatures = UPT1_F_RXCSUM;
        ds->devRead.misc.mtu = dp->cur_mtu;

        /* XXX: ds->devRead.misc.maxNumRxSG */
        ds->devRead.misc.numTxQueues = 1;
        ds->devRead.misc.numRxQueues = 1;
        ds->devRead.misc.queueDescPA = dp->queueDescs.bufPA;
        ds->devRead.misc.queueDescLen = allocSize;

        /* TxQueue and RxQueue information is filled in other functions */
        ds->devRead.intrConf.autoMask = (dp->intrMaskMode == VMXNET3_IMM_AUTO);
        ds->devRead.intrConf.numIntrs = 1;
        /* XXX: ds->intr.modLevels */
        ds->devRead.intrConf.eventIntrIdx = 0;

        VMXNET3_BAR1_PUT32(dp, VMXNET3_REG_DSAL,
            VMXNET3_ADDR_LO(dp->sharedData.bufPA));
        VMXNET3_BAR1_PUT32(dp, VMXNET3_REG_DSAH,
            VMXNET3_ADDR_HI(dp->sharedData.bufPA));

        return (0);
}

/*
 * Destroy the shared data structures of a vmxnet3 device.
 */
static void
vmxnet3_destroy_drivershared(vmxnet3_softc_t *dp)
{
        VMXNET3_BAR1_PUT32(dp, VMXNET3_REG_DSAL, 0);
        VMXNET3_BAR1_PUT32(dp, VMXNET3_REG_DSAH, 0);

        vmxnet3_free_dma_mem(&dp->queueDescs);
        vmxnet3_free_dma_mem(&dp->sharedData);
}

/*
 * Allocate and initialize the command ring of a queue.
 *
 * Returns:
 *      0 on success, non-zero on error.
 */
static int
vmxnet3_alloc_cmdring(vmxnet3_softc_t *dp, vmxnet3_cmdring_t *cmdRing)
{
        size_t ringSize = cmdRing->size * sizeof (Vmxnet3_TxDesc);
        int err;

        if ((err = vmxnet3_alloc_dma_mem_512(dp, &cmdRing->dma, ringSize,
            B_TRUE)) != 0) {
                return (err);
        }
        (void) memset(cmdRing->dma.buf, 0, ringSize);
        cmdRing->avail = cmdRing->size;
        cmdRing->next2fill = 0;
        cmdRing->gen = VMXNET3_INIT_GEN;

        return (0);
}

/*
 * Allocate and initialize the completion ring of a queue.
 *
 * Returns:
 *    DDI_SUCCESS or DDI_FAILURE.
 */
static int
vmxnet3_alloc_compring(vmxnet3_softc_t *dp, vmxnet3_compring_t *compRing)
{
        size_t ringSize = compRing->size * sizeof (Vmxnet3_TxCompDesc);

        if (vmxnet3_alloc_dma_mem_512(dp, &compRing->dma, ringSize,
            B_TRUE) != DDI_SUCCESS) {
                return (DDI_FAILURE);
        }
        (void) memset(compRing->dma.buf, 0, ringSize);
        compRing->next2comp = 0;
        compRing->gen = VMXNET3_INIT_GEN;

        return (DDI_SUCCESS);
}

/*
 * Initialize the tx queue of a vmxnet3 device.
 *
 * Returns:
 *      0 on success, non-zero on failure.
 */
static int
vmxnet3_prepare_txqueue(vmxnet3_softc_t *dp)
{
        Vmxnet3_TxQueueDesc *tqdesc = VMXNET3_TQDESC(dp);
        vmxnet3_txqueue_t *txq = &dp->txQueue;
        int err;

        ASSERT(!(txq->cmdRing.size & VMXNET3_RING_SIZE_MASK));
        ASSERT(!(txq->compRing.size & VMXNET3_RING_SIZE_MASK));
        ASSERT(!txq->cmdRing.dma.buf && !txq->compRing.dma.buf);

        if ((err = vmxnet3_alloc_cmdring(dp, &txq->cmdRing)) != 0) {
                goto error;
        }
        tqdesc->conf.txRingBasePA = txq->cmdRing.dma.bufPA;
        tqdesc->conf.txRingSize = txq->cmdRing.size;
        tqdesc->conf.dataRingBasePA = 0;
        tqdesc->conf.dataRingSize = 0;

        if ((err = vmxnet3_alloc_compring(dp, &txq->compRing)) != 0) {
                goto error_cmdring;
        }
        tqdesc->conf.compRingBasePA = txq->compRing.dma.bufPA;
        tqdesc->conf.compRingSize = txq->compRing.size;

        txq->metaRing = kmem_zalloc(txq->cmdRing.size *
            sizeof (vmxnet3_metatx_t), KM_SLEEP);
        ASSERT(txq->metaRing);

        if ((err = vmxnet3_txqueue_init(dp, txq)) != 0) {
                goto error_mpring;
        }

        return (0);

error_mpring:
        kmem_free(txq->metaRing, txq->cmdRing.size * sizeof (vmxnet3_metatx_t));
        vmxnet3_free_dma_mem(&txq->compRing.dma);
error_cmdring:
        vmxnet3_free_dma_mem(&txq->cmdRing.dma);
error:
        return (err);
}

/*
 * Initialize the rx queue of a vmxnet3 device.
 *
 * Returns:
 *      0 on success, non-zero on failure.
 */
static int
vmxnet3_prepare_rxqueue(vmxnet3_softc_t *dp)
{
        Vmxnet3_RxQueueDesc *rqdesc = VMXNET3_RQDESC(dp);
        vmxnet3_rxqueue_t *rxq = &dp->rxQueue;
        int err = 0;

        ASSERT(!(rxq->cmdRing.size & VMXNET3_RING_SIZE_MASK));
        ASSERT(!(rxq->compRing.size & VMXNET3_RING_SIZE_MASK));
        ASSERT(!rxq->cmdRing.dma.buf && !rxq->compRing.dma.buf);

        if ((err = vmxnet3_alloc_cmdring(dp, &rxq->cmdRing)) != 0) {
                goto error;
        }
        rqdesc->conf.rxRingBasePA[0] = rxq->cmdRing.dma.bufPA;
        rqdesc->conf.rxRingSize[0] = rxq->cmdRing.size;
        rqdesc->conf.rxRingBasePA[1] = 0;
        rqdesc->conf.rxRingSize[1] = 0;

        if ((err = vmxnet3_alloc_compring(dp, &rxq->compRing)) != 0) {
                goto error_cmdring;
        }
        rqdesc->conf.compRingBasePA = rxq->compRing.dma.bufPA;
        rqdesc->conf.compRingSize = rxq->compRing.size;

        rxq->bufRing = kmem_zalloc(rxq->cmdRing.size *
            sizeof (vmxnet3_bufdesc_t), KM_SLEEP);
        ASSERT(rxq->bufRing);

        if ((err = vmxnet3_rxqueue_init(dp, rxq)) != 0) {
                goto error_bufring;
        }

        return (0);

error_bufring:
        kmem_free(rxq->bufRing, rxq->cmdRing.size * sizeof (vmxnet3_bufdesc_t));
        vmxnet3_free_dma_mem(&rxq->compRing.dma);
error_cmdring:
        vmxnet3_free_dma_mem(&rxq->cmdRing.dma);
error:
        return (err);
}

/*
 * Destroy the tx queue of a vmxnet3 device.
 */
static void
vmxnet3_destroy_txqueue(vmxnet3_softc_t *dp)
{
        vmxnet3_txqueue_t *txq = &dp->txQueue;

        ASSERT(txq->metaRing);
        ASSERT(txq->cmdRing.dma.buf && txq->compRing.dma.buf);

        vmxnet3_txqueue_fini(dp, txq);

        kmem_free(txq->metaRing, txq->cmdRing.size * sizeof (vmxnet3_metatx_t));

        vmxnet3_free_dma_mem(&txq->cmdRing.dma);
        vmxnet3_free_dma_mem(&txq->compRing.dma);
}

/*
 * Destroy the rx queue of a vmxnet3 device.
 */
static void
vmxnet3_destroy_rxqueue(vmxnet3_softc_t *dp)
{
        vmxnet3_rxqueue_t *rxq = &dp->rxQueue;

        ASSERT(rxq->bufRing);
        ASSERT(rxq->cmdRing.dma.buf && rxq->compRing.dma.buf);

        vmxnet3_rxqueue_fini(dp, rxq);

        kmem_free(rxq->bufRing, rxq->cmdRing.size * sizeof (vmxnet3_bufdesc_t));

        vmxnet3_free_dma_mem(&rxq->cmdRing.dma);
        vmxnet3_free_dma_mem(&rxq->compRing.dma);
}

/*
 * Apply new RX filters settings to a vmxnet3 device.
 */
static void
vmxnet3_refresh_rxfilter(vmxnet3_softc_t *dp)
{
        Vmxnet3_DriverShared *ds = VMXNET3_DS(dp);

        ds->devRead.rxFilterConf.rxMode = dp->rxMode;
        VMXNET3_BAR1_PUT32(dp, VMXNET3_REG_CMD, VMXNET3_CMD_UPDATE_RX_MODE);
}

/*
 * Fetch the link state of a vmxnet3 device.
 */
static void
vmxnet3_refresh_linkstate(vmxnet3_softc_t *dp)
{
        uint32_t ret32;

        VMXNET3_BAR1_PUT32(dp, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK);
        ret32 = VMXNET3_BAR1_GET32(dp, VMXNET3_REG_CMD);
        if (ret32 & 1) {
                dp->linkState = LINK_STATE_UP;
                dp->linkSpeed = (ret32 >> 16) * 1000000ULL;
        } else {
                dp->linkState = LINK_STATE_DOWN;
                dp->linkSpeed = 0;
        }
}

/*
 * Start a vmxnet3 device: allocate and initialize the shared data
 * structures and send a start command to the device.
 *
 * Returns:
 *      0 on success, non-zero error on failure.
 */
static int
vmxnet3_start(void *data)
{
        vmxnet3_softc_t *dp = data;
        Vmxnet3_TxQueueDesc *tqdesc;
        Vmxnet3_RxQueueDesc *rqdesc;
        int txQueueSize, rxQueueSize;
        uint32_t ret32;
        int err, dmaerr;

        VMXNET3_DEBUG(dp, 1, "start()\n");

        /*
         * Allocate vmxnet3's shared data and advertise its PA
         */
        if ((err = vmxnet3_prepare_drivershared(dp)) != 0) {
                VMXNET3_WARN(dp, "vmxnet3_prepare_drivershared() failed: %d",
                    err);
                goto error;
        }
        tqdesc = VMXNET3_TQDESC(dp);
        rqdesc = VMXNET3_RQDESC(dp);

        /*
         * Create and initialize the tx queue
         */
        txQueueSize = vmxnet3_getprop(dp, "TxRingSize", 32, 4096,
            VMXNET3_DEF_TX_RING_SIZE);
        if (!(txQueueSize & VMXNET3_RING_SIZE_MASK)) {
                dp->txQueue.cmdRing.size = txQueueSize;
                dp->txQueue.compRing.size = txQueueSize;
                dp->txQueue.sharedCtrl = &tqdesc->ctrl;
                if ((err = vmxnet3_prepare_txqueue(dp)) != 0) {
                        VMXNET3_WARN(dp, "vmxnet3_prepare_txqueue() failed: %d",
                            err);
                        goto error_shared_data;
                }
        } else {
                VMXNET3_WARN(dp, "invalid tx ring size (%d)\n", txQueueSize);
                err = EINVAL;
                goto error_shared_data;
        }

        /*
         * Create and initialize the rx queue
         */
        rxQueueSize = vmxnet3_getprop(dp, "RxRingSize", 32, 4096,
            VMXNET3_DEF_RX_RING_SIZE);
        if (!(rxQueueSize & VMXNET3_RING_SIZE_MASK)) {
                dp->rxQueue.cmdRing.size = rxQueueSize;
                dp->rxQueue.compRing.size = rxQueueSize;
                dp->rxQueue.sharedCtrl = &rqdesc->ctrl;
                if ((err = vmxnet3_prepare_rxqueue(dp)) != 0) {
                        VMXNET3_WARN(dp, "vmxnet3_prepare_rxqueue() failed: %d",
                            err);
                        goto error_tx_queue;
                }
        } else {
                VMXNET3_WARN(dp, "invalid rx ring size (%d)\n", rxQueueSize);
                err = EINVAL;
                goto error_tx_queue;
        }

        /*
         * Allocate the Tx DMA handle
         */
        if ((dmaerr = ddi_dma_alloc_handle(dp->dip, &vmxnet3_dma_attrs_tx,
            DDI_DMA_SLEEP, NULL, &dp->txDmaHandle)) != DDI_SUCCESS) {
                VMXNET3_WARN(dp, "ddi_dma_alloc_handle() failed: %d", dmaerr);
                err = vmxnet3_dmaerr2errno(dmaerr);
                goto error_rx_queue;
        }

        /*
         * Activate the device
         */
        VMXNET3_BAR1_PUT32(dp, VMXNET3_REG_CMD, VMXNET3_CMD_ACTIVATE_DEV);
        ret32 = VMXNET3_BAR1_GET32(dp, VMXNET3_REG_CMD);
        if (ret32) {
                VMXNET3_WARN(dp, "ACTIVATE_DEV failed: 0x%x\n", ret32);
                err = ENXIO;
                goto error_txhandle;
        }
        dp->devEnabled = B_TRUE;

        VMXNET3_BAR0_PUT32(dp, VMXNET3_REG_RXPROD,
            dp->txQueue.cmdRing.size - 1);

        /*
         * Update the RX filters, must be done after ACTIVATE_DEV
         */
        dp->rxMode = VMXNET3_RXM_UCAST | VMXNET3_RXM_BCAST;
        vmxnet3_refresh_rxfilter(dp);

        /*
         * Get the link state now because no events will be generated
         */
        vmxnet3_refresh_linkstate(dp);
        mac_link_update(dp->mac, dp->linkState);

        /*
         * Finally, unmask the interrupt
         */
        VMXNET3_BAR0_PUT32(dp, VMXNET3_REG_IMR, 0);

        return (0);

error_txhandle:
        ddi_dma_free_handle(&dp->txDmaHandle);
error_rx_queue:
        vmxnet3_destroy_rxqueue(dp);
error_tx_queue:
        vmxnet3_destroy_txqueue(dp);
error_shared_data:
        vmxnet3_destroy_drivershared(dp);
error:
        return (err);
}

/*
 * Stop a vmxnet3 device: send a stop command to the device and
 * de-allocate the shared data structures.
 */
static void
vmxnet3_stop(void *data)
{
        vmxnet3_softc_t *dp = data;

        VMXNET3_DEBUG(dp, 1, "stop()\n");

        /*
         * Take the 2 locks related to asynchronous events.
         * These events should always check dp->devEnabled before poking dp.
         */
        mutex_enter(&dp->intrLock);
        mutex_enter(&dp->rxPoolLock);
        VMXNET3_BAR0_PUT32(dp, VMXNET3_REG_IMR, 1);
        dp->devEnabled = B_FALSE;
        VMXNET3_BAR1_PUT32(dp, VMXNET3_REG_CMD, VMXNET3_CMD_QUIESCE_DEV);
        mutex_exit(&dp->rxPoolLock);
        mutex_exit(&dp->intrLock);

        ddi_dma_free_handle(&dp->txDmaHandle);

        vmxnet3_destroy_rxqueue(dp);
        vmxnet3_destroy_txqueue(dp);

        vmxnet3_destroy_drivershared(dp);
}

/*
 * Set or unset promiscuous mode on a vmxnet3 device.
 */
static int
vmxnet3_setpromisc(void *data, boolean_t promisc)
{
        vmxnet3_softc_t *dp = data;

        VMXNET3_DEBUG(dp, 2, "setpromisc(%s)\n", promisc ? "TRUE" : "FALSE");

        if (promisc) {
                dp->rxMode |= VMXNET3_RXM_PROMISC;
        } else {
                dp->rxMode &= ~VMXNET3_RXM_PROMISC;
        }

        vmxnet3_refresh_rxfilter(dp);

        return (0);
}

/*
 * Add or remove a multicast address from/to a vmxnet3 device.
 *
 * Returns:
 *      0 on success, non-zero on failure.
 */
static int
vmxnet3_multicst(void *data, boolean_t add, const uint8_t *macaddr)
{
        vmxnet3_softc_t *dp = data;
        vmxnet3_dmabuf_t newMfTable;
        int ret = 0;
        uint16_t macIdx;
        size_t allocSize;

        VMXNET3_DEBUG(dp, 2, "multicst(%s, "MACADDR_FMT")\n",
            add ? "add" : "remove", MACADDR_FMT_ARGS(macaddr));

        /*
         * First lookup the position of the given MAC to check if it is
         * present in the existing MF table.
         */
        for (macIdx = 0; macIdx < dp->mfTable.bufLen; macIdx += 6) {
                if (memcmp(&dp->mfTable.buf[macIdx], macaddr, 6) == 0) {
                        break;
                }
        }

        /*
         * Check for 2 situations we can handle gracefully by bailing out:
         * Adding an already existing filter or removing a non-existing one.
         */
        if (add && macIdx < dp->mfTable.bufLen) {
                VMXNET3_WARN(dp, MACADDR_FMT " already in MC filter list "
                    "@ %u\n", MACADDR_FMT_ARGS(macaddr), macIdx / 6);
                ASSERT(B_FALSE);
                goto done;
        }
        if (!add && macIdx == dp->mfTable.bufLen) {
                VMXNET3_WARN(dp, MACADDR_FMT " not in MC filter list @ %u\n",
                    MACADDR_FMT_ARGS(macaddr), macIdx / 6);
                ASSERT(B_FALSE);
                goto done;
        }

        /*
         * Create the new MF table
         */
        allocSize = dp->mfTable.bufLen + (add ? 6 : -6);
        if (allocSize) {
                ret = vmxnet3_alloc_dma_mem_1(dp, &newMfTable, allocSize,
                    B_TRUE);
                ASSERT(ret == 0);
                if (add) {
                        (void) memcpy(newMfTable.buf, dp->mfTable.buf,
                            dp->mfTable.bufLen);
                        (void) memcpy(newMfTable.buf + dp->mfTable.bufLen,
                            macaddr, 6);
                } else {
                        (void) memcpy(newMfTable.buf, dp->mfTable.buf,
                            macIdx);
                        (void) memcpy(newMfTable.buf + macIdx,
                            dp->mfTable.buf + macIdx + 6,
                            dp->mfTable.bufLen - macIdx - 6);
                }
        } else {
                newMfTable.buf = NULL;
                newMfTable.bufPA = 0;
                newMfTable.bufLen = 0;
        }

        /*
         * Now handle 2 corner cases: if we're creating the first filter or
         * removing the last one, we have to update rxMode accordingly.
         */
        if (add && newMfTable.bufLen == 6) {
                ASSERT(!(dp->rxMode & VMXNET3_RXM_MCAST));
                dp->rxMode |= VMXNET3_RXM_MCAST;
                vmxnet3_refresh_rxfilter(dp);
        }
        if (!add && dp->mfTable.bufLen == 6) {
                ASSERT(newMfTable.buf == NULL);
                ASSERT(dp->rxMode & VMXNET3_RXM_MCAST);
                dp->rxMode &= ~VMXNET3_RXM_MCAST;
                vmxnet3_refresh_rxfilter(dp);
        }

        /*
         * Now replace the old MF table with the new one
         */
        if (dp->mfTable.buf) {
                vmxnet3_free_dma_mem(&dp->mfTable);
        }
        dp->mfTable = newMfTable;
        VMXNET3_DS(dp)->devRead.rxFilterConf.mfTablePA = newMfTable.bufPA;
        VMXNET3_DS(dp)->devRead.rxFilterConf.mfTableLen = newMfTable.bufLen;

done:
        /* Always update the filters */
        VMXNET3_BAR1_PUT32(dp, VMXNET3_REG_CMD, VMXNET3_CMD_UPDATE_MAC_FILTERS);

        return (ret);
}

/*
 * Set the mac address of a vmxnet3 device.
 *
 * Returns:
 *      0
 */
static int
vmxnet3_unicst(void *data, const uint8_t *macaddr)
{
        vmxnet3_softc_t *dp = data;
        uint32_t val32;

        VMXNET3_DEBUG(dp, 2, "unicst("MACADDR_FMT")\n",
            MACADDR_FMT_ARGS(macaddr));

        val32 = *((uint32_t *)(macaddr + 0));
        VMXNET3_BAR1_PUT32(dp, VMXNET3_REG_MACL, val32);
        val32 = *((uint16_t *)(macaddr + 4));
        VMXNET3_BAR1_PUT32(dp, VMXNET3_REG_MACH, val32);

        (void) memcpy(dp->macaddr, macaddr, 6);

        return (0);
}

/*
 * Change the MTU as seen by the driver. This is only supported when
 * the mac is stopped.
 *
 * Returns:
 *      EBUSY if the device is enabled.
 *      EINVAL for invalid MTU values.
 *      0 on success.
 */
static int
vmxnet3_change_mtu(vmxnet3_softc_t *dp, uint32_t new_mtu)
{
        int ret;

        if (dp->devEnabled)
                return (EBUSY);

        if (new_mtu == dp->cur_mtu) {
                VMXNET3_WARN(dp, "New MTU is same as old mtu : %d.\n", new_mtu);
                return (0);
        }

        if (new_mtu < VMXNET3_MIN_MTU || new_mtu > VMXNET3_MAX_MTU) {
                VMXNET3_WARN(dp, "New MTU not in valid range [%d, %d].\n",
                    VMXNET3_MIN_MTU, VMXNET3_MAX_MTU);
                return (EINVAL);
        } else if (new_mtu > ETHERMTU && !dp->allow_jumbo) {
                VMXNET3_WARN(dp, "MTU cannot be greater than %d because "
                    "accept-jumbo is not enabled.\n", ETHERMTU);
                return (EINVAL);
        }

        dp->cur_mtu = new_mtu;
        dp->alloc_ok = VMXNET3_ALLOC_OK(dp);

        if ((ret = mac_maxsdu_update(dp->mac, new_mtu)) != 0)
                VMXNET3_WARN(dp, "Unable to update mac with %d mtu: %d",
                    new_mtu, ret);

        return (ret);
}

/* ARGSUSED */
static int
vmxnet3_get_prop(void *data, const char *prop_name, mac_prop_id_t prop_id,
    uint_t prop_val_size, void *prop_val)
{
        vmxnet3_softc_t *dp = data;
        int ret = 0;

        switch (prop_id) {
        case MAC_PROP_MTU:
                ASSERT(prop_val_size >= sizeof (uint32_t));
                bcopy(&dp->cur_mtu, prop_val, sizeof (uint32_t));
                break;
        default:
                VMXNET3_WARN(dp, "vmxnet3_get_prop property %d not supported",
                    prop_id);
                ret = ENOTSUP;
        }
        return (ret);
}

/* ARGSUSED */
static int
vmxnet3_set_prop(void *data, const char *prop_name, mac_prop_id_t prop_id,
    uint_t prop_val_size, const void *prop_val)
{
        vmxnet3_softc_t *dp = data;
        int ret;

        switch (prop_id) {
        case MAC_PROP_MTU: {
                uint32_t new_mtu;
                ASSERT(prop_val_size >= sizeof (uint32_t));
                bcopy(prop_val, &new_mtu, sizeof (new_mtu));
                ret = vmxnet3_change_mtu(dp, new_mtu);
                break;
        }
        default:
                VMXNET3_WARN(dp, "vmxnet3_set_prop property %d not supported",
                    prop_id);
                ret = ENOTSUP;
        }

        return (ret);
}

/* ARGSUSED */
static void
vmxnet3_prop_info(void *data, const char *prop_name, mac_prop_id_t prop_id,
    mac_prop_info_handle_t prop_handle)
{
        vmxnet3_softc_t *dp = data;

        switch (prop_id) {
        case MAC_PROP_MTU:
                mac_prop_info_set_range_uint32(prop_handle, VMXNET3_MIN_MTU,
                    VMXNET3_MAX_MTU);
                break;
        default:
                VMXNET3_WARN(dp, "vmxnet3_prop_info: property %d not supported",
                    prop_id);
        }
}

/*
 * DDI/DDK callback to handle IOCTL in driver. Currently it only handles
 * ND_SET ioctl. Rest all are ignored. The ND_SET is used to set/reset
 * accept-jumbo ndd parameted for the interface.
 *
 * Side effects:
 *      MTU can be changed and device can be reset. An ACK or NACK is conveyed
 *      to the calling function from the mblk which was used to call this
 *      function.
 */
static void
vmxnet3_ioctl(void *arg, queue_t *wq, mblk_t *mp)
{
        vmxnet3_softc_t *dp = arg;
        int ret = EINVAL;
        IOCP iocp;
        mblk_t *mp1;
        char *valp, *param;
        int data;

        iocp = (void *)mp->b_rptr;
        iocp->ioc_error = 0;

        switch (iocp->ioc_cmd) {
        case ND_SET:
                /*
                 * The mblk in continuation would contain the ndd parameter name
                 * and data value to be set
                 */
                mp1 = mp->b_cont;
                if (!mp1) {
                        VMXNET3_WARN(dp, "Error locating parameter name.\n");
                        ret = EINVAL;
                        break;
                }

                /* Force null termination */
                mp1->b_datap->db_lim[-1] = '\0';

                /*
                 * From /usr/src/uts/common/inet/nd.c : nd_getset()
                 * "logic throughout nd_xxx assumes single data block for ioctl.
                 *  However, existing code sends in some big buffers."
                 */
                if (mp1->b_cont) {
                        freemsg(mp1->b_cont);
                        mp1->b_cont = NULL;
                }

                valp = (char *)mp1->b_rptr;     /* Points to param name */
                ASSERT(valp);
                param = valp;
                VMXNET3_DEBUG(dp, 3, "ND Set ioctl for %s\n", param);

                /*
                 * Go past the end of this null terminated string to get the
                 * data value.
                 */
                while (*valp && valp <= (char *)mp1->b_wptr)
                        valp++;

                if (valp > (char *)mp1->b_wptr) {
                        /*
                         * We are already beyond the readable area of mblk and
                         * still haven't found the end of param string.
                         */
                        VMXNET3_WARN(dp,
                            "No data value found to be set to param\n");
                        data = -1;
                } else {
                        /* Now this points to data string */
                        valp++;
                        /* Get numeric value of first letter */
                        data = (int)*valp - (int)'0';
                }

                if (strcmp("accept-jumbo", param) == 0) {
                        if (data == 1) {
                                VMXNET3_DEBUG(dp, 2,
                                    "Accepting jumbo frames\n");
                                dp->allow_jumbo = B_TRUE;
                                ret = vmxnet3_change_mtu(dp, VMXNET3_MAX_MTU);
                        } else if (data == 0) {
                                VMXNET3_DEBUG(dp, 2,
                                    "Rejecting jumbo frames\n");
                                dp->allow_jumbo = B_FALSE;
                                ret = vmxnet3_change_mtu(dp, ETHERMTU);
                        } else {
                                VMXNET3_WARN(dp, "Invalid data value to be set,"
                                    " use 0 or 1\n");
                                ret = -1;
                        }
                }
                freemsg(mp1);
                mp->b_cont = NULL;
                break;

        default:
                if (mp->b_cont) {
                        freemsg(mp->b_cont);
                        mp->b_cont = NULL;
                }
                ret = -1;
                break;
        }

        if (ret == 0)
                miocack(wq, mp, 0, 0);
        else
                miocnak(wq, mp, 0, EINVAL);
}

/*
 * Get the capabilities of a vmxnet3 device.
 *
 * Returns:
 *      B_TRUE if the capability is supported, B_FALSE otherwise.
 */
static boolean_t
vmxnet3_getcapab(void *data, mac_capab_t capab, void *arg)
{
        vmxnet3_softc_t *dp = data;
        boolean_t ret;

        switch (capab) {
        case MAC_CAPAB_HCKSUM: {
                uint32_t *txflags = arg;
                *txflags = HCKSUM_INET_PARTIAL;
                ret = B_TRUE;
                break;
        }
        case MAC_CAPAB_LSO: {
                mac_capab_lso_t *lso = arg;
                lso->lso_flags = LSO_TX_BASIC_TCP_IPV4;
                lso->lso_basic_tcp_ipv4.lso_max = IP_MAXPACKET;
                ret = vmxnet3_getprop(dp, "EnableLSO", 0, 1, 1);
                break;
        }
        default:
                ret = B_FALSE;
        }

        VMXNET3_DEBUG(dp, 2, "getcapab(0x%x) -> %s\n", capab,
            ret ? "yes" : "no");

        return (ret);
}

/*
 * Reset a vmxnet3 device. Only to be used when the device is wedged.
 *
 * Side effects:
 *      The device is reset.
 */
static void
vmxnet3_reset(void *data)
{
        int ret;

        vmxnet3_softc_t *dp = data;

        VMXNET3_DEBUG(dp, 1, "vmxnet3_reset()\n");

        atomic_inc_32(&dp->reset_count);
        vmxnet3_stop(dp);
        VMXNET3_BAR1_PUT32(dp, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV);
        if ((ret = vmxnet3_start(dp)) != 0)
                VMXNET3_WARN(dp, "failed to reset the device: %d", ret);
}

/*
 * Process pending events on a vmxnet3 device.
 *
 * Returns:
 *      B_TRUE if the link state changed, B_FALSE otherwise.
 */
static boolean_t
vmxnet3_intr_events(vmxnet3_softc_t *dp)
{
        Vmxnet3_DriverShared *ds = VMXNET3_DS(dp);
        boolean_t linkStateChanged = B_FALSE;
        uint32_t events = ds->ecr;

        if (events) {
                VMXNET3_DEBUG(dp, 2, "events(0x%x)\n", events);
                if (events & (VMXNET3_ECR_RQERR | VMXNET3_ECR_TQERR)) {
                        Vmxnet3_TxQueueDesc *tqdesc = VMXNET3_TQDESC(dp);
                        Vmxnet3_RxQueueDesc *rqdesc = VMXNET3_RQDESC(dp);

                        VMXNET3_BAR1_PUT32(dp, VMXNET3_REG_CMD,
                            VMXNET3_CMD_GET_QUEUE_STATUS);
                        if (tqdesc->status.stopped) {
                                VMXNET3_WARN(dp, "tq error 0x%x\n",
                                    tqdesc->status.error);
                        }
                        if (rqdesc->status.stopped) {
                                VMXNET3_WARN(dp, "rq error 0x%x\n",
                                    rqdesc->status.error);
                        }

                        if (ddi_taskq_dispatch(dp->resetTask, vmxnet3_reset,
                            dp, DDI_NOSLEEP) == DDI_SUCCESS) {
                                VMXNET3_WARN(dp, "reset scheduled\n");
                        } else {
                                VMXNET3_WARN(dp,
                                    "ddi_taskq_dispatch() failed\n");
                        }
                }
                if (events & VMXNET3_ECR_LINK) {
                        vmxnet3_refresh_linkstate(dp);
                        linkStateChanged = B_TRUE;
                }
                if (events & VMXNET3_ECR_DIC) {
                        VMXNET3_DEBUG(dp, 1, "device implementation change\n");
                }
                VMXNET3_BAR1_PUT32(dp, VMXNET3_REG_ECR, events);
        }

        return (linkStateChanged);
}

/*
 * Interrupt handler of a vmxnet3 device.
 *
 * Returns:
 *      DDI_INTR_CLAIMED or DDI_INTR_UNCLAIMED.
 */
/* ARGSUSED1 */
static uint_t
vmxnet3_intr(caddr_t data1, caddr_t data2)
{
        vmxnet3_softc_t *dp = (void *) data1;

        VMXNET3_DEBUG(dp, 3, "intr()\n");

        mutex_enter(&dp->intrLock);

        if (dp->devEnabled) {
                boolean_t linkStateChanged;
                boolean_t mustUpdateTx;
                mblk_t *mps;

                if (dp->intrType == DDI_INTR_TYPE_FIXED &&
                    !VMXNET3_BAR1_GET32(dp, VMXNET3_REG_ICR)) {
                        goto intr_unclaimed;
                }

                if (dp->intrMaskMode == VMXNET3_IMM_ACTIVE) {
                        VMXNET3_BAR0_PUT32(dp, VMXNET3_REG_IMR, 1);
                }

                linkStateChanged = vmxnet3_intr_events(dp);
                mustUpdateTx = vmxnet3_tx_complete(dp, &dp->txQueue);
                mps = vmxnet3_rx_intr(dp, &dp->rxQueue);

                mutex_exit(&dp->intrLock);
                VMXNET3_BAR0_PUT32(dp, VMXNET3_REG_IMR, 0);

                if (linkStateChanged) {
                        mac_link_update(dp->mac, dp->linkState);
                }
                if (mustUpdateTx) {
                        mac_tx_update(dp->mac);
                }
                if (mps) {
                        mac_rx(dp->mac, NULL, mps);
                }

                return (DDI_INTR_CLAIMED);
        }

intr_unclaimed:
        mutex_exit(&dp->intrLock);
        return (DDI_INTR_UNCLAIMED);
}

static int
vmxnet3_kstat_update(kstat_t *ksp, int rw)
{
        vmxnet3_softc_t *dp = ksp->ks_private;
        vmxnet3_kstats_t *statp = ksp->ks_data;

        if (rw == KSTAT_WRITE)
                return (EACCES);

        statp->reset_count.value.ul = dp->reset_count;
        statp->tx_pullup_needed.value.ul = dp->tx_pullup_needed;
        statp->tx_ring_full.value.ul = dp->tx_ring_full;
        statp->rx_alloc_buf.value.ul = dp->rx_alloc_buf;
        statp->rx_pool_empty.value.ul = dp->rx_pool_empty;
        statp->rx_num_bufs.value.ul = dp->rx_num_bufs;

        return (0);
}

static int
vmxnet3_kstat_init(vmxnet3_softc_t *dp)
{
        vmxnet3_kstats_t *statp;

        dp->devKstats = kstat_create(VMXNET3_MODNAME, dp->instance,
            "statistics", "dev",  KSTAT_TYPE_NAMED,
            sizeof (vmxnet3_kstats_t) / sizeof (kstat_named_t), 0);
        if (dp->devKstats == NULL)
                return (DDI_FAILURE);

        dp->devKstats->ks_update = vmxnet3_kstat_update;
        dp->devKstats->ks_private = dp;

        statp = dp->devKstats->ks_data;

        kstat_named_init(&statp->reset_count, "reset_count", KSTAT_DATA_ULONG);
        kstat_named_init(&statp->tx_pullup_needed, "tx_pullup_needed",
            KSTAT_DATA_ULONG);
        kstat_named_init(&statp->tx_ring_full, "tx_ring_full",
            KSTAT_DATA_ULONG);
        kstat_named_init(&statp->rx_alloc_buf, "rx_alloc_buf",
            KSTAT_DATA_ULONG);
        kstat_named_init(&statp->rx_pool_empty, "rx_pool_empty",
            KSTAT_DATA_ULONG);
        kstat_named_init(&statp->rx_num_bufs, "rx_num_bufs",
            KSTAT_DATA_ULONG);

        kstat_install(dp->devKstats);

        return (DDI_SUCCESS);
}

/*
 * Probe and attach a vmxnet3 instance to the stack.
 *
 * Returns:
 *      DDI_SUCCESS or DDI_FAILURE.
 */
static int
vmxnet3_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
{
        vmxnet3_softc_t *dp;
        mac_register_t *macr;
        uint16_t vendorId, devId, ret16;
        uint32_t ret32;
        int ret, err;
        uint_t uret;

        if (cmd != DDI_ATTACH) {
                goto error;
        }

        /*
         * Allocate the soft state
         */
        dp = kmem_zalloc(sizeof (vmxnet3_softc_t), KM_SLEEP);
        ASSERT(dp);

        dp->dip = dip;
        dp->instance = ddi_get_instance(dip);
        dp->cur_mtu = ETHERMTU;
        dp->allow_jumbo = B_TRUE;
        dp->alloc_ok = VMXNET3_ALLOC_OK(dp);

        VMXNET3_DEBUG(dp, 1, "attach()\n");

        ddi_set_driver_private(dip, dp);

        /*
         * Get access to the PCI bus configuration space
         */
        if (pci_config_setup(dip, &dp->pciHandle) != DDI_SUCCESS) {
                VMXNET3_WARN(dp, "pci_config_setup() failed\n");
                goto error_soft_state;
        }

        /*
         * Make sure the chip is a vmxnet3 device
         */
        vendorId = pci_config_get16(dp->pciHandle, PCI_CONF_VENID);
        devId = pci_config_get16(dp->pciHandle, PCI_CONF_DEVID);
        if (vendorId != PCI_VENDOR_ID_VMWARE ||
            devId != PCI_DEVICE_ID_VMWARE_VMXNET3) {
                VMXNET3_WARN(dp, "wrong PCI venid/devid (0x%x, 0x%x)\n",
                    vendorId, devId);
                goto error_pci_config;
        }

        /*
         * Make sure we can access the registers through the I/O space
         */
        ret16 = pci_config_get16(dp->pciHandle, PCI_CONF_COMM);
        ret16 |= PCI_COMM_IO | PCI_COMM_ME;
        pci_config_put16(dp->pciHandle, PCI_CONF_COMM, ret16);

        /*
         * Map the I/O space in memory
         */
        if (ddi_regs_map_setup(dip, 1, &dp->bar0, 0, 0, &vmxnet3_dev_attr,
            &dp->bar0Handle) != DDI_SUCCESS) {
                VMXNET3_WARN(dp, "ddi_regs_map_setup() for BAR0 failed\n");
                goto error_pci_config;
        }

        if (ddi_regs_map_setup(dip, 2, &dp->bar1, 0, 0, &vmxnet3_dev_attr,
            &dp->bar1Handle) != DDI_SUCCESS) {
                VMXNET3_WARN(dp, "ddi_regs_map_setup() for BAR1 failed\n");
                goto error_regs_map_0;
        }

        /*
         * Check the version number of the virtual device
         */
        if (VMXNET3_BAR1_GET32(dp, VMXNET3_REG_VRRS) & 1) {
                VMXNET3_BAR1_PUT32(dp, VMXNET3_REG_VRRS, 1);
        } else {
                VMXNET3_WARN(dp, "incompatible h/w version\n");
                goto error_regs_map_1;
        }

        if (VMXNET3_BAR1_GET32(dp, VMXNET3_REG_UVRS) & 1) {
                VMXNET3_BAR1_PUT32(dp, VMXNET3_REG_UVRS, 1);
        } else {
                VMXNET3_WARN(dp, "incompatible upt version\n");
                goto error_regs_map_1;
        }

        if (vmxnet3_kstat_init(dp) != DDI_SUCCESS) {
                VMXNET3_WARN(dp, "unable to initialize kstats");
                goto error_regs_map_1;
        }

        /*
         * Read the MAC address from the device
         */
        ret32 = VMXNET3_BAR1_GET32(dp, VMXNET3_REG_MACL);
        *((uint32_t *)(dp->macaddr + 0)) = ret32;
        ret32 = VMXNET3_BAR1_GET32(dp, VMXNET3_REG_MACH);
        *((uint16_t *)(dp->macaddr + 4)) = ret32;

        /*
         * Register with the MAC framework
         */
        if (!(macr = mac_alloc(MAC_VERSION))) {
                VMXNET3_WARN(dp, "mac_alloc() failed\n");
                goto error_kstat;
        }

        macr->m_type_ident = MAC_PLUGIN_IDENT_ETHER;
        macr->m_driver = dp;
        macr->m_dip = dip;
        macr->m_instance = 0;
        macr->m_src_addr = dp->macaddr;
        macr->m_dst_addr = NULL;
        macr->m_callbacks = &vmxnet3_mac_callbacks;
        macr->m_min_sdu = 0;
        macr->m_max_sdu = ETHERMTU;
        macr->m_margin = VLAN_TAGSZ;
        macr->m_pdata = NULL;
        macr->m_pdata_size = 0;

        ret = mac_register(macr, &dp->mac);
        mac_free(macr);
        if (ret != DDI_SUCCESS) {
                VMXNET3_WARN(dp, "mac_register() failed\n");
                goto error_kstat;
        }

        /*
         * Register the interrupt(s) in this order of preference:
         * MSI-X, MSI, INTx
         */
        VMXNET3_BAR1_PUT32(dp, VMXNET3_REG_CMD, VMXNET3_CMD_GET_CONF_INTR);
        ret32 = VMXNET3_BAR1_GET32(dp, VMXNET3_REG_CMD);
        switch (ret32 & 0x3) {
        case VMXNET3_IT_AUTO:
        case VMXNET3_IT_MSIX:
                dp->intrType = DDI_INTR_TYPE_MSIX;
                err = ddi_intr_alloc(dip, &dp->intrHandle, dp->intrType, 0, 1,
                    &ret, DDI_INTR_ALLOC_STRICT);
                if (err == DDI_SUCCESS)
                        break;
                VMXNET3_DEBUG(dp, 2, "DDI_INTR_TYPE_MSIX failed, err:%d\n",
                    err);
                /* FALLTHROUGH */
        case VMXNET3_IT_MSI:
                dp->intrType = DDI_INTR_TYPE_MSI;
                if (ddi_intr_alloc(dip, &dp->intrHandle, dp->intrType, 0, 1,
                    &ret, DDI_INTR_ALLOC_STRICT) == DDI_SUCCESS)
                        break;
                VMXNET3_DEBUG(dp, 2, "DDI_INTR_TYPE_MSI failed\n");
                /* FALLTHROUGH */
        case VMXNET3_IT_INTX:
                dp->intrType = DDI_INTR_TYPE_FIXED;
                if (ddi_intr_alloc(dip, &dp->intrHandle, dp->intrType, 0, 1,
                    &ret, DDI_INTR_ALLOC_STRICT) == DDI_SUCCESS) {
                        break;
                }
                VMXNET3_DEBUG(dp, 2, "DDI_INTR_TYPE_INTX failed\n");
                /* FALLTHROUGH */
        default:
                VMXNET3_WARN(dp, "ddi_intr_alloc() failed\n");
                goto error_mac;
        }
        dp->intrMaskMode = (ret32 >> 2) & 0x3;
        if (dp->intrMaskMode == VMXNET3_IMM_LAZY) {
                VMXNET3_WARN(dp, "Lazy masking is not supported\n");
                goto error_intr;
        }

        if (ddi_intr_get_pri(dp->intrHandle, &uret) != DDI_SUCCESS) {
                VMXNET3_WARN(dp, "ddi_intr_get_pri() failed\n");
                goto error_intr;
        }

        VMXNET3_DEBUG(dp, 2, "intrType=0x%x, intrMaskMode=0x%x, intrPrio=%u\n",
            dp->intrType, dp->intrMaskMode, uret);

        /*
         * Create a task queue to reset the device if it wedges.
         */
        dp->resetTask = ddi_taskq_create(dip, "vmxnet3_reset_task", 1,
            TASKQ_DEFAULTPRI, 0);
        if (!dp->resetTask) {
                VMXNET3_WARN(dp, "ddi_taskq_create() failed()\n");
                goto error_intr;
        }

        /*
         * Initialize our mutexes now that we know the interrupt priority
         * This _must_ be done before ddi_intr_enable()
         */
        mutex_init(&dp->intrLock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(uret));
        mutex_init(&dp->txLock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(uret));
        mutex_init(&dp->rxPoolLock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(uret));

        if (ddi_intr_add_handler(dp->intrHandle, vmxnet3_intr,
            dp, NULL) != DDI_SUCCESS) {
                VMXNET3_WARN(dp, "ddi_intr_add_handler() failed\n");
                goto error_mutexes;
        }

        err = ddi_intr_get_cap(dp->intrHandle, &dp->intrCap);
        if (err != DDI_SUCCESS) {
                VMXNET3_WARN(dp, "ddi_intr_get_cap() failed %d", err);
                goto error_intr_handler;
        }

        if (dp->intrCap & DDI_INTR_FLAG_BLOCK) {
                err = ddi_intr_block_enable(&dp->intrHandle, 1);
                if (err != DDI_SUCCESS) {
                        VMXNET3_WARN(dp, "ddi_intr_block_enable() failed, "
                            "err:%d\n", err);
                        goto error_intr_handler;
                }
        } else {
                err = ddi_intr_enable(dp->intrHandle);
                if ((err != DDI_SUCCESS)) {
                        VMXNET3_WARN(dp, "ddi_intr_enable() failed, err:%d\n",
                            err);
                        goto error_intr_handler;
                }
        }

        return (DDI_SUCCESS);

error_intr_handler:
        (void) ddi_intr_remove_handler(dp->intrHandle);
error_mutexes:
        mutex_destroy(&dp->rxPoolLock);
        mutex_destroy(&dp->txLock);
        mutex_destroy(&dp->intrLock);
        ddi_taskq_destroy(dp->resetTask);
error_intr:
        (void) ddi_intr_free(dp->intrHandle);
error_mac:
        (void) mac_unregister(dp->mac);
error_kstat:
        kstat_delete(dp->devKstats);
error_regs_map_1:
        ddi_regs_map_free(&dp->bar1Handle);
error_regs_map_0:
        ddi_regs_map_free(&dp->bar0Handle);
error_pci_config:
        pci_config_teardown(&dp->pciHandle);
error_soft_state:
        kmem_free(dp, sizeof (vmxnet3_softc_t));
error:
        return (DDI_FAILURE);
}

/*
 * Detach a vmxnet3 instance from the stack.
 *
 * Returns:
 *      DDI_SUCCESS or DDI_FAILURE.
 */
static int
vmxnet3_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
{
        vmxnet3_softc_t *dp = ddi_get_driver_private(dip);
        unsigned int retries = 0;
        int ret;

        VMXNET3_DEBUG(dp, 1, "detach()\n");

        if (cmd != DDI_DETACH) {
                return (DDI_FAILURE);
        }

        while (dp->rx_num_bufs > 0) {
                if (retries++ < 10) {
                        VMXNET3_WARN(dp, "rx pending (%u), waiting 1 second\n",
                            dp->rx_num_bufs);
                        delay(drv_usectohz(1000000));
                } else {
                        VMXNET3_WARN(dp, "giving up\n");
                        return (DDI_FAILURE);
                }
        }

        if (dp->intrCap & DDI_INTR_FLAG_BLOCK) {
                ret = ddi_intr_block_disable(&dp->intrHandle, 1);
        } else {
                ret = ddi_intr_disable(dp->intrHandle);
        }
        if (ret != DDI_SUCCESS) {
                VMXNET3_WARN(dp, "unable to disable interrupts");
                return (DDI_FAILURE);
        }
        if (ddi_intr_remove_handler(dp->intrHandle) != DDI_SUCCESS) {
                VMXNET3_WARN(dp, "unable to remove interrupt handler");
                return (DDI_FAILURE);
        }
        (void) ddi_intr_free(dp->intrHandle);

        VERIFY(mac_unregister(dp->mac) == 0);

        kstat_delete(dp->devKstats);

        if (dp->mfTable.buf) {
                vmxnet3_free_dma_mem(&dp->mfTable);
        }

        mutex_destroy(&dp->rxPoolLock);
        mutex_destroy(&dp->txLock);
        mutex_destroy(&dp->intrLock);
        ddi_taskq_destroy(dp->resetTask);

        ddi_regs_map_free(&dp->bar1Handle);
        ddi_regs_map_free(&dp->bar0Handle);
        pci_config_teardown(&dp->pciHandle);

        kmem_free(dp, sizeof (vmxnet3_softc_t));

        return (DDI_SUCCESS);
}

/*
 * Structures used by the module loader
 */

#define VMXNET3_IDENT "VMware Ethernet v3 " VMXNET3_DRIVER_VERSION_STRING

DDI_DEFINE_STREAM_OPS(
        vmxnet3_dev_ops,
        nulldev,
        nulldev,
        vmxnet3_attach,
        vmxnet3_detach,
        nodev,
        NULL,
        D_NEW | D_MP,
        NULL,
        ddi_quiesce_not_supported);

static struct modldrv vmxnet3_modldrv = {
        &mod_driverops,         /* drv_modops */
        VMXNET3_IDENT,          /* drv_linkinfo */
        &vmxnet3_dev_ops        /* drv_dev_ops */
};

static struct modlinkage vmxnet3_modlinkage = {
        MODREV_1,                       /* ml_rev */
        { &vmxnet3_modldrv, NULL }      /* ml_linkage */
};

/* Module load entry point */
int
_init(void)
{
        int ret;

        mac_init_ops(&vmxnet3_dev_ops, VMXNET3_MODNAME);
        ret = mod_install(&vmxnet3_modlinkage);
        if (ret != DDI_SUCCESS) {
                mac_fini_ops(&vmxnet3_dev_ops);
        }

        return (ret);
}

/* Module unload entry point */
int
_fini(void)
{
        int ret;

        ret = mod_remove(&vmxnet3_modlinkage);
        if (ret == DDI_SUCCESS) {
                mac_fini_ops(&vmxnet3_dev_ops);
        }

        return (ret);
}

/* Module info entry point */
int
_info(struct modinfo *modinfop)
{
        return (mod_info(&vmxnet3_modlinkage, modinfop));
}

void
vmxnet3_log(int level, vmxnet3_softc_t *dp, char *fmt, ...)
{
        dev_err(dp->dip, level, fmt);
}