Merge remote-tracking branch 'origin/master'

[unleashed/lotheac.git] / usr / src / uts / common / io / ib / adapters / hermon / hermon_fm.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
 */

/*
 * hermon_fm.c
 *    Hermon (InfiniBand) HCA Driver Fault Management Routines
 *
 * [Hermon FM Implementation]
 *
 * Hermon FM recovers the system from a HW error situation and/or isolates a
 * HW error by calling the FMA acc handle check functions. (calling
 * ddi_fm_acc_err_get()) If a HW error is detected when either
 * ddi_fm_acc_err_get() is called, to determine whether or not the error is
 * transient, the I/O operation causing the error will retry up to three times.
 *
 * (Basic HW error recovery)
 *
 *        |
 *  .---->*
 *  |     |
 *  |   issue an I/O request via PIO
 *  |     |
 *  |     |
 *  |   check acc handle
 *  |     |
 *  |     |
 *  `--< a HW error detected && retry count < 3 >
 *        |
 *        v
 *
 * When a HW error is detected, to provide the error information for users to
 * isolate the faulted HW, Hermon FM issues Solaris FMA ereports as follows.
 *
 *  * PIO transient error
 *         invalid_state => unaffected
 *
 *  * PIO persistent error
 *         invalid_state => lost
 *
 *  * PIO fatal error
 *         invalid_state => lost => panic
 *
 *  * Hermon HCA firmware error
 *         invalid_state => degraded
 *
 *  * Other Hermon HCA specific errors
 *         uncorrect => unaffected
 *              or
 *         correct => unaffected
 *
 * (Restrictions)
 *
 * The current implementation has the following restrictions.
 *  * No runtime check/protection
 *  * No detach time check/protection
 *  * No DMA check/protection
 *
 * See the Hermon FMA portfolio in detail.
 */

#include <sys/types.h>
#include <sys/conf.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/sysmacros.h>
#include <sys/list.h>
#include <sys/modhash.h>

#include <sys/ib/adapters/hermon/hermon.h>

/*
 * Hermon driver has to disable its FM functionality
 * if this "fm_capable" variable is defined or has a value
 * in /kernel/drv/hermon.conf.
 */
static char *fm_cap = "fm-capable";     /* FM capability */

static hermon_hca_fm_t hca_fm;          /* Hermon HCA FM Structure */

static void i_hca_fm_ereport(dev_info_t *, int, char *);
static void i_hca_fm_init(struct i_hca_fm *);
static void i_hca_fm_fini(struct i_hca_fm *);
static int i_hca_regs_map_setup(struct i_hca_fm *, dev_info_t *, uint_t,
    caddr_t *, offset_t, offset_t, ddi_device_acc_attr_t *, ddi_acc_handle_t *);
static void i_hca_regs_map_free(struct i_hca_fm *, ddi_acc_handle_t *);
static int i_hca_pci_config_setup(struct i_hca_fm *, dev_info_t *,
    ddi_acc_handle_t *);
static void i_hca_pci_config_teardown(struct i_hca_fm *, ddi_acc_handle_t *);
static int i_hca_pio_start(dev_info_t *, struct i_hca_acc_handle *,
    hermon_test_t *);
static int i_hca_pio_end(dev_info_t *, struct i_hca_acc_handle *, int *,
    hermon_test_t *);
static struct i_hca_acc_handle *i_hca_get_acc_handle(struct i_hca_fm *,
    ddi_acc_handle_t);

/* forward declaration for hermon_fm_{init, fini}() */
#ifdef FMA_TEST
static void i_hca_test_init(mod_hash_t **, mod_hash_t **);
static void i_hca_test_fini(mod_hash_t **, mod_hash_t **);
#endif /* FMA_TEST */

/*
 * Hermon FM Functions
 *
 * These functions are based on the HCA FM common interface
 * defined below, but specific to the Hermon HCA FM capabilities.
 */

/*
 *  void
 *  hermon_hca_fm_init(hermon_state_t *state, hermon_hca_fm_t *hca)
 *
 *  Overview
 *      hermon_hca_fm_init() initializes the Hermon FM resources.
 *
 *  Argument
 *      state: pointer to Hermon state structure
 *      hca: pointer to Hermon FM structure
 *
 *  Return value
 *      Nothing
 *
 *  Caller's context
 *      hermon_hca_fm_init() can be called in user or kernel context only.
 */
static void
hermon_hca_fm_init(hermon_state_t *state, hermon_hca_fm_t *hca_fm)
{
        state->hs_fm_hca_fm = hca_fm;
        i_hca_fm_init((struct i_hca_fm *)hca_fm);
}


/*
 *  void
 *  hermon_hca_fm_fini(hermon_state_t *state)
 *
 *  Overview
 *      hermon_hca_fm_fini() releases the Hermon FM resources.
 *
 *  Argument
 *      state: pointer to Hermon state structure
 *
 *  Return value
 *      Nothing
 *
 *  Caller's context
 *      hermon_hca_fm_fini() can be called in user or kernel context only.
 */
static void
hermon_hca_fm_fini(hermon_state_t *state)
{
        i_hca_fm_fini((struct i_hca_fm *)state->hs_fm_hca_fm);
        state->hs_fm_hca_fm = NULL;
}

/*
 *  void
 *  hermon_clr_state_nolock(hermon_state_t *state, int fm_state)
 *
 *  Overview
 *      hermon_clr_state() drops the specified state from Hermon FM state
 *      without the mutex locks.
 *
 *  Argument
 *      state: pointer to Hermon state structure
 *      fm_state: Hermon FM state, which is composed of:
 *              HCA_NO_FM       Hermom FM is not supported
 *              HCA_PIO_FM      PIO is fma-protected
 *              HCA_DMA_FM      DMA is fma-protected
 *              HCA_EREPORT_FM  FMA ereport is available
 *              HCA_ERRCB_FM    FMA error callback is supported
 *              HCA_ATTCH_FM    HCA FM attach mode
 *              HCA_RUNTM_FM    HCA FM runtime mode
 *
 *  Return value
 *      Nothing
 *
 *  Caller's context
 *      hermon_clr_state() can be called in user, kernel, interrupt context
 *      or high interrupt context.
 */
void
hermon_clr_state_nolock(hermon_state_t *state, int fm_state)
{
        extern void membar_sync(void);

        state->hs_fm_state &= ~fm_state;
        membar_sync();
}


/*
 *  void
 *  hermon_clr_state(hermon_state_t *state, int fm_state)
 *
 *  Overview
 *      hermon_clr_state() drops the specified state from Hermon FM state.
 *
 *  Argument
 *      state: pointer to Hermon state structure
 *      fm_state: Hermon FM state, which is composed of:
 *              HCA_NO_FM       Hermom FM is not supported
 *              HCA_PIO_FM      PIO is fma-protected
 *              HCA_DMA_FM      DMA is fma-protected
 *              HCA_EREPORT_FM  FMA ereport is available
 *              HCA_ERRCB_FM    FMA error callback is supported
 *              HCA_ATTCH_FM    HCA FM attach mode
 *              HCA_RUNTM_FM    HCA FM runtime mode
 *
 *  Return value
 *      Nothing
 *
 *  Caller's context
 *      hermon_clr_state() can be called in user, kernel or interrupt context.
 */
static void
hermon_clr_state(hermon_state_t *state, int fm_state)
{
        ASSERT(fm_state != HCA_NO_FM);

        mutex_enter(&state->hs_fm_lock);
        hermon_clr_state_nolock(state, fm_state);
        mutex_exit(&state->hs_fm_lock);
}


/*
 *  void
 *  hermon_set_state(hermon_state_t *state, int fm_state)
 *
 *  Overview
 *      hermon_set_state() sets Hermon FM state.
 *
 *  Argument
 *      state: pointer to Hermon state structure
 *      fm_state: Hermon FM state, which is composed of:
 *              HCA_NO_FM       Hermom FM is not supported
 *              HCA_PIO_FM      PIO is fma-protected
 *              HCA_DMA_FM      DMA is fma-protected
 *              HCA_EREPORT_FM  FMA ereport is available
 *              HCA_ERRCB_FM    FMA error callback is supported
 *              HCA_ATTCH_FM    HCA FM attach mode
 *              HCA_RUNTM_FM    HCA FM runtime mode
 *
 *  Return value
 *      Nothing
 *
 *  Caller's context
 *      hermon_set_state() can be called in user, kernel or interrupt context.
 */
static void
hermon_set_state(hermon_state_t *state, int fm_state)
{
        extern void membar_sync(void);

        mutex_enter(&state->hs_fm_lock);
        if (fm_state == HCA_NO_FM) {
                state->hs_fm_state = HCA_NO_FM;
        } else {
                state->hs_fm_state |= fm_state;
        }
        membar_sync();
        mutex_exit(&state->hs_fm_lock);
}


/*
 *  int
 *  hermon_get_state(hermon_state_t *state)
 *
 *  Overview
 *      hermon_get_state() returns the current Hermon FM state.
 *
 *  Argument
 *      state: pointer to Hermon state structure
 *
 *  Return value
 *      fm_state: Hermon FM state, which is composed of:
 *              HCA_NO_FM       Hermom FM is not supported
 *              HCA_PIO_FM      PIO is fma-protected
 *              HCA_DMA_FM      DMA is fma-protected
 *              HCA_EREPORT_FM  FMA ereport is available
 *              HCA_ERRCB_FM    FMA error callback is supported
 *              HCA_ATTCH_FM    HCA FM attach mode
 *              HCA_RUNTM_FM    HCA FM runtime mode
 *
 *  Caller's context
 *      hermon_get_state() can be called in user, kernel or interrupt context.
 */
int
hermon_get_state(hermon_state_t *state)
{
        return (state->hs_fm_state);
}


/*
 *  void
 *  hermon_fm_init(hermon_state_t *state)
 *
 *  Overview
 *      hermon_fm_init() is a Hermon FM initialization function which registers
 *      some FMA functions such as the ereport and the acc check capabilities
 *      for Hermon. If the "fm_disable" property in /kernel/drv/hermon.conf is
 *      defined (and/or its value is set), then the Hermon FM capabilities will
 *      drop, and only the default capabilities (the ereport and error callback
 *      capabilities) are available (and the action against HW errors is
 *      issuing an ereport then panicking the system).
 *
 *  Argument
 *      state: pointer to Hermon state structure
 *
 *  Return value
 *      Nothing
 *
 *  Caller's context
 *      hermon_fm_init() can be called in user or kernel context only.
 */
void
hermon_fm_init(hermon_state_t *state)
{
        ddi_iblock_cookie_t iblk;

        /*
         * Check the "fm_disable" property. If it's defined,
         * use the Solaris FMA default action for Hermon.
         */
        if (ddi_getprop(DDI_DEV_T_NONE, state->hs_dip, DDI_PROP_DONTPASS,
            "fm_disable", 0) != 0) {
                state->hs_fm_disable = 1;
        }

        /* If hs_fm_diable is set, then skip the rest */
        if (state->hs_fm_disable) {
                hermon_set_state(state, HCA_NO_FM);
                return;
        }

        /* Set the Hermon FM attach mode */
        hermon_set_state(state, HCA_ATTCH_FM);

        /* Initialize the Solaris FMA capabilities for the Hermon FM support */
        state->hs_fm_capabilities = ddi_prop_get_int(DDI_DEV_T_ANY,
            state->hs_dip, DDI_PROP_DONTPASS, fm_cap,
            DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE);

        /*
         * The Hermon FM uses the ereport and acc check capabilites only,
         * but both of them should be available. If either is not, turn
         * hs_fm_disable on and behave in the same way as the "fm_diable"
         * property is set.
         */
        if (state->hs_fm_capabilities !=
            (DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE)) {
                state->hs_fm_disable = 1;
                hermon_set_state(state, HCA_NO_FM);
                HERMON_ATTACH_MSG(state->hs_attach_buf,
                    "Hermon FM capability fails");
                return;
        }

        /* Initialize the HCA FM resources */
        hermon_hca_fm_init(state, &hca_fm);

        /* Initialize the fm state lock */
        mutex_init(&state->hs_fm_lock, NULL, MUTEX_DRIVER, NULL);

        /* Register the capabilities with the IO fault services */
        ddi_fm_init(state->hs_dip, &state->hs_fm_capabilities, &iblk);

        /* Set up the pci ereport capabilities if the ereport is capable */
        if (DDI_FM_EREPORT_CAP(state->hs_fm_capabilities)) {
                pci_ereport_setup(state->hs_dip);
        }

        /* Set the Hermon FM state */
        hermon_set_state(state, HCA_PIO_FM | HCA_EREPORT_FM);

#ifdef FMA_TEST
        i_hca_test_init(&state->hs_fm_test_hash, &state->hs_fm_id_hash);
#endif /* FMA_TEST */
}


/*
 *  void
 *  hermon_fm_fini(hermon_state_t *state)
 *
 *  Overview
 *      hermon_fm_fini() is a Hermon FM finalization function which de-registers
 *      Solaris FMA functions set to Hermon.
 *
 *  Argument
 *      state: pointer to Hermon state structure
 *
 *  Return value
 *      Nothing
 *
 *  Caller's context
 *      hermon_fm_fini() can be called in user or kernel context only.
 */
void
hermon_fm_fini(hermon_state_t *state)
{
        /*
         * If hermon_fm_diable is set or there is no FM service provided,
         * then skip the rest.
         */
        if (state->hs_fm_disable || hermon_get_state(state) == HCA_NO_FM) {
                return;
        }

        ASSERT(!(hermon_get_state(state) & HCA_ERRCB_FM));

#ifdef FMA_TEST
        i_hca_test_fini(&state->hs_fm_test_hash, &state->hs_fm_id_hash);
#endif /* FMA_TEST */

        /* Set the Hermon FM state to no support */
        hermon_set_state(state, HCA_NO_FM);

        /* Release HCA FM resources */
        hermon_hca_fm_fini(state);

        /*
         * Release any resources allocated by pci_ereport_setup()
         */
        if (DDI_FM_EREPORT_CAP(state->hs_fm_capabilities)) {
                pci_ereport_teardown(state->hs_dip);
        }

        /* De-register the Hermon FM from the IO fault services */
        ddi_fm_fini(state->hs_dip);
}


/*
 *  int
 *  hermon_fm_ereport_init(hermon_state_t *state)
 *
 *  Overview
 *      hermon_fm_ereport_init() changes the Hermon FM state to the ereport
 *      only mode during the driver attach.
 *
 *  Argument
 *      state: pointer to Hermon state structure
 *
 *  Return value
 *      DDI_SUCCESS
 *      DDI_FAILURE
 *
 *  Caller's context
 *      hermon_fm_ereport_init() can be called in user or kernel context only.
 */
int
hermon_fm_ereport_init(hermon_state_t *state)
{
        ddi_iblock_cookie_t iblk;
        hermon_cfg_profile_t *cfgprof;
        hermon_hw_querydevlim_t *devlim;
        hermon_rsrc_hw_entry_info_t entry_info;
        hermon_rsrc_pool_info_t *rsrc_pool;
        uint64_t offset, num, max, num_prealloc;
        ddi_device_acc_attr_t dev_attr = {
                DDI_DEVICE_ATTR_V0,
                DDI_STRUCTURE_LE_ACC,
                DDI_STRICTORDER_ACC,
                DDI_DEFAULT_ACC
        };
        char *rsrc_name;
        extern void membar_sync(void);

        /* Stop the poll thread while the FM state is being changed */
        state->hs_fm_poll_suspend = B_TRUE;
        membar_sync();

        /*
         * Disable the Hermon interrupt after the interrupt capability flag
         * is checked.
         */
        if (state->hs_intrmsi_cap & DDI_INTR_FLAG_BLOCK) {
                if (ddi_intr_block_disable
                    (&state->hs_intrmsi_hdl[0], 1) != DDI_SUCCESS) {
                        return (DDI_FAILURE);
                }
        } else {
                if (ddi_intr_disable
                    (state->hs_intrmsi_hdl[0]) != DDI_SUCCESS) {
                        return (DDI_FAILURE);
                }
        }

        /*
         * Release any resources allocated by pci_ereport_setup()
         */
        if (DDI_FM_EREPORT_CAP(state->hs_fm_capabilities)) {
                pci_ereport_teardown(state->hs_dip);
        }

        /* De-register the Hermon FM from the IO fault services */
        ddi_fm_fini(state->hs_dip);

        /* Re-initialize fm ereport with the ereport only */
        state->hs_fm_capabilities = ddi_prop_get_int(DDI_DEV_T_ANY,
            state->hs_dip, DDI_PROP_DONTPASS, fm_cap,
            DDI_FM_EREPORT_CAPABLE);

        /*
         * Now that the Hermon FM uses the ereport capability only,
         * If it's not set, turn hs_fm_disable on and behave in the
         * same way as the "fm_diable" property is set.
         */
        if (state->hs_fm_capabilities != DDI_FM_EREPORT_CAPABLE) {
                HERMON_ATTACH_MSG(state->hs_attach_buf,
                    "Hermon FM ereport fails (ereport mode)");
                goto error;
        }

        /* Re-register the ereport capability with the IO fault services */
        ddi_fm_init(state->hs_dip, &state->hs_fm_capabilities, &iblk);

        /* Initialize the pci ereport capabilities if the ereport is capable */
        if (DDI_FM_EREPORT_CAP(state->hs_fm_capabilities)) {
                pci_ereport_setup(state->hs_dip);
        }

        /* Setup for PCI config read/write of HCA device */
        if (pci_config_setup(state->hs_dip, &state->hs_reg_pcihdl) !=
            DDI_SUCCESS) {
                HERMON_ATTACH_MSG(state->hs_attach_buf,
                    "PCI config mapping fails (ereport mode)");
                goto error;
        }

        /* Allocate the regular access handle for MSI-X tables */
        if (ddi_regs_map_setup(state->hs_dip, state->hs_msix_tbl_rnumber,
            (caddr_t *)&state->hs_msix_tbl_addr, state->hs_msix_tbl_offset,
            state->hs_msix_tbl_size, &dev_attr,
            &state->hs_reg_msix_tblhdl) != DDI_SUCCESS) {
                HERMON_ATTACH_MSG(state->hs_attach_buf,
                    "MSI-X Table mapping fails (ereport mode)");
                goto error;
        }

        /* Allocate the regular access handle for MSI-X PBA */
        if (ddi_regs_map_setup(state->hs_dip, state->hs_msix_pba_rnumber,
            (caddr_t *)&state->hs_msix_pba_addr, state->hs_msix_pba_offset,
            state->hs_msix_pba_size, &dev_attr,
            &state->hs_reg_msix_pbahdl) != DDI_SUCCESS) {
                HERMON_ATTACH_MSG(state->hs_attach_buf,
                    "MSI-X PBA mapping fails (ereport mode)");
                goto error;
        }

        /* Allocate the regular access handle for Hermon CMD I/O space */
        if (ddi_regs_map_setup(state->hs_dip, HERMON_CMD_BAR,
            &state->hs_reg_cmd_baseaddr, 0, 0, &state->hs_reg_accattr,
            &state->hs_reg_cmdhdl) != DDI_SUCCESS) {
                HERMON_ATTACH_MSG(state->hs_attach_buf,
                    "CMD_BAR mapping fails (ereport mode)");
                goto error;
        }

        /* Reset the host command register */
        state->hs_cmd_regs.hcr = (hermon_hw_hcr_t *)
            ((uintptr_t)state->hs_reg_cmd_baseaddr + HERMON_CMD_HCR_OFFSET);

        /* Reset the software reset register */
        state->hs_cmd_regs.sw_reset = (uint32_t *)
            ((uintptr_t)state->hs_reg_cmd_baseaddr +
            HERMON_CMD_SW_RESET_OFFSET);

        /* Reset the software reset register semaphore */
        state->hs_cmd_regs.sw_semaphore = (uint32_t *)
            ((uintptr_t)state->hs_reg_cmd_baseaddr +
            HERMON_CMD_SW_SEMAPHORE_OFFSET);

        /* Calculate the clear interrupt register offset */
        offset = state->hs_fw.clr_intr_offs & HERMON_CMD_OFFSET_MASK;

        /* Reset the clear interrupt address */
        state->hs_cmd_regs.clr_intr = (uint64_t *)
            (uintptr_t)(state->hs_reg_cmd_baseaddr + offset);

        /* Reset the internal error buffer address */
        state->hs_cmd_regs.fw_err_buf = (uint32_t *)(uintptr_t)
            (state->hs_reg_cmd_baseaddr + state->hs_fw.error_buf_addr);

        /* Check if the blue flame is enabled, and set the offset value */
        if (state->hs_devlim.blu_flm) {
                offset = (uint64_t)1 <<
                    (state->hs_devlim.log_max_uar_sz + 20);
        } else {
                offset = 0;
        }

        /* Allocate the regular access handle for Hermon UAR I/O space */
        if (ddi_regs_map_setup(state->hs_dip, HERMON_UAR_BAR,
            &state->hs_reg_uar_baseaddr, 0, offset,
            &state->hs_reg_accattr, &state->hs_reg_uarhdl) != DDI_SUCCESS) {
                HERMON_ATTACH_MSG(state->hs_attach_buf,
                    "UAR BAR mapping fails (ereport mode)");
                goto error;
        }

        hermon_eq_reset_uar_baseaddr(state);

        /* Drop the Hermon FM Attach Mode */
        hermon_clr_state(state, HCA_ATTCH_FM);

        /* Set the Hermon FM Runtime Mode */
        hermon_set_state(state, HCA_RUNTM_FM);

        /* Free up Hermon UAR page #1 */
        hermon_rsrc_free(state, &state->hs_uarkpg_rsrc);

        /* Free up the UAR pool */
        entry_info.hwi_rsrcpool = &state->hs_rsrc_hdl[HERMON_UARPG];
        hermon_rsrc_hw_entries_fini(state, &entry_info);

        /* Re-allocate the UAR pool */
        cfgprof = state->hs_cfg_profile;
        devlim  = &state->hs_devlim;
        num                       = ((uint64_t)1 << cfgprof->cp_log_num_uar);
        max                       = num;
        num_prealloc              = max(devlim->num_rsvd_uar, 128);
        rsrc_pool                 = &state->hs_rsrc_hdl[HERMON_UARPG];
        rsrc_pool->rsrc_type      = HERMON_UARPG;
        rsrc_pool->rsrc_loc       = HERMON_IN_UAR;
        rsrc_pool->rsrc_pool_size = (num << PAGESHIFT);
        rsrc_pool->rsrc_shift     = PAGESHIFT;
        rsrc_pool->rsrc_quantum   = (uint_t)PAGESIZE;
        rsrc_pool->rsrc_align     = PAGESIZE;
        rsrc_pool->rsrc_state     = state;
        rsrc_pool->rsrc_start     = (void *)state->hs_reg_uar_baseaddr;
        rsrc_name = kmem_zalloc(HERMON_RSRC_NAME_MAXLEN, KM_SLEEP);
        HERMON_RSRC_NAME(rsrc_name, HERMON_UAR_PAGE_VMEM_RUNTM);
        entry_info.hwi_num        = num;
        entry_info.hwi_max        = max;
        entry_info.hwi_prealloc   = num_prealloc;
        entry_info.hwi_rsrcpool   = rsrc_pool;
        entry_info.hwi_rsrcname   = rsrc_name;
        if (hermon_rsrc_hw_entries_init(state, &entry_info) != DDI_SUCCESS) {
                kmem_free(rsrc_name, HERMON_RSRC_NAME_MAXLEN);
                goto error;
        }
        kmem_free(rsrc_name, HERMON_RSRC_NAME_MAXLEN);

        /* Re-allocate the kernel UAR page */
        if (hermon_rsrc_alloc(state, HERMON_UARPG, 1, HERMON_SLEEP,
            &state->hs_uarkpg_rsrc) != DDI_SUCCESS) {
                goto error;
        }

        /* Setup pointer to kernel UAR page */
        state->hs_uar = (hermon_hw_uar_t *)state->hs_uarkpg_rsrc->hr_addr;

        /* Now drop the the Hermon PIO FM */
        hermon_clr_state(state, HCA_PIO_FM);

        /* Release the MSI-X Table access handle */
        if (state->hs_fm_msix_tblhdl) {
                hermon_regs_map_free(state, &state->hs_fm_msix_tblhdl);
                state->hs_fm_msix_tblhdl = NULL;
        }

        /* Release the MSI-X PBA access handle */
        if (state->hs_fm_msix_pbahdl) {
                hermon_regs_map_free(state, &state->hs_fm_msix_pbahdl);
                state->hs_fm_msix_pbahdl = NULL;
        }

        /* Release the pci config space access handle */
        if (state->hs_fm_pcihdl) {
                hermon_regs_map_free(state, &state->hs_fm_pcihdl);
                state->hs_fm_pcihdl = NULL;
        }

        /* Release the cmd protected access handle */
        if (state->hs_fm_cmdhdl) {
                hermon_regs_map_free(state, &state->hs_fm_cmdhdl);
                state->hs_fm_cmdhdl = NULL;
        }

        /* Release the uar fma-protected access handle */
        if (state->hs_fm_uarhdl) {
                hermon_regs_map_free(state, &state->hs_fm_uarhdl);
                state->hs_fm_uarhdl = NULL;
        }

        /* Enable the Hermon interrupt again */
        if (state->hs_intrmsi_cap & DDI_INTR_FLAG_BLOCK) {
                if (ddi_intr_block_enable
                    (&state->hs_intrmsi_hdl[0], 1) != DDI_SUCCESS) {
                        return (DDI_FAILURE);
                }
        } else {
                if (ddi_intr_enable
                    (state->hs_intrmsi_hdl[0]) != DDI_SUCCESS) {
                        return (DDI_FAILURE);
                }
        }

        /* Restart the poll thread */
        state->hs_fm_poll_suspend = B_FALSE;

        return (DDI_SUCCESS);

error:
        /* Enable the Hermon interrupt again */
        if (state->hs_intrmsi_cap & DDI_INTR_FLAG_BLOCK) {
                (void) ddi_intr_block_enable(&state->hs_intrmsi_hdl[0], 1);
        } else {
                (void) ddi_intr_enable(state->hs_intrmsi_hdl[0]);
        }
        return (DDI_FAILURE);
}


/*
 *  int
 *  hermon_regs_map_setup(hermon_state_t *state, uint_t rnumber, caddr_t *addrp,
 *      offset_t offset, offset_t len, ddi_device_acc_attr_t *accattrp,
 *      ddi_acc_handle_t *handle)
 *
 *  Overview
 *      This is a wrapper function of i_hca_regs_map_setup() for Hermon FM so
 *      that it calls i_hca_regs_map_setup() inside after it checks the
 *      "fm_disable" configuration property. If the "fm_disable" is described
 *      in /kernel/drv/hermon.conf, the function calls ddi_regs_map_setup()
 *      directly instead.
 *      See i_hca_regs_map_setup() in detail.
 *
 *  Argument
 *      state: pointer to Hermon state structure
 *      rnumber: index number to the register address space set
 *      addrp: platform-dependent value (same as ddi_regs_map_setup())
 *      offset: offset into the register address space
 *      len: address space length to be mapped
 *      accattrp: pointer to device access attribute structure
 *      handle: pointer to ddi_acc_handle_t used for HCA FM
 *
 *  Return value
 *      ddi function status value which are:
 *              DDI_SUCCESS
 *              DDI_FAILURE
 *              DDI_ME_RNUMBER_RNGE
 *              DDI_REGS_ACC_CONFLICT
 *
 *  Caller's context
 *      hermon_regs_map_setup() can be called in user or kernel context only.
 */
int
hermon_regs_map_setup(hermon_state_t *state, uint_t rnumber, caddr_t *addrp,
        offset_t offset, offset_t len, ddi_device_acc_attr_t *accattrp,
        ddi_acc_handle_t *handle)
{
        if (state->hs_fm_disable) {
                return (ddi_regs_map_setup(state->hs_dip, rnumber, addrp,
                    offset, len, accattrp, handle));
        } else {
                return (i_hca_regs_map_setup(state->hs_fm_hca_fm, state->hs_dip,
                    rnumber, addrp, offset, len, accattrp, handle));
        }
}


/*
 *  void
 *  hermon_regs_map_free(hermon_state_t *state, ddi_acc_handle_t *handlep)
 *
 *  Overview
 *      This is a wrapper function of i_hca_regs_map_free() for Hermon FM so
 *      that it calls i_hca_regs_map_free() inside after it checks the
 *      "fm_disable" configuration property. If the "fm_disable" is described
 *      in /kernel/drv/hermon.conf, the function calls ddi_regs_map_fre()
 *      directly instead.  See i_hca_regs_map_free() in detail.
 *
 *  Argument
 *      state: pointer to Hermon state structure
 *      handle: pointer to ddi_acc_handle_t used for HCA FM
 *
 *  Return value
 *      Nothing
 *
 *  Caller's context
 *      hermon_regs_map_free() can be called in user or kernel context only.
 *
 *  Note that the handle passed to hermon_regs_map_free() is NULL-cleared
 *  after this function is called.
 */
void
hermon_regs_map_free(hermon_state_t *state, ddi_acc_handle_t *handle)
{
        if (state->hs_fm_disable) {
                ddi_regs_map_free(handle);
                *handle = NULL;
        } else {
                i_hca_regs_map_free(state->hs_fm_hca_fm, handle);
        }
}


/*
 *  int
 *  hermon_pci_config_setup(hermon_state_t *state, ddi_acc_handle_t *handle)
 *
 *  Overview
 *      This is a wrapper function of i_hca_pci_config_setup() for Hermon FM so
 *      that it calls i_hca_pci_config_setup() inside after it checks the
 *      "fm-disable" configuration property. If the "fm_disable" is described
 *      in /kernel/drv/hermon.conf, the function calls pci_config_setup()
 *      directly instead. See i_hca_pci_config_setup() in detail.
 *
 *  Argument
 *      state: pointer to Hermon state structure
 *      handle: pointer to ddi_acc_handle_t used for HCA FM
 *
 *  Return value
 *      ddi function status value which are:
 *              DDI_SUCCESS
 *              DDI_FAILURE
 *
 *  Caller's context
 *      hermon_pci_config_setup() can be called in user or kernel context only.
 */
int
hermon_pci_config_setup(hermon_state_t *state, ddi_acc_handle_t *handle)
{
        if (state->hs_fm_disable) {
                return (pci_config_setup(state->hs_dip, handle));
        } else {
                /* Check Hermon FM and Solaris FMA capability flags */
                ASSERT((hermon_get_state(state) & HCA_PIO_FM &&
                    DDI_FM_ACC_ERR_CAP(ddi_fm_capable(state->hs_dip))) ||
                    (!(hermon_get_state(state) & HCA_PIO_FM) &&
                    !DDI_FM_ACC_ERR_CAP(ddi_fm_capable(state->hs_dip))));
                return (i_hca_pci_config_setup(state->hs_fm_hca_fm,
                    state->hs_dip, handle));
        }
}


/*
 *  void
 *  hermon_pci_config_teardown(hermon_state_t *state, ddi_acc_handle_t *handle)
 *
 *  Overview
 *      This is a wrapper function of i_hca_pci_config_teardown() for Hermon
 *      FM so that it calls i_hca_pci_config_teardown() inside after it checks
 *      the "fm-disable" configuration property. If the "fm_disable" is
 *      described in /kernel/drv/hermon.conf, the function calls
 *      pci_config_teardown() directly instead.
 *      See i_hca_pci_config_teardown() in detail.
 *
 *  Argument
 *      state: pointer to Hermon state structure
 *      handle: pointer to ddi_acc_handle_t used for HCA FM
 *
 *  Return value
 *      Nothing
 *
 *  Caller's context
 *      hermon_pci_config_teardown() can be called in user or kernel context
 *      only.
 */
void
hermon_pci_config_teardown(hermon_state_t *state, ddi_acc_handle_t *handle)
{
        if (state->hs_fm_disable) {
                pci_config_teardown(handle);
                *handle = NULL;
        } else {
                i_hca_pci_config_teardown(state->hs_fm_hca_fm, handle);
        }
}


/*
 *  boolean_t
 *  hermon_init_failure(hermon_state_t *state)
 *
 *  Overview
 *      hermon_init_failure() tells if HW errors are detected in
 *      the Hermon driver attach.
 *
 *  Argument
 *      state: pointer to Hermon state structure
 *
 *  Return value
 *      B_TRUE          HW errors detected during attach
 *      B_FALSE         No HW errors during attach
 *
 *  Caller's context
 *      hermon_init_failure() can be called in user, kernel, interrupt
 *      context or high interrupt context.
 */
boolean_t
hermon_init_failure(hermon_state_t *state)
{
        ddi_acc_handle_t hdl;
        ddi_fm_error_t derr;

        if (!(hermon_get_state(state) & HCA_PIO_FM))
                return (B_FALSE);

        /* check if fatal errors occur during attach */
        if (state->hs_fm_async_fatal)
                return (B_TRUE);

        hdl = hermon_get_uarhdl(state);
        /* Get the PIO error against UAR I/O space */
        ddi_fm_acc_err_get(hdl, &derr, DDI_FME_VERSION);
        if (derr.fme_status != DDI_FM_OK) {
                return (B_TRUE);
        }

        hdl = hermon_get_cmdhdl(state);
        /* Get the PIO error againsts CMD I/O space */
        ddi_fm_acc_err_get(hdl, &derr, DDI_FME_VERSION);
        if (derr.fme_status != DDI_FM_OK) {
                return (B_TRUE);
        }

        return (B_FALSE);
}


/*
 *  void
 *  hermon_fm_ereport(hermon_state_t *state, int type, int detail)
 *
 *  Overview
 *      hermon_fm_ereport() is a Hermon FM ereport function used
 *      to issue a Solaris FMA ereport. See Hermon FM comments at the
 *      beginning of this file in detail.
 *
 *  Argument
 *      state: pointer to Hermon state structure
 *      type: error type
 *              HCA_SYS_ERR     FMA reporting HW error
 *              HCA_IBA_ERR     HCA specific HW error
 *      detail: HW error hint implying which ereport is issued
 *              HCA_ERR_TRANSIENT       HW transienet error
 *              HCA_ERR_NON_FATAL       HW persistent error
 *              HCA_ERR_FATAL           HW fatal error
 *              HCA_ERR_SRV_LOST        IB service lost due to HW error
 *              HCA_ERR_DEGRADED        Hermon driver and/or uDAPL degraded
 *                                      due to HW error
 *              HCA_ERR_IOCTL           HW error detected in user conetxt
 *                                      (especially in ioctl())
 *
 *  Return value
 *      Nothing
 *
 *  Caller's context
 *      hermon_fm_ereport() can be called in user, kernel, interrupt context
 *      or high interrupt context.
 */
void
hermon_fm_ereport(hermon_state_t *state, int type, int detail)
{
        /*
         * If hermon_fm_diable is set or there is no FM ereport service
         * provided, then skip the rest.
         */
        if (state->hs_fm_disable ||
            !(hermon_get_state(state) & HCA_EREPORT_FM)) {
                return;
        }

        switch (type) {

        case HCA_SYS_ERR:
                switch (detail) {
                case HCA_ERR_TRANSIENT:
                case HCA_ERR_IOCTL:
                        ddi_fm_service_impact(state->hs_dip,
                            DDI_SERVICE_UNAFFECTED);
                        break;
                case HCA_ERR_NON_FATAL:
                        /* Nothing */
                        break;
                case HCA_ERR_SRV_LOST:
                        ddi_fm_service_impact(state->hs_dip,
                            DDI_SERVICE_LOST);
                        break;
                case HCA_ERR_DEGRADED:
                        switch (state->hs_fm_degraded_reason) {
                        case HCA_FW_CORRUPT:
                                i_hca_fm_ereport(state->hs_dip, type,
                                    DDI_FM_DEVICE_FW_CORRUPT);
                                break;
                        case HCA_FW_MISMATCH:
                                i_hca_fm_ereport(state->hs_dip, type,
                                    DDI_FM_DEVICE_FW_MISMATCH);
                                break;
                        case HCA_FW_MISC:
                        default:
                                i_hca_fm_ereport(state->hs_dip, type,
                                    DDI_FM_DEVICE_INTERN_UNCORR);
                                break;
                        }
                        ddi_fm_service_impact(state->hs_dip,
                            DDI_SERVICE_DEGRADED);
                        break;
                case HCA_ERR_FATAL:
                        ddi_fm_service_impact(state->hs_dip,
                            DDI_SERVICE_LOST);
                        state->hs_fm_async_fatal = B_TRUE;
                        break;
                default:
                        cmn_err(CE_WARN, "hermon_fm_ereport: Unknown error. "
                            "type = %d, detail = %d\n.", type, detail);
                }
                break;

        case HCA_IBA_ERR:
                switch (detail) {
                case HCA_ERR_TRANSIENT:
                        i_hca_fm_ereport(state->hs_dip, type,
                            DDI_FM_DEVICE_INTERN_UNCORR);
                        ddi_fm_service_impact(state->hs_dip,
                            DDI_SERVICE_UNAFFECTED);
                        break;
                case HCA_ERR_SRV_LOST:
                        cmn_err(CE_WARN, "hermon_fm_ereport: not supported "
                            "error. type = %d, detail = %d\n.", type, detail);
                        break;
                case HCA_ERR_DEGRADED:
                        switch (state->hs_fm_degraded_reason) {
                        case HCA_FW_CORRUPT:
                                i_hca_fm_ereport(state->hs_dip, type,
                                    DDI_FM_DEVICE_FW_CORRUPT);
                                break;
                        case HCA_FW_MISMATCH:
                                i_hca_fm_ereport(state->hs_dip, type,
                                    DDI_FM_DEVICE_FW_MISMATCH);
                                break;
                        case HCA_FW_MISC:
                        default:
                                i_hca_fm_ereport(state->hs_dip, type,
                                    DDI_FM_DEVICE_INTERN_UNCORR);
                                break;
                        }
                        ddi_fm_service_impact(state->hs_dip,
                            DDI_SERVICE_DEGRADED);
                        break;
                case HCA_ERR_IOCTL:
                case HCA_ERR_NON_FATAL:
                        i_hca_fm_ereport(state->hs_dip, type,
                            DDI_FM_DEVICE_INTERN_UNCORR);
                        ddi_fm_service_impact(state->hs_dip,
                            DDI_SERVICE_UNAFFECTED);
                        break;
                case HCA_ERR_FATAL:
                        if (hermon_get_state(state) & HCA_PIO_FM) {
                                if (servicing_interrupt()) {
                                        atomic_inc_32(&state->
                                            hs_fm_async_errcnt);
                                } else {
                                        i_hca_fm_ereport(state->hs_dip, type,
                                            DDI_FM_DEVICE_INTERN_UNCORR);
                                        ddi_fm_service_impact(state->hs_dip,
                                            DDI_SERVICE_LOST);
                                }
                                state->hs_fm_async_fatal = B_TRUE;
                        } else {
                                i_hca_fm_ereport(state->hs_dip, type,
                                    DDI_FM_DEVICE_INTERN_UNCORR);
                                ddi_fm_service_impact(state->hs_dip,
                                    DDI_SERVICE_LOST);
                                cmn_err(CE_PANIC,
                                    "Hermon Fatal Internal Error. "
                                    "Hermon state=0x%p", (void *)state);
                        }
                        break;
                default:
                        cmn_err(CE_WARN, "hermon_fm_ereport: Unknown error. "
                            "type = %d, detail = %d\n.", type, detail);
                }
                break;

        default:
                cmn_err(CE_WARN, "hermon_fm_ereport: Unknown type "
                    "type = %d, detail = %d\n.", type, detail);
                break;
        }
}


/*
 *  uchar_t
 *  hermon_devacc_attr_version(hermon_state_t *)
 *
 *  Overview
 *      hermon_devacc_attr_version() returns the ddi device attribute
 *      version.
 *
 *  Argument
 *      state: pointer to Hermon state structure
 *
 *  Return value
 *      dev_acc_attr_version value
 *              DDI_DEVICE_ATTR_V0      Hermon FM disabled
 *              DDI_DEVICE_ATTR_V1      Hermon FM enabled
 *
 *  Caller's context
 *      hermon_devacc_attr_version() can be called in user, kernel, interrupt
 *      context or high interrupt context.
 */
ushort_t
hermon_devacc_attr_version(hermon_state_t *state)
{
        if (state->hs_fm_disable) {
                return (DDI_DEVICE_ATTR_V0);
        } else {
                return (DDI_DEVICE_ATTR_V1);
        }
}


/*
 *  uchar_t
 *  hermon_devacc_attr_access(hermon_state_t *)
 *
 *  Overview
 *      hermon_devacc_attr_access() returns devacc_attr_access error
 *      protection types.
 *
 *  Argument
 *      state: pointer to Hermon state structure
 *
 *  Return value
 *      dev_acc_attr_access error protection type
 *              DDI_DEFAULT_ACC         Hermon FM disabled for PIO
 *              DDI_FLAGERR_ACC         Hermon FM enabled for PIO
 *
 *  Caller's context
 *      hermon_devacc_attr_access() can be called in user, kernel, interrupt
 *      context or high interrupt context.
 */
uchar_t
hermon_devacc_attr_access(hermon_state_t *state)
{
        if (state->hs_fm_disable) {
                return (DDI_DEFAULT_ACC);
        } else {
                return (DDI_FLAGERR_ACC);
        }
}


/*
 *  int
 *  hermon_PIO_start(hermon_state_t *state, ddi_acc_handle_t handle,
 *      hermon_test_t *tst)
 *
 *  Overview
 *      hermon_PIO_start() should be called before Hermon driver issues PIOs
 *      against I/O space. If Hermon FM is disabled, this function returns
 *      HCA_PIO_OK always. See i_hca_pio_start() in detail.
 *
 *  Argument
 *      state: pointer to Hermon state structure
 *      handle: pointer to ddi_acc_handle_t used for HCA FM
 *      tst: pointer to HCA FM function test structure. If the structure
 *           is not used, the NULL value must be passed instead.
 *
 *  Return value
 *      error status showing whether or not this error can retry
 *      HCA_PIO_OK              No HW errors
 *      HCA_PIO_TRANSIENT       This error could be transient
 *      HCA_PIO_PERSISTENT      This error is persistent
 *
 *  Caller's context
 *      hermon_PIO_start() can be called in user, kernel or interrupt context.
 */
int
hermon_PIO_start(hermon_state_t *state, ddi_acc_handle_t handle,
    hermon_test_t *tst)
{
        if (state->hs_fm_disable) {
                return (HCA_PIO_OK);
        } else {
                struct i_hca_acc_handle *handlep =
                    i_hca_get_acc_handle(state->hs_fm_hca_fm, handle);
                ASSERT(handlep != NULL);
                return (i_hca_pio_start(state->hs_dip, handlep, tst));
        }
}


/*
 *  int
 *  hermon_PIO_end(hermon_state_t *state, ddi_acc_handle_t handle, int *cnt,
 *      hermon_test_t *tst)
 *
 *  Overview
 *      hermon_PIO_end() should be called after Hermon driver issues PIOs
 *      against I/O space. If Hermon FM is disabled, this function returns
 *      HCA_PIO_OK always. See i_hca_pio_end() in detail.
 *
 *  Argument
 *      state: pointer to Hermon state structure
 *      handle: pointer to ddi_acc_handle_t used for HCA FM
 *      cnt: pointer to the counter variable which holds the nubmer of retry
 *           (HCA_PIO_RETRY_CNT) when a HW error is detected.
 *      tst: pointer to HCA FM function test structure. If the structure
 *           is not used, the NULL value must be passed instead.
 *
 *  Return value
 *      error status showing whether or not this error can retry
 *      HCA_PIO_OK              No HW errors
 *      HCA_PIO_TRANSIENT       This error could be transient
 *      HCA_PIO_PERSISTENT      This error is persistent
 *
 *  Caller's context
 *      hermon_PIO_end() can be called in user, kernel or interrupt context.
 */
int
hermon_PIO_end(hermon_state_t *state, ddi_acc_handle_t handle, int *cnt,
    hermon_test_t *tst)
{
        if (state->hs_fm_disable) {
                return (HCA_PIO_OK);
        } else {
                struct i_hca_acc_handle *handlep =
                    i_hca_get_acc_handle(state->hs_fm_hca_fm, handle);
                ASSERT(handlep != NULL);
                return (i_hca_pio_end(state->hs_dip, handlep, cnt, tst));
        }
}


/*
 *  ddi_acc_handle_t
 *  hermon_get_cmdhdl(hermon_state_t *state)
 *
 *  Overview
 *      hermon_get_cmdhdl() returns either the fma-protected access handle or
 *      the regular ddi-access handle depending on the Hermon FM state for
 *      Hermon command I/O space.
 *
 *  Argument
 *      state: pointer to Hermon state structure
 *
 *  Return value
 *      the access handle for pio requests
 *
 *  Caller's context
 *      hermon_get_cmdhdl() can be called in user, kernel, interrupt context
 *      or high interrupt context.
 */
ddi_acc_handle_t
hermon_get_cmdhdl(hermon_state_t *state)
{
        return (state->hs_fm_disable || hermon_get_state(state) & HCA_PIO_FM ?
            state->hs_fm_cmdhdl : state->hs_reg_cmdhdl);
}


/*
 *  ddi_acc_handle_t
 *  hermon_get_uarhdl(hermon_state_t *state)
 *
 *  Overview
 *      hermon_get_uarhdl() returns either the fma-protected access handle or
 *      the regular ddi-access handle depending on the Hermon FM state for
 *      Hermon UAR I/O space.
 *
 *  Argument
 *      state: pointer to Hermon state structure
 *
 *  Return value
 *      the access handle for pio requests
 *
 *  Caller's context
 *      hermon_get_uarhdl() can be called in user, kernel, interrupt context
 *      or high interrupt context.
 */
ddi_acc_handle_t
hermon_get_uarhdl(hermon_state_t *state)
{
        return (state->hs_fm_disable || hermon_get_state(state) & HCA_PIO_FM ?
            state->hs_fm_uarhdl : state->hs_reg_uarhdl);
}


/*
 *  ddi_acc_handle_t
 *  hermon_rsrc_alloc_uarhdl(hermon_state_t *state)
 *
 *  Overview
 *      hermon_rsrc_alloc_uarhdl() returns either the fma-protected access
 *      handle or the regular ddi-access handle depending on the Hermon FM
 *      state for Hermon UAR I/O space as well as hermon_get_uarhdl(), but
 *      this function is dedicated to the UAR resource allocator.
 *
 *  Argument
 *      state: pointer to Hermon state structure
 *
 *  Return value
 *      the access handle for pio requests
 *
 *  Caller's context
 *      hermon_rsrc_alloc_uarhdl() can be called in user, kernel, interrupt
 *      or high interrupt context.
 */
ddi_acc_handle_t
hermon_rsrc_alloc_uarhdl(hermon_state_t *state)
{
        return (state->hs_fm_disable || hermon_get_state(state) & HCA_ATTCH_FM ?
            state->hs_fm_uarhdl : state->hs_reg_uarhdl);
}

/*
 *  ddi_acc_handle_t
 *  hermon_get_pcihdl(hermon_state_t *state)
 *
 *  Overview
 *      hermon_get_pcihdl() returns either the fma-protected access
 *      handle or the regular ddi-access handle to access the PCI config
 *      space. Whether or not which handle is returned at the moment depends
 *      on the Hermon FM state.
 *
 *  Argument
 *      state: pointer to Hermon state structure
 *
 *  Return value
 *      the access handle to PCI config space
 *
 *  Caller's context
 *      hermon_get_pcihdl() can be called in user, kernel, interrupt
 *      or high interrupt context.
 */
ddi_acc_handle_t
hermon_get_pcihdl(hermon_state_t *state)
{
        return (state->hs_fm_disable || hermon_get_state(state) & HCA_ATTCH_FM ?
            state->hs_fm_pcihdl : state->hs_reg_pcihdl);
}


/*
 *  ddi_acc_handle_t
 *  hermon_get_msix_tblhdl(hermon_state_t *state)
 *
 *  Overview
 *      hermon_get_msix_tblhdl() returns either the fma-protected access
 *      handle or the regular ddi-access handle to access the MSI-X tables.
 *      Whether or not which handle is returned at the moment depends on
 *      the Hermon FM state.
 *
 *  Argument
 *      state: pointer to Hermon state structure
 *
 *  Return value
 *      the access handle to MSI-X tables
 *
 *  Caller's context
 *      hermon_get_msix_tblhdl() can be called in user, kernel, interrupt
 *      context or high interrupt context.
 */
ddi_acc_handle_t
hermon_get_msix_tblhdl(hermon_state_t *state)
{
        return (state->hs_fm_disable || hermon_get_state(state) & HCA_ATTCH_FM ?
            state->hs_fm_msix_tblhdl : state->hs_reg_msix_tblhdl);
}


/*
 *  ddi_acc_handle_t
 *  hermon_get_msix_pbahdl(hermon_state_t *state)
 *
 *  Overview
 *      hermon_get_msix_pbahdl() returns either the fma-protected access
 *      handle or the regular ddi-access handle to access the MSI-X PBA.
 *      Whether or not which handle is returned at the moment depends on
 *      the Hermon FM state.
 *
 *  Argument
 *      state: pointer to Hermon state structure
 *
 *  Return value
 *      the access handle to MSI-X PBA
 *
 *  Caller's context
 *      hermon_get_msix_pbahdl() can be called in user, kernel, interrupt
 *      context or high interrupt context.
 */
ddi_acc_handle_t
hermon_get_msix_pbahdl(hermon_state_t *state)
{
        return (state->hs_fm_disable || hermon_get_state(state) & HCA_ATTCH_FM ?
            state->hs_fm_msix_pbahdl : state->hs_reg_msix_pbahdl);
}


/*
 *  void
 *  hermon_inter_err_chk(void *arg)
 *
 *  Overview
 *      hermon_inter_err_chk() periodically checks the internal error buffer
 *      to pick up a Hermon asynchronous internal error.
 *
 *      Note that this internal error can be notified if the interrupt is
 *      registered, but even so there are some cases that an interrupt against
 *      it cannot be raised so that Hermon RPM recommeds to poll this internal
 *      error buffer periodically instead. This function is invoked at
 *      10ms interval in kernel context though the function itself can be
 *      called in interrupt context.
 *
 *  Argument
 *      arg: pointer to Hermon state structure
 *
 *  Return value
 *      Nothing
 *
 *  Caller's context
 *      hermon_inter_err_chk() can be called in user, kernel, interrupt
 *      context or high interrupt context.
 *
 */
void
hermon_inter_err_chk(void *arg)
{
        uint32_t        word;
        ddi_acc_handle_t cmdhdl;
        hermon_state_t *state = (hermon_state_t *)arg;

        /* initialize the FMA retry loop */
        hermon_pio_init(fm_loop_cnt, fm_status, fm_test);

#ifdef FMA_TEST
        if (hermon_test_num != 0) {
                return;
        }
#endif
        if (state->hs_fm_poll_suspend) {
                return;
        }

        /* Get the access handle for Hermon CMD I/O space */
        cmdhdl = hermon_get_cmdhdl(state);

        /* the FMA retry loop starts. */
        hermon_pio_start(state, cmdhdl, pio_error, fm_loop_cnt, fm_status,
            fm_test);

        word = ddi_get32(cmdhdl, state->hs_cmd_regs.fw_err_buf);

        /* the FMA retry loop ends. */
        hermon_pio_end(state, cmdhdl, pio_error, fm_loop_cnt, fm_status,
            fm_test);

        if (word != 0) {
                HERMON_FMANOTE(state, HERMON_FMA_INTERNAL);
                /* if fm_disable is on, Hermon FM functions don't work */
                if (state->hs_fm_disable) {
                        cmn_err(CE_PANIC,
                            "Hermon Fatal Internal Error. "
                            "Hermon state=0x%p", (void *)state);
                } else {
                        hermon_fm_ereport(state, HCA_IBA_ERR, HCA_ERR_FATAL);
                }
        }

        /* issue the ereport pended in the interrupt context */
        if (state->hs_fm_async_errcnt > 0) {
                hermon_fm_ereport(state, HCA_IBA_ERR, HCA_ERR_FATAL);
                atomic_dec_32(&state->hs_fm_async_errcnt);
        }

        return;

pio_error:
        hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_FATAL);
}


/*
 *  boolean_t
 *  hermon_cmd_retry_ok(hermon_cmd_post_t *cmd, int status)
 *
 *  Overview
 *      In the case that a HW error is detected, if it can be isolated
 *      enough, Hermon FM retries the operation which caused the error.
 *      However, this retry can induce another error; since the retry is
 *      achieved as a block basis, not a statement basis, once the state
 *      was set inside the Hermon HW already in the previous operation, the
 *      retry can cause for example, a CMD_BAD_SYS_STATE error, as a result.
 *      In this case, CMD_BAD_SYS_STATE should be taken as a side effect
 *      but a harmless result. hermon_cmd_retry_ok() checks this kind of
 *      situation then returns if the state Hermon CMD returns is OK or not.
 *
 *  Argument
 *      cmd: pointer to hermon_cmd_post_t structure
 *      status: Hermon CMD status
 *
 *  Return value
 *      B_TRUE          this state is no problem
 *      B_FALSE         this state should be taken as an error
 *
 *  Caller's context
 *      hermon_cmd_retry_ok() can be called in user, kernel, interrupt
 *      context or high interrupt context.
 *
 *  Note that status except for HERMON_CMD_SUCCESS shouldn't be accepted
 *  in the debug module to catch a hidden software bug, so that ASSERT()
 *  is enabled in the case.
 */
boolean_t
hermon_cmd_retry_ok(hermon_cmd_post_t *cmd, int status)
{
        if (status == HERMON_CMD_SUCCESS)
                return (B_TRUE);

        /*
         * The wrong status such as HERMON_CMD_BAD_SYS_STATE or
         * HERMON_CMD_BAD_RES_STATE can return as a side effect
         * because of the Hermon FM operation retry when a PIO
         * error is detected during the I/O transaction. In the
         * case, the driver may set the same value in Hermon
         * though it was set already, then Hermon returns HERMON_
         * CMD_BAD_{RES,SYS}_STATE as a result, which should be
         * taken as OK.
         */
        switch (cmd->cp_opcode) {
        case INIT_HCA:
                /*
                 * HERMON_CMD_BAD_SYS_STATE can be gotten in case of
                 * ICM not mapped or HCA already initialized.
                 */
                if (status == HERMON_CMD_BAD_SYS_STATE)
                        return (B_TRUE);
                return (B_FALSE);

        case CLOSE_HCA:
                /*
                 * HERMON_CMD_BAD_SYS_STATE can be gotten in case of Firmware
                 * area is not mapped or HCA already closed.
                 */
                if (status == HERMON_CMD_BAD_SYS_STATE)
                        return (B_TRUE);
                return (B_FALSE);

        case CLOSE_PORT:
                /*
                 * HERMON_CMD_BAD_SYS_STATE can be gotten in case of HCA not
                 * initialized or in case that IB ports are already down.
                 */
                if (status == HERMON_CMD_BAD_SYS_STATE)
                        return (B_TRUE);
                return (B_FALSE);

        case SW2HW_MPT:
                /*
                 * HERMON_CMD_BAD_RES_STATE can be gotten in case of MPT
                 * entry already in hardware ownership.
                 */
                if (status == HERMON_CMD_BAD_RES_STATE)
                        return (B_TRUE);
                return (B_FALSE);

        case HW2SW_MPT:
                /*
                 * HERMON_CMD_BAD_RES_STATE can be gotten in case of MPT
                 * entry already in software ownership.
                 */
                if (status == HERMON_CMD_BAD_RES_STATE)
                        return (B_TRUE);
                return (B_FALSE);

        case SW2HW_EQ:
                /*
                 * HERMON_CMD_BAD_RES_STATE can be gotten in case of EQ
                 * entry already in hardware ownership.
                 */
                if (status == HERMON_CMD_BAD_RES_STATE)
                        return (B_TRUE);
                return (B_FALSE);

        case HW2SW_EQ:
                /*
                 * HERMON_CMD_BAD_RES_STATE can be gotten in case of EQ
                 * entry already in software ownership.
                 */
                if (status == HERMON_CMD_BAD_RES_STATE)
                        return (B_TRUE);
                return (B_FALSE);

        case SW2HW_CQ:
                /*
                 * HERMON_CMD_BAD_RES_STATE can be gotten in case of CQ
                 * entry already in hardware ownership.
                 */
                if (status == HERMON_CMD_BAD_RES_STATE)
                        return (B_TRUE);
                return (B_FALSE);

        case HW2SW_CQ:
                /*
                 * HERMON_CMD_BAD_RES_STATE can be gotten in case of CQ
                 * entry already in software ownership.
                 */
                if (status == HERMON_CMD_BAD_RES_STATE)
                        return (B_TRUE);
                return (B_FALSE);

        case SW2HW_SRQ:
                /*
                 * HERMON_CMD_BAD_RES_STATE can be gotten in case of SRQ
                 * entry already in hardware ownership.
                 */
                if (status == HERMON_CMD_BAD_RES_STATE)
                        return (B_TRUE);
                return (B_FALSE);

        case HW2SW_SRQ:
                /*
                 * HERMON_CMD_BAD_RES_STATE can be gotten in case of SRQ
                 * entry already in software ownership.
                 */
                if (status == HERMON_CMD_BAD_RES_STATE)
                        return (B_TRUE);
                return (B_FALSE);
        default:
                break;
        }

        /* other cases */
        return (B_FALSE);
}


#ifdef FMA_TEST

/*
 * Hermon FMA test variables
 */
#define FMA_TEST_HASHSZ 64
int hermon_test_num;                    /* predefined testset */

static struct i_hca_fm_test *i_hca_test_register(char *, int, int,
    void (*)(struct i_hca_fm_test *, ddi_fm_error_t *),
    void *, mod_hash_t *, mod_hash_t *, int);
static void i_hca_test_free_item(mod_hash_val_t);
static void i_hca_test_set_item(int, struct i_hca_fm_test *);
static void hermon_trigger_pio_error(hermon_test_t *, ddi_fm_error_t *);

/*
 * Hermon FMA Function Test Interface
 */

/* Attach Errors */

#define ATTACH_TS       (HCA_TEST_TRANSIENT | HCA_TEST_ATTACH | HCA_TEST_START)
#define ATTACH_TE       (HCA_TEST_TRANSIENT | HCA_TEST_ATTACH | HCA_TEST_END)

#define ATTACH_PS       (HCA_TEST_PERSISTENT | HCA_TEST_ATTACH | HCA_TEST_START)
#define ATTACH_PE       (HCA_TEST_PERSISTENT | HCA_TEST_ATTACH | HCA_TEST_END)

static hermon_test_t testset[] = {
/* Initial Value */
{0, 0, 0, NULL, 0, 0, NULL, NULL, NULL},        /* 0 */

/* PIO Transient Errors */
{0, HCA_TEST_PIO, ATTACH_TS, NULL, /* attach/transient/start/propagate */
    HCA_PIO_RETRY_CNT, 0, NULL, NULL, NULL},    /* 1 */
{0, HCA_TEST_PIO, ATTACH_TE, NULL, /* attach/transient/end/propagate */
    HCA_PIO_RETRY_CNT, 0, NULL, NULL, NULL},    /* 2 */

/* PIO Persistent Errors */
{0, HCA_TEST_PIO, ATTACH_PS, NULL, /* attach/persistent/start/propagate */
    0, 0, NULL, NULL, NULL},                    /* 3 */
{0, HCA_TEST_PIO, ATTACH_PE, NULL, /* attach/persistent/end/propagate */
    0, 0, NULL, NULL, NULL},                    /* 4 */

};


/*
 *  void
 *  hermon_trigger_pio_error(hermon_test_t *tst, ddi_fm_error_t *derr)
 *
 *  Overview
 *      hermon_trigger_pio_error() is a PIO error injection function
 *      to cause a pseduo PIO error.
 *
 *  Argument
 *      tst: pointer to HCA FM function test structure. If the structure
 *           is not used, the NULL value must be passed instead.
 *      derr: pointer to ddi_fm_error_t structure
 *
 *  Return value
 *      Nothing
 *
 *  Caller's context
 *      hermon_trigger_pio_error() can be called in user, kernel, interrupt
 *      context or high interrupt context.
 */
static void
hermon_trigger_pio_error(hermon_test_t *tst, ddi_fm_error_t *derr)
{
        hermon_state_t *state = (hermon_state_t *)tst->private;
        derr->fme_status = DDI_FM_OK;

        if (tst->type != HCA_TEST_PIO) {
                return;
        }

        if ((tst->trigger & HCA_TEST_ATTACH &&
            i_ddi_node_state(state->hs_dip) < DS_ATTACHED &&
            hermon_get_state(state) & HCA_PIO_FM)) {
                if (tst->trigger & HCA_TEST_PERSISTENT) {
                        i_hca_fm_ereport(state->hs_dip, HCA_IBA_ERR,
                            DDI_FM_DEVICE_INVAL_STATE);
                        derr->fme_status = DDI_FM_NONFATAL;
                        return;
                } else if (tst->trigger & HCA_TEST_TRANSIENT &&
                    tst->errcnt) {
                        i_hca_fm_ereport(state->hs_dip, HCA_IBA_ERR,
                            DDI_FM_DEVICE_INVAL_STATE);
                        derr->fme_status = DDI_FM_NONFATAL;
                        tst->errcnt--;
                        return;
                }
        }
}


/*
 *  struct hermon_fm_test *
 *  hermon_test_register(hermon_state_t *state, char *filename, int linenum,
 *      int type)
 *
 *  Overview
 *      hermon_test_register() registers a Hermon FM test item for the
 *      function test.
 *
 *  Argument
 *      state: pointer to Hermon state structure
 *      filename: source file name where the function call is implemented
 *                This value is usually a __FILE__  pre-defined macro.
 *      linenum: line number where the function call is described in the
 *               file specified above.
 *               This value is usually a __LINE__ pre-defined macro.
 *      type: HW error type
 *                      HCA_TEST_PIO    pio error
 *                      HCA_TEST_IBA    ib specific error
 *
 *  Return value
 *      pointer to Hermon FM function test structure registered.
 *
 *  Caller's context
 *      hermon_test_register() can be called in user, kernel or interrupt
 *      context.
 *
 *  Note that no test item is registered if Hermon FM is disabled.
 */
hermon_test_t *
hermon_test_register(hermon_state_t *state, char *filename, int linenum,
    int type)
{
        void (*pio_injection)(struct i_hca_fm_test *, ddi_fm_error_t *) =
            (void (*)(struct i_hca_fm_test *, ddi_fm_error_t *))
            hermon_trigger_pio_error;

        if (state->hs_fm_disable)
                return (NULL);

        return ((hermon_test_t *)i_hca_test_register(filename, linenum, type,
            pio_injection, (void *)state, state->hs_fm_test_hash,
            state->hs_fm_id_hash, hermon_test_num));
}
#endif /* FMA_TEST */


/*
 * HCA FM Common Interface
 *
 * These functions should be used for any HCA drivers, but probably
 * customized for their own HW design and/or FM implementation.
 * Customized functins should have the driver name prefix such as
 * hermon_xxxx() and be defined separately but whose functions should
 * call the common interface inside.
 */

/*
 *  void
 *  i_hca_fm_init(struct i_hca_fm *hca_fm)
 *
 *  Overview
 *      i_hca_fm_init() is an initialization function which sets up the acc
 *      handle kmem_cache if this function is called the first time.
 *
 *  Argument
 *      hca_fm: pointer to HCA FM structure
 *
 *  Return value
 *      Nothing
 *
 *  Caller's context
 *      i_hca_fm_init() can be called in user or kernel context, but cannot
 *      be called in interrupt context.
 */
static void
i_hca_fm_init(struct i_hca_fm *hca_fm)
{

        mutex_enter(&hca_fm->lock);

        ++hca_fm->ref_cnt;
        if (hca_fm->fm_acc_cache == NULL) {
                hca_fm->fm_acc_cache = kmem_cache_create("hca_fm_acc_handle",
                    sizeof (struct i_hca_acc_handle), 0, NULL,
                    NULL, NULL, NULL, NULL, 0);
        }

        mutex_exit(&hca_fm->lock);
}


/*
 *  void
 *  i_hca_fm_fini(struct i_hca_fm *hca_fm)
 *
 *  Overview
 *      i_hca_fm_fini() is a finalization function which frees up the acc
 *      handle kmem_cache if this function is called the last time.
 *
 *  Argument
 *      hca_fm: pointer to HCA FM structure
 *
 *  Return value
 *      Nothing
 *
 *  Caller's context
 *      i_hca_fm_fini() can be called in user or kernel context, but cannot
 *      be called in interrupt context.
 */
static void
i_hca_fm_fini(struct i_hca_fm *hca_fm)
{
        mutex_enter(&hca_fm->lock);

        if (--hca_fm->ref_cnt == 0) {

                if (hca_fm->fm_acc_cache) {
                        kmem_cache_destroy(hca_fm->fm_acc_cache);
                        hca_fm->fm_acc_cache = NULL;
                }
        }

        mutex_exit(&hca_fm->lock);
}


/*
 *  void
 *  i_hca_fm_ereport(dev_info_t *dip, int type, char *detail)
 *
 *  Overview
 *      i_hca_fm_ereport() is a wrapper function of ddi_fm_ereport_post() but
 *      generates an ena before it calls ddi_fm_ereport_post() for HCA
 *      specific HW errors.
 *
 *  Argument
 *      dip: pointer to this device dev_info structure
 *      type: error type
 *              HCA_SYS_ERR     FMA reporting HW error
 *              HCA_IBA_ERR     HCA specific HW error
 *      detail: definition of leaf driver detected ereports which is one of:
 *              DDI_FM_DEVICE_INVAL_STATE
 *              DDI_FM_DEVICE_NO_RESPONSE
 *              DDI_FM_DEVICE_STALL
 *              DDI_FM_DEVICE_BADINT_LIMIT
 *              DDI_FM_DEVICE_INTERN_CORR
 *              DDI_FM_DEVICE_INTERN_UNCORR
 *
 *  Return value
 *      Nothing
 *
 *  Caller's context
 *      i_hca_fm_ereport() can be called in user, kernel or interrupt context.
 */
static void
i_hca_fm_ereport(dev_info_t *dip, int type, char *detail)
{
        uint64_t ena;
        char buf[FM_MAX_CLASS];

        (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail);

        ena = fm_ena_generate(0, FM_ENA_FMT1);
        if (type == HCA_IBA_ERR) {
                /* this is an error of its own */
                ena = fm_ena_increment(ena);
        }

        ddi_fm_ereport_post(dip, buf, ena, DDI_NOSLEEP,
            FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0, NULL);
}


/*
 * struct i_hca_acc_handle *
 * i_hca_get_acc_handle(struct i_hca_fm *hca_fm, ddi_acc_handle_t handle)
 *
 *  Overview
 *      i_hca_get_acc_handle() returns ddi_acc_handle_t used for HCA FM.
 *
 *  Argument
 *      hca_fm: pointer to HCA FM structure
 *      handle: ddi_acc_handle_t
 *
 *  Return value
 *      handle: pointer to ddi_acc_handle_t used for HCA FM
 *
 *  Caller's context
 *      i_hca_get_acc_handle() can be called in user, kernel or interrupt
 *      context.
 */
static struct i_hca_acc_handle *
i_hca_get_acc_handle(struct i_hca_fm *hca_fm, ddi_acc_handle_t handle)
{
        struct i_hca_acc_handle *hdlp;

        /* Retrieve the HCA FM access handle */
        mutex_enter(&hca_fm->lock);

        for (hdlp = hca_fm->hdl; hdlp != NULL; hdlp = hdlp->next) {
                if (hdlp->save_hdl == handle) {
                        mutex_exit(&hca_fm->lock);
                        return (hdlp);
                }
        }

        mutex_exit(&hca_fm->lock);
        return (hdlp);
}


/*
 *  int
 *  i_hca_regs_map_setup(struct i_hca_fm *hca_fm, dev_info_t *dip,
 *      uint_t rnumber, caddr_t *addrp, offset_t offset, offset_t len,
 *      ddi_device_acc_attr_t *accattrp, ddi_acc_handle_t *handle)
 *
 *  Overview
 *      i_hca_regs_map_setup() is a wrapper function of ddi_regs_map_setup(),
 *      but allocates the HCA FM acc handle structure and initializes it.
 *
 *  Argument
 *      hca_fm: pointer to HCA FM structure
 *      dip: pointer to this device dev_info structure
 *      rnumber: index number to the register address space set
 *      addrp: platform-dependent value (same as ddi_regs_map_setup())
 *      offset: offset into the register address space
 *      len: address space length to be mapped
 *      accattrp: pointer to device access attribute structure
 *      handle: pointer to ddi_acc_handle_t used for HCA FM
 *
 *  Return value
 *      ddi function status value which are:
 *              DDI_SUCCESS
 *              DDI_FAILURE
 *              DDI_ME_RNUMBER_RNGE
 *              DDI_REGS_ACC_CONFLICT
 *
 *  Caller's context
 *      i_hca_regs_map_setup() can be called in user or kernel context only.
 */
static int
i_hca_regs_map_setup(struct i_hca_fm *hca_fm, dev_info_t *dip, uint_t rnumber,
    caddr_t *addrp, offset_t offset, offset_t len,
    ddi_device_acc_attr_t *accattrp, ddi_acc_handle_t *handle)
{
        int status;
        struct i_hca_acc_handle *handlep, *hdlp, *last;

        /* Allocate an access handle */
        if ((status = ddi_regs_map_setup(dip, rnumber, addrp, offset,
            len, accattrp, handle)) != DDI_SUCCESS) {
                return (status);
        }

        /* Allocate HCA FM acc handle structure */
        handlep = kmem_cache_alloc(hca_fm->fm_acc_cache, KM_SLEEP);

        /* Initialize fields */
        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*handlep))
        handlep->next = NULL;
        handlep->save_hdl = (*handle);
        handlep->thread_cnt = 0;
        mutex_init(&handlep->lock, NULL, MUTEX_DRIVER, NULL);

        /* Register this handle */
        mutex_enter(&hca_fm->lock);
        for (last = hdlp = hca_fm->hdl; hdlp != NULL; hdlp = hdlp->next) {
                last = hdlp;
        }
        if (last == NULL) {
                hca_fm->hdl = handlep;
        } else {
                last->next = handlep;
        }
        mutex_exit(&hca_fm->lock);

        return (status);
}


/*
 *  void
 *  i_hca_regs_map_free(struct i_hca_fm *hca_fm, ddi_acc_handle_t *handlep)
 *
 *  Overview
 *      i_hca_regs_map_setup() is a wrapper function of ddi_regs_map_free(),
 *      and frees the HCA FM acc handle structure allocated by
 *      i_hca_regs_map_setup().
 *
 *  Argument
 *      hca_fm: pointer to HCA FM structure
 *      handle: pointer to ddi_acc_handle_t used for HCA FM
 *
 *  Return value
 *      Nothing
 *
 *  Caller's context
 *      i_hca_regs_map_free() can be called in user or kernel context only.
 *
 *  Note that the handle passed to i_hca_regs_map_free() is NULL-cleared
 *  after this function is called.
 */
static void
i_hca_regs_map_free(struct i_hca_fm *hca_fm, ddi_acc_handle_t *handle)
{
        struct i_hca_acc_handle *handlep, *hdlp, *prev;

        /* De-register this handle */
        mutex_enter(&hca_fm->lock);
        for (prev = hdlp = hca_fm->hdl; hdlp != NULL; hdlp = hdlp->next) {
                if (hdlp->save_hdl == *handle)
                        break;
                prev = hdlp;
        }
        ASSERT(prev != NULL && hdlp != NULL);
        if (hdlp != prev) {
                prev->next = hdlp->next;
        } else {
                hca_fm->hdl = hdlp->next;
        }
        handlep = hdlp;
        mutex_exit(&hca_fm->lock);

        mutex_destroy(&handlep->lock);
        handlep->save_hdl = NULL;
        kmem_cache_free(hca_fm->fm_acc_cache, handlep);

        /* Release this handle */
        ddi_regs_map_free(handle);
        *handle = NULL;
}


/*
 *  int
 *  i_hca_pci_config_setup(struct i_hca_fm *hca_fm, dev_info_t *dip,
 *      ddi_acc_handle_t *handle, boolean_t fm_protect)
 *
 *  Overview
 *      i_hca_pci_config_setup() is a wrapper function of pci_config_setup(),
 *      but allocates the HCA FM acc handle structure and initializes it.
 *
 *  Argument
 *      hca_fm: pointer to HCA FM structure
 *      dip: pointer to this device dev_info structure
 *      handle: pointer to ddi_acc_handle_t used for HCA PCI config space
 *              with FMA
 *      fm_protect: flag to tell if an fma-protected access handle should
 *              be used
 *
 *  Return value
 *      ddi function status value which are:
 *              DDI_SUCCESS
 *              DDI_FAILURE
 *
 *  Caller's context
 *      i_hca_pci_config_setup() can be called in user or kernel context only.
 */
static int
i_hca_pci_config_setup(struct i_hca_fm *hca_fm, dev_info_t *dip,
    ddi_acc_handle_t *handle)
{
        int status;
        struct i_hca_acc_handle *handlep, *hdlp, *last;

        /* Allocate an access handle */
        if ((status = pci_config_setup(dip, handle)) != DDI_SUCCESS) {
                return (status);
        }

        /* Allocate HCA FM acc handle structure */
        handlep = kmem_cache_alloc(hca_fm->fm_acc_cache, KM_SLEEP);

        /* Initialize fields */
        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*handlep))
        handlep->next = NULL;
        handlep->save_hdl = (*handle);
        handlep->thread_cnt = 0;
        mutex_init(&handlep->lock, NULL, MUTEX_DRIVER, NULL);

        /* Register this handle */
        mutex_enter(&hca_fm->lock);
        for (last = hdlp = hca_fm->hdl; hdlp != NULL; hdlp = hdlp->next) {
                last = hdlp;
        }
        if (last == NULL) {
                hca_fm->hdl = handlep;
        } else {
                last->next = handlep;
        }
        mutex_exit(&hca_fm->lock);

        return (status);
}


/*
 *  void
 *  i_hca_pci_config_teardown(struct i_hca_fm *hca_fm,
 *      ddi_acc_handle_t *handlep)
 *
 *  Overview
 *      i_hca_pci_config_teardown() is a wrapper function of
 *      pci_config_teardown(), and frees the HCA FM acc handle structure
 *      allocated by i_hca_pci_config_setup().
 *
 *  Argument
 *      hca_fm: pointer to HCA FM structure
 *      handle: pointer to ddi_acc_handle_t used for HCA FM
 *
 *  Return value
 *      Nothing
 *
 *  Caller's context
 *      i_hca_pci_config_teardown() can be called in user or kernel context
 *      only.
 *
 *  Note that the handle passed to i_hca_pci_config_teardown() is NULL-cleared
 *  after this function is called.
 */
static void
i_hca_pci_config_teardown(struct i_hca_fm *hca_fm, ddi_acc_handle_t *handle)
{
        struct i_hca_acc_handle *handlep, *hdlp, *prev;

        /* De-register this handle */
        mutex_enter(&hca_fm->lock);
        for (prev = hdlp = hca_fm->hdl; hdlp != NULL; hdlp = hdlp->next) {
                if (hdlp->save_hdl == *handle)
                        break;
                prev = hdlp;
        }
        ASSERT(prev != NULL && hdlp != NULL);
        if (hdlp != prev) {
                prev->next = hdlp->next;
        } else {
                hca_fm->hdl = hdlp->next;
        }
        handlep = hdlp;
        mutex_exit(&hca_fm->lock);

        mutex_destroy(&handlep->lock);
        handlep->save_hdl = NULL;
        kmem_cache_free(hca_fm->fm_acc_cache, handlep);

        /* Release this handle */
        pci_config_teardown(handle);
        *handle = NULL;
}


/*
 *  int
 *  i_hca_pio_start(dev_info_t *dip, struct i_acc_handle *handle,
 *      struct i_hca_fm_test *tst)
 *
 *  Overview
 *      i_hca_pio_start() is one of a pair of HCA FM fuctions for PIO, which
 *      should be called before HCA drivers issue PIOs against I/O space.
 *      See HCA FM comments at the beginning of this file in detail.
 *
 *  Argument
 *      dip: pointer to this device dev_info structure
 *      handle: pointer to ddi_acc_handle_t used for HCA FM
 *      tst: pointer to HCA FM function test structure. If the structure
 *           is not used, the NULL value must be passed instead.
 *
 *  Return value
 *      error status showing whether or not this error can retry
 *      HCA_PIO_OK              No HW errors
 *      HCA_PIO_TRANSIENT       This error could be transient
 *      HCA_PIO_PERSISTENT      This error is persistent
 *
 *  Caller's context
 *      i_hca_pio_start() can be called in user, kernel or interrupt context.
 */
/* ARGSUSED */
static int
i_hca_pio_start(dev_info_t *dip, struct i_hca_acc_handle *hdlp,
    struct i_hca_fm_test *tst)
{
        ddi_fm_error_t derr;

        /* Count up the number of threads issuing this PIO */
        mutex_enter(&hdlp->lock);
        hdlp->thread_cnt++;
        mutex_exit(&hdlp->lock);

        /* Get the PIO error via FMA */
        ddi_fm_acc_err_get(fm_acc_hdl(hdlp), &derr, DDI_FME_VERSION);

#ifdef FMA_TEST
        /* Trigger PIO errors */
        if (tst != NULL && tst->trigger & HCA_TEST_START) {
                (*tst->pio_injection)(tst, &derr);
        }
#endif /* FMA_TEST */

        switch (derr.fme_status) {
        case DDI_FM_OK:
                /* Not have to clear the fma error log */
                return (HCA_PIO_OK);

        case DDI_FM_NONFATAL:
                /* Now clear this error */
                ddi_fm_acc_err_clear(fm_acc_hdl(hdlp), DDI_FME_VERSION);

                /* Log this error and notify it as a persistent error */
                ddi_fm_service_impact(dip, DDI_SERVICE_LOST);
                return (HCA_PIO_PERSISTENT);

        /* In theory, this shouldn't happen */
        case DDI_FM_FATAL:
        case DDI_FM_UNKNOWN:
        default:
                cmn_err(CE_WARN, "Unknown HCA HW error status (%d)",
                    derr.fme_status);
                /* Return this as a persistent error */
                return (HCA_PIO_PERSISTENT);
        }
}


/*
 *  int
 *  i_hca_pio_end(dev_info_t *dip, ddi_acc_handle_t handle, int *cnt,
 *      struct i_hca_fm_test *tst)
 *
 *  Overview
 *      i_hca_pio_end() is the other of a pair of HCA FM fuctions for PIO,
 *      which should be called after HCA drivers issue PIOs against I/O space.
 *      See HCA FM comments at the beginning of this file in detail.
 *
 *  Argument
 *      dip: pointer to this device dev_info structure
 *      handle: pointer to ddi_acc_handle_t used for HCA FM
 *      cnt: pointer to the counter variable which holds the nubmer of retry
 *           when a HW error is detected.
 *      tst: pointer to HCA FM function test structure. If the structure
 *           is not used, the NULL value must be passed instead.
 *
 *  Return value
 *      error status showing whether or not this error can retry
 *      HCA_PIO_OK              No HW errors
 *      HCA_PIO_TRANSIENT       This error could be transient
 *      HCA_PIO_PERSISTENT      This error is persistent
 *
 *  Caller's context
 *      i_hca_pio_end() can be called in user, kernel or interrupt context.
 */
/* ARGSUSED */
static int
i_hca_pio_end(dev_info_t *dip, struct i_hca_acc_handle *hdlp, int *cnt,
    struct i_hca_fm_test *tst)
{
        ddi_fm_error_t derr;

        /* Get the PIO error via FMA */
        ddi_fm_acc_err_get(fm_acc_hdl(hdlp), &derr, DDI_FME_VERSION);

#ifdef FMA_TEST
        /* Trigger PIO errors */
        if (tst != NULL && tst->trigger & HCA_TEST_END) {
                (*tst->pio_injection)(tst, &derr);
        }
#endif /* FMA_TEST */

        /* Evaluate the PIO error */
        switch (derr.fme_status) {
        case DDI_FM_OK:
                /* Count down the number of threads issuing this PIO */
                mutex_enter(&hdlp->lock);
                hdlp->thread_cnt--;
                mutex_exit(&hdlp->lock);

                /* Not have to clear the fma error log */
                return (HCA_PIO_OK);

        case DDI_FM_NONFATAL:
                /* Now clear this error */
                ddi_fm_acc_err_clear(fm_acc_hdl(hdlp), DDI_FME_VERSION);

                /*
                 * Check if this error comes from another thread running
                 * with the same handle almost at the same time.
                 */
                mutex_enter(&hdlp->lock);
                if (hdlp->thread_cnt > 1) {
                        /* Count down the number of threads */
                        hdlp->thread_cnt--;
                        mutex_exit(&hdlp->lock);

                        /* Return this as a persistent error */
                        return (HCA_PIO_PERSISTENT);
                }
                mutex_exit(&hdlp->lock);

                /* Now determine if this error is persistent or not */
                if (--(*cnt) >= 0)  {
                        return (HCA_PIO_TRANSIENT);
                } else {
                        /* Count down the number of threads */
                        mutex_enter(&hdlp->lock);
                        hdlp->thread_cnt--;
                        mutex_exit(&hdlp->lock);
                        return (HCA_PIO_PERSISTENT);
                }

        /* In theory, this shouldn't happen */
        case DDI_FM_FATAL:
        case DDI_FM_UNKNOWN:
        default:
                cmn_err(CE_WARN, "Unknown HCA HW error status (%d)",
                    derr.fme_status);
                /* Return this as a persistent error */
                return (HCA_PIO_PERSISTENT);
        }
}


/*
 * HCA FM Test Interface
 *
 * These functions should be used for any HCA drivers, but probably
 * customized for their own HW design and/or FM implementation.
 * Customized functins should have the driver name prefix such as
 * hermon_xxxx() and be defined separately but whose function should
 * call the common interface inside.
 */

#ifdef FMA_TEST
static int test_num;            /* serial number */
static kmutex_t i_hca_test_lock;        /* lock for serial numer */

/*
 *  void
 *  i_hca_test_init(mod_hash_t **strHashp, mod_hash_t **idHashp)
 *
 *  Overview
 *      i_hca_test_init() creates two hash tables, one of which is for string,
 *      and the other of which is for ID, then saves pointers to arguments
 *      passed. This function uses the mod_hash utilities to manage the
 *      hash tables. About the mod_hash, see kernel/os/modhash.c.
 *
 *  Argument
 *      strHashp: pointer to String hash table pointer
 *      idHashp: pointer to ID hash table pointer
 *
 *  Return value
 *      Nothing
 *
 *  Caller's context
 *      i_hca_test_init() can be called in user or kernel context only.
 */
static void
i_hca_test_init(mod_hash_t **strHashp, mod_hash_t **idHashp)
{
        *idHashp = mod_hash_create_idhash("HCA_FMA_id_hash",
            FMA_TEST_HASHSZ, mod_hash_null_valdtor);

        *strHashp = mod_hash_create_strhash("HCA_FMA_test_hash",
            FMA_TEST_HASHSZ, i_hca_test_free_item);
}


/*
 *  void
 *  i_hca_test_fini(mod_hash_t **strHashp, mod_hash_t **idHashp)
 *
 *  Overview
 *      i_hca_test_fini() releases two hash tables used for HCA FM test.
 *
 *  Argument
 *      strHashp: pointer to String hash table pointer
 *      idHashp: pointer to ID hash table pointer
 *
 *  Return value
 *      Nothing
 *
 *  Caller's context
 *      i_hca_test_fini() can be called in user, kernel or interrupt context.
 *
 */
static void
i_hca_test_fini(mod_hash_t **strHashp, mod_hash_t **idHashp)
{
        mod_hash_destroy_hash(*strHashp);
        *strHashp = NULL;

        mod_hash_destroy_hash(*idHashp);
        *idHashp = NULL;
}


/*
 *  struct i_hca_fm_test *
 *  i_hca_test_register(char *filename, int linenum, int type,
 *      void (*pio_injection)(struct i_hca_fm_test *, ddi_fm_error_t *),
 *      void *private, mod_hash_t *strHash, mod_hash_t *idHash, int preTestNum)
 *
 *  Overview
 *      i_hca_test_register() registers an HCA FM test item against HCA FM
 *      function callings specified with the file name and the line number
 *      (passed as the arguments).
 *
 *  Argument
 *      filename: source file name where the function call is implemented
 *                This value is usually a __FILE__  pre-defined macro.
 *      linenum: line number where the function call is described in the
 *               file specified above.
 *               This value is usually a __LINE__ pre-defined macro.
 *      type: HW error type
 *                      HCA_TEST_PIO    pio error
 *                      HCA_TEST_IBA    ib specific error
 *      pio_injection: pio error injection callback function invoked when the
 *                     function specified above (with the file name and the
 *                     line number) is executed. If the function is not a PIO,
 *                     request, this parameter should be NULL.
 *      private: the argument passed to either of injection functions when
 *               they're invoked.
 *      strHashp: pointer to String hash table
 *      idHashp: pointer to ID hash table
 *      preTestNum: the index of the pre-defined testset for this test item.
 *
 *  Return value
 *      pointer to HCA FM function test structure registered.
 *
 *  Caller's context
 *      i_hca_test_register() can be called in user, kernel or interrupt
 *      context.
 *
 */
static struct i_hca_fm_test *
i_hca_test_register(char *filename, int linenum, int type,
    void (*pio_injection)(struct i_hca_fm_test *, ddi_fm_error_t *),
    void *private, mod_hash_t *strHash, mod_hash_t *idHash, int preTestNum)
{
        struct i_hca_fm_test *t_item;
        char key_buf[255], *hash_key;
        int status;

        (void) sprintf(key_buf, "%s:%d", filename, linenum);
        hash_key = kmem_zalloc(strlen(key_buf) + 1, KM_NOSLEEP);

        if (hash_key == NULL)
                cmn_err(CE_PANIC, "No memory for HCA FMA Test.");

        bcopy(key_buf, hash_key, strlen(key_buf));

        status = mod_hash_find(strHash, (mod_hash_key_t)hash_key,
            (mod_hash_val_t *)&t_item);

        switch (status) {
        case MH_ERR_NOTFOUND:
                t_item = (struct i_hca_fm_test *)
                    kmem_alloc(sizeof (struct i_hca_fm_test), KM_NOSLEEP);
                if (t_item == NULL)
                        cmn_err(CE_PANIC, "No memory for HCA FMA Test.");

                /* Set the error number */
                mutex_enter(&i_hca_test_lock);
                t_item->num = test_num++;
                mutex_exit(&i_hca_test_lock);

                /* Set type and other static information */
                t_item->type = type;
                t_item->line_num = linenum;
                t_item->file_name = filename;
                t_item->hash_key = hash_key;
                t_item->private = private;
                t_item->pio_injection = pio_injection;

                /* Set the pre-defined hermon test item */
                i_hca_test_set_item(preTestNum, (struct i_hca_fm_test *)t_item);

                status = mod_hash_insert(strHash, (mod_hash_key_t)
                    hash_key, (mod_hash_val_t)t_item);
                ASSERT(status == 0);

                status = mod_hash_insert(idHash, (mod_hash_key_t)
                    (uintptr_t)t_item->num, (mod_hash_val_t)t_item);
                ASSERT(status == 0);
                break;

        case MH_ERR_NOMEM:
                cmn_err(CE_PANIC, "No memory for HCA FMA Test.");
                break;

        case MH_ERR_DUPLICATE:
                cmn_err(CE_PANIC, "HCA FMA Test Internal Error.");
                break;
        default:
                /* OK, this is already registered. */
                kmem_free(hash_key, strlen(key_buf) + 1);
                break;
        }
        return (t_item);
}


/*
 *  void
 *  i_hca_test_set_item(int num, struct i_hca_fm_test *t_item)
 *
 *  Overview
 *      i_hca_test_set_item() is a private function used in
 *      i_hca_test_register() above. This function sets the testset specified
 *      (with the index number) to HCA FM function test structure.
 *
 *  Argument
 *      num: index to test set (testset structure array)
 *      t_item: pointer to HCA fM function test structure
 *
 *  Return value
 *      Nothing
 *
 *  Caller's context
 *      i_hca_test_set_item() can be called in user, kernel, interrupt
 *      context or hight interrupt context.
 *
 */
static void
i_hca_test_set_item(int num, struct i_hca_fm_test *t_item)
{
        if (num < 0 || num >= sizeof (testset) / sizeof (hermon_test_t) ||
            testset[num].type != t_item->type) {
                t_item->trigger = testset[0].trigger;
                t_item->errcnt = testset[0].errcnt;
                return;
        }

        /* Set the testsuite */
        t_item->trigger = testset[num].trigger;
        t_item->errcnt = testset[num].errcnt;
}


/*
 *  void
 *  i_hca_test_free_item(mod_hash_val_t val)
 *
 *  Overview
 *      i_hca_test_free_item() is a private function used to free HCA FM
 *      function test structure when i_hca_test_fini() is called. This function
 *      is registered as a destructor when the hash table is created in
 *      i_hca_test_init().
 *
 *  Argument
 *      val: pointer to the value stored in hash table (pointer to HCA FM
 *           function test structure)
 *
 *  Return value
 *      Nothing
 *
 *  Caller's context
 *      i_hca_test_free_item() can be called in user, kernel or interrupt
 *      context.
 *
 */
static void
i_hca_test_free_item(mod_hash_val_t val)
{
        struct i_hca_fm_test *t_item = (struct i_hca_fm_test *)val;
        kmem_free(t_item, sizeof (struct i_hca_fm_test));
}
#endif /* FMA_TEST */