docs/how-to-build.md: use proper markup for directory names

[unleashed/tickless.git] / include / sys / ib / adapters / hermon / hermon.h

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
 */

#ifndef _SYS_IB_ADAPTERS_HERMON_H
#define _SYS_IB_ADAPTERS_HERMON_H

/*
 * hermon.h
 *    Contains the #defines and typedefs necessary for the Hermon softstate
 *    structure and for proper attach() and detach() processing.  Also
 *    includes all the other Hermon header files (and so is the only header
 *    file that is directly included by the Hermon source files).
 *    Lastly, this file includes everything necessary for implementing the
 *    devmap interface and for maintaining the "mapped resource database".
 */

#include <sys/types.h>
#include <sys/conf.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/tnf_probe.h>
#include <sys/taskq.h>
#include <sys/atomic.h>
#ifdef FMA_TEST
#include <sys/modhash.h>
#endif

#include <sys/ib/ibtl/ibci.h>
#include <sys/ib/ibtl/impl/ibtl_util.h>
#include <sys/ib/adapters/mlnx_umap.h>

/*
 * First include all the Hermon typedefs, then include all the other Hermon
 * specific headers (many of which depend on the typedefs having already
 * been defined).
 */
#include <sys/ib/adapters/hermon/hermon_typedef.h>
#include <sys/ib/adapters/hermon/hermon_hw.h>

#include <sys/ib/adapters/hermon/hermon_agents.h>
#include <sys/ib/adapters/hermon/hermon_cfg.h>
#include <sys/ib/adapters/hermon/hermon_cmd.h>
#include <sys/ib/adapters/hermon/hermon_cq.h>
#include <sys/ib/adapters/hermon/hermon_event.h>
#include <sys/ib/adapters/hermon/hermon_fcoib.h>
#include <sys/ib/adapters/hermon/hermon_ioctl.h>
#include <sys/ib/adapters/hermon/hermon_misc.h>
#include <sys/ib/adapters/hermon/hermon_mr.h>
#include <sys/ib/adapters/hermon/hermon_wr.h>
#include <sys/ib/adapters/hermon/hermon_qp.h>
#include <sys/ib/adapters/hermon/hermon_srq.h>
#include <sys/ib/adapters/hermon/hermon_rsrc.h>
#include <sys/ib/adapters/hermon/hermon_fm.h>

#ifdef __cplusplus
extern "C" {
#endif

/*
 * Number of initial states to setup. Used in call to ddi_soft_state_init()
 */
#define HERMON_INITIAL_STATES           3

/*
 * Macro and defines used to calculate device instance number from minor
 * number (and vice versa).
 */
#define HERMON_MINORNUM_SHIFT           3
#define HERMON_DEV_INSTANCE(dev)        (getminor((dev)) &      \
        ((1 << HERMON_MINORNUM_SHIFT) - 1))

/*
 * Locations for the various Hermon hardware CMD,UAR & MSIx PCIe BARs
 */
#define HERMON_CMD_BAR                  1 /* device config space */
#define HERMON_UAR_BAR                  2 /* UAR Region */
#define HERMON_MSIX_BAR                 3 /* MSI-X Table */

#define HERMON_ONCLOSE_FLASH_INPROGRESS         (1 << 0)

#define HERMON_MSIX_MAX                 256 /* max # of interrupt vectors */

/*
 * VPD header size - or more rightfully, the area of interest for fwflash
 *      There's more, but we don't need it for our use so we don't read it
 */
#define HERMON_VPD_HDR_DWSIZE           0x10 /* 16 Dwords */
#define HERMON_VPD_HDR_BSIZE            0x40 /* 64 Bytes */

/*
 * Offsets to be used w/ reset to save/restore PCI capability stuff
 */
#define HERMON_PCI_CAP_DEV_OFFS         0x08
#define HERMON_PCI_CAP_LNK_OFFS         0x10


/*
 * Some defines for the software reset.  These define the value that should
 * be written to begin the reset (HERMON_SW_RESET_START), the delay before
 * beginning to poll for completion (HERMON_SW_RESET_DELAY), the in-between
 * polling delay (HERMON_SW_RESET_POLL_DELAY), and the value that indicates
 * that the reset has not completed (HERMON_SW_RESET_NOTDONE).
 */
#define HERMON_SW_RESET_START           0x00000001
#define HERMON_SW_RESET_DELAY           1000000  /* 1000 ms, per 0.36 PRM */
#define HERMON_SW_RESET_POLL_DELAY      100      /* 100 us */
#define HERMON_SW_RESET_NOTDONE         0xFFFFFFFF

/*
 * These defines are used in the Hermon software reset operation.  They define
 * the total number PCI registers to read/restore during the reset.  And they
 * also specify two config registers which should not be read or restored.
 */
#define HERMON_SW_RESET_NUMREGS         0x40
#define HERMON_SW_RESET_REG22_RSVD      0x16    /* 22 dec */
#define HERMON_SW_RESET_REG23_RSVD      0x17    /* 23 dec */

/*
 * Macro used to output HCA warning messages.  Note: HCA warning messages
 * are only generated when an unexpected condition has been detected.  This
 * can be the result of a software bug or some other problem.  Previously
 * this was used for hardware errors, but those now use HERMON_FMANOTE
 * instead, indicating that the driver state is more likely in an
 * unpredictable state, and that shutdown/restart is suggested.
 *
 * HERMON_WARNING messages are not considered important enough to print
 * to the console, just to the message log.
 */
#define HERMON_WARNING(state, string)                                   \
        cmn_err(CE_CONT, "!hermon%d: %s\n", (state)->hs_instance, string)

/*
 * Macro used to set attach failure messages.  Also, the attach message buf
 * size is set here.
 */
#define HERMON_ATTACH_MSGSIZE   80
#define HERMON_ATTACH_MSG(attach_buf, attach_msg)               \
        (void) snprintf((attach_buf), HERMON_ATTACH_MSGSIZE, (attach_msg));
#define HERMON_ATTACH_MSG_INIT(attach_buf)                      \
        (attach_buf)[0] = '\0';

/*
 * Macros used for controlling whether or not event callbacks will be forwarded
 * to the IBTF.  This is necessary because there are certain race conditions
 * that can occur (e.g. calling IBTF with an asynch event before the IBTF
 * registration has successfully completed or handling an event after we've
 * detached from the IBTF.)
 *
 * HERMON_ENABLE_IBTF_CALLB() initializes the "hs_ibtfpriv" field in the Hermon
 *    softstate.  When "hs_ibtfpriv" is non-NULL, it is OK to forward asynch
 *    and CQ events to the IBTF.
 *
 * HERMON_DO_IBTF_ASYNC_CALLB() and HERMON_DO_IBTF_CQ_CALLB() both set and clear
 *    the "hs_in_evcallb" flag, as necessary, to indicate that an IBTF
 *    callback is currently in progress.  This is necessary so that we can
 *    block on this condition in hermon_detach().
 *
 * HERMON_QUIESCE_IBTF_CALLB() is used in hermon_detach() to set the
 *    "hs_ibtfpriv" to NULL (thereby disabling any further IBTF callbacks)
 *    and to poll on the "hs_in_evcallb" flag.  When this flag is zero, all
 *    IBTF callbacks have quiesced and it is safe to continue with detach
 *    (i.e. continue detaching from IBTF).
 */
#define HERMON_ENABLE_IBTF_CALLB(state, tmp_ibtfpriv)           \
        (state)->hs_ibtfpriv = (tmp_ibtfpriv);

#define HERMON_DO_IBTF_ASYNC_CALLB(state, type, event)  \
        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS((state)->hs_in_evcallb))   \
        (state)->hs_in_evcallb = 1;                                     \
        ibc_async_handler((state)->hs_ibtfpriv, (type), (event));       \
        (state)->hs_in_evcallb = 0;

#define HERMON_DO_IBTF_CQ_CALLB(state, cq)                      \
        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS((state)->hs_in_evcallb))   \
        (state)->hs_in_evcallb = 1;                                     \
        ibc_cq_handler((state)->hs_ibtfpriv, (cq)->cq_hdlrarg);         \
        (state)->hs_in_evcallb = 0;

#define HERMON_QUIESCE_IBTF_CALLB(state)                        \
{                                                                       \
        uint_t          count = 0;                                      \
                                                                        \
        state->hs_ibtfpriv = NULL;                                      \
        while (((state)->hs_in_evcallb != 0) &&                         \
            (count++ < HERMON_QUIESCE_IBTF_CALLB_POLL_MAX)) {           \
                drv_usecwait(HERMON_QUIESCE_IBTF_CALLB_POLL_DELAY);     \
        }                                                               \
}

/*
 * Defines used by the HERMON_QUIESCE_IBTF_CALLB() macro to determine the
 * duration and number of times (at maximum) to poll while waiting for IBTF
 * callbacks to quiesce.
 */
#define HERMON_QUIESCE_IBTF_CALLB_POLL_DELAY    1
#define HERMON_QUIESCE_IBTF_CALLB_POLL_MAX      1000000

/*
 * Macros to retrieve PCI id's of the device
 */
#define HERMON_DDI_PROP_GET(dip, property) \
        (ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, \
            property, -1))

#define HERMON_GET_VENDOR_ID(dip)       HERMON_DDI_PROP_GET(dip, "vendor-id")
#define HERMON_GET_DEVICE_ID(dip)       HERMON_DDI_PROP_GET(dip, "device-id")
#define HERMON_GET_REVISION_ID(dip)     HERMON_DDI_PROP_GET(dip, "revision-id")

/*
 * Defines used to record the device mode to which Hermon driver has been
 * attached.  HERMON_MAINTENANCE_MODE is used when the device has
 * come up in the "maintenance mode".  In this mode, no InfiniBand interfaces
 * are enabled, but the device's firmware can be updated/flashed (and
 * test/debug interfaces should be useable).
 * HERMON_HCA_MODE isused when the device has come up in the
 * normal HCA mode.  In this mode, all necessary InfiniBand interfaces are
 * enabled (and, if necessary, HERMON firmware can be updated/flashed).
 */
#define HERMON_MAINTENANCE_MODE 1
#define HERMON_HCA_MODE         2

/*
 * Used to determine if the device is operational, or not in maintenance mode.
 * This means either the driver has attached successfully against an hermon
 * device in hermon compatibility mode, or against a hermon device in full HCA
 * mode.
 */
#define HERMON_IS_OPERATIONAL(mode)                             \
        (mode == HERMON_HCA_MODE)

/*
 * The following define is used (in hermon_umap_db_set_onclose_cb()) to
 * indicate that a cleanup callback is needed to undo initialization done
 * by the firmware flash burn code.
 */
#define HERMON_ONCLOSE_FLASH_INPROGRESS         (1 << 0)

/*
 * The following enumerated type and structures are used during driver
 * initialization.  Note: The HERMON_DRV_CLEANUP_ALL type is used as a marker
 * for end of the cleanup steps.  No cleanup steps should be added after
 * HERMON_DRV_CLEANUP_ALL.  Any addition steps should be added before it.
 */
typedef enum {
        HERMON_DRV_CLEANUP_LEVEL0,
        HERMON_DRV_CLEANUP_LEVEL1,
        HERMON_DRV_CLEANUP_LEVEL2,
        HERMON_DRV_CLEANUP_LEVEL3,
        HERMON_DRV_CLEANUP_LEVEL4,
        HERMON_DRV_CLEANUP_LEVEL5,
        HERMON_DRV_CLEANUP_LEVEL6,
        HERMON_DRV_CLEANUP_LEVEL7,
        HERMON_DRV_CLEANUP_LEVEL8,
        HERMON_DRV_CLEANUP_LEVEL9,
        HERMON_DRV_CLEANUP_LEVEL10,
        HERMON_DRV_CLEANUP_LEVEL11,
        HERMON_DRV_CLEANUP_LEVEL12,
        HERMON_DRV_CLEANUP_LEVEL13,
        HERMON_DRV_CLEANUP_LEVEL14,
        HERMON_DRV_CLEANUP_LEVEL15,
        HERMON_DRV_CLEANUP_LEVEL16,
        HERMON_DRV_CLEANUP_LEVEL17,
        HERMON_DRV_CLEANUP_LEVEL18,
        HERMON_DRV_CLEANUP_LEVEL19,
        /* No more driver cleanup steps below this point! */
        HERMON_DRV_CLEANUP_ALL
} hermon_drv_cleanup_level_t;

/*
 * The hermon_dma_info_t structure is used to store information related to
 * the various ICM resources' DMA allocations.  The related ICM table and
 * virtual address are stored here.  The DMA and Access handles are stored
 * here.  Also, the allocation length and virtual (host) address.
 */
struct hermon_dma_info_s {
        ddi_dma_handle_t        dma_hdl;
        ddi_acc_handle_t        acc_hdl;
        uint64_t                icmaddr;        /* ICM virtual address */
        uint64_t                vaddr;          /* host virtual address */
        uint_t                  length;         /* length requested */
        uint_t                  icm_refcnt;     /* refcnt */
};
_NOTE(SCHEME_PROTECTS_DATA("safe sharing",
    hermon_dma_info_s::icm_refcnt))


/*
 * The hermon_cmd_reg_t structure is used to hold the address of the each of
 * the most frequently accessed hardware registers.  Specifically, it holds
 * the HCA Command Registers (HCR, used to pass command and mailbox
 * information back and forth to Hermon firmware) and the lock used to guarantee
 * mutually exclusive access to the registers.
 * Related to this, is the "clr_int" register which is used to clear the
 * interrupt once all EQs have been serviced.
 * Finally, there is the software reset register which is used to reinitialize
 * the Hermon device and to put it into a known state at driver startup time.
 * Below we also have the offsets (into the CMD register space) for each of
 * the various registers.
 */
typedef struct hermon_cmd_reg_s {
        hermon_hw_hcr_t *hcr;
        kmutex_t        hcr_lock;
        uint64_t        *clr_intr;
        uint64_t        *eq_arm;
        uint64_t        *eq_set_ci;
        uint32_t        *sw_reset;
        uint32_t        *sw_semaphore;
        uint32_t        *fw_err_buf;
} hermon_cmd_reg_t;
_NOTE(MUTEX_PROTECTS_DATA(hermon_cmd_reg_t::hcr_lock,
    hermon_cmd_reg_t::hcr))

/* SOME TEMPORARY PRINTING THINGS */
#define HERMON_PRINT_CI         (0x01 << 0)
#define HERMON_PRINT_MEM        (0x01 << 1)
#define HERMON_PRINT_CQ         (0x01 << 2)


#define HD_PRINT(state, mask)   \
        if (state->hs_debug_lev & mask)

/* END PRINTING THINGS */

/*
 * The hermon_state_t structure is the HCA software state structure.  It
 * contains all the pointers and placeholder for everything that the HCA
 * driver needs to properly operate.  One of these structures exists for
 * every instance of the HCA driver.
 */
struct hermon_state_s {
        dev_info_t              *hs_dip;
        int                     hs_instance;

        /* PCI device, vendor, and revision IDs */
        uint16_t                hs_vendor_id;
        uint16_t                hs_device_id;
        uint8_t                 hs_revision_id;

        /*
         * DMA information for the InfiniHost Context Memory (ICM),
         * ICM Auxiliary allocation and the firmware. Also, record
         * of ICM and ICMA sizes, in bytes.
         */

        uint64_t                hs_icm_sz;
        hermon_icm_table_t      *hs_icm;
        uint64_t                hs_icma_sz;
        hermon_dma_info_t       hs_icma_dma;
        hermon_dma_info_t       hs_fw_dma;

        /* Hermon interrupt/MSI information */
        int                     hs_intr_types_avail;
        uint_t                  hs_intr_type_chosen;
        int                     hs_intrmsi_count;
        int                     hs_intrmsi_avail;
        int                     hs_intrmsi_allocd;
        ddi_intr_handle_t       hs_intrmsi_hdl[HERMON_MSIX_MAX];
        uint_t                  hs_intrmsi_pri;
        int                     hs_intrmsi_cap;
        ddi_cb_handle_t         hs_intr_cb_hdl;

        /* Do not use reserved EQs */
        uint_t                  hs_rsvd_eqs;
        uint_t                  hs_cq_erreqnum;

        /* cq_sched data */
        kmutex_t                hs_cq_sched_lock;
        hermon_cq_sched_t       *hs_cq_sched_array;
        hermon_cq_sched_t       hs_cq_sched_default;
        uint_t                  hs_cq_sched_array_size;

        /* hermon HCA name and HCA part number */
        char                    hs_hca_name[64];
        char                    hs_hca_pn[64];
        int                     hs_hca_pn_len;

        /* Hermon device operational mode */
        int                     hs_operational_mode;

        /* Attach buffer saved per state to store detailed attach errors */
        char                    hs_attach_buf[HERMON_ATTACH_MSGSIZE];

        /* Hermon NodeGUID, SystemImageGUID, and NodeDescription */
        uint64_t                hs_nodeguid;
        uint64_t                hs_sysimgguid;
        char                    hs_nodedesc[64];

        /* Info passed to IBTF during registration */
        ibc_hca_info_t          hs_ibtfinfo;
        ibc_clnt_hdl_t          hs_ibtfpriv;

        /*
         * Hermon register mapping.  Holds the device access attributes,
         * kernel mapped addresses, and DDI access handles for both
         * Hermon's CMD and UAR BARs.
         */
        ddi_device_acc_attr_t   hs_reg_accattr;
        caddr_t                 hs_reg_cmd_baseaddr;    /* Hermon CMD BAR */
        ddi_acc_handle_t        hs_reg_cmdhdl;
        caddr_t                 hs_reg_uar_baseaddr;    /* Hermon UAR BAR */
        ddi_acc_handle_t        hs_reg_uarhdl;
        caddr_t                 hs_reg_msi_baseaddr;    /* Hermon MSIx BAR */
        ddi_acc_handle_t        hs_reg_msihdl;

        /*
         * Some additional things for UAR Pages
         */
        uint64_t                hs_kernel_uar_index;    /* kernel UAR index */
        uint64_t                hs_bf_offset;           /* offset from UAR */
                                                        /* Bar to Blueflame */
        caddr_t                 hs_reg_bf_baseaddr;     /* blueflame base */
        ddi_acc_handle_t        hs_reg_bfhdl;           /* blueflame handle */


        /*
         * Hermon PCI config space registers.  This array is used to
         * save and restore the PCI config registers before and after a
         * software reset.
         */
        uint32_t                hs_cfg_data[HERMON_SW_RESET_NUMREGS];
        /* for reset per Linux driver */
        uint32_t                hs_pci_cap_offset;
        uint32_t                hs_pci_cap_devctl;
        uint32_t                hs_pci_cap_lnkctl;

        /*
         * Hermon UAR page resources.  Holds the resource pointers for
         * UAR page #0 (reserved) and for UAR page #1 (used for kernel
         * driver doorbells).  In addition, we save a pointer to the
         * UAR page #1 doorbells which will be used throughout the driver
         * whenever it is necessary to ring one of them.  And, in case we
         * are unable to do 64-bit writes to the page (because of system
         * architecture), we include a lock (to ensure atomic 64-bit access).
         */
        hermon_rsrc_t           *hs_uarpg0_rsrc_rsrvd;
        hermon_rsrc_t           *hs_uarkpg_rsrc;
        hermon_hw_uar_t         *hs_uar;
        kmutex_t                hs_uar_lock;

        /*
         * Used during a call to open() if we are in maintenance mode, this
         * field serves as a semi-unique rolling count index value, used only
         * in the setup of umap_db entries.  This is primarily needed to
         * firmware device access ioctl operations can still be guaranteed to
         * close in the event of an unplanned process exit, even in maintenance
         * mode.
         */
        uint_t                  hs_open_ar_indx;

        /*
         * Hermon command registers.  This structure contains the addresses
         * for each of the most frequently accessed CMD registers.  Since
         * almost all accesses to the Hermon hardware are through the Hermon
         * command interface (i.e. the HCR), we save away the pointer to
         * the HCR, as well as pointers to the ECR and INT registers (as
         * well as their corresponding "clear" registers) for interrupt
         * processing.  And we also save away a pointer to the software
         * reset register (see above).
         */
        hermon_cmd_reg_t        hs_cmd_regs;
        uint32_t                hs_cmd_toggle;

        /*
         * Hermon resource pointers.  The following are pointers to the
         * kmem cache (from which the Hermon resource handles are allocated),
         * and the array of "resource pools" (which store all the pertinent
         * information necessary to manage each of the various types of
         * resources that are used by the driver.  See hermon_rsrc.h for
         * more detail.
         */
        kmem_cache_t            *hs_rsrc_cache;
        hermon_rsrc_pool_info_t *hs_rsrc_hdl;

        /*
         * Hermon mailbox lists.  These hold the information necessary to
         * manage the pools of pre-allocated Hermon mailboxes (both "In" and
         * "Out" type).  See hermon_cmd.h for more detail.
         */
        hermon_mboxlist_t       hs_in_mblist;
        hermon_mboxlist_t       hs_out_mblist;

        /*
         * Hermon interrupt mailbox lists.  We allocate both an "In" mailbox
         * and an "Out" type mailbox for the interrupt context.  This is in
         * order to guarantee that a mailbox entry will always be available in
         * the interrupt context, and we can NOSLEEP without having to worry
         * about possible failure allocating the mbox.  We create this as an
         * mboxlist so that we have the potential for having multiple mboxes
         * available based on the number of interrupts we can receive at once.
         */
        hermon_mboxlist_t       hs_in_intr_mblist;
        hermon_mboxlist_t       hs_out_intr_mblist;

        /*
         * Hermon outstanding command list.  Used to hold all the information
         * necessary to manage the Hermon "outstanding command list".  See
         * hermon_cmd.h for more detail.
         */
        hermon_cmdlist_t        hs_cmd_list;

        /*
         * This structure contains the Hermon driver's "configuration profile".
         * This is the collected set of configuration information, such as
         * number of QPs, CQs, mailboxes and other resources, sizes of
         * individual resources, other system level configuration information,
         * etc.  See hermon_cfg.h for more detail.
         */
        hermon_cfg_profile_t    *hs_cfg_profile;

        /*
         * This flag contains the profile setting, selecting which profile the
         * driver would use.  This is needed in the case where we have to
         * fallback to a smaller profile based on some DDR conditions.  If we
         * don't fallback, then it is set to the size of DDR in the system.
         */
        uint32_t                hs_cfg_profile_setting;

        /*
         * The following are a collection of resource handles used by the
         * Hermon driver (internally).  First is the protection domain (PD)
         * handle that is used when mapping all kernel memory (work queues,
         * completion queues, etc).  Next is an array of EQ handles.  This
         * array is indexed by EQ number and allows the Hermon driver to quickly
         * convert an EQ number into the software structure associated with the
         * given EQ.  Likewise, we have three arrays for CQ, QP and SRQ
         * handles.  These arrays are also indexed by CQ, QP or SRQ number and
         * allow the driver to quickly find the corresponding CQ, QP or SRQ
         * software structure.  Note: while the EQ table is of fixed size
         * (because there are a maximum of 64 EQs), each of the CQ, QP and SRQ
         * handle lists must be allocated at driver startup.
         */
        hermon_pdhdl_t          hs_pdhdl_internal;
        hermon_eqhdl_t          hs_eqhdl[HERMON_NUM_EQ];
        kmutex_t                hs_dbr_lock;    /* lock for dbr mgmt */

        /* linked list of kernel dbr resources */
        hermon_dbr_info_t       *hs_kern_dbr;

        /* linked list of non-kernel dbr resources */
        hermon_user_dbr_t       *hs_user_dbr;

        /*
         * The AVL tree is used to store information regarding QP number
         * allocations.  The lock protects access to the AVL tree.
         */
        avl_tree_t              hs_qpn_avl;
        kmutex_t                hs_qpn_avl_lock;

        /*
         * This field is used to indicate whether or not the Hermon driver is
         * currently in an IBTF event callback elsewhere in the system.  Note:
         * It is "volatile" because we intend to poll on this value - in
         * hermon_detach() - until we are assured that no further IBTF callbacks
         * are currently being processed.
         */
        volatile uint32_t       hs_in_evcallb;

        /*
         * The following structures are used to store the results of several
         * device query commands passed to the Hermon hardware at startup.
         * Specifically, we have hung onto the results of QUERY_DDR (which
         * gives information about how much DDR memory is present and where
         * it is located), QUERY_FW (which gives information about firmware
         * version numbers and the location and extent of firmware's footprint
         * in DDR, QUERY_DEVLIM (which gives the device limitations/resource
         * maximums) and QUERY_PORT (where some of the specs from DEVLIM moved),
         * QUERY_ADAPTER (which gives additional miscellaneous
         * information), and INIT/QUERY_HCA (which serves the purpose of
         * recording what configuration information was passed to the firmware
         * when the HCA was initialized).
         */
        struct hermon_hw_queryfw_s      hs_fw;
        struct hermon_hw_querydevlim_s  hs_devlim;
        struct hermon_hw_query_port_s   hs_queryport;
        struct hermon_hw_set_port_s     *hs_initport;
        struct hermon_hw_queryadapter_s hs_adapter;
        struct hermon_hw_initqueryhca_s hs_hcaparams;

        /*
         * The following are used for managing special QP resources.
         * Specifically, we have a lock, a set of flags (in "hs_spec_qpflags")
         * used to track the special QP resources, and two Hermon resource
         * handle pointers.  Each resource handle actually corresponds to two
         * consecutive QP contexts (one per port) for each special QP type.
         */
        kmutex_t                hs_spec_qplock;
        uint_t                  hs_spec_qpflags;
        hermon_rsrc_t           *hs_spec_qp0;
        hermon_rsrc_t           *hs_spec_qp1;
        /*
         * For Hermon, you have to alloc 8 qp's total, but the last 4 are
         * unused/reserved.  The following represents the handle for those
         * last 4 qp's
         */
        hermon_rsrc_t           *hs_spec_qp_unused;

        /*
         * Related in some ways to the special QP handling above are these
         * resources which are used specifically for implementing the Hermon
         * agents (SMA, PMA, and BMA).  Although, each of these agents does
         * little more that intercept the appropriate incoming MAD and forward
         * it along to the firmware (see hermon_agents.c for more details), we
         * do still use a task queue to queue them up.  We can also configure
         * the driver to force firmware handling for certain classes of MAD,
         * and, therefore, we require the agent list and number of agents
         * in order to know what needs to be torn down at detach() time.
         */
        hermon_agent_list_t     *hs_agents;
        ddi_taskq_t             *hs_taskq_agents;
        uint_t                  hs_num_agents;

        /*
         * Multicast group lists.  These are used to track the "shadow" MCG
         * lists that speed up the processing of attach and detach multicast
         * group operations.  See hermon_misc.h for more details.  Note: we
         * need the pointer to the "temporary" MCG entry here primarily
         * because the size of a given MCG entry is configurable.  Therefore,
         * it is impossible to put this variable on the stack.  And rather
         * than allocate and deallocate the entry multiple times, we choose
         * instead to preallocate it once and reuse it over and over again.
         */
        kmutex_t                hs_mcglock;
        hermon_mcghdl_t         hs_mcghdl;
        hermon_hw_mcg_t         *hs_mcgtmp;

        /*
         * Cache of the pkey table, sgid (guid-only) tables, and
         * sgid (subnet) prefix.  These arrays are set
         * during port_query, and mainly used for generating MLX GSI wqes.
         */
        ib_pkey_t               *hs_pkey[HERMON_MAX_PORTS];
        ib_sn_prefix_t          hs_sn_prefix[HERMON_MAX_PORTS];
        ib_guid_t               *hs_guid[HERMON_MAX_PORTS];

        /*
         * Used for tracking Hermon kstat information
         */
        hermon_ks_info_t        *hs_ks_info;

        /*
         * Used for Hermon info ioctl used by VTS
         */
        kmutex_t                hs_info_lock;

        /*
         * Used for Hermon FW flash burning.  They are used exclusively
         * within the ioctl calls for use when accessing the hermon
         * flash device.
         */
        kmutex_t                hs_fw_flashlock;
        int                     hs_fw_flashstarted;
        dev_t                   hs_fw_flashdev;
        uint32_t                hs_fw_log_sector_sz;
        uint32_t                hs_fw_device_sz;
        uint32_t                hs_fw_flashbank;
        uint32_t                *hs_fw_sector;
        uint32_t                hs_fw_gpio[4];
        int                     hs_fw_cmdset;

        /*
         * Used for Hermon FM. They are basically used to manage
         * the toggle switch to enable/disable Hermon FM.
         * Please see the comment in hermon_fm.c.
         */
        int                     hs_fm_capabilities; /* FM capabilities */
        int                     hs_fm_disable;  /* Hermon FM disable flag */
        int                     hs_fm_state;    /* Hermon FM state */
        boolean_t               hs_fm_async_fatal; /* async internal error */
        uint32_t                hs_fm_async_errcnt; /* async error count */
        boolean_t               hs_fm_poll_suspend; /* poll thread suspend */
        kmutex_t                hs_fm_lock;     /* mutex for state */
        hermon_hca_fm_t         *hs_fm_hca_fm;  /* HCA FM pointer */
        ddi_acc_handle_t        hs_fm_cmdhdl;   /* fm-protected CMD hdl */
        ddi_acc_handle_t        hs_fm_uarhdl;   /* fm-protected UAR hdl */
        ddi_device_acc_attr_t   hs_fm_accattr;  /* fm-protected acc attr */
        ddi_periodic_t          hs_fm_poll_thread; /* fma poll thread */
        int32_t                 hs_fm_degraded_reason;  /* degradation cause */
#ifdef FMA_TEST
        mod_hash_t              *hs_fm_test_hash; /* testset */
        mod_hash_t              *hs_fm_id_hash; /* testid */
#endif
        /* FCoIB data */
        hermon_fcoib_t          hs_fcoib;
        boolean_t               hs_fcoib_may_be_running; /* cq_poll test */

        /*
         * Hermon fastreboot support. To sw-reset Hermon HCA, the driver
         * needs to save/restore MSI-X tables and PBA. Those members are
         * used for the purpose.
         */
        /* Access handle for PCI config space */
        ddi_acc_handle_t        hs_reg_pcihdl;          /* PCI cfg handle */
        ddi_acc_handle_t        hs_fm_pcihdl;           /*      fm handle */
        ushort_t                hs_caps_ptr;            /* MSI-X caps */
        ushort_t                hs_msix_ctrl;           /* MSI-X ctrl */

        /* members to handle MSI-X tables */
        ddi_acc_handle_t        hs_reg_msix_tblhdl;     /* MSI-X table handle */
        ddi_acc_handle_t        hs_fm_msix_tblhdl;      /*      fm handle */
        char                    *hs_msix_tbl_addr;      /* MSI-X table addr */
        char                    *hs_msix_tbl_entries;   /* MSI-X table entry */
        size_t                  hs_msix_tbl_size;       /* MSI-X table size */
        uint32_t                hs_msix_tbl_offset;     /* MSI-X table offset */
        uint32_t                hs_msix_tbl_rnumber;    /* MSI-X table reg# */

        /* members to handle MSI-X PBA */
        ddi_acc_handle_t        hs_reg_msix_pbahdl;     /* MSI-X PBA handle */
        ddi_acc_handle_t        hs_fm_msix_pbahdl;      /*      fm handle */
        char                    *hs_msix_pba_addr;      /* MSI-X PBA addr */
        char                    *hs_msix_pba_entries;   /* MSI-X PBA entry */
        size_t                  hs_msix_pba_size;       /* MSI-X PBA size */
        uint32_t                hs_msix_pba_offset;     /* MSI-X PBA offset */
        uint32_t                hs_msix_pba_rnumber;    /* MSI-X PBA reg# */

        boolean_t               hs_quiescing;           /* in fastreboot */
};
_NOTE(MUTEX_PROTECTS_DATA(hermon_state_s::hs_fw_flashlock,
    hermon_state_s::hs_fw_flashstarted
    hermon_state_s::hs_fw_flashdev
    hermon_state_s::hs_fw_log_sector_sz
    hermon_state_s::hs_fw_device_sz))
_NOTE(MUTEX_PROTECTS_DATA(hermon_state_s::hs_spec_qplock,
    hermon_state_s::hs_spec_qpflags
    hermon_state_s::hs_spec_qp0
    hermon_state_s::hs_spec_qp1))
_NOTE(MUTEX_PROTECTS_DATA(hermon_state_s::hs_mcglock,
    hermon_state_s::hs_mcghdl
    hermon_state_s::hs_mcgtmp))
_NOTE(DATA_READABLE_WITHOUT_LOCK(hermon_state_s::hs_in_evcallb
    hermon_state_s::hs_fw_log_sector_sz
    hermon_state_s::hs_fw_device_sz
    hermon_state_s::hs_spec_qpflags
    hermon_state_s::hs_spec_qp0
    hermon_state_s::hs_spec_qp1))
_NOTE(MUTEX_PROTECTS_DATA(hermon_state_s::hs_qpn_avl_lock,
    hermon_state_s::hs_qpn_avl))
_NOTE(SCHEME_PROTECTS_DATA("safe sharing",
    hermon_state_s::hs_fm_async_fatal
    hermon_state_s::hs_fw_sector))

/*
 * HERMON_IN_FASTREBOOT() shows if Hermon driver is at fastreboot.
 * This macro should be used to check if the mutex lock can be used
 * since the lock cannot be used if the driver is in the quiesce mode.
 */
#define HERMON_IN_FASTREBOOT(state)     (state->hs_quiescing == B_TRUE)

/*
 * Bit positions in the "hs_spec_qpflags" field above.  The flags are (from
 * least significant to most): (QP0,Port1), (QP0,Port2), (QP1,Port1), and
 * (QP1,Port2).  The masks are there to help with some specific allocation
 * and freeing operations
 */
#define HERMON_SPECIAL_QP0_RSRC         0
#define HERMON_SPECIAL_QP0_RSRC_MASK    0x3
#define HERMON_SPECIAL_QP1_RSRC         2
#define HERMON_SPECIAL_QP1_RSRC_MASK    0xC


/*
 * These flags specifies additional behaviors on database access.
 * HERMON_UMAP_DB_REMOVE, for example, specifies that (if found) the database
 * entry should be removed from the database.  HERMON_UMAP_DB_IGNORE_INSTANCE
 * specifies that a particular database query should ignore value in the
 * "tdb_instance" field as a criterion for the search.
 */
#define HERMON_UMAP_DB_REMOVE           (1 << 0)
#define HERMON_UMAP_DB_IGNORE_INSTANCE  (1 << 1)

/*
 * The hermon_umap_db_t structure contains what is referred to throughout the
 * driver code as the "userland resources database".  This structure contains
 * all the necessary information to track resources that have been prepared
 * for direct-from-userland access.  There is an AVL tree ("hdl_umapdb_avl")
 * which consists of the "hermon_umap_db_entry_t" (below) and a lock to ensure
 * atomic access when adding or removing entries from the database.
 */
typedef struct hermon_umap_db_s {
        kmutex_t                hdl_umapdb_lock;
        avl_tree_t              hdl_umapdb_avl;
} hermon_umap_db_t;

/*
 * The hermon_umap_db_priv_t structure currently contains information necessary
 * to provide the "on close" callback to the firmware flash interfaces.  It
 * is intended that this structure could be extended to enable other "on
 * close" callbacks as well.
 */
typedef struct hermon_umap_db_priv_s {
        int             (*hdp_cb)(void *);
        void            *hdp_arg;
} hermon_umap_db_priv_t;

/*
 * The hermon_umap_db_common_t structure contains fields which are common
 * between the database entries ("hermon_umap_db_entry_t") and the structure
 * used to contain the search criteria ("hermon_umap_db_query_t").  This
 * structure contains a key, a resource type (described above), an instance
 * (corresponding to the driver instance which inserted the database entry),
 * and a "value" field.  Typically, "hdb_value" is a pointer to a Hermon
 * resource object.  Although for memory regions, the value field corresponds
 * to the ddi_umem_cookie_t for the pinned userland memory.
 * The structure also includes a placeholder for private data ("hdb_priv").
 * Currently this data is being used for holding "on close" callback
 * information to allow certain kinds of cleanup even if a userland process
 * prematurely exits.
 */
typedef struct hermon_umap_db_common_s {
        uint64_t                hdb_key;
        uint64_t                hdb_value;
        uint_t                  hdb_type;
        uint_t                  hdb_instance;
        void                    *hdb_priv;
} hermon_umap_db_common_t;

/*
 * The hermon_umap_db_entry_t structure is the entry in "userland resources
 * database".  As required by the AVL framework, each entry contains an
 * "avl_node_t".  Then, as required to implement the database, each entry
 * contains a "hermon_umap_db_common_t" structure used to contain all of the
 * relevant entries.
 */
typedef struct hermon_umap_db_entry_s {
        avl_node_t              hdbe_avlnode;
        hermon_umap_db_common_t hdbe_common;
} hermon_umap_db_entry_t;

/*
 * The hermon_umap_db_query_t structure is used in queries to the "userland
 * resources database".  In addition to the "hermon_umap_db_common_t" structure
 * used to contain the various search criteria, this structure also contains
 * a flags field "hqdb_flags" which can be used to specify additional behaviors
 * (as described above).  Specifically, the flags field can be used to specify
 * that an entry should be removed from the database, if found, and to
 * specify whether the database lookup should consider "tdb_instance" in the
 * search.
 */
typedef struct hermon_umap_db_query_s {
        uint_t                  hqdb_flags;
        hermon_umap_db_common_t hqdb_common;
} hermon_umap_db_query_t;
_NOTE(MUTEX_PROTECTS_DATA(hermon_umap_db_s::hdl_umapdb_lock,
    hermon_umap_db_entry_s::hdbe_avlnode
    hermon_umap_db_entry_s::hdbe_common.hdb_key
    hermon_umap_db_entry_s::hdbe_common.hdb_value
    hermon_umap_db_entry_s::hdbe_common.hdb_type
    hermon_umap_db_entry_s::hdbe_common.hdb_instance))

/*
 * The hermon_devmap_track_t structure contains all the necessary information
 * to track resources that have been mapped through devmap.  There is a
 * back-pointer to the Hermon softstate, the logical offset corresponding with
 * the mapped resource, the size of the mapped resource (zero indicates an
 * "invalid mapping"), and a reference count and lock used to determine when
 * to free the structure (specifically, this is necessary to handle partial
 * unmappings).
 */
typedef struct hermon_devmap_track_s {
        hermon_state_t  *hdt_state;
        uint64_t        hdt_offset;
        uint_t          hdt_size;
        int             hdt_refcnt;
        kmutex_t        hdt_lock;
} hermon_devmap_track_t;

#define HERMON_ICM_SPLIT        64
#define HERMON_ICM_SPAN         4096

#define hermon_bitmap(bitmap, dma_info, icm_table, split_index, num_to_hdl) \
        bitmap = (icm_table)->icm_bitmap[split_index];          \
        if (bitmap == NULL) {                                   \
                _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*(icm_table))) \
                int num_spans = (icm_table)->num_spans;         \
                bitmap =                                        \
                (icm_table)->icm_bitmap[split_index] =          \
                    kmem_zalloc((num_spans + 7) / 8, KM_SLEEP); \
                ASSERT((icm_table)->icm_dma[split_index] == NULL); \
                (icm_table)->icm_dma[split_index] =             \
                    kmem_zalloc(num_spans * sizeof (hermon_dma_info_t), \
                    KM_SLEEP);                                  \
                if (num_to_hdl) {                               \
                        ASSERT((icm_table)->num_to_hdl[split_index] == NULL); \
                        (icm_table)->num_to_hdl[split_index] =  \
                            kmem_zalloc(num_spans *             \
                            sizeof (void **), KM_SLEEP);        \
                }                                               \
        }                                                       \
        dma_info = (icm_table)->icm_dma[split_index]

/*
 * The hermon_icm_table_t encodes data pertaining to a given ICM table, and
 * holds an array of hermon_dma_info_t's related to its backing memory. Each
 * ICM table is sized during initialization, but real memory is allocated
 * and mapped into and out of ICM in the device throughout the life of the
 * instance. We use a bitmap to determine whether or not a given ICM object
 * has memory backing it or not, and an array of hermon_dma_info_t's to house
 * the actual allocations. Memory is allocated in chunks of span_size, stored
 * in the icm_dma array, and can later be looked up by using the bitmap index.
 * The total number of ICM spans is equal to table_size / span_size. We also
 * keep track of the ICM characteristics, such as ICM object size and the
 * number of entries in the ICM area.
 */
struct hermon_icm_table_s {
        kmutex_t                icm_table_lock;
        kcondvar_t              icm_table_cv;
        uint8_t                 icm_busy;
        hermon_rsrc_type_t      icm_type;
        uint64_t                icm_baseaddr;
        uint64_t                table_size;
        uint64_t                num_entries;    /* maximum #entries */
        uint32_t                object_size;
        uint32_t                span;           /* #rsrc's per span */
        uint32_t                num_spans;      /* #dmainfos in icm_dma */
        uint32_t                split_shift;
        uint32_t                span_mask;
        uint32_t                span_shift;
        uint32_t                rsrc_mask;
        uint16_t                log_num_entries;
        uint16_t                log_object_size;
        /* three arrays of pointers, each pointer points to arrays */
        uint8_t                 *icm_bitmap[HERMON_ICM_SPLIT];
        hermon_dma_info_t       *icm_dma[HERMON_ICM_SPLIT];
        void                    ***num_to_hdl[HERMON_ICM_SPLIT]; /* qp/cq/srq */
};
/*
 * Split the rsrc index into three pieces:
 *
 *      index1 - icm_bitmap[HERMON_ICM_SPLIT], icm_dma[HERMON_ICM_SPLIT]
 *      index2 - bitmap[], dma[]
 *      offset - rsrc within the icm mapping
 */
#define hermon_index(index1, index2, rindx, table, offset)              \
        index1 = (rindx) >> table->split_shift;                         \
        index2 = ((rindx) & table->span_mask) >> table->span_shift;     \
        offset = (rindx) & table->rsrc_mask

/* Defined in hermon.c */
int hermon_dma_alloc(hermon_state_t *state, hermon_dma_info_t *dma_info,
    uint16_t opcode);
void hermon_dma_attr_init(hermon_state_t *state, ddi_dma_attr_t *dma_attr);
void hermon_dma_free(hermon_dma_info_t *info);
int hermon_icm_alloc(hermon_state_t *state, hermon_rsrc_type_t type,
    uint32_t icm_index1, uint32_t icm_index2);
void hermon_icm_free(hermon_state_t *state, hermon_rsrc_type_t type,
    uint32_t icm_index1, uint32_t icm_index2);
void *hermon_icm_num_to_hdl(hermon_state_t *state, hermon_rsrc_type_t type,
    uint32_t idx);
void hermon_icm_set_num_to_hdl(hermon_state_t *state, hermon_rsrc_type_t type,
    uint32_t idx, void *hdl);
int hermon_device_mode(hermon_state_t *state);

/* Defined in hermon_umap.c */
int hermon_devmap(dev_t dev, devmap_cookie_t dhp, offset_t off, size_t len,
    size_t *maplen, uint_t model);
ibt_status_t hermon_umap_ci_data_in(hermon_state_t *state,
    ibt_ci_data_flags_t flags, ibt_object_type_t object, void *hdl,
    void *data_p, size_t data_sz);
ibt_status_t hermon_umap_ci_data_out(hermon_state_t *state,
    ibt_ci_data_flags_t flags, ibt_object_type_t object, void *hdl,
    void *data_p, size_t data_sz);
void hermon_umap_db_init(void);
void hermon_umap_db_fini(void);
hermon_umap_db_entry_t *hermon_umap_db_alloc(uint_t instance, uint64_t key,
    uint_t type, uint64_t value);
void hermon_umap_db_free(hermon_umap_db_entry_t *umapdb);
void hermon_umap_db_add(hermon_umap_db_entry_t *umapdb);
void hermon_umap_db_add_nolock(hermon_umap_db_entry_t *umapdb);
int hermon_umap_db_find(uint_t instance, uint64_t key, uint_t type,
    uint64_t *value, uint_t flags, hermon_umap_db_entry_t **umapdb);
int hermon_umap_db_find_nolock(uint_t instance, uint64_t key, uint_t type,
    uint64_t *value, uint_t flags, hermon_umap_db_entry_t **umapdb);
void hermon_umap_umemlock_cb(ddi_umem_cookie_t *umem_cookie);
int hermon_umap_db_set_onclose_cb(dev_t dev, uint64_t flag,
    int (*callback)(void *), void *arg);
int hermon_umap_db_clear_onclose_cb(dev_t dev, uint64_t flag);
int hermon_umap_db_handle_onclose_cb(hermon_umap_db_priv_t *priv);
int hermon_rsrc_hw_entries_init(hermon_state_t *state,
    hermon_rsrc_hw_entry_info_t *info);
void hermon_rsrc_hw_entries_fini(hermon_state_t *state,
    hermon_rsrc_hw_entry_info_t *info);

#ifdef __cplusplus
}
#endif

#endif  /* _SYS_IB_ADAPTERS_HERMON_H */