include/sys/ib/adapters/tavor/tavor_cq.h

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21
  22 /*
  23  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
  24  * Use is subject to license terms.
  25  */
  26
  27 #ifndef _SYS_IB_ADAPTERS_TAVOR_CQ_H
  28 #define _SYS_IB_ADAPTERS_TAVOR_CQ_H
  29
  30 /*
  31  * tavor_cq.h
  32  *    Contains all of the prototypes, #defines, and structures necessary
  33  *    for the Completion Queue Processing routines.
  34  *    Specifically it contains the various completion types, flags,
  35  *    structures used for managing Tavor completion queues, and prototypes
  36  *    for many of the functions consumed by other parts of the Tavor driver
  37  *    (including those routines directly exposed through the IBTF CI
  38  *    interface).
  39  */
  40
  41 #include <sys/types.h>
  42 #include <sys/conf.h>
  43 #include <sys/ddi.h>
  44 #include <sys/sunddi.h>
  45
  46 #include <sys/ib/adapters/tavor/tavor_misc.h>
  47
  48 #ifdef __cplusplus
  49 extern "C" {
  50 #endif
  51
  52 /*
  53  * The following defines specify the default number of Completion Queues (CQ)
  54  * their maximum size.  Settings exist for the supported DDR DIMM sizes of
  55  * 128MB and 256MB.  If a DIMM greater than 256 is found, then the 256MB
  56  * profile is used.  See tavor_cfg.c for more discussion on config profiles.
  57  *
  58  * For manual configuration (not using config profiles), these values are
  59  * controllable through the "tavor_log_max_cq_sz" and "tavor_log_num_cq"
  60  * configuration variables, respectively. To override config profile settings
  61  * the 'tavor_alt_config_enable' configuration variable must first be set.
  62  *
  63  * Note: We also have a define for the minimum size of a CQ.  CQs allocated
  64  * with size 0, 1, 2, or 3 will always get back a CQ of size 4.  This is the
  65  * smallest size that Tavor hardware and software can correctly handle.
  66  */
  67 #define TAVOR_NUM_CQ_SHIFT_128          0x10
  68 #define TAVOR_NUM_CQ_SHIFT_256          0x11
  69 #define TAVOR_CQ_SZ_SHIFT               0x10
  70 #define TAVOR_CQ_SZ                     (1 << TAVOR_CQ_SZ_SHIFT)
  71 #define TAVOR_CQ_MIN_SIZE               0x3
  72
  73 /*
  74  * Minimal configuration values.
  75  */
  76 #define TAVOR_NUM_CQ_SHIFT_MIN          0xC
  77 #define TAVOR_CQ_SZ_SHIFT_MIN           0xC
  78
  79 /*
  80  * The following macro determines whether the contents of CQ memory (CQEs)
  81  * need to be sync'd (with ddi_dma_sync()).  This decision is based on whether
  82  * the CQ memory is in DDR memory (no sync) or system memory (sync required).
  83  * Note: It doesn't make much sense to put CQEs in DDR memory (since they are
  84  * primarily written by HW and read by the CPU), but the driver does support
  85  * that possibility.  And it also supports the possibility that if a CQ in
  86  * system memory is mapped DDI_DMA_CONSISTENT, it can be configured to not be
  87  * sync'd because of the "sync override" parameter in the config profile.
  88  */
  89 #define TAVOR_CQ_IS_SYNC_REQ(state, cqinfo)                             \
  90         ((((((state)->ts_cfg_profile->cp_streaming_consistent) &&       \
  91         ((state)->ts_cfg_profile->cp_consistent_syncoverride))) ||      \
  92         ((cqinfo).qa_location == TAVOR_QUEUE_LOCATION_INDDR))    \
  93         ? 0 : 1)
  94
  95 /*
  96  * The following defines specify the size of the individual Completion Queue
  97  * Context (CQC) entries
  98  */
  99 #define TAVOR_CQC_SIZE_SHIFT            0x6
 100 #define TAVOR_CQC_SIZE                  (1 << TAVOR_CQC_SIZE_SHIFT)
 101
 102 /*
 103  * These are the defines for the Tavor CQ completion statuses.  They are
 104  * specified by the Tavor register specification.
 105  */
 106 #define TAVOR_CQE_SUCCESS               0x0
 107 #define TAVOR_CQE_LOC_LEN_ERR           0x1
 108 #define TAVOR_CQE_LOC_OP_ERR            0x2
 109 #define TAVOR_CQE_LOC_EEC_ERR           0x3     /* unsupported: RD */
 110 #define TAVOR_CQE_LOC_PROT_ERR          0x4
 111 #define TAVOR_CQE_WR_FLUSHED_ERR        0x5
 112 #define TAVOR_CQE_MW_BIND_ERR           0x6
 113 #define TAVOR_CQE_BAD_RESPONSE_ERR      0x10
 114 #define TAVOR_CQE_LOCAL_ACCESS_ERR      0x11
 115 #define TAVOR_CQE_REM_INV_REQ_ERR       0x12
 116 #define TAVOR_CQE_REM_ACC_ERR           0x13
 117 #define TAVOR_CQE_REM_OP_ERR            0x14
 118 #define TAVOR_CQE_TRANS_TO_ERR          0x15
 119 #define TAVOR_CQE_RNRNAK_TO_ERR         0x16
 120 #define TAVOR_CQE_LOCAL_RDD_VIO_ERR     0x20    /* unsupported: RD */
 121 #define TAVOR_CQE_REM_INV_RD_REQ_ERR    0x21    /* unsupported: RD */
 122 #define TAVOR_CQE_EEC_REM_ABORTED_ERR   0x22    /* unsupported: RD */
 123 #define TAVOR_CQE_INV_EEC_NUM_ERR       0x23    /* unsupported: RD */
 124 #define TAVOR_CQE_INV_EEC_STATE_ERR     0x24    /* unsupported: RD */
 125
 126 /*
 127  * These are the defines for the Tavor CQ entry types.  They are also
 128  * specified by the Tavor register specification.  They indicate what type
 129  * of work request is completing (for successful completions).  Note: The
 130  * "SND" or "RCV" in each define is used to indicate whether the completion
 131  * work request was from the Send work queue or the Receive work queue on
 132  * the associated QP.
 133  */
 134 #define TAVOR_CQE_SND_RDMAWR            0x8
 135 #define TAVOR_CQE_SND_RDMAWR_IMM        0x9
 136 #define TAVOR_CQE_SND_SEND              0xA
 137 #define TAVOR_CQE_SND_SEND_IMM          0xB
 138 #define TAVOR_CQE_SND_RDMARD            0x10
 139 #define TAVOR_CQE_SND_ATOMIC_CS         0x11
 140 #define TAVOR_CQE_SND_ATOMIC_FA         0x12
 141 #define TAVOR_CQE_SND_BIND_MW           0x18
 142 #define TAVOR_CQE_RCV_RECV_IMM          0x3
 143 #define TAVOR_CQE_RCV_RECV_IMM2         0x5
 144 #define TAVOR_CQE_RCV_RECV              0x2
 145 #define TAVOR_CQE_RCV_RECV2             0x4
 146 #define TAVOR_CQE_RCV_RDMAWR_IMM        0x9
 147 #define TAVOR_CQE_RCV_RDMAWR_IMM2       0xB
 148
 149 /* Define for maximum CQ number mask (CQ number is 24 bits) */
 150 #define TAVOR_CQ_MAXNUMBER_MSK          0xFFFFFF
 151
 152 /*
 153  * This define and the following macro are used to find an event queue for a
 154  * new CQ based on its completion queue number.  Note:  This is a rather
 155  * simple method that we use today.  We simply choose from one of the first
 156  * 32 EQs based on the 5 least significant bits of the CQ number.
 157  */
 158 #define TAVOR_CQ_TO_EQ_MASK             0x1F
 159 #define TAVOR_CQ_EQNUM_GET(cqnum)       ((cqnum) & TAVOR_CQ_TO_EQ_MASK)
 160
 161 /*
 162  * The following macro is even simpler than the above one.  This is used to
 163  * find an event queue for CQ errors for a new CQ.  In theory we could do this
 164  * based on the CQ's number (as we do above).  Today, however, all CQ error
 165  * events go to one specific EQ (i.e. EQ #32).
 166  */
 167 #define TAVOR_CQ_ERREQNUM_GET(cqnum)    0x20
 168
 169 /*
 170  * The following defines are used for Tavor CQ error handling.  Note: For
 171  * CQEs which correspond to error events, the Tavor device requires some
 172  * special handling by software.  These defines are used to identify and
 173  * extract the necessary information from each error CQE, including status
 174  * code (above), doorbell count, and whether a error completion is for a
 175  * send or receive work request.
 176  */
 177 #define TAVOR_CQE_ERR_STATUS_SHIFT      24
 178 #define TAVOR_CQE_ERR_STATUS_MASK       0xFF
 179 #define TAVOR_CQE_ERR_DBDCNT_MASK       0xFFFF
 180 #define TAVOR_CQE_SEND_ERR_OPCODE       0xFF
 181 #define TAVOR_CQE_RECV_ERR_OPCODE       0xFE
 182 #define TAVOR_CQ_SYNC_AND_DB            0
 183 #define TAVOR_CQ_RECYCLE_ENTRY          1
 184
 185 /* Defines for tracking whether a CQ is being used with special QP or not */
 186 #define TAVOR_CQ_IS_NORMAL              0
 187 #define TAVOR_CQ_IS_SPECIAL             1
 188
 189 /*
 190  * The tavor_sw_cq_s structure is also referred to using the "tavor_cqhdl_t"
 191  * typedef (see tavor_typedef.h).  It encodes all the information necessary
 192  * to track the various resources needed to allocate, initialize, poll, resize,
 193  * and (later) free a completion queue (CQ).
 194  *
 195  * Specifically, it has a consumer index and a lock to ensure single threaded
 196  * access to it.  It has pointers to the various resources allocated for the
 197  * completion queue, i.e. a CQC resource and the memory for the completion
 198  * queue itself.  It has flags to indicate whether the CQ requires
 199  * ddi_dma_sync() ("cq_sync").  It also has a reference count and the number(s)
 200  * of the EQs to which it is associated (for success and for errors).
 201  *
 202  * Additionally, it has a pointer to the associated MR handle (for the mapped
 203  * queue memory) and a void pointer that holds the argument that should be
 204  * passed back to the IBTF when events are generated on the CQ.
 205  *
 206  * We also have the always necessary backpointer to the resource for the
 207  * CQ handle structure itself.  But we also have pointers to the "Work Request
 208  * ID" processing lists (both the lock and the regular list, as well as the
 209  * head and tail for the "reapable" list).  See tavor_wrid.c for more details.
 210  */
 211 struct tavor_sw_cq_s {
 212         kmutex_t                cq_lock;
 213         uint32_t                cq_consindx;
 214         uint32_t                cq_cqnum;
 215         tavor_hw_cqe_t          *cq_buf;
 216         tavor_mrhdl_t           cq_mrhdl;
 217         uint32_t                cq_bufsz;
 218         uint_t                  cq_sync;
 219         uint_t                  cq_refcnt;
 220         uint32_t                cq_eqnum;
 221         uint32_t                cq_erreqnum;
 222         uint_t                  cq_is_special;
 223         uint_t                  cq_is_umap;
 224         uint32_t                cq_uarpg;
 225         devmap_cookie_t         cq_umap_dhp;
 226         tavor_rsrc_t            *cq_cqcrsrcp;
 227         tavor_rsrc_t            *cq_rsrcp;
 228
 229         void                    *cq_hdlrarg;
 230
 231         /* For Work Request ID processing */
 232         kmutex_t                cq_wrid_wqhdr_lock;
 233         avl_tree_t              cq_wrid_wqhdr_avl_tree;
 234         tavor_wrid_list_hdr_t   *cq_wrid_reap_head;
 235         tavor_wrid_list_hdr_t   *cq_wrid_reap_tail;
 236
 237         struct tavor_qalloc_info_s cq_cqinfo;
 238 };
 239 _NOTE(READ_ONLY_DATA(tavor_sw_cq_s::cq_cqnum
 240     tavor_sw_cq_s::cq_eqnum
 241     tavor_sw_cq_s::cq_erreqnum
 242     tavor_sw_cq_s::cq_cqcrsrcp
 243     tavor_sw_cq_s::cq_rsrcp
 244     tavor_sw_cq_s::cq_hdlrarg
 245     tavor_sw_cq_s::cq_is_umap
 246     tavor_sw_cq_s::cq_uarpg))
 247 _NOTE(DATA_READABLE_WITHOUT_LOCK(tavor_sw_cq_s::cq_bufsz
 248     tavor_sw_cq_s::cq_cqinfo))
 249 _NOTE(MUTEX_PROTECTS_DATA(tavor_sw_cq_s::cq_lock,
 250     tavor_sw_cq_s::cq_consindx
 251     tavor_sw_cq_s::cq_buf
 252     tavor_sw_cq_s::cq_mrhdl
 253     tavor_sw_cq_s::cq_sync
 254     tavor_sw_cq_s::cq_refcnt
 255     tavor_sw_cq_s::cq_is_special
 256     tavor_sw_cq_s::cq_umap_dhp))
 257
 258 int tavor_cq_alloc(tavor_state_t *state, ibt_cq_hdl_t ibt_cqhdl,
 259     ibt_cq_attr_t *attr_p, uint_t *actual_size, tavor_cqhdl_t *cqhdl,
 260     uint_t sleepflag);
 261 int tavor_cq_free(tavor_state_t *state, tavor_cqhdl_t *cqhdl,
 262     uint_t sleepflag);
 263 int tavor_cq_resize(tavor_state_t *state, tavor_cqhdl_t cqhdl,
 264     uint_t req_size, uint_t *actual_size, uint_t sleepflag);
 265 int tavor_cq_notify(tavor_state_t *state, tavor_cqhdl_t cqhdl,
 266     ibt_cq_notify_flags_t flags);
 267 int tavor_cq_poll(tavor_state_t *state, tavor_cqhdl_t cqhdl, ibt_wc_t *wc_p,
 268     uint_t num_wc, uint_t *num_polled);
 269 int tavor_cq_handler(tavor_state_t *state, tavor_eqhdl_t eq,
 270     tavor_hw_eqe_t *eqe);
 271 int tavor_cq_err_handler(tavor_state_t *state, tavor_eqhdl_t eq,
 272     tavor_hw_eqe_t *eqe);
 273 int tavor_cq_refcnt_inc(tavor_cqhdl_t cq, uint_t is_special);
 274 void tavor_cq_refcnt_dec(tavor_cqhdl_t cq);
 275 tavor_cqhdl_t tavor_cqhdl_from_cqnum(tavor_state_t *state, uint_t cqnum);
 276 void tavor_cq_srq_entries_flush(tavor_state_t *state, tavor_qphdl_t qp);
 277 #ifdef __cplusplus
 278 }
 279 #endif
 280
 281 #endif  /* _SYS_IB_ADAPTERS_TAVOR_CQ_H */