include/sys/ib/adapters/tavor/tavor_wr.h

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21
  22 /*
  23  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
  24  * Use is subject to license terms.
  25  */
  26
  27 #ifndef _SYS_IB_ADAPTERS_TAVOR_WR_H
  28 #define _SYS_IB_ADAPTERS_TAVOR_WR_H
  29
  30 /*
  31  * tavor_wr.h
  32  *    Contains all of the prototypes, #defines, and structures necessary
  33  *    for the Tavor Work Request Processing Routines
  34  *    Specifically it contains #defines, macros, and prototypes for each of
  35  *    building each of the various types of WQE and for managing the WRID
  36  *    tracking mechanisms.
  37  */
  38
  39 #include <sys/types.h>
  40 #include <sys/conf.h>
  41 #include <sys/ddi.h>
  42 #include <sys/sunddi.h>
  43
  44 #ifdef __cplusplus
  45 extern "C" {
  46 #endif
  47
  48 /*
  49  * The following macro is used to convert WQE address and size into the
  50  * "wqeaddrsz" value needed in the tavor_wrid_entry_t (see below).
  51  */
  52 #define TAVOR_QP_WQEADDRSZ(addr, size)                                  \
  53         ((((uintptr_t)(addr)) & ~TAVOR_WQE_NDS_MASK) |                  \
  54         ((size) & TAVOR_WQE_NDS_MASK))
  55
  56 /*
  57  * The following macros are used to calculate pointers to the Send or Receive
  58  * (or SRQ) WQEs on a given QP, respectively
  59  */
  60 #define TAVOR_QP_SQ_ENTRY(qp, tail)                                     \
  61         ((uint64_t *)((uintptr_t)((qp)->qp_sq_buf) +                    \
  62         ((tail) << (qp)->qp_sq_log_wqesz)))
  63 #define TAVOR_QP_RQ_ENTRY(qp, tail)                                     \
  64         ((uint64_t *)((uintptr_t)((qp)->qp_rq_buf) +                    \
  65         ((tail) << (qp)->qp_rq_log_wqesz)))
  66 #define TAVOR_SRQ_WQ_ENTRY(srq, tail)                                   \
  67         ((uint64_t *)((uintptr_t)((srq)->srq_wq_buf) +                  \
  68         ((tail) << (srq)->srq_wq_log_wqesz)))
  69
  70 /*
  71  * The following macro is used to calculate the 'wqe_index' field during SRQ
  72  * operation.  This returns the index based on the WQE size, that can be used
  73  * to reference WQEs in an SRQ.
  74  */
  75 #define TAVOR_SRQ_WQE_INDEX(srq_base_addr, wqe_addr, log_wqesz)         \
  76         (((uint32_t)(uintptr_t)wqe_addr -                               \
  77         (uint32_t)(uintptr_t)srq_base_addr) >> log_wqesz)
  78
  79 #define TAVOR_SRQ_WQE_ADDR(srq, wqe_index)                              \
  80         ((uint64_t *)((uintptr_t)srq->srq_wq_buf +                      \
  81         (wqe_index << srq->srq_wq_log_wqesz)))
  82
  83 /*
  84  * The following macros are used to access specific fields in Directed Route
  85  * MAD packets.  We can extract the MgmtClass, "hop pointer", and "hop count".
  86  * We can also update the "hop pointer" as appropriate.  Note:  Again, because
  87  * of the limited amount of direct handling the Tavor hardware does on special
  88  * QP request (specifically on Directed Route MADs), the driver needs to
  89  * update (as necessary) the "hop pointer" value depending on whether a MAD
  90  * is outbound or inbound (i.e. depending on the relationship between "hop
  91  * pointer" and "hop count" in the given MAD)
  92  */
  93 #define TAVOR_SPECIAL_QP_DRMAD_GET_MGMTCLASS(mgmtclass, offset, va, len) \
  94         if (((mgmtclass) == NULL) && ((offset) + (len) > 1)) {           \
  95             (mgmtclass) = &((uint8_t *)(uintptr_t)(va))[1 - (offset)];   \
  96         }
  97 #define TAVOR_SPECIAL_QP_DRMAD_GET_HOPPOINTER(hp, offset, va, len)      \
  98         if (((hp) == NULL) &&                                           \
  99             ((offset) + (len) > 6)) {                                   \
 100             (hp) = &((uint8_t *)(uintptr_t)(va))[6 - (offset)];         \
 101         }
 102 #define TAVOR_SPECIAL_QP_DRMAD_GET_HOPCOUNT(hc, offset, va, len)        \
 103         if (((hc) == NULL) &&                                           \
 104             ((offset) + (len) > 7)) {                                   \
 105             (hc) = &((uint8_t *)(uintptr_t)(va))[7 - (offset)];         \
 106         }
 107 #define TAVOR_SPECIAL_QP_DRMAD_DO_HOPPOINTER_MODIFY(mgmtclass, hp, hc)  \
 108         if ((mgmtclass) == 0x81) {                                      \
 109                 if ((hp) < (hc)) {                                      \
 110                         (hp) = (hp) + 1;                                \
 111                 } else if ((hp) > (hc)) {                               \
 112                         (hp) = (hp) - 1;                                \
 113                 }                                                       \
 114         }
 115
 116
 117 /*
 118  * The tavor_wrid_entry_s structure is used internally by the Tavor
 119  * driver to contain all the information necessary for tracking WRIDs.
 120  * Specifically, this structure contains the 64-bit WRID, the 32-bit quantity
 121  * called "wr_wqeaddrsz" (which can also be found in every CQE), and the
 122  * "wr_signaled_dbd" information which indicates whether a given entry was
 123  * signaled or not and whether a doorbell was subsequently rung for this
 124  * particular work request.  Note: the latter piece of information is
 125  * particularly useful during completion processing on errored CQEs.
 126  */
 127 struct tavor_wrid_entry_s {
 128         uint64_t                wr_wrid;
 129         uint32_t                wr_wqeaddrsz;
 130         uint32_t                wr_signaled_dbd;
 131 };
 132 #define TAVOR_WRID_ENTRY_SIGNALED       (1 << 0)
 133 #define TAVOR_WRID_ENTRY_DOORBELLED     (1 << 1)
 134
 135 /*
 136  * The tavor_sw_wqe_dbinfo_t structure is used internally by the Tavor
 137  * driver to return information (from the tavor_wqe_mlx_build_nextctl() and
 138  * tavor_wqe_send_build_nextctl() routines) regarding the type of Tavor
 139  * doorbell necessary.
 140  */
 141 typedef struct tavor_sw_wqe_dbinfo_s {
 142         uint_t  db_nopcode;
 143         uint_t  db_fence;
 144 } tavor_sw_wqe_dbinfo_t;
 145
 146 /*
 147  * The Work Queue Lock (WQL) structure.  Each WQHDR (tavor_workq_hdr_t defined
 148  * below) must lock access to the wridlist during any wridlist manipulation.
 149  * Also, any Shared Receive Queue (SRQ) must also be able to lock the wridlist
 150  * since it maintains wridlist's differently than normal QPs.  This
 151  * 'tavor_wq_lock_t' structure is shared and accessible through the WQ or the
 152  * SRQ, and refcnt is maintained.  The last entity to decrement use of the
 153  * lock, also will free up the memory.
 154  */
 155 struct tavor_wq_lock_s {
 156         kmutex_t        wql_lock;
 157         uint_t          wql_refcnt;
 158 };
 159
 160 /*
 161  * The tavor_wrid_list_hdr_s structure is used internally by the Tavor driver
 162  * to track all the information necessary to manage a queue of WRID entries
 163  * (the tavor_wrid_entry_s struct above).
 164  * It contains some information regarding the status of a given WRID list
 165  * (e.g. head index, tail index, queue full condition, etc.).  Note:  Although
 166  * some of this information is also kept by the tavor_workq_hdr_s below, what
 167  * is kept here may, in fact, represent the state of an old WRID list.  It
 168  * could be different from what is kept in the tavor_workq_hdr_s because this
 169  * WRID list may no longer be the active WRID list.  If it is an active list,
 170  * however, then both sets of information should be up-to-date and consistent.
 171  * Several of these structures are chained together on each work queue header
 172  * to form a linked list (using the "wl_next" and "wl_prev").  These structs,
 173  * in turn, each have a pointer to a queue of WRID entries.  They also each
 174  * have a pointer to the next "reapable" entry ("wl_reap_next") which is only
 175  * used when a WRID list has been retired and is ready to be freed up.
 176  * Lastly, it has a backpointer to the work queue header to which the WRID
 177  * list belongs (this is for proper handling on removal).
 178  */
 179 struct tavor_wrid_list_hdr_s {
 180         tavor_wrid_list_hdr_t   *wl_next;
 181         tavor_wrid_list_hdr_t   *wl_prev;
 182         tavor_wrid_list_hdr_t   *wl_reap_next;
 183         tavor_workq_hdr_t       *wl_wqhdr;
 184
 185         tavor_wrid_entry_t      *wl_wre;
 186         tavor_wrid_entry_t      *wl_wre_old_tail;
 187         uint32_t                wl_size;
 188         uint32_t                wl_full;
 189         uint32_t                wl_head;
 190         uint32_t                wl_tail;
 191
 192         /* For SRQ */
 193         uint_t                  wl_srq_en;
 194         int                     wl_free_list_indx;
 195         ddi_acc_handle_t        wl_acchdl;
 196         uint32_t                *wl_srq_wq_buf;
 197         uint32_t                wl_srq_wq_bufsz;
 198         uint64_t                wl_srq_desc_off;
 199         uint32_t                wl_srq_log_wqesz;
 200 };
 201 _NOTE(MUTEX_PROTECTS_DATA(tavor_sw_cq_s::cq_wrid_wqhdr_lock,
 202     tavor_wrid_list_hdr_s::wl_next
 203     tavor_wrid_list_hdr_s::wl_prev
 204     tavor_wrid_list_hdr_s::wl_wqhdr))
 205 _NOTE(MUTEX_PROTECTS_DATA(tavor_wq_lock_s::wql_lock,
 206     tavor_wrid_list_hdr_s::wl_wre
 207     tavor_wrid_list_hdr_s::wl_wre_old_tail
 208     tavor_wrid_list_hdr_s::wl_size
 209     tavor_wrid_list_hdr_s::wl_full
 210     tavor_wrid_list_hdr_s::wl_head
 211     tavor_wrid_list_hdr_s::wl_tail
 212     tavor_wrid_list_hdr_s::wl_srq_en
 213     tavor_wrid_list_hdr_s::wl_free_list_indx
 214     tavor_wrid_list_hdr_s::wl_acchdl
 215     tavor_wrid_list_hdr_s::wl_srq_wq_buf
 216     tavor_wrid_list_hdr_s::wl_srq_desc_off
 217     tavor_wrid_list_hdr_s::wl_srq_log_wqesz))
 218
 219 /*
 220  * The tavor_workq_hdr_s structure is used internally by the Tavor driver to
 221  * track all the information necessary to manage the work queues associated
 222  * with a given completion queue.  It contains much of the information
 223  * regarding the status of a given work queue (e.g. head index, tail index,
 224  * queue full condition, etc.).  Note:  This information is kept here (i.e.
 225  * associated with a completion queue) rather than as part of the QP because
 226  * the queue pair may potentially be destroyed while outstanding CQEs still
 227  * remain on the CQ.
 228  * Several of these structures are chained together on each CQ to form a
 229  * linked list (using the "wq_next" and "wq_prev").  These headers, in turn,
 230  * link to the containers for the individual WRID entries (managed with the
 231  * tavor_wrid_list_hdr_s structs above).  Note: We keep a list of these
 232  * tavor_wrid_list_hdr_s because a given QP may be used, destroyed (or
 233  * transition to "Reset"), and then reused.  The list helps us track where
 234  * to put new WRID entries and where to pull old entries from.
 235  * The "wq_qpn" (QP number) and "wq_send_or_recv" (TAVOR_WR_SEND or
 236  * TAVOR_WR_RECV) are used to uniquely identify the given work queue.
 237  * Lookups into the work queue list (to find a given work queue) will use
 238  * these two fields as identifiers.
 239  */
 240 struct tavor_workq_hdr_s {
 241         avl_node_t              wq_avl_link;
 242         uint32_t                wq_qpn;
 243         uint32_t                wq_type;
 244
 245         tavor_wq_lock_t         *wq_wrid_wql;
 246
 247         uint32_t                wq_size;
 248         uint32_t                wq_head;
 249         uint32_t                wq_tail;
 250         uint32_t                wq_full;
 251         tavor_wrid_list_hdr_t   *wq_wrid_poll;
 252         tavor_wrid_list_hdr_t   *wq_wrid_post;
 253 };
 254 _NOTE(MUTEX_PROTECTS_DATA(tavor_sw_cq_s::cq_wrid_wqhdr_lock,
 255     tavor_workq_hdr_s::wq_avl_link
 256     tavor_workq_hdr_s::wq_qpn
 257     tavor_workq_hdr_s::wq_type
 258     tavor_sw_cq_s::cq_wrid_reap_head
 259     tavor_sw_cq_s::cq_wrid_reap_tail))
 260 _NOTE(MUTEX_PROTECTS_DATA(tavor_wq_lock_s::wql_lock,
 261     tavor_workq_hdr_s::wq_size
 262     tavor_workq_hdr_s::wq_head
 263     tavor_workq_hdr_s::wq_tail
 264     tavor_workq_hdr_s::wq_full
 265     tavor_workq_hdr_s::wq_wrid_poll
 266     tavor_workq_hdr_s::wq_wrid_post
 267     tavor_wrid_list_hdr_s::wl_wre
 268     tavor_wrid_list_hdr_s::wl_wre_old_tail
 269     tavor_wrid_list_hdr_s::wl_size
 270     tavor_wrid_list_hdr_s::wl_full
 271     tavor_wrid_list_hdr_s::wl_head
 272     tavor_wrid_list_hdr_s::wl_tail))
 273 _NOTE(MUTEX_PROTECTS_DATA(tavor_sw_cq_s::cq_wrid_wqhdr_lock,
 274     tavor_wrid_list_hdr_s::wl_reap_next))
 275 _NOTE(LOCK_ORDER(tavor_sw_cq_s::cq_lock
 276     tavor_sw_cq_s::cq_wrid_wqhdr_lock
 277     tavor_wq_lock_s::wql_lock))
 278 #define TAVOR_WR_RECV                   0x0
 279 #define TAVOR_WR_SEND                   0x1
 280 #define TAVOR_WR_SRQ                    0x2
 281
 282 extern int tavor_wrid_wqhdr_compare(const void *p1, const void *p2);
 283 typedef struct tavor_workq_compare_s {
 284         uint32_t cmp_type;
 285         uint32_t cmp_qpn;
 286 } tavor_workq_compare_t;
 287
 288 /* For Work Request posting */
 289 int tavor_post_send(tavor_state_t *state, tavor_qphdl_t qphdl,
 290     ibt_send_wr_t *wr_p, uint_t num_wr, uint_t *num_posted);
 291 int tavor_post_recv(tavor_state_t *state, tavor_qphdl_t qphdl,
 292     ibt_recv_wr_t *wr_p, uint_t num_wr, uint_t *num_posted);
 293 int tavor_post_srq(tavor_state_t *state, tavor_srqhdl_t srqhdl,
 294     ibt_recv_wr_t *wr_p, uint_t num_wr, uint_t *num_posted);
 295
 296 /* For WRID handling */
 297 int tavor_wrid_from_reset_handling(tavor_state_t *state, tavor_qphdl_t qp);
 298 void tavor_wrid_to_reset_handling(tavor_state_t *state, tavor_qphdl_t qp);
 299 void tavor_wrid_add_entry(tavor_workq_hdr_t *wq, uint64_t wrid,
 300     uint32_t wqeaddr_sz, uint_t signaled_dbd);
 301 void tavor_wrid_add_entry_srq(tavor_srqhdl_t srq, uint64_t wrid,
 302     uint_t signaled_dbd);
 303 uint64_t tavor_wrid_get_entry(tavor_cqhdl_t cqhdl, tavor_hw_cqe_t *cqe,
 304     tavor_wrid_entry_t *wre);
 305 tavor_wq_lock_t *tavor_wrid_wql_create(tavor_state_t *state);
 306 tavor_wrid_list_hdr_t *tavor_wrid_get_list(uint32_t size);
 307 void tavor_wrid_list_srq_init(tavor_wrid_list_hdr_t *r_wridlist,
 308     tavor_srqhdl_t srq, uint_t wq_start);
 309 void tavor_wrid_cq_reap(tavor_cqhdl_t cq);
 310 void tavor_wrid_cq_force_reap(tavor_cqhdl_t cq);
 311 void tavor_wql_refcnt_dec(tavor_wq_lock_t *wq_lock);
 312 void tavor_wql_refcnt_inc(tavor_wq_lock_t *wq_lock);
 313 tavor_wrid_entry_t *tavor_wrid_find_match_srq(tavor_wrid_list_hdr_t *wq,
 314     tavor_cqhdl_t cq, tavor_hw_cqe_t *cqe);
 315
 316 #ifdef __cplusplus
 317 }
 318 #endif
 319
 320 #endif  /* _SYS_IB_ADAPTERS_TAVOR_WR_H */