include/sys/ib/ibtl/ibti.h

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
  23  */
  24
  25 #ifndef _SYS_IB_IBTL_IBTI_H
  26 #define _SYS_IB_IBTL_IBTI_H
  27
  28 /*
  29  * ibti.h
  30  *
  31  * This file contains the IBTI prototypes and associated data structures.
  32  * It is the only header file that should be included by IBTI clients.
  33  */
  34 #include <sys/ib/ibtl/ibti_common.h>
  35
  36 #ifdef __cplusplus
  37 extern "C" {
  38 #endif
  39
  40 /*
  41  * Allocate channel flags.
  42  */
  43 typedef enum ibt_chan_alloc_flags_e {
  44         IBT_ACHAN_NO_FLAGS              = 0,
  45         IBT_ACHAN_CLONE                 = (1 << 0),
  46         IBT_ACHAN_USER_MAP              = (1 << 1),
  47         IBT_ACHAN_DEFER_ALLOC           = (1 << 2),
  48         IBT_ACHAN_USES_SRQ              = (1 << 3),
  49         IBT_ACHAN_USES_RSS              = (1 << 4),
  50
  51         /* UD variants for FC support */
  52         IBT_ACHAN_USES_RFCI             = (1 << 5),     /* from RFCI pool */
  53         IBT_ACHAN_USES_FCMD             = (1 << 6),
  54         IBT_ACHAN_USES_FEXCH            = (1 << 7)      /* from FEXCH pool */
  55 } ibt_chan_alloc_flags_t;
  56
  57
  58 /*
  59  * Allocate RC channel ibt_alloc_rc_channel() argument.
  60  */
  61 typedef struct ibt_rc_chan_alloc_args_s {
  62         ibt_attr_flags_t        rc_flags;       /* Signal type etc */
  63         ibt_cep_flags_t         rc_control;
  64         uint8_t                 rc_hca_port_num;
  65         ibt_chan_sizes_t        rc_sizes;
  66         ibt_cq_hdl_t            rc_scq; /* Send CQ */
  67         ibt_cq_hdl_t            rc_rcq; /* Recv CQ */
  68         ibt_pd_hdl_t            rc_pd;  /* PD */
  69         ibt_channel_hdl_t       rc_clone_chan;  /* Optional Clone handle */
  70         ibt_srq_hdl_t           rc_srq;         /* Optional Shared Rcv Queue */
  71 } ibt_rc_chan_alloc_args_t;
  72
  73 /*
  74  * RC channel query attributes structure.
  75  */
  76 typedef struct ibt_rc_chan_query_attr_s {
  77         ib_guid_t               rc_hca_guid;    /* Local HCA GUID */
  78         ibt_cq_hdl_t            rc_scq;         /* SendCQ handle */
  79         ibt_cq_hdl_t            rc_rcq;         /* RecvCQ handle */
  80         ibt_pd_hdl_t            rc_pd;          /* PD Handle. */
  81         ibt_cep_state_t         rc_state;       /* Channel state */
  82         ib_mtu_t                rc_path_mtu;
  83         uint8_t                 rc_path_retry_cnt:3;
  84         ibt_rnr_retry_cnt_t     rc_path_rnr_retry_cnt;
  85         ibt_rnr_nak_time_t      rc_min_rnr_nak; /* min RNR-NAK timer */
  86         ibt_cep_path_t          rc_prim_path;
  87         ibt_cep_path_t          rc_alt_path;
  88         ibt_chan_sizes_t        rc_chan_sizes;  /* Queue/SGL sizes */
  89         uint8_t                 rc_rdma_ra_out; /* max RDMA-Reads/Atomics out */
  90         uint8_t                 rc_rdma_ra_in;  /* max RDMA-Reads/Atomics in */
  91         ibt_attr_flags_t        rc_flags;       /* SQ Signaling Type etc */
  92         ibt_cep_flags_t         rc_control;     /* Control Flags */
  93         ibt_cep_cmstate_t       rc_mig_state;
  94         ib_qpn_t                rc_qpn;         /* Local QPN */
  95         ib_qpn_t                rc_dst_qpn;     /* Destination QPN */
  96         ibt_srq_hdl_t           rc_srq;         /* Optional Shared Rcv Queue */
  97 } ibt_rc_chan_query_attr_t;
  98
  99 /*
 100  * RC Channel Modify Attributes definition.
 101  *
 102  * It is only possible to modify a channel that has previously been
 103  * opened. The channel must either be in operational state (IBT_STATE_RTS)
 104  * or paused (IBT_STATE_SQD). If channel is in paused state, then a modify
 105  * operation will unpause the channel.
 106  *
 107  * See the ibt_modify_rc_channel() for details of the required corresponding
 108  * modify flags.
 109  * Not specified attributes should be set to "NULL" or "0".
 110  */
 111 typedef struct ibt_rc_chan_modify_attr_s {
 112         ibt_cep_flags_t         rc_control;     /* Channel Control Flags */
 113         uint_t                  rc_sq_sz;       /* Set SQ Max outstanding WRs */
 114         uint_t                  rc_rq_sz;       /* Set RQ Max outstanding WRs */
 115
 116         ibt_adds_vect_t         rc_prim_adds_vect; /* Primary Path Address */
 117         ibt_adds_vect_t         rc_alt_adds_vect; /* Alternate Path Address */
 118         uint8_t                 rc_path_retry_cnt:3;
 119         ibt_rnr_retry_cnt_t     rc_path_rnr_retry_cnt;
 120         ibt_rnr_nak_time_t      rc_min_rnr_nak; /* min RNR-NAK timer */
 121         uint8_t                 rc_prim_port_num; /* Port of Primary Path */
 122         uint8_t                 rc_alt_port_num; /* Port of Alternate Path */
 123         uint8_t                 rc_rdma_ra_out; /* Initiator Depth, Number of */
 124                                                 /* RDMA RD's & Atomics */
 125                                                 /* outstanding. */
 126         uint8_t                 rc_rdma_ra_in; /* Responder resources for */
 127                                                 /* handling incoming RDMA rds */
 128                                                 /* and Atomics. */
 129 } ibt_rc_chan_modify_attr_t;
 130
 131 /*
 132  * UD remote destination query attributes
 133  */
 134 typedef struct ibt_ud_dest_query_attr_s {
 135         ibt_hca_hdl_t           ud_hca_hdl;     /* Local HCA Handle */
 136         ib_qpn_t                ud_dst_qpn;     /* Destination QPN */
 137         ib_qkey_t               ud_qkey;        /* Q_Key */
 138         ibt_adds_vect_t         ud_addr_vect;   /* Address Information */
 139         ibt_pd_hdl_t            ud_pd;
 140 } ibt_ud_dest_query_attr_t;
 141
 142 /*
 143  * Allocate UD channel ibt_alloc_ud_channel() arguments; see below at
 144  * ibt_alloc_ud_channel() for a description of what's required and optional.
 145  */
 146 typedef struct ibt_ud_chan_alloc_args_s {
 147         ibt_attr_flags_t        ud_flags;       /* Sig type etc */
 148         uint8_t                 ud_hca_port_num;
 149         uint16_t                ud_pkey_ix;     /* P_Key Index */
 150         ibt_chan_sizes_t        ud_sizes;
 151         ib_qkey_t               ud_qkey;        /* Q_Key */
 152         ibt_cq_hdl_t            ud_scq;         /* Send CQ */
 153         ibt_cq_hdl_t            ud_rcq;         /* Recv CQ */
 154         ibt_pd_hdl_t            ud_pd;          /* PD */
 155         ibt_channel_hdl_t       ud_clone_chan;  /* Optional clone handle */
 156         ibt_srq_hdl_t           ud_srq;         /* Optional Shared Rcv Queue */
 157         ibt_rss_attr_t          ud_rss;
 158         ibt_fc_attr_t           ud_fc;
 159 } ibt_ud_chan_alloc_args_t;
 160
 161 /*
 162  * UD channel query attributes.
 163  */
 164 typedef struct ibt_ud_chan_query_attr_s {
 165         ib_qpn_t                ud_qpn;                 /* QPN */
 166         ib_guid_t               ud_hca_guid;            /* Local HCA GUID */
 167         ibt_cq_hdl_t            ud_scq;                 /* SendCQ handle. */
 168         ibt_cq_hdl_t            ud_rcq;                 /* RecvCQ handle. */
 169         ibt_pd_hdl_t            ud_pd;                  /* PD Handle. */
 170         uint8_t                 ud_hca_port_num;        /* Local HCA port */
 171         ibt_cep_state_t         ud_state;               /* Channel state */
 172         uint16_t                ud_pkey_ix;             /* P_Key Index */
 173         ib_qkey_t               ud_qkey;                /* Q_Key */
 174         ibt_chan_sizes_t        ud_chan_sizes;          /* Queue/SGL sizes */
 175         ibt_attr_flags_t        ud_flags;               /* Signaling Type etc */
 176         ibt_srq_hdl_t           ud_srq;         /* Optional Shared Rcv Queue */
 177         ibt_rss_attr_t          ud_rss;
 178         ibt_fc_attr_t           ud_fc;
 179         ibt_fexch_query_attr_t  ud_query_fc;    /* query only set */
 180 } ibt_ud_chan_query_attr_t;
 181
 182 /*
 183  * UD Channel Modify Attributes definition.
 184  *
 185  * It is only possible to modify a channel that is either in the operational
 186  * state (IBT_STATE_RTS) or paused (IBT_STATE_SQD). If channel is in paused
 187  * state, then a modify operation will unpause the channel.
 188  *
 189  * See the ibt_modify_ud_channel() for details of the required corresponding
 190  * modify flags.
 191  */
 192 typedef struct ibt_ud_chan_modify_attr_s {
 193         uint_t                  ud_sq_sz;       /* Set SQ Max outstanding WRs */
 194         uint_t                  ud_rq_sz;       /* Set RQ Max outstanding WRs */
 195         ib_qkey_t               ud_qkey;        /* Set Q_Key */
 196         ibt_rss_attr_t          ud_rss;         /* Set RSS stuff */
 197         ibt_fc_attr_t           ud_fc;
 198 } ibt_ud_chan_modify_attr_t;
 199
 200
 201 /*
 202  * FUNCTION PROTOTYPES
 203  */
 204
 205 /*
 206  * CONNECTION ESTABLISHMENT/TEAR DOWN FUNCTIONS.
 207  */
 208
 209 /*
 210  * ibt_alloc_rc_channel
 211  *      Allocates a RC communication channels that satisfy the specified
 212  *      channel attributes.
 213  */
 214 ibt_status_t ibt_alloc_rc_channel(ibt_hca_hdl_t hca_hdl,
 215     ibt_chan_alloc_flags_t flags, ibt_rc_chan_alloc_args_t *args,
 216     ibt_channel_hdl_t *rc_chan_p, ibt_chan_sizes_t *sizes);
 217
 218 /*
 219  * ibt_flush_channel
 220  *      Flush the specified channel. Outstanding work requests are flushed
 221  *      so that the client can do the associated clean up. After that, the
 222  *      client will usually deregister the previously registered memory,
 223  *      then free the channel by calling ibt_free_channel().  RC channels
 224  *      that have been successfully opened will fail this call, as they
 225  *      need to instead be handled by ibt_close_rc_channel().
 226  */
 227 ibt_status_t ibt_flush_channel(ibt_channel_hdl_t chan);
 228
 229 /*
 230  * ibt_free_channel
 231  *      Releases the resources associated with the specified channel
 232  */
 233 ibt_status_t ibt_free_channel(ibt_channel_hdl_t chan);
 234
 235 /*
 236  * ibt_query_rc_channel
 237  *      Query an RC channel's attributes. Should only be called on an
 238  *      opened RC channel. If called on a channel before it is opened,
 239  *      some channel attributes may change when the channel is opened.
 240  */
 241 ibt_status_t ibt_query_rc_channel(ibt_channel_hdl_t rc_chan,
 242     ibt_rc_chan_query_attr_t *chan_attrs);
 243
 244 /*
 245  * ibt_modify_rc_channel()
 246  *      Modifies a previous opened operational or paused RC channel's
 247  *      attributes.
 248  *
 249  * NOTE:
 250  *      It is only possible to modify a channel that has previously been opened.
 251  *      The channel must either be in operational state (IBT_STATE_RTS) or
 252  *      paused (IBT_STATE_SQD). If channel is in paused state, then a modify
 253  *      operation will will unpause the channel.
 254  */
 255 ibt_status_t ibt_modify_rc_channel(ibt_channel_hdl_t rc_chan,
 256     ibt_cep_modify_flags_t flags, ibt_rc_chan_modify_attr_t *attrs,
 257     ibt_queue_sizes_t *actual_sz);
 258
 259 /*
 260  * ibt_alloc_ud_channel
 261  *      Allocate UD channels that satisfy the specified channel attributes.
 262  */
 263 ibt_status_t ibt_alloc_ud_channel(ibt_hca_hdl_t hca_hdl,
 264     ibt_chan_alloc_flags_t flags, ibt_ud_chan_alloc_args_t *args,
 265     ibt_channel_hdl_t *ud_chan_p, ibt_chan_sizes_t *sizes);
 266
 267 /*
 268  * ibt_alloc_ud_channel_range
 269  *      Allocate a range of UD channels that have consecutive QPNs for RSS.
 270  */
 271 ibt_status_t ibt_alloc_ud_channel_range(ibt_hca_hdl_t hca_hdl,
 272     uint_t log2, ibt_chan_alloc_flags_t flags,
 273     ibt_ud_chan_alloc_args_t *args, ibt_cq_hdl_t *send_cq_p,
 274     ibt_cq_hdl_t *recv_cq_p, ib_qpn_t *base_qpn_p,
 275     ibt_channel_hdl_t *ud_chan_p, ibt_chan_sizes_t *sizes);
 276
 277 /*
 278  * ibt_query_ud_channel
 279  *      Query a UD channel's attributes.
 280  */
 281 ibt_status_t ibt_query_ud_channel(ibt_channel_hdl_t ud_chan,
 282     ibt_ud_chan_query_attr_t *ud_chan_attrs);
 283
 284 /*
 285  * ibt_modify_ud_channel()
 286  *      Modifies an UD channel's attributes, as specified by a
 287  *      ibt_cep_modify_flags_t parameter to those specified in the
 288  *      ibt_ud_chan_modify_attr_t structure.
 289  *
 290  * NOTE:
 291  *      It is only possible to modify a channel that is either in the
 292  *      operational state (IBT_STATE_RTS) or paused (IBT_STATE_SQD). If
 293  *      channel is in paused state, then a modify operation will unpause the
 294  *      channel.
 295  */
 296 ibt_status_t ibt_modify_ud_channel(ibt_channel_hdl_t ud_chan,
 297     ibt_cep_modify_flags_t flags, ibt_ud_chan_modify_attr_t *attrs,
 298     ibt_queue_sizes_t *actual_sz);
 299
 300 /*
 301  * ibt_recover_ud_channel()
 302  *      Recover an UD Channel which has transitioned to SQ Error state. The
 303  *      ibt_recover_ud_channel() transitions the channel from SQ Error state
 304  *      to Ready-To-Send channel state.
 305  *
 306  *      If a work request posted to a UD channel's send queue completes with
 307  *      an error (see ibt_wc_status_t), the channel gets transitioned to SQ
 308  *      Error state. In order to reuse this channel, ibt_recover_ud_channel()
 309  *      can be used to recover the channel to a usable (Ready-to-Send) state.
 310  */
 311 ibt_status_t ibt_recover_ud_channel(ibt_channel_hdl_t ud_chan);
 312
 313
 314 /*
 315  * ibt_alloc_ud_dest
 316  *      Allocate a UD destination handle.  This allocates local resources
 317  *      that will need subsequent modification/initialization before use
 318  *      (in send work requests).  Functions that can be used to do this are
 319  *      ibt_modify_ud_dest (data supplied by caller), ibt_modify_reply_ud_dest
 320  *      (data supplied from a successfully completed receive work request),
 321  *      and ibt_request_ud_dest (data retrieved using SIDR protocol).
 322  */
 323 ibt_status_t ibt_alloc_ud_dest(ibt_hca_hdl_t hca_hdl,
 324     ibt_ud_dest_flags_t flags, ibt_pd_hdl_t pd, ibt_ud_dest_hdl_t *ud_dest_p);
 325
 326 /*
 327  * ibt_modify_ud_dest
 328  *      Modify a previously allocated UD destination handle from the
 329  *      argument data.  After the ud_dest has already been made usable,
 330  *      the adds_vect argument to this function is optional (NULL).
 331  */
 332 ibt_status_t ibt_modify_ud_dest(ibt_ud_dest_hdl_t ud_dest, ib_qkey_t qkey,
 333     ib_qpn_t dest_qpn, ibt_adds_vect_t *adds_vect);
 334
 335 /*
 336  * ibt_modify_reply_ud_dest
 337  *      Modify a previously allocated UD destination handle, so that it
 338  *      can be used to reply to the sender of the datagram contained in the
 339  *      specified work request completion.
 340  */
 341 ibt_status_t ibt_modify_reply_ud_dest(ibt_channel_hdl_t ud_chan,
 342     ibt_ud_dest_hdl_t ud_dest, ib_qkey_t qkey, ibt_wc_t *wc,
 343     ib_vaddr_t recv_buf);
 344
 345 /*
 346  * ibt_request_ud_dest
 347  *      Modify a previously allocated UD destination handle based on the
 348  *      data retrieved by making an SIDR request.
 349  */
 350 ibt_status_t ibt_request_ud_dest(ibt_ud_dest_hdl_t ud_dest,
 351     ibt_execution_mode_t mode, ibt_ud_dest_attr_t *dest_attrs,
 352     ibt_ud_returns_t *ud_ret_args);
 353
 354 /*
 355  * ibt_free_ud_dest
 356  *      Releases the resources associated with the specified UD destination
 357  *      handle.
 358  */
 359 ibt_status_t ibt_free_ud_dest(ibt_ud_dest_hdl_t ud_dest);
 360
 361 /*
 362  * ibt_query_ud_dest
 363  *      Query a UD destination's attributes.
 364  */
 365 ibt_status_t ibt_query_ud_dest(ibt_ud_dest_hdl_t ud_dest,
 366     ibt_ud_dest_query_attr_t *dest_attrs);
 367
 368
 369 /*
 370  * ibt_is_privileged_ud_dest
 371  *      Determine if a UD destination Handle is a privileged handle.
 372  */
 373 boolean_t ibt_is_privileged_ud_dest(ibt_ud_dest_hdl_t ud_dest);
 374
 375
 376 /*
 377  * ibt_update_channel_qkey
 378  *
 379  * ibt_update_channel_qkey() set's the Qkey in the specified channel context
 380  * to the Qkey in the specified destination handle. This function can be used
 381  * to enable sends to a privileged destination. All posted Send Work Requests
 382  * that contain a privileged destination handle now use the Qkey in the
 383  * channel context.
 384  *
 385  * ibt_update_channel_qkey() can also be used to enable the caller to receive
 386  * from the specified remote destination on the specified channel.
 387  *
 388  */
 389 ibt_status_t ibt_update_channel_qkey(ibt_channel_hdl_t ud_chan,
 390     ibt_ud_dest_hdl_t ud_dest);
 391
 392
 393 /*
 394  * ibt_set_chan_private()
 395  * ibt_get_chan_private()
 396  *      Set/get a pointer to client private data.
 397  *      Applicable for both RC and UD channels.
 398  */
 399 void ibt_set_chan_private(ibt_channel_hdl_t chan, void *clnt_private);
 400
 401 void *ibt_get_chan_private(ibt_channel_hdl_t chan);
 402
 403 /*
 404  * ibt_channel_to_hca_guid()
 405  *
 406  *      A helper function to retrieve HCA GUID for the specified Channel.
 407  */
 408 ib_guid_t ibt_channel_to_hca_guid(ibt_channel_hdl_t chan);
 409
 410 #ifdef __cplusplus
 411 }
 412 #endif
 413
 414 #endif /* _SYS_IB_IBTL_IBTI_H */