drivers/infiniband/hw/efa/efa_io_defs.h

   1 /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
   2 /*
   3  * Copyright 2018-2024 Amazon.com, Inc. or its affiliates. All rights reserved.
   4  */
   5
   6 #ifndef _EFA_IO_H_
   7 #define _EFA_IO_H_
   8
   9 #define EFA_IO_TX_DESC_NUM_BUFS              2
  10 #define EFA_IO_TX_DESC_NUM_RDMA_BUFS         1
  11 #define EFA_IO_TX_DESC_INLINE_MAX_SIZE       32
  12 #define EFA_IO_TX_DESC_IMM_DATA_SIZE         4
  13 #define EFA_IO_TX_DESC_INLINE_PBL_SIZE       1
  14
  15 enum efa_io_queue_type {
  16         /* send queue (of a QP) */
  17         EFA_IO_SEND_QUEUE                           = 1,
  18         /* recv queue (of a QP) */
  19         EFA_IO_RECV_QUEUE                           = 2,
  20 };
  21
  22 enum efa_io_send_op_type {
  23         /* send message */
  24         EFA_IO_SEND                                 = 0,
  25         /* RDMA read */
  26         EFA_IO_RDMA_READ                            = 1,
  27         /* RDMA write */
  28         EFA_IO_RDMA_WRITE                           = 2,
  29         /* Fast MR registration */
  30         EFA_IO_FAST_REG                             = 3,
  31         /* Fast MR invalidation */
  32         EFA_IO_FAST_INV                             = 4,
  33 };
  34
  35 enum efa_io_comp_status {
  36         /* Successful completion */
  37         EFA_IO_COMP_STATUS_OK                       = 0,
  38         /* Flushed during QP destroy */
  39         EFA_IO_COMP_STATUS_FLUSHED                  = 1,
  40         /* Internal QP error */
  41         EFA_IO_COMP_STATUS_LOCAL_ERROR_QP_INTERNAL_ERROR = 2,
  42         /* Unsupported operation */
  43         EFA_IO_COMP_STATUS_LOCAL_ERROR_UNSUPPORTED_OP = 3,
  44         /* Bad AH */
  45         EFA_IO_COMP_STATUS_LOCAL_ERROR_INVALID_AH   = 4,
  46         /* LKEY not registered or does not match IOVA */
  47         EFA_IO_COMP_STATUS_LOCAL_ERROR_INVALID_LKEY = 5,
  48         /* Message too long */
  49         EFA_IO_COMP_STATUS_LOCAL_ERROR_BAD_LENGTH   = 6,
  50         /* RKEY not registered or does not match remote IOVA */
  51         EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_ADDRESS = 7,
  52         /* Connection was reset by remote side */
  53         EFA_IO_COMP_STATUS_REMOTE_ERROR_ABORT       = 8,
  54         /* Bad dest QP number (QP does not exist or is in error state) */
  55         EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_DEST_QPN = 9,
  56         /* Destination resource not ready (no WQEs posted on RQ) */
  57         EFA_IO_COMP_STATUS_REMOTE_ERROR_RNR         = 10,
  58         /* Receiver SGL too short */
  59         EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_LENGTH  = 11,
  60         /* Unexpected status returned by responder */
  61         EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_STATUS  = 12,
  62         /* Unresponsive remote - was previously responsive */
  63         EFA_IO_COMP_STATUS_LOCAL_ERROR_UNRESP_REMOTE = 13,
  64         /* No valid AH at remote side (required for RDMA operations) */
  65         EFA_IO_COMP_STATUS_REMOTE_ERROR_UNKNOWN_PEER = 14,
  66         /* Unreachable remote - never received a response */
  67         EFA_IO_COMP_STATUS_LOCAL_ERROR_UNREACH_REMOTE = 15,
  68 };
  69
  70 enum efa_io_frwr_pbl_mode {
  71         EFA_IO_FRWR_INLINE_PBL                      = 0,
  72         EFA_IO_FRWR_DIRECT_PBL                      = 1,
  73 };
  74
  75 struct efa_io_tx_meta_desc {
  76         /* Verbs-generated Request ID */
  77         u16 req_id;
  78
  79         /*
  80          * control flags
  81          * 3:0 : op_type - enum efa_io_send_op_type
  82          * 4 : has_imm - immediate_data field carries valid
  83          *    data.
  84          * 5 : inline_msg - inline mode - inline message data
  85          *    follows this descriptor (no buffer descriptors).
  86          *    Note that it is different from immediate data
  87          * 6 : meta_extension - Extended metadata. MBZ
  88          * 7 : meta_desc - Indicates metadata descriptor.
  89          *    Must be set.
  90          */
  91         u8 ctrl1;
  92
  93         /*
  94          * control flags
  95          * 0 : phase
  96          * 1 : reserved25 - MBZ
  97          * 2 : first - Indicates first descriptor in
  98          *    transaction. Must be set.
  99          * 3 : last - Indicates last descriptor in
 100          *    transaction. Must be set.
 101          * 4 : comp_req - Indicates whether completion should
 102          *    be posted, after packet is transmitted. Valid only
 103          *    for the first descriptor
 104          * 7:5 : reserved29 - MBZ
 105          */
 106         u8 ctrl2;
 107
 108         u16 dest_qp_num;
 109
 110         /*
 111          * If inline_msg bit is set, length of inline message in bytes,
 112          * otherwise length of SGL (number of buffers).
 113          */
 114         u16 length;
 115
 116         /*
 117          * immediate data: if has_imm is set, then this field is included within
 118          * Tx message and reported in remote Rx completion.
 119          */
 120         u32 immediate_data;
 121
 122         u16 ah;
 123
 124         u16 reserved;
 125
 126         /* Queue key */
 127         u32 qkey;
 128
 129         u8 reserved2[12];
 130 };
 131
 132 /*
 133  * Tx queue buffer descriptor, for any transport type. Preceded by metadata
 134  * descriptor.
 135  */
 136 struct efa_io_tx_buf_desc {
 137         /* length in bytes */
 138         u32 length;
 139
 140         /*
 141          * 23:0 : lkey - local memory translation key
 142          * 31:24 : reserved - MBZ
 143          */
 144         u32 lkey;
 145
 146         /* Buffer address bits[31:0] */
 147         u32 buf_addr_lo;
 148
 149         /* Buffer address bits[63:32] */
 150         u32 buf_addr_hi;
 151 };
 152
 153 struct efa_io_remote_mem_addr {
 154         /* length in bytes */
 155         u32 length;
 156
 157         /* remote memory translation key */
 158         u32 rkey;
 159
 160         /* Buffer address bits[31:0] */
 161         u32 buf_addr_lo;
 162
 163         /* Buffer address bits[63:32] */
 164         u32 buf_addr_hi;
 165 };
 166
 167 struct efa_io_rdma_req {
 168         /* Remote memory address */
 169         struct efa_io_remote_mem_addr remote_mem;
 170
 171         /* Local memory address */
 172         struct efa_io_tx_buf_desc local_mem[1];
 173 };
 174
 175 struct efa_io_fast_mr_reg_req {
 176         /* Updated local key of the MR after lkey/rkey increment */
 177         u32 lkey;
 178
 179         /*
 180          * permissions
 181          * 0 : local_write_enable - Local write permissions:
 182          *    must be set for RQ buffers and buffers posted for
 183          *    RDMA Read requests
 184          * 1 : remote_write_enable - Remote write
 185          *    permissions: must be set to enable RDMA write to
 186          *    the region
 187          * 2 : remote_read_enable - Remote read permissions:
 188          *    must be set to enable RDMA read from the region
 189          * 7:3 : reserved2 - MBZ
 190          */
 191         u8 permissions;
 192
 193         /*
 194          * control flags
 195          * 4:0 : phys_page_size_shift - page size is (1 <<
 196          *    phys_page_size_shift)
 197          * 6:5 : pbl_mode - enum efa_io_frwr_pbl_mode
 198          * 7 : reserved - MBZ
 199          */
 200         u8 flags;
 201
 202         /* MBZ */
 203         u8 reserved[2];
 204
 205         /* IO Virtual Address associated with this MR */
 206         u64 iova;
 207
 208         /* Memory region length, in bytes */
 209         u64 mr_length;
 210
 211         /* Physical Buffer List, each element is page-aligned. */
 212         union {
 213                 /*
 214                  * Inline array of physical page addresses (optimization
 215                  * for short region activation).
 216                  */
 217                 u64 inline_array[1];
 218
 219                 /* points to PBL (Currently only direct) */
 220                 u64 dma_addr;
 221         } pbl;
 222 };
 223
 224 struct efa_io_fast_mr_inv_req {
 225         /* Local key of the MR to invalidate */
 226         u32 lkey;
 227
 228         /* MBZ */
 229         u8 reserved[28];
 230 };
 231
 232 /*
 233  * Tx WQE, composed of tx meta descriptors followed by either tx buffer
 234  * descriptors or inline data
 235  */
 236 struct efa_io_tx_wqe {
 237         /* TX meta */
 238         struct efa_io_tx_meta_desc meta;
 239
 240         union {
 241                 /* Send buffer descriptors */
 242                 struct efa_io_tx_buf_desc sgl[2];
 243
 244                 u8 inline_data[32];
 245
 246                 /* RDMA local and remote memory addresses */
 247                 struct efa_io_rdma_req rdma_req;
 248
 249                 /* Fast registration */
 250                 struct efa_io_fast_mr_reg_req reg_mr_req;
 251
 252                 /* Fast invalidation */
 253                 struct efa_io_fast_mr_inv_req inv_mr_req;
 254         } data;
 255 };
 256
 257 /*
 258  * Rx buffer descriptor; RX WQE is composed of one or more RX buffer
 259  * descriptors.
 260  */
 261 struct efa_io_rx_desc {
 262         /* Buffer address bits[31:0] */
 263         u32 buf_addr_lo;
 264
 265         /* Buffer Pointer[63:32] */
 266         u32 buf_addr_hi;
 267
 268         /* Verbs-generated request id. */
 269         u16 req_id;
 270
 271         /* Length in bytes. */
 272         u16 length;
 273
 274         /*
 275          * LKey and control flags
 276          * 23:0 : lkey
 277          * 29:24 : reserved - MBZ
 278          * 30 : first - Indicates first descriptor in WQE
 279          * 31 : last - Indicates last descriptor in WQE
 280          */
 281         u32 lkey_ctrl;
 282 };
 283
 284 /* Common IO completion descriptor */
 285 struct efa_io_cdesc_common {
 286         /*
 287          * verbs-generated request ID, as provided in the completed tx or rx
 288          * descriptor.
 289          */
 290         u16 req_id;
 291
 292         u8 status;
 293
 294         /*
 295          * flags
 296          * 0 : phase - Phase bit
 297          * 2:1 : q_type - enum efa_io_queue_type: send/recv
 298          * 3 : has_imm - indicates that immediate data is
 299          *    present - for RX completions only
 300          * 6:4 : op_type - enum efa_io_send_op_type
 301          * 7 : unsolicited - indicates that there is no
 302          *    matching request - for RDMA with imm. RX only
 303          */
 304         u8 flags;
 305
 306         /* local QP number */
 307         u16 qp_num;
 308 };
 309
 310 /* Tx completion descriptor */
 311 struct efa_io_tx_cdesc {
 312         /* Common completion info */
 313         struct efa_io_cdesc_common common;
 314
 315         /* MBZ */
 316         u16 reserved16;
 317 };
 318
 319 /* Rx Completion Descriptor */
 320 struct efa_io_rx_cdesc {
 321         /* Common completion info */
 322         struct efa_io_cdesc_common common;
 323
 324         /* Transferred length bits[15:0] */
 325         u16 length;
 326
 327         /* Remote Address Handle FW index, 0xFFFF indicates invalid ah */
 328         u16 ah;
 329
 330         u16 src_qp_num;
 331
 332         /* Immediate data */
 333         u32 imm;
 334 };
 335
 336 /* Rx Completion Descriptor RDMA write info */
 337 struct efa_io_rx_cdesc_rdma_write {
 338         /* Transferred length bits[31:16] */
 339         u16 length_hi;
 340 };
 341
 342 /* Extended Rx Completion Descriptor */
 343 struct efa_io_rx_cdesc_ex {
 344         /* Base RX completion info */
 345         struct efa_io_rx_cdesc base;
 346
 347         union {
 348                 struct efa_io_rx_cdesc_rdma_write rdma_write;
 349
 350                 /*
 351                  * Valid only in case of unknown AH (0xFFFF) and CQ
 352                  * set_src_addr is enabled.
 353                  */
 354                 u8 src_addr[16];
 355         } u;
 356 };
 357
 358 /* tx_meta_desc */
 359 #define EFA_IO_TX_META_DESC_OP_TYPE_MASK                    GENMASK(3, 0)
 360 #define EFA_IO_TX_META_DESC_HAS_IMM_MASK                    BIT(4)
 361 #define EFA_IO_TX_META_DESC_INLINE_MSG_MASK                 BIT(5)
 362 #define EFA_IO_TX_META_DESC_META_EXTENSION_MASK             BIT(6)
 363 #define EFA_IO_TX_META_DESC_META_DESC_MASK                  BIT(7)
 364 #define EFA_IO_TX_META_DESC_PHASE_MASK                      BIT(0)
 365 #define EFA_IO_TX_META_DESC_FIRST_MASK                      BIT(2)
 366 #define EFA_IO_TX_META_DESC_LAST_MASK                       BIT(3)
 367 #define EFA_IO_TX_META_DESC_COMP_REQ_MASK                   BIT(4)
 368
 369 /* tx_buf_desc */
 370 #define EFA_IO_TX_BUF_DESC_LKEY_MASK                        GENMASK(23, 0)
 371
 372 /* fast_mr_reg_req */
 373 #define EFA_IO_FAST_MR_REG_REQ_LOCAL_WRITE_ENABLE_MASK      BIT(0)
 374 #define EFA_IO_FAST_MR_REG_REQ_REMOTE_WRITE_ENABLE_MASK     BIT(1)
 375 #define EFA_IO_FAST_MR_REG_REQ_REMOTE_READ_ENABLE_MASK      BIT(2)
 376 #define EFA_IO_FAST_MR_REG_REQ_PHYS_PAGE_SIZE_SHIFT_MASK    GENMASK(4, 0)
 377 #define EFA_IO_FAST_MR_REG_REQ_PBL_MODE_MASK                GENMASK(6, 5)
 378
 379 /* rx_desc */
 380 #define EFA_IO_RX_DESC_LKEY_MASK                            GENMASK(23, 0)
 381 #define EFA_IO_RX_DESC_FIRST_MASK                           BIT(30)
 382 #define EFA_IO_RX_DESC_LAST_MASK                            BIT(31)
 383
 384 /* cdesc_common */
 385 #define EFA_IO_CDESC_COMMON_PHASE_MASK                      BIT(0)
 386 #define EFA_IO_CDESC_COMMON_Q_TYPE_MASK                     GENMASK(2, 1)
 387 #define EFA_IO_CDESC_COMMON_HAS_IMM_MASK                    BIT(3)
 388 #define EFA_IO_CDESC_COMMON_OP_TYPE_MASK                    GENMASK(6, 4)
 389 #define EFA_IO_CDESC_COMMON_UNSOLICITED_MASK                BIT(7)
 390
 391 #endif /* _EFA_IO_H_ */