drivers/infiniband/hw/erdma/erdma_hw.h

   1 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
   2
   3 /* Authors: Cheng Xu <chengyou@linux.alibaba.com> */
   4 /*          Kai Shen <kaishen@linux.alibaba.com> */
   5 /* Copyright (c) 2020-2022, Alibaba Group. */
   6
   7 #ifndef __ERDMA_HW_H__
   8 #define __ERDMA_HW_H__
   9
  10 #include <linux/kernel.h>
  11 #include <linux/types.h>
  12
  13 /* PCIe device related definition. */
  14 #define ERDMA_PCI_WIDTH 64
  15 #define ERDMA_FUNC_BAR 0
  16 #define ERDMA_MISX_BAR 2
  17
  18 #define ERDMA_BAR_MASK (BIT(ERDMA_FUNC_BAR) | BIT(ERDMA_MISX_BAR))
  19
  20 /* MSI-X related. */
  21 #define ERDMA_NUM_MSIX_VEC 32U
  22 #define ERDMA_MSIX_VECTOR_CMDQ 0
  23
  24 /* PCIe Bar0 Registers. */
  25 #define ERDMA_REGS_VERSION_REG 0x0
  26 #define ERDMA_REGS_DEV_CTRL_REG 0x10
  27 #define ERDMA_REGS_DEV_ST_REG 0x14
  28 #define ERDMA_REGS_NETDEV_MAC_L_REG 0x18
  29 #define ERDMA_REGS_NETDEV_MAC_H_REG 0x1C
  30 #define ERDMA_REGS_CMDQ_SQ_ADDR_L_REG 0x20
  31 #define ERDMA_REGS_CMDQ_SQ_ADDR_H_REG 0x24
  32 #define ERDMA_REGS_CMDQ_CQ_ADDR_L_REG 0x28
  33 #define ERDMA_REGS_CMDQ_CQ_ADDR_H_REG 0x2C
  34 #define ERDMA_REGS_CMDQ_DEPTH_REG 0x30
  35 #define ERDMA_REGS_CMDQ_EQ_DEPTH_REG 0x34
  36 #define ERDMA_REGS_CMDQ_EQ_ADDR_L_REG 0x38
  37 #define ERDMA_REGS_CMDQ_EQ_ADDR_H_REG 0x3C
  38 #define ERDMA_REGS_AEQ_ADDR_L_REG 0x40
  39 #define ERDMA_REGS_AEQ_ADDR_H_REG 0x44
  40 #define ERDMA_REGS_AEQ_DEPTH_REG 0x48
  41 #define ERDMA_REGS_GRP_NUM_REG 0x4c
  42 #define ERDMA_REGS_AEQ_DB_REG 0x50
  43 #define ERDMA_CMDQ_SQ_DB_HOST_ADDR_REG 0x60
  44 #define ERDMA_CMDQ_CQ_DB_HOST_ADDR_REG 0x68
  45 #define ERDMA_CMDQ_EQ_DB_HOST_ADDR_REG 0x70
  46 #define ERDMA_AEQ_DB_HOST_ADDR_REG 0x78
  47 #define ERDMA_REGS_STATS_TSO_IN_PKTS_REG 0x80
  48 #define ERDMA_REGS_STATS_TSO_OUT_PKTS_REG 0x88
  49 #define ERDMA_REGS_STATS_TSO_OUT_BYTES_REG 0x90
  50 #define ERDMA_REGS_STATS_TX_DROP_PKTS_REG 0x98
  51 #define ERDMA_REGS_STATS_TX_BPS_METER_DROP_PKTS_REG 0xa0
  52 #define ERDMA_REGS_STATS_TX_PPS_METER_DROP_PKTS_REG 0xa8
  53 #define ERDMA_REGS_STATS_RX_PKTS_REG 0xc0
  54 #define ERDMA_REGS_STATS_RX_BYTES_REG 0xc8
  55 #define ERDMA_REGS_STATS_RX_DROP_PKTS_REG 0xd0
  56 #define ERDMA_REGS_STATS_RX_BPS_METER_DROP_PKTS_REG 0xd8
  57 #define ERDMA_REGS_STATS_RX_PPS_METER_DROP_PKTS_REG 0xe0
  58 #define ERDMA_REGS_CEQ_DB_BASE_REG 0x100
  59 #define ERDMA_CMDQ_SQDB_REG 0x200
  60 #define ERDMA_CMDQ_CQDB_REG 0x300
  61
  62 /* DEV_CTRL_REG details. */
  63 #define ERDMA_REG_DEV_CTRL_RESET_MASK 0x00000001
  64 #define ERDMA_REG_DEV_CTRL_INIT_MASK 0x00000002
  65
  66 /* DEV_ST_REG details. */
  67 #define ERDMA_REG_DEV_ST_RESET_DONE_MASK 0x00000001U
  68 #define ERDMA_REG_DEV_ST_INIT_DONE_MASK 0x00000002U
  69
  70 /* eRDMA PCIe DBs definition. */
  71 #define ERDMA_BAR_DB_SPACE_BASE 4096
  72
  73 #define ERDMA_BAR_SQDB_SPACE_OFFSET ERDMA_BAR_DB_SPACE_BASE
  74 #define ERDMA_BAR_SQDB_SPACE_SIZE (384 * 1024)
  75
  76 #define ERDMA_BAR_RQDB_SPACE_OFFSET \
  77         (ERDMA_BAR_SQDB_SPACE_OFFSET + ERDMA_BAR_SQDB_SPACE_SIZE)
  78 #define ERDMA_BAR_RQDB_SPACE_SIZE (96 * 1024)
  79
  80 #define ERDMA_BAR_CQDB_SPACE_OFFSET \
  81         (ERDMA_BAR_RQDB_SPACE_OFFSET + ERDMA_BAR_RQDB_SPACE_SIZE)
  82
  83 #define ERDMA_SDB_SHARED_PAGE_INDEX 95
  84
  85 /* Doorbell related. */
  86 #define ERDMA_DB_SIZE 8
  87
  88 #define ERDMA_CQDB_IDX_MASK GENMASK_ULL(63, 56)
  89 #define ERDMA_CQDB_CQN_MASK GENMASK_ULL(55, 32)
  90 #define ERDMA_CQDB_ARM_MASK BIT_ULL(31)
  91 #define ERDMA_CQDB_SOL_MASK BIT_ULL(30)
  92 #define ERDMA_CQDB_CMDSN_MASK GENMASK_ULL(29, 28)
  93 #define ERDMA_CQDB_CI_MASK GENMASK_ULL(23, 0)
  94
  95 #define ERDMA_EQDB_ARM_MASK BIT(31)
  96 #define ERDMA_EQDB_CI_MASK GENMASK_ULL(23, 0)
  97
  98 #define ERDMA_PAGE_SIZE_SUPPORT 0x7FFFF000
  99
 100 /* Hardware page size definition */
 101 #define ERDMA_HW_PAGE_SHIFT 12
 102 #define ERDMA_HW_PAGE_SIZE 4096
 103
 104 /* WQE related. */
 105 #define EQE_SIZE 16
 106 #define EQE_SHIFT 4
 107 #define RQE_SIZE 32
 108 #define RQE_SHIFT 5
 109 #define CQE_SIZE 32
 110 #define CQE_SHIFT 5
 111 #define SQEBB_SIZE 32
 112 #define SQEBB_SHIFT 5
 113 #define SQEBB_MASK (~(SQEBB_SIZE - 1))
 114 #define SQEBB_ALIGN(size) ((size + SQEBB_SIZE - 1) & SQEBB_MASK)
 115 #define SQEBB_COUNT(size) (SQEBB_ALIGN(size) >> SQEBB_SHIFT)
 116
 117 #define ERDMA_MAX_SQE_SIZE 128
 118 #define ERDMA_MAX_WQEBB_PER_SQE 4
 119
 120 /* CMDQ related. */
 121 #define ERDMA_CMDQ_MAX_OUTSTANDING 128
 122 #define ERDMA_CMDQ_SQE_SIZE 128
 123
 124 /* cmdq sub module definition. */
 125 enum CMDQ_WQE_SUB_MOD {
 126         CMDQ_SUBMOD_RDMA = 0,
 127         CMDQ_SUBMOD_COMMON = 1
 128 };
 129
 130 enum CMDQ_RDMA_OPCODE {
 131         CMDQ_OPCODE_QUERY_DEVICE = 0,
 132         CMDQ_OPCODE_CREATE_QP = 1,
 133         CMDQ_OPCODE_DESTROY_QP = 2,
 134         CMDQ_OPCODE_MODIFY_QP = 3,
 135         CMDQ_OPCODE_CREATE_CQ = 4,
 136         CMDQ_OPCODE_DESTROY_CQ = 5,
 137         CMDQ_OPCODE_REFLUSH = 6,
 138         CMDQ_OPCODE_REG_MR = 8,
 139         CMDQ_OPCODE_DEREG_MR = 9
 140 };
 141
 142 enum CMDQ_COMMON_OPCODE {
 143         CMDQ_OPCODE_CREATE_EQ = 0,
 144         CMDQ_OPCODE_DESTROY_EQ = 1,
 145         CMDQ_OPCODE_QUERY_FW_INFO = 2,
 146         CMDQ_OPCODE_CONF_MTU = 3,
 147         CMDQ_OPCODE_GET_STATS = 4,
 148         CMDQ_OPCODE_CONF_DEVICE = 5,
 149         CMDQ_OPCODE_ALLOC_DB = 8,
 150         CMDQ_OPCODE_FREE_DB = 9,
 151 };
 152
 153 /* cmdq-SQE HDR */
 154 #define ERDMA_CMD_HDR_WQEBB_CNT_MASK GENMASK_ULL(54, 52)
 155 #define ERDMA_CMD_HDR_CONTEXT_COOKIE_MASK GENMASK_ULL(47, 32)
 156 #define ERDMA_CMD_HDR_SUB_MOD_MASK GENMASK_ULL(25, 24)
 157 #define ERDMA_CMD_HDR_OPCODE_MASK GENMASK_ULL(23, 16)
 158 #define ERDMA_CMD_HDR_WQEBB_INDEX_MASK GENMASK_ULL(15, 0)
 159
 160 struct erdma_cmdq_destroy_cq_req {
 161         u64 hdr;
 162         u32 cqn;
 163 };
 164
 165 #define ERDMA_EQ_TYPE_AEQ 0
 166 #define ERDMA_EQ_TYPE_CEQ 1
 167
 168 struct erdma_cmdq_create_eq_req {
 169         u64 hdr;
 170         u64 qbuf_addr;
 171         u8 vector_idx;
 172         u8 eqn;
 173         u8 depth;
 174         u8 qtype;
 175         u32 db_dma_addr_l;
 176         u32 db_dma_addr_h;
 177 };
 178
 179 struct erdma_cmdq_destroy_eq_req {
 180         u64 hdr;
 181         u64 rsvd0;
 182         u8 vector_idx;
 183         u8 eqn;
 184         u8 rsvd1;
 185         u8 qtype;
 186 };
 187
 188 /* config device cfg */
 189 #define ERDMA_CMD_CONFIG_DEVICE_PS_EN_MASK BIT(31)
 190 #define ERDMA_CMD_CONFIG_DEVICE_PGSHIFT_MASK GENMASK(4, 0)
 191
 192 struct erdma_cmdq_config_device_req {
 193         u64 hdr;
 194         u32 cfg;
 195         u32 rsvd[5];
 196 };
 197
 198 struct erdma_cmdq_config_mtu_req {
 199         u64 hdr;
 200         u32 mtu;
 201 };
 202
 203 /* ext db requests(alloc and free) cfg */
 204 #define ERDMA_CMD_EXT_DB_CQ_EN_MASK BIT(2)
 205 #define ERDMA_CMD_EXT_DB_RQ_EN_MASK BIT(1)
 206 #define ERDMA_CMD_EXT_DB_SQ_EN_MASK BIT(0)
 207
 208 struct erdma_cmdq_ext_db_req {
 209         u64 hdr;
 210         u32 cfg;
 211         u16 rdb_off;
 212         u16 sdb_off;
 213         u16 rsvd0;
 214         u16 cdb_off;
 215         u32 rsvd1[3];
 216 };
 217
 218 /* alloc db response qword 0 definition */
 219 #define ERDMA_CMD_ALLOC_DB_RESP_RDB_MASK GENMASK_ULL(63, 48)
 220 #define ERDMA_CMD_ALLOC_DB_RESP_CDB_MASK GENMASK_ULL(47, 32)
 221 #define ERDMA_CMD_ALLOC_DB_RESP_SDB_MASK GENMASK_ULL(15, 0)
 222
 223 /* create_cq cfg0 */
 224 #define ERDMA_CMD_CREATE_CQ_DEPTH_MASK GENMASK(31, 24)
 225 #define ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK GENMASK(23, 20)
 226 #define ERDMA_CMD_CREATE_CQ_CQN_MASK GENMASK(19, 0)
 227
 228 /* create_cq cfg1 */
 229 #define ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK GENMASK(31, 16)
 230 #define ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK BIT(15)
 231 #define ERDMA_CMD_CREATE_CQ_MTT_DB_CFG_MASK BIT(11)
 232 #define ERDMA_CMD_CREATE_CQ_EQN_MASK GENMASK(9, 0)
 233
 234 /* create_cq cfg2 */
 235 #define ERDMA_CMD_CREATE_CQ_DB_CFG_MASK GENMASK(15, 0)
 236
 237 struct erdma_cmdq_create_cq_req {
 238         u64 hdr;
 239         u32 cfg0;
 240         u32 qbuf_addr_l;
 241         u32 qbuf_addr_h;
 242         u32 cfg1;
 243         u64 cq_dbrec_dma;
 244         u32 first_page_offset;
 245         u32 cfg2;
 246 };
 247
 248 /* regmr/deregmr cfg0 */
 249 #define ERDMA_CMD_MR_VALID_MASK BIT(31)
 250 #define ERDMA_CMD_MR_VERSION_MASK GENMASK(30, 28)
 251 #define ERDMA_CMD_MR_KEY_MASK GENMASK(27, 20)
 252 #define ERDMA_CMD_MR_MPT_IDX_MASK GENMASK(19, 0)
 253
 254 /* regmr cfg1 */
 255 #define ERDMA_CMD_REGMR_PD_MASK GENMASK(31, 12)
 256 #define ERDMA_CMD_REGMR_TYPE_MASK GENMASK(7, 6)
 257 #define ERDMA_CMD_REGMR_RIGHT_MASK GENMASK(5, 1)
 258
 259 /* regmr cfg2 */
 260 #define ERDMA_CMD_REGMR_PAGESIZE_MASK GENMASK(31, 27)
 261 #define ERDMA_CMD_REGMR_MTT_PAGESIZE_MASK GENMASK(26, 24)
 262 #define ERDMA_CMD_REGMR_MTT_LEVEL_MASK GENMASK(21, 20)
 263 #define ERDMA_CMD_REGMR_MTT_CNT_MASK GENMASK(19, 0)
 264
 265 struct erdma_cmdq_reg_mr_req {
 266         u64 hdr;
 267         u32 cfg0;
 268         u32 cfg1;
 269         u64 start_va;
 270         u32 size;
 271         u32 cfg2;
 272         union {
 273                 u64 phy_addr[4];
 274                 struct {
 275                         u64 rsvd;
 276                         u32 size_h;
 277                         u32 mtt_cnt_h;
 278                 };
 279         };
 280 };
 281
 282 struct erdma_cmdq_dereg_mr_req {
 283         u64 hdr;
 284         u32 cfg;
 285 };
 286
 287 /* modify qp cfg */
 288 #define ERDMA_CMD_MODIFY_QP_STATE_MASK GENMASK(31, 24)
 289 #define ERDMA_CMD_MODIFY_QP_CC_MASK GENMASK(23, 20)
 290 #define ERDMA_CMD_MODIFY_QP_QPN_MASK GENMASK(19, 0)
 291
 292 struct erdma_cmdq_modify_qp_req {
 293         u64 hdr;
 294         u32 cfg;
 295         u32 cookie;
 296         __be32 dip;
 297         __be32 sip;
 298         __be16 sport;
 299         __be16 dport;
 300         u32 send_nxt;
 301         u32 recv_nxt;
 302 };
 303
 304 /* create qp cfg0 */
 305 #define ERDMA_CMD_CREATE_QP_SQ_DEPTH_MASK GENMASK(31, 20)
 306 #define ERDMA_CMD_CREATE_QP_QPN_MASK GENMASK(19, 0)
 307
 308 /* create qp cfg1 */
 309 #define ERDMA_CMD_CREATE_QP_RQ_DEPTH_MASK GENMASK(31, 20)
 310 #define ERDMA_CMD_CREATE_QP_PD_MASK GENMASK(19, 0)
 311
 312 /* create qp cqn_mtt_cfg */
 313 #define ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK GENMASK(31, 28)
 314 #define ERDMA_CMD_CREATE_QP_DB_CFG_MASK BIT(25)
 315 #define ERDMA_CMD_CREATE_QP_CQN_MASK GENMASK(23, 0)
 316
 317 /* create qp mtt_cfg */
 318 #define ERDMA_CMD_CREATE_QP_PAGE_OFFSET_MASK GENMASK(31, 12)
 319 #define ERDMA_CMD_CREATE_QP_MTT_CNT_MASK GENMASK(11, 1)
 320 #define ERDMA_CMD_CREATE_QP_MTT_LEVEL_MASK BIT(0)
 321
 322 /* create qp db cfg */
 323 #define ERDMA_CMD_CREATE_QP_SQDB_CFG_MASK GENMASK(31, 16)
 324 #define ERDMA_CMD_CREATE_QP_RQDB_CFG_MASK GENMASK(15, 0)
 325
 326 #define ERDMA_CMDQ_CREATE_QP_RESP_COOKIE_MASK GENMASK_ULL(31, 0)
 327
 328 struct erdma_cmdq_create_qp_req {
 329         u64 hdr;
 330         u32 cfg0;
 331         u32 cfg1;
 332         u32 sq_cqn_mtt_cfg;
 333         u32 rq_cqn_mtt_cfg;
 334         u64 sq_buf_addr;
 335         u64 rq_buf_addr;
 336         u32 sq_mtt_cfg;
 337         u32 rq_mtt_cfg;
 338         u64 sq_dbrec_dma;
 339         u64 rq_dbrec_dma;
 340
 341         u64 sq_mtt_entry[3];
 342         u64 rq_mtt_entry[3];
 343
 344         u32 db_cfg;
 345 };
 346
 347 struct erdma_cmdq_destroy_qp_req {
 348         u64 hdr;
 349         u32 qpn;
 350 };
 351
 352 struct erdma_cmdq_reflush_req {
 353         u64 hdr;
 354         u32 qpn;
 355         u32 sq_pi;
 356         u32 rq_pi;
 357 };
 358
 359 #define ERDMA_HW_RESP_SIZE 256
 360
 361 struct erdma_cmdq_query_req {
 362         u64 hdr;
 363         u32 rsvd;
 364         u32 index;
 365
 366         u64 target_addr;
 367         u32 target_length;
 368 };
 369
 370 #define ERDMA_HW_RESP_MAGIC 0x5566
 371
 372 struct erdma_cmdq_query_resp_hdr {
 373         u16 magic;
 374         u8 ver;
 375         u8 length;
 376
 377         u32 index;
 378         u32 rsvd[2];
 379 };
 380
 381 struct erdma_cmdq_query_stats_resp {
 382         struct erdma_cmdq_query_resp_hdr hdr;
 383
 384         u64 tx_req_cnt;
 385         u64 tx_packets_cnt;
 386         u64 tx_bytes_cnt;
 387         u64 tx_drop_packets_cnt;
 388         u64 tx_bps_meter_drop_packets_cnt;
 389         u64 tx_pps_meter_drop_packets_cnt;
 390         u64 rx_packets_cnt;
 391         u64 rx_bytes_cnt;
 392         u64 rx_drop_packets_cnt;
 393         u64 rx_bps_meter_drop_packets_cnt;
 394         u64 rx_pps_meter_drop_packets_cnt;
 395 };
 396
 397 /* cap qword 0 definition */
 398 #define ERDMA_CMD_DEV_CAP_MAX_CQE_MASK GENMASK_ULL(47, 40)
 399 #define ERDMA_CMD_DEV_CAP_FLAGS_MASK GENMASK_ULL(31, 24)
 400 #define ERDMA_CMD_DEV_CAP_MAX_RECV_WR_MASK GENMASK_ULL(23, 16)
 401 #define ERDMA_CMD_DEV_CAP_MAX_MR_SIZE_MASK GENMASK_ULL(7, 0)
 402
 403 /* cap qword 1 definition */
 404 #define ERDMA_CMD_DEV_CAP_DMA_LOCAL_KEY_MASK GENMASK_ULL(63, 32)
 405 #define ERDMA_CMD_DEV_CAP_DEFAULT_CC_MASK GENMASK_ULL(31, 28)
 406 #define ERDMA_CMD_DEV_CAP_QBLOCK_MASK GENMASK_ULL(27, 16)
 407 #define ERDMA_CMD_DEV_CAP_MAX_MW_MASK GENMASK_ULL(7, 0)
 408
 409 #define ERDMA_NQP_PER_QBLOCK 1024
 410
 411 enum {
 412         ERDMA_DEV_CAP_FLAGS_ATOMIC = 1 << 7,
 413         ERDMA_DEV_CAP_FLAGS_MTT_VA = 1 << 5,
 414         ERDMA_DEV_CAP_FLAGS_EXTEND_DB = 1 << 3,
 415 };
 416
 417 #define ERDMA_CMD_INFO0_FW_VER_MASK GENMASK_ULL(31, 0)
 418
 419 /* CQE hdr */
 420 #define ERDMA_CQE_HDR_OWNER_MASK BIT(31)
 421 #define ERDMA_CQE_HDR_OPCODE_MASK GENMASK(23, 16)
 422 #define ERDMA_CQE_HDR_QTYPE_MASK GENMASK(15, 8)
 423 #define ERDMA_CQE_HDR_SYNDROME_MASK GENMASK(7, 0)
 424
 425 #define ERDMA_CQE_QTYPE_SQ 0
 426 #define ERDMA_CQE_QTYPE_RQ 1
 427 #define ERDMA_CQE_QTYPE_CMDQ 2
 428
 429 struct erdma_cqe {
 430         __be32 hdr;
 431         __be32 qe_idx;
 432         __be32 qpn;
 433         union {
 434                 __le32 imm_data;
 435                 __be32 inv_rkey;
 436         };
 437         __be32 size;
 438         __be32 rsvd[3];
 439 };
 440
 441 struct erdma_sge {
 442         __aligned_le64 addr;
 443         __le32 length;
 444         __le32 key;
 445 };
 446
 447 /* Receive Queue Element */
 448 struct erdma_rqe {
 449         __le16 qe_idx;
 450         __le16 rsvd0;
 451         __le32 qpn;
 452         __le32 rsvd1;
 453         __le32 rsvd2;
 454         __le64 to;
 455         __le32 length;
 456         __le32 stag;
 457 };
 458
 459 /* SQE */
 460 #define ERDMA_SQE_HDR_SGL_LEN_MASK GENMASK_ULL(63, 56)
 461 #define ERDMA_SQE_HDR_WQEBB_CNT_MASK GENMASK_ULL(54, 52)
 462 #define ERDMA_SQE_HDR_QPN_MASK GENMASK_ULL(51, 32)
 463 #define ERDMA_SQE_HDR_OPCODE_MASK GENMASK_ULL(31, 27)
 464 #define ERDMA_SQE_HDR_DWQE_MASK BIT_ULL(26)
 465 #define ERDMA_SQE_HDR_INLINE_MASK BIT_ULL(25)
 466 #define ERDMA_SQE_HDR_FENCE_MASK BIT_ULL(24)
 467 #define ERDMA_SQE_HDR_SE_MASK BIT_ULL(23)
 468 #define ERDMA_SQE_HDR_CE_MASK BIT_ULL(22)
 469 #define ERDMA_SQE_HDR_WQEBB_INDEX_MASK GENMASK_ULL(15, 0)
 470
 471 /* REG MR attrs */
 472 #define ERDMA_SQE_MR_ACCESS_MASK GENMASK(5, 1)
 473 #define ERDMA_SQE_MR_MTT_TYPE_MASK GENMASK(7, 6)
 474 #define ERDMA_SQE_MR_MTT_CNT_MASK GENMASK(31, 12)
 475
 476 struct erdma_write_sqe {
 477         __le64 hdr;
 478         __be32 imm_data;
 479         __le32 length;
 480
 481         __le32 sink_stag;
 482         __le32 sink_to_l;
 483         __le32 sink_to_h;
 484
 485         __le32 rsvd;
 486
 487         struct erdma_sge sgl[];
 488 };
 489
 490 struct erdma_send_sqe {
 491         __le64 hdr;
 492         union {
 493                 __be32 imm_data;
 494                 __le32 invalid_stag;
 495         };
 496
 497         __le32 length;
 498         struct erdma_sge sgl[];
 499 };
 500
 501 struct erdma_readreq_sqe {
 502         __le64 hdr;
 503         __le32 invalid_stag;
 504         __le32 length;
 505         __le32 sink_stag;
 506         __le32 sink_to_l;
 507         __le32 sink_to_h;
 508         __le32 rsvd;
 509 };
 510
 511 struct erdma_atomic_sqe {
 512         __le64 hdr;
 513         __le64 rsvd;
 514         __le64 fetchadd_swap_data;
 515         __le64 cmp_data;
 516
 517         struct erdma_sge remote;
 518         struct erdma_sge sgl;
 519 };
 520
 521 struct erdma_reg_mr_sqe {
 522         __le64 hdr;
 523         __le64 addr;
 524         __le32 length;
 525         __le32 stag;
 526         __le32 attrs;
 527         __le32 rsvd;
 528 };
 529
 530 /* EQ related. */
 531 #define ERDMA_DEFAULT_EQ_DEPTH 4096
 532
 533 /* ceqe */
 534 #define ERDMA_CEQE_HDR_DB_MASK BIT_ULL(63)
 535 #define ERDMA_CEQE_HDR_PI_MASK GENMASK_ULL(55, 32)
 536 #define ERDMA_CEQE_HDR_O_MASK BIT_ULL(31)
 537 #define ERDMA_CEQE_HDR_CQN_MASK GENMASK_ULL(19, 0)
 538
 539 /* aeqe */
 540 #define ERDMA_AEQE_HDR_O_MASK BIT(31)
 541 #define ERDMA_AEQE_HDR_TYPE_MASK GENMASK(23, 16)
 542 #define ERDMA_AEQE_HDR_SUBTYPE_MASK GENMASK(7, 0)
 543
 544 #define ERDMA_AE_TYPE_QP_FATAL_EVENT 0
 545 #define ERDMA_AE_TYPE_QP_ERQ_ERR_EVENT 1
 546 #define ERDMA_AE_TYPE_ACC_ERR_EVENT 2
 547 #define ERDMA_AE_TYPE_CQ_ERR 3
 548 #define ERDMA_AE_TYPE_OTHER_ERROR 4
 549
 550 struct erdma_aeqe {
 551         __le32 hdr;
 552         __le32 event_data0;
 553         __le32 event_data1;
 554         __le32 rsvd;
 555 };
 556
 557 enum erdma_opcode {
 558         ERDMA_OP_WRITE = 0,
 559         ERDMA_OP_READ = 1,
 560         ERDMA_OP_SEND = 2,
 561         ERDMA_OP_SEND_WITH_IMM = 3,
 562
 563         ERDMA_OP_RECEIVE = 4,
 564         ERDMA_OP_RECV_IMM = 5,
 565         ERDMA_OP_RECV_INV = 6,
 566
 567         ERDMA_OP_RSVD0 = 7,
 568         ERDMA_OP_RSVD1 = 8,
 569         ERDMA_OP_WRITE_WITH_IMM = 9,
 570
 571         ERDMA_OP_RSVD2 = 10,
 572         ERDMA_OP_RSVD3 = 11,
 573
 574         ERDMA_OP_RSP_SEND_IMM = 12,
 575         ERDMA_OP_SEND_WITH_INV = 13,
 576
 577         ERDMA_OP_REG_MR = 14,
 578         ERDMA_OP_LOCAL_INV = 15,
 579         ERDMA_OP_READ_WITH_INV = 16,
 580         ERDMA_OP_ATOMIC_CAS = 17,
 581         ERDMA_OP_ATOMIC_FAA = 18,
 582         ERDMA_NUM_OPCODES = 19,
 583         ERDMA_OP_INVALID = ERDMA_NUM_OPCODES + 1
 584 };
 585
 586 enum erdma_wc_status {
 587         ERDMA_WC_SUCCESS = 0,
 588         ERDMA_WC_GENERAL_ERR = 1,
 589         ERDMA_WC_RECV_WQE_FORMAT_ERR = 2,
 590         ERDMA_WC_RECV_STAG_INVALID_ERR = 3,
 591         ERDMA_WC_RECV_ADDR_VIOLATION_ERR = 4,
 592         ERDMA_WC_RECV_RIGHT_VIOLATION_ERR = 5,
 593         ERDMA_WC_RECV_PDID_ERR = 6,
 594         ERDMA_WC_RECV_WARRPING_ERR = 7,
 595         ERDMA_WC_SEND_WQE_FORMAT_ERR = 8,
 596         ERDMA_WC_SEND_WQE_ORD_EXCEED = 9,
 597         ERDMA_WC_SEND_STAG_INVALID_ERR = 10,
 598         ERDMA_WC_SEND_ADDR_VIOLATION_ERR = 11,
 599         ERDMA_WC_SEND_RIGHT_VIOLATION_ERR = 12,
 600         ERDMA_WC_SEND_PDID_ERR = 13,
 601         ERDMA_WC_SEND_WARRPING_ERR = 14,
 602         ERDMA_WC_FLUSH_ERR = 15,
 603         ERDMA_WC_RETRY_EXC_ERR = 16,
 604         ERDMA_NUM_WC_STATUS
 605 };
 606
 607 enum erdma_vendor_err {
 608         ERDMA_WC_VENDOR_NO_ERR = 0,
 609         ERDMA_WC_VENDOR_INVALID_RQE = 1,
 610         ERDMA_WC_VENDOR_RQE_INVALID_STAG = 2,
 611         ERDMA_WC_VENDOR_RQE_ADDR_VIOLATION = 3,
 612         ERDMA_WC_VENDOR_RQE_ACCESS_RIGHT_ERR = 4,
 613         ERDMA_WC_VENDOR_RQE_INVALID_PD = 5,
 614         ERDMA_WC_VENDOR_RQE_WRAP_ERR = 6,
 615         ERDMA_WC_VENDOR_INVALID_SQE = 0x20,
 616         ERDMA_WC_VENDOR_ZERO_ORD = 0x21,
 617         ERDMA_WC_VENDOR_SQE_INVALID_STAG = 0x30,
 618         ERDMA_WC_VENDOR_SQE_ADDR_VIOLATION = 0x31,
 619         ERDMA_WC_VENDOR_SQE_ACCESS_ERR = 0x32,
 620         ERDMA_WC_VENDOR_SQE_INVALID_PD = 0x33,
 621         ERDMA_WC_VENDOR_SQE_WARP_ERR = 0x34
 622 };
 623
 624 #endif