drivers/infiniband/hw/mana/mana_ib.h

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2 /*
   3  * Copyright (c) 2022 Microsoft Corporation. All rights reserved.
   4  */
   5
   6 #ifndef _MANA_IB_H_
   7 #define _MANA_IB_H_
   8
   9 #include <rdma/ib_verbs.h>
  10 #include <rdma/ib_mad.h>
  11 #include <rdma/ib_umem.h>
  12 #include <rdma/mana-abi.h>
  13 #include <rdma/uverbs_ioctl.h>
  14
  15 #include <net/mana/mana.h>
  16
  17 #define PAGE_SZ_BM                                                             \
  18         (SZ_4K | SZ_8K | SZ_16K | SZ_32K | SZ_64K | SZ_128K | SZ_256K |        \
  19          SZ_512K | SZ_1M | SZ_2M)
  20
  21 /* MANA doesn't have any limit for MR size */
  22 #define MANA_IB_MAX_MR_SIZE     U64_MAX
  23
  24 /*
  25  * The hardware limit of number of MRs is greater than maximum number of MRs
  26  * that can possibly represent in 24 bits
  27  */
  28 #define MANA_IB_MAX_MR          0xFFFFFFu
  29
  30 /*
  31  * The CA timeout is approx. 260ms (4us * 2^(DELAY))
  32  */
  33 #define MANA_CA_ACK_DELAY       16
  34
  35 struct mana_ib_adapter_caps {
  36         u32 max_sq_id;
  37         u32 max_rq_id;
  38         u32 max_cq_id;
  39         u32 max_qp_count;
  40         u32 max_cq_count;
  41         u32 max_mr_count;
  42         u32 max_pd_count;
  43         u32 max_inbound_read_limit;
  44         u32 max_outbound_read_limit;
  45         u32 mw_count;
  46         u32 max_srq_count;
  47         u32 max_qp_wr;
  48         u32 max_send_sge_count;
  49         u32 max_recv_sge_count;
  50         u32 max_inline_data_size;
  51 };
  52
  53 struct mana_ib_queue {
  54         struct ib_umem *umem;
  55         u64 gdma_region;
  56         u64 id;
  57 };
  58
  59 struct mana_ib_dev {
  60         struct ib_device ib_dev;
  61         struct gdma_dev *gdma_dev;
  62         mana_handle_t adapter_handle;
  63         struct gdma_queue *fatal_err_eq;
  64         struct gdma_queue **eqs;
  65         struct xarray qp_table_wq;
  66         struct mana_ib_adapter_caps adapter_caps;
  67 };
  68
  69 struct mana_ib_wq {
  70         struct ib_wq ibwq;
  71         struct mana_ib_queue queue;
  72         int wqe;
  73         u32 wq_buf_size;
  74         mana_handle_t rx_object;
  75 };
  76
  77 struct mana_ib_pd {
  78         struct ib_pd ibpd;
  79         u32 pdn;
  80         mana_handle_t pd_handle;
  81
  82         /* Mutex for sharing access to vport_use_count */
  83         struct mutex vport_mutex;
  84         int vport_use_count;
  85
  86         bool tx_shortform_allowed;
  87         u32 tx_vp_offset;
  88 };
  89
  90 struct mana_ib_mr {
  91         struct ib_mr ibmr;
  92         struct ib_umem *umem;
  93         mana_handle_t mr_handle;
  94 };
  95
  96 struct mana_ib_cq {
  97         struct ib_cq ibcq;
  98         struct mana_ib_queue queue;
  99         int cqe;
 100         u32 comp_vector;
 101         mana_handle_t  cq_handle;
 102 };
 103
 104 enum mana_rc_queue_type {
 105         MANA_RC_SEND_QUEUE_REQUESTER = 0,
 106         MANA_RC_SEND_QUEUE_RESPONDER,
 107         MANA_RC_SEND_QUEUE_FMR,
 108         MANA_RC_RECV_QUEUE_REQUESTER,
 109         MANA_RC_RECV_QUEUE_RESPONDER,
 110         MANA_RC_QUEUE_TYPE_MAX,
 111 };
 112
 113 struct mana_ib_rc_qp {
 114         struct mana_ib_queue queues[MANA_RC_QUEUE_TYPE_MAX];
 115 };
 116
 117 struct mana_ib_qp {
 118         struct ib_qp ibqp;
 119
 120         mana_handle_t qp_handle;
 121         union {
 122                 struct mana_ib_queue raw_sq;
 123                 struct mana_ib_rc_qp rc_qp;
 124         };
 125
 126         /* The port on the IB device, starting with 1 */
 127         u32 port;
 128
 129         refcount_t              refcount;
 130         struct completion       free;
 131 };
 132
 133 struct mana_ib_ucontext {
 134         struct ib_ucontext ibucontext;
 135         u32 doorbell;
 136 };
 137
 138 struct mana_ib_rwq_ind_table {
 139         struct ib_rwq_ind_table ib_ind_table;
 140 };
 141
 142 enum mana_ib_command_code {
 143         MANA_IB_GET_ADAPTER_CAP = 0x30001,
 144         MANA_IB_CREATE_ADAPTER  = 0x30002,
 145         MANA_IB_DESTROY_ADAPTER = 0x30003,
 146         MANA_IB_CONFIG_IP_ADDR  = 0x30004,
 147         MANA_IB_CONFIG_MAC_ADDR = 0x30005,
 148         MANA_IB_CREATE_CQ       = 0x30008,
 149         MANA_IB_DESTROY_CQ      = 0x30009,
 150         MANA_IB_CREATE_RC_QP    = 0x3000a,
 151         MANA_IB_DESTROY_RC_QP   = 0x3000b,
 152         MANA_IB_SET_QP_STATE    = 0x3000d,
 153 };
 154
 155 struct mana_ib_query_adapter_caps_req {
 156         struct gdma_req_hdr hdr;
 157 }; /*HW Data */
 158
 159 struct mana_ib_query_adapter_caps_resp {
 160         struct gdma_resp_hdr hdr;
 161         u32 max_sq_id;
 162         u32 max_rq_id;
 163         u32 max_cq_id;
 164         u32 max_qp_count;
 165         u32 max_cq_count;
 166         u32 max_mr_count;
 167         u32 max_pd_count;
 168         u32 max_inbound_read_limit;
 169         u32 max_outbound_read_limit;
 170         u32 mw_count;
 171         u32 max_srq_count;
 172         u32 max_requester_sq_size;
 173         u32 max_responder_sq_size;
 174         u32 max_requester_rq_size;
 175         u32 max_responder_rq_size;
 176         u32 max_send_sge_count;
 177         u32 max_recv_sge_count;
 178         u32 max_inline_data_size;
 179 }; /* HW Data */
 180
 181 struct mana_rnic_create_adapter_req {
 182         struct gdma_req_hdr hdr;
 183         u32 notify_eq_id;
 184         u32 reserved;
 185         u64 feature_flags;
 186 }; /*HW Data */
 187
 188 struct mana_rnic_create_adapter_resp {
 189         struct gdma_resp_hdr hdr;
 190         mana_handle_t adapter;
 191 }; /* HW Data */
 192
 193 struct mana_rnic_destroy_adapter_req {
 194         struct gdma_req_hdr hdr;
 195         mana_handle_t adapter;
 196 }; /*HW Data */
 197
 198 struct mana_rnic_destroy_adapter_resp {
 199         struct gdma_resp_hdr hdr;
 200 }; /* HW Data */
 201
 202 enum mana_ib_addr_op {
 203         ADDR_OP_ADD = 1,
 204         ADDR_OP_REMOVE = 2,
 205 };
 206
 207 enum sgid_entry_type {
 208         SGID_TYPE_IPV4 = 1,
 209         SGID_TYPE_IPV6 = 2,
 210 };
 211
 212 struct mana_rnic_config_addr_req {
 213         struct gdma_req_hdr hdr;
 214         mana_handle_t adapter;
 215         enum mana_ib_addr_op op;
 216         enum sgid_entry_type sgid_type;
 217         u8 ip_addr[16];
 218 }; /* HW Data */
 219
 220 struct mana_rnic_config_addr_resp {
 221         struct gdma_resp_hdr hdr;
 222 }; /* HW Data */
 223
 224 struct mana_rnic_config_mac_addr_req {
 225         struct gdma_req_hdr hdr;
 226         mana_handle_t adapter;
 227         enum mana_ib_addr_op op;
 228         u8 mac_addr[ETH_ALEN];
 229         u8 reserved[6];
 230 }; /* HW Data */
 231
 232 struct mana_rnic_config_mac_addr_resp {
 233         struct gdma_resp_hdr hdr;
 234 }; /* HW Data */
 235
 236 struct mana_rnic_create_cq_req {
 237         struct gdma_req_hdr hdr;
 238         mana_handle_t adapter;
 239         u64 gdma_region;
 240         u32 eq_id;
 241         u32 doorbell_page;
 242 }; /* HW Data */
 243
 244 struct mana_rnic_create_cq_resp {
 245         struct gdma_resp_hdr hdr;
 246         mana_handle_t cq_handle;
 247         u32 cq_id;
 248         u32 reserved;
 249 }; /* HW Data */
 250
 251 struct mana_rnic_destroy_cq_req {
 252         struct gdma_req_hdr hdr;
 253         mana_handle_t adapter;
 254         mana_handle_t cq_handle;
 255 }; /* HW Data */
 256
 257 struct mana_rnic_destroy_cq_resp {
 258         struct gdma_resp_hdr hdr;
 259 }; /* HW Data */
 260
 261 enum mana_rnic_create_rc_flags {
 262         MANA_RC_FLAG_NO_FMR = 2,
 263 };
 264
 265 struct mana_rnic_create_qp_req {
 266         struct gdma_req_hdr hdr;
 267         mana_handle_t adapter;
 268         mana_handle_t pd_handle;
 269         mana_handle_t send_cq_handle;
 270         mana_handle_t recv_cq_handle;
 271         u64 dma_region[MANA_RC_QUEUE_TYPE_MAX];
 272         u64 deprecated[2];
 273         u64 flags;
 274         u32 doorbell_page;
 275         u32 max_send_wr;
 276         u32 max_recv_wr;
 277         u32 max_send_sge;
 278         u32 max_recv_sge;
 279         u32 reserved;
 280 }; /* HW Data */
 281
 282 struct mana_rnic_create_qp_resp {
 283         struct gdma_resp_hdr hdr;
 284         mana_handle_t rc_qp_handle;
 285         u32 queue_ids[MANA_RC_QUEUE_TYPE_MAX];
 286         u32 reserved;
 287 }; /* HW Data*/
 288
 289 struct mana_rnic_destroy_rc_qp_req {
 290         struct gdma_req_hdr hdr;
 291         mana_handle_t adapter;
 292         mana_handle_t rc_qp_handle;
 293 }; /* HW Data */
 294
 295 struct mana_rnic_destroy_rc_qp_resp {
 296         struct gdma_resp_hdr hdr;
 297 }; /* HW Data */
 298
 299 struct mana_ib_ah_attr {
 300         u8 src_addr[16];
 301         u8 dest_addr[16];
 302         u8 src_mac[ETH_ALEN];
 303         u8 dest_mac[ETH_ALEN];
 304         u8 src_addr_type;
 305         u8 dest_addr_type;
 306         u8 hop_limit;
 307         u8 traffic_class;
 308         u16 src_port;
 309         u16 dest_port;
 310         u32 reserved;
 311 };
 312
 313 struct mana_rnic_set_qp_state_req {
 314         struct gdma_req_hdr hdr;
 315         mana_handle_t adapter;
 316         mana_handle_t qp_handle;
 317         u64 attr_mask;
 318         u32 qp_state;
 319         u32 path_mtu;
 320         u32 rq_psn;
 321         u32 sq_psn;
 322         u32 dest_qpn;
 323         u32 max_dest_rd_atomic;
 324         u32 retry_cnt;
 325         u32 rnr_retry;
 326         u32 min_rnr_timer;
 327         u32 reserved;
 328         struct mana_ib_ah_attr ah_attr;
 329 }; /* HW Data */
 330
 331 struct mana_rnic_set_qp_state_resp {
 332         struct gdma_resp_hdr hdr;
 333 }; /* HW Data */
 334
 335 static inline struct gdma_context *mdev_to_gc(struct mana_ib_dev *mdev)
 336 {
 337         return mdev->gdma_dev->gdma_context;
 338 }
 339
 340 static inline struct mana_ib_qp *mana_get_qp_ref(struct mana_ib_dev *mdev,
 341                                                  uint32_t qid)
 342 {
 343         struct mana_ib_qp *qp;
 344         unsigned long flag;
 345
 346         xa_lock_irqsave(&mdev->qp_table_wq, flag);
 347         qp = xa_load(&mdev->qp_table_wq, qid);
 348         if (qp)
 349                 refcount_inc(&qp->refcount);
 350         xa_unlock_irqrestore(&mdev->qp_table_wq, flag);
 351         return qp;
 352 }
 353
 354 static inline void mana_put_qp_ref(struct mana_ib_qp *qp)
 355 {
 356         if (refcount_dec_and_test(&qp->refcount))
 357                 complete(&qp->free);
 358 }
 359
 360 static inline struct net_device *mana_ib_get_netdev(struct ib_device *ibdev, u32 port)
 361 {
 362         struct mana_ib_dev *mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
 363         struct gdma_context *gc = mdev_to_gc(mdev);
 364         struct mana_context *mc = gc->mana.driver_data;
 365
 366         if (port < 1 || port > mc->num_ports)
 367                 return NULL;
 368         return mc->ports[port - 1];
 369 }
 370
 371 static inline void copy_in_reverse(u8 *dst, const u8 *src, u32 size)
 372 {
 373         u32 i;
 374
 375         for (i = 0; i < size; i++)
 376                 dst[size - 1 - i] = src[i];
 377 }
 378
 379 int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq);
 380 void mana_ib_remove_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq);
 381
 382 int mana_ib_create_zero_offset_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem,
 383                                           mana_handle_t *gdma_region);
 384
 385 int mana_ib_create_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem,
 386                               mana_handle_t *gdma_region, u64 virt);
 387
 388 int mana_ib_gd_destroy_dma_region(struct mana_ib_dev *dev,
 389                                   mana_handle_t gdma_region);
 390
 391 int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size,
 392                          struct mana_ib_queue *queue);
 393 void mana_ib_destroy_queue(struct mana_ib_dev *mdev, struct mana_ib_queue *queue);
 394
 395 struct ib_wq *mana_ib_create_wq(struct ib_pd *pd,
 396                                 struct ib_wq_init_attr *init_attr,
 397                                 struct ib_udata *udata);
 398
 399 int mana_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr,
 400                       u32 wq_attr_mask, struct ib_udata *udata);
 401
 402 int mana_ib_destroy_wq(struct ib_wq *ibwq, struct ib_udata *udata);
 403
 404 int mana_ib_create_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_table,
 405                                  struct ib_rwq_ind_table_init_attr *init_attr,
 406                                  struct ib_udata *udata);
 407
 408 int mana_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl);
 409
 410 struct ib_mr *mana_ib_get_dma_mr(struct ib_pd *ibpd, int access_flags);
 411
 412 struct ib_mr *mana_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 413                                   u64 iova, int access_flags,
 414                                   struct ib_udata *udata);
 415
 416 int mana_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
 417
 418 int mana_ib_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *qp_init_attr,
 419                       struct ib_udata *udata);
 420
 421 int mana_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 422                       int attr_mask, struct ib_udata *udata);
 423
 424 int mana_ib_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata);
 425
 426 int mana_ib_cfg_vport(struct mana_ib_dev *dev, u32 port_id,
 427                       struct mana_ib_pd *pd, u32 doorbell_id);
 428 void mana_ib_uncfg_vport(struct mana_ib_dev *dev, struct mana_ib_pd *pd,
 429                          u32 port);
 430
 431 int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 432                       struct uverbs_attr_bundle *attrs);
 433
 434 int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);
 435
 436 int mana_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
 437 int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
 438
 439 int mana_ib_alloc_ucontext(struct ib_ucontext *ibcontext,
 440                            struct ib_udata *udata);
 441 void mana_ib_dealloc_ucontext(struct ib_ucontext *ibcontext);
 442
 443 int mana_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma);
 444
 445 int mana_ib_get_port_immutable(struct ib_device *ibdev, u32 port_num,
 446                                struct ib_port_immutable *immutable);
 447 int mana_ib_query_device(struct ib_device *ibdev, struct ib_device_attr *props,
 448                          struct ib_udata *uhw);
 449 int mana_ib_query_port(struct ib_device *ibdev, u32 port,
 450                        struct ib_port_attr *props);
 451 int mana_ib_query_gid(struct ib_device *ibdev, u32 port, int index,
 452                       union ib_gid *gid);
 453
 454 void mana_ib_disassociate_ucontext(struct ib_ucontext *ibcontext);
 455
 456 int mana_ib_gd_query_adapter_caps(struct mana_ib_dev *mdev);
 457
 458 int mana_ib_create_eqs(struct mana_ib_dev *mdev);
 459
 460 void mana_ib_destroy_eqs(struct mana_ib_dev *mdev);
 461
 462 int mana_ib_gd_create_rnic_adapter(struct mana_ib_dev *mdev);
 463
 464 int mana_ib_gd_destroy_rnic_adapter(struct mana_ib_dev *mdev);
 465
 466 int mana_ib_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey);
 467
 468 enum rdma_link_layer mana_ib_get_link_layer(struct ib_device *device, u32 port_num);
 469
 470 int mana_ib_gd_add_gid(const struct ib_gid_attr *attr, void **context);
 471
 472 int mana_ib_gd_del_gid(const struct ib_gid_attr *attr, void **context);
 473
 474 int mana_ib_gd_config_mac(struct mana_ib_dev *mdev, enum mana_ib_addr_op op, u8 *mac);
 475
 476 int mana_ib_gd_create_cq(struct mana_ib_dev *mdev, struct mana_ib_cq *cq, u32 doorbell);
 477
 478 int mana_ib_gd_destroy_cq(struct mana_ib_dev *mdev, struct mana_ib_cq *cq);
 479
 480 int mana_ib_gd_create_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp,
 481                             struct ib_qp_init_attr *attr, u32 doorbell, u64 flags);
 482 int mana_ib_gd_destroy_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp);
 483 #endif