drivers/infiniband/hw/qedr/qedr.h

   1 /* QLogic qedr NIC Driver
   2  * Copyright (c) 2015-2016  QLogic Corporation
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and /or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32 #ifndef __QEDR_H__
  33 #define __QEDR_H__
  34
  35 #include <linux/pci.h>
  36 #include <linux/xarray.h>
  37 #include <rdma/ib_addr.h>
  38 #include <linux/qed/qed_if.h>
  39 #include <linux/qed/qed_chain.h>
  40 #include <linux/qed/qed_rdma_if.h>
  41 #include <linux/qed/qede_rdma.h>
  42 #include <linux/qed/roce_common.h>
  43 #include <linux/completion.h>
  44 #include "qedr_hsi_rdma.h"
  45
  46 #define QEDR_NODE_DESC "QLogic 579xx RoCE HCA"
  47 #define DP_NAME(_dev) dev_name(&(_dev)->ibdev.dev)
  48 #define IS_IWARP(_dev) ((_dev)->rdma_type == QED_RDMA_TYPE_IWARP)
  49 #define IS_ROCE(_dev) ((_dev)->rdma_type == QED_RDMA_TYPE_ROCE)
  50
  51 #define DP_DEBUG(dev, module, fmt, ...)                                 \
  52         pr_debug("(%s) " module ": " fmt,                               \
  53                  DP_NAME(dev) ? DP_NAME(dev) : "", ## __VA_ARGS__)
  54
  55 #define QEDR_MSG_INIT "INIT"
  56 #define QEDR_MSG_MISC "MISC"
  57 #define QEDR_MSG_CQ   "  CQ"
  58 #define QEDR_MSG_MR   "  MR"
  59 #define QEDR_MSG_RQ   "  RQ"
  60 #define QEDR_MSG_SQ   "  SQ"
  61 #define QEDR_MSG_QP   "  QP"
  62 #define QEDR_MSG_SRQ  " SRQ"
  63 #define QEDR_MSG_GSI  " GSI"
  64 #define QEDR_MSG_IWARP  " IW"
  65
  66 #define QEDR_CQ_MAGIC_NUMBER    (0x11223344)
  67
  68 #define FW_PAGE_SIZE            (RDMA_RING_PAGE_SIZE)
  69 #define FW_PAGE_SHIFT           (12)
  70
  71 struct qedr_dev;
  72
  73 struct qedr_cnq {
  74         struct qedr_dev         *dev;
  75         struct qed_chain        pbl;
  76         struct qed_sb_info      *sb;
  77         char                    name[32];
  78         u64                     n_comp;
  79         __le16                  *hw_cons_ptr;
  80         u8                      index;
  81 };
  82
  83 #define QEDR_MAX_SGID 128
  84
  85 struct qedr_device_attr {
  86         u32     vendor_id;
  87         u32     vendor_part_id;
  88         u32     hw_ver;
  89         u64     fw_ver;
  90         u64     node_guid;
  91         u64     sys_image_guid;
  92         u8      max_cnq;
  93         u8      max_sge;
  94         u16     max_inline;
  95         u32     max_sqe;
  96         u32     max_rqe;
  97         u8      max_qp_resp_rd_atomic_resc;
  98         u8      max_qp_req_rd_atomic_resc;
  99         u64     max_dev_resp_rd_atomic_resc;
 100         u32     max_cq;
 101         u32     max_qp;
 102         u32     max_mr;
 103         u64     max_mr_size;
 104         u32     max_cqe;
 105         u32     max_mw;
 106         u32     max_fmr;
 107         u32     max_mr_mw_fmr_pbl;
 108         u64     max_mr_mw_fmr_size;
 109         u32     max_pd;
 110         u32     max_ah;
 111         u8      max_pkey;
 112         u32     max_srq;
 113         u32     max_srq_wr;
 114         u8      max_srq_sge;
 115         u8      max_stats_queues;
 116         u32     dev_caps;
 117
 118         u64     page_size_caps;
 119         u8      dev_ack_delay;
 120         u32     reserved_lkey;
 121         u32     bad_pkey_counter;
 122         struct qed_rdma_events events;
 123 };
 124
 125 #define QEDR_ENET_STATE_BIT     (0)
 126
 127 struct qedr_dev {
 128         struct ib_device        ibdev;
 129         struct qed_dev          *cdev;
 130         struct pci_dev          *pdev;
 131         struct net_device       *ndev;
 132
 133         enum ib_atomic_cap      atomic_cap;
 134
 135         void *rdma_ctx;
 136         struct qedr_device_attr attr;
 137
 138         const struct qed_rdma_ops *ops;
 139         struct qed_int_info     int_info;
 140
 141         struct qed_sb_info      *sb_array;
 142         struct qedr_cnq         *cnq_array;
 143         int                     num_cnq;
 144         int                     sb_start;
 145
 146         void __iomem            *db_addr;
 147         u64                     db_phys_addr;
 148         u32                     db_size;
 149         u16                     dpi;
 150
 151         union ib_gid *sgid_tbl;
 152
 153         /* Lock for sgid table */
 154         spinlock_t sgid_lock;
 155
 156         u64                     guid;
 157
 158         u32                     dp_module;
 159         u8                      dp_level;
 160         u8                      num_hwfns;
 161 #define QEDR_IS_CMT(dev)        ((dev)->num_hwfns > 1)
 162         u8                      affin_hwfn_idx;
 163         u8                      gsi_ll2_handle;
 164
 165         uint                    wq_multiplier;
 166         u8                      gsi_ll2_mac_address[ETH_ALEN];
 167         int                     gsi_qp_created;
 168         struct qedr_cq          *gsi_sqcq;
 169         struct qedr_cq          *gsi_rqcq;
 170         struct qedr_qp          *gsi_qp;
 171         enum qed_rdma_type      rdma_type;
 172         struct xarray           qps;
 173         struct xarray           srqs;
 174         struct workqueue_struct *iwarp_wq;
 175         u16                     iwarp_max_mtu;
 176
 177         unsigned long enet_state;
 178
 179         u8 user_dpm_enabled;
 180 };
 181
 182 #define QEDR_MAX_SQ_PBL                 (0x8000)
 183 #define QEDR_MAX_SQ_PBL_ENTRIES         (0x10000 / sizeof(void *))
 184 #define QEDR_SQE_ELEMENT_SIZE           (sizeof(struct rdma_sq_sge))
 185 #define QEDR_MAX_SQE_ELEMENTS_PER_SQE   (ROCE_REQ_MAX_SINGLE_SQ_WQE_SIZE / \
 186                                          QEDR_SQE_ELEMENT_SIZE)
 187 #define QEDR_MAX_SQE_ELEMENTS_PER_PAGE  ((RDMA_RING_PAGE_SIZE) / \
 188                                          QEDR_SQE_ELEMENT_SIZE)
 189 #define QEDR_MAX_SQE                    ((QEDR_MAX_SQ_PBL_ENTRIES) *\
 190                                          (RDMA_RING_PAGE_SIZE) / \
 191                                          (QEDR_SQE_ELEMENT_SIZE) /\
 192                                          (QEDR_MAX_SQE_ELEMENTS_PER_SQE))
 193 /* RQ */
 194 #define QEDR_MAX_RQ_PBL                 (0x2000)
 195 #define QEDR_MAX_RQ_PBL_ENTRIES         (0x10000 / sizeof(void *))
 196 #define QEDR_RQE_ELEMENT_SIZE           (sizeof(struct rdma_rq_sge))
 197 #define QEDR_MAX_RQE_ELEMENTS_PER_RQE   (RDMA_MAX_SGE_PER_RQ_WQE)
 198 #define QEDR_MAX_RQE_ELEMENTS_PER_PAGE  ((RDMA_RING_PAGE_SIZE) / \
 199                                          QEDR_RQE_ELEMENT_SIZE)
 200 #define QEDR_MAX_RQE                    ((QEDR_MAX_RQ_PBL_ENTRIES) *\
 201                                          (RDMA_RING_PAGE_SIZE) / \
 202                                          (QEDR_RQE_ELEMENT_SIZE) /\
 203                                          (QEDR_MAX_RQE_ELEMENTS_PER_RQE))
 204
 205 #define QEDR_CQE_SIZE   (sizeof(union rdma_cqe))
 206 #define QEDR_MAX_CQE_PBL_SIZE (512 * 1024)
 207 #define QEDR_MAX_CQE_PBL_ENTRIES (((QEDR_MAX_CQE_PBL_SIZE) / \
 208                                   sizeof(u64)) - 1)
 209 #define QEDR_MAX_CQES ((u32)((QEDR_MAX_CQE_PBL_ENTRIES) * \
 210                              (QED_CHAIN_PAGE_SIZE) / QEDR_CQE_SIZE))
 211
 212 #define QEDR_ROCE_MAX_CNQ_SIZE          (0x4000)
 213
 214 #define QEDR_MAX_PORT                   (1)
 215 #define QEDR_PORT                       (1)
 216
 217 #define QEDR_UVERBS(CMD_NAME) (1ull << IB_USER_VERBS_CMD_##CMD_NAME)
 218
 219 #define QEDR_ROCE_PKEY_MAX 1
 220 #define QEDR_ROCE_PKEY_TABLE_LEN 1
 221 #define QEDR_ROCE_PKEY_DEFAULT 0xffff
 222
 223 struct qedr_pbl {
 224         struct list_head list_entry;
 225         void *va;
 226         dma_addr_t pa;
 227 };
 228
 229 struct qedr_ucontext {
 230         struct ib_ucontext ibucontext;
 231         struct qedr_dev *dev;
 232         struct qedr_pd *pd;
 233         void __iomem *dpi_addr;
 234         struct rdma_user_mmap_entry *db_mmap_entry;
 235         u64 dpi_phys_addr;
 236         u32 dpi_size;
 237         u16 dpi;
 238         bool db_rec;
 239 };
 240
 241 union db_prod32 {
 242         struct rdma_pwm_val16_data data;
 243         u32 raw;
 244 };
 245
 246 union db_prod64 {
 247         struct rdma_pwm_val32_data data;
 248         u64 raw;
 249 };
 250
 251 enum qedr_cq_type {
 252         QEDR_CQ_TYPE_GSI,
 253         QEDR_CQ_TYPE_KERNEL,
 254         QEDR_CQ_TYPE_USER,
 255 };
 256
 257 struct qedr_pbl_info {
 258         u32 num_pbls;
 259         u32 num_pbes;
 260         u32 pbl_size;
 261         u32 pbe_size;
 262         bool two_layered;
 263 };
 264
 265 struct qedr_userq {
 266         struct ib_umem *umem;
 267         struct qedr_pbl_info pbl_info;
 268         struct qedr_pbl *pbl_tbl;
 269         u64 buf_addr;
 270         size_t buf_len;
 271
 272         /* doorbell recovery */
 273         void __iomem *db_addr;
 274         struct qedr_user_db_rec *db_rec_data;
 275         struct rdma_user_mmap_entry *db_mmap_entry;
 276         void __iomem *db_rec_db2_addr;
 277         union db_prod32 db_rec_db2_data;
 278 };
 279
 280 struct qedr_cq {
 281         struct ib_cq ibcq;
 282
 283         enum qedr_cq_type cq_type;
 284         u32 sig;
 285
 286         u16 icid;
 287
 288         /* Lock to protect multiplem CQ's */
 289         spinlock_t cq_lock;
 290         u8 arm_flags;
 291         struct qed_chain pbl;
 292
 293         void __iomem *db_addr;
 294         union db_prod64 db;
 295
 296         u8 pbl_toggle;
 297         union rdma_cqe *latest_cqe;
 298         union rdma_cqe *toggle_cqe;
 299
 300         u32 cq_cons;
 301
 302         struct qedr_userq q;
 303         u8 destroyed;
 304         u16 cnq_notif;
 305 };
 306
 307 struct qedr_pd {
 308         struct ib_pd ibpd;
 309         u32 pd_id;
 310         struct qedr_ucontext *uctx;
 311 };
 312
 313 struct qedr_qp_hwq_info {
 314         /* WQE Elements */
 315         struct qed_chain pbl;
 316         u64 p_phys_addr_tbl;
 317         u32 max_sges;
 318
 319         /* WQE */
 320         u16 prod;
 321         u16 cons;
 322         u16 wqe_cons;
 323         u16 gsi_cons;
 324         u16 max_wr;
 325
 326         /* DB */
 327         void __iomem *db;
 328         union db_prod32 db_data;
 329
 330         void __iomem *iwarp_db2;
 331         union db_prod32 iwarp_db2_data;
 332 };
 333
 334 #define QEDR_INC_SW_IDX(p_info, index)                                  \
 335         do {                                                            \
 336                 p_info->index = (p_info->index + 1) &                   \
 337                                 qed_chain_get_capacity(p_info->pbl)     \
 338         } while (0)
 339
 340 struct qedr_srq_hwq_info {
 341         u32 max_sges;
 342         u32 max_wr;
 343         struct qed_chain pbl;
 344         u64 p_phys_addr_tbl;
 345         u32 wqe_prod;
 346         u32 sge_prod;
 347         u32 wr_prod_cnt;
 348         u32 wr_cons_cnt;
 349         u32 num_elems;
 350
 351         u32 *virt_prod_pair_addr;
 352         dma_addr_t phy_prod_pair_addr;
 353 };
 354
 355 struct qedr_srq {
 356         struct ib_srq ibsrq;
 357         struct qedr_dev *dev;
 358
 359         struct qedr_userq       usrq;
 360         struct qedr_srq_hwq_info hw_srq;
 361         struct ib_umem *prod_umem;
 362         u16 srq_id;
 363         u32 srq_limit;
 364         /* lock to protect srq recv post */
 365         spinlock_t lock;
 366 };
 367
 368 enum qedr_qp_err_bitmap {
 369         QEDR_QP_ERR_SQ_FULL = 1,
 370         QEDR_QP_ERR_RQ_FULL = 2,
 371         QEDR_QP_ERR_BAD_SR = 4,
 372         QEDR_QP_ERR_BAD_RR = 8,
 373         QEDR_QP_ERR_SQ_PBL_FULL = 16,
 374         QEDR_QP_ERR_RQ_PBL_FULL = 32,
 375 };
 376
 377 enum qedr_qp_create_type {
 378         QEDR_QP_CREATE_NONE,
 379         QEDR_QP_CREATE_USER,
 380         QEDR_QP_CREATE_KERNEL,
 381 };
 382
 383 enum qedr_iwarp_cm_flags {
 384         QEDR_IWARP_CM_WAIT_FOR_CONNECT    = BIT(0),
 385         QEDR_IWARP_CM_WAIT_FOR_DISCONNECT = BIT(1),
 386 };
 387
 388 struct qedr_qp {
 389         struct ib_qp ibqp;      /* must be first */
 390         struct qedr_dev *dev;
 391         struct qedr_qp_hwq_info sq;
 392         struct qedr_qp_hwq_info rq;
 393
 394         u32 max_inline_data;
 395
 396         /* Lock for QP's */
 397         spinlock_t q_lock;
 398         struct qedr_cq *sq_cq;
 399         struct qedr_cq *rq_cq;
 400         struct qedr_srq *srq;
 401         enum qed_roce_qp_state state;
 402         u32 id;
 403         struct qedr_pd *pd;
 404         enum ib_qp_type qp_type;
 405         enum qedr_qp_create_type create_type;
 406         struct qed_rdma_qp *qed_qp;
 407         u32 qp_id;
 408         u16 icid;
 409         u16 mtu;
 410         int sgid_idx;
 411         u32 rq_psn;
 412         u32 sq_psn;
 413         u32 qkey;
 414         u32 dest_qp_num;
 415
 416         /* Relevant to qps created from kernel space only (ULPs) */
 417         u8 prev_wqe_size;
 418         u16 wqe_cons;
 419         u32 err_bitmap;
 420         bool signaled;
 421
 422         /* SQ shadow */
 423         struct {
 424                 u64 wr_id;
 425                 enum ib_wc_opcode opcode;
 426                 u32 bytes_len;
 427                 u8 wqe_size;
 428                 bool signaled;
 429                 dma_addr_t icrc_mapping;
 430                 u32 *icrc;
 431                 struct qedr_mr *mr;
 432         } *wqe_wr_id;
 433
 434         /* RQ shadow */
 435         struct {
 436                 u64 wr_id;
 437                 struct ib_sge sg_list[RDMA_MAX_SGE_PER_RQ_WQE];
 438                 u8 wqe_size;
 439
 440                 u8 smac[ETH_ALEN];
 441                 u16 vlan;
 442                 int rc;
 443         } *rqe_wr_id;
 444
 445         /* Relevant to qps created from user space only (applications) */
 446         struct qedr_userq usq;
 447         struct qedr_userq urq;
 448
 449         /* synchronization objects used with iwarp ep */
 450         struct kref refcnt;
 451         struct completion iwarp_cm_comp;
 452         unsigned long iwarp_cm_flags; /* enum iwarp_cm_flags */
 453 };
 454
 455 struct qedr_ah {
 456         struct ib_ah ibah;
 457         struct rdma_ah_attr attr;
 458 };
 459
 460 enum qedr_mr_type {
 461         QEDR_MR_USER,
 462         QEDR_MR_KERNEL,
 463         QEDR_MR_DMA,
 464         QEDR_MR_FRMR,
 465 };
 466
 467 struct mr_info {
 468         struct qedr_pbl *pbl_table;
 469         struct qedr_pbl_info pbl_info;
 470         struct list_head free_pbl_list;
 471         struct list_head inuse_pbl_list;
 472         u32 completed;
 473         u32 completed_handled;
 474 };
 475
 476 struct qedr_mr {
 477         struct ib_mr ibmr;
 478         struct ib_umem *umem;
 479
 480         struct qed_rdma_register_tid_in_params hw_mr;
 481         enum qedr_mr_type type;
 482
 483         struct qedr_dev *dev;
 484         struct mr_info info;
 485
 486         u64 *pages;
 487         u32 npages;
 488 };
 489
 490 struct qedr_user_mmap_entry {
 491         struct rdma_user_mmap_entry rdma_entry;
 492         struct qedr_dev *dev;
 493         union {
 494                 u64 io_address;
 495                 void *address;
 496         };
 497         size_t length;
 498         u16 dpi;
 499         u8 mmap_flag;
 500 };
 501
 502 #define SET_FIELD2(value, name, flag) ((value) |= ((flag) << (name ## _SHIFT)))
 503
 504 #define QEDR_RESP_IMM   (RDMA_CQE_RESPONDER_IMM_FLG_MASK << \
 505                          RDMA_CQE_RESPONDER_IMM_FLG_SHIFT)
 506 #define QEDR_RESP_RDMA  (RDMA_CQE_RESPONDER_RDMA_FLG_MASK << \
 507                          RDMA_CQE_RESPONDER_RDMA_FLG_SHIFT)
 508 #define QEDR_RESP_INV   (RDMA_CQE_RESPONDER_INV_FLG_MASK << \
 509                          RDMA_CQE_RESPONDER_INV_FLG_SHIFT)
 510
 511 static inline void qedr_inc_sw_cons(struct qedr_qp_hwq_info *info)
 512 {
 513         info->cons = (info->cons + 1) % info->max_wr;
 514         info->wqe_cons++;
 515 }
 516
 517 static inline void qedr_inc_sw_prod(struct qedr_qp_hwq_info *info)
 518 {
 519         info->prod = (info->prod + 1) % info->max_wr;
 520 }
 521
 522 static inline int qedr_get_dmac(struct qedr_dev *dev,
 523                                 struct rdma_ah_attr *ah_attr, u8 *mac_addr)
 524 {
 525         union ib_gid zero_sgid = { { 0 } };
 526         struct in6_addr in6;
 527         const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr);
 528         u8 *dmac;
 529
 530         if (!memcmp(&grh->dgid, &zero_sgid, sizeof(union ib_gid))) {
 531                 DP_ERR(dev, "Local port GID not supported\n");
 532                 eth_zero_addr(mac_addr);
 533                 return -EINVAL;
 534         }
 535
 536         memcpy(&in6, grh->dgid.raw, sizeof(in6));
 537         dmac = rdma_ah_retrieve_dmac(ah_attr);
 538         if (!dmac)
 539                 return -EINVAL;
 540         ether_addr_copy(mac_addr, dmac);
 541
 542         return 0;
 543 }
 544
 545 struct qedr_iw_listener {
 546         struct qedr_dev *dev;
 547         struct iw_cm_id *cm_id;
 548         int             backlog;
 549         void            *qed_handle;
 550 };
 551
 552 struct qedr_iw_ep {
 553         struct qedr_dev *dev;
 554         struct iw_cm_id *cm_id;
 555         struct qedr_qp  *qp;
 556         void            *qed_context;
 557         struct kref     refcnt;
 558 };
 559
 560 static inline
 561 struct qedr_ucontext *get_qedr_ucontext(struct ib_ucontext *ibucontext)
 562 {
 563         return container_of(ibucontext, struct qedr_ucontext, ibucontext);
 564 }
 565
 566 static inline struct qedr_dev *get_qedr_dev(struct ib_device *ibdev)
 567 {
 568         return container_of(ibdev, struct qedr_dev, ibdev);
 569 }
 570
 571 static inline struct qedr_pd *get_qedr_pd(struct ib_pd *ibpd)
 572 {
 573         return container_of(ibpd, struct qedr_pd, ibpd);
 574 }
 575
 576 static inline struct qedr_cq *get_qedr_cq(struct ib_cq *ibcq)
 577 {
 578         return container_of(ibcq, struct qedr_cq, ibcq);
 579 }
 580
 581 static inline struct qedr_qp *get_qedr_qp(struct ib_qp *ibqp)
 582 {
 583         return container_of(ibqp, struct qedr_qp, ibqp);
 584 }
 585
 586 static inline struct qedr_ah *get_qedr_ah(struct ib_ah *ibah)
 587 {
 588         return container_of(ibah, struct qedr_ah, ibah);
 589 }
 590
 591 static inline struct qedr_mr *get_qedr_mr(struct ib_mr *ibmr)
 592 {
 593         return container_of(ibmr, struct qedr_mr, ibmr);
 594 }
 595
 596 static inline struct qedr_srq *get_qedr_srq(struct ib_srq *ibsrq)
 597 {
 598         return container_of(ibsrq, struct qedr_srq, ibsrq);
 599 }
 600
 601 static inline struct qedr_user_mmap_entry *
 602 get_qedr_mmap_entry(struct rdma_user_mmap_entry *rdma_entry)
 603 {
 604         return container_of(rdma_entry, struct qedr_user_mmap_entry,
 605                             rdma_entry);
 606 }
 607 #endif