include/net/mana/gdma.h

   1 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
   2 /* Copyright (c) 2021, Microsoft Corporation. */
   3
   4 #ifndef _GDMA_H
   5 #define _GDMA_H
   6
   7 #include <linux/dma-mapping.h>
   8 #include <linux/netdevice.h>
   9
  10 #include "shm_channel.h"
  11
  12 #define GDMA_STATUS_MORE_ENTRIES        0x00000105
  13
  14 /* Structures labeled with "HW DATA" are exchanged with the hardware. All of
  15  * them are naturally aligned and hence don't need __packed.
  16  */
  17
  18 enum gdma_request_type {
  19         GDMA_VERIFY_VF_DRIVER_VERSION   = 1,
  20         GDMA_QUERY_MAX_RESOURCES        = 2,
  21         GDMA_LIST_DEVICES               = 3,
  22         GDMA_REGISTER_DEVICE            = 4,
  23         GDMA_DEREGISTER_DEVICE          = 5,
  24         GDMA_GENERATE_TEST_EQE          = 10,
  25         GDMA_CREATE_QUEUE               = 12,
  26         GDMA_DISABLE_QUEUE              = 13,
  27         GDMA_ALLOCATE_RESOURCE_RANGE    = 22,
  28         GDMA_DESTROY_RESOURCE_RANGE     = 24,
  29         GDMA_CREATE_DMA_REGION          = 25,
  30         GDMA_DMA_REGION_ADD_PAGES       = 26,
  31         GDMA_DESTROY_DMA_REGION         = 27,
  32         GDMA_CREATE_PD                  = 29,
  33         GDMA_DESTROY_PD                 = 30,
  34         GDMA_CREATE_MR                  = 31,
  35         GDMA_DESTROY_MR                 = 32,
  36         GDMA_QUERY_HWC_TIMEOUT          = 84, /* 0x54 */
  37 };
  38
  39 #define GDMA_RESOURCE_DOORBELL_PAGE     27
  40
  41 enum gdma_queue_type {
  42         GDMA_INVALID_QUEUE,
  43         GDMA_SQ,
  44         GDMA_RQ,
  45         GDMA_CQ,
  46         GDMA_EQ,
  47 };
  48
  49 enum gdma_work_request_flags {
  50         GDMA_WR_NONE                    = 0,
  51         GDMA_WR_OOB_IN_SGL              = BIT(0),
  52         GDMA_WR_PAD_BY_SGE0             = BIT(1),
  53 };
  54
  55 enum gdma_eqe_type {
  56         GDMA_EQE_COMPLETION             = 3,
  57         GDMA_EQE_TEST_EVENT             = 64,
  58         GDMA_EQE_HWC_INIT_EQ_ID_DB      = 129,
  59         GDMA_EQE_HWC_INIT_DATA          = 130,
  60         GDMA_EQE_HWC_INIT_DONE          = 131,
  61         GDMA_EQE_HWC_SOC_RECONFIG       = 132,
  62         GDMA_EQE_HWC_SOC_RECONFIG_DATA  = 133,
  63         GDMA_EQE_RNIC_QP_FATAL          = 176,
  64 };
  65
  66 enum {
  67         GDMA_DEVICE_NONE        = 0,
  68         GDMA_DEVICE_HWC         = 1,
  69         GDMA_DEVICE_MANA        = 2,
  70         GDMA_DEVICE_MANA_IB     = 3,
  71 };
  72
  73 struct gdma_resource {
  74         /* Protect the bitmap */
  75         spinlock_t lock;
  76
  77         /* The bitmap size in bits. */
  78         u32 size;
  79
  80         /* The bitmap tracks the resources. */
  81         unsigned long *map;
  82 };
  83
  84 union gdma_doorbell_entry {
  85         u64     as_uint64;
  86
  87         struct {
  88                 u64 id          : 24;
  89                 u64 reserved    : 8;
  90                 u64 tail_ptr    : 31;
  91                 u64 arm         : 1;
  92         } cq;
  93
  94         struct {
  95                 u64 id          : 24;
  96                 u64 wqe_cnt     : 8;
  97                 u64 tail_ptr    : 32;
  98         } rq;
  99
 100         struct {
 101                 u64 id          : 24;
 102                 u64 reserved    : 8;
 103                 u64 tail_ptr    : 32;
 104         } sq;
 105
 106         struct {
 107                 u64 id          : 16;
 108                 u64 reserved    : 16;
 109                 u64 tail_ptr    : 31;
 110                 u64 arm         : 1;
 111         } eq;
 112 }; /* HW DATA */
 113
 114 struct gdma_msg_hdr {
 115         u32 hdr_type;
 116         u32 msg_type;
 117         u16 msg_version;
 118         u16 hwc_msg_id;
 119         u32 msg_size;
 120 }; /* HW DATA */
 121
 122 struct gdma_dev_id {
 123         union {
 124                 struct {
 125                         u16 type;
 126                         u16 instance;
 127                 };
 128
 129                 u32 as_uint32;
 130         };
 131 }; /* HW DATA */
 132
 133 struct gdma_req_hdr {
 134         struct gdma_msg_hdr req;
 135         struct gdma_msg_hdr resp; /* The expected response */
 136         struct gdma_dev_id dev_id;
 137         u32 activity_id;
 138 }; /* HW DATA */
 139
 140 struct gdma_resp_hdr {
 141         struct gdma_msg_hdr response;
 142         struct gdma_dev_id dev_id;
 143         u32 activity_id;
 144         u32 status;
 145         u32 reserved;
 146 }; /* HW DATA */
 147
 148 struct gdma_general_req {
 149         struct gdma_req_hdr hdr;
 150 }; /* HW DATA */
 151
 152 #define GDMA_MESSAGE_V1 1
 153 #define GDMA_MESSAGE_V2 2
 154 #define GDMA_MESSAGE_V3 3
 155
 156 struct gdma_general_resp {
 157         struct gdma_resp_hdr hdr;
 158 }; /* HW DATA */
 159
 160 #define GDMA_STANDARD_HEADER_TYPE 0
 161
 162 static inline void mana_gd_init_req_hdr(struct gdma_req_hdr *hdr, u32 code,
 163                                         u32 req_size, u32 resp_size)
 164 {
 165         hdr->req.hdr_type = GDMA_STANDARD_HEADER_TYPE;
 166         hdr->req.msg_type = code;
 167         hdr->req.msg_version = GDMA_MESSAGE_V1;
 168         hdr->req.msg_size = req_size;
 169
 170         hdr->resp.hdr_type = GDMA_STANDARD_HEADER_TYPE;
 171         hdr->resp.msg_type = code;
 172         hdr->resp.msg_version = GDMA_MESSAGE_V1;
 173         hdr->resp.msg_size = resp_size;
 174 }
 175
 176 /* The 16-byte struct is part of the GDMA work queue entry (WQE). */
 177 struct gdma_sge {
 178         u64 address;
 179         u32 mem_key;
 180         u32 size;
 181 }; /* HW DATA */
 182
 183 struct gdma_wqe_request {
 184         struct gdma_sge *sgl;
 185         u32 num_sge;
 186
 187         u32 inline_oob_size;
 188         const void *inline_oob_data;
 189
 190         u32 flags;
 191         u32 client_data_unit;
 192 };
 193
 194 enum gdma_page_type {
 195         GDMA_PAGE_TYPE_4K,
 196 };
 197
 198 #define GDMA_INVALID_DMA_REGION 0
 199
 200 struct gdma_mem_info {
 201         struct device *dev;
 202
 203         dma_addr_t dma_handle;
 204         void *virt_addr;
 205         u64 length;
 206
 207         /* Allocated by the PF driver */
 208         u64 dma_region_handle;
 209 };
 210
 211 #define REGISTER_ATB_MST_MKEY_LOWER_SIZE 8
 212
 213 struct gdma_dev {
 214         struct gdma_context *gdma_context;
 215
 216         struct gdma_dev_id dev_id;
 217
 218         u32 pdid;
 219         u32 doorbell;
 220         u32 gpa_mkey;
 221
 222         /* GDMA driver specific pointer */
 223         void *driver_data;
 224
 225         struct auxiliary_device *adev;
 226 };
 227
 228 /* MANA_PAGE_SIZE is the DMA unit */
 229 #define MANA_PAGE_SHIFT 12
 230 #define MANA_PAGE_SIZE BIT(MANA_PAGE_SHIFT)
 231 #define MANA_PAGE_ALIGN(x) ALIGN((x), MANA_PAGE_SIZE)
 232 #define MANA_PAGE_ALIGNED(addr) IS_ALIGNED((unsigned long)(addr), MANA_PAGE_SIZE)
 233 #define MANA_PFN(a) ((a) >> MANA_PAGE_SHIFT)
 234
 235 /* Required by HW */
 236 #define MANA_MIN_QSIZE MANA_PAGE_SIZE
 237
 238 #define GDMA_CQE_SIZE 64
 239 #define GDMA_EQE_SIZE 16
 240 #define GDMA_MAX_SQE_SIZE 512
 241 #define GDMA_MAX_RQE_SIZE 256
 242
 243 #define GDMA_COMP_DATA_SIZE 0x3C
 244
 245 #define GDMA_EVENT_DATA_SIZE 0xC
 246
 247 /* The WQE size must be a multiple of the Basic Unit, which is 32 bytes. */
 248 #define GDMA_WQE_BU_SIZE 32
 249
 250 #define INVALID_PDID            UINT_MAX
 251 #define INVALID_DOORBELL        UINT_MAX
 252 #define INVALID_MEM_KEY         UINT_MAX
 253 #define INVALID_QUEUE_ID        UINT_MAX
 254 #define INVALID_PCI_MSIX_INDEX  UINT_MAX
 255
 256 struct gdma_comp {
 257         u32 cqe_data[GDMA_COMP_DATA_SIZE / 4];
 258         u32 wq_num;
 259         bool is_sq;
 260 };
 261
 262 struct gdma_event {
 263         u32 details[GDMA_EVENT_DATA_SIZE / 4];
 264         u8  type;
 265 };
 266
 267 struct gdma_queue;
 268
 269 struct mana_eq {
 270         struct gdma_queue       *eq;
 271         struct dentry           *mana_eq_debugfs;
 272 };
 273
 274 typedef void gdma_eq_callback(void *context, struct gdma_queue *q,
 275                               struct gdma_event *e);
 276
 277 typedef void gdma_cq_callback(void *context, struct gdma_queue *q);
 278
 279 /* The 'head' is the producer index. For SQ/RQ, when the driver posts a WQE
 280  * (Note: the WQE size must be a multiple of the 32-byte Basic Unit), the
 281  * driver increases the 'head' in BUs rather than in bytes, and notifies
 282  * the HW of the updated head. For EQ/CQ, the driver uses the 'head' to track
 283  * the HW head, and increases the 'head' by 1 for every processed EQE/CQE.
 284  *
 285  * The 'tail' is the consumer index for SQ/RQ. After the CQE of the SQ/RQ is
 286  * processed, the driver increases the 'tail' to indicate that WQEs have
 287  * been consumed by the HW, so the driver can post new WQEs into the SQ/RQ.
 288  *
 289  * The driver doesn't use the 'tail' for EQ/CQ, because the driver ensures
 290  * that the EQ/CQ is big enough so they can't overflow, and the driver uses
 291  * the owner bits mechanism to detect if the queue has become empty.
 292  */
 293 struct gdma_queue {
 294         struct gdma_dev *gdma_dev;
 295
 296         enum gdma_queue_type type;
 297         u32 id;
 298
 299         struct gdma_mem_info mem_info;
 300
 301         void *queue_mem_ptr;
 302         u32 queue_size;
 303
 304         bool monitor_avl_buf;
 305
 306         u32 head;
 307         u32 tail;
 308         struct list_head entry;
 309
 310         /* Extra fields specific to EQ/CQ. */
 311         union {
 312                 struct {
 313                         bool disable_needed;
 314
 315                         gdma_eq_callback *callback;
 316                         void *context;
 317
 318                         unsigned int msix_index;
 319
 320                         u32 log2_throttle_limit;
 321                 } eq;
 322
 323                 struct {
 324                         gdma_cq_callback *callback;
 325                         void *context;
 326
 327                         struct gdma_queue *parent; /* For CQ/EQ relationship */
 328                 } cq;
 329         };
 330 };
 331
 332 struct gdma_queue_spec {
 333         enum gdma_queue_type type;
 334         bool monitor_avl_buf;
 335         unsigned int queue_size;
 336
 337         /* Extra fields specific to EQ/CQ. */
 338         union {
 339                 struct {
 340                         gdma_eq_callback *callback;
 341                         void *context;
 342
 343                         unsigned long log2_throttle_limit;
 344                         unsigned int msix_index;
 345                 } eq;
 346
 347                 struct {
 348                         gdma_cq_callback *callback;
 349                         void *context;
 350
 351                         struct gdma_queue *parent_eq;
 352
 353                 } cq;
 354         };
 355 };
 356
 357 #define MANA_IRQ_NAME_SZ 32
 358
 359 struct gdma_irq_context {
 360         void (*handler)(void *arg);
 361         /* Protect the eq_list */
 362         spinlock_t lock;
 363         struct list_head eq_list;
 364         char name[MANA_IRQ_NAME_SZ];
 365 };
 366
 367 struct gdma_context {
 368         struct device           *dev;
 369         struct dentry           *mana_pci_debugfs;
 370
 371         /* Per-vPort max number of queues */
 372         unsigned int            max_num_queues;
 373         unsigned int            max_num_msix;
 374         unsigned int            num_msix_usable;
 375         struct gdma_irq_context *irq_contexts;
 376
 377         /* L2 MTU */
 378         u16 adapter_mtu;
 379
 380         /* This maps a CQ index to the queue structure. */
 381         unsigned int            max_num_cqs;
 382         struct gdma_queue       **cq_table;
 383
 384         /* Protect eq_test_event and test_event_eq_id  */
 385         struct mutex            eq_test_event_mutex;
 386         struct completion       eq_test_event;
 387         u32                     test_event_eq_id;
 388
 389         bool                    is_pf;
 390         phys_addr_t             bar0_pa;
 391         void __iomem            *bar0_va;
 392         void __iomem            *shm_base;
 393         void __iomem            *db_page_base;
 394         phys_addr_t             phys_db_page_base;
 395         u32 db_page_size;
 396         int                     numa_node;
 397
 398         /* Shared memory chanenl (used to bootstrap HWC) */
 399         struct shm_channel      shm_channel;
 400
 401         /* Hardware communication channel (HWC) */
 402         struct gdma_dev         hwc;
 403
 404         /* Azure network adapter */
 405         struct gdma_dev         mana;
 406
 407         /* Azure RDMA adapter */
 408         struct gdma_dev         mana_ib;
 409 };
 410
 411 #define MAX_NUM_GDMA_DEVICES    4
 412
 413 static inline bool mana_gd_is_mana(struct gdma_dev *gd)
 414 {
 415         return gd->dev_id.type == GDMA_DEVICE_MANA;
 416 }
 417
 418 static inline bool mana_gd_is_hwc(struct gdma_dev *gd)
 419 {
 420         return gd->dev_id.type == GDMA_DEVICE_HWC;
 421 }
 422
 423 u8 *mana_gd_get_wqe_ptr(const struct gdma_queue *wq, u32 wqe_offset);
 424 u32 mana_gd_wq_avail_space(struct gdma_queue *wq);
 425
 426 int mana_gd_test_eq(struct gdma_context *gc, struct gdma_queue *eq);
 427
 428 int mana_gd_create_hwc_queue(struct gdma_dev *gd,
 429                              const struct gdma_queue_spec *spec,
 430                              struct gdma_queue **queue_ptr);
 431
 432 int mana_gd_create_mana_eq(struct gdma_dev *gd,
 433                            const struct gdma_queue_spec *spec,
 434                            struct gdma_queue **queue_ptr);
 435
 436 int mana_gd_create_mana_wq_cq(struct gdma_dev *gd,
 437                               const struct gdma_queue_spec *spec,
 438                               struct gdma_queue **queue_ptr);
 439
 440 void mana_gd_destroy_queue(struct gdma_context *gc, struct gdma_queue *queue);
 441
 442 int mana_gd_poll_cq(struct gdma_queue *cq, struct gdma_comp *comp, int num_cqe);
 443
 444 void mana_gd_ring_cq(struct gdma_queue *cq, u8 arm_bit);
 445
 446 struct gdma_wqe {
 447         u32 reserved    :24;
 448         u32 last_vbytes :8;
 449
 450         union {
 451                 u32 flags;
 452
 453                 struct {
 454                         u32 num_sge             :8;
 455                         u32 inline_oob_size_div4:3;
 456                         u32 client_oob_in_sgl   :1;
 457                         u32 reserved1           :4;
 458                         u32 client_data_unit    :14;
 459                         u32 reserved2           :2;
 460                 };
 461         };
 462 }; /* HW DATA */
 463
 464 #define INLINE_OOB_SMALL_SIZE 8
 465 #define INLINE_OOB_LARGE_SIZE 24
 466
 467 #define MAX_TX_WQE_SIZE 512
 468 #define MAX_RX_WQE_SIZE 256
 469
 470 #define MAX_TX_WQE_SGL_ENTRIES  ((GDMA_MAX_SQE_SIZE -                      \
 471                         sizeof(struct gdma_sge) - INLINE_OOB_SMALL_SIZE) / \
 472                         sizeof(struct gdma_sge))
 473
 474 #define MAX_RX_WQE_SGL_ENTRIES  ((GDMA_MAX_RQE_SIZE -                      \
 475                         sizeof(struct gdma_sge)) / sizeof(struct gdma_sge))
 476
 477 struct gdma_cqe {
 478         u32 cqe_data[GDMA_COMP_DATA_SIZE / 4];
 479
 480         union {
 481                 u32 as_uint32;
 482
 483                 struct {
 484                         u32 wq_num      : 24;
 485                         u32 is_sq       : 1;
 486                         u32 reserved    : 4;
 487                         u32 owner_bits  : 3;
 488                 };
 489         } cqe_info;
 490 }; /* HW DATA */
 491
 492 #define GDMA_CQE_OWNER_BITS 3
 493
 494 #define GDMA_CQE_OWNER_MASK ((1 << GDMA_CQE_OWNER_BITS) - 1)
 495
 496 #define SET_ARM_BIT 1
 497
 498 #define GDMA_EQE_OWNER_BITS 3
 499
 500 union gdma_eqe_info {
 501         u32 as_uint32;
 502
 503         struct {
 504                 u32 type        : 8;
 505                 u32 reserved1   : 8;
 506                 u32 client_id   : 2;
 507                 u32 reserved2   : 11;
 508                 u32 owner_bits  : 3;
 509         };
 510 }; /* HW DATA */
 511
 512 #define GDMA_EQE_OWNER_MASK ((1 << GDMA_EQE_OWNER_BITS) - 1)
 513 #define INITIALIZED_OWNER_BIT(log2_num_entries) (1UL << (log2_num_entries))
 514
 515 struct gdma_eqe {
 516         u32 details[GDMA_EVENT_DATA_SIZE / 4];
 517         u32 eqe_info;
 518 }; /* HW DATA */
 519
 520 #define GDMA_REG_DB_PAGE_OFFSET 8
 521 #define GDMA_REG_DB_PAGE_SIZE   0x10
 522 #define GDMA_REG_SHM_OFFSET     0x18
 523
 524 #define GDMA_PF_REG_DB_PAGE_SIZE        0xD0
 525 #define GDMA_PF_REG_DB_PAGE_OFF         0xC8
 526 #define GDMA_PF_REG_SHM_OFF             0x70
 527
 528 #define GDMA_SRIOV_REG_CFG_BASE_OFF     0x108
 529
 530 #define MANA_PF_DEVICE_ID 0x00B9
 531 #define MANA_VF_DEVICE_ID 0x00BA
 532
 533 struct gdma_posted_wqe_info {
 534         u32 wqe_size_in_bu;
 535 };
 536
 537 /* GDMA_GENERATE_TEST_EQE */
 538 struct gdma_generate_test_event_req {
 539         struct gdma_req_hdr hdr;
 540         u32 queue_index;
 541 }; /* HW DATA */
 542
 543 /* GDMA_VERIFY_VF_DRIVER_VERSION */
 544 enum {
 545         GDMA_PROTOCOL_V1        = 1,
 546         GDMA_PROTOCOL_FIRST     = GDMA_PROTOCOL_V1,
 547         GDMA_PROTOCOL_LAST      = GDMA_PROTOCOL_V1,
 548 };
 549
 550 #define GDMA_DRV_CAP_FLAG_1_EQ_SHARING_MULTI_VPORT BIT(0)
 551
 552 /* Advertise to the NIC firmware: the NAPI work_done variable race is fixed,
 553  * so the driver is able to reliably support features like busy_poll.
 554  */
 555 #define GDMA_DRV_CAP_FLAG_1_NAPI_WKDONE_FIX BIT(2)
 556 #define GDMA_DRV_CAP_FLAG_1_HWC_TIMEOUT_RECONFIG BIT(3)
 557 #define GDMA_DRV_CAP_FLAG_1_VARIABLE_INDIRECTION_TABLE_SUPPORT BIT(5)
 558
 559 #define GDMA_DRV_CAP_FLAGS1 \
 560         (GDMA_DRV_CAP_FLAG_1_EQ_SHARING_MULTI_VPORT | \
 561          GDMA_DRV_CAP_FLAG_1_NAPI_WKDONE_FIX | \
 562          GDMA_DRV_CAP_FLAG_1_HWC_TIMEOUT_RECONFIG | \
 563          GDMA_DRV_CAP_FLAG_1_VARIABLE_INDIRECTION_TABLE_SUPPORT)
 564
 565 #define GDMA_DRV_CAP_FLAGS2 0
 566
 567 #define GDMA_DRV_CAP_FLAGS3 0
 568
 569 #define GDMA_DRV_CAP_FLAGS4 0
 570
 571 struct gdma_verify_ver_req {
 572         struct gdma_req_hdr hdr;
 573
 574         /* Mandatory fields required for protocol establishment */
 575         u64 protocol_ver_min;
 576         u64 protocol_ver_max;
 577
 578         /* Gdma Driver Capability Flags */
 579         u64 gd_drv_cap_flags1;
 580         u64 gd_drv_cap_flags2;
 581         u64 gd_drv_cap_flags3;
 582         u64 gd_drv_cap_flags4;
 583
 584         /* Advisory fields */
 585         u64 drv_ver;
 586         u32 os_type; /* Linux = 0x10; Windows = 0x20; Other = 0x30 */
 587         u32 reserved;
 588         u32 os_ver_major;
 589         u32 os_ver_minor;
 590         u32 os_ver_build;
 591         u32 os_ver_platform;
 592         u64 reserved_2;
 593         u8 os_ver_str1[128];
 594         u8 os_ver_str2[128];
 595         u8 os_ver_str3[128];
 596         u8 os_ver_str4[128];
 597 }; /* HW DATA */
 598
 599 struct gdma_verify_ver_resp {
 600         struct gdma_resp_hdr hdr;
 601         u64 gdma_protocol_ver;
 602         u64 pf_cap_flags1;
 603         u64 pf_cap_flags2;
 604         u64 pf_cap_flags3;
 605         u64 pf_cap_flags4;
 606 }; /* HW DATA */
 607
 608 /* GDMA_QUERY_MAX_RESOURCES */
 609 struct gdma_query_max_resources_resp {
 610         struct gdma_resp_hdr hdr;
 611         u32 status;
 612         u32 max_sq;
 613         u32 max_rq;
 614         u32 max_cq;
 615         u32 max_eq;
 616         u32 max_db;
 617         u32 max_mst;
 618         u32 max_cq_mod_ctx;
 619         u32 max_mod_cq;
 620         u32 max_msix;
 621 }; /* HW DATA */
 622
 623 /* GDMA_LIST_DEVICES */
 624 struct gdma_list_devices_resp {
 625         struct gdma_resp_hdr hdr;
 626         u32 num_of_devs;
 627         u32 reserved;
 628         struct gdma_dev_id devs[64];
 629 }; /* HW DATA */
 630
 631 /* GDMA_REGISTER_DEVICE */
 632 struct gdma_register_device_resp {
 633         struct gdma_resp_hdr hdr;
 634         u32 pdid;
 635         u32 gpa_mkey;
 636         u32 db_id;
 637 }; /* HW DATA */
 638
 639 struct gdma_allocate_resource_range_req {
 640         struct gdma_req_hdr hdr;
 641         u32 resource_type;
 642         u32 num_resources;
 643         u32 alignment;
 644         u32 allocated_resources;
 645 };
 646
 647 struct gdma_allocate_resource_range_resp {
 648         struct gdma_resp_hdr hdr;
 649         u32 allocated_resources;
 650 };
 651
 652 struct gdma_destroy_resource_range_req {
 653         struct gdma_req_hdr hdr;
 654         u32 resource_type;
 655         u32 num_resources;
 656         u32 allocated_resources;
 657 };
 658
 659 /* GDMA_CREATE_QUEUE */
 660 struct gdma_create_queue_req {
 661         struct gdma_req_hdr hdr;
 662         u32 type;
 663         u32 reserved1;
 664         u32 pdid;
 665         u32 doolbell_id;
 666         u64 gdma_region;
 667         u32 reserved2;
 668         u32 queue_size;
 669         u32 log2_throttle_limit;
 670         u32 eq_pci_msix_index;
 671         u32 cq_mod_ctx_id;
 672         u32 cq_parent_eq_id;
 673         u8  rq_drop_on_overrun;
 674         u8  rq_err_on_wqe_overflow;
 675         u8  rq_chain_rec_wqes;
 676         u8  sq_hw_db;
 677         u32 reserved3;
 678 }; /* HW DATA */
 679
 680 struct gdma_create_queue_resp {
 681         struct gdma_resp_hdr hdr;
 682         u32 queue_index;
 683 }; /* HW DATA */
 684
 685 /* GDMA_DISABLE_QUEUE */
 686 struct gdma_disable_queue_req {
 687         struct gdma_req_hdr hdr;
 688         u32 type;
 689         u32 queue_index;
 690         u32 alloc_res_id_on_creation;
 691 }; /* HW DATA */
 692
 693 /* GDMA_QUERY_HWC_TIMEOUT */
 694 struct gdma_query_hwc_timeout_req {
 695         struct gdma_req_hdr hdr;
 696         u32 timeout_ms;
 697         u32 reserved;
 698 };
 699
 700 struct gdma_query_hwc_timeout_resp {
 701         struct gdma_resp_hdr hdr;
 702         u32 timeout_ms;
 703         u32 reserved;
 704 };
 705
 706 enum atb_page_size {
 707         ATB_PAGE_SIZE_4K,
 708         ATB_PAGE_SIZE_8K,
 709         ATB_PAGE_SIZE_16K,
 710         ATB_PAGE_SIZE_32K,
 711         ATB_PAGE_SIZE_64K,
 712         ATB_PAGE_SIZE_128K,
 713         ATB_PAGE_SIZE_256K,
 714         ATB_PAGE_SIZE_512K,
 715         ATB_PAGE_SIZE_1M,
 716         ATB_PAGE_SIZE_2M,
 717         ATB_PAGE_SIZE_MAX,
 718 };
 719
 720 enum gdma_mr_access_flags {
 721         GDMA_ACCESS_FLAG_LOCAL_READ = BIT_ULL(0),
 722         GDMA_ACCESS_FLAG_LOCAL_WRITE = BIT_ULL(1),
 723         GDMA_ACCESS_FLAG_REMOTE_READ = BIT_ULL(2),
 724         GDMA_ACCESS_FLAG_REMOTE_WRITE = BIT_ULL(3),
 725         GDMA_ACCESS_FLAG_REMOTE_ATOMIC = BIT_ULL(4),
 726 };
 727
 728 /* GDMA_CREATE_DMA_REGION */
 729 struct gdma_create_dma_region_req {
 730         struct gdma_req_hdr hdr;
 731
 732         /* The total size of the DMA region */
 733         u64 length;
 734
 735         /* The offset in the first page */
 736         u32 offset_in_page;
 737
 738         /* enum gdma_page_type */
 739         u32 gdma_page_type;
 740
 741         /* The total number of pages */
 742         u32 page_count;
 743
 744         /* If page_addr_list_len is smaller than page_count,
 745          * the remaining page addresses will be added via the
 746          * message GDMA_DMA_REGION_ADD_PAGES.
 747          */
 748         u32 page_addr_list_len;
 749         u64 page_addr_list[];
 750 }; /* HW DATA */
 751
 752 struct gdma_create_dma_region_resp {
 753         struct gdma_resp_hdr hdr;
 754         u64 dma_region_handle;
 755 }; /* HW DATA */
 756
 757 /* GDMA_DMA_REGION_ADD_PAGES */
 758 struct gdma_dma_region_add_pages_req {
 759         struct gdma_req_hdr hdr;
 760
 761         u64 dma_region_handle;
 762
 763         u32 page_addr_list_len;
 764         u32 reserved3;
 765
 766         u64 page_addr_list[];
 767 }; /* HW DATA */
 768
 769 /* GDMA_DESTROY_DMA_REGION */
 770 struct gdma_destroy_dma_region_req {
 771         struct gdma_req_hdr hdr;
 772
 773         u64 dma_region_handle;
 774 }; /* HW DATA */
 775
 776 enum gdma_pd_flags {
 777         GDMA_PD_FLAG_INVALID = 0,
 778 };
 779
 780 struct gdma_create_pd_req {
 781         struct gdma_req_hdr hdr;
 782         enum gdma_pd_flags flags;
 783         u32 reserved;
 784 };/* HW DATA */
 785
 786 struct gdma_create_pd_resp {
 787         struct gdma_resp_hdr hdr;
 788         u64 pd_handle;
 789         u32 pd_id;
 790         u32 reserved;
 791 };/* HW DATA */
 792
 793 struct gdma_destroy_pd_req {
 794         struct gdma_req_hdr hdr;
 795         u64 pd_handle;
 796 };/* HW DATA */
 797
 798 struct gdma_destory_pd_resp {
 799         struct gdma_resp_hdr hdr;
 800 };/* HW DATA */
 801
 802 enum gdma_mr_type {
 803         /* Guest Virtual Address - MRs of this type allow access
 804          * to memory mapped by PTEs associated with this MR using a virtual
 805          * address that is set up in the MST
 806          */
 807         GDMA_MR_TYPE_GVA = 2,
 808 };
 809
 810 struct gdma_create_mr_params {
 811         u64 pd_handle;
 812         enum gdma_mr_type mr_type;
 813         union {
 814                 struct {
 815                         u64 dma_region_handle;
 816                         u64 virtual_address;
 817                         enum gdma_mr_access_flags access_flags;
 818                 } gva;
 819         };
 820 };
 821
 822 struct gdma_create_mr_request {
 823         struct gdma_req_hdr hdr;
 824         u64 pd_handle;
 825         enum gdma_mr_type mr_type;
 826         u32 reserved_1;
 827
 828         union {
 829                 struct {
 830                         u64 dma_region_handle;
 831                         u64 virtual_address;
 832                         enum gdma_mr_access_flags access_flags;
 833                 } gva;
 834
 835         };
 836         u32 reserved_2;
 837 };/* HW DATA */
 838
 839 struct gdma_create_mr_response {
 840         struct gdma_resp_hdr hdr;
 841         u64 mr_handle;
 842         u32 lkey;
 843         u32 rkey;
 844 };/* HW DATA */
 845
 846 struct gdma_destroy_mr_request {
 847         struct gdma_req_hdr hdr;
 848         u64 mr_handle;
 849 };/* HW DATA */
 850
 851 struct gdma_destroy_mr_response {
 852         struct gdma_resp_hdr hdr;
 853 };/* HW DATA */
 854
 855 int mana_gd_verify_vf_version(struct pci_dev *pdev);
 856
 857 int mana_gd_register_device(struct gdma_dev *gd);
 858 int mana_gd_deregister_device(struct gdma_dev *gd);
 859
 860 int mana_gd_post_work_request(struct gdma_queue *wq,
 861                               const struct gdma_wqe_request *wqe_req,
 862                               struct gdma_posted_wqe_info *wqe_info);
 863
 864 int mana_gd_post_and_ring(struct gdma_queue *queue,
 865                           const struct gdma_wqe_request *wqe,
 866                           struct gdma_posted_wqe_info *wqe_info);
 867
 868 int mana_gd_alloc_res_map(u32 res_avail, struct gdma_resource *r);
 869 void mana_gd_free_res_map(struct gdma_resource *r);
 870
 871 void mana_gd_wq_ring_doorbell(struct gdma_context *gc,
 872                               struct gdma_queue *queue);
 873
 874 int mana_gd_alloc_memory(struct gdma_context *gc, unsigned int length,
 875                          struct gdma_mem_info *gmi);
 876
 877 void mana_gd_free_memory(struct gdma_mem_info *gmi);
 878
 879 int mana_gd_send_request(struct gdma_context *gc, u32 req_len, const void *req,
 880                          u32 resp_len, void *resp);
 881
 882 int mana_gd_destroy_dma_region(struct gdma_context *gc, u64 dma_region_handle);
 883 void mana_register_debugfs(void);
 884 void mana_unregister_debugfs(void);
 885
 886 #endif /* _GDMA_H */