drivers/infiniband/hw/erdma/erdma_eq.c

   1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
   2
   3 /* Authors: Cheng Xu <chengyou@linux.alibaba.com> */
   4 /*          Kai Shen <kaishen@linux.alibaba.com> */
   5 /* Copyright (c) 2020-2022, Alibaba Group. */
   6
   7 #include "erdma_verbs.h"
   8
   9 #define MAX_POLL_CHUNK_SIZE 16
  10
  11 void notify_eq(struct erdma_eq *eq)
  12 {
  13         u64 db_data = FIELD_PREP(ERDMA_EQDB_CI_MASK, eq->ci) |
  14                       FIELD_PREP(ERDMA_EQDB_ARM_MASK, 1);
  15
  16         *eq->dbrec = db_data;
  17         writeq(db_data, eq->db);
  18
  19         atomic64_inc(&eq->notify_num);
  20 }
  21
  22 void *get_next_valid_eqe(struct erdma_eq *eq)
  23 {
  24         u64 *eqe = get_queue_entry(eq->qbuf, eq->ci, eq->depth, EQE_SHIFT);
  25         u32 owner = FIELD_GET(ERDMA_CEQE_HDR_O_MASK, READ_ONCE(*eqe));
  26
  27         return owner ^ !!(eq->ci & eq->depth) ? eqe : NULL;
  28 }
  29
  30 void erdma_aeq_event_handler(struct erdma_dev *dev)
  31 {
  32         struct erdma_aeqe *aeqe;
  33         u32 cqn, qpn;
  34         struct erdma_qp *qp;
  35         struct erdma_cq *cq;
  36         struct ib_event event;
  37         u32 poll_cnt = 0;
  38
  39         memset(&event, 0, sizeof(event));
  40
  41         while (poll_cnt < MAX_POLL_CHUNK_SIZE) {
  42                 aeqe = get_next_valid_eqe(&dev->aeq);
  43                 if (!aeqe)
  44                         break;
  45
  46                 dma_rmb();
  47
  48                 dev->aeq.ci++;
  49                 atomic64_inc(&dev->aeq.event_num);
  50                 poll_cnt++;
  51
  52                 if (FIELD_GET(ERDMA_AEQE_HDR_TYPE_MASK,
  53                               le32_to_cpu(aeqe->hdr)) == ERDMA_AE_TYPE_CQ_ERR) {
  54                         cqn = le32_to_cpu(aeqe->event_data0);
  55                         cq = find_cq_by_cqn(dev, cqn);
  56                         if (!cq)
  57                                 continue;
  58
  59                         event.device = cq->ibcq.device;
  60                         event.element.cq = &cq->ibcq;
  61                         event.event = IB_EVENT_CQ_ERR;
  62                         if (cq->ibcq.event_handler)
  63                                 cq->ibcq.event_handler(&event,
  64                                                        cq->ibcq.cq_context);
  65                 } else {
  66                         qpn = le32_to_cpu(aeqe->event_data0);
  67                         qp = find_qp_by_qpn(dev, qpn);
  68                         if (!qp)
  69                                 continue;
  70
  71                         event.device = qp->ibqp.device;
  72                         event.element.qp = &qp->ibqp;
  73                         event.event = IB_EVENT_QP_FATAL;
  74                         if (qp->ibqp.event_handler)
  75                                 qp->ibqp.event_handler(&event,
  76                                                        qp->ibqp.qp_context);
  77                 }
  78         }
  79
  80         notify_eq(&dev->aeq);
  81 }
  82
  83 int erdma_eq_common_init(struct erdma_dev *dev, struct erdma_eq *eq, u32 depth)
  84 {
  85         u32 buf_size = depth << EQE_SHIFT;
  86
  87         eq->qbuf = dma_alloc_coherent(&dev->pdev->dev, buf_size,
  88                                       &eq->qbuf_dma_addr, GFP_KERNEL);
  89         if (!eq->qbuf)
  90                 return -ENOMEM;
  91
  92         eq->dbrec = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &eq->dbrec_dma);
  93         if (!eq->dbrec)
  94                 goto err_free_qbuf;
  95
  96         spin_lock_init(&eq->lock);
  97         atomic64_set(&eq->event_num, 0);
  98         atomic64_set(&eq->notify_num, 0);
  99         eq->ci = 0;
 100         eq->depth = depth;
 101
 102         return 0;
 103
 104 err_free_qbuf:
 105         dma_free_coherent(&dev->pdev->dev, buf_size, eq->qbuf,
 106                           eq->qbuf_dma_addr);
 107
 108         return -ENOMEM;
 109 }
 110
 111 void erdma_eq_destroy(struct erdma_dev *dev, struct erdma_eq *eq)
 112 {
 113         dma_pool_free(dev->db_pool, eq->dbrec, eq->dbrec_dma);
 114         dma_free_coherent(&dev->pdev->dev, eq->depth << EQE_SHIFT, eq->qbuf,
 115                           eq->qbuf_dma_addr);
 116 }
 117
 118 int erdma_aeq_init(struct erdma_dev *dev)
 119 {
 120         struct erdma_eq *eq = &dev->aeq;
 121         int ret;
 122
 123         ret = erdma_eq_common_init(dev, &dev->aeq, ERDMA_DEFAULT_EQ_DEPTH);
 124         if (ret)
 125                 return ret;
 126
 127         eq->db = dev->func_bar + ERDMA_REGS_AEQ_DB_REG;
 128
 129         erdma_reg_write32(dev, ERDMA_REGS_AEQ_ADDR_H_REG,
 130                           upper_32_bits(eq->qbuf_dma_addr));
 131         erdma_reg_write32(dev, ERDMA_REGS_AEQ_ADDR_L_REG,
 132                           lower_32_bits(eq->qbuf_dma_addr));
 133         erdma_reg_write32(dev, ERDMA_REGS_AEQ_DEPTH_REG, eq->depth);
 134         erdma_reg_write64(dev, ERDMA_AEQ_DB_HOST_ADDR_REG, eq->dbrec_dma);
 135
 136         return 0;
 137 }
 138
 139 void erdma_ceq_completion_handler(struct erdma_eq_cb *ceq_cb)
 140 {
 141         struct erdma_dev *dev = ceq_cb->dev;
 142         struct erdma_cq *cq;
 143         u32 poll_cnt = 0;
 144         u64 *ceqe;
 145         int cqn;
 146
 147         if (!ceq_cb->ready)
 148                 return;
 149
 150         while (poll_cnt < MAX_POLL_CHUNK_SIZE) {
 151                 ceqe = get_next_valid_eqe(&ceq_cb->eq);
 152                 if (!ceqe)
 153                         break;
 154
 155                 dma_rmb();
 156                 ceq_cb->eq.ci++;
 157                 poll_cnt++;
 158                 cqn = FIELD_GET(ERDMA_CEQE_HDR_CQN_MASK, READ_ONCE(*ceqe));
 159
 160                 cq = find_cq_by_cqn(dev, cqn);
 161                 if (!cq)
 162                         continue;
 163
 164                 if (rdma_is_kernel_res(&cq->ibcq.res))
 165                         cq->kern_cq.cmdsn++;
 166
 167                 if (cq->ibcq.comp_handler)
 168                         cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
 169         }
 170
 171         notify_eq(&ceq_cb->eq);
 172 }
 173
 174 static irqreturn_t erdma_intr_ceq_handler(int irq, void *data)
 175 {
 176         struct erdma_eq_cb *ceq_cb = data;
 177
 178         tasklet_schedule(&ceq_cb->tasklet);
 179
 180         return IRQ_HANDLED;
 181 }
 182
 183 static void erdma_intr_ceq_task(unsigned long data)
 184 {
 185         erdma_ceq_completion_handler((struct erdma_eq_cb *)data);
 186 }
 187
 188 static int erdma_set_ceq_irq(struct erdma_dev *dev, u16 ceqn)
 189 {
 190         struct erdma_eq_cb *eqc = &dev->ceqs[ceqn];
 191         int err;
 192
 193         snprintf(eqc->irq.name, ERDMA_IRQNAME_SIZE, "erdma-ceq%u@pci:%s", ceqn,
 194                  pci_name(dev->pdev));
 195         eqc->irq.msix_vector = pci_irq_vector(dev->pdev, ceqn + 1);
 196
 197         tasklet_init(&dev->ceqs[ceqn].tasklet, erdma_intr_ceq_task,
 198                      (unsigned long)&dev->ceqs[ceqn]);
 199
 200         cpumask_set_cpu(cpumask_local_spread(ceqn + 1, dev->attrs.numa_node),
 201                         &eqc->irq.affinity_hint_mask);
 202
 203         err = request_irq(eqc->irq.msix_vector, erdma_intr_ceq_handler, 0,
 204                           eqc->irq.name, eqc);
 205         if (err) {
 206                 dev_err(&dev->pdev->dev, "failed to request_irq(%d)\n", err);
 207                 return err;
 208         }
 209
 210         irq_set_affinity_hint(eqc->irq.msix_vector,
 211                               &eqc->irq.affinity_hint_mask);
 212
 213         return 0;
 214 }
 215
 216 static void erdma_free_ceq_irq(struct erdma_dev *dev, u16 ceqn)
 217 {
 218         struct erdma_eq_cb *eqc = &dev->ceqs[ceqn];
 219
 220         irq_set_affinity_hint(eqc->irq.msix_vector, NULL);
 221         free_irq(eqc->irq.msix_vector, eqc);
 222 }
 223
 224 static int create_eq_cmd(struct erdma_dev *dev, u32 eqn, struct erdma_eq *eq)
 225 {
 226         struct erdma_cmdq_create_eq_req req;
 227
 228         erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,
 229                                 CMDQ_OPCODE_CREATE_EQ);
 230         req.eqn = eqn;
 231         req.depth = ilog2(eq->depth);
 232         req.qbuf_addr = eq->qbuf_dma_addr;
 233         req.qtype = ERDMA_EQ_TYPE_CEQ;
 234         /* Vector index is the same as EQN. */
 235         req.vector_idx = eqn;
 236         req.db_dma_addr_l = lower_32_bits(eq->dbrec_dma);
 237         req.db_dma_addr_h = upper_32_bits(eq->dbrec_dma);
 238
 239         return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
 240 }
 241
 242 static int erdma_ceq_init_one(struct erdma_dev *dev, u16 ceqn)
 243 {
 244         struct erdma_eq *eq = &dev->ceqs[ceqn].eq;
 245         int ret;
 246
 247         ret = erdma_eq_common_init(dev, eq, ERDMA_DEFAULT_EQ_DEPTH);
 248         if (ret)
 249                 return ret;
 250
 251         eq->db = dev->func_bar + ERDMA_REGS_CEQ_DB_BASE_REG +
 252                  (ceqn + 1) * ERDMA_DB_SIZE;
 253         dev->ceqs[ceqn].dev = dev;
 254         dev->ceqs[ceqn].ready = true;
 255
 256         /* CEQ indexed from 1, 0 rsvd for CMDQ-EQ. */
 257         ret = create_eq_cmd(dev, ceqn + 1, eq);
 258         if (ret) {
 259                 erdma_eq_destroy(dev, eq);
 260                 dev->ceqs[ceqn].ready = false;
 261         }
 262
 263         return ret;
 264 }
 265
 266 static void erdma_ceq_uninit_one(struct erdma_dev *dev, u16 ceqn)
 267 {
 268         struct erdma_eq *eq = &dev->ceqs[ceqn].eq;
 269         struct erdma_cmdq_destroy_eq_req req;
 270         int err;
 271
 272         dev->ceqs[ceqn].ready = 0;
 273
 274         erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,
 275                                 CMDQ_OPCODE_DESTROY_EQ);
 276         /* CEQ indexed from 1, 0 rsvd for CMDQ-EQ. */
 277         req.eqn = ceqn + 1;
 278         req.qtype = ERDMA_EQ_TYPE_CEQ;
 279         req.vector_idx = ceqn + 1;
 280
 281         err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
 282         if (err)
 283                 return;
 284
 285         erdma_eq_destroy(dev, eq);
 286 }
 287
 288 int erdma_ceqs_init(struct erdma_dev *dev)
 289 {
 290         u32 i, j;
 291         int err;
 292
 293         for (i = 0; i < dev->attrs.irq_num - 1; i++) {
 294                 err = erdma_ceq_init_one(dev, i);
 295                 if (err)
 296                         goto out_err;
 297
 298                 err = erdma_set_ceq_irq(dev, i);
 299                 if (err) {
 300                         erdma_ceq_uninit_one(dev, i);
 301                         goto out_err;
 302                 }
 303         }
 304
 305         return 0;
 306
 307 out_err:
 308         for (j = 0; j < i; j++) {
 309                 erdma_free_ceq_irq(dev, j);
 310                 erdma_ceq_uninit_one(dev, j);
 311         }
 312
 313         return err;
 314 }
 315
 316 void erdma_ceqs_uninit(struct erdma_dev *dev)
 317 {
 318         u32 i;
 319
 320         for (i = 0; i < dev->attrs.irq_num - 1; i++) {
 321                 erdma_free_ceq_irq(dev, i);
 322                 erdma_ceq_uninit_one(dev, i);
 323         }
 324 }