drivers/infiniband/hw/hns/hns_roce_srq.c

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 /*
   3  * Copyright (c) 2018 Hisilicon Limited.
   4  */
   5
   6 #include <linux/pci.h>
   7 #include <rdma/ib_umem.h>
   8 #include <rdma/uverbs_ioctl.h>
   9 #include "hns_roce_device.h"
  10 #include "hns_roce_cmd.h"
  11 #include "hns_roce_hem.h"
  12
  13 void hns_roce_srq_event(struct hns_roce_dev *hr_dev, u32 srqn, int event_type)
  14 {
  15         struct hns_roce_srq_table *srq_table = &hr_dev->srq_table;
  16         struct hns_roce_srq *srq;
  17
  18         xa_lock(&srq_table->xa);
  19         srq = xa_load(&srq_table->xa, srqn & (hr_dev->caps.num_srqs - 1));
  20         if (srq)
  21                 refcount_inc(&srq->refcount);
  22         xa_unlock(&srq_table->xa);
  23
  24         if (!srq) {
  25                 dev_warn(hr_dev->dev, "Async event for bogus SRQ %08x\n", srqn);
  26                 return;
  27         }
  28
  29         srq->event(srq, event_type);
  30
  31         if (refcount_dec_and_test(&srq->refcount))
  32                 complete(&srq->free);
  33 }
  34
  35 static void hns_roce_ib_srq_event(struct hns_roce_srq *srq,
  36                                   enum hns_roce_event event_type)
  37 {
  38         struct hns_roce_dev *hr_dev = to_hr_dev(srq->ibsrq.device);
  39         struct ib_srq *ibsrq = &srq->ibsrq;
  40         struct ib_event event;
  41
  42         if (ibsrq->event_handler) {
  43                 event.device      = ibsrq->device;
  44                 event.element.srq = ibsrq;
  45                 switch (event_type) {
  46                 case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:
  47                         event.event = IB_EVENT_SRQ_LIMIT_REACHED;
  48                         break;
  49                 case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:
  50                         event.event = IB_EVENT_SRQ_ERR;
  51                         break;
  52                 default:
  53                         dev_err(hr_dev->dev,
  54                            "hns_roce:Unexpected event type 0x%x on SRQ %06lx\n",
  55                            event_type, srq->srqn);
  56                         return;
  57                 }
  58
  59                 ibsrq->event_handler(&event, ibsrq->srq_context);
  60         }
  61 }
  62
  63 static int alloc_srqn(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq)
  64 {
  65         struct hns_roce_ida *srq_ida = &hr_dev->srq_table.srq_ida;
  66         int id;
  67
  68         id = ida_alloc_range(&srq_ida->ida, srq_ida->min, srq_ida->max,
  69                              GFP_KERNEL);
  70         if (id < 0) {
  71                 ibdev_err(&hr_dev->ib_dev, "failed to alloc srq(%d).\n", id);
  72                 return -ENOMEM;
  73         }
  74
  75         srq->srqn = id;
  76
  77         return 0;
  78 }
  79
  80 static void free_srqn(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq)
  81 {
  82         ida_free(&hr_dev->srq_table.srq_ida.ida, (int)srq->srqn);
  83 }
  84
  85 static int hns_roce_create_srqc(struct hns_roce_dev *hr_dev,
  86                                 struct hns_roce_srq *srq)
  87 {
  88         struct ib_device *ibdev = &hr_dev->ib_dev;
  89         struct hns_roce_cmd_mailbox *mailbox;
  90         int ret;
  91
  92         mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
  93         if (IS_ERR(mailbox)) {
  94                 ibdev_err(ibdev, "failed to alloc mailbox for SRQC.\n");
  95                 return PTR_ERR(mailbox);
  96         }
  97
  98         ret = hr_dev->hw->write_srqc(srq, mailbox->buf);
  99         if (ret) {
 100                 ibdev_err(ibdev, "failed to write SRQC.\n");
 101                 goto err_mbox;
 102         }
 103
 104         ret = hns_roce_create_hw_ctx(hr_dev, mailbox, HNS_ROCE_CMD_CREATE_SRQ,
 105                                      srq->srqn);
 106         if (ret)
 107                 ibdev_err(ibdev, "failed to config SRQC, ret = %d.\n", ret);
 108
 109 err_mbox:
 110         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
 111         return ret;
 112 }
 113
 114 static int alloc_srqc(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq)
 115 {
 116         struct hns_roce_srq_table *srq_table = &hr_dev->srq_table;
 117         struct ib_device *ibdev = &hr_dev->ib_dev;
 118         int ret;
 119
 120         ret = hns_roce_table_get(hr_dev, &srq_table->table, srq->srqn);
 121         if (ret) {
 122                 ibdev_err(ibdev, "failed to get SRQC table, ret = %d.\n", ret);
 123                 return ret;
 124         }
 125
 126         ret = xa_err(xa_store_irq(&srq_table->xa, srq->srqn, srq, GFP_KERNEL));
 127         if (ret) {
 128                 ibdev_err(ibdev, "failed to store SRQC, ret = %d.\n", ret);
 129                 goto err_put;
 130         }
 131
 132         ret = hns_roce_create_srqc(hr_dev, srq);
 133         if (ret)
 134                 goto err_xa;
 135
 136         return 0;
 137
 138 err_xa:
 139         xa_erase_irq(&srq_table->xa, srq->srqn);
 140 err_put:
 141         hns_roce_table_put(hr_dev, &srq_table->table, srq->srqn);
 142
 143         return ret;
 144 }
 145
 146 static void free_srqc(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq)
 147 {
 148         struct hns_roce_srq_table *srq_table = &hr_dev->srq_table;
 149         int ret;
 150
 151         ret = hns_roce_destroy_hw_ctx(hr_dev, HNS_ROCE_CMD_DESTROY_SRQ,
 152                                       srq->srqn);
 153         if (ret)
 154                 dev_err_ratelimited(hr_dev->dev, "DESTROY_SRQ failed (%d) for SRQN %06lx\n",
 155                                     ret, srq->srqn);
 156
 157         xa_erase_irq(&srq_table->xa, srq->srqn);
 158
 159         if (refcount_dec_and_test(&srq->refcount))
 160                 complete(&srq->free);
 161         wait_for_completion(&srq->free);
 162
 163         hns_roce_table_put(hr_dev, &srq_table->table, srq->srqn);
 164 }
 165
 166 static int alloc_srq_idx(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq,
 167                          struct ib_udata *udata, unsigned long addr)
 168 {
 169         struct hns_roce_idx_que *idx_que = &srq->idx_que;
 170         struct ib_device *ibdev = &hr_dev->ib_dev;
 171         struct hns_roce_buf_attr buf_attr = {};
 172         int ret;
 173
 174         srq->idx_que.entry_shift = ilog2(HNS_ROCE_IDX_QUE_ENTRY_SZ);
 175
 176         buf_attr.page_shift = hr_dev->caps.idx_buf_pg_sz + PAGE_SHIFT;
 177         buf_attr.region[0].size = to_hr_hem_entries_size(srq->wqe_cnt,
 178                                         srq->idx_que.entry_shift);
 179         buf_attr.region[0].hopnum = hr_dev->caps.idx_hop_num;
 180         buf_attr.region_count = 1;
 181
 182         ret = hns_roce_mtr_create(hr_dev, &idx_que->mtr, &buf_attr,
 183                                   hr_dev->caps.idx_ba_pg_sz + PAGE_SHIFT,
 184                                   udata, addr);
 185         if (ret) {
 186                 ibdev_err(ibdev,
 187                           "failed to alloc SRQ idx mtr, ret = %d.\n", ret);
 188                 return ret;
 189         }
 190
 191         if (!udata) {
 192                 idx_que->bitmap = bitmap_zalloc(srq->wqe_cnt, GFP_KERNEL);
 193                 if (!idx_que->bitmap) {
 194                         ibdev_err(ibdev, "failed to alloc SRQ idx bitmap.\n");
 195                         ret = -ENOMEM;
 196                         goto err_idx_mtr;
 197                 }
 198         }
 199
 200         idx_que->head = 0;
 201         idx_que->tail = 0;
 202
 203         return 0;
 204 err_idx_mtr:
 205         hns_roce_mtr_destroy(hr_dev, &idx_que->mtr);
 206
 207         return ret;
 208 }
 209
 210 static void free_srq_idx(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq)
 211 {
 212         struct hns_roce_idx_que *idx_que = &srq->idx_que;
 213
 214         bitmap_free(idx_que->bitmap);
 215         idx_que->bitmap = NULL;
 216         hns_roce_mtr_destroy(hr_dev, &idx_que->mtr);
 217 }
 218
 219 static int alloc_srq_wqe_buf(struct hns_roce_dev *hr_dev,
 220                              struct hns_roce_srq *srq,
 221                              struct ib_udata *udata, unsigned long addr)
 222 {
 223         struct ib_device *ibdev = &hr_dev->ib_dev;
 224         struct hns_roce_buf_attr buf_attr = {};
 225         int ret;
 226
 227         srq->wqe_shift = ilog2(roundup_pow_of_two(max(HNS_ROCE_SGE_SIZE,
 228                                                       HNS_ROCE_SGE_SIZE *
 229                                                       srq->max_gs)));
 230
 231         buf_attr.page_shift = hr_dev->caps.srqwqe_buf_pg_sz + PAGE_SHIFT;
 232         buf_attr.region[0].size = to_hr_hem_entries_size(srq->wqe_cnt,
 233                                                          srq->wqe_shift);
 234         buf_attr.region[0].hopnum = hr_dev->caps.srqwqe_hop_num;
 235         buf_attr.region_count = 1;
 236
 237         ret = hns_roce_mtr_create(hr_dev, &srq->buf_mtr, &buf_attr,
 238                                   hr_dev->caps.srqwqe_ba_pg_sz + PAGE_SHIFT,
 239                                   udata, addr);
 240         if (ret)
 241                 ibdev_err(ibdev,
 242                           "failed to alloc SRQ buf mtr, ret = %d.\n", ret);
 243
 244         return ret;
 245 }
 246
 247 static void free_srq_wqe_buf(struct hns_roce_dev *hr_dev,
 248                              struct hns_roce_srq *srq)
 249 {
 250         hns_roce_mtr_destroy(hr_dev, &srq->buf_mtr);
 251 }
 252
 253 static int alloc_srq_wrid(struct hns_roce_srq *srq)
 254 {
 255         srq->wrid = kvmalloc_array(srq->wqe_cnt, sizeof(u64), GFP_KERNEL);
 256         if (!srq->wrid)
 257                 return -ENOMEM;
 258
 259         return 0;
 260 }
 261
 262 static void free_srq_wrid(struct hns_roce_srq *srq)
 263 {
 264         kvfree(srq->wrid);
 265         srq->wrid = NULL;
 266 }
 267
 268 static u32 proc_srq_sge(struct hns_roce_dev *dev, struct hns_roce_srq *hr_srq,
 269                         bool user)
 270 {
 271         u32 max_sge = dev->caps.max_srq_sges;
 272
 273         if (dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)
 274                 return max_sge;
 275
 276         /* Reserve SGEs only for HIP08 in kernel; The userspace driver will
 277          * calculate number of max_sge with reserved SGEs when allocating wqe
 278          * buf, so there is no need to do this again in kernel. But the number
 279          * may exceed the capacity of SGEs recorded in the firmware, so the
 280          * kernel driver should just adapt the value accordingly.
 281          */
 282         if (user)
 283                 max_sge = roundup_pow_of_two(max_sge + 1);
 284         else
 285                 hr_srq->rsv_sge = 1;
 286
 287         return max_sge;
 288 }
 289
 290 static int set_srq_basic_param(struct hns_roce_srq *srq,
 291                                struct ib_srq_init_attr *init_attr,
 292                                struct ib_udata *udata)
 293 {
 294         struct hns_roce_dev *hr_dev = to_hr_dev(srq->ibsrq.device);
 295         struct ib_srq_attr *attr = &init_attr->attr;
 296         u32 max_sge;
 297
 298         max_sge = proc_srq_sge(hr_dev, srq, !!udata);
 299         if (attr->max_wr > hr_dev->caps.max_srq_wrs ||
 300             attr->max_sge > max_sge || !attr->max_sge) {
 301                 ibdev_err(&hr_dev->ib_dev,
 302                           "invalid SRQ attr, depth = %u, sge = %u.\n",
 303                           attr->max_wr, attr->max_sge);
 304                 return -EINVAL;
 305         }
 306
 307         attr->max_wr = max_t(u32, attr->max_wr, HNS_ROCE_MIN_SRQ_WQE_NUM);
 308         srq->wqe_cnt = roundup_pow_of_two(attr->max_wr);
 309         srq->max_gs = roundup_pow_of_two(attr->max_sge + srq->rsv_sge);
 310
 311         attr->max_wr = srq->wqe_cnt;
 312         attr->max_sge = srq->max_gs - srq->rsv_sge;
 313         attr->srq_limit = 0;
 314
 315         return 0;
 316 }
 317
 318 static void set_srq_ext_param(struct hns_roce_srq *srq,
 319                               struct ib_srq_init_attr *init_attr)
 320 {
 321         srq->cqn = ib_srq_has_cq(init_attr->srq_type) ?
 322                    to_hr_cq(init_attr->ext.cq)->cqn : 0;
 323
 324         srq->xrcdn = (init_attr->srq_type == IB_SRQT_XRC) ?
 325                      to_hr_xrcd(init_attr->ext.xrc.xrcd)->xrcdn : 0;
 326 }
 327
 328 static int set_srq_param(struct hns_roce_srq *srq,
 329                          struct ib_srq_init_attr *init_attr,
 330                          struct ib_udata *udata)
 331 {
 332         int ret;
 333
 334         ret = set_srq_basic_param(srq, init_attr, udata);
 335         if (ret)
 336                 return ret;
 337
 338         set_srq_ext_param(srq, init_attr);
 339
 340         return 0;
 341 }
 342
 343 static int alloc_srq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq,
 344                          struct ib_udata *udata)
 345 {
 346         struct hns_roce_ib_create_srq ucmd = {};
 347         int ret;
 348
 349         if (udata) {
 350                 ret = ib_copy_from_udata(&ucmd, udata,
 351                                          min(udata->inlen, sizeof(ucmd)));
 352                 if (ret) {
 353                         ibdev_err(&hr_dev->ib_dev,
 354                                   "failed to copy SRQ udata, ret = %d.\n",
 355                                   ret);
 356                         return ret;
 357                 }
 358         }
 359
 360         ret = alloc_srq_idx(hr_dev, srq, udata, ucmd.que_addr);
 361         if (ret)
 362                 return ret;
 363
 364         ret = alloc_srq_wqe_buf(hr_dev, srq, udata, ucmd.buf_addr);
 365         if (ret)
 366                 goto err_idx;
 367
 368         if (!udata) {
 369                 ret = alloc_srq_wrid(srq);
 370                 if (ret)
 371                         goto err_wqe_buf;
 372         }
 373
 374         return 0;
 375
 376 err_wqe_buf:
 377         free_srq_wqe_buf(hr_dev, srq);
 378 err_idx:
 379         free_srq_idx(hr_dev, srq);
 380
 381         return ret;
 382 }
 383
 384 static void free_srq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq)
 385 {
 386         free_srq_wrid(srq);
 387         free_srq_wqe_buf(hr_dev, srq);
 388         free_srq_idx(hr_dev, srq);
 389 }
 390
 391 static int get_srq_ucmd(struct hns_roce_srq *srq, struct ib_udata *udata,
 392                         struct hns_roce_ib_create_srq *ucmd)
 393 {
 394         struct ib_device *ibdev = srq->ibsrq.device;
 395         int ret;
 396
 397         ret = ib_copy_from_udata(ucmd, udata, min(udata->inlen, sizeof(*ucmd)));
 398         if (ret) {
 399                 ibdev_err(ibdev, "failed to copy SRQ udata, ret = %d.\n", ret);
 400                 return ret;
 401         }
 402
 403         return 0;
 404 }
 405
 406 static void free_srq_db(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq,
 407                         struct ib_udata *udata)
 408 {
 409         struct hns_roce_ucontext *uctx;
 410
 411         if (!(srq->cap_flags & HNS_ROCE_SRQ_CAP_RECORD_DB))
 412                 return;
 413
 414         srq->cap_flags &= ~HNS_ROCE_SRQ_CAP_RECORD_DB;
 415         if (udata) {
 416                 uctx = rdma_udata_to_drv_context(udata,
 417                                                  struct hns_roce_ucontext,
 418                                                  ibucontext);
 419                 hns_roce_db_unmap_user(uctx, &srq->rdb);
 420         } else {
 421                 hns_roce_free_db(hr_dev, &srq->rdb);
 422         }
 423 }
 424
 425 static int alloc_srq_db(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq,
 426                         struct ib_udata *udata,
 427                         struct hns_roce_ib_create_srq_resp *resp)
 428 {
 429         struct hns_roce_ib_create_srq ucmd = {};
 430         struct hns_roce_ucontext *uctx;
 431         int ret;
 432
 433         if (udata) {
 434                 ret = get_srq_ucmd(srq, udata, &ucmd);
 435                 if (ret)
 436                         return ret;
 437
 438                 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ_RECORD_DB) &&
 439                     (ucmd.req_cap_flags & HNS_ROCE_SRQ_CAP_RECORD_DB)) {
 440                         uctx = rdma_udata_to_drv_context(udata,
 441                                         struct hns_roce_ucontext, ibucontext);
 442                         ret = hns_roce_db_map_user(uctx, ucmd.db_addr,
 443                                                    &srq->rdb);
 444                         if (ret)
 445                                 return ret;
 446
 447                         srq->cap_flags |= HNS_ROCE_RSP_SRQ_CAP_RECORD_DB;
 448                 }
 449         } else {
 450                 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ_RECORD_DB) {
 451                         ret = hns_roce_alloc_db(hr_dev, &srq->rdb, 1);
 452                         if (ret)
 453                                 return ret;
 454
 455                         *srq->rdb.db_record = 0;
 456                         srq->cap_flags |= HNS_ROCE_RSP_SRQ_CAP_RECORD_DB;
 457                 }
 458                 srq->db_reg = hr_dev->reg_base + SRQ_DB_REG;
 459         }
 460
 461         return 0;
 462 }
 463
 464 int hns_roce_create_srq(struct ib_srq *ib_srq,
 465                         struct ib_srq_init_attr *init_attr,
 466                         struct ib_udata *udata)
 467 {
 468         struct hns_roce_dev *hr_dev = to_hr_dev(ib_srq->device);
 469         struct hns_roce_ib_create_srq_resp resp = {};
 470         struct hns_roce_srq *srq = to_hr_srq(ib_srq);
 471         int ret;
 472
 473         mutex_init(&srq->mutex);
 474         spin_lock_init(&srq->lock);
 475
 476         ret = set_srq_param(srq, init_attr, udata);
 477         if (ret)
 478                 goto err_out;
 479
 480         ret = alloc_srq_buf(hr_dev, srq, udata);
 481         if (ret)
 482                 goto err_out;
 483
 484         ret = alloc_srq_db(hr_dev, srq, udata, &resp);
 485         if (ret)
 486                 goto err_srq_buf;
 487
 488         ret = alloc_srqn(hr_dev, srq);
 489         if (ret)
 490                 goto err_srq_db;
 491
 492         ret = alloc_srqc(hr_dev, srq);
 493         if (ret)
 494                 goto err_srqn;
 495
 496         if (udata) {
 497                 resp.cap_flags = srq->cap_flags;
 498                 resp.srqn = srq->srqn;
 499                 if (ib_copy_to_udata(udata, &resp,
 500                                      min(udata->outlen, sizeof(resp)))) {
 501                         ret = -EFAULT;
 502                         goto err_srqc;
 503                 }
 504         }
 505
 506         srq->event = hns_roce_ib_srq_event;
 507         refcount_set(&srq->refcount, 1);
 508         init_completion(&srq->free);
 509
 510         return 0;
 511
 512 err_srqc:
 513         free_srqc(hr_dev, srq);
 514 err_srqn:
 515         free_srqn(hr_dev, srq);
 516 err_srq_db:
 517         free_srq_db(hr_dev, srq, udata);
 518 err_srq_buf:
 519         free_srq_buf(hr_dev, srq);
 520 err_out:
 521         mutex_destroy(&srq->mutex);
 522         atomic64_inc(&hr_dev->dfx_cnt[HNS_ROCE_DFX_SRQ_CREATE_ERR_CNT]);
 523
 524         return ret;
 525 }
 526
 527 int hns_roce_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
 528 {
 529         struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
 530         struct hns_roce_srq *srq = to_hr_srq(ibsrq);
 531
 532         free_srqc(hr_dev, srq);
 533         free_srqn(hr_dev, srq);
 534         free_srq_db(hr_dev, srq, udata);
 535         free_srq_buf(hr_dev, srq);
 536         mutex_destroy(&srq->mutex);
 537         return 0;
 538 }
 539
 540 void hns_roce_init_srq_table(struct hns_roce_dev *hr_dev)
 541 {
 542         struct hns_roce_srq_table *srq_table = &hr_dev->srq_table;
 543         struct hns_roce_ida *srq_ida = &srq_table->srq_ida;
 544
 545         xa_init(&srq_table->xa);
 546
 547         ida_init(&srq_ida->ida);
 548         srq_ida->max = hr_dev->caps.num_srqs - 1;
 549         srq_ida->min = hr_dev->caps.reserved_srqs;
 550 }