drivers/infiniband/hw/erdma/erdma_qp.c

   1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
   2
   3 /* Authors: Cheng Xu <chengyou@linux.alibaba.com> */
   4 /*          Kai Shen <kaishen@linux.alibaba.com> */
   5 /* Copyright (c) 2020-2021, Alibaba Group */
   6 /* Authors: Bernard Metzler <bmt@zurich.ibm.com> */
   7 /* Copyright (c) 2008-2019, IBM Corporation */
   8
   9 #include "erdma_cm.h"
  10 #include "erdma_verbs.h"
  11
  12 void erdma_qp_llp_close(struct erdma_qp *qp)
  13 {
  14         struct erdma_qp_attrs qp_attrs;
  15
  16         down_write(&qp->state_lock);
  17
  18         switch (qp->attrs.state) {
  19         case ERDMA_QP_STATE_RTS:
  20         case ERDMA_QP_STATE_RTR:
  21         case ERDMA_QP_STATE_IDLE:
  22         case ERDMA_QP_STATE_TERMINATE:
  23                 qp_attrs.state = ERDMA_QP_STATE_CLOSING;
  24                 erdma_modify_qp_internal(qp, &qp_attrs, ERDMA_QP_ATTR_STATE);
  25                 break;
  26         case ERDMA_QP_STATE_CLOSING:
  27                 qp->attrs.state = ERDMA_QP_STATE_IDLE;
  28                 break;
  29         default:
  30                 break;
  31         }
  32
  33         if (qp->cep) {
  34                 erdma_cep_put(qp->cep);
  35                 qp->cep = NULL;
  36         }
  37
  38         up_write(&qp->state_lock);
  39 }
  40
  41 struct ib_qp *erdma_get_ibqp(struct ib_device *ibdev, int id)
  42 {
  43         struct erdma_qp *qp = find_qp_by_qpn(to_edev(ibdev), id);
  44
  45         if (qp)
  46                 return &qp->ibqp;
  47
  48         return NULL;
  49 }
  50
  51 static int erdma_modify_qp_state_to_rts(struct erdma_qp *qp,
  52                                         struct erdma_qp_attrs *attrs,
  53                                         enum erdma_qp_attr_mask mask)
  54 {
  55         int ret;
  56         struct erdma_dev *dev = qp->dev;
  57         struct erdma_cmdq_modify_qp_req req;
  58         struct tcp_sock *tp;
  59         struct erdma_cep *cep = qp->cep;
  60         struct sockaddr_storage local_addr, remote_addr;
  61
  62         if (!(mask & ERDMA_QP_ATTR_LLP_HANDLE))
  63                 return -EINVAL;
  64
  65         if (!(mask & ERDMA_QP_ATTR_MPA))
  66                 return -EINVAL;
  67
  68         ret = getname_local(cep->sock, &local_addr);
  69         if (ret < 0)
  70                 return ret;
  71
  72         ret = getname_peer(cep->sock, &remote_addr);
  73         if (ret < 0)
  74                 return ret;
  75
  76         qp->attrs.state = ERDMA_QP_STATE_RTS;
  77
  78         tp = tcp_sk(qp->cep->sock->sk);
  79
  80         erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
  81                                 CMDQ_OPCODE_MODIFY_QP);
  82
  83         req.cfg = FIELD_PREP(ERDMA_CMD_MODIFY_QP_STATE_MASK, qp->attrs.state) |
  84                   FIELD_PREP(ERDMA_CMD_MODIFY_QP_CC_MASK, qp->attrs.cc) |
  85                   FIELD_PREP(ERDMA_CMD_MODIFY_QP_QPN_MASK, QP_ID(qp));
  86
  87         req.cookie = be32_to_cpu(qp->cep->mpa.ext_data.cookie);
  88         req.dip = to_sockaddr_in(remote_addr).sin_addr.s_addr;
  89         req.sip = to_sockaddr_in(local_addr).sin_addr.s_addr;
  90         req.dport = to_sockaddr_in(remote_addr).sin_port;
  91         req.sport = to_sockaddr_in(local_addr).sin_port;
  92
  93         req.send_nxt = tp->snd_nxt;
  94         /* rsvd tcp seq for mpa-rsp in server. */
  95         if (qp->attrs.qp_type == ERDMA_QP_PASSIVE)
  96                 req.send_nxt += MPA_DEFAULT_HDR_LEN + qp->attrs.pd_len;
  97         req.recv_nxt = tp->rcv_nxt;
  98
  99         return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
 100 }
 101
 102 static int erdma_modify_qp_state_to_stop(struct erdma_qp *qp,
 103                                          struct erdma_qp_attrs *attrs,
 104                                          enum erdma_qp_attr_mask mask)
 105 {
 106         struct erdma_dev *dev = qp->dev;
 107         struct erdma_cmdq_modify_qp_req req;
 108
 109         qp->attrs.state = attrs->state;
 110
 111         erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
 112                                 CMDQ_OPCODE_MODIFY_QP);
 113
 114         req.cfg = FIELD_PREP(ERDMA_CMD_MODIFY_QP_STATE_MASK, attrs->state) |
 115                   FIELD_PREP(ERDMA_CMD_MODIFY_QP_QPN_MASK, QP_ID(qp));
 116
 117         return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
 118 }
 119
 120 int erdma_modify_qp_internal(struct erdma_qp *qp, struct erdma_qp_attrs *attrs,
 121                              enum erdma_qp_attr_mask mask)
 122 {
 123         bool need_reflush = false;
 124         int drop_conn, ret = 0;
 125
 126         if (!mask)
 127                 return 0;
 128
 129         if (!(mask & ERDMA_QP_ATTR_STATE))
 130                 return 0;
 131
 132         switch (qp->attrs.state) {
 133         case ERDMA_QP_STATE_IDLE:
 134         case ERDMA_QP_STATE_RTR:
 135                 if (attrs->state == ERDMA_QP_STATE_RTS) {
 136                         ret = erdma_modify_qp_state_to_rts(qp, attrs, mask);
 137                 } else if (attrs->state == ERDMA_QP_STATE_ERROR) {
 138                         qp->attrs.state = ERDMA_QP_STATE_ERROR;
 139                         need_reflush = true;
 140                         if (qp->cep) {
 141                                 erdma_cep_put(qp->cep);
 142                                 qp->cep = NULL;
 143                         }
 144                         ret = erdma_modify_qp_state_to_stop(qp, attrs, mask);
 145                 }
 146                 break;
 147         case ERDMA_QP_STATE_RTS:
 148                 drop_conn = 0;
 149
 150                 if (attrs->state == ERDMA_QP_STATE_CLOSING ||
 151                     attrs->state == ERDMA_QP_STATE_TERMINATE ||
 152                     attrs->state == ERDMA_QP_STATE_ERROR) {
 153                         ret = erdma_modify_qp_state_to_stop(qp, attrs, mask);
 154                         drop_conn = 1;
 155                         need_reflush = true;
 156                 }
 157
 158                 if (drop_conn)
 159                         erdma_qp_cm_drop(qp);
 160
 161                 break;
 162         case ERDMA_QP_STATE_TERMINATE:
 163                 if (attrs->state == ERDMA_QP_STATE_ERROR)
 164                         qp->attrs.state = ERDMA_QP_STATE_ERROR;
 165                 break;
 166         case ERDMA_QP_STATE_CLOSING:
 167                 if (attrs->state == ERDMA_QP_STATE_IDLE) {
 168                         qp->attrs.state = ERDMA_QP_STATE_IDLE;
 169                 } else if (attrs->state == ERDMA_QP_STATE_ERROR) {
 170                         ret = erdma_modify_qp_state_to_stop(qp, attrs, mask);
 171                         qp->attrs.state = ERDMA_QP_STATE_ERROR;
 172                 } else if (attrs->state != ERDMA_QP_STATE_CLOSING) {
 173                         return -ECONNABORTED;
 174                 }
 175                 break;
 176         default:
 177                 break;
 178         }
 179
 180         if (need_reflush && !ret && rdma_is_kernel_res(&qp->ibqp.res)) {
 181                 qp->flags |= ERDMA_QP_IN_FLUSHING;
 182                 mod_delayed_work(qp->dev->reflush_wq, &qp->reflush_dwork,
 183                                  usecs_to_jiffies(100));
 184         }
 185
 186         return ret;
 187 }
 188
 189 static void erdma_qp_safe_free(struct kref *ref)
 190 {
 191         struct erdma_qp *qp = container_of(ref, struct erdma_qp, ref);
 192
 193         complete(&qp->safe_free);
 194 }
 195
 196 void erdma_qp_put(struct erdma_qp *qp)
 197 {
 198         WARN_ON(kref_read(&qp->ref) < 1);
 199         kref_put(&qp->ref, erdma_qp_safe_free);
 200 }
 201
 202 void erdma_qp_get(struct erdma_qp *qp)
 203 {
 204         kref_get(&qp->ref);
 205 }
 206
 207 static int fill_inline_data(struct erdma_qp *qp,
 208                             const struct ib_send_wr *send_wr, u16 wqe_idx,
 209                             u32 sgl_offset, __le32 *length_field)
 210 {
 211         u32 remain_size, copy_size, data_off, bytes = 0;
 212         char *data;
 213         int i = 0;
 214
 215         wqe_idx += (sgl_offset >> SQEBB_SHIFT);
 216         sgl_offset &= (SQEBB_SIZE - 1);
 217         data = get_queue_entry(qp->kern_qp.sq_buf, wqe_idx, qp->attrs.sq_size,
 218                                SQEBB_SHIFT);
 219
 220         while (i < send_wr->num_sge) {
 221                 bytes += send_wr->sg_list[i].length;
 222                 if (bytes > (int)ERDMA_MAX_INLINE)
 223                         return -EINVAL;
 224
 225                 remain_size = send_wr->sg_list[i].length;
 226                 data_off = 0;
 227
 228                 while (1) {
 229                         copy_size = min(remain_size, SQEBB_SIZE - sgl_offset);
 230
 231                         memcpy(data + sgl_offset,
 232                                (void *)(uintptr_t)send_wr->sg_list[i].addr +
 233                                        data_off,
 234                                copy_size);
 235                         remain_size -= copy_size;
 236                         data_off += copy_size;
 237                         sgl_offset += copy_size;
 238                         wqe_idx += (sgl_offset >> SQEBB_SHIFT);
 239                         sgl_offset &= (SQEBB_SIZE - 1);
 240
 241                         data = get_queue_entry(qp->kern_qp.sq_buf, wqe_idx,
 242                                                qp->attrs.sq_size, SQEBB_SHIFT);
 243                         if (!remain_size)
 244                                 break;
 245                 }
 246
 247                 i++;
 248         }
 249         *length_field = cpu_to_le32(bytes);
 250
 251         return bytes;
 252 }
 253
 254 static int fill_sgl(struct erdma_qp *qp, const struct ib_send_wr *send_wr,
 255                     u16 wqe_idx, u32 sgl_offset, __le32 *length_field)
 256 {
 257         int i = 0;
 258         u32 bytes = 0;
 259         char *sgl;
 260
 261         if (send_wr->num_sge > qp->dev->attrs.max_send_sge)
 262                 return -EINVAL;
 263
 264         if (sgl_offset & 0xF)
 265                 return -EINVAL;
 266
 267         while (i < send_wr->num_sge) {
 268                 wqe_idx += (sgl_offset >> SQEBB_SHIFT);
 269                 sgl_offset &= (SQEBB_SIZE - 1);
 270                 sgl = get_queue_entry(qp->kern_qp.sq_buf, wqe_idx,
 271                                       qp->attrs.sq_size, SQEBB_SHIFT);
 272
 273                 bytes += send_wr->sg_list[i].length;
 274                 memcpy(sgl + sgl_offset, &send_wr->sg_list[i],
 275                        sizeof(struct ib_sge));
 276
 277                 sgl_offset += sizeof(struct ib_sge);
 278                 i++;
 279         }
 280
 281         *length_field = cpu_to_le32(bytes);
 282         return 0;
 283 }
 284
 285 static int erdma_push_one_sqe(struct erdma_qp *qp, u16 *pi,
 286                               const struct ib_send_wr *send_wr)
 287 {
 288         u32 wqe_size, wqebb_cnt, hw_op, flags, sgl_offset;
 289         u32 idx = *pi & (qp->attrs.sq_size - 1);
 290         enum ib_wr_opcode op = send_wr->opcode;
 291         struct erdma_atomic_sqe *atomic_sqe;
 292         struct erdma_readreq_sqe *read_sqe;
 293         struct erdma_reg_mr_sqe *regmr_sge;
 294         struct erdma_write_sqe *write_sqe;
 295         struct erdma_send_sqe *send_sqe;
 296         struct ib_rdma_wr *rdma_wr;
 297         struct erdma_sge *sge;
 298         __le32 *length_field;
 299         struct erdma_mr *mr;
 300         u64 wqe_hdr, *entry;
 301         u32 attrs;
 302         int ret;
 303
 304         entry = get_queue_entry(qp->kern_qp.sq_buf, idx, qp->attrs.sq_size,
 305                                 SQEBB_SHIFT);
 306
 307         /* Clear the SQE header section. */
 308         *entry = 0;
 309
 310         qp->kern_qp.swr_tbl[idx] = send_wr->wr_id;
 311         flags = send_wr->send_flags;
 312         wqe_hdr = FIELD_PREP(
 313                 ERDMA_SQE_HDR_CE_MASK,
 314                 ((flags & IB_SEND_SIGNALED) || qp->kern_qp.sig_all) ? 1 : 0);
 315         wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_SE_MASK,
 316                               flags & IB_SEND_SOLICITED ? 1 : 0);
 317         wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_FENCE_MASK,
 318                               flags & IB_SEND_FENCE ? 1 : 0);
 319         wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_INLINE_MASK,
 320                               flags & IB_SEND_INLINE ? 1 : 0);
 321         wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_QPN_MASK, QP_ID(qp));
 322
 323         switch (op) {
 324         case IB_WR_RDMA_WRITE:
 325         case IB_WR_RDMA_WRITE_WITH_IMM:
 326                 hw_op = ERDMA_OP_WRITE;
 327                 if (op == IB_WR_RDMA_WRITE_WITH_IMM)
 328                         hw_op = ERDMA_OP_WRITE_WITH_IMM;
 329                 wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_OPCODE_MASK, hw_op);
 330                 rdma_wr = container_of(send_wr, struct ib_rdma_wr, wr);
 331                 write_sqe = (struct erdma_write_sqe *)entry;
 332
 333                 write_sqe->imm_data = send_wr->ex.imm_data;
 334                 write_sqe->sink_stag = cpu_to_le32(rdma_wr->rkey);
 335                 write_sqe->sink_to_h =
 336                         cpu_to_le32(upper_32_bits(rdma_wr->remote_addr));
 337                 write_sqe->sink_to_l =
 338                         cpu_to_le32(lower_32_bits(rdma_wr->remote_addr));
 339
 340                 length_field = &write_sqe->length;
 341                 wqe_size = sizeof(struct erdma_write_sqe);
 342                 sgl_offset = wqe_size;
 343                 break;
 344         case IB_WR_RDMA_READ:
 345         case IB_WR_RDMA_READ_WITH_INV:
 346                 read_sqe = (struct erdma_readreq_sqe *)entry;
 347                 if (unlikely(send_wr->num_sge != 1))
 348                         return -EINVAL;
 349                 hw_op = ERDMA_OP_READ;
 350                 if (op == IB_WR_RDMA_READ_WITH_INV) {
 351                         hw_op = ERDMA_OP_READ_WITH_INV;
 352                         read_sqe->invalid_stag =
 353                                 cpu_to_le32(send_wr->ex.invalidate_rkey);
 354                 }
 355
 356                 wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_OPCODE_MASK, hw_op);
 357                 rdma_wr = container_of(send_wr, struct ib_rdma_wr, wr);
 358                 read_sqe->length = cpu_to_le32(send_wr->sg_list[0].length);
 359                 read_sqe->sink_stag = cpu_to_le32(send_wr->sg_list[0].lkey);
 360                 read_sqe->sink_to_l =
 361                         cpu_to_le32(lower_32_bits(send_wr->sg_list[0].addr));
 362                 read_sqe->sink_to_h =
 363                         cpu_to_le32(upper_32_bits(send_wr->sg_list[0].addr));
 364
 365                 sge = get_queue_entry(qp->kern_qp.sq_buf, idx + 1,
 366                                       qp->attrs.sq_size, SQEBB_SHIFT);
 367                 sge->addr = cpu_to_le64(rdma_wr->remote_addr);
 368                 sge->key = cpu_to_le32(rdma_wr->rkey);
 369                 sge->length = cpu_to_le32(send_wr->sg_list[0].length);
 370                 wqe_size = sizeof(struct erdma_readreq_sqe) +
 371                            send_wr->num_sge * sizeof(struct ib_sge);
 372
 373                 goto out;
 374         case IB_WR_SEND:
 375         case IB_WR_SEND_WITH_IMM:
 376         case IB_WR_SEND_WITH_INV:
 377                 send_sqe = (struct erdma_send_sqe *)entry;
 378                 hw_op = ERDMA_OP_SEND;
 379                 if (op == IB_WR_SEND_WITH_IMM) {
 380                         hw_op = ERDMA_OP_SEND_WITH_IMM;
 381                         send_sqe->imm_data = send_wr->ex.imm_data;
 382                 } else if (op == IB_WR_SEND_WITH_INV) {
 383                         hw_op = ERDMA_OP_SEND_WITH_INV;
 384                         send_sqe->invalid_stag =
 385                                 cpu_to_le32(send_wr->ex.invalidate_rkey);
 386                 }
 387                 wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_OPCODE_MASK, hw_op);
 388                 length_field = &send_sqe->length;
 389                 wqe_size = sizeof(struct erdma_send_sqe);
 390                 sgl_offset = wqe_size;
 391
 392                 break;
 393         case IB_WR_REG_MR:
 394                 wqe_hdr |=
 395                         FIELD_PREP(ERDMA_SQE_HDR_OPCODE_MASK, ERDMA_OP_REG_MR);
 396                 regmr_sge = (struct erdma_reg_mr_sqe *)entry;
 397                 mr = to_emr(reg_wr(send_wr)->mr);
 398
 399                 mr->access = ERDMA_MR_ACC_LR |
 400                              to_erdma_access_flags(reg_wr(send_wr)->access);
 401                 regmr_sge->addr = cpu_to_le64(mr->ibmr.iova);
 402                 regmr_sge->length = cpu_to_le32(mr->ibmr.length);
 403                 regmr_sge->stag = cpu_to_le32(reg_wr(send_wr)->key);
 404                 attrs = FIELD_PREP(ERDMA_SQE_MR_ACCESS_MASK, mr->access) |
 405                         FIELD_PREP(ERDMA_SQE_MR_MTT_CNT_MASK,
 406                                    mr->mem.mtt_nents);
 407
 408                 if (mr->mem.mtt_nents <= ERDMA_MAX_INLINE_MTT_ENTRIES) {
 409                         attrs |= FIELD_PREP(ERDMA_SQE_MR_MTT_TYPE_MASK, 0);
 410                         /* Copy SGLs to SQE content to accelerate */
 411                         memcpy(get_queue_entry(qp->kern_qp.sq_buf, idx + 1,
 412                                                qp->attrs.sq_size, SQEBB_SHIFT),
 413                                mr->mem.mtt->buf, MTT_SIZE(mr->mem.mtt_nents));
 414                         wqe_size = sizeof(struct erdma_reg_mr_sqe) +
 415                                    MTT_SIZE(mr->mem.mtt_nents);
 416                 } else {
 417                         attrs |= FIELD_PREP(ERDMA_SQE_MR_MTT_TYPE_MASK, 1);
 418                         wqe_size = sizeof(struct erdma_reg_mr_sqe);
 419                 }
 420
 421                 regmr_sge->attrs = cpu_to_le32(attrs);
 422                 goto out;
 423         case IB_WR_LOCAL_INV:
 424                 wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_OPCODE_MASK,
 425                                       ERDMA_OP_LOCAL_INV);
 426                 regmr_sge = (struct erdma_reg_mr_sqe *)entry;
 427                 regmr_sge->stag = cpu_to_le32(send_wr->ex.invalidate_rkey);
 428                 wqe_size = sizeof(struct erdma_reg_mr_sqe);
 429                 goto out;
 430         case IB_WR_ATOMIC_CMP_AND_SWP:
 431         case IB_WR_ATOMIC_FETCH_AND_ADD:
 432                 atomic_sqe = (struct erdma_atomic_sqe *)entry;
 433                 if (op == IB_WR_ATOMIC_CMP_AND_SWP) {
 434                         wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_OPCODE_MASK,
 435                                               ERDMA_OP_ATOMIC_CAS);
 436                         atomic_sqe->fetchadd_swap_data =
 437                                 cpu_to_le64(atomic_wr(send_wr)->swap);
 438                         atomic_sqe->cmp_data =
 439                                 cpu_to_le64(atomic_wr(send_wr)->compare_add);
 440                 } else {
 441                         wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_OPCODE_MASK,
 442                                               ERDMA_OP_ATOMIC_FAA);
 443                         atomic_sqe->fetchadd_swap_data =
 444                                 cpu_to_le64(atomic_wr(send_wr)->compare_add);
 445                 }
 446
 447                 sge = get_queue_entry(qp->kern_qp.sq_buf, idx + 1,
 448                                       qp->attrs.sq_size, SQEBB_SHIFT);
 449                 sge->addr = cpu_to_le64(atomic_wr(send_wr)->remote_addr);
 450                 sge->key = cpu_to_le32(atomic_wr(send_wr)->rkey);
 451                 sge++;
 452
 453                 sge->addr = cpu_to_le64(send_wr->sg_list[0].addr);
 454                 sge->key = cpu_to_le32(send_wr->sg_list[0].lkey);
 455                 sge->length = cpu_to_le32(send_wr->sg_list[0].length);
 456
 457                 wqe_size = sizeof(*atomic_sqe);
 458                 goto out;
 459         default:
 460                 return -EOPNOTSUPP;
 461         }
 462
 463         if (flags & IB_SEND_INLINE) {
 464                 ret = fill_inline_data(qp, send_wr, idx, sgl_offset,
 465                                        length_field);
 466                 if (ret < 0)
 467                         return -EINVAL;
 468                 wqe_size += ret;
 469                 wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_SGL_LEN_MASK, ret);
 470         } else {
 471                 ret = fill_sgl(qp, send_wr, idx, sgl_offset, length_field);
 472                 if (ret)
 473                         return -EINVAL;
 474                 wqe_size += send_wr->num_sge * sizeof(struct ib_sge);
 475                 wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_SGL_LEN_MASK,
 476                                       send_wr->num_sge);
 477         }
 478
 479 out:
 480         wqebb_cnt = SQEBB_COUNT(wqe_size);
 481         wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_WQEBB_CNT_MASK, wqebb_cnt - 1);
 482         *pi += wqebb_cnt;
 483         wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_WQEBB_INDEX_MASK, *pi);
 484
 485         *entry = wqe_hdr;
 486
 487         return 0;
 488 }
 489
 490 static void kick_sq_db(struct erdma_qp *qp, u16 pi)
 491 {
 492         u64 db_data = FIELD_PREP(ERDMA_SQE_HDR_QPN_MASK, QP_ID(qp)) |
 493                       FIELD_PREP(ERDMA_SQE_HDR_WQEBB_INDEX_MASK, pi);
 494
 495         *(u64 *)qp->kern_qp.sq_dbrec = db_data;
 496         writeq(db_data, qp->kern_qp.hw_sq_db);
 497 }
 498
 499 int erdma_post_send(struct ib_qp *ibqp, const struct ib_send_wr *send_wr,
 500                     const struct ib_send_wr **bad_send_wr)
 501 {
 502         struct erdma_qp *qp = to_eqp(ibqp);
 503         int ret = 0;
 504         const struct ib_send_wr *wr = send_wr;
 505         unsigned long flags;
 506         u16 sq_pi;
 507
 508         if (!send_wr)
 509                 return -EINVAL;
 510
 511         spin_lock_irqsave(&qp->lock, flags);
 512         sq_pi = qp->kern_qp.sq_pi;
 513
 514         while (wr) {
 515                 if ((u16)(sq_pi - qp->kern_qp.sq_ci) >= qp->attrs.sq_size) {
 516                         ret = -ENOMEM;
 517                         *bad_send_wr = send_wr;
 518                         break;
 519                 }
 520
 521                 ret = erdma_push_one_sqe(qp, &sq_pi, wr);
 522                 if (ret) {
 523                         *bad_send_wr = wr;
 524                         break;
 525                 }
 526                 qp->kern_qp.sq_pi = sq_pi;
 527                 kick_sq_db(qp, sq_pi);
 528
 529                 wr = wr->next;
 530         }
 531         spin_unlock_irqrestore(&qp->lock, flags);
 532
 533         if (unlikely(qp->flags & ERDMA_QP_IN_FLUSHING))
 534                 mod_delayed_work(qp->dev->reflush_wq, &qp->reflush_dwork,
 535                                  usecs_to_jiffies(100));
 536
 537         return ret;
 538 }
 539
 540 static int erdma_post_recv_one(struct erdma_qp *qp,
 541                                const struct ib_recv_wr *recv_wr)
 542 {
 543         struct erdma_rqe *rqe =
 544                 get_queue_entry(qp->kern_qp.rq_buf, qp->kern_qp.rq_pi,
 545                                 qp->attrs.rq_size, RQE_SHIFT);
 546
 547         rqe->qe_idx = cpu_to_le16(qp->kern_qp.rq_pi + 1);
 548         rqe->qpn = cpu_to_le32(QP_ID(qp));
 549
 550         if (recv_wr->num_sge == 0) {
 551                 rqe->length = 0;
 552         } else if (recv_wr->num_sge == 1) {
 553                 rqe->stag = cpu_to_le32(recv_wr->sg_list[0].lkey);
 554                 rqe->to = cpu_to_le64(recv_wr->sg_list[0].addr);
 555                 rqe->length = cpu_to_le32(recv_wr->sg_list[0].length);
 556         } else {
 557                 return -EINVAL;
 558         }
 559
 560         *(u64 *)qp->kern_qp.rq_dbrec = *(u64 *)rqe;
 561         writeq(*(u64 *)rqe, qp->kern_qp.hw_rq_db);
 562
 563         qp->kern_qp.rwr_tbl[qp->kern_qp.rq_pi & (qp->attrs.rq_size - 1)] =
 564                 recv_wr->wr_id;
 565         qp->kern_qp.rq_pi++;
 566
 567         return 0;
 568 }
 569
 570 int erdma_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *recv_wr,
 571                     const struct ib_recv_wr **bad_recv_wr)
 572 {
 573         const struct ib_recv_wr *wr = recv_wr;
 574         struct erdma_qp *qp = to_eqp(ibqp);
 575         unsigned long flags;
 576         int ret;
 577
 578         spin_lock_irqsave(&qp->lock, flags);
 579
 580         while (wr) {
 581                 ret = erdma_post_recv_one(qp, wr);
 582                 if (ret) {
 583                         *bad_recv_wr = wr;
 584                         break;
 585                 }
 586                 wr = wr->next;
 587         }
 588
 589         spin_unlock_irqrestore(&qp->lock, flags);
 590
 591         if (unlikely(qp->flags & ERDMA_QP_IN_FLUSHING))
 592                 mod_delayed_work(qp->dev->reflush_wq, &qp->reflush_dwork,
 593                                  usecs_to_jiffies(100));
 594
 595         return ret;
 596 }