net/smc/smc_wr.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Shared Memory Communications over RDMA (SMC-R) and RoCE
   4  *
   5  * Work Requests exploiting Infiniband API
   6  *
   7  * Work requests (WR) of type ib_post_send or ib_post_recv respectively
   8  * are submitted to either RC SQ or RC RQ respectively
   9  * (reliably connected send/receive queue)
  10  * and become work queue entries (WQEs).
  11  * While an SQ WR/WQE is pending, we track it until transmission completion.
  12  * Through a send or receive completion queue (CQ) respectively,
  13  * we get completion queue entries (CQEs) [aka work completions (WCs)].
  14  * Since the CQ callback is called from IRQ context, we split work by using
  15  * bottom halves implemented by tasklets.
  16  *
  17  * SMC uses this to exchange LLC (link layer control)
  18  * and CDC (connection data control) messages.
  19  *
  20  * Copyright IBM Corp. 2016
  21  *
  22  * Author(s):  Steffen Maier <maier@linux.vnet.ibm.com>
  23  */
  24
  25 #include <linux/atomic.h>
  26 #include <linux/hashtable.h>
  27 #include <linux/wait.h>
  28 #include <rdma/ib_verbs.h>
  29 #include <asm/div64.h>
  30
  31 #include "smc.h"
  32 #include "smc_wr.h"
  33
  34 #define SMC_WR_MAX_POLL_CQE 10  /* max. # of compl. queue elements in 1 poll */
  35
  36 #define SMC_WR_RX_HASH_BITS 4
  37 static DEFINE_HASHTABLE(smc_wr_rx_hash, SMC_WR_RX_HASH_BITS);
  38 static DEFINE_SPINLOCK(smc_wr_rx_hash_lock);
  39
  40 struct smc_wr_tx_pend { /* control data for a pending send request */
  41         u64                     wr_id;          /* work request id sent */
  42         smc_wr_tx_handler       handler;
  43         enum ib_wc_status       wc_status;      /* CQE status */
  44         struct smc_link         *link;
  45         u32                     idx;
  46         struct smc_wr_tx_pend_priv priv;
  47 };
  48
  49 /******************************** send queue *********************************/
  50
  51 /*------------------------------- completion --------------------------------*/
  52
  53 static inline int smc_wr_tx_find_pending_index(struct smc_link *link, u64 wr_id)
  54 {
  55         u32 i;
  56
  57         for (i = 0; i < link->wr_tx_cnt; i++) {
  58                 if (link->wr_tx_pends[i].wr_id == wr_id)
  59                         return i;
  60         }
  61         return link->wr_tx_cnt;
  62 }
  63
  64 static inline void smc_wr_tx_process_cqe(struct ib_wc *wc)
  65 {
  66         struct smc_wr_tx_pend pnd_snd;
  67         struct smc_link *link;
  68         u32 pnd_snd_idx;
  69         int i;
  70
  71         link = wc->qp->qp_context;
  72
  73         if (wc->opcode == IB_WC_REG_MR) {
  74                 if (wc->status)
  75                         link->wr_reg_state = FAILED;
  76                 else
  77                         link->wr_reg_state = CONFIRMED;
  78                 wake_up(&link->wr_reg_wait);
  79                 return;
  80         }
  81
  82         pnd_snd_idx = smc_wr_tx_find_pending_index(link, wc->wr_id);
  83         if (pnd_snd_idx == link->wr_tx_cnt)
  84                 return;
  85         link->wr_tx_pends[pnd_snd_idx].wc_status = wc->status;
  86         memcpy(&pnd_snd, &link->wr_tx_pends[pnd_snd_idx], sizeof(pnd_snd));
  87         /* clear the full struct smc_wr_tx_pend including .priv */
  88         memset(&link->wr_tx_pends[pnd_snd_idx], 0,
  89                sizeof(link->wr_tx_pends[pnd_snd_idx]));
  90         memset(&link->wr_tx_bufs[pnd_snd_idx], 0,
  91                sizeof(link->wr_tx_bufs[pnd_snd_idx]));
  92         if (!test_and_clear_bit(pnd_snd_idx, link->wr_tx_mask))
  93                 return;
  94         if (wc->status) {
  95                 for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) {
  96                         /* clear full struct smc_wr_tx_pend including .priv */
  97                         memset(&link->wr_tx_pends[i], 0,
  98                                sizeof(link->wr_tx_pends[i]));
  99                         memset(&link->wr_tx_bufs[i], 0,
 100                                sizeof(link->wr_tx_bufs[i]));
 101                         clear_bit(i, link->wr_tx_mask);
 102                 }
 103                 /* terminate connections of this link group abnormally */
 104                 smc_lgr_terminate(smc_get_lgr(link));
 105         }
 106         if (pnd_snd.handler)
 107                 pnd_snd.handler(&pnd_snd.priv, link, wc->status);
 108         wake_up(&link->wr_tx_wait);
 109 }
 110
 111 static void smc_wr_tx_tasklet_fn(unsigned long data)
 112 {
 113         struct smc_ib_device *dev = (struct smc_ib_device *)data;
 114         struct ib_wc wc[SMC_WR_MAX_POLL_CQE];
 115         int i = 0, rc;
 116         int polled = 0;
 117
 118 again:
 119         polled++;
 120         do {
 121                 memset(&wc, 0, sizeof(wc));
 122                 rc = ib_poll_cq(dev->roce_cq_send, SMC_WR_MAX_POLL_CQE, wc);
 123                 if (polled == 1) {
 124                         ib_req_notify_cq(dev->roce_cq_send,
 125                                          IB_CQ_NEXT_COMP |
 126                                          IB_CQ_REPORT_MISSED_EVENTS);
 127                 }
 128                 if (!rc)
 129                         break;
 130                 for (i = 0; i < rc; i++)
 131                         smc_wr_tx_process_cqe(&wc[i]);
 132         } while (rc > 0);
 133         if (polled == 1)
 134                 goto again;
 135 }
 136
 137 void smc_wr_tx_cq_handler(struct ib_cq *ib_cq, void *cq_context)
 138 {
 139         struct smc_ib_device *dev = (struct smc_ib_device *)cq_context;
 140
 141         tasklet_schedule(&dev->send_tasklet);
 142 }
 143
 144 /*---------------------------- request submission ---------------------------*/
 145
 146 static inline int smc_wr_tx_get_free_slot_index(struct smc_link *link, u32 *idx)
 147 {
 148         *idx = link->wr_tx_cnt;
 149         for_each_clear_bit(*idx, link->wr_tx_mask, link->wr_tx_cnt) {
 150                 if (!test_and_set_bit(*idx, link->wr_tx_mask))
 151                         return 0;
 152         }
 153         *idx = link->wr_tx_cnt;
 154         return -EBUSY;
 155 }
 156
 157 /**
 158  * smc_wr_tx_get_free_slot() - returns buffer for message assembly,
 159  *                      and sets info for pending transmit tracking
 160  * @link:               Pointer to smc_link used to later send the message.
 161  * @handler:            Send completion handler function pointer.
 162  * @wr_buf:             Out value returns pointer to message buffer.
 163  * @wr_pend_priv:       Out value returns pointer serving as handler context.
 164  *
 165  * Return: 0 on success, or -errno on error.
 166  */
 167 int smc_wr_tx_get_free_slot(struct smc_link *link,
 168                             smc_wr_tx_handler handler,
 169                             struct smc_wr_buf **wr_buf,
 170                             struct smc_wr_tx_pend_priv **wr_pend_priv)
 171 {
 172         struct smc_wr_tx_pend *wr_pend;
 173         u32 idx = link->wr_tx_cnt;
 174         struct ib_send_wr *wr_ib;
 175         u64 wr_id;
 176         int rc;
 177
 178         *wr_buf = NULL;
 179         *wr_pend_priv = NULL;
 180         if (in_softirq()) {
 181                 rc = smc_wr_tx_get_free_slot_index(link, &idx);
 182                 if (rc)
 183                         return rc;
 184         } else {
 185                 rc = wait_event_timeout(
 186                         link->wr_tx_wait,
 187                         link->state == SMC_LNK_INACTIVE ||
 188                         (smc_wr_tx_get_free_slot_index(link, &idx) != -EBUSY),
 189                         SMC_WR_TX_WAIT_FREE_SLOT_TIME);
 190                 if (!rc) {
 191                         /* timeout - terminate connections */
 192                         smc_lgr_terminate(smc_get_lgr(link));
 193                         return -EPIPE;
 194                 }
 195                 if (idx == link->wr_tx_cnt)
 196                         return -EPIPE;
 197         }
 198         wr_id = smc_wr_tx_get_next_wr_id(link);
 199         wr_pend = &link->wr_tx_pends[idx];
 200         wr_pend->wr_id = wr_id;
 201         wr_pend->handler = handler;
 202         wr_pend->link = link;
 203         wr_pend->idx = idx;
 204         wr_ib = &link->wr_tx_ibs[idx];
 205         wr_ib->wr_id = wr_id;
 206         *wr_buf = &link->wr_tx_bufs[idx];
 207         *wr_pend_priv = &wr_pend->priv;
 208         return 0;
 209 }
 210
 211 int smc_wr_tx_put_slot(struct smc_link *link,
 212                        struct smc_wr_tx_pend_priv *wr_pend_priv)
 213 {
 214         struct smc_wr_tx_pend *pend;
 215
 216         pend = container_of(wr_pend_priv, struct smc_wr_tx_pend, priv);
 217         if (pend->idx < link->wr_tx_cnt) {
 218                 u32 idx = pend->idx;
 219
 220                 /* clear the full struct smc_wr_tx_pend including .priv */
 221                 memset(&link->wr_tx_pends[pend->idx], 0,
 222                        sizeof(link->wr_tx_pends[pend->idx]));
 223                 memset(&link->wr_tx_bufs[pend->idx], 0,
 224                        sizeof(link->wr_tx_bufs[pend->idx]));
 225                 test_and_clear_bit(idx, link->wr_tx_mask);
 226                 return 1;
 227         }
 228
 229         return 0;
 230 }
 231
 232 /* Send prepared WR slot via ib_post_send.
 233  * @priv: pointer to smc_wr_tx_pend_priv identifying prepared message buffer
 234  */
 235 int smc_wr_tx_send(struct smc_link *link, struct smc_wr_tx_pend_priv *priv)
 236 {
 237         struct smc_wr_tx_pend *pend;
 238         int rc;
 239
 240         ib_req_notify_cq(link->smcibdev->roce_cq_send,
 241                          IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS);
 242         pend = container_of(priv, struct smc_wr_tx_pend, priv);
 243         rc = ib_post_send(link->roce_qp, &link->wr_tx_ibs[pend->idx], NULL);
 244         if (rc) {
 245                 smc_wr_tx_put_slot(link, priv);
 246                 smc_lgr_terminate(smc_get_lgr(link));
 247         }
 248         return rc;
 249 }
 250
 251 /* Register a memory region and wait for result. */
 252 int smc_wr_reg_send(struct smc_link *link, struct ib_mr *mr)
 253 {
 254         int rc;
 255
 256         ib_req_notify_cq(link->smcibdev->roce_cq_send,
 257                          IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS);
 258         link->wr_reg_state = POSTED;
 259         link->wr_reg.wr.wr_id = (u64)(uintptr_t)mr;
 260         link->wr_reg.mr = mr;
 261         link->wr_reg.key = mr->rkey;
 262         rc = ib_post_send(link->roce_qp, &link->wr_reg.wr, NULL);
 263         if (rc)
 264                 return rc;
 265
 266         rc = wait_event_interruptible_timeout(link->wr_reg_wait,
 267                                               (link->wr_reg_state != POSTED),
 268                                               SMC_WR_REG_MR_WAIT_TIME);
 269         if (!rc) {
 270                 /* timeout - terminate connections */
 271                 smc_lgr_terminate(smc_get_lgr(link));
 272                 return -EPIPE;
 273         }
 274         if (rc == -ERESTARTSYS)
 275                 return -EINTR;
 276         switch (link->wr_reg_state) {
 277         case CONFIRMED:
 278                 rc = 0;
 279                 break;
 280         case FAILED:
 281                 rc = -EIO;
 282                 break;
 283         case POSTED:
 284                 rc = -EPIPE;
 285                 break;
 286         }
 287         return rc;
 288 }
 289
 290 void smc_wr_tx_dismiss_slots(struct smc_link *link, u8 wr_tx_hdr_type,
 291                              smc_wr_tx_filter filter,
 292                              smc_wr_tx_dismisser dismisser,
 293                              unsigned long data)
 294 {
 295         struct smc_wr_tx_pend_priv *tx_pend;
 296         struct smc_wr_rx_hdr *wr_tx;
 297         int i;
 298
 299         for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) {
 300                 wr_tx = (struct smc_wr_rx_hdr *)&link->wr_tx_bufs[i];
 301                 if (wr_tx->type != wr_tx_hdr_type)
 302                         continue;
 303                 tx_pend = &link->wr_tx_pends[i].priv;
 304                 if (filter(tx_pend, data))
 305                         dismisser(tx_pend);
 306         }
 307 }
 308
 309 /****************************** receive queue ********************************/
 310
 311 int smc_wr_rx_register_handler(struct smc_wr_rx_handler *handler)
 312 {
 313         struct smc_wr_rx_handler *h_iter;
 314         int rc = 0;
 315
 316         spin_lock(&smc_wr_rx_hash_lock);
 317         hash_for_each_possible(smc_wr_rx_hash, h_iter, list, handler->type) {
 318                 if (h_iter->type == handler->type) {
 319                         rc = -EEXIST;
 320                         goto out_unlock;
 321                 }
 322         }
 323         hash_add(smc_wr_rx_hash, &handler->list, handler->type);
 324 out_unlock:
 325         spin_unlock(&smc_wr_rx_hash_lock);
 326         return rc;
 327 }
 328
 329 /* Demultiplex a received work request based on the message type to its handler.
 330  * Relies on smc_wr_rx_hash having been completely filled before any IB WRs,
 331  * and not being modified any more afterwards so we don't need to lock it.
 332  */
 333 static inline void smc_wr_rx_demultiplex(struct ib_wc *wc)
 334 {
 335         struct smc_link *link = (struct smc_link *)wc->qp->qp_context;
 336         struct smc_wr_rx_handler *handler;
 337         struct smc_wr_rx_hdr *wr_rx;
 338         u64 temp_wr_id;
 339         u32 index;
 340
 341         if (wc->byte_len < sizeof(*wr_rx))
 342                 return; /* short message */
 343         temp_wr_id = wc->wr_id;
 344         index = do_div(temp_wr_id, link->wr_rx_cnt);
 345         wr_rx = (struct smc_wr_rx_hdr *)&link->wr_rx_bufs[index];
 346         hash_for_each_possible(smc_wr_rx_hash, handler, list, wr_rx->type) {
 347                 if (handler->type == wr_rx->type)
 348                         handler->handler(wc, wr_rx);
 349         }
 350 }
 351
 352 static inline void smc_wr_rx_process_cqes(struct ib_wc wc[], int num)
 353 {
 354         struct smc_link *link;
 355         int i;
 356
 357         for (i = 0; i < num; i++) {
 358                 link = wc[i].qp->qp_context;
 359                 if (wc[i].status == IB_WC_SUCCESS) {
 360                         link->wr_rx_tstamp = jiffies;
 361                         smc_wr_rx_demultiplex(&wc[i]);
 362                         smc_wr_rx_post(link); /* refill WR RX */
 363                 } else {
 364                         /* handle status errors */
 365                         switch (wc[i].status) {
 366                         case IB_WC_RETRY_EXC_ERR:
 367                         case IB_WC_RNR_RETRY_EXC_ERR:
 368                         case IB_WC_WR_FLUSH_ERR:
 369                                 /* terminate connections of this link group
 370                                  * abnormally
 371                                  */
 372                                 smc_lgr_terminate(smc_get_lgr(link));
 373                                 break;
 374                         default:
 375                                 smc_wr_rx_post(link); /* refill WR RX */
 376                                 break;
 377                         }
 378                 }
 379         }
 380 }
 381
 382 static void smc_wr_rx_tasklet_fn(unsigned long data)
 383 {
 384         struct smc_ib_device *dev = (struct smc_ib_device *)data;
 385         struct ib_wc wc[SMC_WR_MAX_POLL_CQE];
 386         int polled = 0;
 387         int rc;
 388
 389 again:
 390         polled++;
 391         do {
 392                 memset(&wc, 0, sizeof(wc));
 393                 rc = ib_poll_cq(dev->roce_cq_recv, SMC_WR_MAX_POLL_CQE, wc);
 394                 if (polled == 1) {
 395                         ib_req_notify_cq(dev->roce_cq_recv,
 396                                          IB_CQ_SOLICITED_MASK
 397                                          | IB_CQ_REPORT_MISSED_EVENTS);
 398                 }
 399                 if (!rc)
 400                         break;
 401                 smc_wr_rx_process_cqes(&wc[0], rc);
 402         } while (rc > 0);
 403         if (polled == 1)
 404                 goto again;
 405 }
 406
 407 void smc_wr_rx_cq_handler(struct ib_cq *ib_cq, void *cq_context)
 408 {
 409         struct smc_ib_device *dev = (struct smc_ib_device *)cq_context;
 410
 411         tasklet_schedule(&dev->recv_tasklet);
 412 }
 413
 414 int smc_wr_rx_post_init(struct smc_link *link)
 415 {
 416         u32 i;
 417         int rc = 0;
 418
 419         for (i = 0; i < link->wr_rx_cnt; i++)
 420                 rc = smc_wr_rx_post(link);
 421         return rc;
 422 }
 423
 424 /***************************** init, exit, misc ******************************/
 425
 426 void smc_wr_remember_qp_attr(struct smc_link *lnk)
 427 {
 428         struct ib_qp_attr *attr = &lnk->qp_attr;
 429         struct ib_qp_init_attr init_attr;
 430
 431         memset(attr, 0, sizeof(*attr));
 432         memset(&init_attr, 0, sizeof(init_attr));
 433         ib_query_qp(lnk->roce_qp, attr,
 434                     IB_QP_STATE |
 435                     IB_QP_CUR_STATE |
 436                     IB_QP_PKEY_INDEX |
 437                     IB_QP_PORT |
 438                     IB_QP_QKEY |
 439                     IB_QP_AV |
 440                     IB_QP_PATH_MTU |
 441                     IB_QP_TIMEOUT |
 442                     IB_QP_RETRY_CNT |
 443                     IB_QP_RNR_RETRY |
 444                     IB_QP_RQ_PSN |
 445                     IB_QP_ALT_PATH |
 446                     IB_QP_MIN_RNR_TIMER |
 447                     IB_QP_SQ_PSN |
 448                     IB_QP_PATH_MIG_STATE |
 449                     IB_QP_CAP |
 450                     IB_QP_DEST_QPN,
 451                     &init_attr);
 452
 453         lnk->wr_tx_cnt = min_t(size_t, SMC_WR_BUF_CNT,
 454                                lnk->qp_attr.cap.max_send_wr);
 455         lnk->wr_rx_cnt = min_t(size_t, SMC_WR_BUF_CNT * 3,
 456                                lnk->qp_attr.cap.max_recv_wr);
 457 }
 458
 459 static void smc_wr_init_sge(struct smc_link *lnk)
 460 {
 461         u32 i;
 462
 463         for (i = 0; i < lnk->wr_tx_cnt; i++) {
 464                 lnk->wr_tx_sges[i].addr =
 465                         lnk->wr_tx_dma_addr + i * SMC_WR_BUF_SIZE;
 466                 lnk->wr_tx_sges[i].length = SMC_WR_TX_SIZE;
 467                 lnk->wr_tx_sges[i].lkey = lnk->roce_pd->local_dma_lkey;
 468                 lnk->wr_tx_ibs[i].next = NULL;
 469                 lnk->wr_tx_ibs[i].sg_list = &lnk->wr_tx_sges[i];
 470                 lnk->wr_tx_ibs[i].num_sge = 1;
 471                 lnk->wr_tx_ibs[i].opcode = IB_WR_SEND;
 472                 lnk->wr_tx_ibs[i].send_flags =
 473                         IB_SEND_SIGNALED | IB_SEND_SOLICITED;
 474         }
 475         for (i = 0; i < lnk->wr_rx_cnt; i++) {
 476                 lnk->wr_rx_sges[i].addr =
 477                         lnk->wr_rx_dma_addr + i * SMC_WR_BUF_SIZE;
 478                 lnk->wr_rx_sges[i].length = SMC_WR_BUF_SIZE;
 479                 lnk->wr_rx_sges[i].lkey = lnk->roce_pd->local_dma_lkey;
 480                 lnk->wr_rx_ibs[i].next = NULL;
 481                 lnk->wr_rx_ibs[i].sg_list = &lnk->wr_rx_sges[i];
 482                 lnk->wr_rx_ibs[i].num_sge = 1;
 483         }
 484         lnk->wr_reg.wr.next = NULL;
 485         lnk->wr_reg.wr.num_sge = 0;
 486         lnk->wr_reg.wr.send_flags = IB_SEND_SIGNALED;
 487         lnk->wr_reg.wr.opcode = IB_WR_REG_MR;
 488         lnk->wr_reg.access = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE;
 489 }
 490
 491 void smc_wr_free_link(struct smc_link *lnk)
 492 {
 493         struct ib_device *ibdev;
 494
 495         memset(lnk->wr_tx_mask, 0,
 496                BITS_TO_LONGS(SMC_WR_BUF_CNT) * sizeof(*lnk->wr_tx_mask));
 497
 498         if (!lnk->smcibdev)
 499                 return;
 500         ibdev = lnk->smcibdev->ibdev;
 501
 502         if (lnk->wr_rx_dma_addr) {
 503                 ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr,
 504                                     SMC_WR_BUF_SIZE * lnk->wr_rx_cnt,
 505                                     DMA_FROM_DEVICE);
 506                 lnk->wr_rx_dma_addr = 0;
 507         }
 508         if (lnk->wr_tx_dma_addr) {
 509                 ib_dma_unmap_single(ibdev, lnk->wr_tx_dma_addr,
 510                                     SMC_WR_BUF_SIZE * lnk->wr_tx_cnt,
 511                                     DMA_TO_DEVICE);
 512                 lnk->wr_tx_dma_addr = 0;
 513         }
 514 }
 515
 516 void smc_wr_free_link_mem(struct smc_link *lnk)
 517 {
 518         kfree(lnk->wr_tx_pends);
 519         lnk->wr_tx_pends = NULL;
 520         kfree(lnk->wr_tx_mask);
 521         lnk->wr_tx_mask = NULL;
 522         kfree(lnk->wr_tx_sges);
 523         lnk->wr_tx_sges = NULL;
 524         kfree(lnk->wr_rx_sges);
 525         lnk->wr_rx_sges = NULL;
 526         kfree(lnk->wr_rx_ibs);
 527         lnk->wr_rx_ibs = NULL;
 528         kfree(lnk->wr_tx_ibs);
 529         lnk->wr_tx_ibs = NULL;
 530         kfree(lnk->wr_tx_bufs);
 531         lnk->wr_tx_bufs = NULL;
 532         kfree(lnk->wr_rx_bufs);
 533         lnk->wr_rx_bufs = NULL;
 534 }
 535
 536 int smc_wr_alloc_link_mem(struct smc_link *link)
 537 {
 538         /* allocate link related memory */
 539         link->wr_tx_bufs = kcalloc(SMC_WR_BUF_CNT, SMC_WR_BUF_SIZE, GFP_KERNEL);
 540         if (!link->wr_tx_bufs)
 541                 goto no_mem;
 542         link->wr_rx_bufs = kcalloc(SMC_WR_BUF_CNT * 3, SMC_WR_BUF_SIZE,
 543                                    GFP_KERNEL);
 544         if (!link->wr_rx_bufs)
 545                 goto no_mem_wr_tx_bufs;
 546         link->wr_tx_ibs = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_ibs[0]),
 547                                   GFP_KERNEL);
 548         if (!link->wr_tx_ibs)
 549                 goto no_mem_wr_rx_bufs;
 550         link->wr_rx_ibs = kcalloc(SMC_WR_BUF_CNT * 3,
 551                                   sizeof(link->wr_rx_ibs[0]),
 552                                   GFP_KERNEL);
 553         if (!link->wr_rx_ibs)
 554                 goto no_mem_wr_tx_ibs;
 555         link->wr_tx_sges = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_sges[0]),
 556                                    GFP_KERNEL);
 557         if (!link->wr_tx_sges)
 558                 goto no_mem_wr_rx_ibs;
 559         link->wr_rx_sges = kcalloc(SMC_WR_BUF_CNT * 3,
 560                                    sizeof(link->wr_rx_sges[0]),
 561                                    GFP_KERNEL);
 562         if (!link->wr_rx_sges)
 563                 goto no_mem_wr_tx_sges;
 564         link->wr_tx_mask = kcalloc(BITS_TO_LONGS(SMC_WR_BUF_CNT),
 565                                    sizeof(*link->wr_tx_mask),
 566                                    GFP_KERNEL);
 567         if (!link->wr_tx_mask)
 568                 goto no_mem_wr_rx_sges;
 569         link->wr_tx_pends = kcalloc(SMC_WR_BUF_CNT,
 570                                     sizeof(link->wr_tx_pends[0]),
 571                                     GFP_KERNEL);
 572         if (!link->wr_tx_pends)
 573                 goto no_mem_wr_tx_mask;
 574         return 0;
 575
 576 no_mem_wr_tx_mask:
 577         kfree(link->wr_tx_mask);
 578 no_mem_wr_rx_sges:
 579         kfree(link->wr_rx_sges);
 580 no_mem_wr_tx_sges:
 581         kfree(link->wr_tx_sges);
 582 no_mem_wr_rx_ibs:
 583         kfree(link->wr_rx_ibs);
 584 no_mem_wr_tx_ibs:
 585         kfree(link->wr_tx_ibs);
 586 no_mem_wr_rx_bufs:
 587         kfree(link->wr_rx_bufs);
 588 no_mem_wr_tx_bufs:
 589         kfree(link->wr_tx_bufs);
 590 no_mem:
 591         return -ENOMEM;
 592 }
 593
 594 void smc_wr_remove_dev(struct smc_ib_device *smcibdev)
 595 {
 596         tasklet_kill(&smcibdev->recv_tasklet);
 597         tasklet_kill(&smcibdev->send_tasklet);
 598 }
 599
 600 void smc_wr_add_dev(struct smc_ib_device *smcibdev)
 601 {
 602         tasklet_init(&smcibdev->recv_tasklet, smc_wr_rx_tasklet_fn,
 603                      (unsigned long)smcibdev);
 604         tasklet_init(&smcibdev->send_tasklet, smc_wr_tx_tasklet_fn,
 605                      (unsigned long)smcibdev);
 606 }
 607
 608 int smc_wr_create_link(struct smc_link *lnk)
 609 {
 610         struct ib_device *ibdev = lnk->smcibdev->ibdev;
 611         int rc = 0;
 612
 613         smc_wr_tx_set_wr_id(&lnk->wr_tx_id, 0);
 614         lnk->wr_rx_id = 0;
 615         lnk->wr_rx_dma_addr = ib_dma_map_single(
 616                 ibdev, lnk->wr_rx_bufs, SMC_WR_BUF_SIZE * lnk->wr_rx_cnt,
 617                 DMA_FROM_DEVICE);
 618         if (ib_dma_mapping_error(ibdev, lnk->wr_rx_dma_addr)) {
 619                 lnk->wr_rx_dma_addr = 0;
 620                 rc = -EIO;
 621                 goto out;
 622         }
 623         lnk->wr_tx_dma_addr = ib_dma_map_single(
 624                 ibdev, lnk->wr_tx_bufs, SMC_WR_BUF_SIZE * lnk->wr_tx_cnt,
 625                 DMA_TO_DEVICE);
 626         if (ib_dma_mapping_error(ibdev, lnk->wr_tx_dma_addr)) {
 627                 rc = -EIO;
 628                 goto dma_unmap;
 629         }
 630         smc_wr_init_sge(lnk);
 631         memset(lnk->wr_tx_mask, 0,
 632                BITS_TO_LONGS(SMC_WR_BUF_CNT) * sizeof(*lnk->wr_tx_mask));
 633         init_waitqueue_head(&lnk->wr_tx_wait);
 634         init_waitqueue_head(&lnk->wr_reg_wait);
 635         return rc;
 636
 637 dma_unmap:
 638         ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr,
 639                             SMC_WR_BUF_SIZE * lnk->wr_rx_cnt,
 640                             DMA_FROM_DEVICE);
 641         lnk->wr_rx_dma_addr = 0;
 642 out:
 643         return rc;
 644 }