drivers/misc/mic/scif/scif_fence.c

   1 /*
   2  * Intel MIC Platform Software Stack (MPSS)
   3  *
   4  * Copyright(c) 2015 Intel Corporation.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License, version 2, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  13  * General Public License for more details.
  14  *
  15  * Intel SCIF driver.
  16  *
  17  */
  18
  19 #include "scif_main.h"
  20
  21 /**
  22  * scif_recv_mark: Handle SCIF_MARK request
  23  * @msg:        Interrupt message
  24  *
  25  * The peer has requested a mark.
  26  */
  27 void scif_recv_mark(struct scif_dev *scifdev, struct scifmsg *msg)
  28 {
  29         struct scif_endpt *ep = (struct scif_endpt *)msg->payload[0];
  30         int mark = 0;
  31         int err;
  32
  33         err = _scif_fence_mark(ep, &mark);
  34         if (err)
  35                 msg->uop = SCIF_MARK_NACK;
  36         else
  37                 msg->uop = SCIF_MARK_ACK;
  38         msg->payload[0] = ep->remote_ep;
  39         msg->payload[2] = mark;
  40         scif_nodeqp_send(ep->remote_dev, msg);
  41 }
  42
  43 /**
  44  * scif_recv_mark_resp: Handle SCIF_MARK_(N)ACK messages.
  45  * @msg:        Interrupt message
  46  *
  47  * The peer has responded to a SCIF_MARK message.
  48  */
  49 void scif_recv_mark_resp(struct scif_dev *scifdev, struct scifmsg *msg)
  50 {
  51         struct scif_endpt *ep = (struct scif_endpt *)msg->payload[0];
  52         struct scif_fence_info *fence_req =
  53                 (struct scif_fence_info *)msg->payload[1];
  54
  55         mutex_lock(&ep->rma_info.rma_lock);
  56         if (msg->uop == SCIF_MARK_ACK) {
  57                 fence_req->state = OP_COMPLETED;
  58                 fence_req->dma_mark = (int)msg->payload[2];
  59         } else {
  60                 fence_req->state = OP_FAILED;
  61         }
  62         mutex_unlock(&ep->rma_info.rma_lock);
  63         complete(&fence_req->comp);
  64 }
  65
  66 /**
  67  * scif_recv_wait: Handle SCIF_WAIT request
  68  * @msg:        Interrupt message
  69  *
  70  * The peer has requested waiting on a fence.
  71  */
  72 void scif_recv_wait(struct scif_dev *scifdev, struct scifmsg *msg)
  73 {
  74         struct scif_endpt *ep = (struct scif_endpt *)msg->payload[0];
  75         struct scif_remote_fence_info *fence;
  76
  77         /*
  78          * Allocate structure for remote fence information and
  79          * send a NACK if the allocation failed. The peer will
  80          * return ENOMEM upon receiving a NACK.
  81          */
  82         fence = kmalloc(sizeof(*fence), GFP_KERNEL);
  83         if (!fence) {
  84                 msg->payload[0] = ep->remote_ep;
  85                 msg->uop = SCIF_WAIT_NACK;
  86                 scif_nodeqp_send(ep->remote_dev, msg);
  87                 return;
  88         }
  89
  90         /* Prepare the fence request */
  91         memcpy(&fence->msg, msg, sizeof(struct scifmsg));
  92         INIT_LIST_HEAD(&fence->list);
  93
  94         /* Insert to the global remote fence request list */
  95         mutex_lock(&scif_info.fencelock);
  96         atomic_inc(&ep->rma_info.fence_refcount);
  97         list_add_tail(&fence->list, &scif_info.fence);
  98         mutex_unlock(&scif_info.fencelock);
  99
 100         schedule_work(&scif_info.misc_work);
 101 }
 102
 103 /**
 104  * scif_recv_wait_resp: Handle SCIF_WAIT_(N)ACK messages.
 105  * @msg:        Interrupt message
 106  *
 107  * The peer has responded to a SCIF_WAIT message.
 108  */
 109 void scif_recv_wait_resp(struct scif_dev *scifdev, struct scifmsg *msg)
 110 {
 111         struct scif_endpt *ep = (struct scif_endpt *)msg->payload[0];
 112         struct scif_fence_info *fence_req =
 113                 (struct scif_fence_info *)msg->payload[1];
 114
 115         mutex_lock(&ep->rma_info.rma_lock);
 116         if (msg->uop == SCIF_WAIT_ACK)
 117                 fence_req->state = OP_COMPLETED;
 118         else
 119                 fence_req->state = OP_FAILED;
 120         mutex_unlock(&ep->rma_info.rma_lock);
 121         complete(&fence_req->comp);
 122 }
 123
 124 /**
 125  * scif_recv_sig_local: Handle SCIF_SIG_LOCAL request
 126  * @msg:        Interrupt message
 127  *
 128  * The peer has requested a signal on a local offset.
 129  */
 130 void scif_recv_sig_local(struct scif_dev *scifdev, struct scifmsg *msg)
 131 {
 132         struct scif_endpt *ep = (struct scif_endpt *)msg->payload[0];
 133         int err;
 134
 135         err = scif_prog_signal(ep, msg->payload[1], msg->payload[2],
 136                                SCIF_WINDOW_SELF);
 137         if (err)
 138                 msg->uop = SCIF_SIG_NACK;
 139         else
 140                 msg->uop = SCIF_SIG_ACK;
 141         msg->payload[0] = ep->remote_ep;
 142         scif_nodeqp_send(ep->remote_dev, msg);
 143 }
 144
 145 /**
 146  * scif_recv_sig_remote: Handle SCIF_SIGNAL_REMOTE request
 147  * @msg:        Interrupt message
 148  *
 149  * The peer has requested a signal on a remote offset.
 150  */
 151 void scif_recv_sig_remote(struct scif_dev *scifdev, struct scifmsg *msg)
 152 {
 153         struct scif_endpt *ep = (struct scif_endpt *)msg->payload[0];
 154         int err;
 155
 156         err = scif_prog_signal(ep, msg->payload[1], msg->payload[2],
 157                                SCIF_WINDOW_PEER);
 158         if (err)
 159                 msg->uop = SCIF_SIG_NACK;
 160         else
 161                 msg->uop = SCIF_SIG_ACK;
 162         msg->payload[0] = ep->remote_ep;
 163         scif_nodeqp_send(ep->remote_dev, msg);
 164 }
 165
 166 /**
 167  * scif_recv_sig_resp: Handle SCIF_SIG_(N)ACK messages.
 168  * @msg:        Interrupt message
 169  *
 170  * The peer has responded to a signal request.
 171  */
 172 void scif_recv_sig_resp(struct scif_dev *scifdev, struct scifmsg *msg)
 173 {
 174         struct scif_endpt *ep = (struct scif_endpt *)msg->payload[0];
 175         struct scif_fence_info *fence_req =
 176                 (struct scif_fence_info *)msg->payload[3];
 177
 178         mutex_lock(&ep->rma_info.rma_lock);
 179         if (msg->uop == SCIF_SIG_ACK)
 180                 fence_req->state = OP_COMPLETED;
 181         else
 182                 fence_req->state = OP_FAILED;
 183         mutex_unlock(&ep->rma_info.rma_lock);
 184         complete(&fence_req->comp);
 185 }
 186
 187 static inline void *scif_get_local_va(off_t off, struct scif_window *window)
 188 {
 189         struct page **pages = window->pinned_pages->pages;
 190         int page_nr = (off - window->offset) >> PAGE_SHIFT;
 191         off_t page_off = off & ~PAGE_MASK;
 192
 193         return page_address(pages[page_nr]) + page_off;
 194 }
 195
 196 static void scif_prog_signal_cb(void *arg)
 197 {
 198         struct scif_cb_arg *cb_arg = arg;
 199
 200         dma_pool_free(cb_arg->ep->remote_dev->signal_pool, cb_arg->status,
 201                       cb_arg->src_dma_addr);
 202         kfree(cb_arg);
 203 }
 204
 205 static int _scif_prog_signal(scif_epd_t epd, dma_addr_t dst, u64 val)
 206 {
 207         struct scif_endpt *ep = (struct scif_endpt *)epd;
 208         struct dma_chan *chan = ep->rma_info.dma_chan;
 209         struct dma_device *ddev = chan->device;
 210         bool x100 = !is_dma_copy_aligned(chan->device, 1, 1, 1);
 211         struct dma_async_tx_descriptor *tx;
 212         struct scif_status *status = NULL;
 213         struct scif_cb_arg *cb_arg = NULL;
 214         dma_addr_t src;
 215         dma_cookie_t cookie;
 216         int err;
 217
 218         tx = ddev->device_prep_dma_memcpy(chan, 0, 0, 0, DMA_PREP_FENCE);
 219         if (!tx) {
 220                 err = -ENOMEM;
 221                 dev_err(&ep->remote_dev->sdev->dev, "%s %d err %d\n",
 222                         __func__, __LINE__, err);
 223                 goto alloc_fail;
 224         }
 225         cookie = tx->tx_submit(tx);
 226         if (dma_submit_error(cookie)) {
 227                 err = (int)cookie;
 228                 dev_err(&ep->remote_dev->sdev->dev, "%s %d err %d\n",
 229                         __func__, __LINE__, err);
 230                 goto alloc_fail;
 231         }
 232         dma_async_issue_pending(chan);
 233         if (x100) {
 234                 /*
 235                  * For X100 use the status descriptor to write the value to
 236                  * the destination.
 237                  */
 238                 tx = ddev->device_prep_dma_imm_data(chan, dst, val, 0);
 239         } else {
 240                 status = dma_pool_alloc(ep->remote_dev->signal_pool, GFP_KERNEL,
 241                                         &src);
 242                 if (!status) {
 243                         err = -ENOMEM;
 244                         dev_err(&ep->remote_dev->sdev->dev, "%s %d err %d\n",
 245                                 __func__, __LINE__, err);
 246                         goto alloc_fail;
 247                 }
 248                 status->val = val;
 249                 status->src_dma_addr = src;
 250                 status->ep = ep;
 251                 src += offsetof(struct scif_status, val);
 252                 tx = ddev->device_prep_dma_memcpy(chan, dst, src, sizeof(val),
 253                                                   DMA_PREP_INTERRUPT);
 254         }
 255         if (!tx) {
 256                 err = -ENOMEM;
 257                 dev_err(&ep->remote_dev->sdev->dev, "%s %d err %d\n",
 258                         __func__, __LINE__, err);
 259                 goto dma_fail;
 260         }
 261         if (!x100) {
 262                 cb_arg = kmalloc(sizeof(*cb_arg), GFP_KERNEL);
 263                 if (!cb_arg) {
 264                         err = -ENOMEM;
 265                         goto dma_fail;
 266                 }
 267                 cb_arg->src_dma_addr = src;
 268                 cb_arg->status = status;
 269                 cb_arg->ep = ep;
 270                 tx->callback = scif_prog_signal_cb;
 271                 tx->callback_param = cb_arg;
 272         }
 273         cookie = tx->tx_submit(tx);
 274         if (dma_submit_error(cookie)) {
 275                 err = -EIO;
 276                 dev_err(&ep->remote_dev->sdev->dev, "%s %d err %d\n",
 277                         __func__, __LINE__, err);
 278                 goto dma_fail;
 279         }
 280         dma_async_issue_pending(chan);
 281         return 0;
 282 dma_fail:
 283         if (!x100) {
 284                 dma_pool_free(ep->remote_dev->signal_pool, status,
 285                               src - offsetof(struct scif_status, val));
 286                 kfree(cb_arg);
 287         }
 288 alloc_fail:
 289         return err;
 290 }
 291
 292 /*
 293  * scif_prog_signal:
 294  * @epd - Endpoint Descriptor
 295  * @offset - registered address to write @val to
 296  * @val - Value to be written at @offset
 297  * @type - Type of the window.
 298  *
 299  * Arrange to write a value to the registered offset after ensuring that the
 300  * offset provided is indeed valid.
 301  */
 302 int scif_prog_signal(scif_epd_t epd, off_t offset, u64 val,
 303                      enum scif_window_type type)
 304 {
 305         struct scif_endpt *ep = (struct scif_endpt *)epd;
 306         struct scif_window *window = NULL;
 307         struct scif_rma_req req;
 308         dma_addr_t dst_dma_addr;
 309         int err;
 310
 311         mutex_lock(&ep->rma_info.rma_lock);
 312         req.out_window = &window;
 313         req.offset = offset;
 314         req.nr_bytes = sizeof(u64);
 315         req.prot = SCIF_PROT_WRITE;
 316         req.type = SCIF_WINDOW_SINGLE;
 317         if (type == SCIF_WINDOW_SELF)
 318                 req.head = &ep->rma_info.reg_list;
 319         else
 320                 req.head = &ep->rma_info.remote_reg_list;
 321         /* Does a valid window exist? */
 322         err = scif_query_window(&req);
 323         if (err) {
 324                 dev_err(scif_info.mdev.this_device,
 325                         "%s %d err %d\n", __func__, __LINE__, err);
 326                 goto unlock_ret;
 327         }
 328
 329         if (scif_is_mgmt_node() && scifdev_self(ep->remote_dev)) {
 330                 u64 *dst_virt;
 331
 332                 if (type == SCIF_WINDOW_SELF)
 333                         dst_virt = scif_get_local_va(offset, window);
 334                 else
 335                         dst_virt =
 336                         scif_get_local_va(offset, (struct scif_window *)
 337                                           window->peer_window);
 338                 *dst_virt = val;
 339         } else {
 340                 dst_dma_addr = __scif_off_to_dma_addr(window, offset);
 341                 err = _scif_prog_signal(epd, dst_dma_addr, val);
 342         }
 343 unlock_ret:
 344         mutex_unlock(&ep->rma_info.rma_lock);
 345         return err;
 346 }
 347
 348 static int _scif_fence_wait(scif_epd_t epd, int mark)
 349 {
 350         struct scif_endpt *ep = (struct scif_endpt *)epd;
 351         dma_cookie_t cookie = mark & ~SCIF_REMOTE_FENCE;
 352         int err;
 353
 354         /* Wait for DMA callback in scif_fence_mark_cb(..) */
 355         err = wait_event_interruptible_timeout(ep->rma_info.markwq,
 356                                                dma_async_is_tx_complete(
 357                                                ep->rma_info.dma_chan,
 358                                                cookie, NULL, NULL) ==
 359                                                DMA_COMPLETE,
 360                                                SCIF_NODE_ALIVE_TIMEOUT);
 361         if (!err)
 362                 err = -ETIMEDOUT;
 363         else if (err > 0)
 364                 err = 0;
 365         return err;
 366 }
 367
 368 /**
 369  * scif_rma_handle_remote_fences:
 370  *
 371  * This routine services remote fence requests.
 372  */
 373 void scif_rma_handle_remote_fences(void)
 374 {
 375         struct list_head *item, *tmp;
 376         struct scif_remote_fence_info *fence;
 377         struct scif_endpt *ep;
 378         int mark, err;
 379
 380         might_sleep();
 381         mutex_lock(&scif_info.fencelock);
 382         list_for_each_safe(item, tmp, &scif_info.fence) {
 383                 fence = list_entry(item, struct scif_remote_fence_info,
 384                                    list);
 385                 /* Remove fence from global list */
 386                 list_del(&fence->list);
 387
 388                 /* Initiate the fence operation */
 389                 ep = (struct scif_endpt *)fence->msg.payload[0];
 390                 mark = fence->msg.payload[2];
 391                 err = _scif_fence_wait(ep, mark);
 392                 if (err)
 393                         fence->msg.uop = SCIF_WAIT_NACK;
 394                 else
 395                         fence->msg.uop = SCIF_WAIT_ACK;
 396                 fence->msg.payload[0] = ep->remote_ep;
 397                 scif_nodeqp_send(ep->remote_dev, &fence->msg);
 398                 kfree(fence);
 399                 if (!atomic_sub_return(1, &ep->rma_info.fence_refcount))
 400                         schedule_work(&scif_info.misc_work);
 401         }
 402         mutex_unlock(&scif_info.fencelock);
 403 }
 404
 405 static int _scif_send_fence(scif_epd_t epd, int uop, int mark, int *out_mark)
 406 {
 407         int err;
 408         struct scifmsg msg;
 409         struct scif_fence_info *fence_req;
 410         struct scif_endpt *ep = (struct scif_endpt *)epd;
 411
 412         fence_req = kmalloc(sizeof(*fence_req), GFP_KERNEL);
 413         if (!fence_req) {
 414                 err = -ENOMEM;
 415                 goto error;
 416         }
 417
 418         fence_req->state = OP_IN_PROGRESS;
 419         init_completion(&fence_req->comp);
 420
 421         msg.src = ep->port;
 422         msg.uop = uop;
 423         msg.payload[0] = ep->remote_ep;
 424         msg.payload[1] = (u64)fence_req;
 425         if (uop == SCIF_WAIT)
 426                 msg.payload[2] = mark;
 427         spin_lock(&ep->lock);
 428         if (ep->state == SCIFEP_CONNECTED)
 429                 err = scif_nodeqp_send(ep->remote_dev, &msg);
 430         else
 431                 err = -ENOTCONN;
 432         spin_unlock(&ep->lock);
 433         if (err)
 434                 goto error_free;
 435 retry:
 436         /* Wait for a SCIF_WAIT_(N)ACK message */
 437         err = wait_for_completion_timeout(&fence_req->comp,
 438                                           SCIF_NODE_ALIVE_TIMEOUT);
 439         if (!err && scifdev_alive(ep))
 440                 goto retry;
 441         if (!err)
 442                 err = -ENODEV;
 443         if (err > 0)
 444                 err = 0;
 445         mutex_lock(&ep->rma_info.rma_lock);
 446         if (err < 0) {
 447                 if (fence_req->state == OP_IN_PROGRESS)
 448                         fence_req->state = OP_FAILED;
 449         }
 450         if (fence_req->state == OP_FAILED && !err)
 451                 err = -ENOMEM;
 452         if (uop == SCIF_MARK && fence_req->state == OP_COMPLETED)
 453                 *out_mark = SCIF_REMOTE_FENCE | fence_req->dma_mark;
 454         mutex_unlock(&ep->rma_info.rma_lock);
 455 error_free:
 456         kfree(fence_req);
 457 error:
 458         return err;
 459 }
 460
 461 /**
 462  * scif_send_fence_mark:
 463  * @epd: end point descriptor.
 464  * @out_mark: Output DMA mark reported by peer.
 465  *
 466  * Send a remote fence mark request.
 467  */
 468 static int scif_send_fence_mark(scif_epd_t epd, int *out_mark)
 469 {
 470         return _scif_send_fence(epd, SCIF_MARK, 0, out_mark);
 471 }
 472
 473 /**
 474  * scif_send_fence_wait:
 475  * @epd: end point descriptor.
 476  * @mark: DMA mark to wait for.
 477  *
 478  * Send a remote fence wait request.
 479  */
 480 static int scif_send_fence_wait(scif_epd_t epd, int mark)
 481 {
 482         return _scif_send_fence(epd, SCIF_WAIT, mark, NULL);
 483 }
 484
 485 static int _scif_send_fence_signal_wait(struct scif_endpt *ep,
 486                                         struct scif_fence_info *fence_req)
 487 {
 488         int err;
 489
 490 retry:
 491         /* Wait for a SCIF_SIG_(N)ACK message */
 492         err = wait_for_completion_timeout(&fence_req->comp,
 493                                           SCIF_NODE_ALIVE_TIMEOUT);
 494         if (!err && scifdev_alive(ep))
 495                 goto retry;
 496         if (!err)
 497                 err = -ENODEV;
 498         if (err > 0)
 499                 err = 0;
 500         if (err < 0) {
 501                 mutex_lock(&ep->rma_info.rma_lock);
 502                 if (fence_req->state == OP_IN_PROGRESS)
 503                         fence_req->state = OP_FAILED;
 504                 mutex_unlock(&ep->rma_info.rma_lock);
 505         }
 506         if (fence_req->state == OP_FAILED && !err)
 507                 err = -ENXIO;
 508         return err;
 509 }
 510
 511 /**
 512  * scif_send_fence_signal:
 513  * @epd - endpoint descriptor
 514  * @loff - local offset
 515  * @lval - local value to write to loffset
 516  * @roff - remote offset
 517  * @rval - remote value to write to roffset
 518  * @flags - flags
 519  *
 520  * Sends a remote fence signal request
 521  */
 522 static int scif_send_fence_signal(scif_epd_t epd, off_t roff, u64 rval,
 523                                   off_t loff, u64 lval, int flags)
 524 {
 525         int err = 0;
 526         struct scifmsg msg;
 527         struct scif_fence_info *fence_req;
 528         struct scif_endpt *ep = (struct scif_endpt *)epd;
 529
 530         fence_req = kmalloc(sizeof(*fence_req), GFP_KERNEL);
 531         if (!fence_req) {
 532                 err = -ENOMEM;
 533                 goto error;
 534         }
 535
 536         fence_req->state = OP_IN_PROGRESS;
 537         init_completion(&fence_req->comp);
 538         msg.src = ep->port;
 539         if (flags & SCIF_SIGNAL_LOCAL) {
 540                 msg.uop = SCIF_SIG_LOCAL;
 541                 msg.payload[0] = ep->remote_ep;
 542                 msg.payload[1] = roff;
 543                 msg.payload[2] = rval;
 544                 msg.payload[3] = (u64)fence_req;
 545                 spin_lock(&ep->lock);
 546                 if (ep->state == SCIFEP_CONNECTED)
 547                         err = scif_nodeqp_send(ep->remote_dev, &msg);
 548                 else
 549                         err = -ENOTCONN;
 550                 spin_unlock(&ep->lock);
 551                 if (err)
 552                         goto error_free;
 553                 err = _scif_send_fence_signal_wait(ep, fence_req);
 554                 if (err)
 555                         goto error_free;
 556         }
 557         fence_req->state = OP_IN_PROGRESS;
 558
 559         if (flags & SCIF_SIGNAL_REMOTE) {
 560                 msg.uop = SCIF_SIG_REMOTE;
 561                 msg.payload[0] = ep->remote_ep;
 562                 msg.payload[1] = loff;
 563                 msg.payload[2] = lval;
 564                 msg.payload[3] = (u64)fence_req;
 565                 spin_lock(&ep->lock);
 566                 if (ep->state == SCIFEP_CONNECTED)
 567                         err = scif_nodeqp_send(ep->remote_dev, &msg);
 568                 else
 569                         err = -ENOTCONN;
 570                 spin_unlock(&ep->lock);
 571                 if (err)
 572                         goto error_free;
 573                 err = _scif_send_fence_signal_wait(ep, fence_req);
 574         }
 575 error_free:
 576         kfree(fence_req);
 577 error:
 578         return err;
 579 }
 580
 581 static void scif_fence_mark_cb(void *arg)
 582 {
 583         struct scif_endpt *ep = (struct scif_endpt *)arg;
 584
 585         wake_up_interruptible(&ep->rma_info.markwq);
 586         atomic_dec(&ep->rma_info.fence_refcount);
 587 }
 588
 589 /*
 590  * _scif_fence_mark:
 591  *
 592  * @epd - endpoint descriptor
 593  * Set up a mark for this endpoint and return the value of the mark.
 594  */
 595 int _scif_fence_mark(scif_epd_t epd, int *mark)
 596 {
 597         struct scif_endpt *ep = (struct scif_endpt *)epd;
 598         struct dma_chan *chan = ep->rma_info.dma_chan;
 599         struct dma_device *ddev = chan->device;
 600         struct dma_async_tx_descriptor *tx;
 601         dma_cookie_t cookie;
 602         int err;
 603
 604         tx = ddev->device_prep_dma_memcpy(chan, 0, 0, 0, DMA_PREP_FENCE);
 605         if (!tx) {
 606                 err = -ENOMEM;
 607                 dev_err(&ep->remote_dev->sdev->dev, "%s %d err %d\n",
 608                         __func__, __LINE__, err);
 609                 return err;
 610         }
 611         cookie = tx->tx_submit(tx);
 612         if (dma_submit_error(cookie)) {
 613                 err = (int)cookie;
 614                 dev_err(&ep->remote_dev->sdev->dev, "%s %d err %d\n",
 615                         __func__, __LINE__, err);
 616                 return err;
 617         }
 618         dma_async_issue_pending(chan);
 619         tx = ddev->device_prep_dma_interrupt(chan, DMA_PREP_INTERRUPT);
 620         if (!tx) {
 621                 err = -ENOMEM;
 622                 dev_err(&ep->remote_dev->sdev->dev, "%s %d err %d\n",
 623                         __func__, __LINE__, err);
 624                 return err;
 625         }
 626         tx->callback = scif_fence_mark_cb;
 627         tx->callback_param = ep;
 628         *mark = cookie = tx->tx_submit(tx);
 629         if (dma_submit_error(cookie)) {
 630                 err = (int)cookie;
 631                 dev_err(&ep->remote_dev->sdev->dev, "%s %d err %d\n",
 632                         __func__, __LINE__, err);
 633                 return err;
 634         }
 635         atomic_inc(&ep->rma_info.fence_refcount);
 636         dma_async_issue_pending(chan);
 637         return 0;
 638 }
 639
 640 #define SCIF_LOOPB_MAGIC_MARK 0xdead
 641
 642 int scif_fence_mark(scif_epd_t epd, int flags, int *mark)
 643 {
 644         struct scif_endpt *ep = (struct scif_endpt *)epd;
 645         int err = 0;
 646
 647         dev_dbg(scif_info.mdev.this_device,
 648                 "SCIFAPI fence_mark: ep %p flags 0x%x mark 0x%x\n",
 649                 ep, flags, *mark);
 650         err = scif_verify_epd(ep);
 651         if (err)
 652                 return err;
 653
 654         /* Invalid flags? */
 655         if (flags & ~(SCIF_FENCE_INIT_SELF | SCIF_FENCE_INIT_PEER))
 656                 return -EINVAL;
 657
 658         /* At least one of init self or peer RMA should be set */
 659         if (!(flags & (SCIF_FENCE_INIT_SELF | SCIF_FENCE_INIT_PEER)))
 660                 return -EINVAL;
 661
 662         /* Exactly one of init self or peer RMA should be set but not both */
 663         if ((flags & SCIF_FENCE_INIT_SELF) && (flags & SCIF_FENCE_INIT_PEER))
 664                 return -EINVAL;
 665
 666         /*
 667          * Management node loopback does not need to use DMA.
 668          * Return a valid mark to be symmetric.
 669          */
 670         if (scifdev_self(ep->remote_dev) && scif_is_mgmt_node()) {
 671                 *mark = SCIF_LOOPB_MAGIC_MARK;
 672                 return 0;
 673         }
 674
 675         if (flags & SCIF_FENCE_INIT_SELF)
 676                 err = _scif_fence_mark(epd, mark);
 677         else
 678                 err = scif_send_fence_mark(ep, mark);
 679
 680         if (err)
 681                 dev_err(scif_info.mdev.this_device,
 682                         "%s %d err %d\n", __func__, __LINE__, err);
 683         dev_dbg(scif_info.mdev.this_device,
 684                 "SCIFAPI fence_mark: ep %p flags 0x%x mark 0x%x err %d\n",
 685                 ep, flags, *mark, err);
 686         return err;
 687 }
 688 EXPORT_SYMBOL_GPL(scif_fence_mark);
 689
 690 int scif_fence_wait(scif_epd_t epd, int mark)
 691 {
 692         struct scif_endpt *ep = (struct scif_endpt *)epd;
 693         int err = 0;
 694
 695         dev_dbg(scif_info.mdev.this_device,
 696                 "SCIFAPI fence_wait: ep %p mark 0x%x\n",
 697                 ep, mark);
 698         err = scif_verify_epd(ep);
 699         if (err)
 700                 return err;
 701         /*
 702          * Management node loopback does not need to use DMA.
 703          * The only valid mark provided is 0 so simply
 704          * return success if the mark is valid.
 705          */
 706         if (scifdev_self(ep->remote_dev) && scif_is_mgmt_node()) {
 707                 if (mark == SCIF_LOOPB_MAGIC_MARK)
 708                         return 0;
 709                 else
 710                         return -EINVAL;
 711         }
 712         if (mark & SCIF_REMOTE_FENCE)
 713                 err = scif_send_fence_wait(epd, mark);
 714         else
 715                 err = _scif_fence_wait(epd, mark);
 716         if (err < 0)
 717                 dev_err(scif_info.mdev.this_device,
 718                         "%s %d err %d\n", __func__, __LINE__, err);
 719         return err;
 720 }
 721 EXPORT_SYMBOL_GPL(scif_fence_wait);
 722
 723 int scif_fence_signal(scif_epd_t epd, off_t loff, u64 lval,
 724                       off_t roff, u64 rval, int flags)
 725 {
 726         struct scif_endpt *ep = (struct scif_endpt *)epd;
 727         int err = 0;
 728
 729         dev_dbg(scif_info.mdev.this_device,
 730                 "SCIFAPI fence_signal: ep %p loff 0x%lx lval 0x%llx roff 0x%lx rval 0x%llx flags 0x%x\n",
 731                 ep, loff, lval, roff, rval, flags);
 732         err = scif_verify_epd(ep);
 733         if (err)
 734                 return err;
 735
 736         /* Invalid flags? */
 737         if (flags & ~(SCIF_FENCE_INIT_SELF | SCIF_FENCE_INIT_PEER |
 738                         SCIF_SIGNAL_LOCAL | SCIF_SIGNAL_REMOTE))
 739                 return -EINVAL;
 740
 741         /* At least one of init self or peer RMA should be set */
 742         if (!(flags & (SCIF_FENCE_INIT_SELF | SCIF_FENCE_INIT_PEER)))
 743                 return -EINVAL;
 744
 745         /* Exactly one of init self or peer RMA should be set but not both */
 746         if ((flags & SCIF_FENCE_INIT_SELF) && (flags & SCIF_FENCE_INIT_PEER))
 747                 return -EINVAL;
 748
 749         /* At least one of SCIF_SIGNAL_LOCAL or SCIF_SIGNAL_REMOTE required */
 750         if (!(flags & (SCIF_SIGNAL_LOCAL | SCIF_SIGNAL_REMOTE)))
 751                 return -EINVAL;
 752
 753         /* Only Dword offsets allowed */
 754         if ((flags & SCIF_SIGNAL_LOCAL) && (loff & (sizeof(u32) - 1)))
 755                 return -EINVAL;
 756
 757         /* Only Dword aligned offsets allowed */
 758         if ((flags & SCIF_SIGNAL_REMOTE) && (roff & (sizeof(u32) - 1)))
 759                 return -EINVAL;
 760
 761         if (flags & SCIF_FENCE_INIT_PEER) {
 762                 err = scif_send_fence_signal(epd, roff, rval, loff,
 763                                              lval, flags);
 764         } else {
 765                 /* Local Signal in Local RAS */
 766                 if (flags & SCIF_SIGNAL_LOCAL) {
 767                         err = scif_prog_signal(epd, loff, lval,
 768                                                SCIF_WINDOW_SELF);
 769                         if (err)
 770                                 goto error_ret;
 771                 }
 772
 773                 /* Signal in Remote RAS */
 774                 if (flags & SCIF_SIGNAL_REMOTE)
 775                         err = scif_prog_signal(epd, roff,
 776                                                rval, SCIF_WINDOW_PEER);
 777         }
 778 error_ret:
 779         if (err)
 780                 dev_err(scif_info.mdev.this_device,
 781                         "%s %d err %d\n", __func__, __LINE__, err);
 782         return err;
 783 }
 784 EXPORT_SYMBOL_GPL(scif_fence_signal);