drivers/infiniband/sw/rdmavt/rc.c

   1 /*
   2  * Copyright(c) 2016 Intel Corporation.
   3  *
   4  * This file is provided under a dual BSD/GPLv2 license.  When using or
   5  * redistributing this file, you may do so under either license.
   6  *
   7  * GPL LICENSE SUMMARY
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of version 2 of the GNU General Public License as
  11  * published by the Free Software Foundation.
  12  *
  13  * This program is distributed in the hope that it will be useful, but
  14  * WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16  * General Public License for more details.
  17  *
  18  * BSD LICENSE
  19  *
  20  * Redistribution and use in source and binary forms, with or without
  21  * modification, are permitted provided that the following conditions
  22  * are met:
  23  *
  24  *  - Redistributions of source code must retain the above copyright
  25  *    notice, this list of conditions and the following disclaimer.
  26  *  - Redistributions in binary form must reproduce the above copyright
  27  *    notice, this list of conditions and the following disclaimer in
  28  *    the documentation and/or other materials provided with the
  29  *    distribution.
  30  *  - Neither the name of Intel Corporation nor the names of its
  31  *    contributors may be used to endorse or promote products derived
  32  *    from this software without specific prior written permission.
  33  *
  34  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  35  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  36  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  37  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  38  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  39  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  40  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  41  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  42  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  43  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  44  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  45  *
  46  */
  47
  48 #include <rdma/rdmavt_qp.h>
  49 #include <rdma/ib_hdrs.h>
  50
  51 /*
  52  * Convert the AETH credit code into the number of credits.
  53  */
  54 static const u16 credit_table[31] = {
  55         0,                      /* 0 */
  56         1,                      /* 1 */
  57         2,                      /* 2 */
  58         3,                      /* 3 */
  59         4,                      /* 4 */
  60         6,                      /* 5 */
  61         8,                      /* 6 */
  62         12,                     /* 7 */
  63         16,                     /* 8 */
  64         24,                     /* 9 */
  65         32,                     /* A */
  66         48,                     /* B */
  67         64,                     /* C */
  68         96,                     /* D */
  69         128,                    /* E */
  70         192,                    /* F */
  71         256,                    /* 10 */
  72         384,                    /* 11 */
  73         512,                    /* 12 */
  74         768,                    /* 13 */
  75         1024,                   /* 14 */
  76         1536,                   /* 15 */
  77         2048,                   /* 16 */
  78         3072,                   /* 17 */
  79         4096,                   /* 18 */
  80         6144,                   /* 19 */
  81         8192,                   /* 1A */
  82         12288,                  /* 1B */
  83         16384,                  /* 1C */
  84         24576,                  /* 1D */
  85         32768                   /* 1E */
  86 };
  87
  88 /**
  89  * rvt_compute_aeth - compute the AETH (syndrome + MSN)
  90  * @qp: the queue pair to compute the AETH for
  91  *
  92  * Returns the AETH.
  93  */
  94 __be32 rvt_compute_aeth(struct rvt_qp *qp)
  95 {
  96         u32 aeth = qp->r_msn & IB_MSN_MASK;
  97
  98         if (qp->ibqp.srq) {
  99                 /*
 100                  * Shared receive queues don't generate credits.
 101                  * Set the credit field to the invalid value.
 102                  */
 103                 aeth |= IB_AETH_CREDIT_INVAL << IB_AETH_CREDIT_SHIFT;
 104         } else {
 105                 u32 min, max, x;
 106                 u32 credits;
 107                 u32 head;
 108                 u32 tail;
 109
 110                 credits = READ_ONCE(qp->r_rq.kwq->count);
 111                 if (credits == 0) {
 112                         /* sanity check pointers before trusting them */
 113                         if (qp->ip) {
 114                                 head = RDMA_READ_UAPI_ATOMIC(qp->r_rq.wq->head);
 115                                 tail = RDMA_READ_UAPI_ATOMIC(qp->r_rq.wq->tail);
 116                         } else {
 117                                 head = READ_ONCE(qp->r_rq.kwq->head);
 118                                 tail = READ_ONCE(qp->r_rq.kwq->tail);
 119                         }
 120                         if (head >= qp->r_rq.size)
 121                                 head = 0;
 122                         if (tail >= qp->r_rq.size)
 123                                 tail = 0;
 124                         /*
 125                          * Compute the number of credits available (RWQEs).
 126                          * There is a small chance that the pair of reads are
 127                          * not atomic, which is OK, since the fuzziness is
 128                          * resolved as further ACKs go out.
 129                          */
 130                         credits = head - tail;
 131                         if ((int)credits < 0)
 132                                 credits += qp->r_rq.size;
 133                 }
 134                 /*
 135                  * Binary search the credit table to find the code to
 136                  * use.
 137                  */
 138                 min = 0;
 139                 max = 31;
 140                 for (;;) {
 141                         x = (min + max) / 2;
 142                         if (credit_table[x] == credits)
 143                                 break;
 144                         if (credit_table[x] > credits) {
 145                                 max = x;
 146                         } else {
 147                                 if (min == x)
 148                                         break;
 149                                 min = x;
 150                         }
 151                 }
 152                 aeth |= x << IB_AETH_CREDIT_SHIFT;
 153         }
 154         return cpu_to_be32(aeth);
 155 }
 156 EXPORT_SYMBOL(rvt_compute_aeth);
 157
 158 /**
 159  * rvt_get_credit - flush the send work queue of a QP
 160  * @qp: the qp who's send work queue to flush
 161  * @aeth: the Acknowledge Extended Transport Header
 162  *
 163  * The QP s_lock should be held.
 164  */
 165 void rvt_get_credit(struct rvt_qp *qp, u32 aeth)
 166 {
 167         struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device);
 168         u32 credit = (aeth >> IB_AETH_CREDIT_SHIFT) & IB_AETH_CREDIT_MASK;
 169
 170         lockdep_assert_held(&qp->s_lock);
 171         /*
 172          * If the credit is invalid, we can send
 173          * as many packets as we like.  Otherwise, we have to
 174          * honor the credit field.
 175          */
 176         if (credit == IB_AETH_CREDIT_INVAL) {
 177                 if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) {
 178                         qp->s_flags |= RVT_S_UNLIMITED_CREDIT;
 179                         if (qp->s_flags & RVT_S_WAIT_SSN_CREDIT) {
 180                                 qp->s_flags &= ~RVT_S_WAIT_SSN_CREDIT;
 181                                 rdi->driver_f.schedule_send(qp);
 182                         }
 183                 }
 184         } else if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) {
 185                 /* Compute new LSN (i.e., MSN + credit) */
 186                 credit = (aeth + credit_table[credit]) & IB_MSN_MASK;
 187                 if (rvt_cmp_msn(credit, qp->s_lsn) > 0) {
 188                         qp->s_lsn = credit;
 189                         if (qp->s_flags & RVT_S_WAIT_SSN_CREDIT) {
 190                                 qp->s_flags &= ~RVT_S_WAIT_SSN_CREDIT;
 191                                 rdi->driver_f.schedule_send(qp);
 192                         }
 193                 }
 194         }
 195 }
 196 EXPORT_SYMBOL(rvt_get_credit);
 197
 198 /* rvt_restart_sge - rewind the sge state for a wqe */
 199 u32 rvt_restart_sge(struct rvt_sge_state *ss, struct rvt_swqe *wqe, u32 len)
 200 {
 201         ss->sge = wqe->sg_list[0];
 202         ss->sg_list = wqe->sg_list + 1;
 203         ss->num_sge = wqe->wr.num_sge;
 204         ss->total_len = wqe->length;
 205         rvt_skip_sge(ss, len, false);
 206         return wqe->length - len;
 207 }
 208 EXPORT_SYMBOL(rvt_restart_sge);
 209