drivers/infiniband/sw/rdmavt/srq.c

   1 /*
   2  * Copyright(c) 2016 Intel Corporation.
   3  *
   4  * This file is provided under a dual BSD/GPLv2 license.  When using or
   5  * redistributing this file, you may do so under either license.
   6  *
   7  * GPL LICENSE SUMMARY
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of version 2 of the GNU General Public License as
  11  * published by the Free Software Foundation.
  12  *
  13  * This program is distributed in the hope that it will be useful, but
  14  * WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16  * General Public License for more details.
  17  *
  18  * BSD LICENSE
  19  *
  20  * Redistribution and use in source and binary forms, with or without
  21  * modification, are permitted provided that the following conditions
  22  * are met:
  23  *
  24  *  - Redistributions of source code must retain the above copyright
  25  *    notice, this list of conditions and the following disclaimer.
  26  *  - Redistributions in binary form must reproduce the above copyright
  27  *    notice, this list of conditions and the following disclaimer in
  28  *    the documentation and/or other materials provided with the
  29  *    distribution.
  30  *  - Neither the name of Intel Corporation nor the names of its
  31  *    contributors may be used to endorse or promote products derived
  32  *    from this software without specific prior written permission.
  33  *
  34  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  35  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  36  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  37  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  38  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  39  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  40  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  41  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  42  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  43  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  44  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  45  *
  46  */
  47
  48 #include <linux/err.h>
  49 #include <linux/slab.h>
  50 #include <linux/vmalloc.h>
  51 #include <rdma/uverbs_ioctl.h>
  52
  53 #include "srq.h"
  54 #include "vt.h"
  55 #include "qp.h"
  56 /**
  57  * rvt_driver_srq_init - init srq resources on a per driver basis
  58  * @rdi: rvt dev structure
  59  *
  60  * Do any initialization needed when a driver registers with rdmavt.
  61  */
  62 void rvt_driver_srq_init(struct rvt_dev_info *rdi)
  63 {
  64         spin_lock_init(&rdi->n_srqs_lock);
  65         rdi->n_srqs_allocated = 0;
  66 }
  67
  68 /**
  69  * rvt_create_srq - create a shared receive queue
  70  * @ibpd: the protection domain of the SRQ to create
  71  * @srq_init_attr: the attributes of the SRQ
  72  * @udata: data from libibverbs when creating a user SRQ
  73  *
  74  * Return: 0 on success
  75  */
  76 int rvt_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *srq_init_attr,
  77                    struct ib_udata *udata)
  78 {
  79         struct rvt_dev_info *dev = ib_to_rvt(ibsrq->device);
  80         struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq);
  81         u32 sz;
  82         int ret;
  83
  84         if (srq_init_attr->srq_type != IB_SRQT_BASIC)
  85                 return -EOPNOTSUPP;
  86
  87         if (srq_init_attr->attr.max_sge == 0 ||
  88             srq_init_attr->attr.max_sge > dev->dparms.props.max_srq_sge ||
  89             srq_init_attr->attr.max_wr == 0 ||
  90             srq_init_attr->attr.max_wr > dev->dparms.props.max_srq_wr)
  91                 return -EINVAL;
  92
  93         /*
  94          * Need to use vmalloc() if we want to support large #s of entries.
  95          */
  96         srq->rq.size = srq_init_attr->attr.max_wr + 1;
  97         srq->rq.max_sge = srq_init_attr->attr.max_sge;
  98         sz = sizeof(struct ib_sge) * srq->rq.max_sge +
  99                 sizeof(struct rvt_rwqe);
 100         if (rvt_alloc_rq(&srq->rq, srq->rq.size * sz,
 101                          dev->dparms.node, udata)) {
 102                 ret = -ENOMEM;
 103                 goto bail_srq;
 104         }
 105
 106         /*
 107          * Return the address of the RWQ as the offset to mmap.
 108          * See rvt_mmap() for details.
 109          */
 110         if (udata && udata->outlen >= sizeof(__u64)) {
 111                 u32 s = sizeof(struct rvt_rwq) + srq->rq.size * sz;
 112
 113                 srq->ip = rvt_create_mmap_info(dev, s, udata, srq->rq.wq);
 114                 if (IS_ERR(srq->ip)) {
 115                         ret = PTR_ERR(srq->ip);
 116                         goto bail_wq;
 117                 }
 118
 119                 ret = ib_copy_to_udata(udata, &srq->ip->offset,
 120                                        sizeof(srq->ip->offset));
 121                 if (ret)
 122                         goto bail_ip;
 123         }
 124
 125         /*
 126          * ib_create_srq() will initialize srq->ibsrq.
 127          */
 128         spin_lock_init(&srq->rq.lock);
 129         srq->limit = srq_init_attr->attr.srq_limit;
 130
 131         spin_lock(&dev->n_srqs_lock);
 132         if (dev->n_srqs_allocated == dev->dparms.props.max_srq) {
 133                 spin_unlock(&dev->n_srqs_lock);
 134                 ret = -ENOMEM;
 135                 goto bail_ip;
 136         }
 137
 138         dev->n_srqs_allocated++;
 139         spin_unlock(&dev->n_srqs_lock);
 140
 141         if (srq->ip) {
 142                 spin_lock_irq(&dev->pending_lock);
 143                 list_add(&srq->ip->pending_mmaps, &dev->pending_mmaps);
 144                 spin_unlock_irq(&dev->pending_lock);
 145         }
 146
 147         return 0;
 148
 149 bail_ip:
 150         kfree(srq->ip);
 151 bail_wq:
 152         rvt_free_rq(&srq->rq);
 153 bail_srq:
 154         return ret;
 155 }
 156
 157 /**
 158  * rvt_modify_srq - modify a shared receive queue
 159  * @ibsrq: the SRQ to modify
 160  * @attr: the new attributes of the SRQ
 161  * @attr_mask: indicates which attributes to modify
 162  * @udata: user data for libibverbs.so
 163  *
 164  * Return: 0 on success
 165  */
 166 int rvt_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
 167                    enum ib_srq_attr_mask attr_mask,
 168                    struct ib_udata *udata)
 169 {
 170         struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq);
 171         struct rvt_dev_info *dev = ib_to_rvt(ibsrq->device);
 172         struct rvt_rq tmp_rq = {};
 173         int ret = 0;
 174
 175         if (attr_mask & IB_SRQ_MAX_WR) {
 176                 struct rvt_krwq *okwq = NULL;
 177                 struct rvt_rwq *owq = NULL;
 178                 struct rvt_rwqe *p;
 179                 u32 sz, size, n, head, tail;
 180
 181                 /* Check that the requested sizes are below the limits. */
 182                 if ((attr->max_wr > dev->dparms.props.max_srq_wr) ||
 183                     ((attr_mask & IB_SRQ_LIMIT) ?
 184                      attr->srq_limit : srq->limit) > attr->max_wr)
 185                         return -EINVAL;
 186                 sz = sizeof(struct rvt_rwqe) +
 187                         srq->rq.max_sge * sizeof(struct ib_sge);
 188                 size = attr->max_wr + 1;
 189                 if (rvt_alloc_rq(&tmp_rq, size * sz, dev->dparms.node,
 190                                  udata))
 191                         return -ENOMEM;
 192                 /* Check that we can write the offset to mmap. */
 193                 if (udata && udata->inlen >= sizeof(__u64)) {
 194                         __u64 offset_addr;
 195                         __u64 offset = 0;
 196
 197                         ret = ib_copy_from_udata(&offset_addr, udata,
 198                                                  sizeof(offset_addr));
 199                         if (ret)
 200                                 goto bail_free;
 201                         udata->outbuf = (void __user *)
 202                                         (unsigned long)offset_addr;
 203                         ret = ib_copy_to_udata(udata, &offset,
 204                                                sizeof(offset));
 205                         if (ret)
 206                                 goto bail_free;
 207                 }
 208
 209                 spin_lock_irq(&srq->rq.kwq->c_lock);
 210                 /*
 211                  * validate head and tail pointer values and compute
 212                  * the number of remaining WQEs.
 213                  */
 214                 if (udata) {
 215                         owq = srq->rq.wq;
 216                         head = RDMA_READ_UAPI_ATOMIC(owq->head);
 217                         tail = RDMA_READ_UAPI_ATOMIC(owq->tail);
 218                 } else {
 219                         okwq = srq->rq.kwq;
 220                         head = okwq->head;
 221                         tail = okwq->tail;
 222                 }
 223                 if (head >= srq->rq.size || tail >= srq->rq.size) {
 224                         ret = -EINVAL;
 225                         goto bail_unlock;
 226                 }
 227                 n = head;
 228                 if (n < tail)
 229                         n += srq->rq.size - tail;
 230                 else
 231                         n -= tail;
 232                 if (size <= n) {
 233                         ret = -EINVAL;
 234                         goto bail_unlock;
 235                 }
 236                 n = 0;
 237                 p = tmp_rq.kwq->curr_wq;
 238                 while (tail != head) {
 239                         struct rvt_rwqe *wqe;
 240                         int i;
 241
 242                         wqe = rvt_get_rwqe_ptr(&srq->rq, tail);
 243                         p->wr_id = wqe->wr_id;
 244                         p->num_sge = wqe->num_sge;
 245                         for (i = 0; i < wqe->num_sge; i++)
 246                                 p->sg_list[i] = wqe->sg_list[i];
 247                         n++;
 248                         p = (struct rvt_rwqe *)((char *)p + sz);
 249                         if (++tail >= srq->rq.size)
 250                                 tail = 0;
 251                 }
 252                 srq->rq.kwq = tmp_rq.kwq;
 253                 if (udata) {
 254                         srq->rq.wq = tmp_rq.wq;
 255                         RDMA_WRITE_UAPI_ATOMIC(tmp_rq.wq->head, n);
 256                         RDMA_WRITE_UAPI_ATOMIC(tmp_rq.wq->tail, 0);
 257                 } else {
 258                         tmp_rq.kwq->head = n;
 259                         tmp_rq.kwq->tail = 0;
 260                 }
 261                 srq->rq.size = size;
 262                 if (attr_mask & IB_SRQ_LIMIT)
 263                         srq->limit = attr->srq_limit;
 264                 spin_unlock_irq(&srq->rq.kwq->c_lock);
 265
 266                 vfree(owq);
 267                 kvfree(okwq);
 268
 269                 if (srq->ip) {
 270                         struct rvt_mmap_info *ip = srq->ip;
 271                         struct rvt_dev_info *dev = ib_to_rvt(srq->ibsrq.device);
 272                         u32 s = sizeof(struct rvt_rwq) + size * sz;
 273
 274                         rvt_update_mmap_info(dev, ip, s, tmp_rq.wq);
 275
 276                         /*
 277                          * Return the offset to mmap.
 278                          * See rvt_mmap() for details.
 279                          */
 280                         if (udata && udata->inlen >= sizeof(__u64)) {
 281                                 ret = ib_copy_to_udata(udata, &ip->offset,
 282                                                        sizeof(ip->offset));
 283                                 if (ret)
 284                                         return ret;
 285                         }
 286
 287                         /*
 288                          * Put user mapping info onto the pending list
 289                          * unless it already is on the list.
 290                          */
 291                         spin_lock_irq(&dev->pending_lock);
 292                         if (list_empty(&ip->pending_mmaps))
 293                                 list_add(&ip->pending_mmaps,
 294                                          &dev->pending_mmaps);
 295                         spin_unlock_irq(&dev->pending_lock);
 296                 }
 297         } else if (attr_mask & IB_SRQ_LIMIT) {
 298                 spin_lock_irq(&srq->rq.kwq->c_lock);
 299                 if (attr->srq_limit >= srq->rq.size)
 300                         ret = -EINVAL;
 301                 else
 302                         srq->limit = attr->srq_limit;
 303                 spin_unlock_irq(&srq->rq.kwq->c_lock);
 304         }
 305         return ret;
 306
 307 bail_unlock:
 308         spin_unlock_irq(&srq->rq.kwq->c_lock);
 309 bail_free:
 310         rvt_free_rq(&tmp_rq);
 311         return ret;
 312 }
 313
 314 /** rvt_query_srq - query srq data
 315  * @ibsrq: srq to query
 316  * @attr: return info in attr
 317  *
 318  * Return: always 0
 319  */
 320 int rvt_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr)
 321 {
 322         struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq);
 323
 324         attr->max_wr = srq->rq.size - 1;
 325         attr->max_sge = srq->rq.max_sge;
 326         attr->srq_limit = srq->limit;
 327         return 0;
 328 }
 329
 330 /**
 331  * rvt_destroy_srq - destory an srq
 332  * @ibsrq: srq object to destroy
 333  *
 334  */
 335 int rvt_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
 336 {
 337         struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq);
 338         struct rvt_dev_info *dev = ib_to_rvt(ibsrq->device);
 339
 340         spin_lock(&dev->n_srqs_lock);
 341         dev->n_srqs_allocated--;
 342         spin_unlock(&dev->n_srqs_lock);
 343         if (srq->ip)
 344                 kref_put(&srq->ip->ref, rvt_release_mmap_info);
 345         kvfree(srq->rq.kwq);
 346         return 0;
 347 }