2 * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 #include <linux/mlx4/qp.h>
35 #include <linux/mlx4/srq.h>
36 #include <linux/slab.h>
41 static void *get_wqe(struct mlx4_ib_srq
*srq
, int n
)
43 return mlx4_buf_offset(&srq
->buf
, n
<< srq
->msrq
.wqe_shift
);
46 static void mlx4_ib_srq_event(struct mlx4_srq
*srq
, enum mlx4_event type
)
48 struct ib_event event
;
49 struct ib_srq
*ibsrq
= &to_mibsrq(srq
)->ibsrq
;
51 if (ibsrq
->event_handler
) {
52 event
.device
= ibsrq
->device
;
53 event
.element
.srq
= ibsrq
;
55 case MLX4_EVENT_TYPE_SRQ_LIMIT
:
56 event
.event
= IB_EVENT_SRQ_LIMIT_REACHED
;
58 case MLX4_EVENT_TYPE_SRQ_CATAS_ERROR
:
59 event
.event
= IB_EVENT_SRQ_ERR
;
62 printk(KERN_WARNING
"mlx4_ib: Unexpected event type %d "
63 "on SRQ %06x\n", type
, srq
->srqn
);
67 ibsrq
->event_handler(&event
, ibsrq
->srq_context
);
71 struct ib_srq
*mlx4_ib_create_srq(struct ib_pd
*pd
,
72 struct ib_srq_init_attr
*init_attr
,
73 struct ib_udata
*udata
)
75 struct mlx4_ib_dev
*dev
= to_mdev(pd
->device
);
76 struct mlx4_ib_srq
*srq
;
77 struct mlx4_wqe_srq_next_seg
*next
;
78 struct mlx4_wqe_data_seg
*scatter
;
86 /* Sanity check SRQ size before proceeding */
87 if (init_attr
->attr
.max_wr
>= dev
->dev
->caps
.max_srq_wqes
||
88 init_attr
->attr
.max_sge
> dev
->dev
->caps
.max_srq_sge
)
89 return ERR_PTR(-EINVAL
);
91 srq
= kmalloc(sizeof *srq
, GFP_KERNEL
);
93 return ERR_PTR(-ENOMEM
);
95 mutex_init(&srq
->mutex
);
96 spin_lock_init(&srq
->lock
);
97 srq
->msrq
.max
= roundup_pow_of_two(init_attr
->attr
.max_wr
+ 1);
98 srq
->msrq
.max_gs
= init_attr
->attr
.max_sge
;
100 desc_size
= max(32UL,
101 roundup_pow_of_two(sizeof (struct mlx4_wqe_srq_next_seg
) +
103 sizeof (struct mlx4_wqe_data_seg
)));
104 srq
->msrq
.wqe_shift
= ilog2(desc_size
);
106 buf_size
= srq
->msrq
.max
* desc_size
;
109 struct mlx4_ib_create_srq ucmd
;
111 if (ib_copy_from_udata(&ucmd
, udata
, sizeof ucmd
)) {
116 srq
->umem
= ib_umem_get(pd
->uobject
->context
, ucmd
.buf_addr
,
118 if (IS_ERR(srq
->umem
)) {
119 err
= PTR_ERR(srq
->umem
);
123 err
= mlx4_mtt_init(dev
->dev
, ib_umem_page_count(srq
->umem
),
124 ilog2(srq
->umem
->page_size
), &srq
->mtt
);
128 err
= mlx4_ib_umem_write_mtt(dev
, &srq
->mtt
, srq
->umem
);
132 err
= mlx4_ib_db_map_user(to_mucontext(pd
->uobject
->context
),
133 ucmd
.db_addr
, &srq
->db
);
137 err
= mlx4_db_alloc(dev
->dev
, &srq
->db
, 0);
143 if (mlx4_buf_alloc(dev
->dev
, buf_size
, PAGE_SIZE
* 2, &srq
->buf
)) {
149 srq
->tail
= srq
->msrq
.max
- 1;
152 for (i
= 0; i
< srq
->msrq
.max
; ++i
) {
153 next
= get_wqe(srq
, i
);
154 next
->next_wqe_index
=
155 cpu_to_be16((i
+ 1) & (srq
->msrq
.max
- 1));
157 for (scatter
= (void *) (next
+ 1);
158 (void *) scatter
< (void *) next
+ desc_size
;
160 scatter
->lkey
= cpu_to_be32(MLX4_INVALID_LKEY
);
163 err
= mlx4_mtt_init(dev
->dev
, srq
->buf
.npages
, srq
->buf
.page_shift
,
168 err
= mlx4_buf_write_mtt(dev
->dev
, &srq
->mtt
, &srq
->buf
);
172 srq
->wrid
= kmalloc(srq
->msrq
.max
* sizeof (u64
), GFP_KERNEL
);
179 cqn
= (init_attr
->srq_type
== IB_SRQT_XRC
) ?
180 to_mcq(init_attr
->ext
.xrc
.cq
)->mcq
.cqn
: 0;
181 xrcdn
= (init_attr
->srq_type
== IB_SRQT_XRC
) ?
182 to_mxrcd(init_attr
->ext
.xrc
.xrcd
)->xrcdn
:
183 (u16
) dev
->dev
->caps
.reserved_xrcds
;
184 err
= mlx4_srq_alloc(dev
->dev
, to_mpd(pd
)->pdn
, cqn
, xrcdn
, &srq
->mtt
,
185 srq
->db
.dma
, &srq
->msrq
);
189 srq
->msrq
.event
= mlx4_ib_srq_event
;
190 srq
->ibsrq
.ext
.xrc
.srq_num
= srq
->msrq
.srqn
;
193 if (ib_copy_to_udata(udata
, &srq
->msrq
.srqn
, sizeof (__u32
))) {
198 init_attr
->attr
.max_wr
= srq
->msrq
.max
- 1;
204 mlx4_ib_db_unmap_user(to_mucontext(pd
->uobject
->context
), &srq
->db
);
209 mlx4_mtt_cleanup(dev
->dev
, &srq
->mtt
);
213 ib_umem_release(srq
->umem
);
215 mlx4_buf_free(dev
->dev
, buf_size
, &srq
->buf
);
219 mlx4_db_free(dev
->dev
, &srq
->db
);
227 int mlx4_ib_modify_srq(struct ib_srq
*ibsrq
, struct ib_srq_attr
*attr
,
228 enum ib_srq_attr_mask attr_mask
, struct ib_udata
*udata
)
230 struct mlx4_ib_dev
*dev
= to_mdev(ibsrq
->device
);
231 struct mlx4_ib_srq
*srq
= to_msrq(ibsrq
);
234 /* We don't support resizing SRQs (yet?) */
235 if (attr_mask
& IB_SRQ_MAX_WR
)
238 if (attr_mask
& IB_SRQ_LIMIT
) {
239 if (attr
->srq_limit
>= srq
->msrq
.max
)
242 mutex_lock(&srq
->mutex
);
243 ret
= mlx4_srq_arm(dev
->dev
, &srq
->msrq
, attr
->srq_limit
);
244 mutex_unlock(&srq
->mutex
);
253 int mlx4_ib_query_srq(struct ib_srq
*ibsrq
, struct ib_srq_attr
*srq_attr
)
255 struct mlx4_ib_dev
*dev
= to_mdev(ibsrq
->device
);
256 struct mlx4_ib_srq
*srq
= to_msrq(ibsrq
);
260 ret
= mlx4_srq_query(dev
->dev
, &srq
->msrq
, &limit_watermark
);
264 srq_attr
->srq_limit
= limit_watermark
;
265 srq_attr
->max_wr
= srq
->msrq
.max
- 1;
266 srq_attr
->max_sge
= srq
->msrq
.max_gs
;
271 int mlx4_ib_destroy_srq(struct ib_srq
*srq
)
273 struct mlx4_ib_dev
*dev
= to_mdev(srq
->device
);
274 struct mlx4_ib_srq
*msrq
= to_msrq(srq
);
276 mlx4_srq_free(dev
->dev
, &msrq
->msrq
);
277 mlx4_mtt_cleanup(dev
->dev
, &msrq
->mtt
);
280 mlx4_ib_db_unmap_user(to_mucontext(srq
->uobject
->context
), &msrq
->db
);
281 ib_umem_release(msrq
->umem
);
284 mlx4_buf_free(dev
->dev
, msrq
->msrq
.max
<< msrq
->msrq
.wqe_shift
,
286 mlx4_db_free(dev
->dev
, &msrq
->db
);
294 void mlx4_ib_free_srq_wqe(struct mlx4_ib_srq
*srq
, int wqe_index
)
296 struct mlx4_wqe_srq_next_seg
*next
;
298 /* always called with interrupts disabled. */
299 spin_lock(&srq
->lock
);
301 next
= get_wqe(srq
, srq
->tail
);
302 next
->next_wqe_index
= cpu_to_be16(wqe_index
);
303 srq
->tail
= wqe_index
;
305 spin_unlock(&srq
->lock
);
308 int mlx4_ib_post_srq_recv(struct ib_srq
*ibsrq
, struct ib_recv_wr
*wr
,
309 struct ib_recv_wr
**bad_wr
)
311 struct mlx4_ib_srq
*srq
= to_msrq(ibsrq
);
312 struct mlx4_wqe_srq_next_seg
*next
;
313 struct mlx4_wqe_data_seg
*scat
;
319 spin_lock_irqsave(&srq
->lock
, flags
);
321 for (nreq
= 0; wr
; ++nreq
, wr
= wr
->next
) {
322 if (unlikely(wr
->num_sge
> srq
->msrq
.max_gs
)) {
328 if (unlikely(srq
->head
== srq
->tail
)) {
334 srq
->wrid
[srq
->head
] = wr
->wr_id
;
336 next
= get_wqe(srq
, srq
->head
);
337 srq
->head
= be16_to_cpu(next
->next_wqe_index
);
338 scat
= (struct mlx4_wqe_data_seg
*) (next
+ 1);
340 for (i
= 0; i
< wr
->num_sge
; ++i
) {
341 scat
[i
].byte_count
= cpu_to_be32(wr
->sg_list
[i
].length
);
342 scat
[i
].lkey
= cpu_to_be32(wr
->sg_list
[i
].lkey
);
343 scat
[i
].addr
= cpu_to_be64(wr
->sg_list
[i
].addr
);
346 if (i
< srq
->msrq
.max_gs
) {
347 scat
[i
].byte_count
= 0;
348 scat
[i
].lkey
= cpu_to_be32(MLX4_INVALID_LKEY
);
354 srq
->wqe_ctr
+= nreq
;
357 * Make sure that descriptors are written before
362 *srq
->db
.db
= cpu_to_be32(srq
->wqe_ctr
);
365 spin_unlock_irqrestore(&srq
->lock
, flags
);