2 * Copyright(c) 2016 - 2018 Intel Corporation.
4 * This file is provided under a dual BSD/GPLv2 license. When using or
5 * redistributing this file, you may do so under either license.
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of version 2 of the GNU General Public License as
11 * published by the Free Software Foundation.
13 * This program is distributed in the hope that it will be useful, but
14 * WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * General Public License for more details.
20 * Redistribution and use in source and binary forms, with or without
21 * modification, are permitted provided that the following conditions
24 * - Redistributions of source code must retain the above copyright
25 * notice, this list of conditions and the following disclaimer.
26 * - Redistributions in binary form must reproduce the above copyright
27 * notice, this list of conditions and the following disclaimer in
28 * the documentation and/or other materials provided with the
30 * - Neither the name of Intel Corporation nor the names of its
31 * contributors may be used to endorse or promote products derived
32 * from this software without specific prior written permission.
34 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
35 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
36 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
37 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
38 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
39 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
40 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
41 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
42 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
43 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
44 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
48 #include <linux/module.h>
49 #include <linux/kernel.h>
50 #include <linux/dma-mapping.h>
55 #define RVT_UVERBS_ABI_VERSION 2
57 MODULE_LICENSE("Dual BSD/GPL");
58 MODULE_DESCRIPTION("RDMA Verbs Transport Library");
60 static int rvt_init(void)
62 int ret
= rvt_driver_cq_init();
65 pr_err("Error in driver CQ init.\n");
69 module_init(rvt_init
);
71 static void rvt_cleanup(void)
75 module_exit(rvt_cleanup
);
78 * rvt_alloc_device - allocate rdi
79 * @size: how big of a structure to allocate
80 * @nports: number of ports to allocate array slots for
82 * Use IB core device alloc to allocate space for the rdi which is assumed to be
83 * inside of the ib_device. Any extra space that drivers require should be
86 * We also allocate a port array based on the number of ports.
88 * Return: pointer to allocated rdi
90 struct rvt_dev_info
*rvt_alloc_device(size_t size
, int nports
)
92 struct rvt_dev_info
*rdi
;
94 rdi
= container_of(_ib_alloc_device(size
), struct rvt_dev_info
, ibdev
);
98 rdi
->ports
= kcalloc(nports
,
99 sizeof(struct rvt_ibport
**),
102 ib_dealloc_device(&rdi
->ibdev
);
106 EXPORT_SYMBOL(rvt_alloc_device
);
109 * rvt_dealloc_device - deallocate rdi
110 * @rdi: structure to free
112 * Free a structure allocated with rvt_alloc_device()
114 void rvt_dealloc_device(struct rvt_dev_info
*rdi
)
117 ib_dealloc_device(&rdi
->ibdev
);
119 EXPORT_SYMBOL(rvt_dealloc_device
);
121 static int rvt_query_device(struct ib_device
*ibdev
,
122 struct ib_device_attr
*props
,
123 struct ib_udata
*uhw
)
125 struct rvt_dev_info
*rdi
= ib_to_rvt(ibdev
);
127 if (uhw
->inlen
|| uhw
->outlen
)
130 * Return rvt_dev_info.dparms.props contents
132 *props
= rdi
->dparms
.props
;
136 static int rvt_modify_device(struct ib_device
*device
,
137 int device_modify_mask
,
138 struct ib_device_modify
*device_modify
)
141 * There is currently no need to supply this based on qib and hfi1.
142 * Future drivers may need to implement this though.
149 * rvt_query_port: Passes the query port call to the driver
150 * @ibdev: Verbs IB dev
151 * @port_num: port number, 1 based from ib core
152 * @props: structure to hold returned properties
154 * Return: 0 on success
156 static int rvt_query_port(struct ib_device
*ibdev
, u8 port_num
,
157 struct ib_port_attr
*props
)
159 struct rvt_dev_info
*rdi
= ib_to_rvt(ibdev
);
160 struct rvt_ibport
*rvp
;
161 int port_index
= ibport_num_to_idx(ibdev
, port_num
);
166 rvp
= rdi
->ports
[port_index
];
167 /* props being zeroed by the caller, avoid zeroing it here */
168 props
->sm_lid
= rvp
->sm_lid
;
169 props
->sm_sl
= rvp
->sm_sl
;
170 props
->port_cap_flags
= rvp
->port_cap_flags
;
171 props
->max_msg_sz
= 0x80000000;
172 props
->pkey_tbl_len
= rvt_get_npkeys(rdi
);
173 props
->bad_pkey_cntr
= rvp
->pkey_violations
;
174 props
->qkey_viol_cntr
= rvp
->qkey_violations
;
175 props
->subnet_timeout
= rvp
->subnet_timeout
;
176 props
->init_type_reply
= 0;
178 /* Populate the remaining ib_port_attr elements */
179 return rdi
->driver_f
.query_port_state(rdi
, port_num
, props
);
184 * @ibdev: Verbs IB dev
185 * @port_num: Port number, 1 based from ib core
186 * @port_modify_mask: How to change the port
187 * @props: Structure to fill in
189 * Return: 0 on success
191 static int rvt_modify_port(struct ib_device
*ibdev
, u8 port_num
,
192 int port_modify_mask
, struct ib_port_modify
*props
)
194 struct rvt_dev_info
*rdi
= ib_to_rvt(ibdev
);
195 struct rvt_ibport
*rvp
;
197 int port_index
= ibport_num_to_idx(ibdev
, port_num
);
202 rvp
= rdi
->ports
[port_index
];
203 if (port_modify_mask
& IB_PORT_OPA_MASK_CHG
) {
204 rvp
->port_cap3_flags
|= props
->set_port_cap_mask
;
205 rvp
->port_cap3_flags
&= ~props
->clr_port_cap_mask
;
207 rvp
->port_cap_flags
|= props
->set_port_cap_mask
;
208 rvp
->port_cap_flags
&= ~props
->clr_port_cap_mask
;
211 if (props
->set_port_cap_mask
|| props
->clr_port_cap_mask
)
212 rdi
->driver_f
.cap_mask_chg(rdi
, port_num
);
213 if (port_modify_mask
& IB_PORT_SHUTDOWN
)
214 ret
= rdi
->driver_f
.shut_down_port(rdi
, port_num
);
215 if (port_modify_mask
& IB_PORT_RESET_QKEY_CNTR
)
216 rvp
->qkey_violations
= 0;
222 * rvt_query_pkey - Return a pkey from the table at a given index
223 * @ibdev: Verbs IB dev
224 * @port_num: Port number, 1 based from ib core
225 * @index: Index into pkey table
226 * @pkey: returned pkey from the port pkey table
228 * Return: 0 on failure pkey otherwise
230 static int rvt_query_pkey(struct ib_device
*ibdev
, u8 port_num
, u16 index
,
234 * Driver will be responsible for keeping rvt_dev_info.pkey_table up to
235 * date. This function will just return that value. There is no need to
236 * lock, if a stale value is read and sent to the user so be it there is
237 * no way to protect against that anyway.
239 struct rvt_dev_info
*rdi
= ib_to_rvt(ibdev
);
242 port_index
= ibport_num_to_idx(ibdev
, port_num
);
246 if (index
>= rvt_get_npkeys(rdi
))
249 *pkey
= rvt_get_pkey(rdi
, port_index
, index
);
254 * rvt_query_gid - Return a gid from the table
255 * @ibdev: Verbs IB dev
256 * @port_num: Port number, 1 based from ib core
257 * @guid_index: Index in table
258 * @gid: Gid to return
260 * Return: 0 on success
262 static int rvt_query_gid(struct ib_device
*ibdev
, u8 port_num
,
263 int guid_index
, union ib_gid
*gid
)
265 struct rvt_dev_info
*rdi
;
266 struct rvt_ibport
*rvp
;
270 * Driver is responsible for updating the guid table. Which will be used
271 * to craft the return value. This will work similar to how query_pkey()
274 port_index
= ibport_num_to_idx(ibdev
, port_num
);
278 rdi
= ib_to_rvt(ibdev
);
279 rvp
= rdi
->ports
[port_index
];
281 gid
->global
.subnet_prefix
= rvp
->gid_prefix
;
283 return rdi
->driver_f
.get_guid_be(rdi
, rvp
, guid_index
,
284 &gid
->global
.interface_id
);
287 static inline struct rvt_ucontext
*to_iucontext(struct ib_ucontext
290 return container_of(ibucontext
, struct rvt_ucontext
, ibucontext
);
294 * rvt_alloc_ucontext - Allocate a user context
295 * @uctx: Verbs context
296 * @udata: User data allocated
298 static int rvt_alloc_ucontext(struct ib_ucontext
*uctx
, struct ib_udata
*udata
)
304 * rvt_dealloc_ucontext - Free a user context
305 * @context - Free this
307 static void rvt_dealloc_ucontext(struct ib_ucontext
*context
)
312 static int rvt_get_port_immutable(struct ib_device
*ibdev
, u8 port_num
,
313 struct ib_port_immutable
*immutable
)
315 struct rvt_dev_info
*rdi
= ib_to_rvt(ibdev
);
316 struct ib_port_attr attr
;
319 port_index
= ibport_num_to_idx(ibdev
, port_num
);
323 immutable
->core_cap_flags
= rdi
->dparms
.core_cap_flags
;
325 err
= ib_query_port(ibdev
, port_num
, &attr
);
329 immutable
->pkey_tbl_len
= attr
.pkey_tbl_len
;
330 immutable
->gid_tbl_len
= attr
.gid_tbl_len
;
331 immutable
->max_mad_size
= rdi
->dparms
.max_mad_size
;
381 _VERB_IDX_MAX
/* Must always be last! */
384 static const struct ib_device_ops rvt_dev_ops
= {
385 .uverbs_abi_ver
= RVT_UVERBS_ABI_VERSION
,
387 .alloc_fmr
= rvt_alloc_fmr
,
388 .alloc_mr
= rvt_alloc_mr
,
389 .alloc_pd
= rvt_alloc_pd
,
390 .alloc_ucontext
= rvt_alloc_ucontext
,
391 .attach_mcast
= rvt_attach_mcast
,
392 .create_ah
= rvt_create_ah
,
393 .create_cq
= rvt_create_cq
,
394 .create_qp
= rvt_create_qp
,
395 .create_srq
= rvt_create_srq
,
396 .dealloc_fmr
= rvt_dealloc_fmr
,
397 .dealloc_pd
= rvt_dealloc_pd
,
398 .dealloc_ucontext
= rvt_dealloc_ucontext
,
399 .dereg_mr
= rvt_dereg_mr
,
400 .destroy_ah
= rvt_destroy_ah
,
401 .destroy_cq
= rvt_destroy_cq
,
402 .destroy_qp
= rvt_destroy_qp
,
403 .destroy_srq
= rvt_destroy_srq
,
404 .detach_mcast
= rvt_detach_mcast
,
405 .get_dma_mr
= rvt_get_dma_mr
,
406 .get_port_immutable
= rvt_get_port_immutable
,
407 .map_mr_sg
= rvt_map_mr_sg
,
408 .map_phys_fmr
= rvt_map_phys_fmr
,
410 .modify_ah
= rvt_modify_ah
,
411 .modify_device
= rvt_modify_device
,
412 .modify_port
= rvt_modify_port
,
413 .modify_qp
= rvt_modify_qp
,
414 .modify_srq
= rvt_modify_srq
,
415 .poll_cq
= rvt_poll_cq
,
416 .post_recv
= rvt_post_recv
,
417 .post_send
= rvt_post_send
,
418 .post_srq_recv
= rvt_post_srq_recv
,
419 .query_ah
= rvt_query_ah
,
420 .query_device
= rvt_query_device
,
421 .query_gid
= rvt_query_gid
,
422 .query_pkey
= rvt_query_pkey
,
423 .query_port
= rvt_query_port
,
424 .query_qp
= rvt_query_qp
,
425 .query_srq
= rvt_query_srq
,
426 .reg_user_mr
= rvt_reg_user_mr
,
427 .req_notify_cq
= rvt_req_notify_cq
,
428 .resize_cq
= rvt_resize_cq
,
429 .unmap_fmr
= rvt_unmap_fmr
,
431 INIT_RDMA_OBJ_SIZE(ib_ah
, rvt_ah
, ibah
),
432 INIT_RDMA_OBJ_SIZE(ib_cq
, rvt_cq
, ibcq
),
433 INIT_RDMA_OBJ_SIZE(ib_pd
, rvt_pd
, ibpd
),
434 INIT_RDMA_OBJ_SIZE(ib_srq
, rvt_srq
, ibsrq
),
435 INIT_RDMA_OBJ_SIZE(ib_ucontext
, rvt_ucontext
, ibucontext
),
438 static noinline
int check_support(struct rvt_dev_info
*rdi
, int verb
)
443 * These functions are not part of verbs specifically but are
444 * required for rdmavt to function.
446 if ((!rdi
->ibdev
.ops
.init_port
) ||
447 (!rdi
->driver_f
.get_pci_dev
))
453 * rdmavt does not support modify device currently drivers must
456 if (!rdi
->ibdev
.ops
.modify_device
)
461 if (!rdi
->ibdev
.ops
.query_port
)
462 if (!rdi
->driver_f
.query_port_state
)
467 if (!rdi
->ibdev
.ops
.modify_port
)
468 if (!rdi
->driver_f
.cap_mask_chg
||
469 !rdi
->driver_f
.shut_down_port
)
474 if (!rdi
->ibdev
.ops
.query_gid
)
475 if (!rdi
->driver_f
.get_guid_be
)
480 if (!rdi
->ibdev
.ops
.create_qp
)
481 if (!rdi
->driver_f
.qp_priv_alloc
||
482 !rdi
->driver_f
.qp_priv_free
||
483 !rdi
->driver_f
.notify_qp_reset
||
484 !rdi
->driver_f
.flush_qp_waiters
||
485 !rdi
->driver_f
.stop_send_queue
||
486 !rdi
->driver_f
.quiesce_qp
)
491 if (!rdi
->ibdev
.ops
.modify_qp
)
492 if (!rdi
->driver_f
.notify_qp_reset
||
493 !rdi
->driver_f
.schedule_send
||
494 !rdi
->driver_f
.get_pmtu_from_attr
||
495 !rdi
->driver_f
.flush_qp_waiters
||
496 !rdi
->driver_f
.stop_send_queue
||
497 !rdi
->driver_f
.quiesce_qp
||
498 !rdi
->driver_f
.notify_error_qp
||
499 !rdi
->driver_f
.mtu_from_qp
||
500 !rdi
->driver_f
.mtu_to_path_mtu
)
505 if (!rdi
->ibdev
.ops
.destroy_qp
)
506 if (!rdi
->driver_f
.qp_priv_free
||
507 !rdi
->driver_f
.notify_qp_reset
||
508 !rdi
->driver_f
.flush_qp_waiters
||
509 !rdi
->driver_f
.stop_send_queue
||
510 !rdi
->driver_f
.quiesce_qp
)
515 if (!rdi
->ibdev
.ops
.post_send
)
516 if (!rdi
->driver_f
.schedule_send
||
517 !rdi
->driver_f
.do_send
||
528 * rvt_register_device - register a driver
529 * @rdi: main dev structure for all of rdmavt operations
531 * It is up to drivers to allocate the rdi and fill in the appropriate
534 * Return: 0 on success otherwise an errno.
536 int rvt_register_device(struct rvt_dev_info
*rdi
)
544 * Check to ensure drivers have setup the required helpers for the verbs
545 * they want rdmavt to handle
547 for (i
= 0; i
< _VERB_IDX_MAX
; i
++)
548 if (check_support(rdi
, i
)) {
549 pr_err("Driver support req not met at %d\n", i
);
553 ib_set_device_ops(&rdi
->ibdev
, &rvt_dev_ops
);
555 /* Once we get past here we can use rvt_pr macros and tracepoints */
556 trace_rvt_dbg(rdi
, "Driver attempting registration");
560 ret
= rvt_driver_qp_init(rdi
);
562 pr_err("Error in driver QP init.\n");
567 spin_lock_init(&rdi
->n_ahs_lock
);
568 rdi
->n_ahs_allocated
= 0;
570 /* Shared Receive Queue */
571 rvt_driver_srq_init(rdi
);
574 rvt_driver_mcast_init(rdi
);
577 ret
= rvt_driver_mr_init(rdi
);
579 pr_err("Error in driver MR init.\n");
583 /* Memory Working Set Size */
584 ret
= rvt_wss_init(rdi
);
586 rvt_pr_err(rdi
, "Error in WSS init.\n");
590 /* Completion queues */
591 spin_lock_init(&rdi
->n_cqs_lock
);
594 rdi
->ibdev
.dev
.dma_ops
= rdi
->ibdev
.dev
.dma_ops
? : &dma_virt_ops
;
596 /* Protection Domain */
597 spin_lock_init(&rdi
->n_pds_lock
);
598 rdi
->n_pds_allocated
= 0;
601 * There are some things which could be set by underlying drivers but
602 * really should be up to rdmavt to set. For instance drivers can't know
603 * exactly which functions rdmavt supports, nor do they know the ABI
604 * version, so we do all of this sort of stuff here.
606 rdi
->ibdev
.uverbs_cmd_mask
=
607 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT
) |
608 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE
) |
609 (1ull << IB_USER_VERBS_CMD_QUERY_PORT
) |
610 (1ull << IB_USER_VERBS_CMD_ALLOC_PD
) |
611 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD
) |
612 (1ull << IB_USER_VERBS_CMD_CREATE_AH
) |
613 (1ull << IB_USER_VERBS_CMD_MODIFY_AH
) |
614 (1ull << IB_USER_VERBS_CMD_QUERY_AH
) |
615 (1ull << IB_USER_VERBS_CMD_DESTROY_AH
) |
616 (1ull << IB_USER_VERBS_CMD_REG_MR
) |
617 (1ull << IB_USER_VERBS_CMD_DEREG_MR
) |
618 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL
) |
619 (1ull << IB_USER_VERBS_CMD_CREATE_CQ
) |
620 (1ull << IB_USER_VERBS_CMD_RESIZE_CQ
) |
621 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ
) |
622 (1ull << IB_USER_VERBS_CMD_POLL_CQ
) |
623 (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ
) |
624 (1ull << IB_USER_VERBS_CMD_CREATE_QP
) |
625 (1ull << IB_USER_VERBS_CMD_QUERY_QP
) |
626 (1ull << IB_USER_VERBS_CMD_MODIFY_QP
) |
627 (1ull << IB_USER_VERBS_CMD_DESTROY_QP
) |
628 (1ull << IB_USER_VERBS_CMD_POST_SEND
) |
629 (1ull << IB_USER_VERBS_CMD_POST_RECV
) |
630 (1ull << IB_USER_VERBS_CMD_ATTACH_MCAST
) |
631 (1ull << IB_USER_VERBS_CMD_DETACH_MCAST
) |
632 (1ull << IB_USER_VERBS_CMD_CREATE_SRQ
) |
633 (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ
) |
634 (1ull << IB_USER_VERBS_CMD_QUERY_SRQ
) |
635 (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ
) |
636 (1ull << IB_USER_VERBS_CMD_POST_SRQ_RECV
);
637 rdi
->ibdev
.node_type
= RDMA_NODE_IB_CA
;
638 if (!rdi
->ibdev
.num_comp_vectors
)
639 rdi
->ibdev
.num_comp_vectors
= 1;
641 /* We are now good to announce we exist */
642 ret
= ib_register_device(&rdi
->ibdev
, dev_name(&rdi
->ibdev
.dev
));
644 rvt_pr_err(rdi
, "Failed to register driver with ib core.\n");
648 rvt_create_mad_agents(rdi
);
650 rvt_pr_info(rdi
, "Registration with rdmavt done.\n");
663 EXPORT_SYMBOL(rvt_register_device
);
666 * rvt_unregister_device - remove a driver
667 * @rdi: rvt dev struct
669 void rvt_unregister_device(struct rvt_dev_info
*rdi
)
671 trace_rvt_dbg(rdi
, "Driver is unregistering.");
675 rvt_free_mad_agents(rdi
);
677 ib_unregister_device(&rdi
->ibdev
);
682 EXPORT_SYMBOL(rvt_unregister_device
);
685 * rvt_init_port - init internal data for driver port
686 * @rdi: rvt_dev_info struct
688 * @port_index: 0 based index of ports, different from IB core port num
689 * @pkey_table: pkey_table for @port
691 * Keep track of a list of ports. No need to have a detach port.
692 * They persist until the driver goes away.
696 int rvt_init_port(struct rvt_dev_info
*rdi
, struct rvt_ibport
*port
,
697 int port_index
, u16
*pkey_table
)
700 rdi
->ports
[port_index
] = port
;
701 rdi
->ports
[port_index
]->pkey_table
= pkey_table
;
705 EXPORT_SYMBOL(rvt_init_port
);