1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2020 Mellanox Technologies Ltd. */
4 #include <linux/mlx5/driver.h>
7 static int alloc_pd(struct mlx5_vdpa_dev
*dev
, u32
*pdn
, u16 uid
)
9 struct mlx5_core_dev
*mdev
= dev
->mdev
;
11 u32 out
[MLX5_ST_SZ_DW(alloc_pd_out
)] = {};
12 u32 in
[MLX5_ST_SZ_DW(alloc_pd_in
)] = {};
15 MLX5_SET(alloc_pd_in
, in
, opcode
, MLX5_CMD_OP_ALLOC_PD
);
16 MLX5_SET(alloc_pd_in
, in
, uid
, uid
);
18 err
= mlx5_cmd_exec_inout(mdev
, alloc_pd
, in
, out
);
20 *pdn
= MLX5_GET(alloc_pd_out
, out
, pd
);
25 static int dealloc_pd(struct mlx5_vdpa_dev
*dev
, u32 pdn
, u16 uid
)
27 u32 in
[MLX5_ST_SZ_DW(dealloc_pd_in
)] = {};
28 struct mlx5_core_dev
*mdev
= dev
->mdev
;
30 MLX5_SET(dealloc_pd_in
, in
, opcode
, MLX5_CMD_OP_DEALLOC_PD
);
31 MLX5_SET(dealloc_pd_in
, in
, pd
, pdn
);
32 MLX5_SET(dealloc_pd_in
, in
, uid
, uid
);
33 return mlx5_cmd_exec_in(mdev
, dealloc_pd
, in
);
36 static int get_null_mkey(struct mlx5_vdpa_dev
*dev
, u32
*null_mkey
)
38 u32 out
[MLX5_ST_SZ_DW(query_special_contexts_out
)] = {};
39 u32 in
[MLX5_ST_SZ_DW(query_special_contexts_in
)] = {};
40 struct mlx5_core_dev
*mdev
= dev
->mdev
;
43 MLX5_SET(query_special_contexts_in
, in
, opcode
, MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS
);
44 err
= mlx5_cmd_exec_inout(mdev
, query_special_contexts
, in
, out
);
46 *null_mkey
= MLX5_GET(query_special_contexts_out
, out
, null_mkey
);
50 static int create_uctx(struct mlx5_vdpa_dev
*mvdev
, u16
*uid
)
52 u32 out
[MLX5_ST_SZ_DW(create_uctx_out
)] = {};
57 /* 0 means not supported */
58 if (!MLX5_CAP_GEN(mvdev
->mdev
, log_max_uctx
))
61 inlen
= MLX5_ST_SZ_BYTES(create_uctx_in
);
62 in
= kzalloc(inlen
, GFP_KERNEL
);
66 MLX5_SET(create_uctx_in
, in
, opcode
, MLX5_CMD_OP_CREATE_UCTX
);
67 MLX5_SET(create_uctx_in
, in
, uctx
.cap
, MLX5_UCTX_CAP_RAW_TX
);
69 err
= mlx5_cmd_exec(mvdev
->mdev
, in
, inlen
, out
, sizeof(out
));
72 *uid
= MLX5_GET(create_uctx_out
, out
, uid
);
77 static void destroy_uctx(struct mlx5_vdpa_dev
*mvdev
, u32 uid
)
79 u32 out
[MLX5_ST_SZ_DW(destroy_uctx_out
)] = {};
80 u32 in
[MLX5_ST_SZ_DW(destroy_uctx_in
)] = {};
82 MLX5_SET(destroy_uctx_in
, in
, opcode
, MLX5_CMD_OP_DESTROY_UCTX
);
83 MLX5_SET(destroy_uctx_in
, in
, uid
, uid
);
85 mlx5_cmd_exec(mvdev
->mdev
, in
, sizeof(in
), out
, sizeof(out
));
88 int mlx5_vdpa_create_tis(struct mlx5_vdpa_dev
*mvdev
, void *in
, u32
*tisn
)
90 u32 out
[MLX5_ST_SZ_DW(create_tis_out
)] = {};
93 MLX5_SET(create_tis_in
, in
, opcode
, MLX5_CMD_OP_CREATE_TIS
);
94 MLX5_SET(create_tis_in
, in
, uid
, mvdev
->res
.uid
);
95 err
= mlx5_cmd_exec_inout(mvdev
->mdev
, create_tis
, in
, out
);
97 *tisn
= MLX5_GET(create_tis_out
, out
, tisn
);
102 void mlx5_vdpa_destroy_tis(struct mlx5_vdpa_dev
*mvdev
, u32 tisn
)
104 u32 in
[MLX5_ST_SZ_DW(destroy_tis_in
)] = {};
106 MLX5_SET(destroy_tis_in
, in
, opcode
, MLX5_CMD_OP_DESTROY_TIS
);
107 MLX5_SET(destroy_tis_in
, in
, uid
, mvdev
->res
.uid
);
108 MLX5_SET(destroy_tis_in
, in
, tisn
, tisn
);
109 mlx5_cmd_exec_in(mvdev
->mdev
, destroy_tis
, in
);
112 int mlx5_vdpa_create_rqt(struct mlx5_vdpa_dev
*mvdev
, void *in
, int inlen
, u32
*rqtn
)
114 u32 out
[MLX5_ST_SZ_DW(create_rqt_out
)] = {};
117 MLX5_SET(create_rqt_in
, in
, opcode
, MLX5_CMD_OP_CREATE_RQT
);
118 err
= mlx5_cmd_exec(mvdev
->mdev
, in
, inlen
, out
, sizeof(out
));
120 *rqtn
= MLX5_GET(create_rqt_out
, out
, rqtn
);
125 void mlx5_vdpa_destroy_rqt(struct mlx5_vdpa_dev
*mvdev
, u32 rqtn
)
127 u32 in
[MLX5_ST_SZ_DW(destroy_rqt_in
)] = {};
129 MLX5_SET(destroy_rqt_in
, in
, opcode
, MLX5_CMD_OP_DESTROY_RQT
);
130 MLX5_SET(destroy_rqt_in
, in
, uid
, mvdev
->res
.uid
);
131 MLX5_SET(destroy_rqt_in
, in
, rqtn
, rqtn
);
132 mlx5_cmd_exec_in(mvdev
->mdev
, destroy_rqt
, in
);
135 int mlx5_vdpa_create_tir(struct mlx5_vdpa_dev
*mvdev
, void *in
, u32
*tirn
)
137 u32 out
[MLX5_ST_SZ_DW(create_tir_out
)] = {};
140 MLX5_SET(create_tir_in
, in
, opcode
, MLX5_CMD_OP_CREATE_TIR
);
141 err
= mlx5_cmd_exec_inout(mvdev
->mdev
, create_tir
, in
, out
);
143 *tirn
= MLX5_GET(create_tir_out
, out
, tirn
);
148 void mlx5_vdpa_destroy_tir(struct mlx5_vdpa_dev
*mvdev
, u32 tirn
)
150 u32 in
[MLX5_ST_SZ_DW(destroy_tir_in
)] = {};
152 MLX5_SET(destroy_tir_in
, in
, opcode
, MLX5_CMD_OP_DESTROY_TIR
);
153 MLX5_SET(destroy_tir_in
, in
, uid
, mvdev
->res
.uid
);
154 MLX5_SET(destroy_tir_in
, in
, tirn
, tirn
);
155 mlx5_cmd_exec_in(mvdev
->mdev
, destroy_tir
, in
);
158 int mlx5_vdpa_alloc_transport_domain(struct mlx5_vdpa_dev
*mvdev
, u32
*tdn
)
160 u32 out
[MLX5_ST_SZ_DW(alloc_transport_domain_out
)] = {};
161 u32 in
[MLX5_ST_SZ_DW(alloc_transport_domain_in
)] = {};
164 MLX5_SET(alloc_transport_domain_in
, in
, opcode
, MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN
);
165 MLX5_SET(alloc_transport_domain_in
, in
, uid
, mvdev
->res
.uid
);
167 err
= mlx5_cmd_exec_inout(mvdev
->mdev
, alloc_transport_domain
, in
, out
);
169 *tdn
= MLX5_GET(alloc_transport_domain_out
, out
, transport_domain
);
174 void mlx5_vdpa_dealloc_transport_domain(struct mlx5_vdpa_dev
*mvdev
, u32 tdn
)
176 u32 in
[MLX5_ST_SZ_DW(dealloc_transport_domain_in
)] = {};
178 MLX5_SET(dealloc_transport_domain_in
, in
, opcode
, MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN
);
179 MLX5_SET(dealloc_transport_domain_in
, in
, uid
, mvdev
->res
.uid
);
180 MLX5_SET(dealloc_transport_domain_in
, in
, transport_domain
, tdn
);
181 mlx5_cmd_exec_in(mvdev
->mdev
, dealloc_transport_domain
, in
);
184 int mlx5_vdpa_create_mkey(struct mlx5_vdpa_dev
*mvdev
, struct mlx5_core_mkey
*mkey
, u32
*in
,
187 u32 lout
[MLX5_ST_SZ_DW(create_mkey_out
)] = {};
192 MLX5_SET(create_mkey_in
, in
, opcode
, MLX5_CMD_OP_CREATE_MKEY
);
193 MLX5_SET(create_mkey_in
, in
, uid
, mvdev
->res
.uid
);
195 err
= mlx5_cmd_exec(mvdev
->mdev
, in
, inlen
, lout
, sizeof(lout
));
199 mkc
= MLX5_ADDR_OF(create_mkey_in
, in
, memory_key_mkey_entry
);
200 mkey_index
= MLX5_GET(create_mkey_out
, lout
, mkey_index
);
201 mkey
->iova
= MLX5_GET64(mkc
, mkc
, start_addr
);
202 mkey
->size
= MLX5_GET64(mkc
, mkc
, len
);
203 mkey
->key
|= mlx5_idx_to_mkey(mkey_index
);
204 mkey
->pd
= MLX5_GET(mkc
, mkc
, pd
);
208 int mlx5_vdpa_destroy_mkey(struct mlx5_vdpa_dev
*mvdev
, struct mlx5_core_mkey
*mkey
)
210 u32 in
[MLX5_ST_SZ_DW(destroy_mkey_in
)] = {};
212 MLX5_SET(destroy_mkey_in
, in
, uid
, mvdev
->res
.uid
);
213 MLX5_SET(destroy_mkey_in
, in
, opcode
, MLX5_CMD_OP_DESTROY_MKEY
);
214 MLX5_SET(destroy_mkey_in
, in
, mkey_index
, mlx5_mkey_to_idx(mkey
->key
));
215 return mlx5_cmd_exec_in(mvdev
->mdev
, destroy_mkey
, in
);
218 int mlx5_vdpa_alloc_resources(struct mlx5_vdpa_dev
*mvdev
)
220 u64 offset
= MLX5_CAP64_DEV_VDPA_EMULATION(mvdev
->mdev
, doorbell_bar_offset
);
221 struct mlx5_vdpa_resources
*res
= &mvdev
->res
;
222 struct mlx5_core_dev
*mdev
= mvdev
->mdev
;
227 mlx5_vdpa_warn(mvdev
, "resources already allocated\n");
230 mutex_init(&mvdev
->mr
.mkey_mtx
);
231 res
->uar
= mlx5_get_uars_page(mdev
);
232 if (IS_ERR(res
->uar
)) {
233 err
= PTR_ERR(res
->uar
);
237 err
= create_uctx(mvdev
, &res
->uid
);
241 err
= alloc_pd(mvdev
, &res
->pdn
, res
->uid
);
245 err
= get_null_mkey(mvdev
, &res
->null_mkey
);
249 kick_addr
= pci_resource_start(mdev
->pdev
, 0) + offset
;
250 res
->kick_addr
= ioremap(kick_addr
, PAGE_SIZE
);
251 if (!res
->kick_addr
) {
260 dealloc_pd(mvdev
, res
->pdn
, res
->uid
);
262 destroy_uctx(mvdev
, res
->uid
);
264 mlx5_put_uars_page(mdev
, res
->uar
);
266 mutex_destroy(&mvdev
->mr
.mkey_mtx
);
270 void mlx5_vdpa_free_resources(struct mlx5_vdpa_dev
*mvdev
)
272 struct mlx5_vdpa_resources
*res
= &mvdev
->res
;
277 iounmap(res
->kick_addr
);
278 res
->kick_addr
= NULL
;
279 dealloc_pd(mvdev
, res
->pdn
, res
->uid
);
280 destroy_uctx(mvdev
, res
->uid
);
281 mlx5_put_uars_page(mvdev
->mdev
, res
->uar
);
282 mutex_destroy(&mvdev
->mr
.mkey_mtx
);