?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
dm writecache: add cond_resched to loop in persistent_memory_claim()
[linux/fpc-iii.git] / drivers / infiniband / hw / mlx5 / devx.c
blobed10e2f32aab0f7b4a0eb94c9550368d75ce2028
1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3 * Copyright (c) 2018, Mellanox Technologies inc. All rights reserved.
4 */
5
6 #include <rdma/ib_user_verbs.h>
7 #include <rdma/ib_verbs.h>
8 #include <rdma/uverbs_types.h>
9 #include <rdma/uverbs_ioctl.h>
10 #include <rdma/mlx5_user_ioctl_cmds.h>
11 #include <rdma/mlx5_user_ioctl_verbs.h>
12 #include <rdma/ib_umem.h>
13 #include <rdma/uverbs_std_types.h>
14 #include <linux/mlx5/driver.h>
15 #include <linux/mlx5/fs.h>
16 #include "mlx5_ib.h"
17 #include <linux/xarray.h>
18
19 #define UVERBS_MODULE_NAME mlx5_ib
20 #include <rdma/uverbs_named_ioctl.h>
21
22 static void dispatch_event_fd(struct list_head *fd_list, const void *data);
23
24 enum devx_obj_flags {
25 DEVX_OBJ_FLAGS_INDIRECT_MKEY = 1 << 0,
26 DEVX_OBJ_FLAGS_DCT = 1 << 1,
27 DEVX_OBJ_FLAGS_CQ = 1 << 2,
28 };
29
30 struct devx_async_data {
31 struct mlx5_ib_dev *mdev;
32 struct list_head list;
33 struct devx_async_cmd_event_file *ev_file;
34 struct mlx5_async_work cb_work;
35 u16 cmd_out_len;
36 /* must be last field in this structure */
37 struct mlx5_ib_uapi_devx_async_cmd_hdr hdr;
38 };
39
40 struct devx_async_event_data {
41 struct list_head list; /* headed in ev_file->event_list */
42 struct mlx5_ib_uapi_devx_async_event_hdr hdr;
43 };
44
45 /* first level XA value data structure */
46 struct devx_event {
47 struct xarray object_ids; /* second XA level, Key = object id */
48 struct list_head unaffiliated_list;
49 };
50
51 /* second level XA value data structure */
52 struct devx_obj_event {
53 struct rcu_head rcu;
54 struct list_head obj_sub_list;
55 };
56
57 struct devx_event_subscription {
58 struct list_head file_list; /* headed in ev_file->
59 * subscribed_events_list
60 */
61 struct list_head xa_list; /* headed in devx_event->unaffiliated_list or
62 * devx_obj_event->obj_sub_list
63 */
64 struct list_head obj_list; /* headed in devx_object */
65 struct list_head event_list; /* headed in ev_file->event_list or in
66 * temp list via subscription
67 */
68
69 u8 is_cleaned:1;
70 u32 xa_key_level1;
71 u32 xa_key_level2;
72 struct rcu_head rcu;
73 u64 cookie;
74 struct devx_async_event_file *ev_file;
75 struct eventfd_ctx *eventfd;
76 };
77
78 struct devx_async_event_file {
79 struct ib_uobject uobj;
80 /* Head of events that are subscribed to this FD */
81 struct list_head subscribed_events_list;
82 spinlock_t lock;
83 wait_queue_head_t poll_wait;
84 struct list_head event_list;
85 struct mlx5_ib_dev *dev;
86 u8 omit_data:1;
87 u8 is_overflow_err:1;
88 u8 is_destroyed:1;
89 };
90
91 #define MLX5_MAX_DESTROY_INBOX_SIZE_DW MLX5_ST_SZ_DW(delete_fte_in)
92 struct devx_obj {
93 struct mlx5_ib_dev *ib_dev;
94 u64 obj_id;
95 u32 dinlen; /* destroy inbox length */
96 u32 dinbox[MLX5_MAX_DESTROY_INBOX_SIZE_DW];
97 u32 flags;
98 union {
99 struct mlx5_ib_devx_mr devx_mr;
100 struct mlx5_core_dct core_dct;
101 struct mlx5_core_cq core_cq;
102 u32 flow_counter_bulk_size;
103 };
104 struct list_head event_sub; /* holds devx_event_subscription entries */
105 };
106
107 struct devx_umem {
108 struct mlx5_core_dev *mdev;
109 struct ib_umem *umem;
110 u32 page_offset;
111 int page_shift;
112 int ncont;
113 u32 dinlen;
114 u32 dinbox[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)];
115 };
116
117 struct devx_umem_reg_cmd {
118 void *in;
119 u32 inlen;
120 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
121 };
122
123 static struct mlx5_ib_ucontext *
124 devx_ufile2uctx(const struct uverbs_attr_bundle *attrs)
125 {
126 return to_mucontext(ib_uverbs_get_ucontext(attrs));
127 }
128
129 int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user)
130 {
131 u32 in[MLX5_ST_SZ_DW(create_uctx_in)] = {0};
132 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};
133 void *uctx;
134 int err;
135 u16 uid;
136 u32 cap = 0;
137
138 /* 0 means not supported */
139 if (!MLX5_CAP_GEN(dev->mdev, log_max_uctx))
140 return -EINVAL;
141
142 uctx = MLX5_ADDR_OF(create_uctx_in, in, uctx);
143 if (is_user && capable(CAP_NET_RAW) &&
144 (MLX5_CAP_GEN(dev->mdev, uctx_cap) & MLX5_UCTX_CAP_RAW_TX))
145 cap |= MLX5_UCTX_CAP_RAW_TX;
146 if (is_user && capable(CAP_SYS_RAWIO) &&
147 (MLX5_CAP_GEN(dev->mdev, uctx_cap) &
148 MLX5_UCTX_CAP_INTERNAL_DEV_RES))
149 cap |= MLX5_UCTX_CAP_INTERNAL_DEV_RES;
150
151 MLX5_SET(create_uctx_in, in, opcode, MLX5_CMD_OP_CREATE_UCTX);
152 MLX5_SET(uctx, uctx, cap, cap);
153
154 err = mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out));
155 if (err)
156 return err;
157
158 uid = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
159 return uid;
160 }
161
162 void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid)
163 {
164 u32 in[MLX5_ST_SZ_DW(destroy_uctx_in)] = {0};
165 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};
166
167 MLX5_SET(destroy_uctx_in, in, opcode, MLX5_CMD_OP_DESTROY_UCTX);
168 MLX5_SET(destroy_uctx_in, in, uid, uid);
169
170 mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out));
171 }
172
173 bool mlx5_ib_devx_is_flow_dest(void *obj, int *dest_id, int *dest_type)
174 {
175 struct devx_obj *devx_obj = obj;
176 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode);
177
178 switch (opcode) {
179 case MLX5_CMD_OP_DESTROY_TIR:
180 *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR;
181 *dest_id = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox,
182 obj_id);
183 return true;
184
185 case MLX5_CMD_OP_DESTROY_FLOW_TABLE:
186 *dest_type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
187 *dest_id = MLX5_GET(destroy_flow_table_in, devx_obj->dinbox,
188 table_id);
189 return true;
190 default:
191 return false;
192 }
193 }
194
195 bool mlx5_ib_devx_is_flow_counter(void *obj, u32 offset, u32 *counter_id)
196 {
197 struct devx_obj *devx_obj = obj;
198 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode);
199
200 if (opcode == MLX5_CMD_OP_DEALLOC_FLOW_COUNTER) {
201
202 if (offset && offset >= devx_obj->flow_counter_bulk_size)
203 return false;
204
205 *counter_id = MLX5_GET(dealloc_flow_counter_in,
206 devx_obj->dinbox,
207 flow_counter_id);
208 *counter_id += offset;
209 return true;
210 }
211
212 return false;
213 }
214
215 static bool is_legacy_unaffiliated_event_num(u16 event_num)
216 {
217 switch (event_num) {
218 case MLX5_EVENT_TYPE_PORT_CHANGE:
219 return true;
220 default:
221 return false;
222 }
223 }
224
225 static bool is_legacy_obj_event_num(u16 event_num)
226 {
227 switch (event_num) {
228 case MLX5_EVENT_TYPE_PATH_MIG:
229 case MLX5_EVENT_TYPE_COMM_EST:
230 case MLX5_EVENT_TYPE_SQ_DRAINED:
231 case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
232 case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
233 case MLX5_EVENT_TYPE_CQ_ERROR:
234 case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
235 case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
236 case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
237 case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
238 case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
239 case MLX5_EVENT_TYPE_DCT_DRAINED:
240 case MLX5_EVENT_TYPE_COMP:
241 case MLX5_EVENT_TYPE_DCT_KEY_VIOLATION:
242 case MLX5_EVENT_TYPE_XRQ_ERROR:
243 return true;
244 default:
245 return false;
246 }
247 }
248
249 static u16 get_legacy_obj_type(u16 opcode)
250 {
251 switch (opcode) {
252 case MLX5_CMD_OP_CREATE_RQ:
253 return MLX5_EVENT_QUEUE_TYPE_RQ;
254 case MLX5_CMD_OP_CREATE_QP:
255 return MLX5_EVENT_QUEUE_TYPE_QP;
256 case MLX5_CMD_OP_CREATE_SQ:
257 return MLX5_EVENT_QUEUE_TYPE_SQ;
258 case MLX5_CMD_OP_CREATE_DCT:
259 return MLX5_EVENT_QUEUE_TYPE_DCT;
260 default:
261 return 0;
262 }
263 }
264
265 static u16 get_dec_obj_type(struct devx_obj *obj, u16 event_num)
266 {
267 u16 opcode;
268
269 opcode = (obj->obj_id >> 32) & 0xffff;
270
271 if (is_legacy_obj_event_num(event_num))
272 return get_legacy_obj_type(opcode);
273
274 switch (opcode) {
275 case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
276 return (obj->obj_id >> 48);
277 case MLX5_CMD_OP_CREATE_RQ:
278 return MLX5_OBJ_TYPE_RQ;
279 case MLX5_CMD_OP_CREATE_QP:
280 return MLX5_OBJ_TYPE_QP;
281 case MLX5_CMD_OP_CREATE_SQ:
282 return MLX5_OBJ_TYPE_SQ;
283 case MLX5_CMD_OP_CREATE_DCT:
284 return MLX5_OBJ_TYPE_DCT;
285 case MLX5_CMD_OP_CREATE_TIR:
286 return MLX5_OBJ_TYPE_TIR;
287 case MLX5_CMD_OP_CREATE_TIS:
288 return MLX5_OBJ_TYPE_TIS;
289 case MLX5_CMD_OP_CREATE_PSV:
290 return MLX5_OBJ_TYPE_PSV;
291 case MLX5_OBJ_TYPE_MKEY:
292 return MLX5_OBJ_TYPE_MKEY;
293 case MLX5_CMD_OP_CREATE_RMP:
294 return MLX5_OBJ_TYPE_RMP;
295 case MLX5_CMD_OP_CREATE_XRC_SRQ:
296 return MLX5_OBJ_TYPE_XRC_SRQ;
297 case MLX5_CMD_OP_CREATE_XRQ:
298 return MLX5_OBJ_TYPE_XRQ;
299 case MLX5_CMD_OP_CREATE_RQT:
300 return MLX5_OBJ_TYPE_RQT;
301 case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
302 return MLX5_OBJ_TYPE_FLOW_COUNTER;
303 case MLX5_CMD_OP_CREATE_CQ:
304 return MLX5_OBJ_TYPE_CQ;
305 default:
306 return 0;
307 }
308 }
309
310 static u16 get_event_obj_type(unsigned long event_type, struct mlx5_eqe *eqe)
311 {
312 switch (event_type) {
313 case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
314 case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
315 case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
316 case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
317 case MLX5_EVENT_TYPE_PATH_MIG:
318 case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
319 case MLX5_EVENT_TYPE_COMM_EST:
320 case MLX5_EVENT_TYPE_SQ_DRAINED:
321 case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
322 case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
323 return eqe->data.qp_srq.type;
324 case MLX5_EVENT_TYPE_CQ_ERROR:
325 case MLX5_EVENT_TYPE_XRQ_ERROR:
326 return 0;
327 case MLX5_EVENT_TYPE_DCT_DRAINED:
328 case MLX5_EVENT_TYPE_DCT_KEY_VIOLATION:
329 return MLX5_EVENT_QUEUE_TYPE_DCT;
330 default:
331 return MLX5_GET(affiliated_event_header, &eqe->data, obj_type);
332 }
333 }
334
335 static u32 get_dec_obj_id(u64 obj_id)
336 {
337 return (obj_id & 0xffffffff);
338 }
339
340 /*
341 * As the obj_id in the firmware is not globally unique the object type
342 * must be considered upon checking for a valid object id.
343 * For that the opcode of the creator command is encoded as part of the obj_id.
344 */
345 static u64 get_enc_obj_id(u32 opcode, u32 obj_id)
346 {
347 return ((u64)opcode << 32) | obj_id;
348 }
349
350 static u64 devx_get_obj_id(const void *in)
351 {
352 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
353 u64 obj_id;
354
355 switch (opcode) {
356 case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT:
357 case MLX5_CMD_OP_QUERY_GENERAL_OBJECT:
358 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_GENERAL_OBJECT |
359 MLX5_GET(general_obj_in_cmd_hdr, in,
360 obj_type) << 16,
361 MLX5_GET(general_obj_in_cmd_hdr, in,
362 obj_id));
363 break;
364 case MLX5_CMD_OP_QUERY_MKEY:
365 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_MKEY,
366 MLX5_GET(query_mkey_in, in,
367 mkey_index));
368 break;
369 case MLX5_CMD_OP_QUERY_CQ:
370 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ,
371 MLX5_GET(query_cq_in, in, cqn));
372 break;
373 case MLX5_CMD_OP_MODIFY_CQ:
374 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ,
375 MLX5_GET(modify_cq_in, in, cqn));
376 break;
377 case MLX5_CMD_OP_QUERY_SQ:
378 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ,
379 MLX5_GET(query_sq_in, in, sqn));
380 break;
381 case MLX5_CMD_OP_MODIFY_SQ:
382 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ,
383 MLX5_GET(modify_sq_in, in, sqn));
384 break;
385 case MLX5_CMD_OP_QUERY_RQ:
386 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
387 MLX5_GET(query_rq_in, in, rqn));
388 break;
389 case MLX5_CMD_OP_MODIFY_RQ:
390 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
391 MLX5_GET(modify_rq_in, in, rqn));
392 break;
393 case MLX5_CMD_OP_QUERY_RMP:
394 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RMP,
395 MLX5_GET(query_rmp_in, in, rmpn));
396 break;
397 case MLX5_CMD_OP_MODIFY_RMP:
398 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RMP,
399 MLX5_GET(modify_rmp_in, in, rmpn));
400 break;
401 case MLX5_CMD_OP_QUERY_RQT:
402 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT,
403 MLX5_GET(query_rqt_in, in, rqtn));
404 break;
405 case MLX5_CMD_OP_MODIFY_RQT:
406 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT,
407 MLX5_GET(modify_rqt_in, in, rqtn));
408 break;
409 case MLX5_CMD_OP_QUERY_TIR:
410 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR,
411 MLX5_GET(query_tir_in, in, tirn));
412 break;
413 case MLX5_CMD_OP_MODIFY_TIR:
414 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR,
415 MLX5_GET(modify_tir_in, in, tirn));
416 break;
417 case MLX5_CMD_OP_QUERY_TIS:
418 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS,
419 MLX5_GET(query_tis_in, in, tisn));
420 break;
421 case MLX5_CMD_OP_MODIFY_TIS:
422 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS,
423 MLX5_GET(modify_tis_in, in, tisn));
424 break;
425 case MLX5_CMD_OP_QUERY_FLOW_TABLE:
426 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_TABLE,
427 MLX5_GET(query_flow_table_in, in,
428 table_id));
429 break;
430 case MLX5_CMD_OP_MODIFY_FLOW_TABLE:
431 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_TABLE,
432 MLX5_GET(modify_flow_table_in, in,
433 table_id));
434 break;
435 case MLX5_CMD_OP_QUERY_FLOW_GROUP:
436 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_GROUP,
437 MLX5_GET(query_flow_group_in, in,
438 group_id));
439 break;
440 case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY:
441 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY,
442 MLX5_GET(query_fte_in, in,
443 flow_index));
444 break;
445 case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
446 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY,
447 MLX5_GET(set_fte_in, in, flow_index));
448 break;
449 case MLX5_CMD_OP_QUERY_Q_COUNTER:
450 obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_Q_COUNTER,
451 MLX5_GET(query_q_counter_in, in,
452 counter_set_id));
453 break;
454 case MLX5_CMD_OP_QUERY_FLOW_COUNTER:
455 obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_FLOW_COUNTER,
456 MLX5_GET(query_flow_counter_in, in,
457 flow_counter_id));
458 break;
459 case MLX5_CMD_OP_QUERY_MODIFY_HEADER_CONTEXT:
460 obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT,
461 MLX5_GET(general_obj_in_cmd_hdr, in,
462 obj_id));
463 break;
464 case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT:
465 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT,
466 MLX5_GET(query_scheduling_element_in,
467 in, scheduling_element_id));
468 break;
469 case MLX5_CMD_OP_MODIFY_SCHEDULING_ELEMENT:
470 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT,
471 MLX5_GET(modify_scheduling_element_in,
472 in, scheduling_element_id));
473 break;
474 case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
475 obj_id = get_enc_obj_id(MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT,
476 MLX5_GET(add_vxlan_udp_dport_in, in,
477 vxlan_udp_port));
478 break;
479 case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY:
480 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_L2_TABLE_ENTRY,
481 MLX5_GET(query_l2_table_entry_in, in,
482 table_index));
483 break;
484 case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
485 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_L2_TABLE_ENTRY,
486 MLX5_GET(set_l2_table_entry_in, in,
487 table_index));
488 break;
489 case MLX5_CMD_OP_QUERY_QP:
490 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
491 MLX5_GET(query_qp_in, in, qpn));
492 break;
493 case MLX5_CMD_OP_RST2INIT_QP:
494 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
495 MLX5_GET(rst2init_qp_in, in, qpn));
496 break;
497 case MLX5_CMD_OP_INIT2INIT_QP:
498 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
499 MLX5_GET(init2init_qp_in, in, qpn));
500 break;
501 case MLX5_CMD_OP_INIT2RTR_QP:
502 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
503 MLX5_GET(init2rtr_qp_in, in, qpn));
504 break;
505 case MLX5_CMD_OP_RTR2RTS_QP:
506 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
507 MLX5_GET(rtr2rts_qp_in, in, qpn));
508 break;
509 case MLX5_CMD_OP_RTS2RTS_QP:
510 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
511 MLX5_GET(rts2rts_qp_in, in, qpn));
512 break;
513 case MLX5_CMD_OP_SQERR2RTS_QP:
514 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
515 MLX5_GET(sqerr2rts_qp_in, in, qpn));
516 break;
517 case MLX5_CMD_OP_2ERR_QP:
518 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
519 MLX5_GET(qp_2err_in, in, qpn));
520 break;
521 case MLX5_CMD_OP_2RST_QP:
522 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
523 MLX5_GET(qp_2rst_in, in, qpn));
524 break;
525 case MLX5_CMD_OP_QUERY_DCT:
526 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT,
527 MLX5_GET(query_dct_in, in, dctn));
528 break;
529 case MLX5_CMD_OP_QUERY_XRQ:
530 case MLX5_CMD_OP_QUERY_XRQ_DC_PARAMS_ENTRY:
531 case MLX5_CMD_OP_QUERY_XRQ_ERROR_PARAMS:
532 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRQ,
533 MLX5_GET(query_xrq_in, in, xrqn));
534 break;
535 case MLX5_CMD_OP_QUERY_XRC_SRQ:
536 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRC_SRQ,
537 MLX5_GET(query_xrc_srq_in, in,
538 xrc_srqn));
539 break;
540 case MLX5_CMD_OP_ARM_XRC_SRQ:
541 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRC_SRQ,
542 MLX5_GET(arm_xrc_srq_in, in, xrc_srqn));
543 break;
544 case MLX5_CMD_OP_QUERY_SRQ:
545 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SRQ,
546 MLX5_GET(query_srq_in, in, srqn));
547 break;
548 case MLX5_CMD_OP_ARM_RQ:
549 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
550 MLX5_GET(arm_rq_in, in, srq_number));
551 break;
552 case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION:
553 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT,
554 MLX5_GET(drain_dct_in, in, dctn));
555 break;
556 case MLX5_CMD_OP_ARM_XRQ:
557 case MLX5_CMD_OP_SET_XRQ_DC_PARAMS_ENTRY:
558 case MLX5_CMD_OP_RELEASE_XRQ_ERROR:
559 case MLX5_CMD_OP_MODIFY_XRQ:
560 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRQ,
561 MLX5_GET(arm_xrq_in, in, xrqn));
562 break;
563 case MLX5_CMD_OP_QUERY_PACKET_REFORMAT_CONTEXT:
564 obj_id = get_enc_obj_id
565 (MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT,
566 MLX5_GET(query_packet_reformat_context_in,
567 in, packet_reformat_id));
568 break;
569 default:
570 obj_id = 0;
571 }
572
573 return obj_id;
574 }
575
576 static bool devx_is_valid_obj_id(struct uverbs_attr_bundle *attrs,
577 struct ib_uobject *uobj, const void *in)
578 {
579 struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata);
580 u64 obj_id = devx_get_obj_id(in);
581
582 if (!obj_id)
583 return false;
584
585 switch (uobj_get_object_id(uobj)) {
586 case UVERBS_OBJECT_CQ:
587 return get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ,
588 to_mcq(uobj->object)->mcq.cqn) ==
589 obj_id;
590
591 case UVERBS_OBJECT_SRQ:
592 {
593 struct mlx5_core_srq *srq = &(to_msrq(uobj->object)->msrq);
594 u16 opcode;
595
596 switch (srq->common.res) {
597 case MLX5_RES_XSRQ:
598 opcode = MLX5_CMD_OP_CREATE_XRC_SRQ;
599 break;
600 case MLX5_RES_XRQ:
601 opcode = MLX5_CMD_OP_CREATE_XRQ;
602 break;
603 default:
604 if (!dev->mdev->issi)
605 opcode = MLX5_CMD_OP_CREATE_SRQ;
606 else
607 opcode = MLX5_CMD_OP_CREATE_RMP;
608 }
609
610 return get_enc_obj_id(opcode,
611 to_msrq(uobj->object)->msrq.srqn) ==
612 obj_id;
613 }
614
615 case UVERBS_OBJECT_QP:
616 {
617 struct mlx5_ib_qp *qp = to_mqp(uobj->object);
618 enum ib_qp_type qp_type = qp->ibqp.qp_type;
619
620 if (qp_type == IB_QPT_RAW_PACKET ||
621 (qp->flags & MLX5_IB_QP_UNDERLAY)) {
622 struct mlx5_ib_raw_packet_qp *raw_packet_qp =
623 &qp->raw_packet_qp;
624 struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
625 struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
626
627 return (get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
628 rq->base.mqp.qpn) == obj_id ||
629 get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ,
630 sq->base.mqp.qpn) == obj_id ||
631 get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR,
632 rq->tirn) == obj_id ||
633 get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS,
634 sq->tisn) == obj_id);
635 }
636
637 if (qp_type == MLX5_IB_QPT_DCT)
638 return get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT,
639 qp->dct.mdct.mqp.qpn) == obj_id;
640
641 return get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
642 qp->ibqp.qp_num) == obj_id;
643 }
644
645 case UVERBS_OBJECT_WQ:
646 return get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
647 to_mrwq(uobj->object)->core_qp.qpn) ==
648 obj_id;
649
650 case UVERBS_OBJECT_RWQ_IND_TBL:
651 return get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT,
652 to_mrwq_ind_table(uobj->object)->rqtn) ==
653 obj_id;
654
655 case MLX5_IB_OBJECT_DEVX_OBJ:
656 return ((struct devx_obj *)uobj->object)->obj_id == obj_id;
657
658 default:
659 return false;
660 }
661 }
662
663 static void devx_set_umem_valid(const void *in)
664 {
665 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
666
667 switch (opcode) {
668 case MLX5_CMD_OP_CREATE_MKEY:
669 MLX5_SET(create_mkey_in, in, mkey_umem_valid, 1);
670 break;
671 case MLX5_CMD_OP_CREATE_CQ:
672 {
673 void *cqc;
674
675 MLX5_SET(create_cq_in, in, cq_umem_valid, 1);
676 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
677 MLX5_SET(cqc, cqc, dbr_umem_valid, 1);
678 break;
679 }
680 case MLX5_CMD_OP_CREATE_QP:
681 {
682 void *qpc;
683
684 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
685 MLX5_SET(qpc, qpc, dbr_umem_valid, 1);
686 MLX5_SET(create_qp_in, in, wq_umem_valid, 1);
687 break;
688 }
689
690 case MLX5_CMD_OP_CREATE_RQ:
691 {
692 void *rqc, *wq;
693
694 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
695 wq = MLX5_ADDR_OF(rqc, rqc, wq);
696 MLX5_SET(wq, wq, dbr_umem_valid, 1);
697 MLX5_SET(wq, wq, wq_umem_valid, 1);
698 break;
699 }
700
701 case MLX5_CMD_OP_CREATE_SQ:
702 {
703 void *sqc, *wq;
704
705 sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
706 wq = MLX5_ADDR_OF(sqc, sqc, wq);
707 MLX5_SET(wq, wq, dbr_umem_valid, 1);
708 MLX5_SET(wq, wq, wq_umem_valid, 1);
709 break;
710 }
711
712 case MLX5_CMD_OP_MODIFY_CQ:
713 MLX5_SET(modify_cq_in, in, cq_umem_valid, 1);
714 break;
715
716 case MLX5_CMD_OP_CREATE_RMP:
717 {
718 void *rmpc, *wq;
719
720 rmpc = MLX5_ADDR_OF(create_rmp_in, in, ctx);
721 wq = MLX5_ADDR_OF(rmpc, rmpc, wq);
722 MLX5_SET(wq, wq, dbr_umem_valid, 1);
723 MLX5_SET(wq, wq, wq_umem_valid, 1);
724 break;
725 }
726
727 case MLX5_CMD_OP_CREATE_XRQ:
728 {
729 void *xrqc, *wq;
730
731 xrqc = MLX5_ADDR_OF(create_xrq_in, in, xrq_context);
732 wq = MLX5_ADDR_OF(xrqc, xrqc, wq);
733 MLX5_SET(wq, wq, dbr_umem_valid, 1);
734 MLX5_SET(wq, wq, wq_umem_valid, 1);
735 break;
736 }
737
738 case MLX5_CMD_OP_CREATE_XRC_SRQ:
739 {
740 void *xrc_srqc;
741
742 MLX5_SET(create_xrc_srq_in, in, xrc_srq_umem_valid, 1);
743 xrc_srqc = MLX5_ADDR_OF(create_xrc_srq_in, in,
744 xrc_srq_context_entry);
745 MLX5_SET(xrc_srqc, xrc_srqc, dbr_umem_valid, 1);
746 break;
747 }
748
749 default:
750 return;
751 }
752 }
753
754 static bool devx_is_obj_create_cmd(const void *in, u16 *opcode)
755 {
756 *opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
757
758 switch (*opcode) {
759 case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
760 case MLX5_CMD_OP_CREATE_MKEY:
761 case MLX5_CMD_OP_CREATE_CQ:
762 case MLX5_CMD_OP_ALLOC_PD:
763 case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN:
764 case MLX5_CMD_OP_CREATE_RMP:
765 case MLX5_CMD_OP_CREATE_SQ:
766 case MLX5_CMD_OP_CREATE_RQ:
767 case MLX5_CMD_OP_CREATE_RQT:
768 case MLX5_CMD_OP_CREATE_TIR:
769 case MLX5_CMD_OP_CREATE_TIS:
770 case MLX5_CMD_OP_ALLOC_Q_COUNTER:
771 case MLX5_CMD_OP_CREATE_FLOW_TABLE:
772 case MLX5_CMD_OP_CREATE_FLOW_GROUP:
773 case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
774 case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT:
775 case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT:
776 case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT:
777 case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
778 case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
779 case MLX5_CMD_OP_CREATE_QP:
780 case MLX5_CMD_OP_CREATE_SRQ:
781 case MLX5_CMD_OP_CREATE_XRC_SRQ:
782 case MLX5_CMD_OP_CREATE_DCT:
783 case MLX5_CMD_OP_CREATE_XRQ:
784 case MLX5_CMD_OP_ATTACH_TO_MCG:
785 case MLX5_CMD_OP_ALLOC_XRCD:
786 return true;
787 case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
788 {
789 u16 op_mod = MLX5_GET(set_fte_in, in, op_mod);
790 if (op_mod == 0)
791 return true;
792 return false;
793 }
794 case MLX5_CMD_OP_CREATE_PSV:
795 {
796 u8 num_psv = MLX5_GET(create_psv_in, in, num_psv);
797
798 if (num_psv == 1)
799 return true;
800 return false;
801 }
802 default:
803 return false;
804 }
805 }
806
807 static bool devx_is_obj_modify_cmd(const void *in)
808 {
809 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
810
811 switch (opcode) {
812 case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT:
813 case MLX5_CMD_OP_MODIFY_CQ:
814 case MLX5_CMD_OP_MODIFY_RMP:
815 case MLX5_CMD_OP_MODIFY_SQ:
816 case MLX5_CMD_OP_MODIFY_RQ:
817 case MLX5_CMD_OP_MODIFY_RQT:
818 case MLX5_CMD_OP_MODIFY_TIR:
819 case MLX5_CMD_OP_MODIFY_TIS:
820 case MLX5_CMD_OP_MODIFY_FLOW_TABLE:
821 case MLX5_CMD_OP_MODIFY_SCHEDULING_ELEMENT:
822 case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
823 case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
824 case MLX5_CMD_OP_RST2INIT_QP:
825 case MLX5_CMD_OP_INIT2RTR_QP:
826 case MLX5_CMD_OP_INIT2INIT_QP:
827 case MLX5_CMD_OP_RTR2RTS_QP:
828 case MLX5_CMD_OP_RTS2RTS_QP:
829 case MLX5_CMD_OP_SQERR2RTS_QP:
830 case MLX5_CMD_OP_2ERR_QP:
831 case MLX5_CMD_OP_2RST_QP:
832 case MLX5_CMD_OP_ARM_XRC_SRQ:
833 case MLX5_CMD_OP_ARM_RQ:
834 case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION:
835 case MLX5_CMD_OP_ARM_XRQ:
836 case MLX5_CMD_OP_SET_XRQ_DC_PARAMS_ENTRY:
837 case MLX5_CMD_OP_RELEASE_XRQ_ERROR:
838 case MLX5_CMD_OP_MODIFY_XRQ:
839 return true;
840 case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
841 {
842 u16 op_mod = MLX5_GET(set_fte_in, in, op_mod);
843
844 if (op_mod == 1)
845 return true;
846 return false;
847 }
848 default:
849 return false;
850 }
851 }
852
853 static bool devx_is_obj_query_cmd(const void *in)
854 {
855 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
856
857 switch (opcode) {
858 case MLX5_CMD_OP_QUERY_GENERAL_OBJECT:
859 case MLX5_CMD_OP_QUERY_MKEY:
860 case MLX5_CMD_OP_QUERY_CQ:
861 case MLX5_CMD_OP_QUERY_RMP:
862 case MLX5_CMD_OP_QUERY_SQ:
863 case MLX5_CMD_OP_QUERY_RQ:
864 case MLX5_CMD_OP_QUERY_RQT:
865 case MLX5_CMD_OP_QUERY_TIR:
866 case MLX5_CMD_OP_QUERY_TIS:
867 case MLX5_CMD_OP_QUERY_Q_COUNTER:
868 case MLX5_CMD_OP_QUERY_FLOW_TABLE:
869 case MLX5_CMD_OP_QUERY_FLOW_GROUP:
870 case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY:
871 case MLX5_CMD_OP_QUERY_FLOW_COUNTER:
872 case MLX5_CMD_OP_QUERY_MODIFY_HEADER_CONTEXT:
873 case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT:
874 case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY:
875 case MLX5_CMD_OP_QUERY_QP:
876 case MLX5_CMD_OP_QUERY_SRQ:
877 case MLX5_CMD_OP_QUERY_XRC_SRQ:
878 case MLX5_CMD_OP_QUERY_DCT:
879 case MLX5_CMD_OP_QUERY_XRQ:
880 case MLX5_CMD_OP_QUERY_XRQ_DC_PARAMS_ENTRY:
881 case MLX5_CMD_OP_QUERY_XRQ_ERROR_PARAMS:
882 case MLX5_CMD_OP_QUERY_PACKET_REFORMAT_CONTEXT:
883 return true;
884 default:
885 return false;
886 }
887 }
888
889 static bool devx_is_whitelist_cmd(void *in)
890 {
891 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
892
893 switch (opcode) {
894 case MLX5_CMD_OP_QUERY_HCA_CAP:
895 case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT:
896 case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT:
897 return true;
898 default:
899 return false;
900 }
901 }
902
903 static int devx_get_uid(struct mlx5_ib_ucontext *c, void *cmd_in)
904 {
905 if (devx_is_whitelist_cmd(cmd_in)) {
906 struct mlx5_ib_dev *dev;
907
908 if (c->devx_uid)
909 return c->devx_uid;
910
911 dev = to_mdev(c->ibucontext.device);
912 if (dev->devx_whitelist_uid)
913 return dev->devx_whitelist_uid;
914
915 return -EOPNOTSUPP;
916 }
917
918 if (!c->devx_uid)
919 return -EINVAL;
920
921 return c->devx_uid;
922 }
923
924 static bool devx_is_general_cmd(void *in, struct mlx5_ib_dev *dev)
925 {
926 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
927
928 /* Pass all cmds for vhca_tunnel as general, tracking is done in FW */
929 if ((MLX5_CAP_GEN_64(dev->mdev, vhca_tunnel_commands) &&
930 MLX5_GET(general_obj_in_cmd_hdr, in, vhca_tunnel_id)) ||
931 (opcode >= MLX5_CMD_OP_GENERAL_START &&
932 opcode < MLX5_CMD_OP_GENERAL_END))
933 return true;
934
935 switch (opcode) {
936 case MLX5_CMD_OP_QUERY_HCA_CAP:
937 case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT:
938 case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT:
939 case MLX5_CMD_OP_QUERY_VPORT_STATE:
940 case MLX5_CMD_OP_QUERY_ADAPTER:
941 case MLX5_CMD_OP_QUERY_ISSI:
942 case MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT:
943 case MLX5_CMD_OP_QUERY_ROCE_ADDRESS:
944 case MLX5_CMD_OP_QUERY_VNIC_ENV:
945 case MLX5_CMD_OP_QUERY_VPORT_COUNTER:
946 case MLX5_CMD_OP_GET_DROPPED_PACKET_LOG:
947 case MLX5_CMD_OP_NOP:
948 case MLX5_CMD_OP_QUERY_CONG_STATUS:
949 case MLX5_CMD_OP_QUERY_CONG_PARAMS:
950 case MLX5_CMD_OP_QUERY_CONG_STATISTICS:
951 case MLX5_CMD_OP_QUERY_LAG:
952 return true;
953 default:
954 return false;
955 }
956 }
957
958 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_QUERY_EQN)(
959 struct uverbs_attr_bundle *attrs)
960 {
961 struct mlx5_ib_ucontext *c;
962 struct mlx5_ib_dev *dev;
963 int user_vector;
964 int dev_eqn;
965 unsigned int irqn;
966 int err;
967
968 if (uverbs_copy_from(&user_vector, attrs,
969 MLX5_IB_ATTR_DEVX_QUERY_EQN_USER_VEC))
970 return -EFAULT;
971
972 c = devx_ufile2uctx(attrs);
973 if (IS_ERR(c))
974 return PTR_ERR(c);
975 dev = to_mdev(c->ibucontext.device);
976
977 err = mlx5_vector2eqn(dev->mdev, user_vector, &dev_eqn, &irqn);
978 if (err < 0)
979 return err;
980
981 if (uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_QUERY_EQN_DEV_EQN,
982 &dev_eqn, sizeof(dev_eqn)))
983 return -EFAULT;
984
985 return 0;
986 }
987
988 /*
989 *Security note:
990 * The hardware protection mechanism works like this: Each device object that
991 * is subject to UAR doorbells (QP/SQ/CQ) gets a UAR ID (called uar_page in
992 * the device specification manual) upon its creation. Then upon doorbell,
993 * hardware fetches the object context for which the doorbell was rang, and
994 * validates that the UAR through which the DB was rang matches the UAR ID
995 * of the object.
996 * If no match the doorbell is silently ignored by the hardware. Of course,
997 * the user cannot ring a doorbell on a UAR that was not mapped to it.
998 * Now in devx, as the devx kernel does not manipulate the QP/SQ/CQ command
999 * mailboxes (except tagging them with UID), we expose to the user its UAR
1000 * ID, so it can embed it in these objects in the expected specification
1001 * format. So the only thing the user can do is hurt itself by creating a
1002 * QP/SQ/CQ with a UAR ID other than his, and then in this case other users
1003 * may ring a doorbell on its objects.
1004 * The consequence of that will be that another user can schedule a QP/SQ
1005 * of the buggy user for execution (just insert it to the hardware schedule
1006 * queue or arm its CQ for event generation), no further harm is expected.
1007 */
1008 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_QUERY_UAR)(
1009 struct uverbs_attr_bundle *attrs)
1010 {
1011 struct mlx5_ib_ucontext *c;
1012 struct mlx5_ib_dev *dev;
1013 u32 user_idx;
1014 s32 dev_idx;
1015
1016 c = devx_ufile2uctx(attrs);
1017 if (IS_ERR(c))
1018 return PTR_ERR(c);
1019 dev = to_mdev(c->ibucontext.device);
1020
1021 if (uverbs_copy_from(&user_idx, attrs,
1022 MLX5_IB_ATTR_DEVX_QUERY_UAR_USER_IDX))
1023 return -EFAULT;
1024
1025 dev_idx = bfregn_to_uar_index(dev, &c->bfregi, user_idx, true);
1026 if (dev_idx < 0)
1027 return dev_idx;
1028
1029 if (uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_QUERY_UAR_DEV_IDX,
1030 &dev_idx, sizeof(dev_idx)))
1031 return -EFAULT;
1032
1033 return 0;
1034 }
1035
1036 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OTHER)(
1037 struct uverbs_attr_bundle *attrs)
1038 {
1039 struct mlx5_ib_ucontext *c;
1040 struct mlx5_ib_dev *dev;
1041 void *cmd_in = uverbs_attr_get_alloced_ptr(
1042 attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_IN);
1043 int cmd_out_len = uverbs_attr_get_len(attrs,
1044 MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT);
1045 void *cmd_out;
1046 int err;
1047 int uid;
1048
1049 c = devx_ufile2uctx(attrs);
1050 if (IS_ERR(c))
1051 return PTR_ERR(c);
1052 dev = to_mdev(c->ibucontext.device);
1053
1054 uid = devx_get_uid(c, cmd_in);
1055 if (uid < 0)
1056 return uid;
1057
1058 /* Only white list of some general HCA commands are allowed for this method. */
1059 if (!devx_is_general_cmd(cmd_in, dev))
1060 return -EINVAL;
1061
1062 cmd_out = uverbs_zalloc(attrs, cmd_out_len);
1063 if (IS_ERR(cmd_out))
1064 return PTR_ERR(cmd_out);
1065
1066 MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1067 err = mlx5_cmd_exec(dev->mdev, cmd_in,
1068 uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_IN),
1069 cmd_out, cmd_out_len);
1070 if (err)
1071 return err;
1072
1073 return uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT, cmd_out,
1074 cmd_out_len);
1075 }
1076
1077 static void devx_obj_build_destroy_cmd(void *in, void *out, void *din,
1078 u32 *dinlen,
1079 u32 *obj_id)
1080 {
1081 u16 obj_type = MLX5_GET(general_obj_in_cmd_hdr, in, obj_type);
1082 u16 uid = MLX5_GET(general_obj_in_cmd_hdr, in, uid);
1083
1084 *obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
1085 *dinlen = MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr);
1086
1087 MLX5_SET(general_obj_in_cmd_hdr, din, obj_id, *obj_id);
1088 MLX5_SET(general_obj_in_cmd_hdr, din, uid, uid);
1089
1090 switch (MLX5_GET(general_obj_in_cmd_hdr, in, opcode)) {
1091 case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
1092 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);
1093 MLX5_SET(general_obj_in_cmd_hdr, din, obj_type, obj_type);
1094 break;
1095
1096 case MLX5_CMD_OP_CREATE_UMEM:
1097 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1098 MLX5_CMD_OP_DESTROY_UMEM);
1099 break;
1100 case MLX5_CMD_OP_CREATE_MKEY:
1101 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_MKEY);
1102 break;
1103 case MLX5_CMD_OP_CREATE_CQ:
1104 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_CQ);
1105 break;
1106 case MLX5_CMD_OP_ALLOC_PD:
1107 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DEALLOC_PD);
1108 break;
1109 case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN:
1110 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1111 MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN);
1112 break;
1113 case MLX5_CMD_OP_CREATE_RMP:
1114 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_RMP);
1115 break;
1116 case MLX5_CMD_OP_CREATE_SQ:
1117 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_SQ);
1118 break;
1119 case MLX5_CMD_OP_CREATE_RQ:
1120 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_RQ);
1121 break;
1122 case MLX5_CMD_OP_CREATE_RQT:
1123 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_RQT);
1124 break;
1125 case MLX5_CMD_OP_CREATE_TIR:
1126 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_TIR);
1127 break;
1128 case MLX5_CMD_OP_CREATE_TIS:
1129 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_TIS);
1130 break;
1131 case MLX5_CMD_OP_ALLOC_Q_COUNTER:
1132 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1133 MLX5_CMD_OP_DEALLOC_Q_COUNTER);
1134 break;
1135 case MLX5_CMD_OP_CREATE_FLOW_TABLE:
1136 *dinlen = MLX5_ST_SZ_BYTES(destroy_flow_table_in);
1137 *obj_id = MLX5_GET(create_flow_table_out, out, table_id);
1138 MLX5_SET(destroy_flow_table_in, din, other_vport,
1139 MLX5_GET(create_flow_table_in, in, other_vport));
1140 MLX5_SET(destroy_flow_table_in, din, vport_number,
1141 MLX5_GET(create_flow_table_in, in, vport_number));
1142 MLX5_SET(destroy_flow_table_in, din, table_type,
1143 MLX5_GET(create_flow_table_in, in, table_type));
1144 MLX5_SET(destroy_flow_table_in, din, table_id, *obj_id);
1145 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1146 MLX5_CMD_OP_DESTROY_FLOW_TABLE);
1147 break;
1148 case MLX5_CMD_OP_CREATE_FLOW_GROUP:
1149 *dinlen = MLX5_ST_SZ_BYTES(destroy_flow_group_in);
1150 *obj_id = MLX5_GET(create_flow_group_out, out, group_id);
1151 MLX5_SET(destroy_flow_group_in, din, other_vport,
1152 MLX5_GET(create_flow_group_in, in, other_vport));
1153 MLX5_SET(destroy_flow_group_in, din, vport_number,
1154 MLX5_GET(create_flow_group_in, in, vport_number));
1155 MLX5_SET(destroy_flow_group_in, din, table_type,
1156 MLX5_GET(create_flow_group_in, in, table_type));
1157 MLX5_SET(destroy_flow_group_in, din, table_id,
1158 MLX5_GET(create_flow_group_in, in, table_id));
1159 MLX5_SET(destroy_flow_group_in, din, group_id, *obj_id);
1160 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1161 MLX5_CMD_OP_DESTROY_FLOW_GROUP);
1162 break;
1163 case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
1164 *dinlen = MLX5_ST_SZ_BYTES(delete_fte_in);
1165 *obj_id = MLX5_GET(set_fte_in, in, flow_index);
1166 MLX5_SET(delete_fte_in, din, other_vport,
1167 MLX5_GET(set_fte_in, in, other_vport));
1168 MLX5_SET(delete_fte_in, din, vport_number,
1169 MLX5_GET(set_fte_in, in, vport_number));
1170 MLX5_SET(delete_fte_in, din, table_type,
1171 MLX5_GET(set_fte_in, in, table_type));
1172 MLX5_SET(delete_fte_in, din, table_id,
1173 MLX5_GET(set_fte_in, in, table_id));
1174 MLX5_SET(delete_fte_in, din, flow_index, *obj_id);
1175 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1176 MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY);
1177 break;
1178 case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
1179 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1180 MLX5_CMD_OP_DEALLOC_FLOW_COUNTER);
1181 break;
1182 case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT:
1183 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1184 MLX5_CMD_OP_DEALLOC_PACKET_REFORMAT_CONTEXT);
1185 break;
1186 case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT:
1187 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1188 MLX5_CMD_OP_DEALLOC_MODIFY_HEADER_CONTEXT);
1189 break;
1190 case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT:
1191 *dinlen = MLX5_ST_SZ_BYTES(destroy_scheduling_element_in);
1192 *obj_id = MLX5_GET(create_scheduling_element_out, out,
1193 scheduling_element_id);
1194 MLX5_SET(destroy_scheduling_element_in, din,
1195 scheduling_hierarchy,
1196 MLX5_GET(create_scheduling_element_in, in,
1197 scheduling_hierarchy));
1198 MLX5_SET(destroy_scheduling_element_in, din,
1199 scheduling_element_id, *obj_id);
1200 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1201 MLX5_CMD_OP_DESTROY_SCHEDULING_ELEMENT);
1202 break;
1203 case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
1204 *dinlen = MLX5_ST_SZ_BYTES(delete_vxlan_udp_dport_in);
1205 *obj_id = MLX5_GET(add_vxlan_udp_dport_in, in, vxlan_udp_port);
1206 MLX5_SET(delete_vxlan_udp_dport_in, din, vxlan_udp_port, *obj_id);
1207 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1208 MLX5_CMD_OP_DELETE_VXLAN_UDP_DPORT);
1209 break;
1210 case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
1211 *dinlen = MLX5_ST_SZ_BYTES(delete_l2_table_entry_in);
1212 *obj_id = MLX5_GET(set_l2_table_entry_in, in, table_index);
1213 MLX5_SET(delete_l2_table_entry_in, din, table_index, *obj_id);
1214 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1215 MLX5_CMD_OP_DELETE_L2_TABLE_ENTRY);
1216 break;
1217 case MLX5_CMD_OP_CREATE_QP:
1218 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_QP);
1219 break;
1220 case MLX5_CMD_OP_CREATE_SRQ:
1221 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_SRQ);
1222 break;
1223 case MLX5_CMD_OP_CREATE_XRC_SRQ:
1224 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1225 MLX5_CMD_OP_DESTROY_XRC_SRQ);
1226 break;
1227 case MLX5_CMD_OP_CREATE_DCT:
1228 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_DCT);
1229 break;
1230 case MLX5_CMD_OP_CREATE_XRQ:
1231 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_XRQ);
1232 break;
1233 case MLX5_CMD_OP_ATTACH_TO_MCG:
1234 *dinlen = MLX5_ST_SZ_BYTES(detach_from_mcg_in);
1235 MLX5_SET(detach_from_mcg_in, din, qpn,
1236 MLX5_GET(attach_to_mcg_in, in, qpn));
1237 memcpy(MLX5_ADDR_OF(detach_from_mcg_in, din, multicast_gid),
1238 MLX5_ADDR_OF(attach_to_mcg_in, in, multicast_gid),
1239 MLX5_FLD_SZ_BYTES(attach_to_mcg_in, multicast_gid));
1240 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DETACH_FROM_MCG);
1241 break;
1242 case MLX5_CMD_OP_ALLOC_XRCD:
1243 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DEALLOC_XRCD);
1244 break;
1245 case MLX5_CMD_OP_CREATE_PSV:
1246 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1247 MLX5_CMD_OP_DESTROY_PSV);
1248 MLX5_SET(destroy_psv_in, din, psvn,
1249 MLX5_GET(create_psv_out, out, psv0_index));
1250 break;
1251 default:
1252 /* The entry must match to one of the devx_is_obj_create_cmd */
1253 WARN_ON(true);
1254 break;
1255 }
1256 }
1257
1258 static int devx_handle_mkey_indirect(struct devx_obj *obj,
1259 struct mlx5_ib_dev *dev,
1260 void *in, void *out)
1261 {
1262 struct mlx5_ib_devx_mr *devx_mr = &obj->devx_mr;
1263 struct mlx5_core_mkey *mkey;
1264 void *mkc;
1265 u8 key;
1266
1267 mkey = &devx_mr->mmkey;
1268 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1269 key = MLX5_GET(mkc, mkc, mkey_7_0);
1270 mkey->key = mlx5_idx_to_mkey(
1271 MLX5_GET(create_mkey_out, out, mkey_index)) | key;
1272 mkey->type = MLX5_MKEY_INDIRECT_DEVX;
1273 mkey->iova = MLX5_GET64(mkc, mkc, start_addr);
1274 mkey->size = MLX5_GET64(mkc, mkc, len);
1275 mkey->pd = MLX5_GET(mkc, mkc, pd);
1276 devx_mr->ndescs = MLX5_GET(mkc, mkc, translations_octword_size);
1277
1278 return xa_err(xa_store(&dev->odp_mkeys, mlx5_base_mkey(mkey->key), mkey,
1279 GFP_KERNEL));
1280 }
1281
1282 static int devx_handle_mkey_create(struct mlx5_ib_dev *dev,
1283 struct devx_obj *obj,
1284 void *in, int in_len)
1285 {
1286 int min_len = MLX5_BYTE_OFF(create_mkey_in, memory_key_mkey_entry) +
1287 MLX5_FLD_SZ_BYTES(create_mkey_in,
1288 memory_key_mkey_entry);
1289 void *mkc;
1290 u8 access_mode;
1291
1292 if (in_len < min_len)
1293 return -EINVAL;
1294
1295 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1296
1297 access_mode = MLX5_GET(mkc, mkc, access_mode_1_0);
1298 access_mode |= MLX5_GET(mkc, mkc, access_mode_4_2) << 2;
1299
1300 if (access_mode == MLX5_MKC_ACCESS_MODE_KLMS ||
1301 access_mode == MLX5_MKC_ACCESS_MODE_KSM) {
1302 if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING))
1303 obj->flags |= DEVX_OBJ_FLAGS_INDIRECT_MKEY;
1304 return 0;
1305 }
1306
1307 MLX5_SET(create_mkey_in, in, mkey_umem_valid, 1);
1308 return 0;
1309 }
1310
1311 static void devx_cleanup_subscription(struct mlx5_ib_dev *dev,
1312 struct devx_event_subscription *sub)
1313 {
1314 struct devx_event *event;
1315 struct devx_obj_event *xa_val_level2;
1316
1317 if (sub->is_cleaned)
1318 return;
1319
1320 sub->is_cleaned = 1;
1321 list_del_rcu(&sub->xa_list);
1322
1323 if (list_empty(&sub->obj_list))
1324 return;
1325
1326 list_del_rcu(&sub->obj_list);
1327 /* check whether key level 1 for this obj_sub_list is empty */
1328 event = xa_load(&dev->devx_event_table.event_xa,
1329 sub->xa_key_level1);
1330 WARN_ON(!event);
1331
1332 xa_val_level2 = xa_load(&event->object_ids, sub->xa_key_level2);
1333 if (list_empty(&xa_val_level2->obj_sub_list)) {
1334 xa_erase(&event->object_ids,
1335 sub->xa_key_level2);
1336 kfree_rcu(xa_val_level2, rcu);
1337 }
1338 }
1339
1340 static int devx_obj_cleanup(struct ib_uobject *uobject,
1341 enum rdma_remove_reason why,
1342 struct uverbs_attr_bundle *attrs)
1343 {
1344 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
1345 struct mlx5_devx_event_table *devx_event_table;
1346 struct devx_obj *obj = uobject->object;
1347 struct devx_event_subscription *sub_entry, *tmp;
1348 struct mlx5_ib_dev *dev;
1349 int ret;
1350
1351 dev = mlx5_udata_to_mdev(&attrs->driver_udata);
1352 if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY) {
1353 /*
1354 * The pagefault_single_data_segment() does commands against
1355 * the mmkey, we must wait for that to stop before freeing the
1356 * mkey, as another allocation could get the same mkey #.
1357 */
1358 xa_erase(&obj->ib_dev->odp_mkeys,
1359 mlx5_base_mkey(obj->devx_mr.mmkey.key));
1360 synchronize_srcu(&dev->odp_srcu);
1361 }
1362
1363 if (obj->flags & DEVX_OBJ_FLAGS_DCT)
1364 ret = mlx5_core_destroy_dct(obj->ib_dev->mdev, &obj->core_dct);
1365 else if (obj->flags & DEVX_OBJ_FLAGS_CQ)
1366 ret = mlx5_core_destroy_cq(obj->ib_dev->mdev, &obj->core_cq);
1367 else
1368 ret = mlx5_cmd_exec(obj->ib_dev->mdev, obj->dinbox,
1369 obj->dinlen, out, sizeof(out));
1370 if (ib_is_destroy_retryable(ret, why, uobject))
1371 return ret;
1372
1373 devx_event_table = &dev->devx_event_table;
1374
1375 mutex_lock(&devx_event_table->event_xa_lock);
1376 list_for_each_entry_safe(sub_entry, tmp, &obj->event_sub, obj_list)
1377 devx_cleanup_subscription(dev, sub_entry);
1378 mutex_unlock(&devx_event_table->event_xa_lock);
1379
1380 kfree(obj);
1381 return ret;
1382 }
1383
1384 static void devx_cq_comp(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe)
1385 {
1386 struct devx_obj *obj = container_of(mcq, struct devx_obj, core_cq);
1387 struct mlx5_devx_event_table *table;
1388 struct devx_event *event;
1389 struct devx_obj_event *obj_event;
1390 u32 obj_id = mcq->cqn;
1391
1392 table = &obj->ib_dev->devx_event_table;
1393 rcu_read_lock();
1394 event = xa_load(&table->event_xa, MLX5_EVENT_TYPE_COMP);
1395 if (!event)
1396 goto out;
1397
1398 obj_event = xa_load(&event->object_ids, obj_id);
1399 if (!obj_event)
1400 goto out;
1401
1402 dispatch_event_fd(&obj_event->obj_sub_list, eqe);
1403 out:
1404 rcu_read_unlock();
1405 }
1406
1407 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)(
1408 struct uverbs_attr_bundle *attrs)
1409 {
1410 void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN);
1411 int cmd_out_len = uverbs_attr_get_len(attrs,
1412 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT);
1413 int cmd_in_len = uverbs_attr_get_len(attrs,
1414 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN);
1415 void *cmd_out;
1416 struct ib_uobject *uobj = uverbs_attr_get_uobject(
1417 attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_HANDLE);
1418 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1419 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1420 struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device);
1421 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
1422 struct devx_obj *obj;
1423 u16 obj_type = 0;
1424 int err;
1425 int uid;
1426 u32 obj_id;
1427 u16 opcode;
1428
1429 if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))
1430 return -EINVAL;
1431
1432 uid = devx_get_uid(c, cmd_in);
1433 if (uid < 0)
1434 return uid;
1435
1436 if (!devx_is_obj_create_cmd(cmd_in, &opcode))
1437 return -EINVAL;
1438
1439 cmd_out = uverbs_zalloc(attrs, cmd_out_len);
1440 if (IS_ERR(cmd_out))
1441 return PTR_ERR(cmd_out);
1442
1443 obj = kzalloc(sizeof(struct devx_obj), GFP_KERNEL);
1444 if (!obj)
1445 return -ENOMEM;
1446
1447 MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1448 if (opcode == MLX5_CMD_OP_CREATE_MKEY) {
1449 err = devx_handle_mkey_create(dev, obj, cmd_in, cmd_in_len);
1450 if (err)
1451 goto obj_free;
1452 } else {
1453 devx_set_umem_valid(cmd_in);
1454 }
1455
1456 if (opcode == MLX5_CMD_OP_CREATE_DCT) {
1457 obj->flags |= DEVX_OBJ_FLAGS_DCT;
1458 err = mlx5_core_create_dct(dev->mdev, &obj->core_dct,
1459 cmd_in, cmd_in_len,
1460 cmd_out, cmd_out_len);
1461 } else if (opcode == MLX5_CMD_OP_CREATE_CQ) {
1462 obj->flags |= DEVX_OBJ_FLAGS_CQ;
1463 obj->core_cq.comp = devx_cq_comp;
1464 err = mlx5_core_create_cq(dev->mdev, &obj->core_cq,
1465 cmd_in, cmd_in_len, cmd_out,
1466 cmd_out_len);
1467 } else {
1468 err = mlx5_cmd_exec(dev->mdev, cmd_in,
1469 cmd_in_len,
1470 cmd_out, cmd_out_len);
1471 }
1472
1473 if (err)
1474 goto obj_free;
1475
1476 if (opcode == MLX5_CMD_OP_ALLOC_FLOW_COUNTER) {
1477 u8 bulk = MLX5_GET(alloc_flow_counter_in,
1478 cmd_in,
1479 flow_counter_bulk);
1480 obj->flow_counter_bulk_size = 128UL * bulk;
1481 }
1482
1483 uobj->object = obj;
1484 INIT_LIST_HEAD(&obj->event_sub);
1485 obj->ib_dev = dev;
1486 devx_obj_build_destroy_cmd(cmd_in, cmd_out, obj->dinbox, &obj->dinlen,
1487 &obj_id);
1488 WARN_ON(obj->dinlen > MLX5_MAX_DESTROY_INBOX_SIZE_DW * sizeof(u32));
1489
1490 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT, cmd_out, cmd_out_len);
1491 if (err)
1492 goto obj_destroy;
1493
1494 if (opcode == MLX5_CMD_OP_CREATE_GENERAL_OBJECT)
1495 obj_type = MLX5_GET(general_obj_in_cmd_hdr, cmd_in, obj_type);
1496 obj->obj_id = get_enc_obj_id(opcode | obj_type << 16, obj_id);
1497
1498 if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY) {
1499 err = devx_handle_mkey_indirect(obj, dev, cmd_in, cmd_out);
1500 if (err)
1501 goto obj_destroy;
1502 }
1503 return 0;
1504
1505 obj_destroy:
1506 if (obj->flags & DEVX_OBJ_FLAGS_DCT)
1507 mlx5_core_destroy_dct(obj->ib_dev->mdev, &obj->core_dct);
1508 else if (obj->flags & DEVX_OBJ_FLAGS_CQ)
1509 mlx5_core_destroy_cq(obj->ib_dev->mdev, &obj->core_cq);
1510 else
1511 mlx5_cmd_exec(obj->ib_dev->mdev, obj->dinbox, obj->dinlen, out,
1512 sizeof(out));
1513 obj_free:
1514 kfree(obj);
1515 return err;
1516 }
1517
1518 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_MODIFY)(
1519 struct uverbs_attr_bundle *attrs)
1520 {
1521 void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN);
1522 int cmd_out_len = uverbs_attr_get_len(attrs,
1523 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT);
1524 struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs,
1525 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_HANDLE);
1526 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1527 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1528 struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device);
1529 void *cmd_out;
1530 int err;
1531 int uid;
1532
1533 if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))
1534 return -EINVAL;
1535
1536 uid = devx_get_uid(c, cmd_in);
1537 if (uid < 0)
1538 return uid;
1539
1540 if (!devx_is_obj_modify_cmd(cmd_in))
1541 return -EINVAL;
1542
1543 if (!devx_is_valid_obj_id(attrs, uobj, cmd_in))
1544 return -EINVAL;
1545
1546 cmd_out = uverbs_zalloc(attrs, cmd_out_len);
1547 if (IS_ERR(cmd_out))
1548 return PTR_ERR(cmd_out);
1549
1550 MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1551 devx_set_umem_valid(cmd_in);
1552
1553 err = mlx5_cmd_exec(mdev->mdev, cmd_in,
1554 uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN),
1555 cmd_out, cmd_out_len);
1556 if (err)
1557 return err;
1558
1559 return uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT,
1560 cmd_out, cmd_out_len);
1561 }
1562
1563 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_QUERY)(
1564 struct uverbs_attr_bundle *attrs)
1565 {
1566 void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN);
1567 int cmd_out_len = uverbs_attr_get_len(attrs,
1568 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT);
1569 struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs,
1570 MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE);
1571 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1572 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1573 void *cmd_out;
1574 int err;
1575 int uid;
1576 struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device);
1577
1578 if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))
1579 return -EINVAL;
1580
1581 uid = devx_get_uid(c, cmd_in);
1582 if (uid < 0)
1583 return uid;
1584
1585 if (!devx_is_obj_query_cmd(cmd_in))
1586 return -EINVAL;
1587
1588 if (!devx_is_valid_obj_id(attrs, uobj, cmd_in))
1589 return -EINVAL;
1590
1591 cmd_out = uverbs_zalloc(attrs, cmd_out_len);
1592 if (IS_ERR(cmd_out))
1593 return PTR_ERR(cmd_out);
1594
1595 MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1596 err = mlx5_cmd_exec(mdev->mdev, cmd_in,
1597 uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN),
1598 cmd_out, cmd_out_len);
1599 if (err)
1600 return err;
1601
1602 return uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT,
1603 cmd_out, cmd_out_len);
1604 }
1605
1606 struct devx_async_event_queue {
1607 spinlock_t lock;
1608 wait_queue_head_t poll_wait;
1609 struct list_head event_list;
1610 atomic_t bytes_in_use;
1611 u8 is_destroyed:1;
1612 };
1613
1614 struct devx_async_cmd_event_file {
1615 struct ib_uobject uobj;
1616 struct devx_async_event_queue ev_queue;
1617 struct mlx5_async_ctx async_ctx;
1618 };
1619
1620 static void devx_init_event_queue(struct devx_async_event_queue *ev_queue)
1621 {
1622 spin_lock_init(&ev_queue->lock);
1623 INIT_LIST_HEAD(&ev_queue->event_list);
1624 init_waitqueue_head(&ev_queue->poll_wait);
1625 atomic_set(&ev_queue->bytes_in_use, 0);
1626 ev_queue->is_destroyed = 0;
1627 }
1628
1629 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC)(
1630 struct uverbs_attr_bundle *attrs)
1631 {
1632 struct devx_async_cmd_event_file *ev_file;
1633
1634 struct ib_uobject *uobj = uverbs_attr_get_uobject(
1635 attrs, MLX5_IB_ATTR_DEVX_ASYNC_CMD_FD_ALLOC_HANDLE);
1636 struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata);
1637
1638 ev_file = container_of(uobj, struct devx_async_cmd_event_file,
1639 uobj);
1640 devx_init_event_queue(&ev_file->ev_queue);
1641 mlx5_cmd_init_async_ctx(mdev->mdev, &ev_file->async_ctx);
1642 return 0;
1643 }
1644
1645 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC)(
1646 struct uverbs_attr_bundle *attrs)
1647 {
1648 struct ib_uobject *uobj = uverbs_attr_get_uobject(
1649 attrs, MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_HANDLE);
1650 struct devx_async_event_file *ev_file;
1651 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1652 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1653 struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device);
1654 u32 flags;
1655 int err;
1656
1657 err = uverbs_get_flags32(&flags, attrs,
1658 MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_FLAGS,
1659 MLX5_IB_UAPI_DEVX_CR_EV_CH_FLAGS_OMIT_DATA);
1660
1661 if (err)
1662 return err;
1663
1664 ev_file = container_of(uobj, struct devx_async_event_file,
1665 uobj);
1666 spin_lock_init(&ev_file->lock);
1667 INIT_LIST_HEAD(&ev_file->event_list);
1668 init_waitqueue_head(&ev_file->poll_wait);
1669 if (flags & MLX5_IB_UAPI_DEVX_CR_EV_CH_FLAGS_OMIT_DATA)
1670 ev_file->omit_data = 1;
1671 INIT_LIST_HEAD(&ev_file->subscribed_events_list);
1672 ev_file->dev = dev;
1673 get_device(&dev->ib_dev.dev);
1674 return 0;
1675 }
1676
1677 static void devx_query_callback(int status, struct mlx5_async_work *context)
1678 {
1679 struct devx_async_data *async_data =
1680 container_of(context, struct devx_async_data, cb_work);
1681 struct devx_async_cmd_event_file *ev_file = async_data->ev_file;
1682 struct devx_async_event_queue *ev_queue = &ev_file->ev_queue;
1683 unsigned long flags;
1684
1685 /*
1686 * Note that if the struct devx_async_cmd_event_file uobj begins to be
1687 * destroyed it will block at mlx5_cmd_cleanup_async_ctx() until this
1688 * routine returns, ensuring that it always remains valid here.
1689 */
1690 spin_lock_irqsave(&ev_queue->lock, flags);
1691 list_add_tail(&async_data->list, &ev_queue->event_list);
1692 spin_unlock_irqrestore(&ev_queue->lock, flags);
1693
1694 wake_up_interruptible(&ev_queue->poll_wait);
1695 }
1696
1697 #define MAX_ASYNC_BYTES_IN_USE (1024 * 1024) /* 1MB */
1698
1699 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY)(
1700 struct uverbs_attr_bundle *attrs)
1701 {
1702 void *cmd_in = uverbs_attr_get_alloced_ptr(attrs,
1703 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_CMD_IN);
1704 struct ib_uobject *uobj = uverbs_attr_get_uobject(
1705 attrs,
1706 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_HANDLE);
1707 u16 cmd_out_len;
1708 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1709 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1710 struct ib_uobject *fd_uobj;
1711 int err;
1712 int uid;
1713 struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device);
1714 struct devx_async_cmd_event_file *ev_file;
1715 struct devx_async_data *async_data;
1716
1717 if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))
1718 return -EINVAL;
1719
1720 uid = devx_get_uid(c, cmd_in);
1721 if (uid < 0)
1722 return uid;
1723
1724 if (!devx_is_obj_query_cmd(cmd_in))
1725 return -EINVAL;
1726
1727 err = uverbs_get_const(&cmd_out_len, attrs,
1728 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_OUT_LEN);
1729 if (err)
1730 return err;
1731
1732 if (!devx_is_valid_obj_id(attrs, uobj, cmd_in))
1733 return -EINVAL;
1734
1735 fd_uobj = uverbs_attr_get_uobject(attrs,
1736 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_FD);
1737 if (IS_ERR(fd_uobj))
1738 return PTR_ERR(fd_uobj);
1739
1740 ev_file = container_of(fd_uobj, struct devx_async_cmd_event_file,
1741 uobj);
1742
1743 if (atomic_add_return(cmd_out_len, &ev_file->ev_queue.bytes_in_use) >
1744 MAX_ASYNC_BYTES_IN_USE) {
1745 atomic_sub(cmd_out_len, &ev_file->ev_queue.bytes_in_use);
1746 return -EAGAIN;
1747 }
1748
1749 async_data = kvzalloc(struct_size(async_data, hdr.out_data,
1750 cmd_out_len), GFP_KERNEL);
1751 if (!async_data) {
1752 err = -ENOMEM;
1753 goto sub_bytes;
1754 }
1755
1756 err = uverbs_copy_from(&async_data->hdr.wr_id, attrs,
1757 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_WR_ID);
1758 if (err)
1759 goto free_async;
1760
1761 async_data->cmd_out_len = cmd_out_len;
1762 async_data->mdev = mdev;
1763 async_data->ev_file = ev_file;
1764
1765 MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1766 err = mlx5_cmd_exec_cb(&ev_file->async_ctx, cmd_in,
1767 uverbs_attr_get_len(attrs,
1768 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_CMD_IN),
1769 async_data->hdr.out_data,
1770 async_data->cmd_out_len,
1771 devx_query_callback, &async_data->cb_work);
1772
1773 if (err)
1774 goto free_async;
1775
1776 return 0;
1777
1778 free_async:
1779 kvfree(async_data);
1780 sub_bytes:
1781 atomic_sub(cmd_out_len, &ev_file->ev_queue.bytes_in_use);
1782 return err;
1783 }
1784
1785 static void
1786 subscribe_event_xa_dealloc(struct mlx5_devx_event_table *devx_event_table,
1787 u32 key_level1,
1788 bool is_level2,
1789 u32 key_level2)
1790 {
1791 struct devx_event *event;
1792 struct devx_obj_event *xa_val_level2;
1793
1794 /* Level 1 is valid for future use, no need to free */
1795 if (!is_level2)
1796 return;
1797
1798 event = xa_load(&devx_event_table->event_xa, key_level1);
1799 WARN_ON(!event);
1800
1801 xa_val_level2 = xa_load(&event->object_ids,
1802 key_level2);
1803 if (list_empty(&xa_val_level2->obj_sub_list)) {
1804 xa_erase(&event->object_ids,
1805 key_level2);
1806 kfree_rcu(xa_val_level2, rcu);
1807 }
1808 }
1809
1810 static int
1811 subscribe_event_xa_alloc(struct mlx5_devx_event_table *devx_event_table,
1812 u32 key_level1,
1813 bool is_level2,
1814 u32 key_level2)
1815 {
1816 struct devx_obj_event *obj_event;
1817 struct devx_event *event;
1818 int err;
1819
1820 event = xa_load(&devx_event_table->event_xa, key_level1);
1821 if (!event) {
1822 event = kzalloc(sizeof(*event), GFP_KERNEL);
1823 if (!event)
1824 return -ENOMEM;
1825
1826 INIT_LIST_HEAD(&event->unaffiliated_list);
1827 xa_init(&event->object_ids);
1828
1829 err = xa_insert(&devx_event_table->event_xa,
1830 key_level1,
1831 event,
1832 GFP_KERNEL);
1833 if (err) {
1834 kfree(event);
1835 return err;
1836 }
1837 }
1838
1839 if (!is_level2)
1840 return 0;
1841
1842 obj_event = xa_load(&event->object_ids, key_level2);
1843 if (!obj_event) {
1844 obj_event = kzalloc(sizeof(*obj_event), GFP_KERNEL);
1845 if (!obj_event)
1846 /* Level1 is valid for future use, no need to free */
1847 return -ENOMEM;
1848
1849 err = xa_insert(&event->object_ids,
1850 key_level2,
1851 obj_event,
1852 GFP_KERNEL);
1853 if (err)
1854 return err;
1855 INIT_LIST_HEAD(&obj_event->obj_sub_list);
1856 }
1857
1858 return 0;
1859 }
1860
1861 static bool is_valid_events_legacy(int num_events, u16 *event_type_num_list,
1862 struct devx_obj *obj)
1863 {
1864 int i;
1865
1866 for (i = 0; i < num_events; i++) {
1867 if (obj) {
1868 if (!is_legacy_obj_event_num(event_type_num_list[i]))
1869 return false;
1870 } else if (!is_legacy_unaffiliated_event_num(
1871 event_type_num_list[i])) {
1872 return false;
1873 }
1874 }
1875
1876 return true;
1877 }
1878
1879 #define MAX_SUPP_EVENT_NUM 255
1880 static bool is_valid_events(struct mlx5_core_dev *dev,
1881 int num_events, u16 *event_type_num_list,
1882 struct devx_obj *obj)
1883 {
1884 __be64 *aff_events;
1885 __be64 *unaff_events;
1886 int mask_entry;
1887 int mask_bit;
1888 int i;
1889
1890 if (MLX5_CAP_GEN(dev, event_cap)) {
1891 aff_events = MLX5_CAP_DEV_EVENT(dev,
1892 user_affiliated_events);
1893 unaff_events = MLX5_CAP_DEV_EVENT(dev,
1894 user_unaffiliated_events);
1895 } else {
1896 return is_valid_events_legacy(num_events, event_type_num_list,
1897 obj);
1898 }
1899
1900 for (i = 0; i < num_events; i++) {
1901 if (event_type_num_list[i] > MAX_SUPP_EVENT_NUM)
1902 return false;
1903
1904 mask_entry = event_type_num_list[i] / 64;
1905 mask_bit = event_type_num_list[i] % 64;
1906
1907 if (obj) {
1908 /* CQ completion */
1909 if (event_type_num_list[i] == 0)
1910 continue;
1911
1912 if (!(be64_to_cpu(aff_events[mask_entry]) &
1913 (1ull << mask_bit)))
1914 return false;
1915
1916 continue;
1917 }
1918
1919 if (!(be64_to_cpu(unaff_events[mask_entry]) &
1920 (1ull << mask_bit)))
1921 return false;
1922 }
1923
1924 return true;
1925 }
1926
1927 #define MAX_NUM_EVENTS 16
1928 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)(
1929 struct uverbs_attr_bundle *attrs)
1930 {
1931 struct ib_uobject *devx_uobj = uverbs_attr_get_uobject(
1932 attrs,
1933 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_OBJ_HANDLE);
1934 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1935 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1936 struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device);
1937 struct ib_uobject *fd_uobj;
1938 struct devx_obj *obj = NULL;
1939 struct devx_async_event_file *ev_file;
1940 struct mlx5_devx_event_table *devx_event_table = &dev->devx_event_table;
1941 u16 *event_type_num_list;
1942 struct devx_event_subscription *event_sub, *tmp_sub;
1943 struct list_head sub_list;
1944 int redirect_fd;
1945 bool use_eventfd = false;
1946 int num_events;
1947 int num_alloc_xa_entries = 0;
1948 u16 obj_type = 0;
1949 u64 cookie = 0;
1950 u32 obj_id = 0;
1951 int err;
1952 int i;
1953
1954 if (!c->devx_uid)
1955 return -EINVAL;
1956
1957 if (!IS_ERR(devx_uobj)) {
1958 obj = (struct devx_obj *)devx_uobj->object;
1959 if (obj)
1960 obj_id = get_dec_obj_id(obj->obj_id);
1961 }
1962
1963 fd_uobj = uverbs_attr_get_uobject(attrs,
1964 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_HANDLE);
1965 if (IS_ERR(fd_uobj))
1966 return PTR_ERR(fd_uobj);
1967
1968 ev_file = container_of(fd_uobj, struct devx_async_event_file,
1969 uobj);
1970
1971 if (uverbs_attr_is_valid(attrs,
1972 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM)) {
1973 err = uverbs_copy_from(&redirect_fd, attrs,
1974 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM);
1975 if (err)
1976 return err;
1977
1978 use_eventfd = true;
1979 }
1980
1981 if (uverbs_attr_is_valid(attrs,
1982 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE)) {
1983 if (use_eventfd)
1984 return -EINVAL;
1985
1986 err = uverbs_copy_from(&cookie, attrs,
1987 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE);
1988 if (err)
1989 return err;
1990 }
1991
1992 num_events = uverbs_attr_ptr_get_array_size(
1993 attrs, MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST,
1994 sizeof(u16));
1995
1996 if (num_events < 0)
1997 return num_events;
1998
1999 if (num_events > MAX_NUM_EVENTS)
2000 return -EINVAL;
2001
2002 event_type_num_list = uverbs_attr_get_alloced_ptr(attrs,
2003 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST);
2004
2005 if (!is_valid_events(dev->mdev, num_events, event_type_num_list, obj))
2006 return -EINVAL;
2007
2008 INIT_LIST_HEAD(&sub_list);
2009
2010 /* Protect from concurrent subscriptions to same XA entries to allow
2011 * both to succeed
2012 */
2013 mutex_lock(&devx_event_table->event_xa_lock);
2014 for (i = 0; i < num_events; i++) {
2015 u32 key_level1;
2016
2017 if (obj)
2018 obj_type = get_dec_obj_type(obj,
2019 event_type_num_list[i]);
2020 key_level1 = event_type_num_list[i] | obj_type << 16;
2021
2022 err = subscribe_event_xa_alloc(devx_event_table,
2023 key_level1,
2024 obj,
2025 obj_id);
2026 if (err)
2027 goto err;
2028
2029 num_alloc_xa_entries++;
2030 event_sub = kzalloc(sizeof(*event_sub), GFP_KERNEL);
2031 if (!event_sub)
2032 goto err;
2033
2034 list_add_tail(&event_sub->event_list, &sub_list);
2035 uverbs_uobject_get(&ev_file->uobj);
2036 if (use_eventfd) {
2037 event_sub->eventfd =
2038 eventfd_ctx_fdget(redirect_fd);
2039
2040 if (IS_ERR(event_sub->eventfd)) {
2041 err = PTR_ERR(event_sub->eventfd);
2042 event_sub->eventfd = NULL;
2043 goto err;
2044 }
2045 }
2046
2047 event_sub->cookie = cookie;
2048 event_sub->ev_file = ev_file;
2049 /* May be needed upon cleanup the devx object/subscription */
2050 event_sub->xa_key_level1 = key_level1;
2051 event_sub->xa_key_level2 = obj_id;
2052 INIT_LIST_HEAD(&event_sub->obj_list);
2053 }
2054
2055 /* Once all the allocations and the XA data insertions were done we
2056 * can go ahead and add all the subscriptions to the relevant lists
2057 * without concern of a failure.
2058 */
2059 list_for_each_entry_safe(event_sub, tmp_sub, &sub_list, event_list) {
2060 struct devx_event *event;
2061 struct devx_obj_event *obj_event;
2062
2063 list_del_init(&event_sub->event_list);
2064
2065 spin_lock_irq(&ev_file->lock);
2066 list_add_tail_rcu(&event_sub->file_list,
2067 &ev_file->subscribed_events_list);
2068 spin_unlock_irq(&ev_file->lock);
2069
2070 event = xa_load(&devx_event_table->event_xa,
2071 event_sub->xa_key_level1);
2072 WARN_ON(!event);
2073
2074 if (!obj) {
2075 list_add_tail_rcu(&event_sub->xa_list,
2076 &event->unaffiliated_list);
2077 continue;
2078 }
2079
2080 obj_event = xa_load(&event->object_ids, obj_id);
2081 WARN_ON(!obj_event);
2082 list_add_tail_rcu(&event_sub->xa_list,
2083 &obj_event->obj_sub_list);
2084 list_add_tail_rcu(&event_sub->obj_list,
2085 &obj->event_sub);
2086 }
2087
2088 mutex_unlock(&devx_event_table->event_xa_lock);
2089 return 0;
2090
2091 err:
2092 list_for_each_entry_safe(event_sub, tmp_sub, &sub_list, event_list) {
2093 list_del(&event_sub->event_list);
2094
2095 subscribe_event_xa_dealloc(devx_event_table,
2096 event_sub->xa_key_level1,
2097 obj,
2098 obj_id);
2099
2100 if (event_sub->eventfd)
2101 eventfd_ctx_put(event_sub->eventfd);
2102 uverbs_uobject_put(&event_sub->ev_file->uobj);
2103 kfree(event_sub);
2104 }
2105
2106 mutex_unlock(&devx_event_table->event_xa_lock);
2107 return err;
2108 }
2109
2110 static int devx_umem_get(struct mlx5_ib_dev *dev, struct ib_ucontext *ucontext,
2111 struct uverbs_attr_bundle *attrs,
2112 struct devx_umem *obj)
2113 {
2114 u64 addr;
2115 size_t size;
2116 u32 access;
2117 int npages;
2118 int err;
2119 u32 page_mask;
2120
2121 if (uverbs_copy_from(&addr, attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_ADDR) ||
2122 uverbs_copy_from(&size, attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_LEN))
2123 return -EFAULT;
2124
2125 err = uverbs_get_flags32(&access, attrs,
2126 MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS,
2127 IB_ACCESS_LOCAL_WRITE |
2128 IB_ACCESS_REMOTE_WRITE |
2129 IB_ACCESS_REMOTE_READ);
2130 if (err)
2131 return err;
2132
2133 err = ib_check_mr_access(access);
2134 if (err)
2135 return err;
2136
2137 obj->umem = ib_umem_get(&dev->ib_dev, addr, size, access);
2138 if (IS_ERR(obj->umem))
2139 return PTR_ERR(obj->umem);
2140
2141 mlx5_ib_cont_pages(obj->umem, obj->umem->address,
2142 MLX5_MKEY_PAGE_SHIFT_MASK, &npages,
2143 &obj->page_shift, &obj->ncont, NULL);
2144
2145 if (!npages) {
2146 ib_umem_release(obj->umem);
2147 return -EINVAL;
2148 }
2149
2150 page_mask = (1 << obj->page_shift) - 1;
2151 obj->page_offset = obj->umem->address & page_mask;
2152
2153 return 0;
2154 }
2155
2156 static int devx_umem_reg_cmd_alloc(struct uverbs_attr_bundle *attrs,
2157 struct devx_umem *obj,
2158 struct devx_umem_reg_cmd *cmd)
2159 {
2160 cmd->inlen = MLX5_ST_SZ_BYTES(create_umem_in) +
2161 (MLX5_ST_SZ_BYTES(mtt) * obj->ncont);
2162 cmd->in = uverbs_zalloc(attrs, cmd->inlen);
2163 return PTR_ERR_OR_ZERO(cmd->in);
2164 }
2165
2166 static void devx_umem_reg_cmd_build(struct mlx5_ib_dev *dev,
2167 struct devx_umem *obj,
2168 struct devx_umem_reg_cmd *cmd)
2169 {
2170 void *umem;
2171 __be64 *mtt;
2172
2173 umem = MLX5_ADDR_OF(create_umem_in, cmd->in, umem);
2174 mtt = (__be64 *)MLX5_ADDR_OF(umem, umem, mtt);
2175
2176 MLX5_SET(create_umem_in, cmd->in, opcode, MLX5_CMD_OP_CREATE_UMEM);
2177 MLX5_SET64(umem, umem, num_of_mtt, obj->ncont);
2178 MLX5_SET(umem, umem, log_page_size, obj->page_shift -
2179 MLX5_ADAPTER_PAGE_SHIFT);
2180 MLX5_SET(umem, umem, page_offset, obj->page_offset);
2181 mlx5_ib_populate_pas(dev, obj->umem, obj->page_shift, mtt,
2182 (obj->umem->writable ? MLX5_IB_MTT_WRITE : 0) |
2183 MLX5_IB_MTT_READ);
2184 }
2185
2186 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_UMEM_REG)(
2187 struct uverbs_attr_bundle *attrs)
2188 {
2189 struct devx_umem_reg_cmd cmd;
2190 struct devx_umem *obj;
2191 struct ib_uobject *uobj = uverbs_attr_get_uobject(
2192 attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE);
2193 u32 obj_id;
2194 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
2195 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
2196 struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device);
2197 int err;
2198
2199 if (!c->devx_uid)
2200 return -EINVAL;
2201
2202 obj = kzalloc(sizeof(struct devx_umem), GFP_KERNEL);
2203 if (!obj)
2204 return -ENOMEM;
2205
2206 err = devx_umem_get(dev, &c->ibucontext, attrs, obj);
2207 if (err)
2208 goto err_obj_free;
2209
2210 err = devx_umem_reg_cmd_alloc(attrs, obj, &cmd);
2211 if (err)
2212 goto err_umem_release;
2213
2214 devx_umem_reg_cmd_build(dev, obj, &cmd);
2215
2216 MLX5_SET(create_umem_in, cmd.in, uid, c->devx_uid);
2217 err = mlx5_cmd_exec(dev->mdev, cmd.in, cmd.inlen, cmd.out,
2218 sizeof(cmd.out));
2219 if (err)
2220 goto err_umem_release;
2221
2222 obj->mdev = dev->mdev;
2223 uobj->object = obj;
2224 devx_obj_build_destroy_cmd(cmd.in, cmd.out, obj->dinbox, &obj->dinlen, &obj_id);
2225 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID, &obj_id, sizeof(obj_id));
2226 if (err)
2227 goto err_umem_destroy;
2228
2229 return 0;
2230
2231 err_umem_destroy:
2232 mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, cmd.out, sizeof(cmd.out));
2233 err_umem_release:
2234 ib_umem_release(obj->umem);
2235 err_obj_free:
2236 kfree(obj);
2237 return err;
2238 }
2239
2240 static int devx_umem_cleanup(struct ib_uobject *uobject,
2241 enum rdma_remove_reason why,
2242 struct uverbs_attr_bundle *attrs)
2243 {
2244 struct devx_umem *obj = uobject->object;
2245 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
2246 int err;
2247
2248 err = mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, out, sizeof(out));
2249 if (ib_is_destroy_retryable(err, why, uobject))
2250 return err;
2251
2252 ib_umem_release(obj->umem);
2253 kfree(obj);
2254 return 0;
2255 }
2256
2257 static bool is_unaffiliated_event(struct mlx5_core_dev *dev,
2258 unsigned long event_type)
2259 {
2260 __be64 *unaff_events;
2261 int mask_entry;
2262 int mask_bit;
2263
2264 if (!MLX5_CAP_GEN(dev, event_cap))
2265 return is_legacy_unaffiliated_event_num(event_type);
2266
2267 unaff_events = MLX5_CAP_DEV_EVENT(dev,
2268 user_unaffiliated_events);
2269 WARN_ON(event_type > MAX_SUPP_EVENT_NUM);
2270
2271 mask_entry = event_type / 64;
2272 mask_bit = event_type % 64;
2273
2274 if (!(be64_to_cpu(unaff_events[mask_entry]) & (1ull << mask_bit)))
2275 return false;
2276
2277 return true;
2278 }
2279
2280 static u32 devx_get_obj_id_from_event(unsigned long event_type, void *data)
2281 {
2282 struct mlx5_eqe *eqe = data;
2283 u32 obj_id = 0;
2284
2285 switch (event_type) {
2286 case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
2287 case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
2288 case MLX5_EVENT_TYPE_PATH_MIG:
2289 case MLX5_EVENT_TYPE_COMM_EST:
2290 case MLX5_EVENT_TYPE_SQ_DRAINED:
2291 case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
2292 case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
2293 case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
2294 case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
2295 case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
2296 obj_id = be32_to_cpu(eqe->data.qp_srq.qp_srq_n) & 0xffffff;
2297 break;
2298 case MLX5_EVENT_TYPE_XRQ_ERROR:
2299 obj_id = be32_to_cpu(eqe->data.xrq_err.type_xrqn) & 0xffffff;
2300 break;
2301 case MLX5_EVENT_TYPE_DCT_DRAINED:
2302 case MLX5_EVENT_TYPE_DCT_KEY_VIOLATION:
2303 obj_id = be32_to_cpu(eqe->data.dct.dctn) & 0xffffff;
2304 break;
2305 case MLX5_EVENT_TYPE_CQ_ERROR:
2306 obj_id = be32_to_cpu(eqe->data.cq_err.cqn) & 0xffffff;
2307 break;
2308 default:
2309 obj_id = MLX5_GET(affiliated_event_header, &eqe->data, obj_id);
2310 break;
2311 }
2312
2313 return obj_id;
2314 }
2315
2316 static int deliver_event(struct devx_event_subscription *event_sub,
2317 const void *data)
2318 {
2319 struct devx_async_event_file *ev_file;
2320 struct devx_async_event_data *event_data;
2321 unsigned long flags;
2322
2323 ev_file = event_sub->ev_file;
2324
2325 if (ev_file->omit_data) {
2326 spin_lock_irqsave(&ev_file->lock, flags);
2327 if (!list_empty(&event_sub->event_list) ||
2328 ev_file->is_destroyed) {
2329 spin_unlock_irqrestore(&ev_file->lock, flags);
2330 return 0;
2331 }
2332
2333 list_add_tail(&event_sub->event_list, &ev_file->event_list);
2334 spin_unlock_irqrestore(&ev_file->lock, flags);
2335 wake_up_interruptible(&ev_file->poll_wait);
2336 return 0;
2337 }
2338
2339 event_data = kzalloc(sizeof(*event_data) + sizeof(struct mlx5_eqe),
2340 GFP_ATOMIC);
2341 if (!event_data) {
2342 spin_lock_irqsave(&ev_file->lock, flags);
2343 ev_file->is_overflow_err = 1;
2344 spin_unlock_irqrestore(&ev_file->lock, flags);
2345 return -ENOMEM;
2346 }
2347
2348 event_data->hdr.cookie = event_sub->cookie;
2349 memcpy(event_data->hdr.out_data, data, sizeof(struct mlx5_eqe));
2350
2351 spin_lock_irqsave(&ev_file->lock, flags);
2352 if (!ev_file->is_destroyed)
2353 list_add_tail(&event_data->list, &ev_file->event_list);
2354 else
2355 kfree(event_data);
2356 spin_unlock_irqrestore(&ev_file->lock, flags);
2357 wake_up_interruptible(&ev_file->poll_wait);
2358
2359 return 0;
2360 }
2361
2362 static void dispatch_event_fd(struct list_head *fd_list,
2363 const void *data)
2364 {
2365 struct devx_event_subscription *item;
2366
2367 list_for_each_entry_rcu(item, fd_list, xa_list) {
2368 if (item->eventfd)
2369 eventfd_signal(item->eventfd, 1);
2370 else
2371 deliver_event(item, data);
2372 }
2373 }
2374
2375 static int devx_event_notifier(struct notifier_block *nb,
2376 unsigned long event_type, void *data)
2377 {
2378 struct mlx5_devx_event_table *table;
2379 struct mlx5_ib_dev *dev;
2380 struct devx_event *event;
2381 struct devx_obj_event *obj_event;
2382 u16 obj_type = 0;
2383 bool is_unaffiliated;
2384 u32 obj_id;
2385
2386 /* Explicit filtering to kernel events which may occur frequently */
2387 if (event_type == MLX5_EVENT_TYPE_CMD ||
2388 event_type == MLX5_EVENT_TYPE_PAGE_REQUEST)
2389 return NOTIFY_OK;
2390
2391 table = container_of(nb, struct mlx5_devx_event_table, devx_nb.nb);
2392 dev = container_of(table, struct mlx5_ib_dev, devx_event_table);
2393 is_unaffiliated = is_unaffiliated_event(dev->mdev, event_type);
2394
2395 if (!is_unaffiliated)
2396 obj_type = get_event_obj_type(event_type, data);
2397
2398 rcu_read_lock();
2399 event = xa_load(&table->event_xa, event_type | (obj_type << 16));
2400 if (!event) {
2401 rcu_read_unlock();
2402 return NOTIFY_DONE;
2403 }
2404
2405 if (is_unaffiliated) {
2406 dispatch_event_fd(&event->unaffiliated_list, data);
2407 rcu_read_unlock();
2408 return NOTIFY_OK;
2409 }
2410
2411 obj_id = devx_get_obj_id_from_event(event_type, data);
2412 obj_event = xa_load(&event->object_ids, obj_id);
2413 if (!obj_event) {
2414 rcu_read_unlock();
2415 return NOTIFY_DONE;
2416 }
2417
2418 dispatch_event_fd(&obj_event->obj_sub_list, data);
2419
2420 rcu_read_unlock();
2421 return NOTIFY_OK;
2422 }
2423
2424 void mlx5_ib_devx_init_event_table(struct mlx5_ib_dev *dev)
2425 {
2426 struct mlx5_devx_event_table *table = &dev->devx_event_table;
2427
2428 xa_init(&table->event_xa);
2429 mutex_init(&table->event_xa_lock);
2430 MLX5_NB_INIT(&table->devx_nb, devx_event_notifier, NOTIFY_ANY);
2431 mlx5_eq_notifier_register(dev->mdev, &table->devx_nb);
2432 }
2433
2434 void mlx5_ib_devx_cleanup_event_table(struct mlx5_ib_dev *dev)
2435 {
2436 struct mlx5_devx_event_table *table = &dev->devx_event_table;
2437 struct devx_event_subscription *sub, *tmp;
2438 struct devx_event *event;
2439 void *entry;
2440 unsigned long id;
2441
2442 mlx5_eq_notifier_unregister(dev->mdev, &table->devx_nb);
2443 mutex_lock(&dev->devx_event_table.event_xa_lock);
2444 xa_for_each(&table->event_xa, id, entry) {
2445 event = entry;
2446 list_for_each_entry_safe(sub, tmp, &event->unaffiliated_list,
2447 xa_list)
2448 devx_cleanup_subscription(dev, sub);
2449 kfree(entry);
2450 }
2451 mutex_unlock(&dev->devx_event_table.event_xa_lock);
2452 xa_destroy(&table->event_xa);
2453 }
2454
2455 static ssize_t devx_async_cmd_event_read(struct file *filp, char __user *buf,
2456 size_t count, loff_t *pos)
2457 {
2458 struct devx_async_cmd_event_file *comp_ev_file = filp->private_data;
2459 struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue;
2460 struct devx_async_data *event;
2461 int ret = 0;
2462 size_t eventsz;
2463
2464 spin_lock_irq(&ev_queue->lock);
2465
2466 while (list_empty(&ev_queue->event_list)) {
2467 spin_unlock_irq(&ev_queue->lock);
2468
2469 if (filp->f_flags & O_NONBLOCK)
2470 return -EAGAIN;
2471
2472 if (wait_event_interruptible(
2473 ev_queue->poll_wait,
2474 (!list_empty(&ev_queue->event_list) ||
2475 ev_queue->is_destroyed))) {
2476 return -ERESTARTSYS;
2477 }
2478
2479 spin_lock_irq(&ev_queue->lock);
2480 if (ev_queue->is_destroyed) {
2481 spin_unlock_irq(&ev_queue->lock);
2482 return -EIO;
2483 }
2484 }
2485
2486 event = list_entry(ev_queue->event_list.next,
2487 struct devx_async_data, list);
2488 eventsz = event->cmd_out_len +
2489 sizeof(struct mlx5_ib_uapi_devx_async_cmd_hdr);
2490
2491 if (eventsz > count) {
2492 spin_unlock_irq(&ev_queue->lock);
2493 return -ENOSPC;
2494 }
2495
2496 list_del(ev_queue->event_list.next);
2497 spin_unlock_irq(&ev_queue->lock);
2498
2499 if (copy_to_user(buf, &event->hdr, eventsz))
2500 ret = -EFAULT;
2501 else
2502 ret = eventsz;
2503
2504 atomic_sub(event->cmd_out_len, &ev_queue->bytes_in_use);
2505 kvfree(event);
2506 return ret;
2507 }
2508
2509 static __poll_t devx_async_cmd_event_poll(struct file *filp,
2510 struct poll_table_struct *wait)
2511 {
2512 struct devx_async_cmd_event_file *comp_ev_file = filp->private_data;
2513 struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue;
2514 __poll_t pollflags = 0;
2515
2516 poll_wait(filp, &ev_queue->poll_wait, wait);
2517
2518 spin_lock_irq(&ev_queue->lock);
2519 if (ev_queue->is_destroyed)
2520 pollflags = EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
2521 else if (!list_empty(&ev_queue->event_list))
2522 pollflags = EPOLLIN | EPOLLRDNORM;
2523 spin_unlock_irq(&ev_queue->lock);
2524
2525 return pollflags;
2526 }
2527
2528 static const struct file_operations devx_async_cmd_event_fops = {
2529 .owner = THIS_MODULE,
2530 .read = devx_async_cmd_event_read,
2531 .poll = devx_async_cmd_event_poll,
2532 .release = uverbs_uobject_fd_release,
2533 .llseek = no_llseek,
2534 };
2535
2536 static ssize_t devx_async_event_read(struct file *filp, char __user *buf,
2537 size_t count, loff_t *pos)
2538 {
2539 struct devx_async_event_file *ev_file = filp->private_data;
2540 struct devx_event_subscription *event_sub;
2541 struct devx_async_event_data *uninitialized_var(event);
2542 int ret = 0;
2543 size_t eventsz;
2544 bool omit_data;
2545 void *event_data;
2546
2547 omit_data = ev_file->omit_data;
2548
2549 spin_lock_irq(&ev_file->lock);
2550
2551 if (ev_file->is_overflow_err) {
2552 ev_file->is_overflow_err = 0;
2553 spin_unlock_irq(&ev_file->lock);
2554 return -EOVERFLOW;
2555 }
2556
2557
2558 while (list_empty(&ev_file->event_list)) {
2559 spin_unlock_irq(&ev_file->lock);
2560
2561 if (filp->f_flags & O_NONBLOCK)
2562 return -EAGAIN;
2563
2564 if (wait_event_interruptible(ev_file->poll_wait,
2565 (!list_empty(&ev_file->event_list) ||
2566 ev_file->is_destroyed))) {
2567 return -ERESTARTSYS;
2568 }
2569
2570 spin_lock_irq(&ev_file->lock);
2571 if (ev_file->is_destroyed) {
2572 spin_unlock_irq(&ev_file->lock);
2573 return -EIO;
2574 }
2575 }
2576
2577 if (omit_data) {
2578 event_sub = list_first_entry(&ev_file->event_list,
2579 struct devx_event_subscription,
2580 event_list);
2581 eventsz = sizeof(event_sub->cookie);
2582 event_data = &event_sub->cookie;
2583 } else {
2584 event = list_first_entry(&ev_file->event_list,
2585 struct devx_async_event_data, list);
2586 eventsz = sizeof(struct mlx5_eqe) +
2587 sizeof(struct mlx5_ib_uapi_devx_async_event_hdr);
2588 event_data = &event->hdr;
2589 }
2590
2591 if (eventsz > count) {
2592 spin_unlock_irq(&ev_file->lock);
2593 return -EINVAL;
2594 }
2595
2596 if (omit_data)
2597 list_del_init(&event_sub->event_list);
2598 else
2599 list_del(&event->list);
2600
2601 spin_unlock_irq(&ev_file->lock);
2602
2603 if (copy_to_user(buf, event_data, eventsz))
2604 /* This points to an application issue, not a kernel concern */
2605 ret = -EFAULT;
2606 else
2607 ret = eventsz;
2608
2609 if (!omit_data)
2610 kfree(event);
2611 return ret;
2612 }
2613
2614 static __poll_t devx_async_event_poll(struct file *filp,
2615 struct poll_table_struct *wait)
2616 {
2617 struct devx_async_event_file *ev_file = filp->private_data;
2618 __poll_t pollflags = 0;
2619
2620 poll_wait(filp, &ev_file->poll_wait, wait);
2621
2622 spin_lock_irq(&ev_file->lock);
2623 if (ev_file->is_destroyed)
2624 pollflags = EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
2625 else if (!list_empty(&ev_file->event_list))
2626 pollflags = EPOLLIN | EPOLLRDNORM;
2627 spin_unlock_irq(&ev_file->lock);
2628
2629 return pollflags;
2630 }
2631
2632 static void devx_free_subscription(struct rcu_head *rcu)
2633 {
2634 struct devx_event_subscription *event_sub =
2635 container_of(rcu, struct devx_event_subscription, rcu);
2636
2637 if (event_sub->eventfd)
2638 eventfd_ctx_put(event_sub->eventfd);
2639 uverbs_uobject_put(&event_sub->ev_file->uobj);
2640 kfree(event_sub);
2641 }
2642
2643 static const struct file_operations devx_async_event_fops = {
2644 .owner = THIS_MODULE,
2645 .read = devx_async_event_read,
2646 .poll = devx_async_event_poll,
2647 .release = uverbs_uobject_fd_release,
2648 .llseek = no_llseek,
2649 };
2650
2651 static int devx_async_cmd_event_destroy_uobj(struct ib_uobject *uobj,
2652 enum rdma_remove_reason why)
2653 {
2654 struct devx_async_cmd_event_file *comp_ev_file =
2655 container_of(uobj, struct devx_async_cmd_event_file,
2656 uobj);
2657 struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue;
2658 struct devx_async_data *entry, *tmp;
2659
2660 spin_lock_irq(&ev_queue->lock);
2661 ev_queue->is_destroyed = 1;
2662 spin_unlock_irq(&ev_queue->lock);
2663 wake_up_interruptible(&ev_queue->poll_wait);
2664
2665 mlx5_cmd_cleanup_async_ctx(&comp_ev_file->async_ctx);
2666
2667 spin_lock_irq(&comp_ev_file->ev_queue.lock);
2668 list_for_each_entry_safe(entry, tmp,
2669 &comp_ev_file->ev_queue.event_list, list) {
2670 list_del(&entry->list);
2671 kvfree(entry);
2672 }
2673 spin_unlock_irq(&comp_ev_file->ev_queue.lock);
2674 return 0;
2675 };
2676
2677 static int devx_async_event_destroy_uobj(struct ib_uobject *uobj,
2678 enum rdma_remove_reason why)
2679 {
2680 struct devx_async_event_file *ev_file =
2681 container_of(uobj, struct devx_async_event_file,
2682 uobj);
2683 struct devx_event_subscription *event_sub, *event_sub_tmp;
2684 struct mlx5_ib_dev *dev = ev_file->dev;
2685
2686 spin_lock_irq(&ev_file->lock);
2687 ev_file->is_destroyed = 1;
2688
2689 /* free the pending events allocation */
2690 if (ev_file->omit_data) {
2691 struct devx_event_subscription *event_sub, *tmp;
2692
2693 list_for_each_entry_safe(event_sub, tmp, &ev_file->event_list,
2694 event_list)
2695 list_del_init(&event_sub->event_list);
2696
2697 } else {
2698 struct devx_async_event_data *entry, *tmp;
2699
2700 list_for_each_entry_safe(entry, tmp, &ev_file->event_list,
2701 list) {
2702 list_del(&entry->list);
2703 kfree(entry);
2704 }
2705 }
2706
2707 spin_unlock_irq(&ev_file->lock);
2708 wake_up_interruptible(&ev_file->poll_wait);
2709
2710 mutex_lock(&dev->devx_event_table.event_xa_lock);
2711 /* delete the subscriptions which are related to this FD */
2712 list_for_each_entry_safe(event_sub, event_sub_tmp,
2713 &ev_file->subscribed_events_list, file_list) {
2714 devx_cleanup_subscription(dev, event_sub);
2715 list_del_rcu(&event_sub->file_list);
2716 /* subscription may not be used by the read API any more */
2717 call_rcu(&event_sub->rcu, devx_free_subscription);
2718 }
2719 mutex_unlock(&dev->devx_event_table.event_xa_lock);
2720
2721 put_device(&dev->ib_dev.dev);
2722 return 0;
2723 };
2724
2725 DECLARE_UVERBS_NAMED_METHOD(
2726 MLX5_IB_METHOD_DEVX_UMEM_REG,
2727 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE,
2728 MLX5_IB_OBJECT_DEVX_UMEM,
2729 UVERBS_ACCESS_NEW,
2730 UA_MANDATORY),
2731 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_ADDR,
2732 UVERBS_ATTR_TYPE(u64),
2733 UA_MANDATORY),
2734 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_LEN,
2735 UVERBS_ATTR_TYPE(u64),
2736 UA_MANDATORY),
2737 UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS,
2738 enum ib_access_flags),
2739 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID,
2740 UVERBS_ATTR_TYPE(u32),
2741 UA_MANDATORY));
2742
2743 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
2744 MLX5_IB_METHOD_DEVX_UMEM_DEREG,
2745 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_UMEM_DEREG_HANDLE,
2746 MLX5_IB_OBJECT_DEVX_UMEM,
2747 UVERBS_ACCESS_DESTROY,
2748 UA_MANDATORY));
2749
2750 DECLARE_UVERBS_NAMED_METHOD(
2751 MLX5_IB_METHOD_DEVX_QUERY_EQN,
2752 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_QUERY_EQN_USER_VEC,
2753 UVERBS_ATTR_TYPE(u32),
2754 UA_MANDATORY),
2755 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_QUERY_EQN_DEV_EQN,
2756 UVERBS_ATTR_TYPE(u32),
2757 UA_MANDATORY));
2758
2759 DECLARE_UVERBS_NAMED_METHOD(
2760 MLX5_IB_METHOD_DEVX_QUERY_UAR,
2761 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_QUERY_UAR_USER_IDX,
2762 UVERBS_ATTR_TYPE(u32),
2763 UA_MANDATORY),
2764 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_QUERY_UAR_DEV_IDX,
2765 UVERBS_ATTR_TYPE(u32),
2766 UA_MANDATORY));
2767
2768 DECLARE_UVERBS_NAMED_METHOD(
2769 MLX5_IB_METHOD_DEVX_OTHER,
2770 UVERBS_ATTR_PTR_IN(
2771 MLX5_IB_ATTR_DEVX_OTHER_CMD_IN,
2772 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
2773 UA_MANDATORY,
2774 UA_ALLOC_AND_COPY),
2775 UVERBS_ATTR_PTR_OUT(
2776 MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT,
2777 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
2778 UA_MANDATORY));
2779
2780 DECLARE_UVERBS_NAMED_METHOD(
2781 MLX5_IB_METHOD_DEVX_OBJ_CREATE,
2782 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_CREATE_HANDLE,
2783 MLX5_IB_OBJECT_DEVX_OBJ,
2784 UVERBS_ACCESS_NEW,
2785 UA_MANDATORY),
2786 UVERBS_ATTR_PTR_IN(
2787 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN,
2788 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
2789 UA_MANDATORY,
2790 UA_ALLOC_AND_COPY),
2791 UVERBS_ATTR_PTR_OUT(
2792 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT,
2793 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
2794 UA_MANDATORY));
2795
2796 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
2797 MLX5_IB_METHOD_DEVX_OBJ_DESTROY,
2798 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_DESTROY_HANDLE,
2799 MLX5_IB_OBJECT_DEVX_OBJ,
2800 UVERBS_ACCESS_DESTROY,
2801 UA_MANDATORY));
2802
2803 DECLARE_UVERBS_NAMED_METHOD(
2804 MLX5_IB_METHOD_DEVX_OBJ_MODIFY,
2805 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_MODIFY_HANDLE,
2806 UVERBS_IDR_ANY_OBJECT,
2807 UVERBS_ACCESS_WRITE,
2808 UA_MANDATORY),
2809 UVERBS_ATTR_PTR_IN(
2810 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN,
2811 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
2812 UA_MANDATORY,
2813 UA_ALLOC_AND_COPY),
2814 UVERBS_ATTR_PTR_OUT(
2815 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT,
2816 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
2817 UA_MANDATORY));
2818
2819 DECLARE_UVERBS_NAMED_METHOD(
2820 MLX5_IB_METHOD_DEVX_OBJ_QUERY,
2821 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE,
2822 UVERBS_IDR_ANY_OBJECT,
2823 UVERBS_ACCESS_READ,
2824 UA_MANDATORY),
2825 UVERBS_ATTR_PTR_IN(
2826 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN,
2827 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
2828 UA_MANDATORY,
2829 UA_ALLOC_AND_COPY),
2830 UVERBS_ATTR_PTR_OUT(
2831 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT,
2832 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
2833 UA_MANDATORY));
2834
2835 DECLARE_UVERBS_NAMED_METHOD(
2836 MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY,
2837 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE,
2838 UVERBS_IDR_ANY_OBJECT,
2839 UVERBS_ACCESS_READ,
2840 UA_MANDATORY),
2841 UVERBS_ATTR_PTR_IN(
2842 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN,
2843 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
2844 UA_MANDATORY,
2845 UA_ALLOC_AND_COPY),
2846 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_OUT_LEN,
2847 u16, UA_MANDATORY),
2848 UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_FD,
2849 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
2850 UVERBS_ACCESS_READ,
2851 UA_MANDATORY),
2852 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_WR_ID,
2853 UVERBS_ATTR_TYPE(u64),
2854 UA_MANDATORY));
2855
2856 DECLARE_UVERBS_NAMED_METHOD(
2857 MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT,
2858 UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_HANDLE,
2859 MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,
2860 UVERBS_ACCESS_READ,
2861 UA_MANDATORY),
2862 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_OBJ_HANDLE,
2863 MLX5_IB_OBJECT_DEVX_OBJ,
2864 UVERBS_ACCESS_READ,
2865 UA_OPTIONAL),
2866 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST,
2867 UVERBS_ATTR_MIN_SIZE(sizeof(u16)),
2868 UA_MANDATORY,
2869 UA_ALLOC_AND_COPY),
2870 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE,
2871 UVERBS_ATTR_TYPE(u64),
2872 UA_OPTIONAL),
2873 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM,
2874 UVERBS_ATTR_TYPE(u32),
2875 UA_OPTIONAL));
2876
2877 DECLARE_UVERBS_GLOBAL_METHODS(MLX5_IB_OBJECT_DEVX,
2878 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OTHER),
2879 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_UAR),
2880 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_EQN),
2881 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT));
2882
2883 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_DEVX_OBJ,
2884 UVERBS_TYPE_ALLOC_IDR(devx_obj_cleanup),
2885 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_CREATE),
2886 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_DESTROY),
2887 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_MODIFY),
2888 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_QUERY),
2889 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY));
2890
2891 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_DEVX_UMEM,
2892 UVERBS_TYPE_ALLOC_IDR(devx_umem_cleanup),
2893 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_UMEM_REG),
2894 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_UMEM_DEREG));
2895
2896
2897 DECLARE_UVERBS_NAMED_METHOD(
2898 MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC,
2899 UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_ASYNC_CMD_FD_ALLOC_HANDLE,
2900 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
2901 UVERBS_ACCESS_NEW,
2902 UA_MANDATORY));
2903
2904 DECLARE_UVERBS_NAMED_OBJECT(
2905 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
2906 UVERBS_TYPE_ALLOC_FD(sizeof(struct devx_async_cmd_event_file),
2907 devx_async_cmd_event_destroy_uobj,
2908 &devx_async_cmd_event_fops, "[devx_async_cmd]",
2909 O_RDONLY),
2910 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC));
2911
2912 DECLARE_UVERBS_NAMED_METHOD(
2913 MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC,
2914 UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_HANDLE,
2915 MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,
2916 UVERBS_ACCESS_NEW,
2917 UA_MANDATORY),
2918 UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_FLAGS,
2919 enum mlx5_ib_uapi_devx_create_event_channel_flags,
2920 UA_MANDATORY));
2921
2922 DECLARE_UVERBS_NAMED_OBJECT(
2923 MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,
2924 UVERBS_TYPE_ALLOC_FD(sizeof(struct devx_async_event_file),
2925 devx_async_event_destroy_uobj,
2926 &devx_async_event_fops, "[devx_async_event]",
2927 O_RDONLY),
2928 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC));
2929
2930 static bool devx_is_supported(struct ib_device *device)
2931 {
2932 struct mlx5_ib_dev *dev = to_mdev(device);
2933
2934 return MLX5_CAP_GEN(dev->mdev, log_max_uctx);
2935 }
2936
2937 const struct uapi_definition mlx5_ib_devx_defs[] = {
2938 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
2939 MLX5_IB_OBJECT_DEVX,
2940 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
2941 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
2942 MLX5_IB_OBJECT_DEVX_OBJ,
2943 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
2944 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
2945 MLX5_IB_OBJECT_DEVX_UMEM,
2946 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
2947 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
2948 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
2949 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
2950 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
2951 MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,
2952 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
2953 {},
2954 };
Linux with added FPC-III support