?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge remote-tracking branch 'remotes/dgilbert-gitlab/tags/pull-migration-20210726a...
[qemu/armbru.git] / hw / rdma / rdma_backend.c
blob6dcdfbbbe23cf38e7dbce246e2df5eb4e5a7571f
1 /*
2 * QEMU paravirtual RDMA - Generic RDMA backend
3 *
4 * Copyright (C) 2018 Oracle
5 * Copyright (C) 2018 Red Hat Inc
6 *
7 * Authors:
8 * Yuval Shaia <yuval.shaia@oracle.com>
9 * Marcel Apfelbaum <marcel@redhat.com>
10 *
11 * This work is licensed under the terms of the GNU GPL, version 2 or later.
12 * See the COPYING file in the top-level directory.
13 *
14 */
15
16 #include "qemu/osdep.h"
17 #include "qapi/qapi-events-rdma.h"
18
19 #include <infiniband/verbs.h>
20
21 #include "contrib/rdmacm-mux/rdmacm-mux.h"
22 #include "trace.h"
23 #include "rdma_utils.h"
24 #include "rdma_rm.h"
25 #include "rdma_backend.h"
26
27 #define THR_NAME_LEN 16
28 #define THR_POLL_TO 5000
29
30 #define MAD_HDR_SIZE sizeof(struct ibv_grh)
31
32 typedef struct BackendCtx {
33 void *up_ctx;
34 struct ibv_sge sge; /* Used to save MAD recv buffer */
35 RdmaBackendQP *backend_qp; /* To maintain recv buffers */
36 RdmaBackendSRQ *backend_srq;
37 } BackendCtx;
38
39 struct backend_umad {
40 struct ib_user_mad hdr;
41 char mad[RDMA_MAX_PRIVATE_DATA];
42 };
43
44 static void (*comp_handler)(void *ctx, struct ibv_wc *wc);
45
46 static void dummy_comp_handler(void *ctx, struct ibv_wc *wc)
47 {
48 rdma_error_report("No completion handler is registered");
49 }
50
51 static inline void complete_work(enum ibv_wc_status status, uint32_t vendor_err,
52 void *ctx)
53 {
54 struct ibv_wc wc = {};
55
56 wc.status = status;
57 wc.vendor_err = vendor_err;
58
59 comp_handler(ctx, &wc);
60 }
61
62 static void free_cqe_ctx(gpointer data, gpointer user_data)
63 {
64 BackendCtx *bctx;
65 RdmaDeviceResources *rdma_dev_res = user_data;
66 unsigned long cqe_ctx_id = GPOINTER_TO_INT(data);
67
68 bctx = rdma_rm_get_cqe_ctx(rdma_dev_res, cqe_ctx_id);
69 if (bctx) {
70 rdma_rm_dealloc_cqe_ctx(rdma_dev_res, cqe_ctx_id);
71 qatomic_dec(&rdma_dev_res->stats.missing_cqe);
72 }
73 g_free(bctx);
74 }
75
76 static void clean_recv_mads(RdmaBackendDev *backend_dev)
77 {
78 unsigned long cqe_ctx_id;
79
80 do {
81 cqe_ctx_id = rdma_protected_gqueue_pop_int64(&backend_dev->
82 recv_mads_list);
83 if (cqe_ctx_id != -ENOENT) {
84 qatomic_inc(&backend_dev->rdma_dev_res->stats.missing_cqe);
85 free_cqe_ctx(GINT_TO_POINTER(cqe_ctx_id),
86 backend_dev->rdma_dev_res);
87 }
88 } while (cqe_ctx_id != -ENOENT);
89 }
90
91 static int rdma_poll_cq(RdmaDeviceResources *rdma_dev_res, struct ibv_cq *ibcq)
92 {
93 int i, ne, total_ne = 0;
94 BackendCtx *bctx;
95 struct ibv_wc wc[2];
96 RdmaProtectedGSList *cqe_ctx_list;
97
98 WITH_QEMU_LOCK_GUARD(&rdma_dev_res->lock) {
99 do {
100 ne = ibv_poll_cq(ibcq, ARRAY_SIZE(wc), wc);
101
102 trace_rdma_poll_cq(ne, ibcq);
103
104 for (i = 0; i < ne; i++) {
105 bctx = rdma_rm_get_cqe_ctx(rdma_dev_res, wc[i].wr_id);
106 if (unlikely(!bctx)) {
107 rdma_error_report("No matching ctx for req %"PRId64,
108 wc[i].wr_id);
109 continue;
110 }
111
112 comp_handler(bctx->up_ctx, &wc[i]);
113
114 if (bctx->backend_qp) {
115 cqe_ctx_list = &bctx->backend_qp->cqe_ctx_list;
116 } else {
117 cqe_ctx_list = &bctx->backend_srq->cqe_ctx_list;
118 }
119
120 rdma_protected_gslist_remove_int32(cqe_ctx_list, wc[i].wr_id);
121 rdma_rm_dealloc_cqe_ctx(rdma_dev_res, wc[i].wr_id);
122 g_free(bctx);
123 }
124 total_ne += ne;
125 } while (ne > 0);
126 qatomic_sub(&rdma_dev_res->stats.missing_cqe, total_ne);
127 }
128
129 if (ne < 0) {
130 rdma_error_report("ibv_poll_cq fail, rc=%d, errno=%d", ne, errno);
131 }
132
133 rdma_dev_res->stats.completions += total_ne;
134
135 return total_ne;
136 }
137
138 static void *comp_handler_thread(void *arg)
139 {
140 RdmaBackendDev *backend_dev = (RdmaBackendDev *)arg;
141 int rc;
142 struct ibv_cq *ev_cq;
143 void *ev_ctx;
144 int flags;
145 GPollFD pfds[1];
146
147 /* Change to non-blocking mode */
148 flags = fcntl(backend_dev->channel->fd, F_GETFL);
149 rc = fcntl(backend_dev->channel->fd, F_SETFL, flags | O_NONBLOCK);
150 if (rc < 0) {
151 rdma_error_report("Failed to change backend channel FD to non-blocking");
152 return NULL;
153 }
154
155 pfds[0].fd = backend_dev->channel->fd;
156 pfds[0].events = G_IO_IN | G_IO_HUP | G_IO_ERR;
157
158 backend_dev->comp_thread.is_running = true;
159
160 while (backend_dev->comp_thread.run) {
161 do {
162 rc = qemu_poll_ns(pfds, 1, THR_POLL_TO * (int64_t)SCALE_MS);
163 if (!rc) {
164 backend_dev->rdma_dev_res->stats.poll_cq_ppoll_to++;
165 }
166 } while (!rc && backend_dev->comp_thread.run);
167
168 if (backend_dev->comp_thread.run) {
169 rc = ibv_get_cq_event(backend_dev->channel, &ev_cq, &ev_ctx);
170 if (unlikely(rc)) {
171 rdma_error_report("ibv_get_cq_event fail, rc=%d, errno=%d", rc,
172 errno);
173 continue;
174 }
175
176 rc = ibv_req_notify_cq(ev_cq, 0);
177 if (unlikely(rc)) {
178 rdma_error_report("ibv_req_notify_cq fail, rc=%d, errno=%d", rc,
179 errno);
180 }
181
182 backend_dev->rdma_dev_res->stats.poll_cq_from_bk++;
183 rdma_poll_cq(backend_dev->rdma_dev_res, ev_cq);
184
185 ibv_ack_cq_events(ev_cq, 1);
186 }
187 }
188
189 backend_dev->comp_thread.is_running = false;
190
191 qemu_thread_exit(0);
192
193 return NULL;
194 }
195
196 static inline void disable_rdmacm_mux_async(RdmaBackendDev *backend_dev)
197 {
198 qatomic_set(&backend_dev->rdmacm_mux.can_receive, 0);
199 }
200
201 static inline void enable_rdmacm_mux_async(RdmaBackendDev *backend_dev)
202 {
203 qatomic_set(&backend_dev->rdmacm_mux.can_receive, sizeof(RdmaCmMuxMsg));
204 }
205
206 static inline int rdmacm_mux_can_process_async(RdmaBackendDev *backend_dev)
207 {
208 return qatomic_read(&backend_dev->rdmacm_mux.can_receive);
209 }
210
211 static int rdmacm_mux_check_op_status(CharBackend *mad_chr_be)
212 {
213 RdmaCmMuxMsg msg = {};
214 int ret;
215
216 ret = qemu_chr_fe_read_all(mad_chr_be, (uint8_t *)&msg, sizeof(msg));
217 if (ret != sizeof(msg)) {
218 rdma_error_report("Got invalid message from mux: size %d, expecting %d",
219 ret, (int)sizeof(msg));
220 return -EIO;
221 }
222
223 trace_rdmacm_mux_check_op_status(msg.hdr.msg_type, msg.hdr.op_code,
224 msg.hdr.err_code);
225
226 if (msg.hdr.msg_type != RDMACM_MUX_MSG_TYPE_RESP) {
227 rdma_error_report("Got invalid message type %d", msg.hdr.msg_type);
228 return -EIO;
229 }
230
231 if (msg.hdr.err_code != RDMACM_MUX_ERR_CODE_OK) {
232 rdma_error_report("Operation failed in mux, error code %d",
233 msg.hdr.err_code);
234 return -EIO;
235 }
236
237 return 0;
238 }
239
240 static int rdmacm_mux_send(RdmaBackendDev *backend_dev, RdmaCmMuxMsg *msg)
241 {
242 int rc = 0;
243
244 msg->hdr.msg_type = RDMACM_MUX_MSG_TYPE_REQ;
245 trace_rdmacm_mux("send", msg->hdr.msg_type, msg->hdr.op_code);
246 disable_rdmacm_mux_async(backend_dev);
247 rc = qemu_chr_fe_write(backend_dev->rdmacm_mux.chr_be,
248 (const uint8_t *)msg, sizeof(*msg));
249 if (rc != sizeof(*msg)) {
250 enable_rdmacm_mux_async(backend_dev);
251 rdma_error_report("Failed to send request to rdmacm_mux (rc=%d)", rc);
252 return -EIO;
253 }
254
255 rc = rdmacm_mux_check_op_status(backend_dev->rdmacm_mux.chr_be);
256 if (rc) {
257 rdma_error_report("Failed to execute rdmacm_mux request %d (rc=%d)",
258 msg->hdr.op_code, rc);
259 }
260
261 enable_rdmacm_mux_async(backend_dev);
262
263 return 0;
264 }
265
266 static void stop_backend_thread(RdmaBackendThread *thread)
267 {
268 thread->run = false;
269 while (thread->is_running) {
270 sleep(THR_POLL_TO / SCALE_US / 2);
271 }
272 }
273
274 static void start_comp_thread(RdmaBackendDev *backend_dev)
275 {
276 char thread_name[THR_NAME_LEN] = {};
277
278 stop_backend_thread(&backend_dev->comp_thread);
279
280 snprintf(thread_name, sizeof(thread_name), "rdma_comp_%s",
281 ibv_get_device_name(backend_dev->ib_dev));
282 backend_dev->comp_thread.run = true;
283 qemu_thread_create(&backend_dev->comp_thread.thread, thread_name,
284 comp_handler_thread, backend_dev, QEMU_THREAD_DETACHED);
285 }
286
287 void rdma_backend_register_comp_handler(void (*handler)(void *ctx,
288 struct ibv_wc *wc))
289 {
290 comp_handler = handler;
291 }
292
293 void rdma_backend_unregister_comp_handler(void)
294 {
295 rdma_backend_register_comp_handler(dummy_comp_handler);
296 }
297
298 int rdma_backend_query_port(RdmaBackendDev *backend_dev,
299 struct ibv_port_attr *port_attr)
300 {
301 int rc;
302
303 rc = ibv_query_port(backend_dev->context, backend_dev->port_num, port_attr);
304 if (rc) {
305 rdma_error_report("ibv_query_port fail, rc=%d, errno=%d", rc, errno);
306 return -EIO;
307 }
308
309 return 0;
310 }
311
312 void rdma_backend_poll_cq(RdmaDeviceResources *rdma_dev_res, RdmaBackendCQ *cq)
313 {
314 int polled;
315
316 rdma_dev_res->stats.poll_cq_from_guest++;
317 polled = rdma_poll_cq(rdma_dev_res, cq->ibcq);
318 if (!polled) {
319 rdma_dev_res->stats.poll_cq_from_guest_empty++;
320 }
321 }
322
323 static GHashTable *ah_hash;
324
325 static struct ibv_ah *create_ah(RdmaBackendDev *backend_dev, struct ibv_pd *pd,
326 uint8_t sgid_idx, union ibv_gid *dgid)
327 {
328 GBytes *ah_key = g_bytes_new(dgid, sizeof(*dgid));
329 struct ibv_ah *ah = g_hash_table_lookup(ah_hash, ah_key);
330
331 if (ah) {
332 trace_rdma_create_ah_cache_hit(be64_to_cpu(dgid->global.subnet_prefix),
333 be64_to_cpu(dgid->global.interface_id));
334 g_bytes_unref(ah_key);
335 } else {
336 struct ibv_ah_attr ah_attr = {
337 .is_global = 1,
338 .port_num = backend_dev->port_num,
339 .grh.hop_limit = 1,
340 };
341
342 ah_attr.grh.dgid = *dgid;
343 ah_attr.grh.sgid_index = sgid_idx;
344
345 ah = ibv_create_ah(pd, &ah_attr);
346 if (ah) {
347 g_hash_table_insert(ah_hash, ah_key, ah);
348 } else {
349 g_bytes_unref(ah_key);
350 rdma_error_report("Failed to create AH for gid <0x%" PRIx64", 0x%"PRIx64">",
351 be64_to_cpu(dgid->global.subnet_prefix),
352 be64_to_cpu(dgid->global.interface_id));
353 }
354
355 trace_rdma_create_ah_cache_miss(be64_to_cpu(dgid->global.subnet_prefix),
356 be64_to_cpu(dgid->global.interface_id));
357 }
358
359 return ah;
360 }
361
362 static void destroy_ah_hash_key(gpointer data)
363 {
364 g_bytes_unref(data);
365 }
366
367 static void destroy_ah_hast_data(gpointer data)
368 {
369 struct ibv_ah *ah = data;
370
371 ibv_destroy_ah(ah);
372 }
373
374 static void ah_cache_init(void)
375 {
376 ah_hash = g_hash_table_new_full(g_bytes_hash, g_bytes_equal,
377 destroy_ah_hash_key, destroy_ah_hast_data);
378 }
379
380 #ifdef LEGACY_RDMA_REG_MR
381 static int build_host_sge_array(RdmaDeviceResources *rdma_dev_res,
382 struct ibv_sge *sge, uint8_t num_sge,
383 uint64_t *total_length)
384 {
385 RdmaRmMR *mr;
386 int idx;
387
388 for (idx = 0; idx < num_sge; idx++) {
389 mr = rdma_rm_get_mr(rdma_dev_res, sge[idx].lkey);
390 if (unlikely(!mr)) {
391 rdma_error_report("Invalid lkey 0x%x", sge[idx].lkey);
392 return VENDOR_ERR_INVLKEY | sge[idx].lkey;
393 }
394
395 sge[idx].addr = (uintptr_t)mr->virt + sge[idx].addr - mr->start;
396 sge[idx].lkey = rdma_backend_mr_lkey(&mr->backend_mr);
397
398 *total_length += sge[idx].length;
399 }
400
401 return 0;
402 }
403 #else
404 static inline int build_host_sge_array(RdmaDeviceResources *rdma_dev_res,
405 struct ibv_sge *sge, uint8_t num_sge,
406 uint64_t *total_length)
407 {
408 int idx;
409
410 for (idx = 0; idx < num_sge; idx++) {
411 *total_length += sge[idx].length;
412 }
413 return 0;
414 }
415 #endif
416
417 static void trace_mad_message(const char *title, char *buf, int len)
418 {
419 int i;
420 char *b = g_malloc0(len * 3 + 1);
421 char b1[4];
422
423 for (i = 0; i < len; i++) {
424 sprintf(b1, "%.2X ", buf[i] & 0x000000FF);
425 strcat(b, b1);
426 }
427
428 trace_rdma_mad_message(title, len, b);
429
430 g_free(b);
431 }
432
433 static int mad_send(RdmaBackendDev *backend_dev, uint8_t sgid_idx,
434 union ibv_gid *sgid, struct ibv_sge *sge, uint32_t num_sge)
435 {
436 RdmaCmMuxMsg msg = {};
437 char *hdr, *data;
438 int ret;
439
440 if (num_sge != 2) {
441 return -EINVAL;
442 }
443
444 msg.hdr.op_code = RDMACM_MUX_OP_CODE_MAD;
445 memcpy(msg.hdr.sgid.raw, sgid->raw, sizeof(msg.hdr.sgid));
446
447 msg.umad_len = sge[0].length + sge[1].length;
448
449 if (msg.umad_len > sizeof(msg.umad.mad)) {
450 return -ENOMEM;
451 }
452
453 msg.umad.hdr.addr.qpn = htobe32(1);
454 msg.umad.hdr.addr.grh_present = 1;
455 msg.umad.hdr.addr.gid_index = sgid_idx;
456 memcpy(msg.umad.hdr.addr.gid, sgid->raw, sizeof(msg.umad.hdr.addr.gid));
457 msg.umad.hdr.addr.hop_limit = 0xFF;
458
459 hdr = rdma_pci_dma_map(backend_dev->dev, sge[0].addr, sge[0].length);
460 if (!hdr) {
461 return -ENOMEM;
462 }
463 data = rdma_pci_dma_map(backend_dev->dev, sge[1].addr, sge[1].length);
464 if (!data) {
465 rdma_pci_dma_unmap(backend_dev->dev, hdr, sge[0].length);
466 return -ENOMEM;
467 }
468
469 memcpy(&msg.umad.mad[0], hdr, sge[0].length);
470 memcpy(&msg.umad.mad[sge[0].length], data, sge[1].length);
471
472 rdma_pci_dma_unmap(backend_dev->dev, data, sge[1].length);
473 rdma_pci_dma_unmap(backend_dev->dev, hdr, sge[0].length);
474
475 trace_mad_message("send", msg.umad.mad, msg.umad_len);
476
477 ret = rdmacm_mux_send(backend_dev, &msg);
478 if (ret) {
479 rdma_error_report("Failed to send MAD to rdma_umadmux (%d)", ret);
480 return -EIO;
481 }
482
483 return 0;
484 }
485
486 void rdma_backend_post_send(RdmaBackendDev *backend_dev,
487 RdmaBackendQP *qp, uint8_t qp_type,
488 struct ibv_sge *sge, uint32_t num_sge,
489 uint8_t sgid_idx, union ibv_gid *sgid,
490 union ibv_gid *dgid, uint32_t dqpn, uint32_t dqkey,
491 void *ctx)
492 {
493 BackendCtx *bctx;
494 uint32_t bctx_id;
495 int rc;
496 struct ibv_send_wr wr = {}, *bad_wr;
497
498 if (!qp->ibqp) { /* This field is not initialized for QP0 and QP1 */
499 if (qp_type == IBV_QPT_SMI) {
500 rdma_error_report("Got QP0 request");
501 complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_QP0, ctx);
502 } else if (qp_type == IBV_QPT_GSI) {
503 rc = mad_send(backend_dev, sgid_idx, sgid, sge, num_sge);
504 if (rc) {
505 complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_MAD_SEND, ctx);
506 backend_dev->rdma_dev_res->stats.mad_tx_err++;
507 } else {
508 complete_work(IBV_WC_SUCCESS, 0, ctx);
509 backend_dev->rdma_dev_res->stats.mad_tx++;
510 }
511 }
512 return;
513 }
514
515 bctx = g_malloc0(sizeof(*bctx));
516 bctx->up_ctx = ctx;
517 bctx->backend_qp = qp;
518
519 rc = rdma_rm_alloc_cqe_ctx(backend_dev->rdma_dev_res, &bctx_id, bctx);
520 if (unlikely(rc)) {
521 complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_NOMEM, ctx);
522 goto err_free_bctx;
523 }
524
525 rdma_protected_gslist_append_int32(&qp->cqe_ctx_list, bctx_id);
526
527 rc = build_host_sge_array(backend_dev->rdma_dev_res, sge, num_sge,
528 &backend_dev->rdma_dev_res->stats.tx_len);
529 if (rc) {
530 complete_work(IBV_WC_GENERAL_ERR, rc, ctx);
531 goto err_dealloc_cqe_ctx;
532 }
533
534 if (qp_type == IBV_QPT_UD) {
535 wr.wr.ud.ah = create_ah(backend_dev, qp->ibpd, sgid_idx, dgid);
536 if (!wr.wr.ud.ah) {
537 complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_FAIL_BACKEND, ctx);
538 goto err_dealloc_cqe_ctx;
539 }
540 wr.wr.ud.remote_qpn = dqpn;
541 wr.wr.ud.remote_qkey = dqkey;
542 }
543
544 wr.num_sge = num_sge;
545 wr.opcode = IBV_WR_SEND;
546 wr.send_flags = IBV_SEND_SIGNALED;
547 wr.sg_list = sge;
548 wr.wr_id = bctx_id;
549
550 rc = ibv_post_send(qp->ibqp, &wr, &bad_wr);
551 if (rc) {
552 rdma_error_report("ibv_post_send fail, qpn=0x%x, rc=%d, errno=%d",
553 qp->ibqp->qp_num, rc, errno);
554 complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_FAIL_BACKEND, ctx);
555 goto err_dealloc_cqe_ctx;
556 }
557
558 qatomic_inc(&backend_dev->rdma_dev_res->stats.missing_cqe);
559 backend_dev->rdma_dev_res->stats.tx++;
560
561 return;
562
563 err_dealloc_cqe_ctx:
564 backend_dev->rdma_dev_res->stats.tx_err++;
565 rdma_rm_dealloc_cqe_ctx(backend_dev->rdma_dev_res, bctx_id);
566
567 err_free_bctx:
568 g_free(bctx);
569 }
570
571 static unsigned int save_mad_recv_buffer(RdmaBackendDev *backend_dev,
572 struct ibv_sge *sge, uint32_t num_sge,
573 void *ctx)
574 {
575 BackendCtx *bctx;
576 int rc;
577 uint32_t bctx_id;
578
579 if (num_sge != 1) {
580 rdma_error_report("Invalid num_sge (%d), expecting 1", num_sge);
581 return VENDOR_ERR_INV_NUM_SGE;
582 }
583
584 if (sge[0].length < RDMA_MAX_PRIVATE_DATA + sizeof(struct ibv_grh)) {
585 rdma_error_report("Too small buffer for MAD");
586 return VENDOR_ERR_INV_MAD_BUFF;
587 }
588
589 bctx = g_malloc0(sizeof(*bctx));
590
591 rc = rdma_rm_alloc_cqe_ctx(backend_dev->rdma_dev_res, &bctx_id, bctx);
592 if (unlikely(rc)) {
593 g_free(bctx);
594 return VENDOR_ERR_NOMEM;
595 }
596
597 bctx->up_ctx = ctx;
598 bctx->sge = *sge;
599
600 rdma_protected_gqueue_append_int64(&backend_dev->recv_mads_list, bctx_id);
601
602 return 0;
603 }
604
605 void rdma_backend_post_recv(RdmaBackendDev *backend_dev,
606 RdmaBackendQP *qp, uint8_t qp_type,
607 struct ibv_sge *sge, uint32_t num_sge, void *ctx)
608 {
609 BackendCtx *bctx;
610 uint32_t bctx_id;
611 int rc;
612 struct ibv_recv_wr wr = {}, *bad_wr;
613
614 if (!qp->ibqp) { /* This field does not get initialized for QP0 and QP1 */
615 if (qp_type == IBV_QPT_SMI) {
616 rdma_error_report("Got QP0 request");
617 complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_QP0, ctx);
618 }
619 if (qp_type == IBV_QPT_GSI) {
620 rc = save_mad_recv_buffer(backend_dev, sge, num_sge, ctx);
621 if (rc) {
622 complete_work(IBV_WC_GENERAL_ERR, rc, ctx);
623 backend_dev->rdma_dev_res->stats.mad_rx_bufs_err++;
624 } else {
625 backend_dev->rdma_dev_res->stats.mad_rx_bufs++;
626 }
627 }
628 return;
629 }
630
631 bctx = g_malloc0(sizeof(*bctx));
632 bctx->up_ctx = ctx;
633 bctx->backend_qp = qp;
634
635 rc = rdma_rm_alloc_cqe_ctx(backend_dev->rdma_dev_res, &bctx_id, bctx);
636 if (unlikely(rc)) {
637 complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_NOMEM, ctx);
638 goto err_free_bctx;
639 }
640
641 rdma_protected_gslist_append_int32(&qp->cqe_ctx_list, bctx_id);
642
643 rc = build_host_sge_array(backend_dev->rdma_dev_res, sge, num_sge,
644 &backend_dev->rdma_dev_res->stats.rx_bufs_len);
645 if (rc) {
646 complete_work(IBV_WC_GENERAL_ERR, rc, ctx);
647 goto err_dealloc_cqe_ctx;
648 }
649
650 wr.num_sge = num_sge;
651 wr.sg_list = sge;
652 wr.wr_id = bctx_id;
653 rc = ibv_post_recv(qp->ibqp, &wr, &bad_wr);
654 if (rc) {
655 rdma_error_report("ibv_post_recv fail, qpn=0x%x, rc=%d, errno=%d",
656 qp->ibqp->qp_num, rc, errno);
657 complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_FAIL_BACKEND, ctx);
658 goto err_dealloc_cqe_ctx;
659 }
660
661 qatomic_inc(&backend_dev->rdma_dev_res->stats.missing_cqe);
662 backend_dev->rdma_dev_res->stats.rx_bufs++;
663
664 return;
665
666 err_dealloc_cqe_ctx:
667 backend_dev->rdma_dev_res->stats.rx_bufs_err++;
668 rdma_rm_dealloc_cqe_ctx(backend_dev->rdma_dev_res, bctx_id);
669
670 err_free_bctx:
671 g_free(bctx);
672 }
673
674 void rdma_backend_post_srq_recv(RdmaBackendDev *backend_dev,
675 RdmaBackendSRQ *srq, struct ibv_sge *sge,
676 uint32_t num_sge, void *ctx)
677 {
678 BackendCtx *bctx;
679 uint32_t bctx_id;
680 int rc;
681 struct ibv_recv_wr wr = {}, *bad_wr;
682
683 bctx = g_malloc0(sizeof(*bctx));
684 bctx->up_ctx = ctx;
685 bctx->backend_srq = srq;
686
687 rc = rdma_rm_alloc_cqe_ctx(backend_dev->rdma_dev_res, &bctx_id, bctx);
688 if (unlikely(rc)) {
689 complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_NOMEM, ctx);
690 goto err_free_bctx;
691 }
692
693 rdma_protected_gslist_append_int32(&srq->cqe_ctx_list, bctx_id);
694
695 rc = build_host_sge_array(backend_dev->rdma_dev_res, sge, num_sge,
696 &backend_dev->rdma_dev_res->stats.rx_bufs_len);
697 if (rc) {
698 complete_work(IBV_WC_GENERAL_ERR, rc, ctx);
699 goto err_dealloc_cqe_ctx;
700 }
701
702 wr.num_sge = num_sge;
703 wr.sg_list = sge;
704 wr.wr_id = bctx_id;
705 rc = ibv_post_srq_recv(srq->ibsrq, &wr, &bad_wr);
706 if (rc) {
707 rdma_error_report("ibv_post_srq_recv fail, srqn=0x%x, rc=%d, errno=%d",
708 srq->ibsrq->handle, rc, errno);
709 complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_FAIL_BACKEND, ctx);
710 goto err_dealloc_cqe_ctx;
711 }
712
713 qatomic_inc(&backend_dev->rdma_dev_res->stats.missing_cqe);
714 backend_dev->rdma_dev_res->stats.rx_bufs++;
715 backend_dev->rdma_dev_res->stats.rx_srq++;
716
717 return;
718
719 err_dealloc_cqe_ctx:
720 backend_dev->rdma_dev_res->stats.rx_bufs_err++;
721 rdma_rm_dealloc_cqe_ctx(backend_dev->rdma_dev_res, bctx_id);
722
723 err_free_bctx:
724 g_free(bctx);
725 }
726
727 int rdma_backend_create_pd(RdmaBackendDev *backend_dev, RdmaBackendPD *pd)
728 {
729 pd->ibpd = ibv_alloc_pd(backend_dev->context);
730
731 if (!pd->ibpd) {
732 rdma_error_report("ibv_alloc_pd fail, errno=%d", errno);
733 return -EIO;
734 }
735
736 return 0;
737 }
738
739 void rdma_backend_destroy_pd(RdmaBackendPD *pd)
740 {
741 if (pd->ibpd) {
742 ibv_dealloc_pd(pd->ibpd);
743 }
744 }
745
746 int rdma_backend_create_mr(RdmaBackendMR *mr, RdmaBackendPD *pd, void *addr,
747 size_t length, uint64_t guest_start, int access)
748 {
749 #ifdef LEGACY_RDMA_REG_MR
750 mr->ibmr = ibv_reg_mr(pd->ibpd, addr, length, access);
751 #else
752 mr->ibmr = ibv_reg_mr_iova(pd->ibpd, addr, length, guest_start, access);
753 #endif
754 if (!mr->ibmr) {
755 rdma_error_report("ibv_reg_mr fail, errno=%d", errno);
756 return -EIO;
757 }
758
759 mr->ibpd = pd->ibpd;
760
761 return 0;
762 }
763
764 void rdma_backend_destroy_mr(RdmaBackendMR *mr)
765 {
766 if (mr->ibmr) {
767 ibv_dereg_mr(mr->ibmr);
768 }
769 }
770
771 int rdma_backend_create_cq(RdmaBackendDev *backend_dev, RdmaBackendCQ *cq,
772 int cqe)
773 {
774 int rc;
775
776 cq->ibcq = ibv_create_cq(backend_dev->context, cqe + 1, NULL,
777 backend_dev->channel, 0);
778 if (!cq->ibcq) {
779 rdma_error_report("ibv_create_cq fail, errno=%d", errno);
780 return -EIO;
781 }
782
783 rc = ibv_req_notify_cq(cq->ibcq, 0);
784 if (rc) {
785 rdma_warn_report("ibv_req_notify_cq fail, rc=%d, errno=%d", rc, errno);
786 }
787
788 cq->backend_dev = backend_dev;
789
790 return 0;
791 }
792
793 void rdma_backend_destroy_cq(RdmaBackendCQ *cq)
794 {
795 if (cq->ibcq) {
796 ibv_destroy_cq(cq->ibcq);
797 }
798 }
799
800 int rdma_backend_create_qp(RdmaBackendQP *qp, uint8_t qp_type,
801 RdmaBackendPD *pd, RdmaBackendCQ *scq,
802 RdmaBackendCQ *rcq, RdmaBackendSRQ *srq,
803 uint32_t max_send_wr, uint32_t max_recv_wr,
804 uint32_t max_send_sge, uint32_t max_recv_sge)
805 {
806 struct ibv_qp_init_attr attr = {};
807
808 qp->ibqp = 0;
809
810 switch (qp_type) {
811 case IBV_QPT_GSI:
812 return 0;
813
814 case IBV_QPT_RC:
815 /* fall through */
816 case IBV_QPT_UD:
817 /* do nothing */
818 break;
819
820 default:
821 rdma_error_report("Unsupported QP type %d", qp_type);
822 return -EIO;
823 }
824
825 attr.qp_type = qp_type;
826 attr.send_cq = scq->ibcq;
827 attr.recv_cq = rcq->ibcq;
828 attr.cap.max_send_wr = max_send_wr;
829 attr.cap.max_recv_wr = max_recv_wr;
830 attr.cap.max_send_sge = max_send_sge;
831 attr.cap.max_recv_sge = max_recv_sge;
832 if (srq) {
833 attr.srq = srq->ibsrq;
834 }
835
836 qp->ibqp = ibv_create_qp(pd->ibpd, &attr);
837 if (!qp->ibqp) {
838 rdma_error_report("ibv_create_qp fail, errno=%d", errno);
839 return -EIO;
840 }
841
842 rdma_protected_gslist_init(&qp->cqe_ctx_list);
843
844 qp->ibpd = pd->ibpd;
845
846 /* TODO: Query QP to get max_inline_data and save it to be used in send */
847
848 return 0;
849 }
850
851 int rdma_backend_qp_state_init(RdmaBackendDev *backend_dev, RdmaBackendQP *qp,
852 uint8_t qp_type, uint32_t qkey)
853 {
854 struct ibv_qp_attr attr = {};
855 int rc, attr_mask;
856
857 attr_mask = IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT;
858 attr.qp_state = IBV_QPS_INIT;
859 attr.pkey_index = 0;
860 attr.port_num = backend_dev->port_num;
861
862 switch (qp_type) {
863 case IBV_QPT_RC:
864 attr_mask |= IBV_QP_ACCESS_FLAGS;
865 trace_rdma_backend_rc_qp_state_init(qp->ibqp->qp_num);
866 break;
867
868 case IBV_QPT_UD:
869 attr.qkey = qkey;
870 attr_mask |= IBV_QP_QKEY;
871 trace_rdma_backend_ud_qp_state_init(qp->ibqp->qp_num, qkey);
872 break;
873
874 default:
875 rdma_error_report("Unsupported QP type %d", qp_type);
876 return -EIO;
877 }
878
879 rc = ibv_modify_qp(qp->ibqp, &attr, attr_mask);
880 if (rc) {
881 rdma_error_report("ibv_modify_qp fail, rc=%d, errno=%d", rc, errno);
882 return -EIO;
883 }
884
885 return 0;
886 }
887
888 int rdma_backend_qp_state_rtr(RdmaBackendDev *backend_dev, RdmaBackendQP *qp,
889 uint8_t qp_type, uint8_t sgid_idx,
890 union ibv_gid *dgid, uint32_t dqpn,
891 uint32_t rq_psn, uint32_t qkey, bool use_qkey)
892 {
893 struct ibv_qp_attr attr = {};
894 union ibv_gid ibv_gid = {
895 .global.interface_id = dgid->global.interface_id,
896 .global.subnet_prefix = dgid->global.subnet_prefix
897 };
898 int rc, attr_mask;
899
900 attr.qp_state = IBV_QPS_RTR;
901 attr_mask = IBV_QP_STATE;
902
903 qp->sgid_idx = sgid_idx;
904
905 switch (qp_type) {
906 case IBV_QPT_RC:
907 attr.path_mtu = IBV_MTU_1024;
908 attr.dest_qp_num = dqpn;
909 attr.max_dest_rd_atomic = 1;
910 attr.min_rnr_timer = 12;
911 attr.ah_attr.port_num = backend_dev->port_num;
912 attr.ah_attr.is_global = 1;
913 attr.ah_attr.grh.hop_limit = 1;
914 attr.ah_attr.grh.dgid = ibv_gid;
915 attr.ah_attr.grh.sgid_index = qp->sgid_idx;
916 attr.rq_psn = rq_psn;
917
918 attr_mask |= IBV_QP_AV | IBV_QP_PATH_MTU | IBV_QP_DEST_QPN |
919 IBV_QP_RQ_PSN | IBV_QP_MAX_DEST_RD_ATOMIC |
920 IBV_QP_MIN_RNR_TIMER;
921
922 trace_rdma_backend_rc_qp_state_rtr(qp->ibqp->qp_num,
923 be64_to_cpu(ibv_gid.global.
924 subnet_prefix),
925 be64_to_cpu(ibv_gid.global.
926 interface_id),
927 qp->sgid_idx, dqpn, rq_psn);
928 break;
929
930 case IBV_QPT_UD:
931 if (use_qkey) {
932 attr.qkey = qkey;
933 attr_mask |= IBV_QP_QKEY;
934 }
935 trace_rdma_backend_ud_qp_state_rtr(qp->ibqp->qp_num, use_qkey ? qkey :
936 0);
937 break;
938 }
939
940 rc = ibv_modify_qp(qp->ibqp, &attr, attr_mask);
941 if (rc) {
942 rdma_error_report("ibv_modify_qp fail, rc=%d, errno=%d", rc, errno);
943 return -EIO;
944 }
945
946 return 0;
947 }
948
949 int rdma_backend_qp_state_rts(RdmaBackendQP *qp, uint8_t qp_type,
950 uint32_t sq_psn, uint32_t qkey, bool use_qkey)
951 {
952 struct ibv_qp_attr attr = {};
953 int rc, attr_mask;
954
955 attr.qp_state = IBV_QPS_RTS;
956 attr.sq_psn = sq_psn;
957 attr_mask = IBV_QP_STATE | IBV_QP_SQ_PSN;
958
959 switch (qp_type) {
960 case IBV_QPT_RC:
961 attr.timeout = 14;
962 attr.retry_cnt = 7;
963 attr.rnr_retry = 7;
964 attr.max_rd_atomic = 1;
965
966 attr_mask |= IBV_QP_TIMEOUT | IBV_QP_RETRY_CNT | IBV_QP_RNR_RETRY |
967 IBV_QP_MAX_QP_RD_ATOMIC;
968 trace_rdma_backend_rc_qp_state_rts(qp->ibqp->qp_num, sq_psn);
969 break;
970
971 case IBV_QPT_UD:
972 if (use_qkey) {
973 attr.qkey = qkey;
974 attr_mask |= IBV_QP_QKEY;
975 }
976 trace_rdma_backend_ud_qp_state_rts(qp->ibqp->qp_num, sq_psn,
977 use_qkey ? qkey : 0);
978 break;
979 }
980
981 rc = ibv_modify_qp(qp->ibqp, &attr, attr_mask);
982 if (rc) {
983 rdma_error_report("ibv_modify_qp fail, rc=%d, errno=%d", rc, errno);
984 return -EIO;
985 }
986
987 return 0;
988 }
989
990 int rdma_backend_query_qp(RdmaBackendQP *qp, struct ibv_qp_attr *attr,
991 int attr_mask, struct ibv_qp_init_attr *init_attr)
992 {
993 if (!qp->ibqp) {
994 attr->qp_state = IBV_QPS_RTS;
995 return 0;
996 }
997
998 return ibv_query_qp(qp->ibqp, attr, attr_mask, init_attr);
999 }
1000
1001 void rdma_backend_destroy_qp(RdmaBackendQP *qp, RdmaDeviceResources *dev_res)
1002 {
1003 if (qp->ibqp) {
1004 ibv_destroy_qp(qp->ibqp);
1005 }
1006 g_slist_foreach(qp->cqe_ctx_list.list, free_cqe_ctx, dev_res);
1007 rdma_protected_gslist_destroy(&qp->cqe_ctx_list);
1008 }
1009
1010 int rdma_backend_create_srq(RdmaBackendSRQ *srq, RdmaBackendPD *pd,
1011 uint32_t max_wr, uint32_t max_sge,
1012 uint32_t srq_limit)
1013 {
1014 struct ibv_srq_init_attr srq_init_attr = {};
1015
1016 srq_init_attr.attr.max_wr = max_wr;
1017 srq_init_attr.attr.max_sge = max_sge;
1018 srq_init_attr.attr.srq_limit = srq_limit;
1019
1020 srq->ibsrq = ibv_create_srq(pd->ibpd, &srq_init_attr);
1021 if (!srq->ibsrq) {
1022 rdma_error_report("ibv_create_srq failed, errno=%d", errno);
1023 return -EIO;
1024 }
1025
1026 rdma_protected_gslist_init(&srq->cqe_ctx_list);
1027
1028 return 0;
1029 }
1030
1031 int rdma_backend_query_srq(RdmaBackendSRQ *srq, struct ibv_srq_attr *srq_attr)
1032 {
1033 if (!srq->ibsrq) {
1034 return -EINVAL;
1035 }
1036
1037 return ibv_query_srq(srq->ibsrq, srq_attr);
1038 }
1039
1040 int rdma_backend_modify_srq(RdmaBackendSRQ *srq, struct ibv_srq_attr *srq_attr,
1041 int srq_attr_mask)
1042 {
1043 if (!srq->ibsrq) {
1044 return -EINVAL;
1045 }
1046
1047 return ibv_modify_srq(srq->ibsrq, srq_attr, srq_attr_mask);
1048 }
1049
1050 void rdma_backend_destroy_srq(RdmaBackendSRQ *srq, RdmaDeviceResources *dev_res)
1051 {
1052 if (srq->ibsrq) {
1053 ibv_destroy_srq(srq->ibsrq);
1054 }
1055 g_slist_foreach(srq->cqe_ctx_list.list, free_cqe_ctx, dev_res);
1056 rdma_protected_gslist_destroy(&srq->cqe_ctx_list);
1057 }
1058
1059 #define CHK_ATTR(req, dev, member, fmt) ({ \
1060 trace_rdma_check_dev_attr(#member, dev.member, req->member); \
1061 if (req->member > dev.member) { \
1062 rdma_warn_report("%s = "fmt" is higher than host device capability "fmt, \
1063 #member, req->member, dev.member); \
1064 req->member = dev.member; \
1065 } \
1066 })
1067
1068 static int init_device_caps(RdmaBackendDev *backend_dev,
1069 struct ibv_device_attr *dev_attr)
1070 {
1071 struct ibv_device_attr bk_dev_attr;
1072 int rc;
1073
1074 rc = ibv_query_device(backend_dev->context, &bk_dev_attr);
1075 if (rc) {
1076 rdma_error_report("ibv_query_device fail, rc=%d, errno=%d", rc, errno);
1077 return -EIO;
1078 }
1079
1080 dev_attr->max_sge = MAX_SGE;
1081 dev_attr->max_srq_sge = MAX_SGE;
1082
1083 CHK_ATTR(dev_attr, bk_dev_attr, max_mr_size, "%" PRId64);
1084 CHK_ATTR(dev_attr, bk_dev_attr, max_qp, "%d");
1085 CHK_ATTR(dev_attr, bk_dev_attr, max_sge, "%d");
1086 CHK_ATTR(dev_attr, bk_dev_attr, max_cq, "%d");
1087 CHK_ATTR(dev_attr, bk_dev_attr, max_mr, "%d");
1088 CHK_ATTR(dev_attr, bk_dev_attr, max_pd, "%d");
1089 CHK_ATTR(dev_attr, bk_dev_attr, max_qp_rd_atom, "%d");
1090 CHK_ATTR(dev_attr, bk_dev_attr, max_qp_init_rd_atom, "%d");
1091 CHK_ATTR(dev_attr, bk_dev_attr, max_ah, "%d");
1092 CHK_ATTR(dev_attr, bk_dev_attr, max_srq, "%d");
1093
1094 return 0;
1095 }
1096
1097 static inline void build_mad_hdr(struct ibv_grh *grh, union ibv_gid *sgid,
1098 union ibv_gid *my_gid, int paylen)
1099 {
1100 grh->paylen = htons(paylen);
1101 grh->sgid = *sgid;
1102 grh->dgid = *my_gid;
1103 }
1104
1105 static void process_incoming_mad_req(RdmaBackendDev *backend_dev,
1106 RdmaCmMuxMsg *msg)
1107 {
1108 unsigned long cqe_ctx_id;
1109 BackendCtx *bctx;
1110 char *mad;
1111
1112 trace_mad_message("recv", msg->umad.mad, msg->umad_len);
1113
1114 cqe_ctx_id = rdma_protected_gqueue_pop_int64(&backend_dev->recv_mads_list);
1115 if (cqe_ctx_id == -ENOENT) {
1116 rdma_warn_report("No more free MADs buffers, waiting for a while");
1117 sleep(THR_POLL_TO);
1118 return;
1119 }
1120
1121 bctx = rdma_rm_get_cqe_ctx(backend_dev->rdma_dev_res, cqe_ctx_id);
1122 if (unlikely(!bctx)) {
1123 rdma_error_report("No matching ctx for req %ld", cqe_ctx_id);
1124 backend_dev->rdma_dev_res->stats.mad_rx_err++;
1125 return;
1126 }
1127
1128 mad = rdma_pci_dma_map(backend_dev->dev, bctx->sge.addr,
1129 bctx->sge.length);
1130 if (!mad || bctx->sge.length < msg->umad_len + MAD_HDR_SIZE) {
1131 backend_dev->rdma_dev_res->stats.mad_rx_err++;
1132 complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_INV_MAD_BUFF,
1133 bctx->up_ctx);
1134 } else {
1135 struct ibv_wc wc = {};
1136 memset(mad, 0, bctx->sge.length);
1137 build_mad_hdr((struct ibv_grh *)mad,
1138 (union ibv_gid *)&msg->umad.hdr.addr.gid, &msg->hdr.sgid,
1139 msg->umad_len);
1140 memcpy(&mad[MAD_HDR_SIZE], msg->umad.mad, msg->umad_len);
1141 rdma_pci_dma_unmap(backend_dev->dev, mad, bctx->sge.length);
1142
1143 wc.byte_len = msg->umad_len;
1144 wc.status = IBV_WC_SUCCESS;
1145 wc.wc_flags = IBV_WC_GRH;
1146 backend_dev->rdma_dev_res->stats.mad_rx++;
1147 comp_handler(bctx->up_ctx, &wc);
1148 }
1149
1150 g_free(bctx);
1151 rdma_rm_dealloc_cqe_ctx(backend_dev->rdma_dev_res, cqe_ctx_id);
1152 }
1153
1154 static inline int rdmacm_mux_can_receive(void *opaque)
1155 {
1156 RdmaBackendDev *backend_dev = (RdmaBackendDev *)opaque;
1157
1158 return rdmacm_mux_can_process_async(backend_dev);
1159 }
1160
1161 static void rdmacm_mux_read(void *opaque, const uint8_t *buf, int size)
1162 {
1163 RdmaBackendDev *backend_dev = (RdmaBackendDev *)opaque;
1164 RdmaCmMuxMsg *msg = (RdmaCmMuxMsg *)buf;
1165
1166 trace_rdmacm_mux("read", msg->hdr.msg_type, msg->hdr.op_code);
1167
1168 if (msg->hdr.msg_type != RDMACM_MUX_MSG_TYPE_REQ &&
1169 msg->hdr.op_code != RDMACM_MUX_OP_CODE_MAD) {
1170 rdma_error_report("Error: Not a MAD request, skipping");
1171 return;
1172 }
1173 process_incoming_mad_req(backend_dev, msg);
1174 }
1175
1176 static int mad_init(RdmaBackendDev *backend_dev, CharBackend *mad_chr_be)
1177 {
1178 int ret;
1179
1180 backend_dev->rdmacm_mux.chr_be = mad_chr_be;
1181
1182 ret = qemu_chr_fe_backend_connected(backend_dev->rdmacm_mux.chr_be);
1183 if (!ret) {
1184 rdma_error_report("Missing chardev for MAD multiplexer");
1185 return -EIO;
1186 }
1187
1188 rdma_protected_gqueue_init(&backend_dev->recv_mads_list);
1189
1190 enable_rdmacm_mux_async(backend_dev);
1191
1192 qemu_chr_fe_set_handlers(backend_dev->rdmacm_mux.chr_be,
1193 rdmacm_mux_can_receive, rdmacm_mux_read, NULL,
1194 NULL, backend_dev, NULL, true);
1195
1196 return 0;
1197 }
1198
1199 static void mad_stop(RdmaBackendDev *backend_dev)
1200 {
1201 clean_recv_mads(backend_dev);
1202 }
1203
1204 static void mad_fini(RdmaBackendDev *backend_dev)
1205 {
1206 disable_rdmacm_mux_async(backend_dev);
1207 qemu_chr_fe_disconnect(backend_dev->rdmacm_mux.chr_be);
1208 rdma_protected_gqueue_destroy(&backend_dev->recv_mads_list);
1209 }
1210
1211 int rdma_backend_get_gid_index(RdmaBackendDev *backend_dev,
1212 union ibv_gid *gid)
1213 {
1214 union ibv_gid sgid;
1215 int ret;
1216 int i = 0;
1217
1218 do {
1219 ret = ibv_query_gid(backend_dev->context, backend_dev->port_num, i,
1220 &sgid);
1221 i++;
1222 } while (!ret && (memcmp(&sgid, gid, sizeof(*gid))));
1223
1224 trace_rdma_backend_get_gid_index(be64_to_cpu(gid->global.subnet_prefix),
1225 be64_to_cpu(gid->global.interface_id),
1226 i - 1);
1227
1228 return ret ? ret : i - 1;
1229 }
1230
1231 int rdma_backend_add_gid(RdmaBackendDev *backend_dev, const char *ifname,
1232 union ibv_gid *gid)
1233 {
1234 RdmaCmMuxMsg msg = {};
1235 int ret;
1236
1237 trace_rdma_backend_gid_change("add", be64_to_cpu(gid->global.subnet_prefix),
1238 be64_to_cpu(gid->global.interface_id));
1239
1240 msg.hdr.op_code = RDMACM_MUX_OP_CODE_REG;
1241 memcpy(msg.hdr.sgid.raw, gid->raw, sizeof(msg.hdr.sgid));
1242
1243 ret = rdmacm_mux_send(backend_dev, &msg);
1244 if (ret) {
1245 rdma_error_report("Failed to register GID to rdma_umadmux (%d)", ret);
1246 return -EIO;
1247 }
1248
1249 qapi_event_send_rdma_gid_status_changed(ifname, true,
1250 gid->global.subnet_prefix,
1251 gid->global.interface_id);
1252
1253 return ret;
1254 }
1255
1256 int rdma_backend_del_gid(RdmaBackendDev *backend_dev, const char *ifname,
1257 union ibv_gid *gid)
1258 {
1259 RdmaCmMuxMsg msg = {};
1260 int ret;
1261
1262 trace_rdma_backend_gid_change("del", be64_to_cpu(gid->global.subnet_prefix),
1263 be64_to_cpu(gid->global.interface_id));
1264
1265 msg.hdr.op_code = RDMACM_MUX_OP_CODE_UNREG;
1266 memcpy(msg.hdr.sgid.raw, gid->raw, sizeof(msg.hdr.sgid));
1267
1268 ret = rdmacm_mux_send(backend_dev, &msg);
1269 if (ret) {
1270 rdma_error_report("Failed to unregister GID from rdma_umadmux (%d)",
1271 ret);
1272 return -EIO;
1273 }
1274
1275 qapi_event_send_rdma_gid_status_changed(ifname, false,
1276 gid->global.subnet_prefix,
1277 gid->global.interface_id);
1278
1279 return 0;
1280 }
1281
1282 int rdma_backend_init(RdmaBackendDev *backend_dev, PCIDevice *pdev,
1283 RdmaDeviceResources *rdma_dev_res,
1284 const char *backend_device_name, uint8_t port_num,
1285 struct ibv_device_attr *dev_attr, CharBackend *mad_chr_be)
1286 {
1287 int i;
1288 int ret = 0;
1289 int num_ibv_devices;
1290 struct ibv_device **dev_list;
1291
1292 memset(backend_dev, 0, sizeof(*backend_dev));
1293
1294 backend_dev->dev = pdev;
1295 backend_dev->port_num = port_num;
1296 backend_dev->rdma_dev_res = rdma_dev_res;
1297
1298 rdma_backend_register_comp_handler(dummy_comp_handler);
1299
1300 dev_list = ibv_get_device_list(&num_ibv_devices);
1301 if (!dev_list) {
1302 rdma_error_report("Failed to get IB devices list");
1303 return -EIO;
1304 }
1305
1306 if (num_ibv_devices == 0) {
1307 rdma_error_report("No IB devices were found");
1308 ret = -ENXIO;
1309 goto out_free_dev_list;
1310 }
1311
1312 if (backend_device_name) {
1313 for (i = 0; dev_list[i]; ++i) {
1314 if (!strcmp(ibv_get_device_name(dev_list[i]),
1315 backend_device_name)) {
1316 break;
1317 }
1318 }
1319
1320 backend_dev->ib_dev = dev_list[i];
1321 if (!backend_dev->ib_dev) {
1322 rdma_error_report("Failed to find IB device %s",
1323 backend_device_name);
1324 ret = -EIO;
1325 goto out_free_dev_list;
1326 }
1327 } else {
1328 backend_dev->ib_dev = *dev_list;
1329 }
1330
1331 rdma_info_report("uverb device %s", backend_dev->ib_dev->dev_name);
1332
1333 backend_dev->context = ibv_open_device(backend_dev->ib_dev);
1334 if (!backend_dev->context) {
1335 rdma_error_report("Failed to open IB device %s",
1336 ibv_get_device_name(backend_dev->ib_dev));
1337 ret = -EIO;
1338 goto out;
1339 }
1340
1341 backend_dev->channel = ibv_create_comp_channel(backend_dev->context);
1342 if (!backend_dev->channel) {
1343 rdma_error_report("Failed to create IB communication channel");
1344 ret = -EIO;
1345 goto out_close_device;
1346 }
1347
1348 ret = init_device_caps(backend_dev, dev_attr);
1349 if (ret) {
1350 rdma_error_report("Failed to initialize device capabilities");
1351 ret = -EIO;
1352 goto out_destroy_comm_channel;
1353 }
1354
1355
1356 ret = mad_init(backend_dev, mad_chr_be);
1357 if (ret) {
1358 rdma_error_report("Failed to initialize mad");
1359 ret = -EIO;
1360 goto out_destroy_comm_channel;
1361 }
1362
1363 backend_dev->comp_thread.run = false;
1364 backend_dev->comp_thread.is_running = false;
1365
1366 ah_cache_init();
1367
1368 goto out_free_dev_list;
1369
1370 out_destroy_comm_channel:
1371 ibv_destroy_comp_channel(backend_dev->channel);
1372
1373 out_close_device:
1374 ibv_close_device(backend_dev->context);
1375
1376 out_free_dev_list:
1377 ibv_free_device_list(dev_list);
1378
1379 out:
1380 return ret;
1381 }
1382
1383
1384 void rdma_backend_start(RdmaBackendDev *backend_dev)
1385 {
1386 start_comp_thread(backend_dev);
1387 }
1388
1389 void rdma_backend_stop(RdmaBackendDev *backend_dev)
1390 {
1391 mad_stop(backend_dev);
1392 stop_backend_thread(&backend_dev->comp_thread);
1393 }
1394
1395 void rdma_backend_fini(RdmaBackendDev *backend_dev)
1396 {
1397 mad_fini(backend_dev);
1398 g_hash_table_destroy(ah_hash);
1399 ibv_destroy_comp_channel(backend_dev->channel);
1400 ibv_close_device(backend_dev->context);
1401 }
armbru's QEMU hackery