?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'block-5.11-2021-01-10' of git://git.kernel.dk/linux-block
[linux/fpc-iii.git] / drivers / infiniband / core / cma.c
blobc51b84b2d2f3750ef1c9d183640d4119a2a57a20
1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3 * Copyright (c) 2005 Voltaire Inc. All rights reserved.
4 * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved.
5 * Copyright (c) 1999-2019, Mellanox Technologies, Inc. All rights reserved.
6 * Copyright (c) 2005-2006 Intel Corporation. All rights reserved.
7 */
8
9 #include <linux/completion.h>
10 #include <linux/in.h>
11 #include <linux/in6.h>
12 #include <linux/mutex.h>
13 #include <linux/random.h>
14 #include <linux/igmp.h>
15 #include <linux/xarray.h>
16 #include <linux/inetdevice.h>
17 #include <linux/slab.h>
18 #include <linux/module.h>
19 #include <net/route.h>
20
21 #include <net/net_namespace.h>
22 #include <net/netns/generic.h>
23 #include <net/tcp.h>
24 #include <net/ipv6.h>
25 #include <net/ip_fib.h>
26 #include <net/ip6_route.h>
27
28 #include <rdma/rdma_cm.h>
29 #include <rdma/rdma_cm_ib.h>
30 #include <rdma/rdma_netlink.h>
31 #include <rdma/ib.h>
32 #include <rdma/ib_cache.h>
33 #include <rdma/ib_cm.h>
34 #include <rdma/ib_sa.h>
35 #include <rdma/iw_cm.h>
36
37 #include "core_priv.h"
38 #include "cma_priv.h"
39 #include "cma_trace.h"
40
41 MODULE_AUTHOR("Sean Hefty");
42 MODULE_DESCRIPTION("Generic RDMA CM Agent");
43 MODULE_LICENSE("Dual BSD/GPL");
44
45 #define CMA_CM_RESPONSE_TIMEOUT 20
46 #define CMA_QUERY_CLASSPORT_INFO_TIMEOUT 3000
47 #define CMA_MAX_CM_RETRIES 15
48 #define CMA_CM_MRA_SETTING (IB_CM_MRA_FLAG_DELAY | 24)
49 #define CMA_IBOE_PACKET_LIFETIME 18
50 #define CMA_PREFERRED_ROCE_GID_TYPE IB_GID_TYPE_ROCE_UDP_ENCAP
51
52 static const char * const cma_events[] = {
53 [RDMA_CM_EVENT_ADDR_RESOLVED] = "address resolved",
54 [RDMA_CM_EVENT_ADDR_ERROR] = "address error",
55 [RDMA_CM_EVENT_ROUTE_RESOLVED] = "route resolved ",
56 [RDMA_CM_EVENT_ROUTE_ERROR] = "route error",
57 [RDMA_CM_EVENT_CONNECT_REQUEST] = "connect request",
58 [RDMA_CM_EVENT_CONNECT_RESPONSE] = "connect response",
59 [RDMA_CM_EVENT_CONNECT_ERROR] = "connect error",
60 [RDMA_CM_EVENT_UNREACHABLE] = "unreachable",
61 [RDMA_CM_EVENT_REJECTED] = "rejected",
62 [RDMA_CM_EVENT_ESTABLISHED] = "established",
63 [RDMA_CM_EVENT_DISCONNECTED] = "disconnected",
64 [RDMA_CM_EVENT_DEVICE_REMOVAL] = "device removal",
65 [RDMA_CM_EVENT_MULTICAST_JOIN] = "multicast join",
66 [RDMA_CM_EVENT_MULTICAST_ERROR] = "multicast error",
67 [RDMA_CM_EVENT_ADDR_CHANGE] = "address change",
68 [RDMA_CM_EVENT_TIMEWAIT_EXIT] = "timewait exit",
69 };
70
71 static void cma_set_mgid(struct rdma_id_private *id_priv, struct sockaddr *addr,
72 union ib_gid *mgid);
73
74 const char *__attribute_const__ rdma_event_msg(enum rdma_cm_event_type event)
75 {
76 size_t index = event;
77
78 return (index < ARRAY_SIZE(cma_events) && cma_events[index]) ?
79 cma_events[index] : "unrecognized event";
80 }
81 EXPORT_SYMBOL(rdma_event_msg);
82
83 const char *__attribute_const__ rdma_reject_msg(struct rdma_cm_id *id,
84 int reason)
85 {
86 if (rdma_ib_or_roce(id->device, id->port_num))
87 return ibcm_reject_msg(reason);
88
89 if (rdma_protocol_iwarp(id->device, id->port_num))
90 return iwcm_reject_msg(reason);
91
92 WARN_ON_ONCE(1);
93 return "unrecognized transport";
94 }
95 EXPORT_SYMBOL(rdma_reject_msg);
96
97 /**
98 * rdma_is_consumer_reject - return true if the consumer rejected the connect
99 * request.
100 * @id: Communication identifier that received the REJECT event.
101 * @reason: Value returned in the REJECT event status field.
102 */
103 static bool rdma_is_consumer_reject(struct rdma_cm_id *id, int reason)
104 {
105 if (rdma_ib_or_roce(id->device, id->port_num))
106 return reason == IB_CM_REJ_CONSUMER_DEFINED;
107
108 if (rdma_protocol_iwarp(id->device, id->port_num))
109 return reason == -ECONNREFUSED;
110
111 WARN_ON_ONCE(1);
112 return false;
113 }
114
115 const void *rdma_consumer_reject_data(struct rdma_cm_id *id,
116 struct rdma_cm_event *ev, u8 *data_len)
117 {
118 const void *p;
119
120 if (rdma_is_consumer_reject(id, ev->status)) {
121 *data_len = ev->param.conn.private_data_len;
122 p = ev->param.conn.private_data;
123 } else {
124 *data_len = 0;
125 p = NULL;
126 }
127 return p;
128 }
129 EXPORT_SYMBOL(rdma_consumer_reject_data);
130
131 /**
132 * rdma_iw_cm_id() - return the iw_cm_id pointer for this cm_id.
133 * @id: Communication Identifier
134 */
135 struct iw_cm_id *rdma_iw_cm_id(struct rdma_cm_id *id)
136 {
137 struct rdma_id_private *id_priv;
138
139 id_priv = container_of(id, struct rdma_id_private, id);
140 if (id->device->node_type == RDMA_NODE_RNIC)
141 return id_priv->cm_id.iw;
142 return NULL;
143 }
144 EXPORT_SYMBOL(rdma_iw_cm_id);
145
146 /**
147 * rdma_res_to_id() - return the rdma_cm_id pointer for this restrack.
148 * @res: rdma resource tracking entry pointer
149 */
150 struct rdma_cm_id *rdma_res_to_id(struct rdma_restrack_entry *res)
151 {
152 struct rdma_id_private *id_priv =
153 container_of(res, struct rdma_id_private, res);
154
155 return &id_priv->id;
156 }
157 EXPORT_SYMBOL(rdma_res_to_id);
158
159 static int cma_add_one(struct ib_device *device);
160 static void cma_remove_one(struct ib_device *device, void *client_data);
161
162 static struct ib_client cma_client = {
163 .name = "cma",
164 .add = cma_add_one,
165 .remove = cma_remove_one
166 };
167
168 static struct ib_sa_client sa_client;
169 static LIST_HEAD(dev_list);
170 static LIST_HEAD(listen_any_list);
171 static DEFINE_MUTEX(lock);
172 static struct workqueue_struct *cma_wq;
173 static unsigned int cma_pernet_id;
174
175 struct cma_pernet {
176 struct xarray tcp_ps;
177 struct xarray udp_ps;
178 struct xarray ipoib_ps;
179 struct xarray ib_ps;
180 };
181
182 static struct cma_pernet *cma_pernet(struct net *net)
183 {
184 return net_generic(net, cma_pernet_id);
185 }
186
187 static
188 struct xarray *cma_pernet_xa(struct net *net, enum rdma_ucm_port_space ps)
189 {
190 struct cma_pernet *pernet = cma_pernet(net);
191
192 switch (ps) {
193 case RDMA_PS_TCP:
194 return &pernet->tcp_ps;
195 case RDMA_PS_UDP:
196 return &pernet->udp_ps;
197 case RDMA_PS_IPOIB:
198 return &pernet->ipoib_ps;
199 case RDMA_PS_IB:
200 return &pernet->ib_ps;
201 default:
202 return NULL;
203 }
204 }
205
206 struct cma_device {
207 struct list_head list;
208 struct ib_device *device;
209 struct completion comp;
210 refcount_t refcount;
211 struct list_head id_list;
212 enum ib_gid_type *default_gid_type;
213 u8 *default_roce_tos;
214 };
215
216 struct rdma_bind_list {
217 enum rdma_ucm_port_space ps;
218 struct hlist_head owners;
219 unsigned short port;
220 };
221
222 struct class_port_info_context {
223 struct ib_class_port_info *class_port_info;
224 struct ib_device *device;
225 struct completion done;
226 struct ib_sa_query *sa_query;
227 u8 port_num;
228 };
229
230 static int cma_ps_alloc(struct net *net, enum rdma_ucm_port_space ps,
231 struct rdma_bind_list *bind_list, int snum)
232 {
233 struct xarray *xa = cma_pernet_xa(net, ps);
234
235 return xa_insert(xa, snum, bind_list, GFP_KERNEL);
236 }
237
238 static struct rdma_bind_list *cma_ps_find(struct net *net,
239 enum rdma_ucm_port_space ps, int snum)
240 {
241 struct xarray *xa = cma_pernet_xa(net, ps);
242
243 return xa_load(xa, snum);
244 }
245
246 static void cma_ps_remove(struct net *net, enum rdma_ucm_port_space ps,
247 int snum)
248 {
249 struct xarray *xa = cma_pernet_xa(net, ps);
250
251 xa_erase(xa, snum);
252 }
253
254 enum {
255 CMA_OPTION_AFONLY,
256 };
257
258 void cma_dev_get(struct cma_device *cma_dev)
259 {
260 refcount_inc(&cma_dev->refcount);
261 }
262
263 void cma_dev_put(struct cma_device *cma_dev)
264 {
265 if (refcount_dec_and_test(&cma_dev->refcount))
266 complete(&cma_dev->comp);
267 }
268
269 struct cma_device *cma_enum_devices_by_ibdev(cma_device_filter filter,
270 void *cookie)
271 {
272 struct cma_device *cma_dev;
273 struct cma_device *found_cma_dev = NULL;
274
275 mutex_lock(&lock);
276
277 list_for_each_entry(cma_dev, &dev_list, list)
278 if (filter(cma_dev->device, cookie)) {
279 found_cma_dev = cma_dev;
280 break;
281 }
282
283 if (found_cma_dev)
284 cma_dev_get(found_cma_dev);
285 mutex_unlock(&lock);
286 return found_cma_dev;
287 }
288
289 int cma_get_default_gid_type(struct cma_device *cma_dev,
290 unsigned int port)
291 {
292 if (!rdma_is_port_valid(cma_dev->device, port))
293 return -EINVAL;
294
295 return cma_dev->default_gid_type[port - rdma_start_port(cma_dev->device)];
296 }
297
298 int cma_set_default_gid_type(struct cma_device *cma_dev,
299 unsigned int port,
300 enum ib_gid_type default_gid_type)
301 {
302 unsigned long supported_gids;
303
304 if (!rdma_is_port_valid(cma_dev->device, port))
305 return -EINVAL;
306
307 if (default_gid_type == IB_GID_TYPE_IB &&
308 rdma_protocol_roce_eth_encap(cma_dev->device, port))
309 default_gid_type = IB_GID_TYPE_ROCE;
310
311 supported_gids = roce_gid_type_mask_support(cma_dev->device, port);
312
313 if (!(supported_gids & 1 << default_gid_type))
314 return -EINVAL;
315
316 cma_dev->default_gid_type[port - rdma_start_port(cma_dev->device)] =
317 default_gid_type;
318
319 return 0;
320 }
321
322 int cma_get_default_roce_tos(struct cma_device *cma_dev, unsigned int port)
323 {
324 if (!rdma_is_port_valid(cma_dev->device, port))
325 return -EINVAL;
326
327 return cma_dev->default_roce_tos[port - rdma_start_port(cma_dev->device)];
328 }
329
330 int cma_set_default_roce_tos(struct cma_device *cma_dev, unsigned int port,
331 u8 default_roce_tos)
332 {
333 if (!rdma_is_port_valid(cma_dev->device, port))
334 return -EINVAL;
335
336 cma_dev->default_roce_tos[port - rdma_start_port(cma_dev->device)] =
337 default_roce_tos;
338
339 return 0;
340 }
341 struct ib_device *cma_get_ib_dev(struct cma_device *cma_dev)
342 {
343 return cma_dev->device;
344 }
345
346 /*
347 * Device removal can occur at anytime, so we need extra handling to
348 * serialize notifying the user of device removal with other callbacks.
349 * We do this by disabling removal notification while a callback is in process,
350 * and reporting it after the callback completes.
351 */
352
353 struct cma_multicast {
354 struct rdma_id_private *id_priv;
355 struct ib_sa_multicast *sa_mc;
356 struct list_head list;
357 void *context;
358 struct sockaddr_storage addr;
359 u8 join_state;
360 };
361
362 struct cma_work {
363 struct work_struct work;
364 struct rdma_id_private *id;
365 enum rdma_cm_state old_state;
366 enum rdma_cm_state new_state;
367 struct rdma_cm_event event;
368 };
369
370 union cma_ip_addr {
371 struct in6_addr ip6;
372 struct {
373 __be32 pad[3];
374 __be32 addr;
375 } ip4;
376 };
377
378 struct cma_hdr {
379 u8 cma_version;
380 u8 ip_version; /* IP version: 7:4 */
381 __be16 port;
382 union cma_ip_addr src_addr;
383 union cma_ip_addr dst_addr;
384 };
385
386 #define CMA_VERSION 0x00
387
388 struct cma_req_info {
389 struct sockaddr_storage listen_addr_storage;
390 struct sockaddr_storage src_addr_storage;
391 struct ib_device *device;
392 union ib_gid local_gid;
393 __be64 service_id;
394 int port;
395 bool has_gid;
396 u16 pkey;
397 };
398
399 static int cma_comp_exch(struct rdma_id_private *id_priv,
400 enum rdma_cm_state comp, enum rdma_cm_state exch)
401 {
402 unsigned long flags;
403 int ret;
404
405 /*
406 * The FSM uses a funny double locking where state is protected by both
407 * the handler_mutex and the spinlock. State is not allowed to change
408 * to/from a handler_mutex protected value without also holding
409 * handler_mutex.
410 */
411 if (comp == RDMA_CM_CONNECT || exch == RDMA_CM_CONNECT)
412 lockdep_assert_held(&id_priv->handler_mutex);
413
414 spin_lock_irqsave(&id_priv->lock, flags);
415 if ((ret = (id_priv->state == comp)))
416 id_priv->state = exch;
417 spin_unlock_irqrestore(&id_priv->lock, flags);
418 return ret;
419 }
420
421 static inline u8 cma_get_ip_ver(const struct cma_hdr *hdr)
422 {
423 return hdr->ip_version >> 4;
424 }
425
426 static inline void cma_set_ip_ver(struct cma_hdr *hdr, u8 ip_ver)
427 {
428 hdr->ip_version = (ip_ver << 4) | (hdr->ip_version & 0xF);
429 }
430
431 static int cma_igmp_send(struct net_device *ndev, union ib_gid *mgid, bool join)
432 {
433 struct in_device *in_dev = NULL;
434
435 if (ndev) {
436 rtnl_lock();
437 in_dev = __in_dev_get_rtnl(ndev);
438 if (in_dev) {
439 if (join)
440 ip_mc_inc_group(in_dev,
441 *(__be32 *)(mgid->raw + 12));
442 else
443 ip_mc_dec_group(in_dev,
444 *(__be32 *)(mgid->raw + 12));
445 }
446 rtnl_unlock();
447 }
448 return (in_dev) ? 0 : -ENODEV;
449 }
450
451 static void _cma_attach_to_dev(struct rdma_id_private *id_priv,
452 struct cma_device *cma_dev)
453 {
454 cma_dev_get(cma_dev);
455 id_priv->cma_dev = cma_dev;
456 id_priv->id.device = cma_dev->device;
457 id_priv->id.route.addr.dev_addr.transport =
458 rdma_node_get_transport(cma_dev->device->node_type);
459 list_add_tail(&id_priv->list, &cma_dev->id_list);
460 rdma_restrack_add(&id_priv->res);
461
462 trace_cm_id_attach(id_priv, cma_dev->device);
463 }
464
465 static void cma_attach_to_dev(struct rdma_id_private *id_priv,
466 struct cma_device *cma_dev)
467 {
468 _cma_attach_to_dev(id_priv, cma_dev);
469 id_priv->gid_type =
470 cma_dev->default_gid_type[id_priv->id.port_num -
471 rdma_start_port(cma_dev->device)];
472 }
473
474 static void cma_release_dev(struct rdma_id_private *id_priv)
475 {
476 mutex_lock(&lock);
477 list_del(&id_priv->list);
478 cma_dev_put(id_priv->cma_dev);
479 id_priv->cma_dev = NULL;
480 if (id_priv->id.route.addr.dev_addr.sgid_attr) {
481 rdma_put_gid_attr(id_priv->id.route.addr.dev_addr.sgid_attr);
482 id_priv->id.route.addr.dev_addr.sgid_attr = NULL;
483 }
484 mutex_unlock(&lock);
485 }
486
487 static inline struct sockaddr *cma_src_addr(struct rdma_id_private *id_priv)
488 {
489 return (struct sockaddr *) &id_priv->id.route.addr.src_addr;
490 }
491
492 static inline struct sockaddr *cma_dst_addr(struct rdma_id_private *id_priv)
493 {
494 return (struct sockaddr *) &id_priv->id.route.addr.dst_addr;
495 }
496
497 static inline unsigned short cma_family(struct rdma_id_private *id_priv)
498 {
499 return id_priv->id.route.addr.src_addr.ss_family;
500 }
501
502 static int cma_set_qkey(struct rdma_id_private *id_priv, u32 qkey)
503 {
504 struct ib_sa_mcmember_rec rec;
505 int ret = 0;
506
507 if (id_priv->qkey) {
508 if (qkey && id_priv->qkey != qkey)
509 return -EINVAL;
510 return 0;
511 }
512
513 if (qkey) {
514 id_priv->qkey = qkey;
515 return 0;
516 }
517
518 switch (id_priv->id.ps) {
519 case RDMA_PS_UDP:
520 case RDMA_PS_IB:
521 id_priv->qkey = RDMA_UDP_QKEY;
522 break;
523 case RDMA_PS_IPOIB:
524 ib_addr_get_mgid(&id_priv->id.route.addr.dev_addr, &rec.mgid);
525 ret = ib_sa_get_mcmember_rec(id_priv->id.device,
526 id_priv->id.port_num, &rec.mgid,
527 &rec);
528 if (!ret)
529 id_priv->qkey = be32_to_cpu(rec.qkey);
530 break;
531 default:
532 break;
533 }
534 return ret;
535 }
536
537 static void cma_translate_ib(struct sockaddr_ib *sib, struct rdma_dev_addr *dev_addr)
538 {
539 dev_addr->dev_type = ARPHRD_INFINIBAND;
540 rdma_addr_set_sgid(dev_addr, (union ib_gid *) &sib->sib_addr);
541 ib_addr_set_pkey(dev_addr, ntohs(sib->sib_pkey));
542 }
543
544 static int cma_translate_addr(struct sockaddr *addr, struct rdma_dev_addr *dev_addr)
545 {
546 int ret;
547
548 if (addr->sa_family != AF_IB) {
549 ret = rdma_translate_ip(addr, dev_addr);
550 } else {
551 cma_translate_ib((struct sockaddr_ib *) addr, dev_addr);
552 ret = 0;
553 }
554
555 return ret;
556 }
557
558 static const struct ib_gid_attr *
559 cma_validate_port(struct ib_device *device, u8 port,
560 enum ib_gid_type gid_type,
561 union ib_gid *gid,
562 struct rdma_id_private *id_priv)
563 {
564 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
565 int bound_if_index = dev_addr->bound_dev_if;
566 const struct ib_gid_attr *sgid_attr;
567 int dev_type = dev_addr->dev_type;
568 struct net_device *ndev = NULL;
569
570 if (!rdma_dev_access_netns(device, id_priv->id.route.addr.dev_addr.net))
571 return ERR_PTR(-ENODEV);
572
573 if ((dev_type == ARPHRD_INFINIBAND) && !rdma_protocol_ib(device, port))
574 return ERR_PTR(-ENODEV);
575
576 if ((dev_type != ARPHRD_INFINIBAND) && rdma_protocol_ib(device, port))
577 return ERR_PTR(-ENODEV);
578
579 if (dev_type == ARPHRD_ETHER && rdma_protocol_roce(device, port)) {
580 ndev = dev_get_by_index(dev_addr->net, bound_if_index);
581 if (!ndev)
582 return ERR_PTR(-ENODEV);
583 } else {
584 gid_type = IB_GID_TYPE_IB;
585 }
586
587 sgid_attr = rdma_find_gid_by_port(device, gid, gid_type, port, ndev);
588 if (ndev)
589 dev_put(ndev);
590 return sgid_attr;
591 }
592
593 static void cma_bind_sgid_attr(struct rdma_id_private *id_priv,
594 const struct ib_gid_attr *sgid_attr)
595 {
596 WARN_ON(id_priv->id.route.addr.dev_addr.sgid_attr);
597 id_priv->id.route.addr.dev_addr.sgid_attr = sgid_attr;
598 }
599
600 /**
601 * cma_acquire_dev_by_src_ip - Acquire cma device, port, gid attribute
602 * based on source ip address.
603 * @id_priv: cm_id which should be bound to cma device
604 *
605 * cma_acquire_dev_by_src_ip() binds cm id to cma device, port and GID attribute
606 * based on source IP address. It returns 0 on success or error code otherwise.
607 * It is applicable to active and passive side cm_id.
608 */
609 static int cma_acquire_dev_by_src_ip(struct rdma_id_private *id_priv)
610 {
611 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
612 const struct ib_gid_attr *sgid_attr;
613 union ib_gid gid, iboe_gid, *gidp;
614 struct cma_device *cma_dev;
615 enum ib_gid_type gid_type;
616 int ret = -ENODEV;
617 unsigned int port;
618
619 if (dev_addr->dev_type != ARPHRD_INFINIBAND &&
620 id_priv->id.ps == RDMA_PS_IPOIB)
621 return -EINVAL;
622
623 rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr,
624 &iboe_gid);
625
626 memcpy(&gid, dev_addr->src_dev_addr +
627 rdma_addr_gid_offset(dev_addr), sizeof(gid));
628
629 mutex_lock(&lock);
630 list_for_each_entry(cma_dev, &dev_list, list) {
631 rdma_for_each_port (cma_dev->device, port) {
632 gidp = rdma_protocol_roce(cma_dev->device, port) ?
633 &iboe_gid : &gid;
634 gid_type = cma_dev->default_gid_type[port - 1];
635 sgid_attr = cma_validate_port(cma_dev->device, port,
636 gid_type, gidp, id_priv);
637 if (!IS_ERR(sgid_attr)) {
638 id_priv->id.port_num = port;
639 cma_bind_sgid_attr(id_priv, sgid_attr);
640 cma_attach_to_dev(id_priv, cma_dev);
641 ret = 0;
642 goto out;
643 }
644 }
645 }
646 out:
647 mutex_unlock(&lock);
648 return ret;
649 }
650
651 /**
652 * cma_ib_acquire_dev - Acquire cma device, port and SGID attribute
653 * @id_priv: cm id to bind to cma device
654 * @listen_id_priv: listener cm id to match against
655 * @req: Pointer to req structure containaining incoming
656 * request information
657 * cma_ib_acquire_dev() acquires cma device, port and SGID attribute when
658 * rdma device matches for listen_id and incoming request. It also verifies
659 * that a GID table entry is present for the source address.
660 * Returns 0 on success, or returns error code otherwise.
661 */
662 static int cma_ib_acquire_dev(struct rdma_id_private *id_priv,
663 const struct rdma_id_private *listen_id_priv,
664 struct cma_req_info *req)
665 {
666 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
667 const struct ib_gid_attr *sgid_attr;
668 enum ib_gid_type gid_type;
669 union ib_gid gid;
670
671 if (dev_addr->dev_type != ARPHRD_INFINIBAND &&
672 id_priv->id.ps == RDMA_PS_IPOIB)
673 return -EINVAL;
674
675 if (rdma_protocol_roce(req->device, req->port))
676 rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr,
677 &gid);
678 else
679 memcpy(&gid, dev_addr->src_dev_addr +
680 rdma_addr_gid_offset(dev_addr), sizeof(gid));
681
682 gid_type = listen_id_priv->cma_dev->default_gid_type[req->port - 1];
683 sgid_attr = cma_validate_port(req->device, req->port,
684 gid_type, &gid, id_priv);
685 if (IS_ERR(sgid_attr))
686 return PTR_ERR(sgid_attr);
687
688 id_priv->id.port_num = req->port;
689 cma_bind_sgid_attr(id_priv, sgid_attr);
690 /* Need to acquire lock to protect against reader
691 * of cma_dev->id_list such as cma_netdev_callback() and
692 * cma_process_remove().
693 */
694 mutex_lock(&lock);
695 cma_attach_to_dev(id_priv, listen_id_priv->cma_dev);
696 mutex_unlock(&lock);
697 return 0;
698 }
699
700 static int cma_iw_acquire_dev(struct rdma_id_private *id_priv,
701 const struct rdma_id_private *listen_id_priv)
702 {
703 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
704 const struct ib_gid_attr *sgid_attr;
705 struct cma_device *cma_dev;
706 enum ib_gid_type gid_type;
707 int ret = -ENODEV;
708 unsigned int port;
709 union ib_gid gid;
710
711 if (dev_addr->dev_type != ARPHRD_INFINIBAND &&
712 id_priv->id.ps == RDMA_PS_IPOIB)
713 return -EINVAL;
714
715 memcpy(&gid, dev_addr->src_dev_addr +
716 rdma_addr_gid_offset(dev_addr), sizeof(gid));
717
718 mutex_lock(&lock);
719
720 cma_dev = listen_id_priv->cma_dev;
721 port = listen_id_priv->id.port_num;
722 gid_type = listen_id_priv->gid_type;
723 sgid_attr = cma_validate_port(cma_dev->device, port,
724 gid_type, &gid, id_priv);
725 if (!IS_ERR(sgid_attr)) {
726 id_priv->id.port_num = port;
727 cma_bind_sgid_attr(id_priv, sgid_attr);
728 ret = 0;
729 goto out;
730 }
731
732 list_for_each_entry(cma_dev, &dev_list, list) {
733 rdma_for_each_port (cma_dev->device, port) {
734 if (listen_id_priv->cma_dev == cma_dev &&
735 listen_id_priv->id.port_num == port)
736 continue;
737
738 gid_type = cma_dev->default_gid_type[port - 1];
739 sgid_attr = cma_validate_port(cma_dev->device, port,
740 gid_type, &gid, id_priv);
741 if (!IS_ERR(sgid_attr)) {
742 id_priv->id.port_num = port;
743 cma_bind_sgid_attr(id_priv, sgid_attr);
744 ret = 0;
745 goto out;
746 }
747 }
748 }
749
750 out:
751 if (!ret)
752 cma_attach_to_dev(id_priv, cma_dev);
753
754 mutex_unlock(&lock);
755 return ret;
756 }
757
758 /*
759 * Select the source IB device and address to reach the destination IB address.
760 */
761 static int cma_resolve_ib_dev(struct rdma_id_private *id_priv)
762 {
763 struct cma_device *cma_dev, *cur_dev;
764 struct sockaddr_ib *addr;
765 union ib_gid gid, sgid, *dgid;
766 unsigned int p;
767 u16 pkey, index;
768 enum ib_port_state port_state;
769 int i;
770
771 cma_dev = NULL;
772 addr = (struct sockaddr_ib *) cma_dst_addr(id_priv);
773 dgid = (union ib_gid *) &addr->sib_addr;
774 pkey = ntohs(addr->sib_pkey);
775
776 mutex_lock(&lock);
777 list_for_each_entry(cur_dev, &dev_list, list) {
778 rdma_for_each_port (cur_dev->device, p) {
779 if (!rdma_cap_af_ib(cur_dev->device, p))
780 continue;
781
782 if (ib_find_cached_pkey(cur_dev->device, p, pkey, &index))
783 continue;
784
785 if (ib_get_cached_port_state(cur_dev->device, p, &port_state))
786 continue;
787 for (i = 0; !rdma_query_gid(cur_dev->device,
788 p, i, &gid);
789 i++) {
790 if (!memcmp(&gid, dgid, sizeof(gid))) {
791 cma_dev = cur_dev;
792 sgid = gid;
793 id_priv->id.port_num = p;
794 goto found;
795 }
796
797 if (!cma_dev && (gid.global.subnet_prefix ==
798 dgid->global.subnet_prefix) &&
799 port_state == IB_PORT_ACTIVE) {
800 cma_dev = cur_dev;
801 sgid = gid;
802 id_priv->id.port_num = p;
803 goto found;
804 }
805 }
806 }
807 }
808 mutex_unlock(&lock);
809 return -ENODEV;
810
811 found:
812 cma_attach_to_dev(id_priv, cma_dev);
813 mutex_unlock(&lock);
814 addr = (struct sockaddr_ib *)cma_src_addr(id_priv);
815 memcpy(&addr->sib_addr, &sgid, sizeof(sgid));
816 cma_translate_ib(addr, &id_priv->id.route.addr.dev_addr);
817 return 0;
818 }
819
820 static void cma_id_get(struct rdma_id_private *id_priv)
821 {
822 refcount_inc(&id_priv->refcount);
823 }
824
825 static void cma_id_put(struct rdma_id_private *id_priv)
826 {
827 if (refcount_dec_and_test(&id_priv->refcount))
828 complete(&id_priv->comp);
829 }
830
831 static struct rdma_id_private *
832 __rdma_create_id(struct net *net, rdma_cm_event_handler event_handler,
833 void *context, enum rdma_ucm_port_space ps,
834 enum ib_qp_type qp_type, const struct rdma_id_private *parent)
835 {
836 struct rdma_id_private *id_priv;
837
838 id_priv = kzalloc(sizeof *id_priv, GFP_KERNEL);
839 if (!id_priv)
840 return ERR_PTR(-ENOMEM);
841
842 id_priv->state = RDMA_CM_IDLE;
843 id_priv->id.context = context;
844 id_priv->id.event_handler = event_handler;
845 id_priv->id.ps = ps;
846 id_priv->id.qp_type = qp_type;
847 id_priv->tos_set = false;
848 id_priv->timeout_set = false;
849 id_priv->gid_type = IB_GID_TYPE_IB;
850 spin_lock_init(&id_priv->lock);
851 mutex_init(&id_priv->qp_mutex);
852 init_completion(&id_priv->comp);
853 refcount_set(&id_priv->refcount, 1);
854 mutex_init(&id_priv->handler_mutex);
855 INIT_LIST_HEAD(&id_priv->listen_list);
856 INIT_LIST_HEAD(&id_priv->mc_list);
857 get_random_bytes(&id_priv->seq_num, sizeof id_priv->seq_num);
858 id_priv->id.route.addr.dev_addr.net = get_net(net);
859 id_priv->seq_num &= 0x00ffffff;
860
861 rdma_restrack_new(&id_priv->res, RDMA_RESTRACK_CM_ID);
862 if (parent)
863 rdma_restrack_parent_name(&id_priv->res, &parent->res);
864
865 return id_priv;
866 }
867
868 struct rdma_cm_id *
869 __rdma_create_kernel_id(struct net *net, rdma_cm_event_handler event_handler,
870 void *context, enum rdma_ucm_port_space ps,
871 enum ib_qp_type qp_type, const char *caller)
872 {
873 struct rdma_id_private *ret;
874
875 ret = __rdma_create_id(net, event_handler, context, ps, qp_type, NULL);
876 if (IS_ERR(ret))
877 return ERR_CAST(ret);
878
879 rdma_restrack_set_name(&ret->res, caller);
880 return &ret->id;
881 }
882 EXPORT_SYMBOL(__rdma_create_kernel_id);
883
884 struct rdma_cm_id *rdma_create_user_id(rdma_cm_event_handler event_handler,
885 void *context,
886 enum rdma_ucm_port_space ps,
887 enum ib_qp_type qp_type)
888 {
889 struct rdma_id_private *ret;
890
891 ret = __rdma_create_id(current->nsproxy->net_ns, event_handler, context,
892 ps, qp_type, NULL);
893 if (IS_ERR(ret))
894 return ERR_CAST(ret);
895
896 rdma_restrack_set_name(&ret->res, NULL);
897 return &ret->id;
898 }
899 EXPORT_SYMBOL(rdma_create_user_id);
900
901 static int cma_init_ud_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
902 {
903 struct ib_qp_attr qp_attr;
904 int qp_attr_mask, ret;
905
906 qp_attr.qp_state = IB_QPS_INIT;
907 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
908 if (ret)
909 return ret;
910
911 ret = ib_modify_qp(qp, &qp_attr, qp_attr_mask);
912 if (ret)
913 return ret;
914
915 qp_attr.qp_state = IB_QPS_RTR;
916 ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE);
917 if (ret)
918 return ret;
919
920 qp_attr.qp_state = IB_QPS_RTS;
921 qp_attr.sq_psn = 0;
922 ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE | IB_QP_SQ_PSN);
923
924 return ret;
925 }
926
927 static int cma_init_conn_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
928 {
929 struct ib_qp_attr qp_attr;
930 int qp_attr_mask, ret;
931
932 qp_attr.qp_state = IB_QPS_INIT;
933 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
934 if (ret)
935 return ret;
936
937 return ib_modify_qp(qp, &qp_attr, qp_attr_mask);
938 }
939
940 int rdma_create_qp(struct rdma_cm_id *id, struct ib_pd *pd,
941 struct ib_qp_init_attr *qp_init_attr)
942 {
943 struct rdma_id_private *id_priv;
944 struct ib_qp *qp;
945 int ret;
946
947 id_priv = container_of(id, struct rdma_id_private, id);
948 if (id->device != pd->device) {
949 ret = -EINVAL;
950 goto out_err;
951 }
952
953 qp_init_attr->port_num = id->port_num;
954 qp = ib_create_qp(pd, qp_init_attr);
955 if (IS_ERR(qp)) {
956 ret = PTR_ERR(qp);
957 goto out_err;
958 }
959
960 if (id->qp_type == IB_QPT_UD)
961 ret = cma_init_ud_qp(id_priv, qp);
962 else
963 ret = cma_init_conn_qp(id_priv, qp);
964 if (ret)
965 goto out_destroy;
966
967 id->qp = qp;
968 id_priv->qp_num = qp->qp_num;
969 id_priv->srq = (qp->srq != NULL);
970 trace_cm_qp_create(id_priv, pd, qp_init_attr, 0);
971 return 0;
972 out_destroy:
973 ib_destroy_qp(qp);
974 out_err:
975 trace_cm_qp_create(id_priv, pd, qp_init_attr, ret);
976 return ret;
977 }
978 EXPORT_SYMBOL(rdma_create_qp);
979
980 void rdma_destroy_qp(struct rdma_cm_id *id)
981 {
982 struct rdma_id_private *id_priv;
983
984 id_priv = container_of(id, struct rdma_id_private, id);
985 trace_cm_qp_destroy(id_priv);
986 mutex_lock(&id_priv->qp_mutex);
987 ib_destroy_qp(id_priv->id.qp);
988 id_priv->id.qp = NULL;
989 mutex_unlock(&id_priv->qp_mutex);
990 }
991 EXPORT_SYMBOL(rdma_destroy_qp);
992
993 static int cma_modify_qp_rtr(struct rdma_id_private *id_priv,
994 struct rdma_conn_param *conn_param)
995 {
996 struct ib_qp_attr qp_attr;
997 int qp_attr_mask, ret;
998
999 mutex_lock(&id_priv->qp_mutex);
1000 if (!id_priv->id.qp) {
1001 ret = 0;
1002 goto out;
1003 }
1004
1005 /* Need to update QP attributes from default values. */
1006 qp_attr.qp_state = IB_QPS_INIT;
1007 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
1008 if (ret)
1009 goto out;
1010
1011 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
1012 if (ret)
1013 goto out;
1014
1015 qp_attr.qp_state = IB_QPS_RTR;
1016 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
1017 if (ret)
1018 goto out;
1019
1020 BUG_ON(id_priv->cma_dev->device != id_priv->id.device);
1021
1022 if (conn_param)
1023 qp_attr.max_dest_rd_atomic = conn_param->responder_resources;
1024 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
1025 out:
1026 mutex_unlock(&id_priv->qp_mutex);
1027 return ret;
1028 }
1029
1030 static int cma_modify_qp_rts(struct rdma_id_private *id_priv,
1031 struct rdma_conn_param *conn_param)
1032 {
1033 struct ib_qp_attr qp_attr;
1034 int qp_attr_mask, ret;
1035
1036 mutex_lock(&id_priv->qp_mutex);
1037 if (!id_priv->id.qp) {
1038 ret = 0;
1039 goto out;
1040 }
1041
1042 qp_attr.qp_state = IB_QPS_RTS;
1043 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
1044 if (ret)
1045 goto out;
1046
1047 if (conn_param)
1048 qp_attr.max_rd_atomic = conn_param->initiator_depth;
1049 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
1050 out:
1051 mutex_unlock(&id_priv->qp_mutex);
1052 return ret;
1053 }
1054
1055 static int cma_modify_qp_err(struct rdma_id_private *id_priv)
1056 {
1057 struct ib_qp_attr qp_attr;
1058 int ret;
1059
1060 mutex_lock(&id_priv->qp_mutex);
1061 if (!id_priv->id.qp) {
1062 ret = 0;
1063 goto out;
1064 }
1065
1066 qp_attr.qp_state = IB_QPS_ERR;
1067 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, IB_QP_STATE);
1068 out:
1069 mutex_unlock(&id_priv->qp_mutex);
1070 return ret;
1071 }
1072
1073 static int cma_ib_init_qp_attr(struct rdma_id_private *id_priv,
1074 struct ib_qp_attr *qp_attr, int *qp_attr_mask)
1075 {
1076 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
1077 int ret;
1078 u16 pkey;
1079
1080 if (rdma_cap_eth_ah(id_priv->id.device, id_priv->id.port_num))
1081 pkey = 0xffff;
1082 else
1083 pkey = ib_addr_get_pkey(dev_addr);
1084
1085 ret = ib_find_cached_pkey(id_priv->id.device, id_priv->id.port_num,
1086 pkey, &qp_attr->pkey_index);
1087 if (ret)
1088 return ret;
1089
1090 qp_attr->port_num = id_priv->id.port_num;
1091 *qp_attr_mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT;
1092
1093 if (id_priv->id.qp_type == IB_QPT_UD) {
1094 ret = cma_set_qkey(id_priv, 0);
1095 if (ret)
1096 return ret;
1097
1098 qp_attr->qkey = id_priv->qkey;
1099 *qp_attr_mask |= IB_QP_QKEY;
1100 } else {
1101 qp_attr->qp_access_flags = 0;
1102 *qp_attr_mask |= IB_QP_ACCESS_FLAGS;
1103 }
1104 return 0;
1105 }
1106
1107 int rdma_init_qp_attr(struct rdma_cm_id *id, struct ib_qp_attr *qp_attr,
1108 int *qp_attr_mask)
1109 {
1110 struct rdma_id_private *id_priv;
1111 int ret = 0;
1112
1113 id_priv = container_of(id, struct rdma_id_private, id);
1114 if (rdma_cap_ib_cm(id->device, id->port_num)) {
1115 if (!id_priv->cm_id.ib || (id_priv->id.qp_type == IB_QPT_UD))
1116 ret = cma_ib_init_qp_attr(id_priv, qp_attr, qp_attr_mask);
1117 else
1118 ret = ib_cm_init_qp_attr(id_priv->cm_id.ib, qp_attr,
1119 qp_attr_mask);
1120
1121 if (qp_attr->qp_state == IB_QPS_RTR)
1122 qp_attr->rq_psn = id_priv->seq_num;
1123 } else if (rdma_cap_iw_cm(id->device, id->port_num)) {
1124 if (!id_priv->cm_id.iw) {
1125 qp_attr->qp_access_flags = 0;
1126 *qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS;
1127 } else
1128 ret = iw_cm_init_qp_attr(id_priv->cm_id.iw, qp_attr,
1129 qp_attr_mask);
1130 qp_attr->port_num = id_priv->id.port_num;
1131 *qp_attr_mask |= IB_QP_PORT;
1132 } else
1133 ret = -ENOSYS;
1134
1135 if ((*qp_attr_mask & IB_QP_TIMEOUT) && id_priv->timeout_set)
1136 qp_attr->timeout = id_priv->timeout;
1137
1138 return ret;
1139 }
1140 EXPORT_SYMBOL(rdma_init_qp_attr);
1141
1142 static inline bool cma_zero_addr(const struct sockaddr *addr)
1143 {
1144 switch (addr->sa_family) {
1145 case AF_INET:
1146 return ipv4_is_zeronet(((struct sockaddr_in *)addr)->sin_addr.s_addr);
1147 case AF_INET6:
1148 return ipv6_addr_any(&((struct sockaddr_in6 *)addr)->sin6_addr);
1149 case AF_IB:
1150 return ib_addr_any(&((struct sockaddr_ib *)addr)->sib_addr);
1151 default:
1152 return false;
1153 }
1154 }
1155
1156 static inline bool cma_loopback_addr(const struct sockaddr *addr)
1157 {
1158 switch (addr->sa_family) {
1159 case AF_INET:
1160 return ipv4_is_loopback(
1161 ((struct sockaddr_in *)addr)->sin_addr.s_addr);
1162 case AF_INET6:
1163 return ipv6_addr_loopback(
1164 &((struct sockaddr_in6 *)addr)->sin6_addr);
1165 case AF_IB:
1166 return ib_addr_loopback(
1167 &((struct sockaddr_ib *)addr)->sib_addr);
1168 default:
1169 return false;
1170 }
1171 }
1172
1173 static inline bool cma_any_addr(const struct sockaddr *addr)
1174 {
1175 return cma_zero_addr(addr) || cma_loopback_addr(addr);
1176 }
1177
1178 static int cma_addr_cmp(const struct sockaddr *src, const struct sockaddr *dst)
1179 {
1180 if (src->sa_family != dst->sa_family)
1181 return -1;
1182
1183 switch (src->sa_family) {
1184 case AF_INET:
1185 return ((struct sockaddr_in *)src)->sin_addr.s_addr !=
1186 ((struct sockaddr_in *)dst)->sin_addr.s_addr;
1187 case AF_INET6: {
1188 struct sockaddr_in6 *src_addr6 = (struct sockaddr_in6 *)src;
1189 struct sockaddr_in6 *dst_addr6 = (struct sockaddr_in6 *)dst;
1190 bool link_local;
1191
1192 if (ipv6_addr_cmp(&src_addr6->sin6_addr,
1193 &dst_addr6->sin6_addr))
1194 return 1;
1195 link_local = ipv6_addr_type(&dst_addr6->sin6_addr) &
1196 IPV6_ADDR_LINKLOCAL;
1197 /* Link local must match their scope_ids */
1198 return link_local ? (src_addr6->sin6_scope_id !=
1199 dst_addr6->sin6_scope_id) :
1200 0;
1201 }
1202
1203 default:
1204 return ib_addr_cmp(&((struct sockaddr_ib *) src)->sib_addr,
1205 &((struct sockaddr_ib *) dst)->sib_addr);
1206 }
1207 }
1208
1209 static __be16 cma_port(const struct sockaddr *addr)
1210 {
1211 struct sockaddr_ib *sib;
1212
1213 switch (addr->sa_family) {
1214 case AF_INET:
1215 return ((struct sockaddr_in *) addr)->sin_port;
1216 case AF_INET6:
1217 return ((struct sockaddr_in6 *) addr)->sin6_port;
1218 case AF_IB:
1219 sib = (struct sockaddr_ib *) addr;
1220 return htons((u16) (be64_to_cpu(sib->sib_sid) &
1221 be64_to_cpu(sib->sib_sid_mask)));
1222 default:
1223 return 0;
1224 }
1225 }
1226
1227 static inline int cma_any_port(const struct sockaddr *addr)
1228 {
1229 return !cma_port(addr);
1230 }
1231
1232 static void cma_save_ib_info(struct sockaddr *src_addr,
1233 struct sockaddr *dst_addr,
1234 const struct rdma_cm_id *listen_id,
1235 const struct sa_path_rec *path)
1236 {
1237 struct sockaddr_ib *listen_ib, *ib;
1238
1239 listen_ib = (struct sockaddr_ib *) &listen_id->route.addr.src_addr;
1240 if (src_addr) {
1241 ib = (struct sockaddr_ib *)src_addr;
1242 ib->sib_family = AF_IB;
1243 if (path) {
1244 ib->sib_pkey = path->pkey;
1245 ib->sib_flowinfo = path->flow_label;
1246 memcpy(&ib->sib_addr, &path->sgid, 16);
1247 ib->sib_sid = path->service_id;
1248 ib->sib_scope_id = 0;
1249 } else {
1250 ib->sib_pkey = listen_ib->sib_pkey;
1251 ib->sib_flowinfo = listen_ib->sib_flowinfo;
1252 ib->sib_addr = listen_ib->sib_addr;
1253 ib->sib_sid = listen_ib->sib_sid;
1254 ib->sib_scope_id = listen_ib->sib_scope_id;
1255 }
1256 ib->sib_sid_mask = cpu_to_be64(0xffffffffffffffffULL);
1257 }
1258 if (dst_addr) {
1259 ib = (struct sockaddr_ib *)dst_addr;
1260 ib->sib_family = AF_IB;
1261 if (path) {
1262 ib->sib_pkey = path->pkey;
1263 ib->sib_flowinfo = path->flow_label;
1264 memcpy(&ib->sib_addr, &path->dgid, 16);
1265 }
1266 }
1267 }
1268
1269 static void cma_save_ip4_info(struct sockaddr_in *src_addr,
1270 struct sockaddr_in *dst_addr,
1271 struct cma_hdr *hdr,
1272 __be16 local_port)
1273 {
1274 if (src_addr) {
1275 *src_addr = (struct sockaddr_in) {
1276 .sin_family = AF_INET,
1277 .sin_addr.s_addr = hdr->dst_addr.ip4.addr,
1278 .sin_port = local_port,
1279 };
1280 }
1281
1282 if (dst_addr) {
1283 *dst_addr = (struct sockaddr_in) {
1284 .sin_family = AF_INET,
1285 .sin_addr.s_addr = hdr->src_addr.ip4.addr,
1286 .sin_port = hdr->port,
1287 };
1288 }
1289 }
1290
1291 static void cma_save_ip6_info(struct sockaddr_in6 *src_addr,
1292 struct sockaddr_in6 *dst_addr,
1293 struct cma_hdr *hdr,
1294 __be16 local_port)
1295 {
1296 if (src_addr) {
1297 *src_addr = (struct sockaddr_in6) {
1298 .sin6_family = AF_INET6,
1299 .sin6_addr = hdr->dst_addr.ip6,
1300 .sin6_port = local_port,
1301 };
1302 }
1303
1304 if (dst_addr) {
1305 *dst_addr = (struct sockaddr_in6) {
1306 .sin6_family = AF_INET6,
1307 .sin6_addr = hdr->src_addr.ip6,
1308 .sin6_port = hdr->port,
1309 };
1310 }
1311 }
1312
1313 static u16 cma_port_from_service_id(__be64 service_id)
1314 {
1315 return (u16)be64_to_cpu(service_id);
1316 }
1317
1318 static int cma_save_ip_info(struct sockaddr *src_addr,
1319 struct sockaddr *dst_addr,
1320 const struct ib_cm_event *ib_event,
1321 __be64 service_id)
1322 {
1323 struct cma_hdr *hdr;
1324 __be16 port;
1325
1326 hdr = ib_event->private_data;
1327 if (hdr->cma_version != CMA_VERSION)
1328 return -EINVAL;
1329
1330 port = htons(cma_port_from_service_id(service_id));
1331
1332 switch (cma_get_ip_ver(hdr)) {
1333 case 4:
1334 cma_save_ip4_info((struct sockaddr_in *)src_addr,
1335 (struct sockaddr_in *)dst_addr, hdr, port);
1336 break;
1337 case 6:
1338 cma_save_ip6_info((struct sockaddr_in6 *)src_addr,
1339 (struct sockaddr_in6 *)dst_addr, hdr, port);
1340 break;
1341 default:
1342 return -EAFNOSUPPORT;
1343 }
1344
1345 return 0;
1346 }
1347
1348 static int cma_save_net_info(struct sockaddr *src_addr,
1349 struct sockaddr *dst_addr,
1350 const struct rdma_cm_id *listen_id,
1351 const struct ib_cm_event *ib_event,
1352 sa_family_t sa_family, __be64 service_id)
1353 {
1354 if (sa_family == AF_IB) {
1355 if (ib_event->event == IB_CM_REQ_RECEIVED)
1356 cma_save_ib_info(src_addr, dst_addr, listen_id,
1357 ib_event->param.req_rcvd.primary_path);
1358 else if (ib_event->event == IB_CM_SIDR_REQ_RECEIVED)
1359 cma_save_ib_info(src_addr, dst_addr, listen_id, NULL);
1360 return 0;
1361 }
1362
1363 return cma_save_ip_info(src_addr, dst_addr, ib_event, service_id);
1364 }
1365
1366 static int cma_save_req_info(const struct ib_cm_event *ib_event,
1367 struct cma_req_info *req)
1368 {
1369 const struct ib_cm_req_event_param *req_param =
1370 &ib_event->param.req_rcvd;
1371 const struct ib_cm_sidr_req_event_param *sidr_param =
1372 &ib_event->param.sidr_req_rcvd;
1373
1374 switch (ib_event->event) {
1375 case IB_CM_REQ_RECEIVED:
1376 req->device = req_param->listen_id->device;
1377 req->port = req_param->port;
1378 memcpy(&req->local_gid, &req_param->primary_path->sgid,
1379 sizeof(req->local_gid));
1380 req->has_gid = true;
1381 req->service_id = req_param->primary_path->service_id;
1382 req->pkey = be16_to_cpu(req_param->primary_path->pkey);
1383 if (req->pkey != req_param->bth_pkey)
1384 pr_warn_ratelimited("RDMA CMA: got different BTH P_Key (0x%x) and primary path P_Key (0x%x)\n"
1385 "RDMA CMA: in the future this may cause the request to be dropped\n",
1386 req_param->bth_pkey, req->pkey);
1387 break;
1388 case IB_CM_SIDR_REQ_RECEIVED:
1389 req->device = sidr_param->listen_id->device;
1390 req->port = sidr_param->port;
1391 req->has_gid = false;
1392 req->service_id = sidr_param->service_id;
1393 req->pkey = sidr_param->pkey;
1394 if (req->pkey != sidr_param->bth_pkey)
1395 pr_warn_ratelimited("RDMA CMA: got different BTH P_Key (0x%x) and SIDR request payload P_Key (0x%x)\n"
1396 "RDMA CMA: in the future this may cause the request to be dropped\n",
1397 sidr_param->bth_pkey, req->pkey);
1398 break;
1399 default:
1400 return -EINVAL;
1401 }
1402
1403 return 0;
1404 }
1405
1406 static bool validate_ipv4_net_dev(struct net_device *net_dev,
1407 const struct sockaddr_in *dst_addr,
1408 const struct sockaddr_in *src_addr)
1409 {
1410 __be32 daddr = dst_addr->sin_addr.s_addr,
1411 saddr = src_addr->sin_addr.s_addr;
1412 struct fib_result res;
1413 struct flowi4 fl4;
1414 int err;
1415 bool ret;
1416
1417 if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
1418 ipv4_is_lbcast(daddr) || ipv4_is_zeronet(saddr) ||
1419 ipv4_is_zeronet(daddr) || ipv4_is_loopback(daddr) ||
1420 ipv4_is_loopback(saddr))
1421 return false;
1422
1423 memset(&fl4, 0, sizeof(fl4));
1424 fl4.flowi4_iif = net_dev->ifindex;
1425 fl4.daddr = daddr;
1426 fl4.saddr = saddr;
1427
1428 rcu_read_lock();
1429 err = fib_lookup(dev_net(net_dev), &fl4, &res, 0);
1430 ret = err == 0 && FIB_RES_DEV(res) == net_dev;
1431 rcu_read_unlock();
1432
1433 return ret;
1434 }
1435
1436 static bool validate_ipv6_net_dev(struct net_device *net_dev,
1437 const struct sockaddr_in6 *dst_addr,
1438 const struct sockaddr_in6 *src_addr)
1439 {
1440 #if IS_ENABLED(CONFIG_IPV6)
1441 const int strict = ipv6_addr_type(&dst_addr->sin6_addr) &
1442 IPV6_ADDR_LINKLOCAL;
1443 struct rt6_info *rt = rt6_lookup(dev_net(net_dev), &dst_addr->sin6_addr,
1444 &src_addr->sin6_addr, net_dev->ifindex,
1445 NULL, strict);
1446 bool ret;
1447
1448 if (!rt)
1449 return false;
1450
1451 ret = rt->rt6i_idev->dev == net_dev;
1452 ip6_rt_put(rt);
1453
1454 return ret;
1455 #else
1456 return false;
1457 #endif
1458 }
1459
1460 static bool validate_net_dev(struct net_device *net_dev,
1461 const struct sockaddr *daddr,
1462 const struct sockaddr *saddr)
1463 {
1464 const struct sockaddr_in *daddr4 = (const struct sockaddr_in *)daddr;
1465 const struct sockaddr_in *saddr4 = (const struct sockaddr_in *)saddr;
1466 const struct sockaddr_in6 *daddr6 = (const struct sockaddr_in6 *)daddr;
1467 const struct sockaddr_in6 *saddr6 = (const struct sockaddr_in6 *)saddr;
1468
1469 switch (daddr->sa_family) {
1470 case AF_INET:
1471 return saddr->sa_family == AF_INET &&
1472 validate_ipv4_net_dev(net_dev, daddr4, saddr4);
1473
1474 case AF_INET6:
1475 return saddr->sa_family == AF_INET6 &&
1476 validate_ipv6_net_dev(net_dev, daddr6, saddr6);
1477
1478 default:
1479 return false;
1480 }
1481 }
1482
1483 static struct net_device *
1484 roce_get_net_dev_by_cm_event(const struct ib_cm_event *ib_event)
1485 {
1486 const struct ib_gid_attr *sgid_attr = NULL;
1487 struct net_device *ndev;
1488
1489 if (ib_event->event == IB_CM_REQ_RECEIVED)
1490 sgid_attr = ib_event->param.req_rcvd.ppath_sgid_attr;
1491 else if (ib_event->event == IB_CM_SIDR_REQ_RECEIVED)
1492 sgid_attr = ib_event->param.sidr_req_rcvd.sgid_attr;
1493
1494 if (!sgid_attr)
1495 return NULL;
1496
1497 rcu_read_lock();
1498 ndev = rdma_read_gid_attr_ndev_rcu(sgid_attr);
1499 if (IS_ERR(ndev))
1500 ndev = NULL;
1501 else
1502 dev_hold(ndev);
1503 rcu_read_unlock();
1504 return ndev;
1505 }
1506
1507 static struct net_device *cma_get_net_dev(const struct ib_cm_event *ib_event,
1508 struct cma_req_info *req)
1509 {
1510 struct sockaddr *listen_addr =
1511 (struct sockaddr *)&req->listen_addr_storage;
1512 struct sockaddr *src_addr = (struct sockaddr *)&req->src_addr_storage;
1513 struct net_device *net_dev;
1514 const union ib_gid *gid = req->has_gid ? &req->local_gid : NULL;
1515 int err;
1516
1517 err = cma_save_ip_info(listen_addr, src_addr, ib_event,
1518 req->service_id);
1519 if (err)
1520 return ERR_PTR(err);
1521
1522 if (rdma_protocol_roce(req->device, req->port))
1523 net_dev = roce_get_net_dev_by_cm_event(ib_event);
1524 else
1525 net_dev = ib_get_net_dev_by_params(req->device, req->port,
1526 req->pkey,
1527 gid, listen_addr);
1528 if (!net_dev)
1529 return ERR_PTR(-ENODEV);
1530
1531 return net_dev;
1532 }
1533
1534 static enum rdma_ucm_port_space rdma_ps_from_service_id(__be64 service_id)
1535 {
1536 return (be64_to_cpu(service_id) >> 16) & 0xffff;
1537 }
1538
1539 static bool cma_match_private_data(struct rdma_id_private *id_priv,
1540 const struct cma_hdr *hdr)
1541 {
1542 struct sockaddr *addr = cma_src_addr(id_priv);
1543 __be32 ip4_addr;
1544 struct in6_addr ip6_addr;
1545
1546 if (cma_any_addr(addr) && !id_priv->afonly)
1547 return true;
1548
1549 switch (addr->sa_family) {
1550 case AF_INET:
1551 ip4_addr = ((struct sockaddr_in *)addr)->sin_addr.s_addr;
1552 if (cma_get_ip_ver(hdr) != 4)
1553 return false;
1554 if (!cma_any_addr(addr) &&
1555 hdr->dst_addr.ip4.addr != ip4_addr)
1556 return false;
1557 break;
1558 case AF_INET6:
1559 ip6_addr = ((struct sockaddr_in6 *)addr)->sin6_addr;
1560 if (cma_get_ip_ver(hdr) != 6)
1561 return false;
1562 if (!cma_any_addr(addr) &&
1563 memcmp(&hdr->dst_addr.ip6, &ip6_addr, sizeof(ip6_addr)))
1564 return false;
1565 break;
1566 case AF_IB:
1567 return true;
1568 default:
1569 return false;
1570 }
1571
1572 return true;
1573 }
1574
1575 static bool cma_protocol_roce(const struct rdma_cm_id *id)
1576 {
1577 struct ib_device *device = id->device;
1578 const int port_num = id->port_num ?: rdma_start_port(device);
1579
1580 return rdma_protocol_roce(device, port_num);
1581 }
1582
1583 static bool cma_is_req_ipv6_ll(const struct cma_req_info *req)
1584 {
1585 const struct sockaddr *daddr =
1586 (const struct sockaddr *)&req->listen_addr_storage;
1587 const struct sockaddr_in6 *daddr6 = (const struct sockaddr_in6 *)daddr;
1588
1589 /* Returns true if the req is for IPv6 link local */
1590 return (daddr->sa_family == AF_INET6 &&
1591 (ipv6_addr_type(&daddr6->sin6_addr) & IPV6_ADDR_LINKLOCAL));
1592 }
1593
1594 static bool cma_match_net_dev(const struct rdma_cm_id *id,
1595 const struct net_device *net_dev,
1596 const struct cma_req_info *req)
1597 {
1598 const struct rdma_addr *addr = &id->route.addr;
1599
1600 if (!net_dev)
1601 /* This request is an AF_IB request */
1602 return (!id->port_num || id->port_num == req->port) &&
1603 (addr->src_addr.ss_family == AF_IB);
1604
1605 /*
1606 * If the request is not for IPv6 link local, allow matching
1607 * request to any netdevice of the one or multiport rdma device.
1608 */
1609 if (!cma_is_req_ipv6_ll(req))
1610 return true;
1611 /*
1612 * Net namespaces must match, and if the listner is listening
1613 * on a specific netdevice than netdevice must match as well.
1614 */
1615 if (net_eq(dev_net(net_dev), addr->dev_addr.net) &&
1616 (!!addr->dev_addr.bound_dev_if ==
1617 (addr->dev_addr.bound_dev_if == net_dev->ifindex)))
1618 return true;
1619 else
1620 return false;
1621 }
1622
1623 static struct rdma_id_private *cma_find_listener(
1624 const struct rdma_bind_list *bind_list,
1625 const struct ib_cm_id *cm_id,
1626 const struct ib_cm_event *ib_event,
1627 const struct cma_req_info *req,
1628 const struct net_device *net_dev)
1629 {
1630 struct rdma_id_private *id_priv, *id_priv_dev;
1631
1632 lockdep_assert_held(&lock);
1633
1634 if (!bind_list)
1635 return ERR_PTR(-EINVAL);
1636
1637 hlist_for_each_entry(id_priv, &bind_list->owners, node) {
1638 if (cma_match_private_data(id_priv, ib_event->private_data)) {
1639 if (id_priv->id.device == cm_id->device &&
1640 cma_match_net_dev(&id_priv->id, net_dev, req))
1641 return id_priv;
1642 list_for_each_entry(id_priv_dev,
1643 &id_priv->listen_list,
1644 listen_list) {
1645 if (id_priv_dev->id.device == cm_id->device &&
1646 cma_match_net_dev(&id_priv_dev->id,
1647 net_dev, req))
1648 return id_priv_dev;
1649 }
1650 }
1651 }
1652
1653 return ERR_PTR(-EINVAL);
1654 }
1655
1656 static struct rdma_id_private *
1657 cma_ib_id_from_event(struct ib_cm_id *cm_id,
1658 const struct ib_cm_event *ib_event,
1659 struct cma_req_info *req,
1660 struct net_device **net_dev)
1661 {
1662 struct rdma_bind_list *bind_list;
1663 struct rdma_id_private *id_priv;
1664 int err;
1665
1666 err = cma_save_req_info(ib_event, req);
1667 if (err)
1668 return ERR_PTR(err);
1669
1670 *net_dev = cma_get_net_dev(ib_event, req);
1671 if (IS_ERR(*net_dev)) {
1672 if (PTR_ERR(*net_dev) == -EAFNOSUPPORT) {
1673 /* Assuming the protocol is AF_IB */
1674 *net_dev = NULL;
1675 } else {
1676 return ERR_CAST(*net_dev);
1677 }
1678 }
1679
1680 mutex_lock(&lock);
1681 /*
1682 * Net namespace might be getting deleted while route lookup,
1683 * cm_id lookup is in progress. Therefore, perform netdevice
1684 * validation, cm_id lookup under rcu lock.
1685 * RCU lock along with netdevice state check, synchronizes with
1686 * netdevice migrating to different net namespace and also avoids
1687 * case where net namespace doesn't get deleted while lookup is in
1688 * progress.
1689 * If the device state is not IFF_UP, its properties such as ifindex
1690 * and nd_net cannot be trusted to remain valid without rcu lock.
1691 * net/core/dev.c change_net_namespace() ensures to synchronize with
1692 * ongoing operations on net device after device is closed using
1693 * synchronize_net().
1694 */
1695 rcu_read_lock();
1696 if (*net_dev) {
1697 /*
1698 * If netdevice is down, it is likely that it is administratively
1699 * down or it might be migrating to different namespace.
1700 * In that case avoid further processing, as the net namespace
1701 * or ifindex may change.
1702 */
1703 if (((*net_dev)->flags & IFF_UP) == 0) {
1704 id_priv = ERR_PTR(-EHOSTUNREACH);
1705 goto err;
1706 }
1707
1708 if (!validate_net_dev(*net_dev,
1709 (struct sockaddr *)&req->listen_addr_storage,
1710 (struct sockaddr *)&req->src_addr_storage)) {
1711 id_priv = ERR_PTR(-EHOSTUNREACH);
1712 goto err;
1713 }
1714 }
1715
1716 bind_list = cma_ps_find(*net_dev ? dev_net(*net_dev) : &init_net,
1717 rdma_ps_from_service_id(req->service_id),
1718 cma_port_from_service_id(req->service_id));
1719 id_priv = cma_find_listener(bind_list, cm_id, ib_event, req, *net_dev);
1720 err:
1721 rcu_read_unlock();
1722 mutex_unlock(&lock);
1723 if (IS_ERR(id_priv) && *net_dev) {
1724 dev_put(*net_dev);
1725 *net_dev = NULL;
1726 }
1727 return id_priv;
1728 }
1729
1730 static inline u8 cma_user_data_offset(struct rdma_id_private *id_priv)
1731 {
1732 return cma_family(id_priv) == AF_IB ? 0 : sizeof(struct cma_hdr);
1733 }
1734
1735 static void cma_cancel_route(struct rdma_id_private *id_priv)
1736 {
1737 if (rdma_cap_ib_sa(id_priv->id.device, id_priv->id.port_num)) {
1738 if (id_priv->query)
1739 ib_sa_cancel_query(id_priv->query_id, id_priv->query);
1740 }
1741 }
1742
1743 static void cma_cancel_listens(struct rdma_id_private *id_priv)
1744 {
1745 struct rdma_id_private *dev_id_priv;
1746
1747 /*
1748 * Remove from listen_any_list to prevent added devices from spawning
1749 * additional listen requests.
1750 */
1751 mutex_lock(&lock);
1752 list_del(&id_priv->list);
1753
1754 while (!list_empty(&id_priv->listen_list)) {
1755 dev_id_priv = list_entry(id_priv->listen_list.next,
1756 struct rdma_id_private, listen_list);
1757 /* sync with device removal to avoid duplicate destruction */
1758 list_del_init(&dev_id_priv->list);
1759 list_del(&dev_id_priv->listen_list);
1760 mutex_unlock(&lock);
1761
1762 rdma_destroy_id(&dev_id_priv->id);
1763 mutex_lock(&lock);
1764 }
1765 mutex_unlock(&lock);
1766 }
1767
1768 static void cma_cancel_operation(struct rdma_id_private *id_priv,
1769 enum rdma_cm_state state)
1770 {
1771 switch (state) {
1772 case RDMA_CM_ADDR_QUERY:
1773 rdma_addr_cancel(&id_priv->id.route.addr.dev_addr);
1774 break;
1775 case RDMA_CM_ROUTE_QUERY:
1776 cma_cancel_route(id_priv);
1777 break;
1778 case RDMA_CM_LISTEN:
1779 if (cma_any_addr(cma_src_addr(id_priv)) && !id_priv->cma_dev)
1780 cma_cancel_listens(id_priv);
1781 break;
1782 default:
1783 break;
1784 }
1785 }
1786
1787 static void cma_release_port(struct rdma_id_private *id_priv)
1788 {
1789 struct rdma_bind_list *bind_list = id_priv->bind_list;
1790 struct net *net = id_priv->id.route.addr.dev_addr.net;
1791
1792 if (!bind_list)
1793 return;
1794
1795 mutex_lock(&lock);
1796 hlist_del(&id_priv->node);
1797 if (hlist_empty(&bind_list->owners)) {
1798 cma_ps_remove(net, bind_list->ps, bind_list->port);
1799 kfree(bind_list);
1800 }
1801 mutex_unlock(&lock);
1802 }
1803
1804 static void destroy_mc(struct rdma_id_private *id_priv,
1805 struct cma_multicast *mc)
1806 {
1807 if (rdma_cap_ib_mcast(id_priv->id.device, id_priv->id.port_num))
1808 ib_sa_free_multicast(mc->sa_mc);
1809
1810 if (rdma_protocol_roce(id_priv->id.device, id_priv->id.port_num)) {
1811 struct rdma_dev_addr *dev_addr =
1812 &id_priv->id.route.addr.dev_addr;
1813 struct net_device *ndev = NULL;
1814
1815 if (dev_addr->bound_dev_if)
1816 ndev = dev_get_by_index(dev_addr->net,
1817 dev_addr->bound_dev_if);
1818 if (ndev) {
1819 union ib_gid mgid;
1820
1821 cma_set_mgid(id_priv, (struct sockaddr *)&mc->addr,
1822 &mgid);
1823 cma_igmp_send(ndev, &mgid, false);
1824 dev_put(ndev);
1825 }
1826 }
1827 kfree(mc);
1828 }
1829
1830 static void cma_leave_mc_groups(struct rdma_id_private *id_priv)
1831 {
1832 struct cma_multicast *mc;
1833
1834 while (!list_empty(&id_priv->mc_list)) {
1835 mc = list_first_entry(&id_priv->mc_list, struct cma_multicast,
1836 list);
1837 list_del(&mc->list);
1838 destroy_mc(id_priv, mc);
1839 }
1840 }
1841
1842 static void _destroy_id(struct rdma_id_private *id_priv,
1843 enum rdma_cm_state state)
1844 {
1845 cma_cancel_operation(id_priv, state);
1846
1847 if (id_priv->cma_dev) {
1848 if (rdma_cap_ib_cm(id_priv->id.device, 1)) {
1849 if (id_priv->cm_id.ib)
1850 ib_destroy_cm_id(id_priv->cm_id.ib);
1851 } else if (rdma_cap_iw_cm(id_priv->id.device, 1)) {
1852 if (id_priv->cm_id.iw)
1853 iw_destroy_cm_id(id_priv->cm_id.iw);
1854 }
1855 cma_leave_mc_groups(id_priv);
1856 cma_release_dev(id_priv);
1857 }
1858
1859 cma_release_port(id_priv);
1860 cma_id_put(id_priv);
1861 wait_for_completion(&id_priv->comp);
1862
1863 if (id_priv->internal_id)
1864 cma_id_put(id_priv->id.context);
1865
1866 kfree(id_priv->id.route.path_rec);
1867
1868 put_net(id_priv->id.route.addr.dev_addr.net);
1869 rdma_restrack_del(&id_priv->res);
1870 kfree(id_priv);
1871 }
1872
1873 /*
1874 * destroy an ID from within the handler_mutex. This ensures that no other
1875 * handlers can start running concurrently.
1876 */
1877 static void destroy_id_handler_unlock(struct rdma_id_private *id_priv)
1878 __releases(&idprv->handler_mutex)
1879 {
1880 enum rdma_cm_state state;
1881 unsigned long flags;
1882
1883 trace_cm_id_destroy(id_priv);
1884
1885 /*
1886 * Setting the state to destroyed under the handler mutex provides a
1887 * fence against calling handler callbacks. If this is invoked due to
1888 * the failure of a handler callback then it guarentees that no future
1889 * handlers will be called.
1890 */
1891 lockdep_assert_held(&id_priv->handler_mutex);
1892 spin_lock_irqsave(&id_priv->lock, flags);
1893 state = id_priv->state;
1894 id_priv->state = RDMA_CM_DESTROYING;
1895 spin_unlock_irqrestore(&id_priv->lock, flags);
1896 mutex_unlock(&id_priv->handler_mutex);
1897 _destroy_id(id_priv, state);
1898 }
1899
1900 void rdma_destroy_id(struct rdma_cm_id *id)
1901 {
1902 struct rdma_id_private *id_priv =
1903 container_of(id, struct rdma_id_private, id);
1904
1905 mutex_lock(&id_priv->handler_mutex);
1906 destroy_id_handler_unlock(id_priv);
1907 }
1908 EXPORT_SYMBOL(rdma_destroy_id);
1909
1910 static int cma_rep_recv(struct rdma_id_private *id_priv)
1911 {
1912 int ret;
1913
1914 ret = cma_modify_qp_rtr(id_priv, NULL);
1915 if (ret)
1916 goto reject;
1917
1918 ret = cma_modify_qp_rts(id_priv, NULL);
1919 if (ret)
1920 goto reject;
1921
1922 trace_cm_send_rtu(id_priv);
1923 ret = ib_send_cm_rtu(id_priv->cm_id.ib, NULL, 0);
1924 if (ret)
1925 goto reject;
1926
1927 return 0;
1928 reject:
1929 pr_debug_ratelimited("RDMA CM: CONNECT_ERROR: failed to handle reply. status %d\n", ret);
1930 cma_modify_qp_err(id_priv);
1931 trace_cm_send_rej(id_priv);
1932 ib_send_cm_rej(id_priv->cm_id.ib, IB_CM_REJ_CONSUMER_DEFINED,
1933 NULL, 0, NULL, 0);
1934 return ret;
1935 }
1936
1937 static void cma_set_rep_event_data(struct rdma_cm_event *event,
1938 const struct ib_cm_rep_event_param *rep_data,
1939 void *private_data)
1940 {
1941 event->param.conn.private_data = private_data;
1942 event->param.conn.private_data_len = IB_CM_REP_PRIVATE_DATA_SIZE;
1943 event->param.conn.responder_resources = rep_data->responder_resources;
1944 event->param.conn.initiator_depth = rep_data->initiator_depth;
1945 event->param.conn.flow_control = rep_data->flow_control;
1946 event->param.conn.rnr_retry_count = rep_data->rnr_retry_count;
1947 event->param.conn.srq = rep_data->srq;
1948 event->param.conn.qp_num = rep_data->remote_qpn;
1949
1950 event->ece.vendor_id = rep_data->ece.vendor_id;
1951 event->ece.attr_mod = rep_data->ece.attr_mod;
1952 }
1953
1954 static int cma_cm_event_handler(struct rdma_id_private *id_priv,
1955 struct rdma_cm_event *event)
1956 {
1957 int ret;
1958
1959 lockdep_assert_held(&id_priv->handler_mutex);
1960
1961 trace_cm_event_handler(id_priv, event);
1962 ret = id_priv->id.event_handler(&id_priv->id, event);
1963 trace_cm_event_done(id_priv, event, ret);
1964 return ret;
1965 }
1966
1967 static int cma_ib_handler(struct ib_cm_id *cm_id,
1968 const struct ib_cm_event *ib_event)
1969 {
1970 struct rdma_id_private *id_priv = cm_id->context;
1971 struct rdma_cm_event event = {};
1972 enum rdma_cm_state state;
1973 int ret;
1974
1975 mutex_lock(&id_priv->handler_mutex);
1976 state = READ_ONCE(id_priv->state);
1977 if ((ib_event->event != IB_CM_TIMEWAIT_EXIT &&
1978 state != RDMA_CM_CONNECT) ||
1979 (ib_event->event == IB_CM_TIMEWAIT_EXIT &&
1980 state != RDMA_CM_DISCONNECT))
1981 goto out;
1982
1983 switch (ib_event->event) {
1984 case IB_CM_REQ_ERROR:
1985 case IB_CM_REP_ERROR:
1986 event.event = RDMA_CM_EVENT_UNREACHABLE;
1987 event.status = -ETIMEDOUT;
1988 break;
1989 case IB_CM_REP_RECEIVED:
1990 if (state == RDMA_CM_CONNECT &&
1991 (id_priv->id.qp_type != IB_QPT_UD)) {
1992 trace_cm_send_mra(id_priv);
1993 ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0);
1994 }
1995 if (id_priv->id.qp) {
1996 event.status = cma_rep_recv(id_priv);
1997 event.event = event.status ? RDMA_CM_EVENT_CONNECT_ERROR :
1998 RDMA_CM_EVENT_ESTABLISHED;
1999 } else {
2000 event.event = RDMA_CM_EVENT_CONNECT_RESPONSE;
2001 }
2002 cma_set_rep_event_data(&event, &ib_event->param.rep_rcvd,
2003 ib_event->private_data);
2004 break;
2005 case IB_CM_RTU_RECEIVED:
2006 case IB_CM_USER_ESTABLISHED:
2007 event.event = RDMA_CM_EVENT_ESTABLISHED;
2008 break;
2009 case IB_CM_DREQ_ERROR:
2010 event.status = -ETIMEDOUT;
2011 fallthrough;
2012 case IB_CM_DREQ_RECEIVED:
2013 case IB_CM_DREP_RECEIVED:
2014 if (!cma_comp_exch(id_priv, RDMA_CM_CONNECT,
2015 RDMA_CM_DISCONNECT))
2016 goto out;
2017 event.event = RDMA_CM_EVENT_DISCONNECTED;
2018 break;
2019 case IB_CM_TIMEWAIT_EXIT:
2020 event.event = RDMA_CM_EVENT_TIMEWAIT_EXIT;
2021 break;
2022 case IB_CM_MRA_RECEIVED:
2023 /* ignore event */
2024 goto out;
2025 case IB_CM_REJ_RECEIVED:
2026 pr_debug_ratelimited("RDMA CM: REJECTED: %s\n", rdma_reject_msg(&id_priv->id,
2027 ib_event->param.rej_rcvd.reason));
2028 cma_modify_qp_err(id_priv);
2029 event.status = ib_event->param.rej_rcvd.reason;
2030 event.event = RDMA_CM_EVENT_REJECTED;
2031 event.param.conn.private_data = ib_event->private_data;
2032 event.param.conn.private_data_len = IB_CM_REJ_PRIVATE_DATA_SIZE;
2033 break;
2034 default:
2035 pr_err("RDMA CMA: unexpected IB CM event: %d\n",
2036 ib_event->event);
2037 goto out;
2038 }
2039
2040 ret = cma_cm_event_handler(id_priv, &event);
2041 if (ret) {
2042 /* Destroy the CM ID by returning a non-zero value. */
2043 id_priv->cm_id.ib = NULL;
2044 destroy_id_handler_unlock(id_priv);
2045 return ret;
2046 }
2047 out:
2048 mutex_unlock(&id_priv->handler_mutex);
2049 return 0;
2050 }
2051
2052 static struct rdma_id_private *
2053 cma_ib_new_conn_id(const struct rdma_cm_id *listen_id,
2054 const struct ib_cm_event *ib_event,
2055 struct net_device *net_dev)
2056 {
2057 struct rdma_id_private *listen_id_priv;
2058 struct rdma_id_private *id_priv;
2059 struct rdma_cm_id *id;
2060 struct rdma_route *rt;
2061 const sa_family_t ss_family = listen_id->route.addr.src_addr.ss_family;
2062 struct sa_path_rec *path = ib_event->param.req_rcvd.primary_path;
2063 const __be64 service_id =
2064 ib_event->param.req_rcvd.primary_path->service_id;
2065 int ret;
2066
2067 listen_id_priv = container_of(listen_id, struct rdma_id_private, id);
2068 id_priv = __rdma_create_id(listen_id->route.addr.dev_addr.net,
2069 listen_id->event_handler, listen_id->context,
2070 listen_id->ps,
2071 ib_event->param.req_rcvd.qp_type,
2072 listen_id_priv);
2073 if (IS_ERR(id_priv))
2074 return NULL;
2075
2076 id = &id_priv->id;
2077 if (cma_save_net_info((struct sockaddr *)&id->route.addr.src_addr,
2078 (struct sockaddr *)&id->route.addr.dst_addr,
2079 listen_id, ib_event, ss_family, service_id))
2080 goto err;
2081
2082 rt = &id->route;
2083 rt->num_paths = ib_event->param.req_rcvd.alternate_path ? 2 : 1;
2084 rt->path_rec = kmalloc_array(rt->num_paths, sizeof(*rt->path_rec),
2085 GFP_KERNEL);
2086 if (!rt->path_rec)
2087 goto err;
2088
2089 rt->path_rec[0] = *path;
2090 if (rt->num_paths == 2)
2091 rt->path_rec[1] = *ib_event->param.req_rcvd.alternate_path;
2092
2093 if (net_dev) {
2094 rdma_copy_src_l2_addr(&rt->addr.dev_addr, net_dev);
2095 } else {
2096 if (!cma_protocol_roce(listen_id) &&
2097 cma_any_addr(cma_src_addr(id_priv))) {
2098 rt->addr.dev_addr.dev_type = ARPHRD_INFINIBAND;
2099 rdma_addr_set_sgid(&rt->addr.dev_addr, &rt->path_rec[0].sgid);
2100 ib_addr_set_pkey(&rt->addr.dev_addr, be16_to_cpu(rt->path_rec[0].pkey));
2101 } else if (!cma_any_addr(cma_src_addr(id_priv))) {
2102 ret = cma_translate_addr(cma_src_addr(id_priv), &rt->addr.dev_addr);
2103 if (ret)
2104 goto err;
2105 }
2106 }
2107 rdma_addr_set_dgid(&rt->addr.dev_addr, &rt->path_rec[0].dgid);
2108
2109 id_priv->state = RDMA_CM_CONNECT;
2110 return id_priv;
2111
2112 err:
2113 rdma_destroy_id(id);
2114 return NULL;
2115 }
2116
2117 static struct rdma_id_private *
2118 cma_ib_new_udp_id(const struct rdma_cm_id *listen_id,
2119 const struct ib_cm_event *ib_event,
2120 struct net_device *net_dev)
2121 {
2122 const struct rdma_id_private *listen_id_priv;
2123 struct rdma_id_private *id_priv;
2124 struct rdma_cm_id *id;
2125 const sa_family_t ss_family = listen_id->route.addr.src_addr.ss_family;
2126 struct net *net = listen_id->route.addr.dev_addr.net;
2127 int ret;
2128
2129 listen_id_priv = container_of(listen_id, struct rdma_id_private, id);
2130 id_priv = __rdma_create_id(net, listen_id->event_handler,
2131 listen_id->context, listen_id->ps, IB_QPT_UD,
2132 listen_id_priv);
2133 if (IS_ERR(id_priv))
2134 return NULL;
2135
2136 id = &id_priv->id;
2137 if (cma_save_net_info((struct sockaddr *)&id->route.addr.src_addr,
2138 (struct sockaddr *)&id->route.addr.dst_addr,
2139 listen_id, ib_event, ss_family,
2140 ib_event->param.sidr_req_rcvd.service_id))
2141 goto err;
2142
2143 if (net_dev) {
2144 rdma_copy_src_l2_addr(&id->route.addr.dev_addr, net_dev);
2145 } else {
2146 if (!cma_any_addr(cma_src_addr(id_priv))) {
2147 ret = cma_translate_addr(cma_src_addr(id_priv),
2148 &id->route.addr.dev_addr);
2149 if (ret)
2150 goto err;
2151 }
2152 }
2153
2154 id_priv->state = RDMA_CM_CONNECT;
2155 return id_priv;
2156 err:
2157 rdma_destroy_id(id);
2158 return NULL;
2159 }
2160
2161 static void cma_set_req_event_data(struct rdma_cm_event *event,
2162 const struct ib_cm_req_event_param *req_data,
2163 void *private_data, int offset)
2164 {
2165 event->param.conn.private_data = private_data + offset;
2166 event->param.conn.private_data_len = IB_CM_REQ_PRIVATE_DATA_SIZE - offset;
2167 event->param.conn.responder_resources = req_data->responder_resources;
2168 event->param.conn.initiator_depth = req_data->initiator_depth;
2169 event->param.conn.flow_control = req_data->flow_control;
2170 event->param.conn.retry_count = req_data->retry_count;
2171 event->param.conn.rnr_retry_count = req_data->rnr_retry_count;
2172 event->param.conn.srq = req_data->srq;
2173 event->param.conn.qp_num = req_data->remote_qpn;
2174
2175 event->ece.vendor_id = req_data->ece.vendor_id;
2176 event->ece.attr_mod = req_data->ece.attr_mod;
2177 }
2178
2179 static int cma_ib_check_req_qp_type(const struct rdma_cm_id *id,
2180 const struct ib_cm_event *ib_event)
2181 {
2182 return (((ib_event->event == IB_CM_REQ_RECEIVED) &&
2183 (ib_event->param.req_rcvd.qp_type == id->qp_type)) ||
2184 ((ib_event->event == IB_CM_SIDR_REQ_RECEIVED) &&
2185 (id->qp_type == IB_QPT_UD)) ||
2186 (!id->qp_type));
2187 }
2188
2189 static int cma_ib_req_handler(struct ib_cm_id *cm_id,
2190 const struct ib_cm_event *ib_event)
2191 {
2192 struct rdma_id_private *listen_id, *conn_id = NULL;
2193 struct rdma_cm_event event = {};
2194 struct cma_req_info req = {};
2195 struct net_device *net_dev;
2196 u8 offset;
2197 int ret;
2198
2199 listen_id = cma_ib_id_from_event(cm_id, ib_event, &req, &net_dev);
2200 if (IS_ERR(listen_id))
2201 return PTR_ERR(listen_id);
2202
2203 trace_cm_req_handler(listen_id, ib_event->event);
2204 if (!cma_ib_check_req_qp_type(&listen_id->id, ib_event)) {
2205 ret = -EINVAL;
2206 goto net_dev_put;
2207 }
2208
2209 mutex_lock(&listen_id->handler_mutex);
2210 if (READ_ONCE(listen_id->state) != RDMA_CM_LISTEN) {
2211 ret = -ECONNABORTED;
2212 goto err_unlock;
2213 }
2214
2215 offset = cma_user_data_offset(listen_id);
2216 event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
2217 if (ib_event->event == IB_CM_SIDR_REQ_RECEIVED) {
2218 conn_id = cma_ib_new_udp_id(&listen_id->id, ib_event, net_dev);
2219 event.param.ud.private_data = ib_event->private_data + offset;
2220 event.param.ud.private_data_len =
2221 IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE - offset;
2222 } else {
2223 conn_id = cma_ib_new_conn_id(&listen_id->id, ib_event, net_dev);
2224 cma_set_req_event_data(&event, &ib_event->param.req_rcvd,
2225 ib_event->private_data, offset);
2226 }
2227 if (!conn_id) {
2228 ret = -ENOMEM;
2229 goto err_unlock;
2230 }
2231
2232 mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING);
2233 ret = cma_ib_acquire_dev(conn_id, listen_id, &req);
2234 if (ret) {
2235 destroy_id_handler_unlock(conn_id);
2236 goto err_unlock;
2237 }
2238
2239 conn_id->cm_id.ib = cm_id;
2240 cm_id->context = conn_id;
2241 cm_id->cm_handler = cma_ib_handler;
2242
2243 ret = cma_cm_event_handler(conn_id, &event);
2244 if (ret) {
2245 /* Destroy the CM ID by returning a non-zero value. */
2246 conn_id->cm_id.ib = NULL;
2247 mutex_unlock(&listen_id->handler_mutex);
2248 destroy_id_handler_unlock(conn_id);
2249 goto net_dev_put;
2250 }
2251
2252 if (READ_ONCE(conn_id->state) == RDMA_CM_CONNECT &&
2253 conn_id->id.qp_type != IB_QPT_UD) {
2254 trace_cm_send_mra(cm_id->context);
2255 ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0);
2256 }
2257 mutex_unlock(&conn_id->handler_mutex);
2258
2259 err_unlock:
2260 mutex_unlock(&listen_id->handler_mutex);
2261
2262 net_dev_put:
2263 if (net_dev)
2264 dev_put(net_dev);
2265
2266 return ret;
2267 }
2268
2269 __be64 rdma_get_service_id(struct rdma_cm_id *id, struct sockaddr *addr)
2270 {
2271 if (addr->sa_family == AF_IB)
2272 return ((struct sockaddr_ib *) addr)->sib_sid;
2273
2274 return cpu_to_be64(((u64)id->ps << 16) + be16_to_cpu(cma_port(addr)));
2275 }
2276 EXPORT_SYMBOL(rdma_get_service_id);
2277
2278 void rdma_read_gids(struct rdma_cm_id *cm_id, union ib_gid *sgid,
2279 union ib_gid *dgid)
2280 {
2281 struct rdma_addr *addr = &cm_id->route.addr;
2282
2283 if (!cm_id->device) {
2284 if (sgid)
2285 memset(sgid, 0, sizeof(*sgid));
2286 if (dgid)
2287 memset(dgid, 0, sizeof(*dgid));
2288 return;
2289 }
2290
2291 if (rdma_protocol_roce(cm_id->device, cm_id->port_num)) {
2292 if (sgid)
2293 rdma_ip2gid((struct sockaddr *)&addr->src_addr, sgid);
2294 if (dgid)
2295 rdma_ip2gid((struct sockaddr *)&addr->dst_addr, dgid);
2296 } else {
2297 if (sgid)
2298 rdma_addr_get_sgid(&addr->dev_addr, sgid);
2299 if (dgid)
2300 rdma_addr_get_dgid(&addr->dev_addr, dgid);
2301 }
2302 }
2303 EXPORT_SYMBOL(rdma_read_gids);
2304
2305 static int cma_iw_handler(struct iw_cm_id *iw_id, struct iw_cm_event *iw_event)
2306 {
2307 struct rdma_id_private *id_priv = iw_id->context;
2308 struct rdma_cm_event event = {};
2309 int ret = 0;
2310 struct sockaddr *laddr = (struct sockaddr *)&iw_event->local_addr;
2311 struct sockaddr *raddr = (struct sockaddr *)&iw_event->remote_addr;
2312
2313 mutex_lock(&id_priv->handler_mutex);
2314 if (READ_ONCE(id_priv->state) != RDMA_CM_CONNECT)
2315 goto out;
2316
2317 switch (iw_event->event) {
2318 case IW_CM_EVENT_CLOSE:
2319 event.event = RDMA_CM_EVENT_DISCONNECTED;
2320 break;
2321 case IW_CM_EVENT_CONNECT_REPLY:
2322 memcpy(cma_src_addr(id_priv), laddr,
2323 rdma_addr_size(laddr));
2324 memcpy(cma_dst_addr(id_priv), raddr,
2325 rdma_addr_size(raddr));
2326 switch (iw_event->status) {
2327 case 0:
2328 event.event = RDMA_CM_EVENT_ESTABLISHED;
2329 event.param.conn.initiator_depth = iw_event->ird;
2330 event.param.conn.responder_resources = iw_event->ord;
2331 break;
2332 case -ECONNRESET:
2333 case -ECONNREFUSED:
2334 event.event = RDMA_CM_EVENT_REJECTED;
2335 break;
2336 case -ETIMEDOUT:
2337 event.event = RDMA_CM_EVENT_UNREACHABLE;
2338 break;
2339 default:
2340 event.event = RDMA_CM_EVENT_CONNECT_ERROR;
2341 break;
2342 }
2343 break;
2344 case IW_CM_EVENT_ESTABLISHED:
2345 event.event = RDMA_CM_EVENT_ESTABLISHED;
2346 event.param.conn.initiator_depth = iw_event->ird;
2347 event.param.conn.responder_resources = iw_event->ord;
2348 break;
2349 default:
2350 goto out;
2351 }
2352
2353 event.status = iw_event->status;
2354 event.param.conn.private_data = iw_event->private_data;
2355 event.param.conn.private_data_len = iw_event->private_data_len;
2356 ret = cma_cm_event_handler(id_priv, &event);
2357 if (ret) {
2358 /* Destroy the CM ID by returning a non-zero value. */
2359 id_priv->cm_id.iw = NULL;
2360 destroy_id_handler_unlock(id_priv);
2361 return ret;
2362 }
2363
2364 out:
2365 mutex_unlock(&id_priv->handler_mutex);
2366 return ret;
2367 }
2368
2369 static int iw_conn_req_handler(struct iw_cm_id *cm_id,
2370 struct iw_cm_event *iw_event)
2371 {
2372 struct rdma_id_private *listen_id, *conn_id;
2373 struct rdma_cm_event event = {};
2374 int ret = -ECONNABORTED;
2375 struct sockaddr *laddr = (struct sockaddr *)&iw_event->local_addr;
2376 struct sockaddr *raddr = (struct sockaddr *)&iw_event->remote_addr;
2377
2378 event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
2379 event.param.conn.private_data = iw_event->private_data;
2380 event.param.conn.private_data_len = iw_event->private_data_len;
2381 event.param.conn.initiator_depth = iw_event->ird;
2382 event.param.conn.responder_resources = iw_event->ord;
2383
2384 listen_id = cm_id->context;
2385
2386 mutex_lock(&listen_id->handler_mutex);
2387 if (READ_ONCE(listen_id->state) != RDMA_CM_LISTEN)
2388 goto out;
2389
2390 /* Create a new RDMA id for the new IW CM ID */
2391 conn_id = __rdma_create_id(listen_id->id.route.addr.dev_addr.net,
2392 listen_id->id.event_handler,
2393 listen_id->id.context, RDMA_PS_TCP,
2394 IB_QPT_RC, listen_id);
2395 if (IS_ERR(conn_id)) {
2396 ret = -ENOMEM;
2397 goto out;
2398 }
2399 mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING);
2400 conn_id->state = RDMA_CM_CONNECT;
2401
2402 ret = rdma_translate_ip(laddr, &conn_id->id.route.addr.dev_addr);
2403 if (ret) {
2404 mutex_unlock(&listen_id->handler_mutex);
2405 destroy_id_handler_unlock(conn_id);
2406 return ret;
2407 }
2408
2409 ret = cma_iw_acquire_dev(conn_id, listen_id);
2410 if (ret) {
2411 mutex_unlock(&listen_id->handler_mutex);
2412 destroy_id_handler_unlock(conn_id);
2413 return ret;
2414 }
2415
2416 conn_id->cm_id.iw = cm_id;
2417 cm_id->context = conn_id;
2418 cm_id->cm_handler = cma_iw_handler;
2419
2420 memcpy(cma_src_addr(conn_id), laddr, rdma_addr_size(laddr));
2421 memcpy(cma_dst_addr(conn_id), raddr, rdma_addr_size(raddr));
2422
2423 ret = cma_cm_event_handler(conn_id, &event);
2424 if (ret) {
2425 /* User wants to destroy the CM ID */
2426 conn_id->cm_id.iw = NULL;
2427 mutex_unlock(&listen_id->handler_mutex);
2428 destroy_id_handler_unlock(conn_id);
2429 return ret;
2430 }
2431
2432 mutex_unlock(&conn_id->handler_mutex);
2433
2434 out:
2435 mutex_unlock(&listen_id->handler_mutex);
2436 return ret;
2437 }
2438
2439 static int cma_ib_listen(struct rdma_id_private *id_priv)
2440 {
2441 struct sockaddr *addr;
2442 struct ib_cm_id *id;
2443 __be64 svc_id;
2444
2445 addr = cma_src_addr(id_priv);
2446 svc_id = rdma_get_service_id(&id_priv->id, addr);
2447 id = ib_cm_insert_listen(id_priv->id.device,
2448 cma_ib_req_handler, svc_id);
2449 if (IS_ERR(id))
2450 return PTR_ERR(id);
2451 id_priv->cm_id.ib = id;
2452
2453 return 0;
2454 }
2455
2456 static int cma_iw_listen(struct rdma_id_private *id_priv, int backlog)
2457 {
2458 int ret;
2459 struct iw_cm_id *id;
2460
2461 id = iw_create_cm_id(id_priv->id.device,
2462 iw_conn_req_handler,
2463 id_priv);
2464 if (IS_ERR(id))
2465 return PTR_ERR(id);
2466
2467 id->tos = id_priv->tos;
2468 id->tos_set = id_priv->tos_set;
2469 id_priv->cm_id.iw = id;
2470
2471 memcpy(&id_priv->cm_id.iw->local_addr, cma_src_addr(id_priv),
2472 rdma_addr_size(cma_src_addr(id_priv)));
2473
2474 ret = iw_cm_listen(id_priv->cm_id.iw, backlog);
2475
2476 if (ret) {
2477 iw_destroy_cm_id(id_priv->cm_id.iw);
2478 id_priv->cm_id.iw = NULL;
2479 }
2480
2481 return ret;
2482 }
2483
2484 static int cma_listen_handler(struct rdma_cm_id *id,
2485 struct rdma_cm_event *event)
2486 {
2487 struct rdma_id_private *id_priv = id->context;
2488
2489 /* Listening IDs are always destroyed on removal */
2490 if (event->event == RDMA_CM_EVENT_DEVICE_REMOVAL)
2491 return -1;
2492
2493 id->context = id_priv->id.context;
2494 id->event_handler = id_priv->id.event_handler;
2495 trace_cm_event_handler(id_priv, event);
2496 return id_priv->id.event_handler(id, event);
2497 }
2498
2499 static int cma_listen_on_dev(struct rdma_id_private *id_priv,
2500 struct cma_device *cma_dev,
2501 struct rdma_id_private **to_destroy)
2502 {
2503 struct rdma_id_private *dev_id_priv;
2504 struct net *net = id_priv->id.route.addr.dev_addr.net;
2505 int ret;
2506
2507 lockdep_assert_held(&lock);
2508
2509 *to_destroy = NULL;
2510 if (cma_family(id_priv) == AF_IB && !rdma_cap_ib_cm(cma_dev->device, 1))
2511 return 0;
2512
2513 dev_id_priv =
2514 __rdma_create_id(net, cma_listen_handler, id_priv,
2515 id_priv->id.ps, id_priv->id.qp_type, id_priv);
2516 if (IS_ERR(dev_id_priv))
2517 return PTR_ERR(dev_id_priv);
2518
2519 dev_id_priv->state = RDMA_CM_ADDR_BOUND;
2520 memcpy(cma_src_addr(dev_id_priv), cma_src_addr(id_priv),
2521 rdma_addr_size(cma_src_addr(id_priv)));
2522
2523 _cma_attach_to_dev(dev_id_priv, cma_dev);
2524 cma_id_get(id_priv);
2525 dev_id_priv->internal_id = 1;
2526 dev_id_priv->afonly = id_priv->afonly;
2527 dev_id_priv->tos_set = id_priv->tos_set;
2528 dev_id_priv->tos = id_priv->tos;
2529
2530 ret = rdma_listen(&dev_id_priv->id, id_priv->backlog);
2531 if (ret)
2532 goto err_listen;
2533 list_add_tail(&dev_id_priv->listen_list, &id_priv->listen_list);
2534 return 0;
2535 err_listen:
2536 /* Caller must destroy this after releasing lock */
2537 *to_destroy = dev_id_priv;
2538 dev_warn(&cma_dev->device->dev, "RDMA CMA: %s, error %d\n", __func__, ret);
2539 return ret;
2540 }
2541
2542 static int cma_listen_on_all(struct rdma_id_private *id_priv)
2543 {
2544 struct rdma_id_private *to_destroy;
2545 struct cma_device *cma_dev;
2546 int ret;
2547
2548 mutex_lock(&lock);
2549 list_add_tail(&id_priv->list, &listen_any_list);
2550 list_for_each_entry(cma_dev, &dev_list, list) {
2551 ret = cma_listen_on_dev(id_priv, cma_dev, &to_destroy);
2552 if (ret) {
2553 /* Prevent racing with cma_process_remove() */
2554 if (to_destroy)
2555 list_del_init(&to_destroy->list);
2556 goto err_listen;
2557 }
2558 }
2559 mutex_unlock(&lock);
2560 return 0;
2561
2562 err_listen:
2563 list_del(&id_priv->list);
2564 mutex_unlock(&lock);
2565 if (to_destroy)
2566 rdma_destroy_id(&to_destroy->id);
2567 return ret;
2568 }
2569
2570 void rdma_set_service_type(struct rdma_cm_id *id, int tos)
2571 {
2572 struct rdma_id_private *id_priv;
2573
2574 id_priv = container_of(id, struct rdma_id_private, id);
2575 id_priv->tos = (u8) tos;
2576 id_priv->tos_set = true;
2577 }
2578 EXPORT_SYMBOL(rdma_set_service_type);
2579
2580 /**
2581 * rdma_set_ack_timeout() - Set the ack timeout of QP associated
2582 * with a connection identifier.
2583 * @id: Communication identifier to associated with service type.
2584 * @timeout: Ack timeout to set a QP, expressed as 4.096 * 2^(timeout) usec.
2585 *
2586 * This function should be called before rdma_connect() on active side,
2587 * and on passive side before rdma_accept(). It is applicable to primary
2588 * path only. The timeout will affect the local side of the QP, it is not
2589 * negotiated with remote side and zero disables the timer. In case it is
2590 * set before rdma_resolve_route, the value will also be used to determine
2591 * PacketLifeTime for RoCE.
2592 *
2593 * Return: 0 for success
2594 */
2595 int rdma_set_ack_timeout(struct rdma_cm_id *id, u8 timeout)
2596 {
2597 struct rdma_id_private *id_priv;
2598
2599 if (id->qp_type != IB_QPT_RC)
2600 return -EINVAL;
2601
2602 id_priv = container_of(id, struct rdma_id_private, id);
2603 id_priv->timeout = timeout;
2604 id_priv->timeout_set = true;
2605
2606 return 0;
2607 }
2608 EXPORT_SYMBOL(rdma_set_ack_timeout);
2609
2610 static void cma_query_handler(int status, struct sa_path_rec *path_rec,
2611 void *context)
2612 {
2613 struct cma_work *work = context;
2614 struct rdma_route *route;
2615
2616 route = &work->id->id.route;
2617
2618 if (!status) {
2619 route->num_paths = 1;
2620 *route->path_rec = *path_rec;
2621 } else {
2622 work->old_state = RDMA_CM_ROUTE_QUERY;
2623 work->new_state = RDMA_CM_ADDR_RESOLVED;
2624 work->event.event = RDMA_CM_EVENT_ROUTE_ERROR;
2625 work->event.status = status;
2626 pr_debug_ratelimited("RDMA CM: ROUTE_ERROR: failed to query path. status %d\n",
2627 status);
2628 }
2629
2630 queue_work(cma_wq, &work->work);
2631 }
2632
2633 static int cma_query_ib_route(struct rdma_id_private *id_priv,
2634 unsigned long timeout_ms, struct cma_work *work)
2635 {
2636 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
2637 struct sa_path_rec path_rec;
2638 ib_sa_comp_mask comp_mask;
2639 struct sockaddr_in6 *sin6;
2640 struct sockaddr_ib *sib;
2641
2642 memset(&path_rec, 0, sizeof path_rec);
2643
2644 if (rdma_cap_opa_ah(id_priv->id.device, id_priv->id.port_num))
2645 path_rec.rec_type = SA_PATH_REC_TYPE_OPA;
2646 else
2647 path_rec.rec_type = SA_PATH_REC_TYPE_IB;
2648 rdma_addr_get_sgid(dev_addr, &path_rec.sgid);
2649 rdma_addr_get_dgid(dev_addr, &path_rec.dgid);
2650 path_rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr));
2651 path_rec.numb_path = 1;
2652 path_rec.reversible = 1;
2653 path_rec.service_id = rdma_get_service_id(&id_priv->id,
2654 cma_dst_addr(id_priv));
2655
2656 comp_mask = IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID |
2657 IB_SA_PATH_REC_PKEY | IB_SA_PATH_REC_NUMB_PATH |
2658 IB_SA_PATH_REC_REVERSIBLE | IB_SA_PATH_REC_SERVICE_ID;
2659
2660 switch (cma_family(id_priv)) {
2661 case AF_INET:
2662 path_rec.qos_class = cpu_to_be16((u16) id_priv->tos);
2663 comp_mask |= IB_SA_PATH_REC_QOS_CLASS;
2664 break;
2665 case AF_INET6:
2666 sin6 = (struct sockaddr_in6 *) cma_src_addr(id_priv);
2667 path_rec.traffic_class = (u8) (be32_to_cpu(sin6->sin6_flowinfo) >> 20);
2668 comp_mask |= IB_SA_PATH_REC_TRAFFIC_CLASS;
2669 break;
2670 case AF_IB:
2671 sib = (struct sockaddr_ib *) cma_src_addr(id_priv);
2672 path_rec.traffic_class = (u8) (be32_to_cpu(sib->sib_flowinfo) >> 20);
2673 comp_mask |= IB_SA_PATH_REC_TRAFFIC_CLASS;
2674 break;
2675 }
2676
2677 id_priv->query_id = ib_sa_path_rec_get(&sa_client, id_priv->id.device,
2678 id_priv->id.port_num, &path_rec,
2679 comp_mask, timeout_ms,
2680 GFP_KERNEL, cma_query_handler,
2681 work, &id_priv->query);
2682
2683 return (id_priv->query_id < 0) ? id_priv->query_id : 0;
2684 }
2685
2686 static void cma_work_handler(struct work_struct *_work)
2687 {
2688 struct cma_work *work = container_of(_work, struct cma_work, work);
2689 struct rdma_id_private *id_priv = work->id;
2690
2691 mutex_lock(&id_priv->handler_mutex);
2692 if (READ_ONCE(id_priv->state) == RDMA_CM_DESTROYING ||
2693 READ_ONCE(id_priv->state) == RDMA_CM_DEVICE_REMOVAL)
2694 goto out_unlock;
2695 if (work->old_state != 0 || work->new_state != 0) {
2696 if (!cma_comp_exch(id_priv, work->old_state, work->new_state))
2697 goto out_unlock;
2698 }
2699
2700 if (cma_cm_event_handler(id_priv, &work->event)) {
2701 cma_id_put(id_priv);
2702 destroy_id_handler_unlock(id_priv);
2703 goto out_free;
2704 }
2705
2706 out_unlock:
2707 mutex_unlock(&id_priv->handler_mutex);
2708 cma_id_put(id_priv);
2709 out_free:
2710 if (work->event.event == RDMA_CM_EVENT_MULTICAST_JOIN)
2711 rdma_destroy_ah_attr(&work->event.param.ud.ah_attr);
2712 kfree(work);
2713 }
2714
2715 static void cma_init_resolve_route_work(struct cma_work *work,
2716 struct rdma_id_private *id_priv)
2717 {
2718 work->id = id_priv;
2719 INIT_WORK(&work->work, cma_work_handler);
2720 work->old_state = RDMA_CM_ROUTE_QUERY;
2721 work->new_state = RDMA_CM_ROUTE_RESOLVED;
2722 work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
2723 }
2724
2725 static void enqueue_resolve_addr_work(struct cma_work *work,
2726 struct rdma_id_private *id_priv)
2727 {
2728 /* Balances with cma_id_put() in cma_work_handler */
2729 cma_id_get(id_priv);
2730
2731 work->id = id_priv;
2732 INIT_WORK(&work->work, cma_work_handler);
2733 work->old_state = RDMA_CM_ADDR_QUERY;
2734 work->new_state = RDMA_CM_ADDR_RESOLVED;
2735 work->event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
2736
2737 queue_work(cma_wq, &work->work);
2738 }
2739
2740 static int cma_resolve_ib_route(struct rdma_id_private *id_priv,
2741 unsigned long timeout_ms)
2742 {
2743 struct rdma_route *route = &id_priv->id.route;
2744 struct cma_work *work;
2745 int ret;
2746
2747 work = kzalloc(sizeof *work, GFP_KERNEL);
2748 if (!work)
2749 return -ENOMEM;
2750
2751 cma_init_resolve_route_work(work, id_priv);
2752
2753 route->path_rec = kmalloc(sizeof *route->path_rec, GFP_KERNEL);
2754 if (!route->path_rec) {
2755 ret = -ENOMEM;
2756 goto err1;
2757 }
2758
2759 ret = cma_query_ib_route(id_priv, timeout_ms, work);
2760 if (ret)
2761 goto err2;
2762
2763 return 0;
2764 err2:
2765 kfree(route->path_rec);
2766 route->path_rec = NULL;
2767 err1:
2768 kfree(work);
2769 return ret;
2770 }
2771
2772 static enum ib_gid_type cma_route_gid_type(enum rdma_network_type network_type,
2773 unsigned long supported_gids,
2774 enum ib_gid_type default_gid)
2775 {
2776 if ((network_type == RDMA_NETWORK_IPV4 ||
2777 network_type == RDMA_NETWORK_IPV6) &&
2778 test_bit(IB_GID_TYPE_ROCE_UDP_ENCAP, &supported_gids))
2779 return IB_GID_TYPE_ROCE_UDP_ENCAP;
2780
2781 return default_gid;
2782 }
2783
2784 /*
2785 * cma_iboe_set_path_rec_l2_fields() is helper function which sets
2786 * path record type based on GID type.
2787 * It also sets up other L2 fields which includes destination mac address
2788 * netdev ifindex, of the path record.
2789 * It returns the netdev of the bound interface for this path record entry.
2790 */
2791 static struct net_device *
2792 cma_iboe_set_path_rec_l2_fields(struct rdma_id_private *id_priv)
2793 {
2794 struct rdma_route *route = &id_priv->id.route;
2795 enum ib_gid_type gid_type = IB_GID_TYPE_ROCE;
2796 struct rdma_addr *addr = &route->addr;
2797 unsigned long supported_gids;
2798 struct net_device *ndev;
2799
2800 if (!addr->dev_addr.bound_dev_if)
2801 return NULL;
2802
2803 ndev = dev_get_by_index(addr->dev_addr.net,
2804 addr->dev_addr.bound_dev_if);
2805 if (!ndev)
2806 return NULL;
2807
2808 supported_gids = roce_gid_type_mask_support(id_priv->id.device,
2809 id_priv->id.port_num);
2810 gid_type = cma_route_gid_type(addr->dev_addr.network,
2811 supported_gids,
2812 id_priv->gid_type);
2813 /* Use the hint from IP Stack to select GID Type */
2814 if (gid_type < ib_network_to_gid_type(addr->dev_addr.network))
2815 gid_type = ib_network_to_gid_type(addr->dev_addr.network);
2816 route->path_rec->rec_type = sa_conv_gid_to_pathrec_type(gid_type);
2817
2818 route->path_rec->roce.route_resolved = true;
2819 sa_path_set_dmac(route->path_rec, addr->dev_addr.dst_dev_addr);
2820 return ndev;
2821 }
2822
2823 int rdma_set_ib_path(struct rdma_cm_id *id,
2824 struct sa_path_rec *path_rec)
2825 {
2826 struct rdma_id_private *id_priv;
2827 struct net_device *ndev;
2828 int ret;
2829
2830 id_priv = container_of(id, struct rdma_id_private, id);
2831 if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED,
2832 RDMA_CM_ROUTE_RESOLVED))
2833 return -EINVAL;
2834
2835 id->route.path_rec = kmemdup(path_rec, sizeof(*path_rec),
2836 GFP_KERNEL);
2837 if (!id->route.path_rec) {
2838 ret = -ENOMEM;
2839 goto err;
2840 }
2841
2842 if (rdma_protocol_roce(id->device, id->port_num)) {
2843 ndev = cma_iboe_set_path_rec_l2_fields(id_priv);
2844 if (!ndev) {
2845 ret = -ENODEV;
2846 goto err_free;
2847 }
2848 dev_put(ndev);
2849 }
2850
2851 id->route.num_paths = 1;
2852 return 0;
2853
2854 err_free:
2855 kfree(id->route.path_rec);
2856 id->route.path_rec = NULL;
2857 err:
2858 cma_comp_exch(id_priv, RDMA_CM_ROUTE_RESOLVED, RDMA_CM_ADDR_RESOLVED);
2859 return ret;
2860 }
2861 EXPORT_SYMBOL(rdma_set_ib_path);
2862
2863 static int cma_resolve_iw_route(struct rdma_id_private *id_priv)
2864 {
2865 struct cma_work *work;
2866
2867 work = kzalloc(sizeof *work, GFP_KERNEL);
2868 if (!work)
2869 return -ENOMEM;
2870
2871 cma_init_resolve_route_work(work, id_priv);
2872 queue_work(cma_wq, &work->work);
2873 return 0;
2874 }
2875
2876 static int get_vlan_ndev_tc(struct net_device *vlan_ndev, int prio)
2877 {
2878 struct net_device *dev;
2879
2880 dev = vlan_dev_real_dev(vlan_ndev);
2881 if (dev->num_tc)
2882 return netdev_get_prio_tc_map(dev, prio);
2883
2884 return (vlan_dev_get_egress_qos_mask(vlan_ndev, prio) &
2885 VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
2886 }
2887
2888 struct iboe_prio_tc_map {
2889 int input_prio;
2890 int output_tc;
2891 bool found;
2892 };
2893
2894 static int get_lower_vlan_dev_tc(struct net_device *dev,
2895 struct netdev_nested_priv *priv)
2896 {
2897 struct iboe_prio_tc_map *map = (struct iboe_prio_tc_map *)priv->data;
2898
2899 if (is_vlan_dev(dev))
2900 map->output_tc = get_vlan_ndev_tc(dev, map->input_prio);
2901 else if (dev->num_tc)
2902 map->output_tc = netdev_get_prio_tc_map(dev, map->input_prio);
2903 else
2904 map->output_tc = 0;
2905 /* We are interested only in first level VLAN device, so always
2906 * return 1 to stop iterating over next level devices.
2907 */
2908 map->found = true;
2909 return 1;
2910 }
2911
2912 static int iboe_tos_to_sl(struct net_device *ndev, int tos)
2913 {
2914 struct iboe_prio_tc_map prio_tc_map = {};
2915 int prio = rt_tos2priority(tos);
2916 struct netdev_nested_priv priv;
2917
2918 /* If VLAN device, get it directly from the VLAN netdev */
2919 if (is_vlan_dev(ndev))
2920 return get_vlan_ndev_tc(ndev, prio);
2921
2922 prio_tc_map.input_prio = prio;
2923 priv.data = (void *)&prio_tc_map;
2924 rcu_read_lock();
2925 netdev_walk_all_lower_dev_rcu(ndev,
2926 get_lower_vlan_dev_tc,
2927 &priv);
2928 rcu_read_unlock();
2929 /* If map is found from lower device, use it; Otherwise
2930 * continue with the current netdevice to get priority to tc map.
2931 */
2932 if (prio_tc_map.found)
2933 return prio_tc_map.output_tc;
2934 else if (ndev->num_tc)
2935 return netdev_get_prio_tc_map(ndev, prio);
2936 else
2937 return 0;
2938 }
2939
2940 static __be32 cma_get_roce_udp_flow_label(struct rdma_id_private *id_priv)
2941 {
2942 struct sockaddr_in6 *addr6;
2943 u16 dport, sport;
2944 u32 hash, fl;
2945
2946 addr6 = (struct sockaddr_in6 *)cma_src_addr(id_priv);
2947 fl = be32_to_cpu(addr6->sin6_flowinfo) & IB_GRH_FLOWLABEL_MASK;
2948 if ((cma_family(id_priv) != AF_INET6) || !fl) {
2949 dport = be16_to_cpu(cma_port(cma_dst_addr(id_priv)));
2950 sport = be16_to_cpu(cma_port(cma_src_addr(id_priv)));
2951 hash = (u32)sport * 31 + dport;
2952 fl = hash & IB_GRH_FLOWLABEL_MASK;
2953 }
2954
2955 return cpu_to_be32(fl);
2956 }
2957
2958 static int cma_resolve_iboe_route(struct rdma_id_private *id_priv)
2959 {
2960 struct rdma_route *route = &id_priv->id.route;
2961 struct rdma_addr *addr = &route->addr;
2962 struct cma_work *work;
2963 int ret;
2964 struct net_device *ndev;
2965
2966 u8 default_roce_tos = id_priv->cma_dev->default_roce_tos[id_priv->id.port_num -
2967 rdma_start_port(id_priv->cma_dev->device)];
2968 u8 tos = id_priv->tos_set ? id_priv->tos : default_roce_tos;
2969
2970
2971 work = kzalloc(sizeof *work, GFP_KERNEL);
2972 if (!work)
2973 return -ENOMEM;
2974
2975 route->path_rec = kzalloc(sizeof *route->path_rec, GFP_KERNEL);
2976 if (!route->path_rec) {
2977 ret = -ENOMEM;
2978 goto err1;
2979 }
2980
2981 route->num_paths = 1;
2982
2983 ndev = cma_iboe_set_path_rec_l2_fields(id_priv);
2984 if (!ndev) {
2985 ret = -ENODEV;
2986 goto err2;
2987 }
2988
2989 rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr,
2990 &route->path_rec->sgid);
2991 rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.dst_addr,
2992 &route->path_rec->dgid);
2993
2994 if (((struct sockaddr *)&id_priv->id.route.addr.dst_addr)->sa_family != AF_IB)
2995 /* TODO: get the hoplimit from the inet/inet6 device */
2996 route->path_rec->hop_limit = addr->dev_addr.hoplimit;
2997 else
2998 route->path_rec->hop_limit = 1;
2999 route->path_rec->reversible = 1;
3000 route->path_rec->pkey = cpu_to_be16(0xffff);
3001 route->path_rec->mtu_selector = IB_SA_EQ;
3002 route->path_rec->sl = iboe_tos_to_sl(ndev, tos);
3003 route->path_rec->traffic_class = tos;
3004 route->path_rec->mtu = iboe_get_mtu(ndev->mtu);
3005 route->path_rec->rate_selector = IB_SA_EQ;
3006 route->path_rec->rate = iboe_get_rate(ndev);
3007 dev_put(ndev);
3008 route->path_rec->packet_life_time_selector = IB_SA_EQ;
3009 /* In case ACK timeout is set, use this value to calculate
3010 * PacketLifeTime. As per IBTA 12.7.34,
3011 * local ACK timeout = (2 * PacketLifeTime + Local CA’s ACK delay).
3012 * Assuming a negligible local ACK delay, we can use
3013 * PacketLifeTime = local ACK timeout/2
3014 * as a reasonable approximation for RoCE networks.
3015 */
3016 route->path_rec->packet_life_time = id_priv->timeout_set ?
3017 id_priv->timeout - 1 : CMA_IBOE_PACKET_LIFETIME;
3018
3019 if (!route->path_rec->mtu) {
3020 ret = -EINVAL;
3021 goto err2;
3022 }
3023
3024 if (rdma_protocol_roce_udp_encap(id_priv->id.device,
3025 id_priv->id.port_num))
3026 route->path_rec->flow_label =
3027 cma_get_roce_udp_flow_label(id_priv);
3028
3029 cma_init_resolve_route_work(work, id_priv);
3030 queue_work(cma_wq, &work->work);
3031
3032 return 0;
3033
3034 err2:
3035 kfree(route->path_rec);
3036 route->path_rec = NULL;
3037 route->num_paths = 0;
3038 err1:
3039 kfree(work);
3040 return ret;
3041 }
3042
3043 int rdma_resolve_route(struct rdma_cm_id *id, unsigned long timeout_ms)
3044 {
3045 struct rdma_id_private *id_priv;
3046 int ret;
3047
3048 id_priv = container_of(id, struct rdma_id_private, id);
3049 if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED, RDMA_CM_ROUTE_QUERY))
3050 return -EINVAL;
3051
3052 cma_id_get(id_priv);
3053 if (rdma_cap_ib_sa(id->device, id->port_num))
3054 ret = cma_resolve_ib_route(id_priv, timeout_ms);
3055 else if (rdma_protocol_roce(id->device, id->port_num))
3056 ret = cma_resolve_iboe_route(id_priv);
3057 else if (rdma_protocol_iwarp(id->device, id->port_num))
3058 ret = cma_resolve_iw_route(id_priv);
3059 else
3060 ret = -ENOSYS;
3061
3062 if (ret)
3063 goto err;
3064
3065 return 0;
3066 err:
3067 cma_comp_exch(id_priv, RDMA_CM_ROUTE_QUERY, RDMA_CM_ADDR_RESOLVED);
3068 cma_id_put(id_priv);
3069 return ret;
3070 }
3071 EXPORT_SYMBOL(rdma_resolve_route);
3072
3073 static void cma_set_loopback(struct sockaddr *addr)
3074 {
3075 switch (addr->sa_family) {
3076 case AF_INET:
3077 ((struct sockaddr_in *) addr)->sin_addr.s_addr = htonl(INADDR_LOOPBACK);
3078 break;
3079 case AF_INET6:
3080 ipv6_addr_set(&((struct sockaddr_in6 *) addr)->sin6_addr,
3081 0, 0, 0, htonl(1));
3082 break;
3083 default:
3084 ib_addr_set(&((struct sockaddr_ib *) addr)->sib_addr,
3085 0, 0, 0, htonl(1));
3086 break;
3087 }
3088 }
3089
3090 static int cma_bind_loopback(struct rdma_id_private *id_priv)
3091 {
3092 struct cma_device *cma_dev, *cur_dev;
3093 union ib_gid gid;
3094 enum ib_port_state port_state;
3095 unsigned int p;
3096 u16 pkey;
3097 int ret;
3098
3099 cma_dev = NULL;
3100 mutex_lock(&lock);
3101 list_for_each_entry(cur_dev, &dev_list, list) {
3102 if (cma_family(id_priv) == AF_IB &&
3103 !rdma_cap_ib_cm(cur_dev->device, 1))
3104 continue;
3105
3106 if (!cma_dev)
3107 cma_dev = cur_dev;
3108
3109 rdma_for_each_port (cur_dev->device, p) {
3110 if (!ib_get_cached_port_state(cur_dev->device, p, &port_state) &&
3111 port_state == IB_PORT_ACTIVE) {
3112 cma_dev = cur_dev;
3113 goto port_found;
3114 }
3115 }
3116 }
3117
3118 if (!cma_dev) {
3119 ret = -ENODEV;
3120 goto out;
3121 }
3122
3123 p = 1;
3124
3125 port_found:
3126 ret = rdma_query_gid(cma_dev->device, p, 0, &gid);
3127 if (ret)
3128 goto out;
3129
3130 ret = ib_get_cached_pkey(cma_dev->device, p, 0, &pkey);
3131 if (ret)
3132 goto out;
3133
3134 id_priv->id.route.addr.dev_addr.dev_type =
3135 (rdma_protocol_ib(cma_dev->device, p)) ?
3136 ARPHRD_INFINIBAND : ARPHRD_ETHER;
3137
3138 rdma_addr_set_sgid(&id_priv->id.route.addr.dev_addr, &gid);
3139 ib_addr_set_pkey(&id_priv->id.route.addr.dev_addr, pkey);
3140 id_priv->id.port_num = p;
3141 cma_attach_to_dev(id_priv, cma_dev);
3142 cma_set_loopback(cma_src_addr(id_priv));
3143 out:
3144 mutex_unlock(&lock);
3145 return ret;
3146 }
3147
3148 static void addr_handler(int status, struct sockaddr *src_addr,
3149 struct rdma_dev_addr *dev_addr, void *context)
3150 {
3151 struct rdma_id_private *id_priv = context;
3152 struct rdma_cm_event event = {};
3153 struct sockaddr *addr;
3154 struct sockaddr_storage old_addr;
3155
3156 mutex_lock(&id_priv->handler_mutex);
3157 if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY,
3158 RDMA_CM_ADDR_RESOLVED))
3159 goto out;
3160
3161 /*
3162 * Store the previous src address, so that if we fail to acquire
3163 * matching rdma device, old address can be restored back, which helps
3164 * to cancel the cma listen operation correctly.
3165 */
3166 addr = cma_src_addr(id_priv);
3167 memcpy(&old_addr, addr, rdma_addr_size(addr));
3168 memcpy(addr, src_addr, rdma_addr_size(src_addr));
3169 if (!status && !id_priv->cma_dev) {
3170 status = cma_acquire_dev_by_src_ip(id_priv);
3171 if (status)
3172 pr_debug_ratelimited("RDMA CM: ADDR_ERROR: failed to acquire device. status %d\n",
3173 status);
3174 } else if (status) {
3175 pr_debug_ratelimited("RDMA CM: ADDR_ERROR: failed to resolve IP. status %d\n", status);
3176 }
3177
3178 if (status) {
3179 memcpy(addr, &old_addr,
3180 rdma_addr_size((struct sockaddr *)&old_addr));
3181 if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED,
3182 RDMA_CM_ADDR_BOUND))
3183 goto out;
3184 event.event = RDMA_CM_EVENT_ADDR_ERROR;
3185 event.status = status;
3186 } else
3187 event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
3188
3189 if (cma_cm_event_handler(id_priv, &event)) {
3190 destroy_id_handler_unlock(id_priv);
3191 return;
3192 }
3193 out:
3194 mutex_unlock(&id_priv->handler_mutex);
3195 }
3196
3197 static int cma_resolve_loopback(struct rdma_id_private *id_priv)
3198 {
3199 struct cma_work *work;
3200 union ib_gid gid;
3201 int ret;
3202
3203 work = kzalloc(sizeof *work, GFP_KERNEL);
3204 if (!work)
3205 return -ENOMEM;
3206
3207 if (!id_priv->cma_dev) {
3208 ret = cma_bind_loopback(id_priv);
3209 if (ret)
3210 goto err;
3211 }
3212
3213 rdma_addr_get_sgid(&id_priv->id.route.addr.dev_addr, &gid);
3214 rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, &gid);
3215
3216 enqueue_resolve_addr_work(work, id_priv);
3217 return 0;
3218 err:
3219 kfree(work);
3220 return ret;
3221 }
3222
3223 static int cma_resolve_ib_addr(struct rdma_id_private *id_priv)
3224 {
3225 struct cma_work *work;
3226 int ret;
3227
3228 work = kzalloc(sizeof *work, GFP_KERNEL);
3229 if (!work)
3230 return -ENOMEM;
3231
3232 if (!id_priv->cma_dev) {
3233 ret = cma_resolve_ib_dev(id_priv);
3234 if (ret)
3235 goto err;
3236 }
3237
3238 rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, (union ib_gid *)
3239 &(((struct sockaddr_ib *) &id_priv->id.route.addr.dst_addr)->sib_addr));
3240
3241 enqueue_resolve_addr_work(work, id_priv);
3242 return 0;
3243 err:
3244 kfree(work);
3245 return ret;
3246 }
3247
3248 static int cma_bind_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
3249 const struct sockaddr *dst_addr)
3250 {
3251 if (!src_addr || !src_addr->sa_family) {
3252 src_addr = (struct sockaddr *) &id->route.addr.src_addr;
3253 src_addr->sa_family = dst_addr->sa_family;
3254 if (IS_ENABLED(CONFIG_IPV6) &&
3255 dst_addr->sa_family == AF_INET6) {
3256 struct sockaddr_in6 *src_addr6 = (struct sockaddr_in6 *) src_addr;
3257 struct sockaddr_in6 *dst_addr6 = (struct sockaddr_in6 *) dst_addr;
3258 src_addr6->sin6_scope_id = dst_addr6->sin6_scope_id;
3259 if (ipv6_addr_type(&dst_addr6->sin6_addr) & IPV6_ADDR_LINKLOCAL)
3260 id->route.addr.dev_addr.bound_dev_if = dst_addr6->sin6_scope_id;
3261 } else if (dst_addr->sa_family == AF_IB) {
3262 ((struct sockaddr_ib *) src_addr)->sib_pkey =
3263 ((struct sockaddr_ib *) dst_addr)->sib_pkey;
3264 }
3265 }
3266 return rdma_bind_addr(id, src_addr);
3267 }
3268
3269 /*
3270 * If required, resolve the source address for bind and leave the id_priv in
3271 * state RDMA_CM_ADDR_BOUND. This oddly uses the state to determine the prior
3272 * calls made by ULP, a previously bound ID will not be re-bound and src_addr is
3273 * ignored.
3274 */
3275 static int resolve_prepare_src(struct rdma_id_private *id_priv,
3276 struct sockaddr *src_addr,
3277 const struct sockaddr *dst_addr)
3278 {
3279 int ret;
3280
3281 memcpy(cma_dst_addr(id_priv), dst_addr, rdma_addr_size(dst_addr));
3282 if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_ADDR_QUERY)) {
3283 /* For a well behaved ULP state will be RDMA_CM_IDLE */
3284 ret = cma_bind_addr(&id_priv->id, src_addr, dst_addr);
3285 if (ret)
3286 goto err_dst;
3287 if (WARN_ON(!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND,
3288 RDMA_CM_ADDR_QUERY))) {
3289 ret = -EINVAL;
3290 goto err_dst;
3291 }
3292 }
3293
3294 if (cma_family(id_priv) != dst_addr->sa_family) {
3295 ret = -EINVAL;
3296 goto err_state;
3297 }
3298 return 0;
3299
3300 err_state:
3301 cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, RDMA_CM_ADDR_BOUND);
3302 err_dst:
3303 memset(cma_dst_addr(id_priv), 0, rdma_addr_size(dst_addr));
3304 return ret;
3305 }
3306
3307 int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
3308 const struct sockaddr *dst_addr, unsigned long timeout_ms)
3309 {
3310 struct rdma_id_private *id_priv =
3311 container_of(id, struct rdma_id_private, id);
3312 int ret;
3313
3314 ret = resolve_prepare_src(id_priv, src_addr, dst_addr);
3315 if (ret)
3316 return ret;
3317
3318 if (cma_any_addr(dst_addr)) {
3319 ret = cma_resolve_loopback(id_priv);
3320 } else {
3321 if (dst_addr->sa_family == AF_IB) {
3322 ret = cma_resolve_ib_addr(id_priv);
3323 } else {
3324 ret = rdma_resolve_ip(cma_src_addr(id_priv), dst_addr,
3325 &id->route.addr.dev_addr,
3326 timeout_ms, addr_handler,
3327 false, id_priv);
3328 }
3329 }
3330 if (ret)
3331 goto err;
3332
3333 return 0;
3334 err:
3335 cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, RDMA_CM_ADDR_BOUND);
3336 return ret;
3337 }
3338 EXPORT_SYMBOL(rdma_resolve_addr);
3339
3340 int rdma_set_reuseaddr(struct rdma_cm_id *id, int reuse)
3341 {
3342 struct rdma_id_private *id_priv;
3343 unsigned long flags;
3344 int ret;
3345
3346 id_priv = container_of(id, struct rdma_id_private, id);
3347 spin_lock_irqsave(&id_priv->lock, flags);
3348 if ((reuse && id_priv->state != RDMA_CM_LISTEN) ||
3349 id_priv->state == RDMA_CM_IDLE) {
3350 id_priv->reuseaddr = reuse;
3351 ret = 0;
3352 } else {
3353 ret = -EINVAL;
3354 }
3355 spin_unlock_irqrestore(&id_priv->lock, flags);
3356 return ret;
3357 }
3358 EXPORT_SYMBOL(rdma_set_reuseaddr);
3359
3360 int rdma_set_afonly(struct rdma_cm_id *id, int afonly)
3361 {
3362 struct rdma_id_private *id_priv;
3363 unsigned long flags;
3364 int ret;
3365
3366 id_priv = container_of(id, struct rdma_id_private, id);
3367 spin_lock_irqsave(&id_priv->lock, flags);
3368 if (id_priv->state == RDMA_CM_IDLE || id_priv->state == RDMA_CM_ADDR_BOUND) {
3369 id_priv->options |= (1 << CMA_OPTION_AFONLY);
3370 id_priv->afonly = afonly;
3371 ret = 0;
3372 } else {
3373 ret = -EINVAL;
3374 }
3375 spin_unlock_irqrestore(&id_priv->lock, flags);
3376 return ret;
3377 }
3378 EXPORT_SYMBOL(rdma_set_afonly);
3379
3380 static void cma_bind_port(struct rdma_bind_list *bind_list,
3381 struct rdma_id_private *id_priv)
3382 {
3383 struct sockaddr *addr;
3384 struct sockaddr_ib *sib;
3385 u64 sid, mask;
3386 __be16 port;
3387
3388 lockdep_assert_held(&lock);
3389
3390 addr = cma_src_addr(id_priv);
3391 port = htons(bind_list->port);
3392
3393 switch (addr->sa_family) {
3394 case AF_INET:
3395 ((struct sockaddr_in *) addr)->sin_port = port;
3396 break;
3397 case AF_INET6:
3398 ((struct sockaddr_in6 *) addr)->sin6_port = port;
3399 break;
3400 case AF_IB:
3401 sib = (struct sockaddr_ib *) addr;
3402 sid = be64_to_cpu(sib->sib_sid);
3403 mask = be64_to_cpu(sib->sib_sid_mask);
3404 sib->sib_sid = cpu_to_be64((sid & mask) | (u64) ntohs(port));
3405 sib->sib_sid_mask = cpu_to_be64(~0ULL);
3406 break;
3407 }
3408 id_priv->bind_list = bind_list;
3409 hlist_add_head(&id_priv->node, &bind_list->owners);
3410 }
3411
3412 static int cma_alloc_port(enum rdma_ucm_port_space ps,
3413 struct rdma_id_private *id_priv, unsigned short snum)
3414 {
3415 struct rdma_bind_list *bind_list;
3416 int ret;
3417
3418 lockdep_assert_held(&lock);
3419
3420 bind_list = kzalloc(sizeof *bind_list, GFP_KERNEL);
3421 if (!bind_list)
3422 return -ENOMEM;
3423
3424 ret = cma_ps_alloc(id_priv->id.route.addr.dev_addr.net, ps, bind_list,
3425 snum);
3426 if (ret < 0)
3427 goto err;
3428
3429 bind_list->ps = ps;
3430 bind_list->port = snum;
3431 cma_bind_port(bind_list, id_priv);
3432 return 0;
3433 err:
3434 kfree(bind_list);
3435 return ret == -ENOSPC ? -EADDRNOTAVAIL : ret;
3436 }
3437
3438 static int cma_port_is_unique(struct rdma_bind_list *bind_list,
3439 struct rdma_id_private *id_priv)
3440 {
3441 struct rdma_id_private *cur_id;
3442 struct sockaddr *daddr = cma_dst_addr(id_priv);
3443 struct sockaddr *saddr = cma_src_addr(id_priv);
3444 __be16 dport = cma_port(daddr);
3445
3446 lockdep_assert_held(&lock);
3447
3448 hlist_for_each_entry(cur_id, &bind_list->owners, node) {
3449 struct sockaddr *cur_daddr = cma_dst_addr(cur_id);
3450 struct sockaddr *cur_saddr = cma_src_addr(cur_id);
3451 __be16 cur_dport = cma_port(cur_daddr);
3452
3453 if (id_priv == cur_id)
3454 continue;
3455
3456 /* different dest port -> unique */
3457 if (!cma_any_port(daddr) &&
3458 !cma_any_port(cur_daddr) &&
3459 (dport != cur_dport))
3460 continue;
3461
3462 /* different src address -> unique */
3463 if (!cma_any_addr(saddr) &&
3464 !cma_any_addr(cur_saddr) &&
3465 cma_addr_cmp(saddr, cur_saddr))
3466 continue;
3467
3468 /* different dst address -> unique */
3469 if (!cma_any_addr(daddr) &&
3470 !cma_any_addr(cur_daddr) &&
3471 cma_addr_cmp(daddr, cur_daddr))
3472 continue;
3473
3474 return -EADDRNOTAVAIL;
3475 }
3476 return 0;
3477 }
3478
3479 static int cma_alloc_any_port(enum rdma_ucm_port_space ps,
3480 struct rdma_id_private *id_priv)
3481 {
3482 static unsigned int last_used_port;
3483 int low, high, remaining;
3484 unsigned int rover;
3485 struct net *net = id_priv->id.route.addr.dev_addr.net;
3486
3487 lockdep_assert_held(&lock);
3488
3489 inet_get_local_port_range(net, &low, &high);
3490 remaining = (high - low) + 1;
3491 rover = prandom_u32() % remaining + low;
3492 retry:
3493 if (last_used_port != rover) {
3494 struct rdma_bind_list *bind_list;
3495 int ret;
3496
3497 bind_list = cma_ps_find(net, ps, (unsigned short)rover);
3498
3499 if (!bind_list) {
3500 ret = cma_alloc_port(ps, id_priv, rover);
3501 } else {
3502 ret = cma_port_is_unique(bind_list, id_priv);
3503 if (!ret)
3504 cma_bind_port(bind_list, id_priv);
3505 }
3506 /*
3507 * Remember previously used port number in order to avoid
3508 * re-using same port immediately after it is closed.
3509 */
3510 if (!ret)
3511 last_used_port = rover;
3512 if (ret != -EADDRNOTAVAIL)
3513 return ret;
3514 }
3515 if (--remaining) {
3516 rover++;
3517 if ((rover < low) || (rover > high))
3518 rover = low;
3519 goto retry;
3520 }
3521 return -EADDRNOTAVAIL;
3522 }
3523
3524 /*
3525 * Check that the requested port is available. This is called when trying to
3526 * bind to a specific port, or when trying to listen on a bound port. In
3527 * the latter case, the provided id_priv may already be on the bind_list, but
3528 * we still need to check that it's okay to start listening.
3529 */
3530 static int cma_check_port(struct rdma_bind_list *bind_list,
3531 struct rdma_id_private *id_priv, uint8_t reuseaddr)
3532 {
3533 struct rdma_id_private *cur_id;
3534 struct sockaddr *addr, *cur_addr;
3535
3536 lockdep_assert_held(&lock);
3537
3538 addr = cma_src_addr(id_priv);
3539 hlist_for_each_entry(cur_id, &bind_list->owners, node) {
3540 if (id_priv == cur_id)
3541 continue;
3542
3543 if (reuseaddr && cur_id->reuseaddr)
3544 continue;
3545
3546 cur_addr = cma_src_addr(cur_id);
3547 if (id_priv->afonly && cur_id->afonly &&
3548 (addr->sa_family != cur_addr->sa_family))
3549 continue;
3550
3551 if (cma_any_addr(addr) || cma_any_addr(cur_addr))
3552 return -EADDRNOTAVAIL;
3553
3554 if (!cma_addr_cmp(addr, cur_addr))
3555 return -EADDRINUSE;
3556 }
3557 return 0;
3558 }
3559
3560 static int cma_use_port(enum rdma_ucm_port_space ps,
3561 struct rdma_id_private *id_priv)
3562 {
3563 struct rdma_bind_list *bind_list;
3564 unsigned short snum;
3565 int ret;
3566
3567 lockdep_assert_held(&lock);
3568
3569 snum = ntohs(cma_port(cma_src_addr(id_priv)));
3570 if (snum < PROT_SOCK && !capable(CAP_NET_BIND_SERVICE))
3571 return -EACCES;
3572
3573 bind_list = cma_ps_find(id_priv->id.route.addr.dev_addr.net, ps, snum);
3574 if (!bind_list) {
3575 ret = cma_alloc_port(ps, id_priv, snum);
3576 } else {
3577 ret = cma_check_port(bind_list, id_priv, id_priv->reuseaddr);
3578 if (!ret)
3579 cma_bind_port(bind_list, id_priv);
3580 }
3581 return ret;
3582 }
3583
3584 static enum rdma_ucm_port_space
3585 cma_select_inet_ps(struct rdma_id_private *id_priv)
3586 {
3587 switch (id_priv->id.ps) {
3588 case RDMA_PS_TCP:
3589 case RDMA_PS_UDP:
3590 case RDMA_PS_IPOIB:
3591 case RDMA_PS_IB:
3592 return id_priv->id.ps;
3593 default:
3594
3595 return 0;
3596 }
3597 }
3598
3599 static enum rdma_ucm_port_space
3600 cma_select_ib_ps(struct rdma_id_private *id_priv)
3601 {
3602 enum rdma_ucm_port_space ps = 0;
3603 struct sockaddr_ib *sib;
3604 u64 sid_ps, mask, sid;
3605
3606 sib = (struct sockaddr_ib *) cma_src_addr(id_priv);
3607 mask = be64_to_cpu(sib->sib_sid_mask) & RDMA_IB_IP_PS_MASK;
3608 sid = be64_to_cpu(sib->sib_sid) & mask;
3609
3610 if ((id_priv->id.ps == RDMA_PS_IB) && (sid == (RDMA_IB_IP_PS_IB & mask))) {
3611 sid_ps = RDMA_IB_IP_PS_IB;
3612 ps = RDMA_PS_IB;
3613 } else if (((id_priv->id.ps == RDMA_PS_IB) || (id_priv->id.ps == RDMA_PS_TCP)) &&
3614 (sid == (RDMA_IB_IP_PS_TCP & mask))) {
3615 sid_ps = RDMA_IB_IP_PS_TCP;
3616 ps = RDMA_PS_TCP;
3617 } else if (((id_priv->id.ps == RDMA_PS_IB) || (id_priv->id.ps == RDMA_PS_UDP)) &&
3618 (sid == (RDMA_IB_IP_PS_UDP & mask))) {
3619 sid_ps = RDMA_IB_IP_PS_UDP;
3620 ps = RDMA_PS_UDP;
3621 }
3622
3623 if (ps) {
3624 sib->sib_sid = cpu_to_be64(sid_ps | ntohs(cma_port((struct sockaddr *) sib)));
3625 sib->sib_sid_mask = cpu_to_be64(RDMA_IB_IP_PS_MASK |
3626 be64_to_cpu(sib->sib_sid_mask));
3627 }
3628 return ps;
3629 }
3630
3631 static int cma_get_port(struct rdma_id_private *id_priv)
3632 {
3633 enum rdma_ucm_port_space ps;
3634 int ret;
3635
3636 if (cma_family(id_priv) != AF_IB)
3637 ps = cma_select_inet_ps(id_priv);
3638 else
3639 ps = cma_select_ib_ps(id_priv);
3640 if (!ps)
3641 return -EPROTONOSUPPORT;
3642
3643 mutex_lock(&lock);
3644 if (cma_any_port(cma_src_addr(id_priv)))
3645 ret = cma_alloc_any_port(ps, id_priv);
3646 else
3647 ret = cma_use_port(ps, id_priv);
3648 mutex_unlock(&lock);
3649
3650 return ret;
3651 }
3652
3653 static int cma_check_linklocal(struct rdma_dev_addr *dev_addr,
3654 struct sockaddr *addr)
3655 {
3656 #if IS_ENABLED(CONFIG_IPV6)
3657 struct sockaddr_in6 *sin6;
3658
3659 if (addr->sa_family != AF_INET6)
3660 return 0;
3661
3662 sin6 = (struct sockaddr_in6 *) addr;
3663
3664 if (!(ipv6_addr_type(&sin6->sin6_addr) & IPV6_ADDR_LINKLOCAL))
3665 return 0;
3666
3667 if (!sin6->sin6_scope_id)
3668 return -EINVAL;
3669
3670 dev_addr->bound_dev_if = sin6->sin6_scope_id;
3671 #endif
3672 return 0;
3673 }
3674
3675 int rdma_listen(struct rdma_cm_id *id, int backlog)
3676 {
3677 struct rdma_id_private *id_priv =
3678 container_of(id, struct rdma_id_private, id);
3679 int ret;
3680
3681 if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_LISTEN)) {
3682 /* For a well behaved ULP state will be RDMA_CM_IDLE */
3683 id->route.addr.src_addr.ss_family = AF_INET;
3684 ret = rdma_bind_addr(id, cma_src_addr(id_priv));
3685 if (ret)
3686 return ret;
3687 if (WARN_ON(!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND,
3688 RDMA_CM_LISTEN)))
3689 return -EINVAL;
3690 }
3691
3692 /*
3693 * Once the ID reaches RDMA_CM_LISTEN it is not allowed to be reusable
3694 * any more, and has to be unique in the bind list.
3695 */
3696 if (id_priv->reuseaddr) {
3697 mutex_lock(&lock);
3698 ret = cma_check_port(id_priv->bind_list, id_priv, 0);
3699 if (!ret)
3700 id_priv->reuseaddr = 0;
3701 mutex_unlock(&lock);
3702 if (ret)
3703 goto err;
3704 }
3705
3706 id_priv->backlog = backlog;
3707 if (id->device) {
3708 if (rdma_cap_ib_cm(id->device, 1)) {
3709 ret = cma_ib_listen(id_priv);
3710 if (ret)
3711 goto err;
3712 } else if (rdma_cap_iw_cm(id->device, 1)) {
3713 ret = cma_iw_listen(id_priv, backlog);
3714 if (ret)
3715 goto err;
3716 } else {
3717 ret = -ENOSYS;
3718 goto err;
3719 }
3720 } else {
3721 ret = cma_listen_on_all(id_priv);
3722 if (ret)
3723 goto err;
3724 }
3725
3726 return 0;
3727 err:
3728 id_priv->backlog = 0;
3729 /*
3730 * All the failure paths that lead here will not allow the req_handler's
3731 * to have run.
3732 */
3733 cma_comp_exch(id_priv, RDMA_CM_LISTEN, RDMA_CM_ADDR_BOUND);
3734 return ret;
3735 }
3736 EXPORT_SYMBOL(rdma_listen);
3737
3738 int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
3739 {
3740 struct rdma_id_private *id_priv;
3741 int ret;
3742 struct sockaddr *daddr;
3743
3744 if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6 &&
3745 addr->sa_family != AF_IB)
3746 return -EAFNOSUPPORT;
3747
3748 id_priv = container_of(id, struct rdma_id_private, id);
3749 if (!cma_comp_exch(id_priv, RDMA_CM_IDLE, RDMA_CM_ADDR_BOUND))
3750 return -EINVAL;
3751
3752 ret = cma_check_linklocal(&id->route.addr.dev_addr, addr);
3753 if (ret)
3754 goto err1;
3755
3756 memcpy(cma_src_addr(id_priv), addr, rdma_addr_size(addr));
3757 if (!cma_any_addr(addr)) {
3758 ret = cma_translate_addr(addr, &id->route.addr.dev_addr);
3759 if (ret)
3760 goto err1;
3761
3762 ret = cma_acquire_dev_by_src_ip(id_priv);
3763 if (ret)
3764 goto err1;
3765 }
3766
3767 if (!(id_priv->options & (1 << CMA_OPTION_AFONLY))) {
3768 if (addr->sa_family == AF_INET)
3769 id_priv->afonly = 1;
3770 #if IS_ENABLED(CONFIG_IPV6)
3771 else if (addr->sa_family == AF_INET6) {
3772 struct net *net = id_priv->id.route.addr.dev_addr.net;
3773
3774 id_priv->afonly = net->ipv6.sysctl.bindv6only;
3775 }
3776 #endif
3777 }
3778 daddr = cma_dst_addr(id_priv);
3779 daddr->sa_family = addr->sa_family;
3780
3781 ret = cma_get_port(id_priv);
3782 if (ret)
3783 goto err2;
3784
3785 return 0;
3786 err2:
3787 if (id_priv->cma_dev)
3788 cma_release_dev(id_priv);
3789 err1:
3790 cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_IDLE);
3791 return ret;
3792 }
3793 EXPORT_SYMBOL(rdma_bind_addr);
3794
3795 static int cma_format_hdr(void *hdr, struct rdma_id_private *id_priv)
3796 {
3797 struct cma_hdr *cma_hdr;
3798
3799 cma_hdr = hdr;
3800 cma_hdr->cma_version = CMA_VERSION;
3801 if (cma_family(id_priv) == AF_INET) {
3802 struct sockaddr_in *src4, *dst4;
3803
3804 src4 = (struct sockaddr_in *) cma_src_addr(id_priv);
3805 dst4 = (struct sockaddr_in *) cma_dst_addr(id_priv);
3806
3807 cma_set_ip_ver(cma_hdr, 4);
3808 cma_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr;
3809 cma_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr;
3810 cma_hdr->port = src4->sin_port;
3811 } else if (cma_family(id_priv) == AF_INET6) {
3812 struct sockaddr_in6 *src6, *dst6;
3813
3814 src6 = (struct sockaddr_in6 *) cma_src_addr(id_priv);
3815 dst6 = (struct sockaddr_in6 *) cma_dst_addr(id_priv);
3816
3817 cma_set_ip_ver(cma_hdr, 6);
3818 cma_hdr->src_addr.ip6 = src6->sin6_addr;
3819 cma_hdr->dst_addr.ip6 = dst6->sin6_addr;
3820 cma_hdr->port = src6->sin6_port;
3821 }
3822 return 0;
3823 }
3824
3825 static int cma_sidr_rep_handler(struct ib_cm_id *cm_id,
3826 const struct ib_cm_event *ib_event)
3827 {
3828 struct rdma_id_private *id_priv = cm_id->context;
3829 struct rdma_cm_event event = {};
3830 const struct ib_cm_sidr_rep_event_param *rep =
3831 &ib_event->param.sidr_rep_rcvd;
3832 int ret;
3833
3834 mutex_lock(&id_priv->handler_mutex);
3835 if (READ_ONCE(id_priv->state) != RDMA_CM_CONNECT)
3836 goto out;
3837
3838 switch (ib_event->event) {
3839 case IB_CM_SIDR_REQ_ERROR:
3840 event.event = RDMA_CM_EVENT_UNREACHABLE;
3841 event.status = -ETIMEDOUT;
3842 break;
3843 case IB_CM_SIDR_REP_RECEIVED:
3844 event.param.ud.private_data = ib_event->private_data;
3845 event.param.ud.private_data_len = IB_CM_SIDR_REP_PRIVATE_DATA_SIZE;
3846 if (rep->status != IB_SIDR_SUCCESS) {
3847 event.event = RDMA_CM_EVENT_UNREACHABLE;
3848 event.status = ib_event->param.sidr_rep_rcvd.status;
3849 pr_debug_ratelimited("RDMA CM: UNREACHABLE: bad SIDR reply. status %d\n",
3850 event.status);
3851 break;
3852 }
3853 ret = cma_set_qkey(id_priv, rep->qkey);
3854 if (ret) {
3855 pr_debug_ratelimited("RDMA CM: ADDR_ERROR: failed to set qkey. status %d\n", ret);
3856 event.event = RDMA_CM_EVENT_ADDR_ERROR;
3857 event.status = ret;
3858 break;
3859 }
3860 ib_init_ah_attr_from_path(id_priv->id.device,
3861 id_priv->id.port_num,
3862 id_priv->id.route.path_rec,
3863 &event.param.ud.ah_attr,
3864 rep->sgid_attr);
3865 event.param.ud.qp_num = rep->qpn;
3866 event.param.ud.qkey = rep->qkey;
3867 event.event = RDMA_CM_EVENT_ESTABLISHED;
3868 event.status = 0;
3869 break;
3870 default:
3871 pr_err("RDMA CMA: unexpected IB CM event: %d\n",
3872 ib_event->event);
3873 goto out;
3874 }
3875
3876 ret = cma_cm_event_handler(id_priv, &event);
3877
3878 rdma_destroy_ah_attr(&event.param.ud.ah_attr);
3879 if (ret) {
3880 /* Destroy the CM ID by returning a non-zero value. */
3881 id_priv->cm_id.ib = NULL;
3882 destroy_id_handler_unlock(id_priv);
3883 return ret;
3884 }
3885 out:
3886 mutex_unlock(&id_priv->handler_mutex);
3887 return 0;
3888 }
3889
3890 static int cma_resolve_ib_udp(struct rdma_id_private *id_priv,
3891 struct rdma_conn_param *conn_param)
3892 {
3893 struct ib_cm_sidr_req_param req;
3894 struct ib_cm_id *id;
3895 void *private_data;
3896 u8 offset;
3897 int ret;
3898
3899 memset(&req, 0, sizeof req);
3900 offset = cma_user_data_offset(id_priv);
3901 req.private_data_len = offset + conn_param->private_data_len;
3902 if (req.private_data_len < conn_param->private_data_len)
3903 return -EINVAL;
3904
3905 if (req.private_data_len) {
3906 private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
3907 if (!private_data)
3908 return -ENOMEM;
3909 } else {
3910 private_data = NULL;
3911 }
3912
3913 if (conn_param->private_data && conn_param->private_data_len)
3914 memcpy(private_data + offset, conn_param->private_data,
3915 conn_param->private_data_len);
3916
3917 if (private_data) {
3918 ret = cma_format_hdr(private_data, id_priv);
3919 if (ret)
3920 goto out;
3921 req.private_data = private_data;
3922 }
3923
3924 id = ib_create_cm_id(id_priv->id.device, cma_sidr_rep_handler,
3925 id_priv);
3926 if (IS_ERR(id)) {
3927 ret = PTR_ERR(id);
3928 goto out;
3929 }
3930 id_priv->cm_id.ib = id;
3931
3932 req.path = id_priv->id.route.path_rec;
3933 req.sgid_attr = id_priv->id.route.addr.dev_addr.sgid_attr;
3934 req.service_id = rdma_get_service_id(&id_priv->id, cma_dst_addr(id_priv));
3935 req.timeout_ms = 1 << (CMA_CM_RESPONSE_TIMEOUT - 8);
3936 req.max_cm_retries = CMA_MAX_CM_RETRIES;
3937
3938 trace_cm_send_sidr_req(id_priv);
3939 ret = ib_send_cm_sidr_req(id_priv->cm_id.ib, &req);
3940 if (ret) {
3941 ib_destroy_cm_id(id_priv->cm_id.ib);
3942 id_priv->cm_id.ib = NULL;
3943 }
3944 out:
3945 kfree(private_data);
3946 return ret;
3947 }
3948
3949 static int cma_connect_ib(struct rdma_id_private *id_priv,
3950 struct rdma_conn_param *conn_param)
3951 {
3952 struct ib_cm_req_param req;
3953 struct rdma_route *route;
3954 void *private_data;
3955 struct ib_cm_id *id;
3956 u8 offset;
3957 int ret;
3958
3959 memset(&req, 0, sizeof req);
3960 offset = cma_user_data_offset(id_priv);
3961 req.private_data_len = offset + conn_param->private_data_len;
3962 if (req.private_data_len < conn_param->private_data_len)
3963 return -EINVAL;
3964
3965 if (req.private_data_len) {
3966 private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
3967 if (!private_data)
3968 return -ENOMEM;
3969 } else {
3970 private_data = NULL;
3971 }
3972
3973 if (conn_param->private_data && conn_param->private_data_len)
3974 memcpy(private_data + offset, conn_param->private_data,
3975 conn_param->private_data_len);
3976
3977 id = ib_create_cm_id(id_priv->id.device, cma_ib_handler, id_priv);
3978 if (IS_ERR(id)) {
3979 ret = PTR_ERR(id);
3980 goto out;
3981 }
3982 id_priv->cm_id.ib = id;
3983
3984 route = &id_priv->id.route;
3985 if (private_data) {
3986 ret = cma_format_hdr(private_data, id_priv);
3987 if (ret)
3988 goto out;
3989 req.private_data = private_data;
3990 }
3991
3992 req.primary_path = &route->path_rec[0];
3993 if (route->num_paths == 2)
3994 req.alternate_path = &route->path_rec[1];
3995
3996 req.ppath_sgid_attr = id_priv->id.route.addr.dev_addr.sgid_attr;
3997 /* Alternate path SGID attribute currently unsupported */
3998 req.service_id = rdma_get_service_id(&id_priv->id, cma_dst_addr(id_priv));
3999 req.qp_num = id_priv->qp_num;
4000 req.qp_type = id_priv->id.qp_type;
4001 req.starting_psn = id_priv->seq_num;
4002 req.responder_resources = conn_param->responder_resources;
4003 req.initiator_depth = conn_param->initiator_depth;
4004 req.flow_control = conn_param->flow_control;
4005 req.retry_count = min_t(u8, 7, conn_param->retry_count);
4006 req.rnr_retry_count = min_t(u8, 7, conn_param->rnr_retry_count);
4007 req.remote_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
4008 req.local_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
4009 req.max_cm_retries = CMA_MAX_CM_RETRIES;
4010 req.srq = id_priv->srq ? 1 : 0;
4011 req.ece.vendor_id = id_priv->ece.vendor_id;
4012 req.ece.attr_mod = id_priv->ece.attr_mod;
4013
4014 trace_cm_send_req(id_priv);
4015 ret = ib_send_cm_req(id_priv->cm_id.ib, &req);
4016 out:
4017 if (ret && !IS_ERR(id)) {
4018 ib_destroy_cm_id(id);
4019 id_priv->cm_id.ib = NULL;
4020 }
4021
4022 kfree(private_data);
4023 return ret;
4024 }
4025
4026 static int cma_connect_iw(struct rdma_id_private *id_priv,
4027 struct rdma_conn_param *conn_param)
4028 {
4029 struct iw_cm_id *cm_id;
4030 int ret;
4031 struct iw_cm_conn_param iw_param;
4032
4033 cm_id = iw_create_cm_id(id_priv->id.device, cma_iw_handler, id_priv);
4034 if (IS_ERR(cm_id))
4035 return PTR_ERR(cm_id);
4036
4037 cm_id->tos = id_priv->tos;
4038 cm_id->tos_set = id_priv->tos_set;
4039 id_priv->cm_id.iw = cm_id;
4040
4041 memcpy(&cm_id->local_addr, cma_src_addr(id_priv),
4042 rdma_addr_size(cma_src_addr(id_priv)));
4043 memcpy(&cm_id->remote_addr, cma_dst_addr(id_priv),
4044 rdma_addr_size(cma_dst_addr(id_priv)));
4045
4046 ret = cma_modify_qp_rtr(id_priv, conn_param);
4047 if (ret)
4048 goto out;
4049
4050 if (conn_param) {
4051 iw_param.ord = conn_param->initiator_depth;
4052 iw_param.ird = conn_param->responder_resources;
4053 iw_param.private_data = conn_param->private_data;
4054 iw_param.private_data_len = conn_param->private_data_len;
4055 iw_param.qpn = id_priv->id.qp ? id_priv->qp_num : conn_param->qp_num;
4056 } else {
4057 memset(&iw_param, 0, sizeof iw_param);
4058 iw_param.qpn = id_priv->qp_num;
4059 }
4060 ret = iw_cm_connect(cm_id, &iw_param);
4061 out:
4062 if (ret) {
4063 iw_destroy_cm_id(cm_id);
4064 id_priv->cm_id.iw = NULL;
4065 }
4066 return ret;
4067 }
4068
4069 /**
4070 * rdma_connect_locked - Initiate an active connection request.
4071 * @id: Connection identifier to connect.
4072 * @conn_param: Connection information used for connected QPs.
4073 *
4074 * Same as rdma_connect() but can only be called from the
4075 * RDMA_CM_EVENT_ROUTE_RESOLVED handler callback.
4076 */
4077 int rdma_connect_locked(struct rdma_cm_id *id,
4078 struct rdma_conn_param *conn_param)
4079 {
4080 struct rdma_id_private *id_priv =
4081 container_of(id, struct rdma_id_private, id);
4082 int ret;
4083
4084 if (!cma_comp_exch(id_priv, RDMA_CM_ROUTE_RESOLVED, RDMA_CM_CONNECT))
4085 return -EINVAL;
4086
4087 if (!id->qp) {
4088 id_priv->qp_num = conn_param->qp_num;
4089 id_priv->srq = conn_param->srq;
4090 }
4091
4092 if (rdma_cap_ib_cm(id->device, id->port_num)) {
4093 if (id->qp_type == IB_QPT_UD)
4094 ret = cma_resolve_ib_udp(id_priv, conn_param);
4095 else
4096 ret = cma_connect_ib(id_priv, conn_param);
4097 } else if (rdma_cap_iw_cm(id->device, id->port_num))
4098 ret = cma_connect_iw(id_priv, conn_param);
4099 else
4100 ret = -ENOSYS;
4101 if (ret)
4102 goto err_state;
4103 return 0;
4104 err_state:
4105 cma_comp_exch(id_priv, RDMA_CM_CONNECT, RDMA_CM_ROUTE_RESOLVED);
4106 return ret;
4107 }
4108 EXPORT_SYMBOL(rdma_connect_locked);
4109
4110 /**
4111 * rdma_connect - Initiate an active connection request.
4112 * @id: Connection identifier to connect.
4113 * @conn_param: Connection information used for connected QPs.
4114 *
4115 * Users must have resolved a route for the rdma_cm_id to connect with by having
4116 * called rdma_resolve_route before calling this routine.
4117 *
4118 * This call will either connect to a remote QP or obtain remote QP information
4119 * for unconnected rdma_cm_id's. The actual operation is based on the
4120 * rdma_cm_id's port space.
4121 */
4122 int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
4123 {
4124 struct rdma_id_private *id_priv =
4125 container_of(id, struct rdma_id_private, id);
4126 int ret;
4127
4128 mutex_lock(&id_priv->handler_mutex);
4129 ret = rdma_connect_locked(id, conn_param);
4130 mutex_unlock(&id_priv->handler_mutex);
4131 return ret;
4132 }
4133 EXPORT_SYMBOL(rdma_connect);
4134
4135 /**
4136 * rdma_connect_ece - Initiate an active connection request with ECE data.
4137 * @id: Connection identifier to connect.
4138 * @conn_param: Connection information used for connected QPs.
4139 * @ece: ECE parameters
4140 *
4141 * See rdma_connect() explanation.
4142 */
4143 int rdma_connect_ece(struct rdma_cm_id *id, struct rdma_conn_param *conn_param,
4144 struct rdma_ucm_ece *ece)
4145 {
4146 struct rdma_id_private *id_priv =
4147 container_of(id, struct rdma_id_private, id);
4148
4149 id_priv->ece.vendor_id = ece->vendor_id;
4150 id_priv->ece.attr_mod = ece->attr_mod;
4151
4152 return rdma_connect(id, conn_param);
4153 }
4154 EXPORT_SYMBOL(rdma_connect_ece);
4155
4156 static int cma_accept_ib(struct rdma_id_private *id_priv,
4157 struct rdma_conn_param *conn_param)
4158 {
4159 struct ib_cm_rep_param rep;
4160 int ret;
4161
4162 ret = cma_modify_qp_rtr(id_priv, conn_param);
4163 if (ret)
4164 goto out;
4165
4166 ret = cma_modify_qp_rts(id_priv, conn_param);
4167 if (ret)
4168 goto out;
4169
4170 memset(&rep, 0, sizeof rep);
4171 rep.qp_num = id_priv->qp_num;
4172 rep.starting_psn = id_priv->seq_num;
4173 rep.private_data = conn_param->private_data;
4174 rep.private_data_len = conn_param->private_data_len;
4175 rep.responder_resources = conn_param->responder_resources;
4176 rep.initiator_depth = conn_param->initiator_depth;
4177 rep.failover_accepted = 0;
4178 rep.flow_control = conn_param->flow_control;
4179 rep.rnr_retry_count = min_t(u8, 7, conn_param->rnr_retry_count);
4180 rep.srq = id_priv->srq ? 1 : 0;
4181 rep.ece.vendor_id = id_priv->ece.vendor_id;
4182 rep.ece.attr_mod = id_priv->ece.attr_mod;
4183
4184 trace_cm_send_rep(id_priv);
4185 ret = ib_send_cm_rep(id_priv->cm_id.ib, &rep);
4186 out:
4187 return ret;
4188 }
4189
4190 static int cma_accept_iw(struct rdma_id_private *id_priv,
4191 struct rdma_conn_param *conn_param)
4192 {
4193 struct iw_cm_conn_param iw_param;
4194 int ret;
4195
4196 if (!conn_param)
4197 return -EINVAL;
4198
4199 ret = cma_modify_qp_rtr(id_priv, conn_param);
4200 if (ret)
4201 return ret;
4202
4203 iw_param.ord = conn_param->initiator_depth;
4204 iw_param.ird = conn_param->responder_resources;
4205 iw_param.private_data = conn_param->private_data;
4206 iw_param.private_data_len = conn_param->private_data_len;
4207 if (id_priv->id.qp) {
4208 iw_param.qpn = id_priv->qp_num;
4209 } else
4210 iw_param.qpn = conn_param->qp_num;
4211
4212 return iw_cm_accept(id_priv->cm_id.iw, &iw_param);
4213 }
4214
4215 static int cma_send_sidr_rep(struct rdma_id_private *id_priv,
4216 enum ib_cm_sidr_status status, u32 qkey,
4217 const void *private_data, int private_data_len)
4218 {
4219 struct ib_cm_sidr_rep_param rep;
4220 int ret;
4221
4222 memset(&rep, 0, sizeof rep);
4223 rep.status = status;
4224 if (status == IB_SIDR_SUCCESS) {
4225 ret = cma_set_qkey(id_priv, qkey);
4226 if (ret)
4227 return ret;
4228 rep.qp_num = id_priv->qp_num;
4229 rep.qkey = id_priv->qkey;
4230
4231 rep.ece.vendor_id = id_priv->ece.vendor_id;
4232 rep.ece.attr_mod = id_priv->ece.attr_mod;
4233 }
4234
4235 rep.private_data = private_data;
4236 rep.private_data_len = private_data_len;
4237
4238 trace_cm_send_sidr_rep(id_priv);
4239 return ib_send_cm_sidr_rep(id_priv->cm_id.ib, &rep);
4240 }
4241
4242 /**
4243 * rdma_accept - Called to accept a connection request or response.
4244 * @id: Connection identifier associated with the request.
4245 * @conn_param: Information needed to establish the connection. This must be
4246 * provided if accepting a connection request. If accepting a connection
4247 * response, this parameter must be NULL.
4248 *
4249 * Typically, this routine is only called by the listener to accept a connection
4250 * request. It must also be called on the active side of a connection if the
4251 * user is performing their own QP transitions.
4252 *
4253 * In the case of error, a reject message is sent to the remote side and the
4254 * state of the qp associated with the id is modified to error, such that any
4255 * previously posted receive buffers would be flushed.
4256 *
4257 * This function is for use by kernel ULPs and must be called from under the
4258 * handler callback.
4259 */
4260 int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
4261 {
4262 struct rdma_id_private *id_priv =
4263 container_of(id, struct rdma_id_private, id);
4264 int ret;
4265
4266 lockdep_assert_held(&id_priv->handler_mutex);
4267
4268 if (READ_ONCE(id_priv->state) != RDMA_CM_CONNECT)
4269 return -EINVAL;
4270
4271 if (!id->qp && conn_param) {
4272 id_priv->qp_num = conn_param->qp_num;
4273 id_priv->srq = conn_param->srq;
4274 }
4275
4276 if (rdma_cap_ib_cm(id->device, id->port_num)) {
4277 if (id->qp_type == IB_QPT_UD) {
4278 if (conn_param)
4279 ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS,
4280 conn_param->qkey,
4281 conn_param->private_data,
4282 conn_param->private_data_len);
4283 else
4284 ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS,
4285 0, NULL, 0);
4286 } else {
4287 if (conn_param)
4288 ret = cma_accept_ib(id_priv, conn_param);
4289 else
4290 ret = cma_rep_recv(id_priv);
4291 }
4292 } else if (rdma_cap_iw_cm(id->device, id->port_num))
4293 ret = cma_accept_iw(id_priv, conn_param);
4294 else
4295 ret = -ENOSYS;
4296
4297 if (ret)
4298 goto reject;
4299
4300 return 0;
4301 reject:
4302 cma_modify_qp_err(id_priv);
4303 rdma_reject(id, NULL, 0, IB_CM_REJ_CONSUMER_DEFINED);
4304 return ret;
4305 }
4306 EXPORT_SYMBOL(rdma_accept);
4307
4308 int rdma_accept_ece(struct rdma_cm_id *id, struct rdma_conn_param *conn_param,
4309 struct rdma_ucm_ece *ece)
4310 {
4311 struct rdma_id_private *id_priv =
4312 container_of(id, struct rdma_id_private, id);
4313
4314 id_priv->ece.vendor_id = ece->vendor_id;
4315 id_priv->ece.attr_mod = ece->attr_mod;
4316
4317 return rdma_accept(id, conn_param);
4318 }
4319 EXPORT_SYMBOL(rdma_accept_ece);
4320
4321 void rdma_lock_handler(struct rdma_cm_id *id)
4322 {
4323 struct rdma_id_private *id_priv =
4324 container_of(id, struct rdma_id_private, id);
4325
4326 mutex_lock(&id_priv->handler_mutex);
4327 }
4328 EXPORT_SYMBOL(rdma_lock_handler);
4329
4330 void rdma_unlock_handler(struct rdma_cm_id *id)
4331 {
4332 struct rdma_id_private *id_priv =
4333 container_of(id, struct rdma_id_private, id);
4334
4335 mutex_unlock(&id_priv->handler_mutex);
4336 }
4337 EXPORT_SYMBOL(rdma_unlock_handler);
4338
4339 int rdma_notify(struct rdma_cm_id *id, enum ib_event_type event)
4340 {
4341 struct rdma_id_private *id_priv;
4342 int ret;
4343
4344 id_priv = container_of(id, struct rdma_id_private, id);
4345 if (!id_priv->cm_id.ib)
4346 return -EINVAL;
4347
4348 switch (id->device->node_type) {
4349 case RDMA_NODE_IB_CA:
4350 ret = ib_cm_notify(id_priv->cm_id.ib, event);
4351 break;
4352 default:
4353 ret = 0;
4354 break;
4355 }
4356 return ret;
4357 }
4358 EXPORT_SYMBOL(rdma_notify);
4359
4360 int rdma_reject(struct rdma_cm_id *id, const void *private_data,
4361 u8 private_data_len, u8 reason)
4362 {
4363 struct rdma_id_private *id_priv;
4364 int ret;
4365
4366 id_priv = container_of(id, struct rdma_id_private, id);
4367 if (!id_priv->cm_id.ib)
4368 return -EINVAL;
4369
4370 if (rdma_cap_ib_cm(id->device, id->port_num)) {
4371 if (id->qp_type == IB_QPT_UD) {
4372 ret = cma_send_sidr_rep(id_priv, IB_SIDR_REJECT, 0,
4373 private_data, private_data_len);
4374 } else {
4375 trace_cm_send_rej(id_priv);
4376 ret = ib_send_cm_rej(id_priv->cm_id.ib, reason, NULL, 0,
4377 private_data, private_data_len);
4378 }
4379 } else if (rdma_cap_iw_cm(id->device, id->port_num)) {
4380 ret = iw_cm_reject(id_priv->cm_id.iw,
4381 private_data, private_data_len);
4382 } else
4383 ret = -ENOSYS;
4384
4385 return ret;
4386 }
4387 EXPORT_SYMBOL(rdma_reject);
4388
4389 int rdma_disconnect(struct rdma_cm_id *id)
4390 {
4391 struct rdma_id_private *id_priv;
4392 int ret;
4393
4394 id_priv = container_of(id, struct rdma_id_private, id);
4395 if (!id_priv->cm_id.ib)
4396 return -EINVAL;
4397
4398 if (rdma_cap_ib_cm(id->device, id->port_num)) {
4399 ret = cma_modify_qp_err(id_priv);
4400 if (ret)
4401 goto out;
4402 /* Initiate or respond to a disconnect. */
4403 trace_cm_disconnect(id_priv);
4404 if (ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0)) {
4405 if (!ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0))
4406 trace_cm_sent_drep(id_priv);
4407 } else {
4408 trace_cm_sent_dreq(id_priv);
4409 }
4410 } else if (rdma_cap_iw_cm(id->device, id->port_num)) {
4411 ret = iw_cm_disconnect(id_priv->cm_id.iw, 0);
4412 } else
4413 ret = -EINVAL;
4414
4415 out:
4416 return ret;
4417 }
4418 EXPORT_SYMBOL(rdma_disconnect);
4419
4420 static void cma_make_mc_event(int status, struct rdma_id_private *id_priv,
4421 struct ib_sa_multicast *multicast,
4422 struct rdma_cm_event *event,
4423 struct cma_multicast *mc)
4424 {
4425 struct rdma_dev_addr *dev_addr;
4426 enum ib_gid_type gid_type;
4427 struct net_device *ndev;
4428
4429 if (!status)
4430 status = cma_set_qkey(id_priv, be32_to_cpu(multicast->rec.qkey));
4431 else
4432 pr_debug_ratelimited("RDMA CM: MULTICAST_ERROR: failed to join multicast. status %d\n",
4433 status);
4434
4435 event->status = status;
4436 event->param.ud.private_data = mc->context;
4437 if (status) {
4438 event->event = RDMA_CM_EVENT_MULTICAST_ERROR;
4439 return;
4440 }
4441
4442 dev_addr = &id_priv->id.route.addr.dev_addr;
4443 ndev = dev_get_by_index(dev_addr->net, dev_addr->bound_dev_if);
4444 gid_type =
4445 id_priv->cma_dev
4446 ->default_gid_type[id_priv->id.port_num -
4447 rdma_start_port(
4448 id_priv->cma_dev->device)];
4449
4450 event->event = RDMA_CM_EVENT_MULTICAST_JOIN;
4451 if (ib_init_ah_from_mcmember(id_priv->id.device, id_priv->id.port_num,
4452 &multicast->rec, ndev, gid_type,
4453 &event->param.ud.ah_attr)) {
4454 event->event = RDMA_CM_EVENT_MULTICAST_ERROR;
4455 goto out;
4456 }
4457
4458 event->param.ud.qp_num = 0xFFFFFF;
4459 event->param.ud.qkey = be32_to_cpu(multicast->rec.qkey);
4460
4461 out:
4462 if (ndev)
4463 dev_put(ndev);
4464 }
4465
4466 static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast)
4467 {
4468 struct cma_multicast *mc = multicast->context;
4469 struct rdma_id_private *id_priv = mc->id_priv;
4470 struct rdma_cm_event event = {};
4471 int ret = 0;
4472
4473 mutex_lock(&id_priv->handler_mutex);
4474 if (READ_ONCE(id_priv->state) == RDMA_CM_DEVICE_REMOVAL ||
4475 READ_ONCE(id_priv->state) == RDMA_CM_DESTROYING)
4476 goto out;
4477
4478 cma_make_mc_event(status, id_priv, multicast, &event, mc);
4479 ret = cma_cm_event_handler(id_priv, &event);
4480 rdma_destroy_ah_attr(&event.param.ud.ah_attr);
4481 if (ret) {
4482 destroy_id_handler_unlock(id_priv);
4483 return 0;
4484 }
4485
4486 out:
4487 mutex_unlock(&id_priv->handler_mutex);
4488 return 0;
4489 }
4490
4491 static void cma_set_mgid(struct rdma_id_private *id_priv,
4492 struct sockaddr *addr, union ib_gid *mgid)
4493 {
4494 unsigned char mc_map[MAX_ADDR_LEN];
4495 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
4496 struct sockaddr_in *sin = (struct sockaddr_in *) addr;
4497 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) addr;
4498
4499 if (cma_any_addr(addr)) {
4500 memset(mgid, 0, sizeof *mgid);
4501 } else if ((addr->sa_family == AF_INET6) &&
4502 ((be32_to_cpu(sin6->sin6_addr.s6_addr32[0]) & 0xFFF0FFFF) ==
4503 0xFF10A01B)) {
4504 /* IPv6 address is an SA assigned MGID. */
4505 memcpy(mgid, &sin6->sin6_addr, sizeof *mgid);
4506 } else if (addr->sa_family == AF_IB) {
4507 memcpy(mgid, &((struct sockaddr_ib *) addr)->sib_addr, sizeof *mgid);
4508 } else if (addr->sa_family == AF_INET6) {
4509 ipv6_ib_mc_map(&sin6->sin6_addr, dev_addr->broadcast, mc_map);
4510 if (id_priv->id.ps == RDMA_PS_UDP)
4511 mc_map[7] = 0x01; /* Use RDMA CM signature */
4512 *mgid = *(union ib_gid *) (mc_map + 4);
4513 } else {
4514 ip_ib_mc_map(sin->sin_addr.s_addr, dev_addr->broadcast, mc_map);
4515 if (id_priv->id.ps == RDMA_PS_UDP)
4516 mc_map[7] = 0x01; /* Use RDMA CM signature */
4517 *mgid = *(union ib_gid *) (mc_map + 4);
4518 }
4519 }
4520
4521 static int cma_join_ib_multicast(struct rdma_id_private *id_priv,
4522 struct cma_multicast *mc)
4523 {
4524 struct ib_sa_mcmember_rec rec;
4525 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
4526 ib_sa_comp_mask comp_mask;
4527 int ret;
4528
4529 ib_addr_get_mgid(dev_addr, &rec.mgid);
4530 ret = ib_sa_get_mcmember_rec(id_priv->id.device, id_priv->id.port_num,
4531 &rec.mgid, &rec);
4532 if (ret)
4533 return ret;
4534
4535 ret = cma_set_qkey(id_priv, 0);
4536 if (ret)
4537 return ret;
4538
4539 cma_set_mgid(id_priv, (struct sockaddr *) &mc->addr, &rec.mgid);
4540 rec.qkey = cpu_to_be32(id_priv->qkey);
4541 rdma_addr_get_sgid(dev_addr, &rec.port_gid);
4542 rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr));
4543 rec.join_state = mc->join_state;
4544
4545 if ((rec.join_state == BIT(SENDONLY_FULLMEMBER_JOIN)) &&
4546 (!ib_sa_sendonly_fullmem_support(&sa_client,
4547 id_priv->id.device,
4548 id_priv->id.port_num))) {
4549 dev_warn(
4550 &id_priv->id.device->dev,
4551 "RDMA CM: port %u Unable to multicast join: SM doesn't support Send Only Full Member option\n",
4552 id_priv->id.port_num);
4553 return -EOPNOTSUPP;
4554 }
4555
4556 comp_mask = IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID |
4557 IB_SA_MCMEMBER_REC_PKEY | IB_SA_MCMEMBER_REC_JOIN_STATE |
4558 IB_SA_MCMEMBER_REC_QKEY | IB_SA_MCMEMBER_REC_SL |
4559 IB_SA_MCMEMBER_REC_FLOW_LABEL |
4560 IB_SA_MCMEMBER_REC_TRAFFIC_CLASS;
4561
4562 if (id_priv->id.ps == RDMA_PS_IPOIB)
4563 comp_mask |= IB_SA_MCMEMBER_REC_RATE |
4564 IB_SA_MCMEMBER_REC_RATE_SELECTOR |
4565 IB_SA_MCMEMBER_REC_MTU_SELECTOR |
4566 IB_SA_MCMEMBER_REC_MTU |
4567 IB_SA_MCMEMBER_REC_HOP_LIMIT;
4568
4569 mc->sa_mc = ib_sa_join_multicast(&sa_client, id_priv->id.device,
4570 id_priv->id.port_num, &rec, comp_mask,
4571 GFP_KERNEL, cma_ib_mc_handler, mc);
4572 return PTR_ERR_OR_ZERO(mc->sa_mc);
4573 }
4574
4575 static void cma_iboe_set_mgid(struct sockaddr *addr, union ib_gid *mgid,
4576 enum ib_gid_type gid_type)
4577 {
4578 struct sockaddr_in *sin = (struct sockaddr_in *)addr;
4579 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr;
4580
4581 if (cma_any_addr(addr)) {
4582 memset(mgid, 0, sizeof *mgid);
4583 } else if (addr->sa_family == AF_INET6) {
4584 memcpy(mgid, &sin6->sin6_addr, sizeof *mgid);
4585 } else {
4586 mgid->raw[0] =
4587 (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) ? 0 : 0xff;
4588 mgid->raw[1] =
4589 (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) ? 0 : 0x0e;
4590 mgid->raw[2] = 0;
4591 mgid->raw[3] = 0;
4592 mgid->raw[4] = 0;
4593 mgid->raw[5] = 0;
4594 mgid->raw[6] = 0;
4595 mgid->raw[7] = 0;
4596 mgid->raw[8] = 0;
4597 mgid->raw[9] = 0;
4598 mgid->raw[10] = 0xff;
4599 mgid->raw[11] = 0xff;
4600 *(__be32 *)(&mgid->raw[12]) = sin->sin_addr.s_addr;
4601 }
4602 }
4603
4604 static int cma_iboe_join_multicast(struct rdma_id_private *id_priv,
4605 struct cma_multicast *mc)
4606 {
4607 struct cma_work *work;
4608 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
4609 int err = 0;
4610 struct sockaddr *addr = (struct sockaddr *)&mc->addr;
4611 struct net_device *ndev = NULL;
4612 struct ib_sa_multicast ib;
4613 enum ib_gid_type gid_type;
4614 bool send_only;
4615
4616 send_only = mc->join_state == BIT(SENDONLY_FULLMEMBER_JOIN);
4617
4618 if (cma_zero_addr(addr))
4619 return -EINVAL;
4620
4621 work = kzalloc(sizeof *work, GFP_KERNEL);
4622 if (!work)
4623 return -ENOMEM;
4624
4625 gid_type = id_priv->cma_dev->default_gid_type[id_priv->id.port_num -
4626 rdma_start_port(id_priv->cma_dev->device)];
4627 cma_iboe_set_mgid(addr, &ib.rec.mgid, gid_type);
4628
4629 ib.rec.pkey = cpu_to_be16(0xffff);
4630 if (id_priv->id.ps == RDMA_PS_UDP)
4631 ib.rec.qkey = cpu_to_be32(RDMA_UDP_QKEY);
4632
4633 if (dev_addr->bound_dev_if)
4634 ndev = dev_get_by_index(dev_addr->net, dev_addr->bound_dev_if);
4635 if (!ndev) {
4636 err = -ENODEV;
4637 goto err_free;
4638 }
4639 ib.rec.rate = iboe_get_rate(ndev);
4640 ib.rec.hop_limit = 1;
4641 ib.rec.mtu = iboe_get_mtu(ndev->mtu);
4642
4643 if (addr->sa_family == AF_INET) {
4644 if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) {
4645 ib.rec.hop_limit = IPV6_DEFAULT_HOPLIMIT;
4646 if (!send_only) {
4647 err = cma_igmp_send(ndev, &ib.rec.mgid,
4648 true);
4649 }
4650 }
4651 } else {
4652 if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP)
4653 err = -ENOTSUPP;
4654 }
4655 dev_put(ndev);
4656 if (err || !ib.rec.mtu) {
4657 if (!err)
4658 err = -EINVAL;
4659 goto err_free;
4660 }
4661 rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr,
4662 &ib.rec.port_gid);
4663 work->id = id_priv;
4664 INIT_WORK(&work->work, cma_work_handler);
4665 cma_make_mc_event(0, id_priv, &ib, &work->event, mc);
4666 /* Balances with cma_id_put() in cma_work_handler */
4667 cma_id_get(id_priv);
4668 queue_work(cma_wq, &work->work);
4669 return 0;
4670
4671 err_free:
4672 kfree(work);
4673 return err;
4674 }
4675
4676 int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,
4677 u8 join_state, void *context)
4678 {
4679 struct rdma_id_private *id_priv =
4680 container_of(id, struct rdma_id_private, id);
4681 struct cma_multicast *mc;
4682 int ret;
4683
4684 /* Not supported for kernel QPs */
4685 if (WARN_ON(id->qp))
4686 return -EINVAL;
4687
4688 /* ULP is calling this wrong. */
4689 if (!id->device || (READ_ONCE(id_priv->state) != RDMA_CM_ADDR_BOUND &&
4690 READ_ONCE(id_priv->state) != RDMA_CM_ADDR_RESOLVED))
4691 return -EINVAL;
4692
4693 mc = kzalloc(sizeof(*mc), GFP_KERNEL);
4694 if (!mc)
4695 return -ENOMEM;
4696
4697 memcpy(&mc->addr, addr, rdma_addr_size(addr));
4698 mc->context = context;
4699 mc->id_priv = id_priv;
4700 mc->join_state = join_state;
4701
4702 if (rdma_protocol_roce(id->device, id->port_num)) {
4703 ret = cma_iboe_join_multicast(id_priv, mc);
4704 if (ret)
4705 goto out_err;
4706 } else if (rdma_cap_ib_mcast(id->device, id->port_num)) {
4707 ret = cma_join_ib_multicast(id_priv, mc);
4708 if (ret)
4709 goto out_err;
4710 } else {
4711 ret = -ENOSYS;
4712 goto out_err;
4713 }
4714
4715 spin_lock(&id_priv->lock);
4716 list_add(&mc->list, &id_priv->mc_list);
4717 spin_unlock(&id_priv->lock);
4718
4719 return 0;
4720 out_err:
4721 kfree(mc);
4722 return ret;
4723 }
4724 EXPORT_SYMBOL(rdma_join_multicast);
4725
4726 void rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr)
4727 {
4728 struct rdma_id_private *id_priv;
4729 struct cma_multicast *mc;
4730
4731 id_priv = container_of(id, struct rdma_id_private, id);
4732 spin_lock_irq(&id_priv->lock);
4733 list_for_each_entry(mc, &id_priv->mc_list, list) {
4734 if (memcmp(&mc->addr, addr, rdma_addr_size(addr)) != 0)
4735 continue;
4736 list_del(&mc->list);
4737 spin_unlock_irq(&id_priv->lock);
4738
4739 WARN_ON(id_priv->cma_dev->device != id->device);
4740 destroy_mc(id_priv, mc);
4741 return;
4742 }
4743 spin_unlock_irq(&id_priv->lock);
4744 }
4745 EXPORT_SYMBOL(rdma_leave_multicast);
4746
4747 static int cma_netdev_change(struct net_device *ndev, struct rdma_id_private *id_priv)
4748 {
4749 struct rdma_dev_addr *dev_addr;
4750 struct cma_work *work;
4751
4752 dev_addr = &id_priv->id.route.addr.dev_addr;
4753
4754 if ((dev_addr->bound_dev_if == ndev->ifindex) &&
4755 (net_eq(dev_net(ndev), dev_addr->net)) &&
4756 memcmp(dev_addr->src_dev_addr, ndev->dev_addr, ndev->addr_len)) {
4757 pr_info("RDMA CM addr change for ndev %s used by id %p\n",
4758 ndev->name, &id_priv->id);
4759 work = kzalloc(sizeof *work, GFP_KERNEL);
4760 if (!work)
4761 return -ENOMEM;
4762
4763 INIT_WORK(&work->work, cma_work_handler);
4764 work->id = id_priv;
4765 work->event.event = RDMA_CM_EVENT_ADDR_CHANGE;
4766 cma_id_get(id_priv);
4767 queue_work(cma_wq, &work->work);
4768 }
4769
4770 return 0;
4771 }
4772
4773 static int cma_netdev_callback(struct notifier_block *self, unsigned long event,
4774 void *ptr)
4775 {
4776 struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
4777 struct cma_device *cma_dev;
4778 struct rdma_id_private *id_priv;
4779 int ret = NOTIFY_DONE;
4780
4781 if (event != NETDEV_BONDING_FAILOVER)
4782 return NOTIFY_DONE;
4783
4784 if (!netif_is_bond_master(ndev))
4785 return NOTIFY_DONE;
4786
4787 mutex_lock(&lock);
4788 list_for_each_entry(cma_dev, &dev_list, list)
4789 list_for_each_entry(id_priv, &cma_dev->id_list, list) {
4790 ret = cma_netdev_change(ndev, id_priv);
4791 if (ret)
4792 goto out;
4793 }
4794
4795 out:
4796 mutex_unlock(&lock);
4797 return ret;
4798 }
4799
4800 static struct notifier_block cma_nb = {
4801 .notifier_call = cma_netdev_callback
4802 };
4803
4804 static void cma_send_device_removal_put(struct rdma_id_private *id_priv)
4805 {
4806 struct rdma_cm_event event = { .event = RDMA_CM_EVENT_DEVICE_REMOVAL };
4807 enum rdma_cm_state state;
4808 unsigned long flags;
4809
4810 mutex_lock(&id_priv->handler_mutex);
4811 /* Record that we want to remove the device */
4812 spin_lock_irqsave(&id_priv->lock, flags);
4813 state = id_priv->state;
4814 if (state == RDMA_CM_DESTROYING || state == RDMA_CM_DEVICE_REMOVAL) {
4815 spin_unlock_irqrestore(&id_priv->lock, flags);
4816 mutex_unlock(&id_priv->handler_mutex);
4817 cma_id_put(id_priv);
4818 return;
4819 }
4820 id_priv->state = RDMA_CM_DEVICE_REMOVAL;
4821 spin_unlock_irqrestore(&id_priv->lock, flags);
4822
4823 if (cma_cm_event_handler(id_priv, &event)) {
4824 /*
4825 * At this point the ULP promises it won't call
4826 * rdma_destroy_id() concurrently
4827 */
4828 cma_id_put(id_priv);
4829 mutex_unlock(&id_priv->handler_mutex);
4830 trace_cm_id_destroy(id_priv);
4831 _destroy_id(id_priv, state);
4832 return;
4833 }
4834 mutex_unlock(&id_priv->handler_mutex);
4835
4836 /*
4837 * If this races with destroy then the thread that first assigns state
4838 * to a destroying does the cancel.
4839 */
4840 cma_cancel_operation(id_priv, state);
4841 cma_id_put(id_priv);
4842 }
4843
4844 static void cma_process_remove(struct cma_device *cma_dev)
4845 {
4846 mutex_lock(&lock);
4847 while (!list_empty(&cma_dev->id_list)) {
4848 struct rdma_id_private *id_priv = list_first_entry(
4849 &cma_dev->id_list, struct rdma_id_private, list);
4850
4851 list_del(&id_priv->listen_list);
4852 list_del_init(&id_priv->list);
4853 cma_id_get(id_priv);
4854 mutex_unlock(&lock);
4855
4856 cma_send_device_removal_put(id_priv);
4857
4858 mutex_lock(&lock);
4859 }
4860 mutex_unlock(&lock);
4861
4862 cma_dev_put(cma_dev);
4863 wait_for_completion(&cma_dev->comp);
4864 }
4865
4866 static int cma_add_one(struct ib_device *device)
4867 {
4868 struct rdma_id_private *to_destroy;
4869 struct cma_device *cma_dev;
4870 struct rdma_id_private *id_priv;
4871 unsigned int i;
4872 unsigned long supported_gids = 0;
4873 int ret;
4874
4875 cma_dev = kmalloc(sizeof(*cma_dev), GFP_KERNEL);
4876 if (!cma_dev)
4877 return -ENOMEM;
4878
4879 cma_dev->device = device;
4880 cma_dev->default_gid_type = kcalloc(device->phys_port_cnt,
4881 sizeof(*cma_dev->default_gid_type),
4882 GFP_KERNEL);
4883 if (!cma_dev->default_gid_type) {
4884 ret = -ENOMEM;
4885 goto free_cma_dev;
4886 }
4887
4888 cma_dev->default_roce_tos = kcalloc(device->phys_port_cnt,
4889 sizeof(*cma_dev->default_roce_tos),
4890 GFP_KERNEL);
4891 if (!cma_dev->default_roce_tos) {
4892 ret = -ENOMEM;
4893 goto free_gid_type;
4894 }
4895
4896 rdma_for_each_port (device, i) {
4897 supported_gids = roce_gid_type_mask_support(device, i);
4898 WARN_ON(!supported_gids);
4899 if (supported_gids & (1 << CMA_PREFERRED_ROCE_GID_TYPE))
4900 cma_dev->default_gid_type[i - rdma_start_port(device)] =
4901 CMA_PREFERRED_ROCE_GID_TYPE;
4902 else
4903 cma_dev->default_gid_type[i - rdma_start_port(device)] =
4904 find_first_bit(&supported_gids, BITS_PER_LONG);
4905 cma_dev->default_roce_tos[i - rdma_start_port(device)] = 0;
4906 }
4907
4908 init_completion(&cma_dev->comp);
4909 refcount_set(&cma_dev->refcount, 1);
4910 INIT_LIST_HEAD(&cma_dev->id_list);
4911 ib_set_client_data(device, &cma_client, cma_dev);
4912
4913 mutex_lock(&lock);
4914 list_add_tail(&cma_dev->list, &dev_list);
4915 list_for_each_entry(id_priv, &listen_any_list, list) {
4916 ret = cma_listen_on_dev(id_priv, cma_dev, &to_destroy);
4917 if (ret)
4918 goto free_listen;
4919 }
4920 mutex_unlock(&lock);
4921
4922 trace_cm_add_one(device);
4923 return 0;
4924
4925 free_listen:
4926 list_del(&cma_dev->list);
4927 mutex_unlock(&lock);
4928
4929 /* cma_process_remove() will delete to_destroy */
4930 cma_process_remove(cma_dev);
4931 kfree(cma_dev->default_roce_tos);
4932 free_gid_type:
4933 kfree(cma_dev->default_gid_type);
4934
4935 free_cma_dev:
4936 kfree(cma_dev);
4937 return ret;
4938 }
4939
4940 static void cma_remove_one(struct ib_device *device, void *client_data)
4941 {
4942 struct cma_device *cma_dev = client_data;
4943
4944 trace_cm_remove_one(device);
4945
4946 mutex_lock(&lock);
4947 list_del(&cma_dev->list);
4948 mutex_unlock(&lock);
4949
4950 cma_process_remove(cma_dev);
4951 kfree(cma_dev->default_roce_tos);
4952 kfree(cma_dev->default_gid_type);
4953 kfree(cma_dev);
4954 }
4955
4956 static int cma_init_net(struct net *net)
4957 {
4958 struct cma_pernet *pernet = cma_pernet(net);
4959
4960 xa_init(&pernet->tcp_ps);
4961 xa_init(&pernet->udp_ps);
4962 xa_init(&pernet->ipoib_ps);
4963 xa_init(&pernet->ib_ps);
4964
4965 return 0;
4966 }
4967
4968 static void cma_exit_net(struct net *net)
4969 {
4970 struct cma_pernet *pernet = cma_pernet(net);
4971
4972 WARN_ON(!xa_empty(&pernet->tcp_ps));
4973 WARN_ON(!xa_empty(&pernet->udp_ps));
4974 WARN_ON(!xa_empty(&pernet->ipoib_ps));
4975 WARN_ON(!xa_empty(&pernet->ib_ps));
4976 }
4977
4978 static struct pernet_operations cma_pernet_operations = {
4979 .init = cma_init_net,
4980 .exit = cma_exit_net,
4981 .id = &cma_pernet_id,
4982 .size = sizeof(struct cma_pernet),
4983 };
4984
4985 static int __init cma_init(void)
4986 {
4987 int ret;
4988
4989 /*
4990 * There is a rare lock ordering dependency in cma_netdev_callback()
4991 * that only happens when bonding is enabled. Teach lockdep that rtnl
4992 * must never be nested under lock so it can find these without having
4993 * to test with bonding.
4994 */
4995 if (IS_ENABLED(CONFIG_LOCKDEP)) {
4996 rtnl_lock();
4997 mutex_lock(&lock);
4998 mutex_unlock(&lock);
4999 rtnl_unlock();
5000 }
5001
5002 cma_wq = alloc_ordered_workqueue("rdma_cm", WQ_MEM_RECLAIM);
5003 if (!cma_wq)
5004 return -ENOMEM;
5005
5006 ret = register_pernet_subsys(&cma_pernet_operations);
5007 if (ret)
5008 goto err_wq;
5009
5010 ib_sa_register_client(&sa_client);
5011 register_netdevice_notifier(&cma_nb);
5012
5013 ret = ib_register_client(&cma_client);
5014 if (ret)
5015 goto err;
5016
5017 ret = cma_configfs_init();
5018 if (ret)
5019 goto err_ib;
5020
5021 return 0;
5022
5023 err_ib:
5024 ib_unregister_client(&cma_client);
5025 err:
5026 unregister_netdevice_notifier(&cma_nb);
5027 ib_sa_unregister_client(&sa_client);
5028 unregister_pernet_subsys(&cma_pernet_operations);
5029 err_wq:
5030 destroy_workqueue(cma_wq);
5031 return ret;
5032 }
5033
5034 static void __exit cma_cleanup(void)
5035 {
5036 cma_configfs_exit();
5037 ib_unregister_client(&cma_client);
5038 unregister_netdevice_notifier(&cma_nb);
5039 ib_sa_unregister_client(&sa_client);
5040 unregister_pernet_subsys(&cma_pernet_operations);
5041 destroy_workqueue(cma_wq);
5042 }
5043
5044 module_init(cma_init);
5045 module_exit(cma_cleanup);
Linux with added FPC-III support