?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
dm writecache: add cond_resched to loop in persistent_memory_claim()
[linux/fpc-iii.git] / drivers / infiniband / core / cm.c
blob1c2bf18cda9f6cabd2c76fd2d32a081e5d5f470f
1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3 * Copyright (c) 2004-2007 Intel Corporation. All rights reserved.
4 * Copyright (c) 2004 Topspin Corporation. All rights reserved.
5 * Copyright (c) 2004, 2005 Voltaire Corporation. All rights reserved.
6 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
7 * Copyright (c) 2019, Mellanox Technologies inc. All rights reserved.
8 */
9
10 #include <linux/completion.h>
11 #include <linux/dma-mapping.h>
12 #include <linux/device.h>
13 #include <linux/module.h>
14 #include <linux/err.h>
15 #include <linux/idr.h>
16 #include <linux/interrupt.h>
17 #include <linux/random.h>
18 #include <linux/rbtree.h>
19 #include <linux/spinlock.h>
20 #include <linux/slab.h>
21 #include <linux/sysfs.h>
22 #include <linux/workqueue.h>
23 #include <linux/kdev_t.h>
24 #include <linux/etherdevice.h>
25
26 #include <rdma/ib_cache.h>
27 #include <rdma/ib_cm.h>
28 #include "cm_msgs.h"
29 #include "core_priv.h"
30
31 MODULE_AUTHOR("Sean Hefty");
32 MODULE_DESCRIPTION("InfiniBand CM");
33 MODULE_LICENSE("Dual BSD/GPL");
34
35 static const char * const ibcm_rej_reason_strs[] = {
36 [IB_CM_REJ_NO_QP] = "no QP",
37 [IB_CM_REJ_NO_EEC] = "no EEC",
38 [IB_CM_REJ_NO_RESOURCES] = "no resources",
39 [IB_CM_REJ_TIMEOUT] = "timeout",
40 [IB_CM_REJ_UNSUPPORTED] = "unsupported",
41 [IB_CM_REJ_INVALID_COMM_ID] = "invalid comm ID",
42 [IB_CM_REJ_INVALID_COMM_INSTANCE] = "invalid comm instance",
43 [IB_CM_REJ_INVALID_SERVICE_ID] = "invalid service ID",
44 [IB_CM_REJ_INVALID_TRANSPORT_TYPE] = "invalid transport type",
45 [IB_CM_REJ_STALE_CONN] = "stale conn",
46 [IB_CM_REJ_RDC_NOT_EXIST] = "RDC not exist",
47 [IB_CM_REJ_INVALID_GID] = "invalid GID",
48 [IB_CM_REJ_INVALID_LID] = "invalid LID",
49 [IB_CM_REJ_INVALID_SL] = "invalid SL",
50 [IB_CM_REJ_INVALID_TRAFFIC_CLASS] = "invalid traffic class",
51 [IB_CM_REJ_INVALID_HOP_LIMIT] = "invalid hop limit",
52 [IB_CM_REJ_INVALID_PACKET_RATE] = "invalid packet rate",
53 [IB_CM_REJ_INVALID_ALT_GID] = "invalid alt GID",
54 [IB_CM_REJ_INVALID_ALT_LID] = "invalid alt LID",
55 [IB_CM_REJ_INVALID_ALT_SL] = "invalid alt SL",
56 [IB_CM_REJ_INVALID_ALT_TRAFFIC_CLASS] = "invalid alt traffic class",
57 [IB_CM_REJ_INVALID_ALT_HOP_LIMIT] = "invalid alt hop limit",
58 [IB_CM_REJ_INVALID_ALT_PACKET_RATE] = "invalid alt packet rate",
59 [IB_CM_REJ_PORT_CM_REDIRECT] = "port CM redirect",
60 [IB_CM_REJ_PORT_REDIRECT] = "port redirect",
61 [IB_CM_REJ_INVALID_MTU] = "invalid MTU",
62 [IB_CM_REJ_INSUFFICIENT_RESP_RESOURCES] = "insufficient resp resources",
63 [IB_CM_REJ_CONSUMER_DEFINED] = "consumer defined",
64 [IB_CM_REJ_INVALID_RNR_RETRY] = "invalid RNR retry",
65 [IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID] = "duplicate local comm ID",
66 [IB_CM_REJ_INVALID_CLASS_VERSION] = "invalid class version",
67 [IB_CM_REJ_INVALID_FLOW_LABEL] = "invalid flow label",
68 [IB_CM_REJ_INVALID_ALT_FLOW_LABEL] = "invalid alt flow label",
69 };
70
71 const char *__attribute_const__ ibcm_reject_msg(int reason)
72 {
73 size_t index = reason;
74
75 if (index < ARRAY_SIZE(ibcm_rej_reason_strs) &&
76 ibcm_rej_reason_strs[index])
77 return ibcm_rej_reason_strs[index];
78 else
79 return "unrecognized reason";
80 }
81 EXPORT_SYMBOL(ibcm_reject_msg);
82
83 struct cm_id_private;
84 static void cm_add_one(struct ib_device *device);
85 static void cm_remove_one(struct ib_device *device, void *client_data);
86 static int cm_send_sidr_rep_locked(struct cm_id_private *cm_id_priv,
87 struct ib_cm_sidr_rep_param *param);
88 static int cm_send_dreq_locked(struct cm_id_private *cm_id_priv,
89 const void *private_data, u8 private_data_len);
90 static int cm_send_drep_locked(struct cm_id_private *cm_id_priv,
91 void *private_data, u8 private_data_len);
92 static int cm_send_rej_locked(struct cm_id_private *cm_id_priv,
93 enum ib_cm_rej_reason reason, void *ari,
94 u8 ari_length, const void *private_data,
95 u8 private_data_len);
96
97 static struct ib_client cm_client = {
98 .name = "cm",
99 .add = cm_add_one,
100 .remove = cm_remove_one
101 };
102
103 static struct ib_cm {
104 spinlock_t lock;
105 struct list_head device_list;
106 rwlock_t device_lock;
107 struct rb_root listen_service_table;
108 u64 listen_service_id;
109 /* struct rb_root peer_service_table; todo: fix peer to peer */
110 struct rb_root remote_qp_table;
111 struct rb_root remote_id_table;
112 struct rb_root remote_sidr_table;
113 struct xarray local_id_table;
114 u32 local_id_next;
115 __be32 random_id_operand;
116 struct list_head timewait_list;
117 struct workqueue_struct *wq;
118 /* Sync on cm change port state */
119 spinlock_t state_lock;
120 } cm;
121
122 /* Counter indexes ordered by attribute ID */
123 enum {
124 CM_REQ_COUNTER,
125 CM_MRA_COUNTER,
126 CM_REJ_COUNTER,
127 CM_REP_COUNTER,
128 CM_RTU_COUNTER,
129 CM_DREQ_COUNTER,
130 CM_DREP_COUNTER,
131 CM_SIDR_REQ_COUNTER,
132 CM_SIDR_REP_COUNTER,
133 CM_LAP_COUNTER,
134 CM_APR_COUNTER,
135 CM_ATTR_COUNT,
136 CM_ATTR_ID_OFFSET = 0x0010,
137 };
138
139 enum {
140 CM_XMIT,
141 CM_XMIT_RETRIES,
142 CM_RECV,
143 CM_RECV_DUPLICATES,
144 CM_COUNTER_GROUPS
145 };
146
147 static char const counter_group_names[CM_COUNTER_GROUPS]
148 [sizeof("cm_rx_duplicates")] = {
149 "cm_tx_msgs", "cm_tx_retries",
150 "cm_rx_msgs", "cm_rx_duplicates"
151 };
152
153 struct cm_counter_group {
154 struct kobject obj;
155 atomic_long_t counter[CM_ATTR_COUNT];
156 };
157
158 struct cm_counter_attribute {
159 struct attribute attr;
160 int index;
161 };
162
163 #define CM_COUNTER_ATTR(_name, _index) \
164 struct cm_counter_attribute cm_##_name##_counter_attr = { \
165 .attr = { .name = __stringify(_name), .mode = 0444 }, \
166 .index = _index \
167 }
168
169 static CM_COUNTER_ATTR(req, CM_REQ_COUNTER);
170 static CM_COUNTER_ATTR(mra, CM_MRA_COUNTER);
171 static CM_COUNTER_ATTR(rej, CM_REJ_COUNTER);
172 static CM_COUNTER_ATTR(rep, CM_REP_COUNTER);
173 static CM_COUNTER_ATTR(rtu, CM_RTU_COUNTER);
174 static CM_COUNTER_ATTR(dreq, CM_DREQ_COUNTER);
175 static CM_COUNTER_ATTR(drep, CM_DREP_COUNTER);
176 static CM_COUNTER_ATTR(sidr_req, CM_SIDR_REQ_COUNTER);
177 static CM_COUNTER_ATTR(sidr_rep, CM_SIDR_REP_COUNTER);
178 static CM_COUNTER_ATTR(lap, CM_LAP_COUNTER);
179 static CM_COUNTER_ATTR(apr, CM_APR_COUNTER);
180
181 static struct attribute *cm_counter_default_attrs[] = {
182 &cm_req_counter_attr.attr,
183 &cm_mra_counter_attr.attr,
184 &cm_rej_counter_attr.attr,
185 &cm_rep_counter_attr.attr,
186 &cm_rtu_counter_attr.attr,
187 &cm_dreq_counter_attr.attr,
188 &cm_drep_counter_attr.attr,
189 &cm_sidr_req_counter_attr.attr,
190 &cm_sidr_rep_counter_attr.attr,
191 &cm_lap_counter_attr.attr,
192 &cm_apr_counter_attr.attr,
193 NULL
194 };
195
196 struct cm_port {
197 struct cm_device *cm_dev;
198 struct ib_mad_agent *mad_agent;
199 struct kobject port_obj;
200 u8 port_num;
201 struct list_head cm_priv_prim_list;
202 struct list_head cm_priv_altr_list;
203 struct cm_counter_group counter_group[CM_COUNTER_GROUPS];
204 };
205
206 struct cm_device {
207 struct list_head list;
208 struct ib_device *ib_device;
209 u8 ack_delay;
210 int going_down;
211 struct cm_port *port[];
212 };
213
214 struct cm_av {
215 struct cm_port *port;
216 union ib_gid dgid;
217 struct rdma_ah_attr ah_attr;
218 u16 pkey_index;
219 u8 timeout;
220 };
221
222 struct cm_work {
223 struct delayed_work work;
224 struct list_head list;
225 struct cm_port *port;
226 struct ib_mad_recv_wc *mad_recv_wc; /* Received MADs */
227 __be32 local_id; /* Established / timewait */
228 __be32 remote_id;
229 struct ib_cm_event cm_event;
230 struct sa_path_rec path[];
231 };
232
233 struct cm_timewait_info {
234 struct cm_work work;
235 struct list_head list;
236 struct rb_node remote_qp_node;
237 struct rb_node remote_id_node;
238 __be64 remote_ca_guid;
239 __be32 remote_qpn;
240 u8 inserted_remote_qp;
241 u8 inserted_remote_id;
242 };
243
244 struct cm_id_private {
245 struct ib_cm_id id;
246
247 struct rb_node service_node;
248 struct rb_node sidr_id_node;
249 spinlock_t lock; /* Do not acquire inside cm.lock */
250 struct completion comp;
251 refcount_t refcount;
252 /* Number of clients sharing this ib_cm_id. Only valid for listeners.
253 * Protected by the cm.lock spinlock. */
254 int listen_sharecount;
255 struct rcu_head rcu;
256
257 struct ib_mad_send_buf *msg;
258 struct cm_timewait_info *timewait_info;
259 /* todo: use alternate port on send failure */
260 struct cm_av av;
261 struct cm_av alt_av;
262
263 void *private_data;
264 __be64 tid;
265 __be32 local_qpn;
266 __be32 remote_qpn;
267 enum ib_qp_type qp_type;
268 __be32 sq_psn;
269 __be32 rq_psn;
270 int timeout_ms;
271 enum ib_mtu path_mtu;
272 __be16 pkey;
273 u8 private_data_len;
274 u8 max_cm_retries;
275 u8 responder_resources;
276 u8 initiator_depth;
277 u8 retry_count;
278 u8 rnr_retry_count;
279 u8 service_timeout;
280 u8 target_ack_delay;
281
282 struct list_head prim_list;
283 struct list_head altr_list;
284 /* Indicates that the send port mad is registered and av is set */
285 int prim_send_port_not_ready;
286 int altr_send_port_not_ready;
287
288 struct list_head work_list;
289 atomic_t work_count;
290 };
291
292 static void cm_work_handler(struct work_struct *work);
293
294 static inline void cm_deref_id(struct cm_id_private *cm_id_priv)
295 {
296 if (refcount_dec_and_test(&cm_id_priv->refcount))
297 complete(&cm_id_priv->comp);
298 }
299
300 static int cm_alloc_msg(struct cm_id_private *cm_id_priv,
301 struct ib_mad_send_buf **msg)
302 {
303 struct ib_mad_agent *mad_agent;
304 struct ib_mad_send_buf *m;
305 struct ib_ah *ah;
306 struct cm_av *av;
307 unsigned long flags, flags2;
308 int ret = 0;
309
310 /* don't let the port to be released till the agent is down */
311 spin_lock_irqsave(&cm.state_lock, flags2);
312 spin_lock_irqsave(&cm.lock, flags);
313 if (!cm_id_priv->prim_send_port_not_ready)
314 av = &cm_id_priv->av;
315 else if (!cm_id_priv->altr_send_port_not_ready &&
316 (cm_id_priv->alt_av.port))
317 av = &cm_id_priv->alt_av;
318 else {
319 pr_info("%s: not valid CM id\n", __func__);
320 ret = -ENODEV;
321 spin_unlock_irqrestore(&cm.lock, flags);
322 goto out;
323 }
324 spin_unlock_irqrestore(&cm.lock, flags);
325 /* Make sure the port haven't released the mad yet */
326 mad_agent = cm_id_priv->av.port->mad_agent;
327 if (!mad_agent) {
328 pr_info("%s: not a valid MAD agent\n", __func__);
329 ret = -ENODEV;
330 goto out;
331 }
332 ah = rdma_create_ah(mad_agent->qp->pd, &av->ah_attr, 0);
333 if (IS_ERR(ah)) {
334 ret = PTR_ERR(ah);
335 goto out;
336 }
337
338 m = ib_create_send_mad(mad_agent, cm_id_priv->id.remote_cm_qpn,
339 av->pkey_index,
340 0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA,
341 GFP_ATOMIC,
342 IB_MGMT_BASE_VERSION);
343 if (IS_ERR(m)) {
344 rdma_destroy_ah(ah, 0);
345 ret = PTR_ERR(m);
346 goto out;
347 }
348
349 /* Timeout set by caller if response is expected. */
350 m->ah = ah;
351 m->retries = cm_id_priv->max_cm_retries;
352
353 refcount_inc(&cm_id_priv->refcount);
354 m->context[0] = cm_id_priv;
355 *msg = m;
356
357 out:
358 spin_unlock_irqrestore(&cm.state_lock, flags2);
359 return ret;
360 }
361
362 static struct ib_mad_send_buf *cm_alloc_response_msg_no_ah(struct cm_port *port,
363 struct ib_mad_recv_wc *mad_recv_wc)
364 {
365 return ib_create_send_mad(port->mad_agent, 1, mad_recv_wc->wc->pkey_index,
366 0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA,
367 GFP_ATOMIC,
368 IB_MGMT_BASE_VERSION);
369 }
370
371 static int cm_create_response_msg_ah(struct cm_port *port,
372 struct ib_mad_recv_wc *mad_recv_wc,
373 struct ib_mad_send_buf *msg)
374 {
375 struct ib_ah *ah;
376
377 ah = ib_create_ah_from_wc(port->mad_agent->qp->pd, mad_recv_wc->wc,
378 mad_recv_wc->recv_buf.grh, port->port_num);
379 if (IS_ERR(ah))
380 return PTR_ERR(ah);
381
382 msg->ah = ah;
383 return 0;
384 }
385
386 static void cm_free_msg(struct ib_mad_send_buf *msg)
387 {
388 if (msg->ah)
389 rdma_destroy_ah(msg->ah, 0);
390 if (msg->context[0])
391 cm_deref_id(msg->context[0]);
392 ib_free_send_mad(msg);
393 }
394
395 static int cm_alloc_response_msg(struct cm_port *port,
396 struct ib_mad_recv_wc *mad_recv_wc,
397 struct ib_mad_send_buf **msg)
398 {
399 struct ib_mad_send_buf *m;
400 int ret;
401
402 m = cm_alloc_response_msg_no_ah(port, mad_recv_wc);
403 if (IS_ERR(m))
404 return PTR_ERR(m);
405
406 ret = cm_create_response_msg_ah(port, mad_recv_wc, m);
407 if (ret) {
408 cm_free_msg(m);
409 return ret;
410 }
411
412 *msg = m;
413 return 0;
414 }
415
416 static void * cm_copy_private_data(const void *private_data,
417 u8 private_data_len)
418 {
419 void *data;
420
421 if (!private_data || !private_data_len)
422 return NULL;
423
424 data = kmemdup(private_data, private_data_len, GFP_KERNEL);
425 if (!data)
426 return ERR_PTR(-ENOMEM);
427
428 return data;
429 }
430
431 static void cm_set_private_data(struct cm_id_private *cm_id_priv,
432 void *private_data, u8 private_data_len)
433 {
434 if (cm_id_priv->private_data && cm_id_priv->private_data_len)
435 kfree(cm_id_priv->private_data);
436
437 cm_id_priv->private_data = private_data;
438 cm_id_priv->private_data_len = private_data_len;
439 }
440
441 static int cm_init_av_for_lap(struct cm_port *port, struct ib_wc *wc,
442 struct ib_grh *grh, struct cm_av *av)
443 {
444 struct rdma_ah_attr new_ah_attr;
445 int ret;
446
447 av->port = port;
448 av->pkey_index = wc->pkey_index;
449
450 /*
451 * av->ah_attr might be initialized based on past wc during incoming
452 * connect request or while sending out connect request. So initialize
453 * a new ah_attr on stack. If initialization fails, old ah_attr is
454 * used for sending any responses. If initialization is successful,
455 * than new ah_attr is used by overwriting old one.
456 */
457 ret = ib_init_ah_attr_from_wc(port->cm_dev->ib_device,
458 port->port_num, wc,
459 grh, &new_ah_attr);
460 if (ret)
461 return ret;
462
463 rdma_move_ah_attr(&av->ah_attr, &new_ah_attr);
464 return 0;
465 }
466
467 static int cm_init_av_for_response(struct cm_port *port, struct ib_wc *wc,
468 struct ib_grh *grh, struct cm_av *av)
469 {
470 av->port = port;
471 av->pkey_index = wc->pkey_index;
472 return ib_init_ah_attr_from_wc(port->cm_dev->ib_device,
473 port->port_num, wc,
474 grh, &av->ah_attr);
475 }
476
477 static int add_cm_id_to_port_list(struct cm_id_private *cm_id_priv,
478 struct cm_av *av,
479 struct cm_port *port)
480 {
481 unsigned long flags;
482 int ret = 0;
483
484 spin_lock_irqsave(&cm.lock, flags);
485
486 if (&cm_id_priv->av == av)
487 list_add_tail(&cm_id_priv->prim_list, &port->cm_priv_prim_list);
488 else if (&cm_id_priv->alt_av == av)
489 list_add_tail(&cm_id_priv->altr_list, &port->cm_priv_altr_list);
490 else
491 ret = -EINVAL;
492
493 spin_unlock_irqrestore(&cm.lock, flags);
494 return ret;
495 }
496
497 static struct cm_port *
498 get_cm_port_from_path(struct sa_path_rec *path, const struct ib_gid_attr *attr)
499 {
500 struct cm_device *cm_dev;
501 struct cm_port *port = NULL;
502 unsigned long flags;
503
504 if (attr) {
505 read_lock_irqsave(&cm.device_lock, flags);
506 list_for_each_entry(cm_dev, &cm.device_list, list) {
507 if (cm_dev->ib_device == attr->device) {
508 port = cm_dev->port[attr->port_num - 1];
509 break;
510 }
511 }
512 read_unlock_irqrestore(&cm.device_lock, flags);
513 } else {
514 /* SGID attribute can be NULL in following
515 * conditions.
516 * (a) Alternative path
517 * (b) IB link layer without GRH
518 * (c) LAP send messages
519 */
520 read_lock_irqsave(&cm.device_lock, flags);
521 list_for_each_entry(cm_dev, &cm.device_list, list) {
522 attr = rdma_find_gid(cm_dev->ib_device,
523 &path->sgid,
524 sa_conv_pathrec_to_gid_type(path),
525 NULL);
526 if (!IS_ERR(attr)) {
527 port = cm_dev->port[attr->port_num - 1];
528 break;
529 }
530 }
531 read_unlock_irqrestore(&cm.device_lock, flags);
532 if (port)
533 rdma_put_gid_attr(attr);
534 }
535 return port;
536 }
537
538 static int cm_init_av_by_path(struct sa_path_rec *path,
539 const struct ib_gid_attr *sgid_attr,
540 struct cm_av *av,
541 struct cm_id_private *cm_id_priv)
542 {
543 struct rdma_ah_attr new_ah_attr;
544 struct cm_device *cm_dev;
545 struct cm_port *port;
546 int ret;
547
548 port = get_cm_port_from_path(path, sgid_attr);
549 if (!port)
550 return -EINVAL;
551 cm_dev = port->cm_dev;
552
553 ret = ib_find_cached_pkey(cm_dev->ib_device, port->port_num,
554 be16_to_cpu(path->pkey), &av->pkey_index);
555 if (ret)
556 return ret;
557
558 av->port = port;
559
560 /*
561 * av->ah_attr might be initialized based on wc or during
562 * request processing time which might have reference to sgid_attr.
563 * So initialize a new ah_attr on stack.
564 * If initialization fails, old ah_attr is used for sending any
565 * responses. If initialization is successful, than new ah_attr
566 * is used by overwriting the old one. So that right ah_attr
567 * can be used to return an error response.
568 */
569 ret = ib_init_ah_attr_from_path(cm_dev->ib_device, port->port_num, path,
570 &new_ah_attr, sgid_attr);
571 if (ret)
572 return ret;
573
574 av->timeout = path->packet_life_time + 1;
575
576 ret = add_cm_id_to_port_list(cm_id_priv, av, port);
577 if (ret) {
578 rdma_destroy_ah_attr(&new_ah_attr);
579 return ret;
580 }
581 rdma_move_ah_attr(&av->ah_attr, &new_ah_attr);
582 return 0;
583 }
584
585 static u32 cm_local_id(__be32 local_id)
586 {
587 return (__force u32) (local_id ^ cm.random_id_operand);
588 }
589
590 static void cm_free_id(__be32 local_id)
591 {
592 xa_erase_irq(&cm.local_id_table, cm_local_id(local_id));
593 }
594
595 static struct cm_id_private *cm_acquire_id(__be32 local_id, __be32 remote_id)
596 {
597 struct cm_id_private *cm_id_priv;
598
599 rcu_read_lock();
600 cm_id_priv = xa_load(&cm.local_id_table, cm_local_id(local_id));
601 if (!cm_id_priv || cm_id_priv->id.remote_id != remote_id ||
602 !refcount_inc_not_zero(&cm_id_priv->refcount))
603 cm_id_priv = NULL;
604 rcu_read_unlock();
605
606 return cm_id_priv;
607 }
608
609 /*
610 * Trivial helpers to strip endian annotation and compare; the
611 * endianness doesn't actually matter since we just need a stable
612 * order for the RB tree.
613 */
614 static int be32_lt(__be32 a, __be32 b)
615 {
616 return (__force u32) a < (__force u32) b;
617 }
618
619 static int be32_gt(__be32 a, __be32 b)
620 {
621 return (__force u32) a > (__force u32) b;
622 }
623
624 static int be64_lt(__be64 a, __be64 b)
625 {
626 return (__force u64) a < (__force u64) b;
627 }
628
629 static int be64_gt(__be64 a, __be64 b)
630 {
631 return (__force u64) a > (__force u64) b;
632 }
633
634 /*
635 * Inserts a new cm_id_priv into the listen_service_table. Returns cm_id_priv
636 * if the new ID was inserted, NULL if it could not be inserted due to a
637 * collision, or the existing cm_id_priv ready for shared usage.
638 */
639 static struct cm_id_private *cm_insert_listen(struct cm_id_private *cm_id_priv,
640 ib_cm_handler shared_handler)
641 {
642 struct rb_node **link = &cm.listen_service_table.rb_node;
643 struct rb_node *parent = NULL;
644 struct cm_id_private *cur_cm_id_priv;
645 __be64 service_id = cm_id_priv->id.service_id;
646 __be64 service_mask = cm_id_priv->id.service_mask;
647 unsigned long flags;
648
649 spin_lock_irqsave(&cm.lock, flags);
650 while (*link) {
651 parent = *link;
652 cur_cm_id_priv = rb_entry(parent, struct cm_id_private,
653 service_node);
654 if ((cur_cm_id_priv->id.service_mask & service_id) ==
655 (service_mask & cur_cm_id_priv->id.service_id) &&
656 (cm_id_priv->id.device == cur_cm_id_priv->id.device)) {
657 /*
658 * Sharing an ib_cm_id with different handlers is not
659 * supported
660 */
661 if (cur_cm_id_priv->id.cm_handler != shared_handler ||
662 cur_cm_id_priv->id.context ||
663 WARN_ON(!cur_cm_id_priv->id.cm_handler)) {
664 spin_unlock_irqrestore(&cm.lock, flags);
665 return NULL;
666 }
667 refcount_inc(&cur_cm_id_priv->refcount);
668 cur_cm_id_priv->listen_sharecount++;
669 spin_unlock_irqrestore(&cm.lock, flags);
670 return cur_cm_id_priv;
671 }
672
673 if (cm_id_priv->id.device < cur_cm_id_priv->id.device)
674 link = &(*link)->rb_left;
675 else if (cm_id_priv->id.device > cur_cm_id_priv->id.device)
676 link = &(*link)->rb_right;
677 else if (be64_lt(service_id, cur_cm_id_priv->id.service_id))
678 link = &(*link)->rb_left;
679 else if (be64_gt(service_id, cur_cm_id_priv->id.service_id))
680 link = &(*link)->rb_right;
681 else
682 link = &(*link)->rb_right;
683 }
684 cm_id_priv->listen_sharecount++;
685 rb_link_node(&cm_id_priv->service_node, parent, link);
686 rb_insert_color(&cm_id_priv->service_node, &cm.listen_service_table);
687 spin_unlock_irqrestore(&cm.lock, flags);
688 return cm_id_priv;
689 }
690
691 static struct cm_id_private * cm_find_listen(struct ib_device *device,
692 __be64 service_id)
693 {
694 struct rb_node *node = cm.listen_service_table.rb_node;
695 struct cm_id_private *cm_id_priv;
696
697 while (node) {
698 cm_id_priv = rb_entry(node, struct cm_id_private, service_node);
699 if ((cm_id_priv->id.service_mask & service_id) ==
700 cm_id_priv->id.service_id &&
701 (cm_id_priv->id.device == device))
702 return cm_id_priv;
703
704 if (device < cm_id_priv->id.device)
705 node = node->rb_left;
706 else if (device > cm_id_priv->id.device)
707 node = node->rb_right;
708 else if (be64_lt(service_id, cm_id_priv->id.service_id))
709 node = node->rb_left;
710 else if (be64_gt(service_id, cm_id_priv->id.service_id))
711 node = node->rb_right;
712 else
713 node = node->rb_right;
714 }
715 return NULL;
716 }
717
718 static struct cm_timewait_info * cm_insert_remote_id(struct cm_timewait_info
719 *timewait_info)
720 {
721 struct rb_node **link = &cm.remote_id_table.rb_node;
722 struct rb_node *parent = NULL;
723 struct cm_timewait_info *cur_timewait_info;
724 __be64 remote_ca_guid = timewait_info->remote_ca_guid;
725 __be32 remote_id = timewait_info->work.remote_id;
726
727 while (*link) {
728 parent = *link;
729 cur_timewait_info = rb_entry(parent, struct cm_timewait_info,
730 remote_id_node);
731 if (be32_lt(remote_id, cur_timewait_info->work.remote_id))
732 link = &(*link)->rb_left;
733 else if (be32_gt(remote_id, cur_timewait_info->work.remote_id))
734 link = &(*link)->rb_right;
735 else if (be64_lt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
736 link = &(*link)->rb_left;
737 else if (be64_gt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
738 link = &(*link)->rb_right;
739 else
740 return cur_timewait_info;
741 }
742 timewait_info->inserted_remote_id = 1;
743 rb_link_node(&timewait_info->remote_id_node, parent, link);
744 rb_insert_color(&timewait_info->remote_id_node, &cm.remote_id_table);
745 return NULL;
746 }
747
748 static struct cm_timewait_info * cm_find_remote_id(__be64 remote_ca_guid,
749 __be32 remote_id)
750 {
751 struct rb_node *node = cm.remote_id_table.rb_node;
752 struct cm_timewait_info *timewait_info;
753
754 while (node) {
755 timewait_info = rb_entry(node, struct cm_timewait_info,
756 remote_id_node);
757 if (be32_lt(remote_id, timewait_info->work.remote_id))
758 node = node->rb_left;
759 else if (be32_gt(remote_id, timewait_info->work.remote_id))
760 node = node->rb_right;
761 else if (be64_lt(remote_ca_guid, timewait_info->remote_ca_guid))
762 node = node->rb_left;
763 else if (be64_gt(remote_ca_guid, timewait_info->remote_ca_guid))
764 node = node->rb_right;
765 else
766 return timewait_info;
767 }
768 return NULL;
769 }
770
771 static struct cm_timewait_info * cm_insert_remote_qpn(struct cm_timewait_info
772 *timewait_info)
773 {
774 struct rb_node **link = &cm.remote_qp_table.rb_node;
775 struct rb_node *parent = NULL;
776 struct cm_timewait_info *cur_timewait_info;
777 __be64 remote_ca_guid = timewait_info->remote_ca_guid;
778 __be32 remote_qpn = timewait_info->remote_qpn;
779
780 while (*link) {
781 parent = *link;
782 cur_timewait_info = rb_entry(parent, struct cm_timewait_info,
783 remote_qp_node);
784 if (be32_lt(remote_qpn, cur_timewait_info->remote_qpn))
785 link = &(*link)->rb_left;
786 else if (be32_gt(remote_qpn, cur_timewait_info->remote_qpn))
787 link = &(*link)->rb_right;
788 else if (be64_lt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
789 link = &(*link)->rb_left;
790 else if (be64_gt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
791 link = &(*link)->rb_right;
792 else
793 return cur_timewait_info;
794 }
795 timewait_info->inserted_remote_qp = 1;
796 rb_link_node(&timewait_info->remote_qp_node, parent, link);
797 rb_insert_color(&timewait_info->remote_qp_node, &cm.remote_qp_table);
798 return NULL;
799 }
800
801 static struct cm_id_private * cm_insert_remote_sidr(struct cm_id_private
802 *cm_id_priv)
803 {
804 struct rb_node **link = &cm.remote_sidr_table.rb_node;
805 struct rb_node *parent = NULL;
806 struct cm_id_private *cur_cm_id_priv;
807 union ib_gid *port_gid = &cm_id_priv->av.dgid;
808 __be32 remote_id = cm_id_priv->id.remote_id;
809
810 while (*link) {
811 parent = *link;
812 cur_cm_id_priv = rb_entry(parent, struct cm_id_private,
813 sidr_id_node);
814 if (be32_lt(remote_id, cur_cm_id_priv->id.remote_id))
815 link = &(*link)->rb_left;
816 else if (be32_gt(remote_id, cur_cm_id_priv->id.remote_id))
817 link = &(*link)->rb_right;
818 else {
819 int cmp;
820 cmp = memcmp(port_gid, &cur_cm_id_priv->av.dgid,
821 sizeof *port_gid);
822 if (cmp < 0)
823 link = &(*link)->rb_left;
824 else if (cmp > 0)
825 link = &(*link)->rb_right;
826 else
827 return cur_cm_id_priv;
828 }
829 }
830 rb_link_node(&cm_id_priv->sidr_id_node, parent, link);
831 rb_insert_color(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table);
832 return NULL;
833 }
834
835 static struct cm_id_private *cm_alloc_id_priv(struct ib_device *device,
836 ib_cm_handler cm_handler,
837 void *context)
838 {
839 struct cm_id_private *cm_id_priv;
840 u32 id;
841 int ret;
842
843 cm_id_priv = kzalloc(sizeof *cm_id_priv, GFP_KERNEL);
844 if (!cm_id_priv)
845 return ERR_PTR(-ENOMEM);
846
847 cm_id_priv->id.state = IB_CM_IDLE;
848 cm_id_priv->id.device = device;
849 cm_id_priv->id.cm_handler = cm_handler;
850 cm_id_priv->id.context = context;
851 cm_id_priv->id.remote_cm_qpn = 1;
852
853 RB_CLEAR_NODE(&cm_id_priv->service_node);
854 RB_CLEAR_NODE(&cm_id_priv->sidr_id_node);
855 spin_lock_init(&cm_id_priv->lock);
856 init_completion(&cm_id_priv->comp);
857 INIT_LIST_HEAD(&cm_id_priv->work_list);
858 INIT_LIST_HEAD(&cm_id_priv->prim_list);
859 INIT_LIST_HEAD(&cm_id_priv->altr_list);
860 atomic_set(&cm_id_priv->work_count, -1);
861 refcount_set(&cm_id_priv->refcount, 1);
862
863 ret = xa_alloc_cyclic_irq(&cm.local_id_table, &id, NULL, xa_limit_32b,
864 &cm.local_id_next, GFP_KERNEL);
865 if (ret < 0)
866 goto error;
867 cm_id_priv->id.local_id = (__force __be32)id ^ cm.random_id_operand;
868
869 return cm_id_priv;
870
871 error:
872 kfree(cm_id_priv);
873 return ERR_PTR(ret);
874 }
875
876 /*
877 * Make the ID visible to the MAD handlers and other threads that use the
878 * xarray.
879 */
880 static void cm_finalize_id(struct cm_id_private *cm_id_priv)
881 {
882 xa_store_irq(&cm.local_id_table, cm_local_id(cm_id_priv->id.local_id),
883 cm_id_priv, GFP_KERNEL);
884 }
885
886 struct ib_cm_id *ib_create_cm_id(struct ib_device *device,
887 ib_cm_handler cm_handler,
888 void *context)
889 {
890 struct cm_id_private *cm_id_priv;
891
892 cm_id_priv = cm_alloc_id_priv(device, cm_handler, context);
893 if (IS_ERR(cm_id_priv))
894 return ERR_CAST(cm_id_priv);
895
896 cm_finalize_id(cm_id_priv);
897 return &cm_id_priv->id;
898 }
899 EXPORT_SYMBOL(ib_create_cm_id);
900
901 static struct cm_work * cm_dequeue_work(struct cm_id_private *cm_id_priv)
902 {
903 struct cm_work *work;
904
905 if (list_empty(&cm_id_priv->work_list))
906 return NULL;
907
908 work = list_entry(cm_id_priv->work_list.next, struct cm_work, list);
909 list_del(&work->list);
910 return work;
911 }
912
913 static void cm_free_work(struct cm_work *work)
914 {
915 if (work->mad_recv_wc)
916 ib_free_recv_mad(work->mad_recv_wc);
917 kfree(work);
918 }
919
920 static inline int cm_convert_to_ms(int iba_time)
921 {
922 /* approximate conversion to ms from 4.096us x 2^iba_time */
923 return 1 << max(iba_time - 8, 0);
924 }
925
926 /*
927 * calculate: 4.096x2^ack_timeout = 4.096x2^ack_delay + 2x4.096x2^life_time
928 * Because of how ack_timeout is stored, adding one doubles the timeout.
929 * To avoid large timeouts, select the max(ack_delay, life_time + 1), and
930 * increment it (round up) only if the other is within 50%.
931 */
932 static u8 cm_ack_timeout(u8 ca_ack_delay, u8 packet_life_time)
933 {
934 int ack_timeout = packet_life_time + 1;
935
936 if (ack_timeout >= ca_ack_delay)
937 ack_timeout += (ca_ack_delay >= (ack_timeout - 1));
938 else
939 ack_timeout = ca_ack_delay +
940 (ack_timeout >= (ca_ack_delay - 1));
941
942 return min(31, ack_timeout);
943 }
944
945 static void cm_cleanup_timewait(struct cm_timewait_info *timewait_info)
946 {
947 if (timewait_info->inserted_remote_id) {
948 rb_erase(&timewait_info->remote_id_node, &cm.remote_id_table);
949 timewait_info->inserted_remote_id = 0;
950 }
951
952 if (timewait_info->inserted_remote_qp) {
953 rb_erase(&timewait_info->remote_qp_node, &cm.remote_qp_table);
954 timewait_info->inserted_remote_qp = 0;
955 }
956 }
957
958 static struct cm_timewait_info * cm_create_timewait_info(__be32 local_id)
959 {
960 struct cm_timewait_info *timewait_info;
961
962 timewait_info = kzalloc(sizeof *timewait_info, GFP_KERNEL);
963 if (!timewait_info)
964 return ERR_PTR(-ENOMEM);
965
966 timewait_info->work.local_id = local_id;
967 INIT_DELAYED_WORK(&timewait_info->work.work, cm_work_handler);
968 timewait_info->work.cm_event.event = IB_CM_TIMEWAIT_EXIT;
969 return timewait_info;
970 }
971
972 static void cm_enter_timewait(struct cm_id_private *cm_id_priv)
973 {
974 int wait_time;
975 unsigned long flags;
976 struct cm_device *cm_dev;
977
978 lockdep_assert_held(&cm_id_priv->lock);
979
980 cm_dev = ib_get_client_data(cm_id_priv->id.device, &cm_client);
981 if (!cm_dev)
982 return;
983
984 spin_lock_irqsave(&cm.lock, flags);
985 cm_cleanup_timewait(cm_id_priv->timewait_info);
986 list_add_tail(&cm_id_priv->timewait_info->list, &cm.timewait_list);
987 spin_unlock_irqrestore(&cm.lock, flags);
988
989 /*
990 * The cm_id could be destroyed by the user before we exit timewait.
991 * To protect against this, we search for the cm_id after exiting
992 * timewait before notifying the user that we've exited timewait.
993 */
994 cm_id_priv->id.state = IB_CM_TIMEWAIT;
995 wait_time = cm_convert_to_ms(cm_id_priv->av.timeout);
996
997 /* Check if the device started its remove_one */
998 spin_lock_irqsave(&cm.lock, flags);
999 if (!cm_dev->going_down)
1000 queue_delayed_work(cm.wq, &cm_id_priv->timewait_info->work.work,
1001 msecs_to_jiffies(wait_time));
1002 spin_unlock_irqrestore(&cm.lock, flags);
1003
1004 cm_id_priv->timewait_info = NULL;
1005 }
1006
1007 static void cm_reset_to_idle(struct cm_id_private *cm_id_priv)
1008 {
1009 unsigned long flags;
1010
1011 lockdep_assert_held(&cm_id_priv->lock);
1012
1013 cm_id_priv->id.state = IB_CM_IDLE;
1014 if (cm_id_priv->timewait_info) {
1015 spin_lock_irqsave(&cm.lock, flags);
1016 cm_cleanup_timewait(cm_id_priv->timewait_info);
1017 spin_unlock_irqrestore(&cm.lock, flags);
1018 kfree(cm_id_priv->timewait_info);
1019 cm_id_priv->timewait_info = NULL;
1020 }
1021 }
1022
1023 static void cm_destroy_id(struct ib_cm_id *cm_id, int err)
1024 {
1025 struct cm_id_private *cm_id_priv;
1026 struct cm_work *work;
1027
1028 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1029 spin_lock_irq(&cm_id_priv->lock);
1030 retest:
1031 switch (cm_id->state) {
1032 case IB_CM_LISTEN:
1033 spin_lock(&cm.lock);
1034 if (--cm_id_priv->listen_sharecount > 0) {
1035 /* The id is still shared. */
1036 WARN_ON(refcount_read(&cm_id_priv->refcount) == 1);
1037 spin_unlock(&cm.lock);
1038 spin_unlock_irq(&cm_id_priv->lock);
1039 cm_deref_id(cm_id_priv);
1040 return;
1041 }
1042 cm_id->state = IB_CM_IDLE;
1043 rb_erase(&cm_id_priv->service_node, &cm.listen_service_table);
1044 RB_CLEAR_NODE(&cm_id_priv->service_node);
1045 spin_unlock(&cm.lock);
1046 break;
1047 case IB_CM_SIDR_REQ_SENT:
1048 cm_id->state = IB_CM_IDLE;
1049 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
1050 break;
1051 case IB_CM_SIDR_REQ_RCVD:
1052 cm_send_sidr_rep_locked(cm_id_priv,
1053 &(struct ib_cm_sidr_rep_param){
1054 .status = IB_SIDR_REJECT });
1055 /* cm_send_sidr_rep_locked will not move to IDLE if it fails */
1056 cm_id->state = IB_CM_IDLE;
1057 break;
1058 case IB_CM_REQ_SENT:
1059 case IB_CM_MRA_REQ_RCVD:
1060 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
1061 cm_send_rej_locked(cm_id_priv, IB_CM_REJ_TIMEOUT,
1062 &cm_id_priv->id.device->node_guid,
1063 sizeof(cm_id_priv->id.device->node_guid),
1064 NULL, 0);
1065 break;
1066 case IB_CM_REQ_RCVD:
1067 if (err == -ENOMEM) {
1068 /* Do not reject to allow future retries. */
1069 cm_reset_to_idle(cm_id_priv);
1070 } else {
1071 cm_send_rej_locked(cm_id_priv,
1072 IB_CM_REJ_CONSUMER_DEFINED, NULL, 0,
1073 NULL, 0);
1074 }
1075 break;
1076 case IB_CM_REP_SENT:
1077 case IB_CM_MRA_REP_RCVD:
1078 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
1079 cm_send_rej_locked(cm_id_priv, IB_CM_REJ_CONSUMER_DEFINED, NULL,
1080 0, NULL, 0);
1081 goto retest;
1082 case IB_CM_MRA_REQ_SENT:
1083 case IB_CM_REP_RCVD:
1084 case IB_CM_MRA_REP_SENT:
1085 cm_send_rej_locked(cm_id_priv, IB_CM_REJ_CONSUMER_DEFINED, NULL,
1086 0, NULL, 0);
1087 break;
1088 case IB_CM_ESTABLISHED:
1089 if (cm_id_priv->qp_type == IB_QPT_XRC_TGT) {
1090 cm_id->state = IB_CM_IDLE;
1091 break;
1092 }
1093 cm_send_dreq_locked(cm_id_priv, NULL, 0);
1094 goto retest;
1095 case IB_CM_DREQ_SENT:
1096 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
1097 cm_enter_timewait(cm_id_priv);
1098 goto retest;
1099 case IB_CM_DREQ_RCVD:
1100 cm_send_drep_locked(cm_id_priv, NULL, 0);
1101 WARN_ON(cm_id->state != IB_CM_TIMEWAIT);
1102 goto retest;
1103 case IB_CM_TIMEWAIT:
1104 /*
1105 * The cm_acquire_id in cm_timewait_handler will stop working
1106 * once we do cm_free_id() below, so just move to idle here for
1107 * consistency.
1108 */
1109 cm_id->state = IB_CM_IDLE;
1110 break;
1111 case IB_CM_IDLE:
1112 break;
1113 }
1114 WARN_ON(cm_id->state != IB_CM_IDLE);
1115
1116 spin_lock(&cm.lock);
1117 /* Required for cleanup paths related cm_req_handler() */
1118 if (cm_id_priv->timewait_info) {
1119 cm_cleanup_timewait(cm_id_priv->timewait_info);
1120 kfree(cm_id_priv->timewait_info);
1121 cm_id_priv->timewait_info = NULL;
1122 }
1123 if (!list_empty(&cm_id_priv->altr_list) &&
1124 (!cm_id_priv->altr_send_port_not_ready))
1125 list_del(&cm_id_priv->altr_list);
1126 if (!list_empty(&cm_id_priv->prim_list) &&
1127 (!cm_id_priv->prim_send_port_not_ready))
1128 list_del(&cm_id_priv->prim_list);
1129 WARN_ON(cm_id_priv->listen_sharecount);
1130 WARN_ON(!RB_EMPTY_NODE(&cm_id_priv->service_node));
1131 if (!RB_EMPTY_NODE(&cm_id_priv->sidr_id_node))
1132 rb_erase(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table);
1133 spin_unlock(&cm.lock);
1134 spin_unlock_irq(&cm_id_priv->lock);
1135
1136 cm_free_id(cm_id->local_id);
1137 cm_deref_id(cm_id_priv);
1138 wait_for_completion(&cm_id_priv->comp);
1139 while ((work = cm_dequeue_work(cm_id_priv)) != NULL)
1140 cm_free_work(work);
1141
1142 rdma_destroy_ah_attr(&cm_id_priv->av.ah_attr);
1143 rdma_destroy_ah_attr(&cm_id_priv->alt_av.ah_attr);
1144 kfree(cm_id_priv->private_data);
1145 kfree_rcu(cm_id_priv, rcu);
1146 }
1147
1148 void ib_destroy_cm_id(struct ib_cm_id *cm_id)
1149 {
1150 cm_destroy_id(cm_id, 0);
1151 }
1152 EXPORT_SYMBOL(ib_destroy_cm_id);
1153
1154 static int cm_init_listen(struct cm_id_private *cm_id_priv, __be64 service_id,
1155 __be64 service_mask)
1156 {
1157 service_mask = service_mask ? service_mask : ~cpu_to_be64(0);
1158 service_id &= service_mask;
1159 if ((service_id & IB_SERVICE_ID_AGN_MASK) == IB_CM_ASSIGN_SERVICE_ID &&
1160 (service_id != IB_CM_ASSIGN_SERVICE_ID))
1161 return -EINVAL;
1162
1163 if (service_id == IB_CM_ASSIGN_SERVICE_ID) {
1164 cm_id_priv->id.service_id = cpu_to_be64(cm.listen_service_id++);
1165 cm_id_priv->id.service_mask = ~cpu_to_be64(0);
1166 } else {
1167 cm_id_priv->id.service_id = service_id;
1168 cm_id_priv->id.service_mask = service_mask;
1169 }
1170 return 0;
1171 }
1172
1173 /**
1174 * ib_cm_listen - Initiates listening on the specified service ID for
1175 * connection and service ID resolution requests.
1176 * @cm_id: Connection identifier associated with the listen request.
1177 * @service_id: Service identifier matched against incoming connection
1178 * and service ID resolution requests. The service ID should be specified
1179 * network-byte order. If set to IB_CM_ASSIGN_SERVICE_ID, the CM will
1180 * assign a service ID to the caller.
1181 * @service_mask: Mask applied to service ID used to listen across a
1182 * range of service IDs. If set to 0, the service ID is matched
1183 * exactly. This parameter is ignored if %service_id is set to
1184 * IB_CM_ASSIGN_SERVICE_ID.
1185 */
1186 int ib_cm_listen(struct ib_cm_id *cm_id, __be64 service_id, __be64 service_mask)
1187 {
1188 struct cm_id_private *cm_id_priv =
1189 container_of(cm_id, struct cm_id_private, id);
1190 unsigned long flags;
1191 int ret;
1192
1193 spin_lock_irqsave(&cm_id_priv->lock, flags);
1194 if (cm_id_priv->id.state != IB_CM_IDLE) {
1195 ret = -EINVAL;
1196 goto out;
1197 }
1198
1199 ret = cm_init_listen(cm_id_priv, service_id, service_mask);
1200 if (ret)
1201 goto out;
1202
1203 if (!cm_insert_listen(cm_id_priv, NULL)) {
1204 ret = -EBUSY;
1205 goto out;
1206 }
1207
1208 cm_id_priv->id.state = IB_CM_LISTEN;
1209 ret = 0;
1210
1211 out:
1212 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1213 return ret;
1214 }
1215 EXPORT_SYMBOL(ib_cm_listen);
1216
1217 /**
1218 * Create a new listening ib_cm_id and listen on the given service ID.
1219 *
1220 * If there's an existing ID listening on that same device and service ID,
1221 * return it.
1222 *
1223 * @device: Device associated with the cm_id. All related communication will
1224 * be associated with the specified device.
1225 * @cm_handler: Callback invoked to notify the user of CM events.
1226 * @service_id: Service identifier matched against incoming connection
1227 * and service ID resolution requests. The service ID should be specified
1228 * network-byte order. If set to IB_CM_ASSIGN_SERVICE_ID, the CM will
1229 * assign a service ID to the caller.
1230 *
1231 * Callers should call ib_destroy_cm_id when done with the listener ID.
1232 */
1233 struct ib_cm_id *ib_cm_insert_listen(struct ib_device *device,
1234 ib_cm_handler cm_handler,
1235 __be64 service_id)
1236 {
1237 struct cm_id_private *listen_id_priv;
1238 struct cm_id_private *cm_id_priv;
1239 int err = 0;
1240
1241 /* Create an ID in advance, since the creation may sleep */
1242 cm_id_priv = cm_alloc_id_priv(device, cm_handler, NULL);
1243 if (IS_ERR(cm_id_priv))
1244 return ERR_CAST(cm_id_priv);
1245
1246 err = cm_init_listen(cm_id_priv, service_id, 0);
1247 if (err)
1248 return ERR_PTR(err);
1249
1250 spin_lock_irq(&cm_id_priv->lock);
1251 listen_id_priv = cm_insert_listen(cm_id_priv, cm_handler);
1252 if (listen_id_priv != cm_id_priv) {
1253 spin_unlock_irq(&cm_id_priv->lock);
1254 ib_destroy_cm_id(&cm_id_priv->id);
1255 if (!listen_id_priv)
1256 return ERR_PTR(-EINVAL);
1257 return &listen_id_priv->id;
1258 }
1259 cm_id_priv->id.state = IB_CM_LISTEN;
1260 spin_unlock_irq(&cm_id_priv->lock);
1261
1262 /*
1263 * A listen ID does not need to be in the xarray since it does not
1264 * receive mads, is not placed in the remote_id or remote_qpn rbtree,
1265 * and does not enter timewait.
1266 */
1267
1268 return &cm_id_priv->id;
1269 }
1270 EXPORT_SYMBOL(ib_cm_insert_listen);
1271
1272 static __be64 cm_form_tid(struct cm_id_private *cm_id_priv)
1273 {
1274 u64 hi_tid, low_tid;
1275
1276 hi_tid = ((u64) cm_id_priv->av.port->mad_agent->hi_tid) << 32;
1277 low_tid = (u64)cm_id_priv->id.local_id;
1278 return cpu_to_be64(hi_tid | low_tid);
1279 }
1280
1281 static void cm_format_mad_hdr(struct ib_mad_hdr *hdr,
1282 __be16 attr_id, __be64 tid)
1283 {
1284 hdr->base_version = IB_MGMT_BASE_VERSION;
1285 hdr->mgmt_class = IB_MGMT_CLASS_CM;
1286 hdr->class_version = IB_CM_CLASS_VERSION;
1287 hdr->method = IB_MGMT_METHOD_SEND;
1288 hdr->attr_id = attr_id;
1289 hdr->tid = tid;
1290 }
1291
1292 static void cm_format_req(struct cm_req_msg *req_msg,
1293 struct cm_id_private *cm_id_priv,
1294 struct ib_cm_req_param *param)
1295 {
1296 struct sa_path_rec *pri_path = param->primary_path;
1297 struct sa_path_rec *alt_path = param->alternate_path;
1298 bool pri_ext = false;
1299
1300 if (pri_path->rec_type == SA_PATH_REC_TYPE_OPA)
1301 pri_ext = opa_is_extended_lid(pri_path->opa.dlid,
1302 pri_path->opa.slid);
1303
1304 cm_format_mad_hdr(&req_msg->hdr, CM_REQ_ATTR_ID,
1305 cm_form_tid(cm_id_priv));
1306
1307 IBA_SET(CM_REQ_LOCAL_COMM_ID, req_msg,
1308 be32_to_cpu(cm_id_priv->id.local_id));
1309 IBA_SET(CM_REQ_SERVICE_ID, req_msg, be64_to_cpu(param->service_id));
1310 IBA_SET(CM_REQ_LOCAL_CA_GUID, req_msg,
1311 be64_to_cpu(cm_id_priv->id.device->node_guid));
1312 IBA_SET(CM_REQ_LOCAL_QPN, req_msg, param->qp_num);
1313 IBA_SET(CM_REQ_INITIATOR_DEPTH, req_msg, param->initiator_depth);
1314 IBA_SET(CM_REQ_REMOTE_CM_RESPONSE_TIMEOUT, req_msg,
1315 param->remote_cm_response_timeout);
1316 cm_req_set_qp_type(req_msg, param->qp_type);
1317 IBA_SET(CM_REQ_END_TO_END_FLOW_CONTROL, req_msg, param->flow_control);
1318 IBA_SET(CM_REQ_STARTING_PSN, req_msg, param->starting_psn);
1319 IBA_SET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg,
1320 param->local_cm_response_timeout);
1321 IBA_SET(CM_REQ_PARTITION_KEY, req_msg,
1322 be16_to_cpu(param->primary_path->pkey));
1323 IBA_SET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg,
1324 param->primary_path->mtu);
1325 IBA_SET(CM_REQ_MAX_CM_RETRIES, req_msg, param->max_cm_retries);
1326
1327 if (param->qp_type != IB_QPT_XRC_INI) {
1328 IBA_SET(CM_REQ_RESPONDER_RESOURCES, req_msg,
1329 param->responder_resources);
1330 IBA_SET(CM_REQ_RETRY_COUNT, req_msg, param->retry_count);
1331 IBA_SET(CM_REQ_RNR_RETRY_COUNT, req_msg,
1332 param->rnr_retry_count);
1333 IBA_SET(CM_REQ_SRQ, req_msg, param->srq);
1334 }
1335
1336 *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg) =
1337 pri_path->sgid;
1338 *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg) =
1339 pri_path->dgid;
1340 if (pri_ext) {
1341 IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg)
1342 ->global.interface_id =
1343 OPA_MAKE_ID(be32_to_cpu(pri_path->opa.slid));
1344 IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg)
1345 ->global.interface_id =
1346 OPA_MAKE_ID(be32_to_cpu(pri_path->opa.dlid));
1347 }
1348 if (pri_path->hop_limit <= 1) {
1349 IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,
1350 be16_to_cpu(pri_ext ? 0 :
1351 htons(ntohl(sa_path_get_slid(
1352 pri_path)))));
1353 IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg,
1354 be16_to_cpu(pri_ext ? 0 :
1355 htons(ntohl(sa_path_get_dlid(
1356 pri_path)))));
1357 } else {
1358 /* Work-around until there's a way to obtain remote LID info */
1359 IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,
1360 be16_to_cpu(IB_LID_PERMISSIVE));
1361 IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg,
1362 be16_to_cpu(IB_LID_PERMISSIVE));
1363 }
1364 IBA_SET(CM_REQ_PRIMARY_FLOW_LABEL, req_msg,
1365 be32_to_cpu(pri_path->flow_label));
1366 IBA_SET(CM_REQ_PRIMARY_PACKET_RATE, req_msg, pri_path->rate);
1367 IBA_SET(CM_REQ_PRIMARY_TRAFFIC_CLASS, req_msg, pri_path->traffic_class);
1368 IBA_SET(CM_REQ_PRIMARY_HOP_LIMIT, req_msg, pri_path->hop_limit);
1369 IBA_SET(CM_REQ_PRIMARY_SL, req_msg, pri_path->sl);
1370 IBA_SET(CM_REQ_PRIMARY_SUBNET_LOCAL, req_msg,
1371 (pri_path->hop_limit <= 1));
1372 IBA_SET(CM_REQ_PRIMARY_LOCAL_ACK_TIMEOUT, req_msg,
1373 cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay,
1374 pri_path->packet_life_time));
1375
1376 if (alt_path) {
1377 bool alt_ext = false;
1378
1379 if (alt_path->rec_type == SA_PATH_REC_TYPE_OPA)
1380 alt_ext = opa_is_extended_lid(alt_path->opa.dlid,
1381 alt_path->opa.slid);
1382
1383 *IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg) =
1384 alt_path->sgid;
1385 *IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg) =
1386 alt_path->dgid;
1387 if (alt_ext) {
1388 IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID,
1389 req_msg)
1390 ->global.interface_id =
1391 OPA_MAKE_ID(be32_to_cpu(alt_path->opa.slid));
1392 IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_REMOTE_PORT_GID,
1393 req_msg)
1394 ->global.interface_id =
1395 OPA_MAKE_ID(be32_to_cpu(alt_path->opa.dlid));
1396 }
1397 if (alt_path->hop_limit <= 1) {
1398 IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg,
1399 be16_to_cpu(
1400 alt_ext ? 0 :
1401 htons(ntohl(sa_path_get_slid(
1402 alt_path)))));
1403 IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg,
1404 be16_to_cpu(
1405 alt_ext ? 0 :
1406 htons(ntohl(sa_path_get_dlid(
1407 alt_path)))));
1408 } else {
1409 IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg,
1410 be16_to_cpu(IB_LID_PERMISSIVE));
1411 IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg,
1412 be16_to_cpu(IB_LID_PERMISSIVE));
1413 }
1414 IBA_SET(CM_REQ_ALTERNATE_FLOW_LABEL, req_msg,
1415 be32_to_cpu(alt_path->flow_label));
1416 IBA_SET(CM_REQ_ALTERNATE_PACKET_RATE, req_msg, alt_path->rate);
1417 IBA_SET(CM_REQ_ALTERNATE_TRAFFIC_CLASS, req_msg,
1418 alt_path->traffic_class);
1419 IBA_SET(CM_REQ_ALTERNATE_HOP_LIMIT, req_msg,
1420 alt_path->hop_limit);
1421 IBA_SET(CM_REQ_ALTERNATE_SL, req_msg, alt_path->sl);
1422 IBA_SET(CM_REQ_ALTERNATE_SUBNET_LOCAL, req_msg,
1423 (alt_path->hop_limit <= 1));
1424 IBA_SET(CM_REQ_ALTERNATE_LOCAL_ACK_TIMEOUT, req_msg,
1425 cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay,
1426 alt_path->packet_life_time));
1427 }
1428
1429 if (param->private_data && param->private_data_len)
1430 IBA_SET_MEM(CM_REQ_PRIVATE_DATA, req_msg, param->private_data,
1431 param->private_data_len);
1432 }
1433
1434 static int cm_validate_req_param(struct ib_cm_req_param *param)
1435 {
1436 if (!param->primary_path)
1437 return -EINVAL;
1438
1439 if (param->qp_type != IB_QPT_RC && param->qp_type != IB_QPT_UC &&
1440 param->qp_type != IB_QPT_XRC_INI)
1441 return -EINVAL;
1442
1443 if (param->private_data &&
1444 param->private_data_len > IB_CM_REQ_PRIVATE_DATA_SIZE)
1445 return -EINVAL;
1446
1447 if (param->alternate_path &&
1448 (param->alternate_path->pkey != param->primary_path->pkey ||
1449 param->alternate_path->mtu != param->primary_path->mtu))
1450 return -EINVAL;
1451
1452 return 0;
1453 }
1454
1455 int ib_send_cm_req(struct ib_cm_id *cm_id,
1456 struct ib_cm_req_param *param)
1457 {
1458 struct cm_id_private *cm_id_priv;
1459 struct cm_req_msg *req_msg;
1460 unsigned long flags;
1461 int ret;
1462
1463 ret = cm_validate_req_param(param);
1464 if (ret)
1465 return ret;
1466
1467 /* Verify that we're not in timewait. */
1468 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1469 spin_lock_irqsave(&cm_id_priv->lock, flags);
1470 if (cm_id->state != IB_CM_IDLE || WARN_ON(cm_id_priv->timewait_info)) {
1471 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1472 ret = -EINVAL;
1473 goto out;
1474 }
1475 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1476
1477 cm_id_priv->timewait_info = cm_create_timewait_info(cm_id_priv->
1478 id.local_id);
1479 if (IS_ERR(cm_id_priv->timewait_info)) {
1480 ret = PTR_ERR(cm_id_priv->timewait_info);
1481 goto out;
1482 }
1483
1484 ret = cm_init_av_by_path(param->primary_path,
1485 param->ppath_sgid_attr, &cm_id_priv->av,
1486 cm_id_priv);
1487 if (ret)
1488 goto out;
1489 if (param->alternate_path) {
1490 ret = cm_init_av_by_path(param->alternate_path, NULL,
1491 &cm_id_priv->alt_av, cm_id_priv);
1492 if (ret)
1493 goto out;
1494 }
1495 cm_id->service_id = param->service_id;
1496 cm_id->service_mask = ~cpu_to_be64(0);
1497 cm_id_priv->timeout_ms = cm_convert_to_ms(
1498 param->primary_path->packet_life_time) * 2 +
1499 cm_convert_to_ms(
1500 param->remote_cm_response_timeout);
1501 cm_id_priv->max_cm_retries = param->max_cm_retries;
1502 cm_id_priv->initiator_depth = param->initiator_depth;
1503 cm_id_priv->responder_resources = param->responder_resources;
1504 cm_id_priv->retry_count = param->retry_count;
1505 cm_id_priv->path_mtu = param->primary_path->mtu;
1506 cm_id_priv->pkey = param->primary_path->pkey;
1507 cm_id_priv->qp_type = param->qp_type;
1508
1509 ret = cm_alloc_msg(cm_id_priv, &cm_id_priv->msg);
1510 if (ret)
1511 goto out;
1512
1513 req_msg = (struct cm_req_msg *) cm_id_priv->msg->mad;
1514 cm_format_req(req_msg, cm_id_priv, param);
1515 cm_id_priv->tid = req_msg->hdr.tid;
1516 cm_id_priv->msg->timeout_ms = cm_id_priv->timeout_ms;
1517 cm_id_priv->msg->context[1] = (void *) (unsigned long) IB_CM_REQ_SENT;
1518
1519 cm_id_priv->local_qpn = cpu_to_be32(IBA_GET(CM_REQ_LOCAL_QPN, req_msg));
1520 cm_id_priv->rq_psn = cpu_to_be32(IBA_GET(CM_REQ_STARTING_PSN, req_msg));
1521
1522 spin_lock_irqsave(&cm_id_priv->lock, flags);
1523 ret = ib_post_send_mad(cm_id_priv->msg, NULL);
1524 if (ret) {
1525 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1526 goto error2;
1527 }
1528 BUG_ON(cm_id->state != IB_CM_IDLE);
1529 cm_id->state = IB_CM_REQ_SENT;
1530 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1531 return 0;
1532
1533 error2: cm_free_msg(cm_id_priv->msg);
1534 out: return ret;
1535 }
1536 EXPORT_SYMBOL(ib_send_cm_req);
1537
1538 static int cm_issue_rej(struct cm_port *port,
1539 struct ib_mad_recv_wc *mad_recv_wc,
1540 enum ib_cm_rej_reason reason,
1541 enum cm_msg_response msg_rejected,
1542 void *ari, u8 ari_length)
1543 {
1544 struct ib_mad_send_buf *msg = NULL;
1545 struct cm_rej_msg *rej_msg, *rcv_msg;
1546 int ret;
1547
1548 ret = cm_alloc_response_msg(port, mad_recv_wc, &msg);
1549 if (ret)
1550 return ret;
1551
1552 /* We just need common CM header information. Cast to any message. */
1553 rcv_msg = (struct cm_rej_msg *) mad_recv_wc->recv_buf.mad;
1554 rej_msg = (struct cm_rej_msg *) msg->mad;
1555
1556 cm_format_mad_hdr(&rej_msg->hdr, CM_REJ_ATTR_ID, rcv_msg->hdr.tid);
1557 IBA_SET(CM_REJ_REMOTE_COMM_ID, rej_msg,
1558 IBA_GET(CM_REJ_LOCAL_COMM_ID, rcv_msg));
1559 IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1560 IBA_GET(CM_REJ_REMOTE_COMM_ID, rcv_msg));
1561 IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, msg_rejected);
1562 IBA_SET(CM_REJ_REASON, rej_msg, reason);
1563
1564 if (ari && ari_length) {
1565 IBA_SET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg, ari_length);
1566 IBA_SET_MEM(CM_REJ_ARI, rej_msg, ari, ari_length);
1567 }
1568
1569 ret = ib_post_send_mad(msg, NULL);
1570 if (ret)
1571 cm_free_msg(msg);
1572
1573 return ret;
1574 }
1575
1576 static bool cm_req_has_alt_path(struct cm_req_msg *req_msg)
1577 {
1578 return ((cpu_to_be16(
1579 IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg))) ||
1580 (ib_is_opa_gid(IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID,
1581 req_msg))));
1582 }
1583
1584 static void cm_path_set_rec_type(struct ib_device *ib_device, u8 port_num,
1585 struct sa_path_rec *path, union ib_gid *gid)
1586 {
1587 if (ib_is_opa_gid(gid) && rdma_cap_opa_ah(ib_device, port_num))
1588 path->rec_type = SA_PATH_REC_TYPE_OPA;
1589 else
1590 path->rec_type = SA_PATH_REC_TYPE_IB;
1591 }
1592
1593 static void cm_format_path_lid_from_req(struct cm_req_msg *req_msg,
1594 struct sa_path_rec *primary_path,
1595 struct sa_path_rec *alt_path)
1596 {
1597 u32 lid;
1598
1599 if (primary_path->rec_type != SA_PATH_REC_TYPE_OPA) {
1600 sa_path_set_dlid(primary_path,
1601 IBA_GET(CM_REQ_PRIMARY_LOCAL_PORT_LID,
1602 req_msg));
1603 sa_path_set_slid(primary_path,
1604 IBA_GET(CM_REQ_PRIMARY_REMOTE_PORT_LID,
1605 req_msg));
1606 } else {
1607 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1608 CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg));
1609 sa_path_set_dlid(primary_path, lid);
1610
1611 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1612 CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg));
1613 sa_path_set_slid(primary_path, lid);
1614 }
1615
1616 if (!cm_req_has_alt_path(req_msg))
1617 return;
1618
1619 if (alt_path->rec_type != SA_PATH_REC_TYPE_OPA) {
1620 sa_path_set_dlid(alt_path,
1621 IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID,
1622 req_msg));
1623 sa_path_set_slid(alt_path,
1624 IBA_GET(CM_REQ_ALTERNATE_REMOTE_PORT_LID,
1625 req_msg));
1626 } else {
1627 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1628 CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg));
1629 sa_path_set_dlid(alt_path, lid);
1630
1631 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1632 CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg));
1633 sa_path_set_slid(alt_path, lid);
1634 }
1635 }
1636
1637 static void cm_format_paths_from_req(struct cm_req_msg *req_msg,
1638 struct sa_path_rec *primary_path,
1639 struct sa_path_rec *alt_path)
1640 {
1641 primary_path->dgid =
1642 *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg);
1643 primary_path->sgid =
1644 *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg);
1645 primary_path->flow_label =
1646 cpu_to_be32(IBA_GET(CM_REQ_PRIMARY_FLOW_LABEL, req_msg));
1647 primary_path->hop_limit = IBA_GET(CM_REQ_PRIMARY_HOP_LIMIT, req_msg);
1648 primary_path->traffic_class =
1649 IBA_GET(CM_REQ_PRIMARY_TRAFFIC_CLASS, req_msg);
1650 primary_path->reversible = 1;
1651 primary_path->pkey =
1652 cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg));
1653 primary_path->sl = IBA_GET(CM_REQ_PRIMARY_SL, req_msg);
1654 primary_path->mtu_selector = IB_SA_EQ;
1655 primary_path->mtu = IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg);
1656 primary_path->rate_selector = IB_SA_EQ;
1657 primary_path->rate = IBA_GET(CM_REQ_PRIMARY_PACKET_RATE, req_msg);
1658 primary_path->packet_life_time_selector = IB_SA_EQ;
1659 primary_path->packet_life_time =
1660 IBA_GET(CM_REQ_PRIMARY_LOCAL_ACK_TIMEOUT, req_msg);
1661 primary_path->packet_life_time -= (primary_path->packet_life_time > 0);
1662 primary_path->service_id =
1663 cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg));
1664 if (sa_path_is_roce(primary_path))
1665 primary_path->roce.route_resolved = false;
1666
1667 if (cm_req_has_alt_path(req_msg)) {
1668 alt_path->dgid = *IBA_GET_MEM_PTR(
1669 CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg);
1670 alt_path->sgid = *IBA_GET_MEM_PTR(
1671 CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg);
1672 alt_path->flow_label = cpu_to_be32(
1673 IBA_GET(CM_REQ_ALTERNATE_FLOW_LABEL, req_msg));
1674 alt_path->hop_limit =
1675 IBA_GET(CM_REQ_ALTERNATE_HOP_LIMIT, req_msg);
1676 alt_path->traffic_class =
1677 IBA_GET(CM_REQ_ALTERNATE_TRAFFIC_CLASS, req_msg);
1678 alt_path->reversible = 1;
1679 alt_path->pkey =
1680 cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg));
1681 alt_path->sl = IBA_GET(CM_REQ_ALTERNATE_SL, req_msg);
1682 alt_path->mtu_selector = IB_SA_EQ;
1683 alt_path->mtu =
1684 IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg);
1685 alt_path->rate_selector = IB_SA_EQ;
1686 alt_path->rate = IBA_GET(CM_REQ_ALTERNATE_PACKET_RATE, req_msg);
1687 alt_path->packet_life_time_selector = IB_SA_EQ;
1688 alt_path->packet_life_time =
1689 IBA_GET(CM_REQ_ALTERNATE_LOCAL_ACK_TIMEOUT, req_msg);
1690 alt_path->packet_life_time -= (alt_path->packet_life_time > 0);
1691 alt_path->service_id =
1692 cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg));
1693
1694 if (sa_path_is_roce(alt_path))
1695 alt_path->roce.route_resolved = false;
1696 }
1697 cm_format_path_lid_from_req(req_msg, primary_path, alt_path);
1698 }
1699
1700 static u16 cm_get_bth_pkey(struct cm_work *work)
1701 {
1702 struct ib_device *ib_dev = work->port->cm_dev->ib_device;
1703 u8 port_num = work->port->port_num;
1704 u16 pkey_index = work->mad_recv_wc->wc->pkey_index;
1705 u16 pkey;
1706 int ret;
1707
1708 ret = ib_get_cached_pkey(ib_dev, port_num, pkey_index, &pkey);
1709 if (ret) {
1710 dev_warn_ratelimited(&ib_dev->dev, "ib_cm: Couldn't retrieve pkey for incoming request (port %d, pkey index %d). %d\n",
1711 port_num, pkey_index, ret);
1712 return 0;
1713 }
1714
1715 return pkey;
1716 }
1717
1718 /**
1719 * Convert OPA SGID to IB SGID
1720 * ULPs (such as IPoIB) do not understand OPA GIDs and will
1721 * reject them as the local_gid will not match the sgid. Therefore,
1722 * change the pathrec's SGID to an IB SGID.
1723 *
1724 * @work: Work completion
1725 * @path: Path record
1726 */
1727 static void cm_opa_to_ib_sgid(struct cm_work *work,
1728 struct sa_path_rec *path)
1729 {
1730 struct ib_device *dev = work->port->cm_dev->ib_device;
1731 u8 port_num = work->port->port_num;
1732
1733 if (rdma_cap_opa_ah(dev, port_num) &&
1734 (ib_is_opa_gid(&path->sgid))) {
1735 union ib_gid sgid;
1736
1737 if (rdma_query_gid(dev, port_num, 0, &sgid)) {
1738 dev_warn(&dev->dev,
1739 "Error updating sgid in CM request\n");
1740 return;
1741 }
1742
1743 path->sgid = sgid;
1744 }
1745 }
1746
1747 static void cm_format_req_event(struct cm_work *work,
1748 struct cm_id_private *cm_id_priv,
1749 struct ib_cm_id *listen_id)
1750 {
1751 struct cm_req_msg *req_msg;
1752 struct ib_cm_req_event_param *param;
1753
1754 req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
1755 param = &work->cm_event.param.req_rcvd;
1756 param->listen_id = listen_id;
1757 param->bth_pkey = cm_get_bth_pkey(work);
1758 param->port = cm_id_priv->av.port->port_num;
1759 param->primary_path = &work->path[0];
1760 cm_opa_to_ib_sgid(work, param->primary_path);
1761 if (cm_req_has_alt_path(req_msg)) {
1762 param->alternate_path = &work->path[1];
1763 cm_opa_to_ib_sgid(work, param->alternate_path);
1764 } else {
1765 param->alternate_path = NULL;
1766 }
1767 param->remote_ca_guid =
1768 cpu_to_be64(IBA_GET(CM_REQ_LOCAL_CA_GUID, req_msg));
1769 param->remote_qkey = IBA_GET(CM_REQ_LOCAL_Q_KEY, req_msg);
1770 param->remote_qpn = IBA_GET(CM_REQ_LOCAL_QPN, req_msg);
1771 param->qp_type = cm_req_get_qp_type(req_msg);
1772 param->starting_psn = IBA_GET(CM_REQ_STARTING_PSN, req_msg);
1773 param->responder_resources = IBA_GET(CM_REQ_INITIATOR_DEPTH, req_msg);
1774 param->initiator_depth = IBA_GET(CM_REQ_RESPONDER_RESOURCES, req_msg);
1775 param->local_cm_response_timeout =
1776 IBA_GET(CM_REQ_REMOTE_CM_RESPONSE_TIMEOUT, req_msg);
1777 param->flow_control = IBA_GET(CM_REQ_END_TO_END_FLOW_CONTROL, req_msg);
1778 param->remote_cm_response_timeout =
1779 IBA_GET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg);
1780 param->retry_count = IBA_GET(CM_REQ_RETRY_COUNT, req_msg);
1781 param->rnr_retry_count = IBA_GET(CM_REQ_RNR_RETRY_COUNT, req_msg);
1782 param->srq = IBA_GET(CM_REQ_SRQ, req_msg);
1783 param->ppath_sgid_attr = cm_id_priv->av.ah_attr.grh.sgid_attr;
1784 work->cm_event.private_data =
1785 IBA_GET_MEM_PTR(CM_REQ_PRIVATE_DATA, req_msg);
1786 }
1787
1788 static void cm_process_work(struct cm_id_private *cm_id_priv,
1789 struct cm_work *work)
1790 {
1791 int ret;
1792
1793 /* We will typically only have the current event to report. */
1794 ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &work->cm_event);
1795 cm_free_work(work);
1796
1797 while (!ret && !atomic_add_negative(-1, &cm_id_priv->work_count)) {
1798 spin_lock_irq(&cm_id_priv->lock);
1799 work = cm_dequeue_work(cm_id_priv);
1800 spin_unlock_irq(&cm_id_priv->lock);
1801 if (!work)
1802 return;
1803
1804 ret = cm_id_priv->id.cm_handler(&cm_id_priv->id,
1805 &work->cm_event);
1806 cm_free_work(work);
1807 }
1808 cm_deref_id(cm_id_priv);
1809 if (ret)
1810 cm_destroy_id(&cm_id_priv->id, ret);
1811 }
1812
1813 static void cm_format_mra(struct cm_mra_msg *mra_msg,
1814 struct cm_id_private *cm_id_priv,
1815 enum cm_msg_response msg_mraed, u8 service_timeout,
1816 const void *private_data, u8 private_data_len)
1817 {
1818 cm_format_mad_hdr(&mra_msg->hdr, CM_MRA_ATTR_ID, cm_id_priv->tid);
1819 IBA_SET(CM_MRA_MESSAGE_MRAED, mra_msg, msg_mraed);
1820 IBA_SET(CM_MRA_LOCAL_COMM_ID, mra_msg,
1821 be32_to_cpu(cm_id_priv->id.local_id));
1822 IBA_SET(CM_MRA_REMOTE_COMM_ID, mra_msg,
1823 be32_to_cpu(cm_id_priv->id.remote_id));
1824 IBA_SET(CM_MRA_SERVICE_TIMEOUT, mra_msg, service_timeout);
1825
1826 if (private_data && private_data_len)
1827 IBA_SET_MEM(CM_MRA_PRIVATE_DATA, mra_msg, private_data,
1828 private_data_len);
1829 }
1830
1831 static void cm_format_rej(struct cm_rej_msg *rej_msg,
1832 struct cm_id_private *cm_id_priv,
1833 enum ib_cm_rej_reason reason, void *ari,
1834 u8 ari_length, const void *private_data,
1835 u8 private_data_len, enum ib_cm_state state)
1836 {
1837 lockdep_assert_held(&cm_id_priv->lock);
1838
1839 cm_format_mad_hdr(&rej_msg->hdr, CM_REJ_ATTR_ID, cm_id_priv->tid);
1840 IBA_SET(CM_REJ_REMOTE_COMM_ID, rej_msg,
1841 be32_to_cpu(cm_id_priv->id.remote_id));
1842
1843 switch (state) {
1844 case IB_CM_REQ_RCVD:
1845 IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg, be32_to_cpu(0));
1846 IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REQ);
1847 break;
1848 case IB_CM_MRA_REQ_SENT:
1849 IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1850 be32_to_cpu(cm_id_priv->id.local_id));
1851 IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REQ);
1852 break;
1853 case IB_CM_REP_RCVD:
1854 case IB_CM_MRA_REP_SENT:
1855 IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1856 be32_to_cpu(cm_id_priv->id.local_id));
1857 IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REP);
1858 break;
1859 default:
1860 IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1861 be32_to_cpu(cm_id_priv->id.local_id));
1862 IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg,
1863 CM_MSG_RESPONSE_OTHER);
1864 break;
1865 }
1866
1867 IBA_SET(CM_REJ_REASON, rej_msg, reason);
1868 if (ari && ari_length) {
1869 IBA_SET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg, ari_length);
1870 IBA_SET_MEM(CM_REJ_ARI, rej_msg, ari, ari_length);
1871 }
1872
1873 if (private_data && private_data_len)
1874 IBA_SET_MEM(CM_REJ_PRIVATE_DATA, rej_msg, private_data,
1875 private_data_len);
1876 }
1877
1878 static void cm_dup_req_handler(struct cm_work *work,
1879 struct cm_id_private *cm_id_priv)
1880 {
1881 struct ib_mad_send_buf *msg = NULL;
1882 int ret;
1883
1884 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
1885 counter[CM_REQ_COUNTER]);
1886
1887 /* Quick state check to discard duplicate REQs. */
1888 spin_lock_irq(&cm_id_priv->lock);
1889 if (cm_id_priv->id.state == IB_CM_REQ_RCVD) {
1890 spin_unlock_irq(&cm_id_priv->lock);
1891 return;
1892 }
1893 spin_unlock_irq(&cm_id_priv->lock);
1894
1895 ret = cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg);
1896 if (ret)
1897 return;
1898
1899 spin_lock_irq(&cm_id_priv->lock);
1900 switch (cm_id_priv->id.state) {
1901 case IB_CM_MRA_REQ_SENT:
1902 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
1903 CM_MSG_RESPONSE_REQ, cm_id_priv->service_timeout,
1904 cm_id_priv->private_data,
1905 cm_id_priv->private_data_len);
1906 break;
1907 case IB_CM_TIMEWAIT:
1908 cm_format_rej((struct cm_rej_msg *)msg->mad, cm_id_priv,
1909 IB_CM_REJ_STALE_CONN, NULL, 0, NULL, 0,
1910 IB_CM_TIMEWAIT);
1911 break;
1912 default:
1913 goto unlock;
1914 }
1915 spin_unlock_irq(&cm_id_priv->lock);
1916
1917 ret = ib_post_send_mad(msg, NULL);
1918 if (ret)
1919 goto free;
1920 return;
1921
1922 unlock: spin_unlock_irq(&cm_id_priv->lock);
1923 free: cm_free_msg(msg);
1924 }
1925
1926 static struct cm_id_private * cm_match_req(struct cm_work *work,
1927 struct cm_id_private *cm_id_priv)
1928 {
1929 struct cm_id_private *listen_cm_id_priv, *cur_cm_id_priv;
1930 struct cm_timewait_info *timewait_info;
1931 struct cm_req_msg *req_msg;
1932 struct ib_cm_id *cm_id;
1933
1934 req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
1935
1936 /* Check for possible duplicate REQ. */
1937 spin_lock_irq(&cm.lock);
1938 timewait_info = cm_insert_remote_id(cm_id_priv->timewait_info);
1939 if (timewait_info) {
1940 cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
1941 timewait_info->work.remote_id);
1942 spin_unlock_irq(&cm.lock);
1943 if (cur_cm_id_priv) {
1944 cm_dup_req_handler(work, cur_cm_id_priv);
1945 cm_deref_id(cur_cm_id_priv);
1946 }
1947 return NULL;
1948 }
1949
1950 /* Check for stale connections. */
1951 timewait_info = cm_insert_remote_qpn(cm_id_priv->timewait_info);
1952 if (timewait_info) {
1953 cm_cleanup_timewait(cm_id_priv->timewait_info);
1954 cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
1955 timewait_info->work.remote_id);
1956
1957 spin_unlock_irq(&cm.lock);
1958 cm_issue_rej(work->port, work->mad_recv_wc,
1959 IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REQ,
1960 NULL, 0);
1961 if (cur_cm_id_priv) {
1962 cm_id = &cur_cm_id_priv->id;
1963 ib_send_cm_dreq(cm_id, NULL, 0);
1964 cm_deref_id(cur_cm_id_priv);
1965 }
1966 return NULL;
1967 }
1968
1969 /* Find matching listen request. */
1970 listen_cm_id_priv = cm_find_listen(
1971 cm_id_priv->id.device,
1972 cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg)));
1973 if (!listen_cm_id_priv) {
1974 cm_cleanup_timewait(cm_id_priv->timewait_info);
1975 spin_unlock_irq(&cm.lock);
1976 cm_issue_rej(work->port, work->mad_recv_wc,
1977 IB_CM_REJ_INVALID_SERVICE_ID, CM_MSG_RESPONSE_REQ,
1978 NULL, 0);
1979 return NULL;
1980 }
1981 refcount_inc(&listen_cm_id_priv->refcount);
1982 spin_unlock_irq(&cm.lock);
1983 return listen_cm_id_priv;
1984 }
1985
1986 /*
1987 * Work-around for inter-subnet connections. If the LIDs are permissive,
1988 * we need to override the LID/SL data in the REQ with the LID information
1989 * in the work completion.
1990 */
1991 static void cm_process_routed_req(struct cm_req_msg *req_msg, struct ib_wc *wc)
1992 {
1993 if (!IBA_GET(CM_REQ_PRIMARY_SUBNET_LOCAL, req_msg)) {
1994 if (cpu_to_be16(IBA_GET(CM_REQ_PRIMARY_LOCAL_PORT_LID,
1995 req_msg)) == IB_LID_PERMISSIVE) {
1996 IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,
1997 be16_to_cpu(ib_lid_be16(wc->slid)));
1998 IBA_SET(CM_REQ_PRIMARY_SL, req_msg, wc->sl);
1999 }
2000
2001 if (cpu_to_be16(IBA_GET(CM_REQ_PRIMARY_REMOTE_PORT_LID,
2002 req_msg)) == IB_LID_PERMISSIVE)
2003 IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg,
2004 wc->dlid_path_bits);
2005 }
2006
2007 if (!IBA_GET(CM_REQ_ALTERNATE_SUBNET_LOCAL, req_msg)) {
2008 if (cpu_to_be16(IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID,
2009 req_msg)) == IB_LID_PERMISSIVE) {
2010 IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg,
2011 be16_to_cpu(ib_lid_be16(wc->slid)));
2012 IBA_SET(CM_REQ_ALTERNATE_SL, req_msg, wc->sl);
2013 }
2014
2015 if (cpu_to_be16(IBA_GET(CM_REQ_ALTERNATE_REMOTE_PORT_LID,
2016 req_msg)) == IB_LID_PERMISSIVE)
2017 IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg,
2018 wc->dlid_path_bits);
2019 }
2020 }
2021
2022 static int cm_req_handler(struct cm_work *work)
2023 {
2024 struct cm_id_private *cm_id_priv, *listen_cm_id_priv;
2025 struct cm_req_msg *req_msg;
2026 const struct ib_global_route *grh;
2027 const struct ib_gid_attr *gid_attr;
2028 int ret;
2029
2030 req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
2031
2032 cm_id_priv =
2033 cm_alloc_id_priv(work->port->cm_dev->ib_device, NULL, NULL);
2034 if (IS_ERR(cm_id_priv))
2035 return PTR_ERR(cm_id_priv);
2036
2037 cm_id_priv->id.remote_id =
2038 cpu_to_be32(IBA_GET(CM_REQ_LOCAL_COMM_ID, req_msg));
2039 cm_id_priv->id.service_id =
2040 cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg));
2041 cm_id_priv->id.service_mask = ~cpu_to_be64(0);
2042 cm_id_priv->tid = req_msg->hdr.tid;
2043 cm_id_priv->timeout_ms = cm_convert_to_ms(
2044 IBA_GET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg));
2045 cm_id_priv->max_cm_retries = IBA_GET(CM_REQ_MAX_CM_RETRIES, req_msg);
2046 cm_id_priv->remote_qpn =
2047 cpu_to_be32(IBA_GET(CM_REQ_LOCAL_QPN, req_msg));
2048 cm_id_priv->initiator_depth =
2049 IBA_GET(CM_REQ_RESPONDER_RESOURCES, req_msg);
2050 cm_id_priv->responder_resources =
2051 IBA_GET(CM_REQ_INITIATOR_DEPTH, req_msg);
2052 cm_id_priv->path_mtu = IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg);
2053 cm_id_priv->pkey = cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg));
2054 cm_id_priv->sq_psn = cpu_to_be32(IBA_GET(CM_REQ_STARTING_PSN, req_msg));
2055 cm_id_priv->retry_count = IBA_GET(CM_REQ_RETRY_COUNT, req_msg);
2056 cm_id_priv->rnr_retry_count = IBA_GET(CM_REQ_RNR_RETRY_COUNT, req_msg);
2057 cm_id_priv->qp_type = cm_req_get_qp_type(req_msg);
2058
2059 ret = cm_init_av_for_response(work->port, work->mad_recv_wc->wc,
2060 work->mad_recv_wc->recv_buf.grh,
2061 &cm_id_priv->av);
2062 if (ret)
2063 goto destroy;
2064 cm_id_priv->timewait_info = cm_create_timewait_info(cm_id_priv->
2065 id.local_id);
2066 if (IS_ERR(cm_id_priv->timewait_info)) {
2067 ret = PTR_ERR(cm_id_priv->timewait_info);
2068 goto destroy;
2069 }
2070 cm_id_priv->timewait_info->work.remote_id = cm_id_priv->id.remote_id;
2071 cm_id_priv->timewait_info->remote_ca_guid =
2072 cpu_to_be64(IBA_GET(CM_REQ_LOCAL_CA_GUID, req_msg));
2073 cm_id_priv->timewait_info->remote_qpn = cm_id_priv->remote_qpn;
2074
2075 /*
2076 * Note that the ID pointer is not in the xarray at this point,
2077 * so this set is only visible to the local thread.
2078 */
2079 cm_id_priv->id.state = IB_CM_REQ_RCVD;
2080
2081 listen_cm_id_priv = cm_match_req(work, cm_id_priv);
2082 if (!listen_cm_id_priv) {
2083 pr_debug("%s: local_id %d, no listen_cm_id_priv\n", __func__,
2084 be32_to_cpu(cm_id_priv->id.local_id));
2085 cm_id_priv->id.state = IB_CM_IDLE;
2086 ret = -EINVAL;
2087 goto destroy;
2088 }
2089
2090 cm_process_routed_req(req_msg, work->mad_recv_wc->wc);
2091
2092 memset(&work->path[0], 0, sizeof(work->path[0]));
2093 if (cm_req_has_alt_path(req_msg))
2094 memset(&work->path[1], 0, sizeof(work->path[1]));
2095 grh = rdma_ah_read_grh(&cm_id_priv->av.ah_attr);
2096 gid_attr = grh->sgid_attr;
2097
2098 if (gid_attr &&
2099 rdma_protocol_roce(work->port->cm_dev->ib_device,
2100 work->port->port_num)) {
2101 work->path[0].rec_type =
2102 sa_conv_gid_to_pathrec_type(gid_attr->gid_type);
2103 } else {
2104 cm_path_set_rec_type(
2105 work->port->cm_dev->ib_device, work->port->port_num,
2106 &work->path[0],
2107 IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID,
2108 req_msg));
2109 }
2110 if (cm_req_has_alt_path(req_msg))
2111 work->path[1].rec_type = work->path[0].rec_type;
2112 cm_format_paths_from_req(req_msg, &work->path[0],
2113 &work->path[1]);
2114 if (cm_id_priv->av.ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE)
2115 sa_path_set_dmac(&work->path[0],
2116 cm_id_priv->av.ah_attr.roce.dmac);
2117 work->path[0].hop_limit = grh->hop_limit;
2118 ret = cm_init_av_by_path(&work->path[0], gid_attr, &cm_id_priv->av,
2119 cm_id_priv);
2120 if (ret) {
2121 int err;
2122
2123 err = rdma_query_gid(work->port->cm_dev->ib_device,
2124 work->port->port_num, 0,
2125 &work->path[0].sgid);
2126 if (err)
2127 ib_send_cm_rej(&cm_id_priv->id, IB_CM_REJ_INVALID_GID,
2128 NULL, 0, NULL, 0);
2129 else
2130 ib_send_cm_rej(&cm_id_priv->id, IB_CM_REJ_INVALID_GID,
2131 &work->path[0].sgid,
2132 sizeof(work->path[0].sgid),
2133 NULL, 0);
2134 goto rejected;
2135 }
2136 if (cm_req_has_alt_path(req_msg)) {
2137 ret = cm_init_av_by_path(&work->path[1], NULL,
2138 &cm_id_priv->alt_av, cm_id_priv);
2139 if (ret) {
2140 ib_send_cm_rej(&cm_id_priv->id,
2141 IB_CM_REJ_INVALID_ALT_GID,
2142 &work->path[0].sgid,
2143 sizeof(work->path[0].sgid), NULL, 0);
2144 goto rejected;
2145 }
2146 }
2147
2148 cm_id_priv->id.cm_handler = listen_cm_id_priv->id.cm_handler;
2149 cm_id_priv->id.context = listen_cm_id_priv->id.context;
2150 cm_format_req_event(work, cm_id_priv, &listen_cm_id_priv->id);
2151
2152 /* Now MAD handlers can see the new ID */
2153 spin_lock_irq(&cm_id_priv->lock);
2154 cm_finalize_id(cm_id_priv);
2155
2156 /* Refcount belongs to the event, pairs with cm_process_work() */
2157 refcount_inc(&cm_id_priv->refcount);
2158 atomic_inc(&cm_id_priv->work_count);
2159 spin_unlock_irq(&cm_id_priv->lock);
2160 cm_process_work(cm_id_priv, work);
2161 /*
2162 * Since this ID was just created and was not made visible to other MAD
2163 * handlers until the cm_finalize_id() above we know that the
2164 * cm_process_work() will deliver the event and the listen_cm_id
2165 * embedded in the event can be derefed here.
2166 */
2167 cm_deref_id(listen_cm_id_priv);
2168 return 0;
2169
2170 rejected:
2171 cm_deref_id(listen_cm_id_priv);
2172 destroy:
2173 ib_destroy_cm_id(&cm_id_priv->id);
2174 return ret;
2175 }
2176
2177 static void cm_format_rep(struct cm_rep_msg *rep_msg,
2178 struct cm_id_private *cm_id_priv,
2179 struct ib_cm_rep_param *param)
2180 {
2181 cm_format_mad_hdr(&rep_msg->hdr, CM_REP_ATTR_ID, cm_id_priv->tid);
2182 IBA_SET(CM_REP_LOCAL_COMM_ID, rep_msg,
2183 be32_to_cpu(cm_id_priv->id.local_id));
2184 IBA_SET(CM_REP_REMOTE_COMM_ID, rep_msg,
2185 be32_to_cpu(cm_id_priv->id.remote_id));
2186 IBA_SET(CM_REP_STARTING_PSN, rep_msg, param->starting_psn);
2187 IBA_SET(CM_REP_RESPONDER_RESOURCES, rep_msg,
2188 param->responder_resources);
2189 IBA_SET(CM_REP_TARGET_ACK_DELAY, rep_msg,
2190 cm_id_priv->av.port->cm_dev->ack_delay);
2191 IBA_SET(CM_REP_FAILOVER_ACCEPTED, rep_msg, param->failover_accepted);
2192 IBA_SET(CM_REP_RNR_RETRY_COUNT, rep_msg, param->rnr_retry_count);
2193 IBA_SET(CM_REP_LOCAL_CA_GUID, rep_msg,
2194 be64_to_cpu(cm_id_priv->id.device->node_guid));
2195
2196 if (cm_id_priv->qp_type != IB_QPT_XRC_TGT) {
2197 IBA_SET(CM_REP_INITIATOR_DEPTH, rep_msg,
2198 param->initiator_depth);
2199 IBA_SET(CM_REP_END_TO_END_FLOW_CONTROL, rep_msg,
2200 param->flow_control);
2201 IBA_SET(CM_REP_SRQ, rep_msg, param->srq);
2202 IBA_SET(CM_REP_LOCAL_QPN, rep_msg, param->qp_num);
2203 } else {
2204 IBA_SET(CM_REP_SRQ, rep_msg, 1);
2205 IBA_SET(CM_REP_LOCAL_EE_CONTEXT_NUMBER, rep_msg, param->qp_num);
2206 }
2207
2208 if (param->private_data && param->private_data_len)
2209 IBA_SET_MEM(CM_REP_PRIVATE_DATA, rep_msg, param->private_data,
2210 param->private_data_len);
2211 }
2212
2213 int ib_send_cm_rep(struct ib_cm_id *cm_id,
2214 struct ib_cm_rep_param *param)
2215 {
2216 struct cm_id_private *cm_id_priv;
2217 struct ib_mad_send_buf *msg;
2218 struct cm_rep_msg *rep_msg;
2219 unsigned long flags;
2220 int ret;
2221
2222 if (param->private_data &&
2223 param->private_data_len > IB_CM_REP_PRIVATE_DATA_SIZE)
2224 return -EINVAL;
2225
2226 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2227 spin_lock_irqsave(&cm_id_priv->lock, flags);
2228 if (cm_id->state != IB_CM_REQ_RCVD &&
2229 cm_id->state != IB_CM_MRA_REQ_SENT) {
2230 pr_debug("%s: local_comm_id %d, cm_id->state: %d\n", __func__,
2231 be32_to_cpu(cm_id_priv->id.local_id), cm_id->state);
2232 ret = -EINVAL;
2233 goto out;
2234 }
2235
2236 ret = cm_alloc_msg(cm_id_priv, &msg);
2237 if (ret)
2238 goto out;
2239
2240 rep_msg = (struct cm_rep_msg *) msg->mad;
2241 cm_format_rep(rep_msg, cm_id_priv, param);
2242 msg->timeout_ms = cm_id_priv->timeout_ms;
2243 msg->context[1] = (void *) (unsigned long) IB_CM_REP_SENT;
2244
2245 ret = ib_post_send_mad(msg, NULL);
2246 if (ret) {
2247 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2248 cm_free_msg(msg);
2249 return ret;
2250 }
2251
2252 cm_id->state = IB_CM_REP_SENT;
2253 cm_id_priv->msg = msg;
2254 cm_id_priv->initiator_depth = param->initiator_depth;
2255 cm_id_priv->responder_resources = param->responder_resources;
2256 cm_id_priv->rq_psn = cpu_to_be32(IBA_GET(CM_REP_STARTING_PSN, rep_msg));
2257 WARN_ONCE(param->qp_num & 0xFF000000,
2258 "IBTA declares QPN to be 24 bits, but it is 0x%X\n",
2259 param->qp_num);
2260 cm_id_priv->local_qpn = cpu_to_be32(param->qp_num & 0xFFFFFF);
2261
2262 out: spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2263 return ret;
2264 }
2265 EXPORT_SYMBOL(ib_send_cm_rep);
2266
2267 static void cm_format_rtu(struct cm_rtu_msg *rtu_msg,
2268 struct cm_id_private *cm_id_priv,
2269 const void *private_data,
2270 u8 private_data_len)
2271 {
2272 cm_format_mad_hdr(&rtu_msg->hdr, CM_RTU_ATTR_ID, cm_id_priv->tid);
2273 IBA_SET(CM_RTU_LOCAL_COMM_ID, rtu_msg,
2274 be32_to_cpu(cm_id_priv->id.local_id));
2275 IBA_SET(CM_RTU_REMOTE_COMM_ID, rtu_msg,
2276 be32_to_cpu(cm_id_priv->id.remote_id));
2277
2278 if (private_data && private_data_len)
2279 IBA_SET_MEM(CM_RTU_PRIVATE_DATA, rtu_msg, private_data,
2280 private_data_len);
2281 }
2282
2283 int ib_send_cm_rtu(struct ib_cm_id *cm_id,
2284 const void *private_data,
2285 u8 private_data_len)
2286 {
2287 struct cm_id_private *cm_id_priv;
2288 struct ib_mad_send_buf *msg;
2289 unsigned long flags;
2290 void *data;
2291 int ret;
2292
2293 if (private_data && private_data_len > IB_CM_RTU_PRIVATE_DATA_SIZE)
2294 return -EINVAL;
2295
2296 data = cm_copy_private_data(private_data, private_data_len);
2297 if (IS_ERR(data))
2298 return PTR_ERR(data);
2299
2300 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2301 spin_lock_irqsave(&cm_id_priv->lock, flags);
2302 if (cm_id->state != IB_CM_REP_RCVD &&
2303 cm_id->state != IB_CM_MRA_REP_SENT) {
2304 pr_debug("%s: local_id %d, cm_id->state %d\n", __func__,
2305 be32_to_cpu(cm_id->local_id), cm_id->state);
2306 ret = -EINVAL;
2307 goto error;
2308 }
2309
2310 ret = cm_alloc_msg(cm_id_priv, &msg);
2311 if (ret)
2312 goto error;
2313
2314 cm_format_rtu((struct cm_rtu_msg *) msg->mad, cm_id_priv,
2315 private_data, private_data_len);
2316
2317 ret = ib_post_send_mad(msg, NULL);
2318 if (ret) {
2319 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2320 cm_free_msg(msg);
2321 kfree(data);
2322 return ret;
2323 }
2324
2325 cm_id->state = IB_CM_ESTABLISHED;
2326 cm_set_private_data(cm_id_priv, data, private_data_len);
2327 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2328 return 0;
2329
2330 error: spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2331 kfree(data);
2332 return ret;
2333 }
2334 EXPORT_SYMBOL(ib_send_cm_rtu);
2335
2336 static void cm_format_rep_event(struct cm_work *work, enum ib_qp_type qp_type)
2337 {
2338 struct cm_rep_msg *rep_msg;
2339 struct ib_cm_rep_event_param *param;
2340
2341 rep_msg = (struct cm_rep_msg *)work->mad_recv_wc->recv_buf.mad;
2342 param = &work->cm_event.param.rep_rcvd;
2343 param->remote_ca_guid =
2344 cpu_to_be64(IBA_GET(CM_REP_LOCAL_CA_GUID, rep_msg));
2345 param->remote_qkey = IBA_GET(CM_REP_LOCAL_Q_KEY, rep_msg);
2346 param->remote_qpn = be32_to_cpu(cm_rep_get_qpn(rep_msg, qp_type));
2347 param->starting_psn = IBA_GET(CM_REP_STARTING_PSN, rep_msg);
2348 param->responder_resources = IBA_GET(CM_REP_INITIATOR_DEPTH, rep_msg);
2349 param->initiator_depth = IBA_GET(CM_REP_RESPONDER_RESOURCES, rep_msg);
2350 param->target_ack_delay = IBA_GET(CM_REP_TARGET_ACK_DELAY, rep_msg);
2351 param->failover_accepted = IBA_GET(CM_REP_FAILOVER_ACCEPTED, rep_msg);
2352 param->flow_control = IBA_GET(CM_REP_END_TO_END_FLOW_CONTROL, rep_msg);
2353 param->rnr_retry_count = IBA_GET(CM_REP_RNR_RETRY_COUNT, rep_msg);
2354 param->srq = IBA_GET(CM_REP_SRQ, rep_msg);
2355 work->cm_event.private_data =
2356 IBA_GET_MEM_PTR(CM_REP_PRIVATE_DATA, rep_msg);
2357 }
2358
2359 static void cm_dup_rep_handler(struct cm_work *work)
2360 {
2361 struct cm_id_private *cm_id_priv;
2362 struct cm_rep_msg *rep_msg;
2363 struct ib_mad_send_buf *msg = NULL;
2364 int ret;
2365
2366 rep_msg = (struct cm_rep_msg *) work->mad_recv_wc->recv_buf.mad;
2367 cm_id_priv = cm_acquire_id(
2368 cpu_to_be32(IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)),
2369 cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg)));
2370 if (!cm_id_priv)
2371 return;
2372
2373 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
2374 counter[CM_REP_COUNTER]);
2375 ret = cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg);
2376 if (ret)
2377 goto deref;
2378
2379 spin_lock_irq(&cm_id_priv->lock);
2380 if (cm_id_priv->id.state == IB_CM_ESTABLISHED)
2381 cm_format_rtu((struct cm_rtu_msg *) msg->mad, cm_id_priv,
2382 cm_id_priv->private_data,
2383 cm_id_priv->private_data_len);
2384 else if (cm_id_priv->id.state == IB_CM_MRA_REP_SENT)
2385 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
2386 CM_MSG_RESPONSE_REP, cm_id_priv->service_timeout,
2387 cm_id_priv->private_data,
2388 cm_id_priv->private_data_len);
2389 else
2390 goto unlock;
2391 spin_unlock_irq(&cm_id_priv->lock);
2392
2393 ret = ib_post_send_mad(msg, NULL);
2394 if (ret)
2395 goto free;
2396 goto deref;
2397
2398 unlock: spin_unlock_irq(&cm_id_priv->lock);
2399 free: cm_free_msg(msg);
2400 deref: cm_deref_id(cm_id_priv);
2401 }
2402
2403 static int cm_rep_handler(struct cm_work *work)
2404 {
2405 struct cm_id_private *cm_id_priv;
2406 struct cm_rep_msg *rep_msg;
2407 int ret;
2408 struct cm_id_private *cur_cm_id_priv;
2409 struct ib_cm_id *cm_id;
2410 struct cm_timewait_info *timewait_info;
2411
2412 rep_msg = (struct cm_rep_msg *)work->mad_recv_wc->recv_buf.mad;
2413 cm_id_priv = cm_acquire_id(
2414 cpu_to_be32(IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)), 0);
2415 if (!cm_id_priv) {
2416 cm_dup_rep_handler(work);
2417 pr_debug("%s: remote_comm_id %d, no cm_id_priv\n", __func__,
2418 IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg));
2419 return -EINVAL;
2420 }
2421
2422 cm_format_rep_event(work, cm_id_priv->qp_type);
2423
2424 spin_lock_irq(&cm_id_priv->lock);
2425 switch (cm_id_priv->id.state) {
2426 case IB_CM_REQ_SENT:
2427 case IB_CM_MRA_REQ_RCVD:
2428 break;
2429 default:
2430 ret = -EINVAL;
2431 pr_debug(
2432 "%s: cm_id_priv->id.state: %d, local_comm_id %d, remote_comm_id %d\n",
2433 __func__, cm_id_priv->id.state,
2434 IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg),
2435 IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg));
2436 spin_unlock_irq(&cm_id_priv->lock);
2437 goto error;
2438 }
2439
2440 cm_id_priv->timewait_info->work.remote_id =
2441 cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg));
2442 cm_id_priv->timewait_info->remote_ca_guid =
2443 cpu_to_be64(IBA_GET(CM_REP_LOCAL_CA_GUID, rep_msg));
2444 cm_id_priv->timewait_info->remote_qpn = cm_rep_get_qpn(rep_msg, cm_id_priv->qp_type);
2445
2446 spin_lock(&cm.lock);
2447 /* Check for duplicate REP. */
2448 if (cm_insert_remote_id(cm_id_priv->timewait_info)) {
2449 spin_unlock(&cm.lock);
2450 spin_unlock_irq(&cm_id_priv->lock);
2451 ret = -EINVAL;
2452 pr_debug("%s: Failed to insert remote id %d\n", __func__,
2453 IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg));
2454 goto error;
2455 }
2456 /* Check for a stale connection. */
2457 timewait_info = cm_insert_remote_qpn(cm_id_priv->timewait_info);
2458 if (timewait_info) {
2459 rb_erase(&cm_id_priv->timewait_info->remote_id_node,
2460 &cm.remote_id_table);
2461 cm_id_priv->timewait_info->inserted_remote_id = 0;
2462 cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
2463 timewait_info->work.remote_id);
2464
2465 spin_unlock(&cm.lock);
2466 spin_unlock_irq(&cm_id_priv->lock);
2467 cm_issue_rej(work->port, work->mad_recv_wc,
2468 IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REP,
2469 NULL, 0);
2470 ret = -EINVAL;
2471 pr_debug(
2472 "%s: Stale connection. local_comm_id %d, remote_comm_id %d\n",
2473 __func__, IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg),
2474 IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg));
2475
2476 if (cur_cm_id_priv) {
2477 cm_id = &cur_cm_id_priv->id;
2478 ib_send_cm_dreq(cm_id, NULL, 0);
2479 cm_deref_id(cur_cm_id_priv);
2480 }
2481
2482 goto error;
2483 }
2484 spin_unlock(&cm.lock);
2485
2486 cm_id_priv->id.state = IB_CM_REP_RCVD;
2487 cm_id_priv->id.remote_id =
2488 cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg));
2489 cm_id_priv->remote_qpn = cm_rep_get_qpn(rep_msg, cm_id_priv->qp_type);
2490 cm_id_priv->initiator_depth =
2491 IBA_GET(CM_REP_RESPONDER_RESOURCES, rep_msg);
2492 cm_id_priv->responder_resources =
2493 IBA_GET(CM_REP_INITIATOR_DEPTH, rep_msg);
2494 cm_id_priv->sq_psn = cpu_to_be32(IBA_GET(CM_REP_STARTING_PSN, rep_msg));
2495 cm_id_priv->rnr_retry_count = IBA_GET(CM_REP_RNR_RETRY_COUNT, rep_msg);
2496 cm_id_priv->target_ack_delay =
2497 IBA_GET(CM_REP_TARGET_ACK_DELAY, rep_msg);
2498 cm_id_priv->av.timeout =
2499 cm_ack_timeout(cm_id_priv->target_ack_delay,
2500 cm_id_priv->av.timeout - 1);
2501 cm_id_priv->alt_av.timeout =
2502 cm_ack_timeout(cm_id_priv->target_ack_delay,
2503 cm_id_priv->alt_av.timeout - 1);
2504
2505 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2506 ret = atomic_inc_and_test(&cm_id_priv->work_count);
2507 if (!ret)
2508 list_add_tail(&work->list, &cm_id_priv->work_list);
2509 spin_unlock_irq(&cm_id_priv->lock);
2510
2511 if (ret)
2512 cm_process_work(cm_id_priv, work);
2513 else
2514 cm_deref_id(cm_id_priv);
2515 return 0;
2516
2517 error:
2518 cm_deref_id(cm_id_priv);
2519 return ret;
2520 }
2521
2522 static int cm_establish_handler(struct cm_work *work)
2523 {
2524 struct cm_id_private *cm_id_priv;
2525 int ret;
2526
2527 /* See comment in cm_establish about lookup. */
2528 cm_id_priv = cm_acquire_id(work->local_id, work->remote_id);
2529 if (!cm_id_priv)
2530 return -EINVAL;
2531
2532 spin_lock_irq(&cm_id_priv->lock);
2533 if (cm_id_priv->id.state != IB_CM_ESTABLISHED) {
2534 spin_unlock_irq(&cm_id_priv->lock);
2535 goto out;
2536 }
2537
2538 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2539 ret = atomic_inc_and_test(&cm_id_priv->work_count);
2540 if (!ret)
2541 list_add_tail(&work->list, &cm_id_priv->work_list);
2542 spin_unlock_irq(&cm_id_priv->lock);
2543
2544 if (ret)
2545 cm_process_work(cm_id_priv, work);
2546 else
2547 cm_deref_id(cm_id_priv);
2548 return 0;
2549 out:
2550 cm_deref_id(cm_id_priv);
2551 return -EINVAL;
2552 }
2553
2554 static int cm_rtu_handler(struct cm_work *work)
2555 {
2556 struct cm_id_private *cm_id_priv;
2557 struct cm_rtu_msg *rtu_msg;
2558 int ret;
2559
2560 rtu_msg = (struct cm_rtu_msg *)work->mad_recv_wc->recv_buf.mad;
2561 cm_id_priv = cm_acquire_id(
2562 cpu_to_be32(IBA_GET(CM_RTU_REMOTE_COMM_ID, rtu_msg)),
2563 cpu_to_be32(IBA_GET(CM_RTU_LOCAL_COMM_ID, rtu_msg)));
2564 if (!cm_id_priv)
2565 return -EINVAL;
2566
2567 work->cm_event.private_data =
2568 IBA_GET_MEM_PTR(CM_RTU_PRIVATE_DATA, rtu_msg);
2569
2570 spin_lock_irq(&cm_id_priv->lock);
2571 if (cm_id_priv->id.state != IB_CM_REP_SENT &&
2572 cm_id_priv->id.state != IB_CM_MRA_REP_RCVD) {
2573 spin_unlock_irq(&cm_id_priv->lock);
2574 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
2575 counter[CM_RTU_COUNTER]);
2576 goto out;
2577 }
2578 cm_id_priv->id.state = IB_CM_ESTABLISHED;
2579
2580 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2581 ret = atomic_inc_and_test(&cm_id_priv->work_count);
2582 if (!ret)
2583 list_add_tail(&work->list, &cm_id_priv->work_list);
2584 spin_unlock_irq(&cm_id_priv->lock);
2585
2586 if (ret)
2587 cm_process_work(cm_id_priv, work);
2588 else
2589 cm_deref_id(cm_id_priv);
2590 return 0;
2591 out:
2592 cm_deref_id(cm_id_priv);
2593 return -EINVAL;
2594 }
2595
2596 static void cm_format_dreq(struct cm_dreq_msg *dreq_msg,
2597 struct cm_id_private *cm_id_priv,
2598 const void *private_data,
2599 u8 private_data_len)
2600 {
2601 cm_format_mad_hdr(&dreq_msg->hdr, CM_DREQ_ATTR_ID,
2602 cm_form_tid(cm_id_priv));
2603 IBA_SET(CM_DREQ_LOCAL_COMM_ID, dreq_msg,
2604 be32_to_cpu(cm_id_priv->id.local_id));
2605 IBA_SET(CM_DREQ_REMOTE_COMM_ID, dreq_msg,
2606 be32_to_cpu(cm_id_priv->id.remote_id));
2607 IBA_SET(CM_DREQ_REMOTE_QPN_EECN, dreq_msg,
2608 be32_to_cpu(cm_id_priv->remote_qpn));
2609
2610 if (private_data && private_data_len)
2611 IBA_SET_MEM(CM_DREQ_PRIVATE_DATA, dreq_msg, private_data,
2612 private_data_len);
2613 }
2614
2615 static int cm_send_dreq_locked(struct cm_id_private *cm_id_priv,
2616 const void *private_data, u8 private_data_len)
2617 {
2618 struct ib_mad_send_buf *msg;
2619 int ret;
2620
2621 lockdep_assert_held(&cm_id_priv->lock);
2622
2623 if (private_data && private_data_len > IB_CM_DREQ_PRIVATE_DATA_SIZE)
2624 return -EINVAL;
2625
2626 if (cm_id_priv->id.state != IB_CM_ESTABLISHED) {
2627 pr_debug("%s: local_id %d, cm_id->state: %d\n", __func__,
2628 be32_to_cpu(cm_id_priv->id.local_id),
2629 cm_id_priv->id.state);
2630 return -EINVAL;
2631 }
2632
2633 if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT ||
2634 cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD)
2635 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2636
2637 ret = cm_alloc_msg(cm_id_priv, &msg);
2638 if (ret) {
2639 cm_enter_timewait(cm_id_priv);
2640 return ret;
2641 }
2642
2643 cm_format_dreq((struct cm_dreq_msg *) msg->mad, cm_id_priv,
2644 private_data, private_data_len);
2645 msg->timeout_ms = cm_id_priv->timeout_ms;
2646 msg->context[1] = (void *) (unsigned long) IB_CM_DREQ_SENT;
2647
2648 ret = ib_post_send_mad(msg, NULL);
2649 if (ret) {
2650 cm_enter_timewait(cm_id_priv);
2651 cm_free_msg(msg);
2652 return ret;
2653 }
2654
2655 cm_id_priv->id.state = IB_CM_DREQ_SENT;
2656 cm_id_priv->msg = msg;
2657 return 0;
2658 }
2659
2660 int ib_send_cm_dreq(struct ib_cm_id *cm_id, const void *private_data,
2661 u8 private_data_len)
2662 {
2663 struct cm_id_private *cm_id_priv =
2664 container_of(cm_id, struct cm_id_private, id);
2665 unsigned long flags;
2666 int ret;
2667
2668 spin_lock_irqsave(&cm_id_priv->lock, flags);
2669 ret = cm_send_dreq_locked(cm_id_priv, private_data, private_data_len);
2670 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2671 return ret;
2672 }
2673 EXPORT_SYMBOL(ib_send_cm_dreq);
2674
2675 static void cm_format_drep(struct cm_drep_msg *drep_msg,
2676 struct cm_id_private *cm_id_priv,
2677 const void *private_data,
2678 u8 private_data_len)
2679 {
2680 cm_format_mad_hdr(&drep_msg->hdr, CM_DREP_ATTR_ID, cm_id_priv->tid);
2681 IBA_SET(CM_DREP_LOCAL_COMM_ID, drep_msg,
2682 be32_to_cpu(cm_id_priv->id.local_id));
2683 IBA_SET(CM_DREP_REMOTE_COMM_ID, drep_msg,
2684 be32_to_cpu(cm_id_priv->id.remote_id));
2685
2686 if (private_data && private_data_len)
2687 IBA_SET_MEM(CM_DREP_PRIVATE_DATA, drep_msg, private_data,
2688 private_data_len);
2689 }
2690
2691 static int cm_send_drep_locked(struct cm_id_private *cm_id_priv,
2692 void *private_data, u8 private_data_len)
2693 {
2694 struct ib_mad_send_buf *msg;
2695 int ret;
2696
2697 lockdep_assert_held(&cm_id_priv->lock);
2698
2699 if (private_data && private_data_len > IB_CM_DREP_PRIVATE_DATA_SIZE)
2700 return -EINVAL;
2701
2702 if (cm_id_priv->id.state != IB_CM_DREQ_RCVD) {
2703 pr_debug(
2704 "%s: local_id %d, cm_idcm_id->state(%d) != IB_CM_DREQ_RCVD\n",
2705 __func__, be32_to_cpu(cm_id_priv->id.local_id),
2706 cm_id_priv->id.state);
2707 kfree(private_data);
2708 return -EINVAL;
2709 }
2710
2711 cm_set_private_data(cm_id_priv, private_data, private_data_len);
2712 cm_enter_timewait(cm_id_priv);
2713
2714 ret = cm_alloc_msg(cm_id_priv, &msg);
2715 if (ret)
2716 return ret;
2717
2718 cm_format_drep((struct cm_drep_msg *) msg->mad, cm_id_priv,
2719 private_data, private_data_len);
2720
2721 ret = ib_post_send_mad(msg, NULL);
2722 if (ret) {
2723 cm_free_msg(msg);
2724 return ret;
2725 }
2726 return 0;
2727 }
2728
2729 int ib_send_cm_drep(struct ib_cm_id *cm_id, const void *private_data,
2730 u8 private_data_len)
2731 {
2732 struct cm_id_private *cm_id_priv =
2733 container_of(cm_id, struct cm_id_private, id);
2734 unsigned long flags;
2735 void *data;
2736 int ret;
2737
2738 data = cm_copy_private_data(private_data, private_data_len);
2739 if (IS_ERR(data))
2740 return PTR_ERR(data);
2741
2742 spin_lock_irqsave(&cm_id_priv->lock, flags);
2743 ret = cm_send_drep_locked(cm_id_priv, data, private_data_len);
2744 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2745 return ret;
2746 }
2747 EXPORT_SYMBOL(ib_send_cm_drep);
2748
2749 static int cm_issue_drep(struct cm_port *port,
2750 struct ib_mad_recv_wc *mad_recv_wc)
2751 {
2752 struct ib_mad_send_buf *msg = NULL;
2753 struct cm_dreq_msg *dreq_msg;
2754 struct cm_drep_msg *drep_msg;
2755 int ret;
2756
2757 ret = cm_alloc_response_msg(port, mad_recv_wc, &msg);
2758 if (ret)
2759 return ret;
2760
2761 dreq_msg = (struct cm_dreq_msg *) mad_recv_wc->recv_buf.mad;
2762 drep_msg = (struct cm_drep_msg *) msg->mad;
2763
2764 cm_format_mad_hdr(&drep_msg->hdr, CM_DREP_ATTR_ID, dreq_msg->hdr.tid);
2765 IBA_SET(CM_DREP_REMOTE_COMM_ID, drep_msg,
2766 IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg));
2767 IBA_SET(CM_DREP_LOCAL_COMM_ID, drep_msg,
2768 IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg));
2769
2770 ret = ib_post_send_mad(msg, NULL);
2771 if (ret)
2772 cm_free_msg(msg);
2773
2774 return ret;
2775 }
2776
2777 static int cm_dreq_handler(struct cm_work *work)
2778 {
2779 struct cm_id_private *cm_id_priv;
2780 struct cm_dreq_msg *dreq_msg;
2781 struct ib_mad_send_buf *msg = NULL;
2782 int ret;
2783
2784 dreq_msg = (struct cm_dreq_msg *)work->mad_recv_wc->recv_buf.mad;
2785 cm_id_priv = cm_acquire_id(
2786 cpu_to_be32(IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg)),
2787 cpu_to_be32(IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg)));
2788 if (!cm_id_priv) {
2789 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
2790 counter[CM_DREQ_COUNTER]);
2791 cm_issue_drep(work->port, work->mad_recv_wc);
2792 pr_debug(
2793 "%s: no cm_id_priv, local_comm_id %d, remote_comm_id %d\n",
2794 __func__, IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg),
2795 IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg));
2796 return -EINVAL;
2797 }
2798
2799 work->cm_event.private_data =
2800 IBA_GET_MEM_PTR(CM_DREQ_PRIVATE_DATA, dreq_msg);
2801
2802 spin_lock_irq(&cm_id_priv->lock);
2803 if (cm_id_priv->local_qpn !=
2804 cpu_to_be32(IBA_GET(CM_DREQ_REMOTE_QPN_EECN, dreq_msg)))
2805 goto unlock;
2806
2807 switch (cm_id_priv->id.state) {
2808 case IB_CM_REP_SENT:
2809 case IB_CM_DREQ_SENT:
2810 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2811 break;
2812 case IB_CM_ESTABLISHED:
2813 if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT ||
2814 cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD)
2815 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2816 break;
2817 case IB_CM_MRA_REP_RCVD:
2818 break;
2819 case IB_CM_TIMEWAIT:
2820 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
2821 counter[CM_DREQ_COUNTER]);
2822 msg = cm_alloc_response_msg_no_ah(work->port, work->mad_recv_wc);
2823 if (IS_ERR(msg))
2824 goto unlock;
2825
2826 cm_format_drep((struct cm_drep_msg *) msg->mad, cm_id_priv,
2827 cm_id_priv->private_data,
2828 cm_id_priv->private_data_len);
2829 spin_unlock_irq(&cm_id_priv->lock);
2830
2831 if (cm_create_response_msg_ah(work->port, work->mad_recv_wc, msg) ||
2832 ib_post_send_mad(msg, NULL))
2833 cm_free_msg(msg);
2834 goto deref;
2835 case IB_CM_DREQ_RCVD:
2836 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
2837 counter[CM_DREQ_COUNTER]);
2838 goto unlock;
2839 default:
2840 pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n",
2841 __func__, be32_to_cpu(cm_id_priv->id.local_id),
2842 cm_id_priv->id.state);
2843 goto unlock;
2844 }
2845 cm_id_priv->id.state = IB_CM_DREQ_RCVD;
2846 cm_id_priv->tid = dreq_msg->hdr.tid;
2847 ret = atomic_inc_and_test(&cm_id_priv->work_count);
2848 if (!ret)
2849 list_add_tail(&work->list, &cm_id_priv->work_list);
2850 spin_unlock_irq(&cm_id_priv->lock);
2851
2852 if (ret)
2853 cm_process_work(cm_id_priv, work);
2854 else
2855 cm_deref_id(cm_id_priv);
2856 return 0;
2857
2858 unlock: spin_unlock_irq(&cm_id_priv->lock);
2859 deref: cm_deref_id(cm_id_priv);
2860 return -EINVAL;
2861 }
2862
2863 static int cm_drep_handler(struct cm_work *work)
2864 {
2865 struct cm_id_private *cm_id_priv;
2866 struct cm_drep_msg *drep_msg;
2867 int ret;
2868
2869 drep_msg = (struct cm_drep_msg *)work->mad_recv_wc->recv_buf.mad;
2870 cm_id_priv = cm_acquire_id(
2871 cpu_to_be32(IBA_GET(CM_DREP_REMOTE_COMM_ID, drep_msg)),
2872 cpu_to_be32(IBA_GET(CM_DREP_LOCAL_COMM_ID, drep_msg)));
2873 if (!cm_id_priv)
2874 return -EINVAL;
2875
2876 work->cm_event.private_data =
2877 IBA_GET_MEM_PTR(CM_DREP_PRIVATE_DATA, drep_msg);
2878
2879 spin_lock_irq(&cm_id_priv->lock);
2880 if (cm_id_priv->id.state != IB_CM_DREQ_SENT &&
2881 cm_id_priv->id.state != IB_CM_DREQ_RCVD) {
2882 spin_unlock_irq(&cm_id_priv->lock);
2883 goto out;
2884 }
2885 cm_enter_timewait(cm_id_priv);
2886
2887 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2888 ret = atomic_inc_and_test(&cm_id_priv->work_count);
2889 if (!ret)
2890 list_add_tail(&work->list, &cm_id_priv->work_list);
2891 spin_unlock_irq(&cm_id_priv->lock);
2892
2893 if (ret)
2894 cm_process_work(cm_id_priv, work);
2895 else
2896 cm_deref_id(cm_id_priv);
2897 return 0;
2898 out:
2899 cm_deref_id(cm_id_priv);
2900 return -EINVAL;
2901 }
2902
2903 static int cm_send_rej_locked(struct cm_id_private *cm_id_priv,
2904 enum ib_cm_rej_reason reason, void *ari,
2905 u8 ari_length, const void *private_data,
2906 u8 private_data_len)
2907 {
2908 enum ib_cm_state state = cm_id_priv->id.state;
2909 struct ib_mad_send_buf *msg;
2910 int ret;
2911
2912 lockdep_assert_held(&cm_id_priv->lock);
2913
2914 if ((private_data && private_data_len > IB_CM_REJ_PRIVATE_DATA_SIZE) ||
2915 (ari && ari_length > IB_CM_REJ_ARI_LENGTH))
2916 return -EINVAL;
2917
2918 switch (state) {
2919 case IB_CM_REQ_SENT:
2920 case IB_CM_MRA_REQ_RCVD:
2921 case IB_CM_REQ_RCVD:
2922 case IB_CM_MRA_REQ_SENT:
2923 case IB_CM_REP_RCVD:
2924 case IB_CM_MRA_REP_SENT:
2925 cm_reset_to_idle(cm_id_priv);
2926 ret = cm_alloc_msg(cm_id_priv, &msg);
2927 if (ret)
2928 return ret;
2929 cm_format_rej((struct cm_rej_msg *)msg->mad, cm_id_priv, reason,
2930 ari, ari_length, private_data, private_data_len,
2931 state);
2932 break;
2933 case IB_CM_REP_SENT:
2934 case IB_CM_MRA_REP_RCVD:
2935 cm_enter_timewait(cm_id_priv);
2936 ret = cm_alloc_msg(cm_id_priv, &msg);
2937 if (ret)
2938 return ret;
2939 cm_format_rej((struct cm_rej_msg *)msg->mad, cm_id_priv, reason,
2940 ari, ari_length, private_data, private_data_len,
2941 state);
2942 break;
2943 default:
2944 pr_debug("%s: local_id %d, cm_id->state: %d\n", __func__,
2945 be32_to_cpu(cm_id_priv->id.local_id),
2946 cm_id_priv->id.state);
2947 return -EINVAL;
2948 }
2949
2950 ret = ib_post_send_mad(msg, NULL);
2951 if (ret) {
2952 cm_free_msg(msg);
2953 return ret;
2954 }
2955
2956 return 0;
2957 }
2958
2959 int ib_send_cm_rej(struct ib_cm_id *cm_id, enum ib_cm_rej_reason reason,
2960 void *ari, u8 ari_length, const void *private_data,
2961 u8 private_data_len)
2962 {
2963 struct cm_id_private *cm_id_priv =
2964 container_of(cm_id, struct cm_id_private, id);
2965 unsigned long flags;
2966 int ret;
2967
2968 spin_lock_irqsave(&cm_id_priv->lock, flags);
2969 ret = cm_send_rej_locked(cm_id_priv, reason, ari, ari_length,
2970 private_data, private_data_len);
2971 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2972 return ret;
2973 }
2974 EXPORT_SYMBOL(ib_send_cm_rej);
2975
2976 static void cm_format_rej_event(struct cm_work *work)
2977 {
2978 struct cm_rej_msg *rej_msg;
2979 struct ib_cm_rej_event_param *param;
2980
2981 rej_msg = (struct cm_rej_msg *)work->mad_recv_wc->recv_buf.mad;
2982 param = &work->cm_event.param.rej_rcvd;
2983 param->ari = IBA_GET_MEM_PTR(CM_REJ_ARI, rej_msg);
2984 param->ari_length = IBA_GET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg);
2985 param->reason = IBA_GET(CM_REJ_REASON, rej_msg);
2986 work->cm_event.private_data =
2987 IBA_GET_MEM_PTR(CM_REJ_PRIVATE_DATA, rej_msg);
2988 }
2989
2990 static struct cm_id_private * cm_acquire_rejected_id(struct cm_rej_msg *rej_msg)
2991 {
2992 struct cm_timewait_info *timewait_info;
2993 struct cm_id_private *cm_id_priv;
2994 __be32 remote_id;
2995
2996 remote_id = cpu_to_be32(IBA_GET(CM_REJ_LOCAL_COMM_ID, rej_msg));
2997
2998 if (IBA_GET(CM_REJ_REASON, rej_msg) == IB_CM_REJ_TIMEOUT) {
2999 spin_lock_irq(&cm.lock);
3000 timewait_info = cm_find_remote_id(
3001 *((__be64 *)IBA_GET_MEM_PTR(CM_REJ_ARI, rej_msg)),
3002 remote_id);
3003 if (!timewait_info) {
3004 spin_unlock_irq(&cm.lock);
3005 return NULL;
3006 }
3007 cm_id_priv =
3008 cm_acquire_id(timewait_info->work.local_id, remote_id);
3009 spin_unlock_irq(&cm.lock);
3010 } else if (IBA_GET(CM_REJ_MESSAGE_REJECTED, rej_msg) ==
3011 CM_MSG_RESPONSE_REQ)
3012 cm_id_priv = cm_acquire_id(
3013 cpu_to_be32(IBA_GET(CM_REJ_REMOTE_COMM_ID, rej_msg)),
3014 0);
3015 else
3016 cm_id_priv = cm_acquire_id(
3017 cpu_to_be32(IBA_GET(CM_REJ_REMOTE_COMM_ID, rej_msg)),
3018 remote_id);
3019
3020 return cm_id_priv;
3021 }
3022
3023 static int cm_rej_handler(struct cm_work *work)
3024 {
3025 struct cm_id_private *cm_id_priv;
3026 struct cm_rej_msg *rej_msg;
3027 int ret;
3028
3029 rej_msg = (struct cm_rej_msg *)work->mad_recv_wc->recv_buf.mad;
3030 cm_id_priv = cm_acquire_rejected_id(rej_msg);
3031 if (!cm_id_priv)
3032 return -EINVAL;
3033
3034 cm_format_rej_event(work);
3035
3036 spin_lock_irq(&cm_id_priv->lock);
3037 switch (cm_id_priv->id.state) {
3038 case IB_CM_REQ_SENT:
3039 case IB_CM_MRA_REQ_RCVD:
3040 case IB_CM_REP_SENT:
3041 case IB_CM_MRA_REP_RCVD:
3042 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
3043 /* fall through */
3044 case IB_CM_REQ_RCVD:
3045 case IB_CM_MRA_REQ_SENT:
3046 if (IBA_GET(CM_REJ_REASON, rej_msg) == IB_CM_REJ_STALE_CONN)
3047 cm_enter_timewait(cm_id_priv);
3048 else
3049 cm_reset_to_idle(cm_id_priv);
3050 break;
3051 case IB_CM_DREQ_SENT:
3052 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
3053 /* fall through */
3054 case IB_CM_REP_RCVD:
3055 case IB_CM_MRA_REP_SENT:
3056 cm_enter_timewait(cm_id_priv);
3057 break;
3058 case IB_CM_ESTABLISHED:
3059 if (cm_id_priv->id.lap_state == IB_CM_LAP_UNINIT ||
3060 cm_id_priv->id.lap_state == IB_CM_LAP_SENT) {
3061 if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT)
3062 ib_cancel_mad(cm_id_priv->av.port->mad_agent,
3063 cm_id_priv->msg);
3064 cm_enter_timewait(cm_id_priv);
3065 break;
3066 }
3067 /* fall through */
3068 default:
3069 pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n",
3070 __func__, be32_to_cpu(cm_id_priv->id.local_id),
3071 cm_id_priv->id.state);
3072 spin_unlock_irq(&cm_id_priv->lock);
3073 ret = -EINVAL;
3074 goto out;
3075 }
3076
3077 ret = atomic_inc_and_test(&cm_id_priv->work_count);
3078 if (!ret)
3079 list_add_tail(&work->list, &cm_id_priv->work_list);
3080 spin_unlock_irq(&cm_id_priv->lock);
3081
3082 if (ret)
3083 cm_process_work(cm_id_priv, work);
3084 else
3085 cm_deref_id(cm_id_priv);
3086 return 0;
3087 out:
3088 cm_deref_id(cm_id_priv);
3089 return -EINVAL;
3090 }
3091
3092 int ib_send_cm_mra(struct ib_cm_id *cm_id,
3093 u8 service_timeout,
3094 const void *private_data,
3095 u8 private_data_len)
3096 {
3097 struct cm_id_private *cm_id_priv;
3098 struct ib_mad_send_buf *msg;
3099 enum ib_cm_state cm_state;
3100 enum ib_cm_lap_state lap_state;
3101 enum cm_msg_response msg_response;
3102 void *data;
3103 unsigned long flags;
3104 int ret;
3105
3106 if (private_data && private_data_len > IB_CM_MRA_PRIVATE_DATA_SIZE)
3107 return -EINVAL;
3108
3109 data = cm_copy_private_data(private_data, private_data_len);
3110 if (IS_ERR(data))
3111 return PTR_ERR(data);
3112
3113 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3114
3115 spin_lock_irqsave(&cm_id_priv->lock, flags);
3116 switch(cm_id_priv->id.state) {
3117 case IB_CM_REQ_RCVD:
3118 cm_state = IB_CM_MRA_REQ_SENT;
3119 lap_state = cm_id->lap_state;
3120 msg_response = CM_MSG_RESPONSE_REQ;
3121 break;
3122 case IB_CM_REP_RCVD:
3123 cm_state = IB_CM_MRA_REP_SENT;
3124 lap_state = cm_id->lap_state;
3125 msg_response = CM_MSG_RESPONSE_REP;
3126 break;
3127 case IB_CM_ESTABLISHED:
3128 if (cm_id->lap_state == IB_CM_LAP_RCVD) {
3129 cm_state = cm_id->state;
3130 lap_state = IB_CM_MRA_LAP_SENT;
3131 msg_response = CM_MSG_RESPONSE_OTHER;
3132 break;
3133 }
3134 /* fall through */
3135 default:
3136 pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n",
3137 __func__, be32_to_cpu(cm_id_priv->id.local_id),
3138 cm_id_priv->id.state);
3139 ret = -EINVAL;
3140 goto error1;
3141 }
3142
3143 if (!(service_timeout & IB_CM_MRA_FLAG_DELAY)) {
3144 ret = cm_alloc_msg(cm_id_priv, &msg);
3145 if (ret)
3146 goto error1;
3147
3148 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
3149 msg_response, service_timeout,
3150 private_data, private_data_len);
3151 ret = ib_post_send_mad(msg, NULL);
3152 if (ret)
3153 goto error2;
3154 }
3155
3156 cm_id->state = cm_state;
3157 cm_id->lap_state = lap_state;
3158 cm_id_priv->service_timeout = service_timeout;
3159 cm_set_private_data(cm_id_priv, data, private_data_len);
3160 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3161 return 0;
3162
3163 error1: spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3164 kfree(data);
3165 return ret;
3166
3167 error2: spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3168 kfree(data);
3169 cm_free_msg(msg);
3170 return ret;
3171 }
3172 EXPORT_SYMBOL(ib_send_cm_mra);
3173
3174 static struct cm_id_private * cm_acquire_mraed_id(struct cm_mra_msg *mra_msg)
3175 {
3176 switch (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg)) {
3177 case CM_MSG_RESPONSE_REQ:
3178 return cm_acquire_id(
3179 cpu_to_be32(IBA_GET(CM_MRA_REMOTE_COMM_ID, mra_msg)),
3180 0);
3181 case CM_MSG_RESPONSE_REP:
3182 case CM_MSG_RESPONSE_OTHER:
3183 return cm_acquire_id(
3184 cpu_to_be32(IBA_GET(CM_MRA_REMOTE_COMM_ID, mra_msg)),
3185 cpu_to_be32(IBA_GET(CM_MRA_LOCAL_COMM_ID, mra_msg)));
3186 default:
3187 return NULL;
3188 }
3189 }
3190
3191 static int cm_mra_handler(struct cm_work *work)
3192 {
3193 struct cm_id_private *cm_id_priv;
3194 struct cm_mra_msg *mra_msg;
3195 int timeout, ret;
3196
3197 mra_msg = (struct cm_mra_msg *)work->mad_recv_wc->recv_buf.mad;
3198 cm_id_priv = cm_acquire_mraed_id(mra_msg);
3199 if (!cm_id_priv)
3200 return -EINVAL;
3201
3202 work->cm_event.private_data =
3203 IBA_GET_MEM_PTR(CM_MRA_PRIVATE_DATA, mra_msg);
3204 work->cm_event.param.mra_rcvd.service_timeout =
3205 IBA_GET(CM_MRA_SERVICE_TIMEOUT, mra_msg);
3206 timeout = cm_convert_to_ms(IBA_GET(CM_MRA_SERVICE_TIMEOUT, mra_msg)) +
3207 cm_convert_to_ms(cm_id_priv->av.timeout);
3208
3209 spin_lock_irq(&cm_id_priv->lock);
3210 switch (cm_id_priv->id.state) {
3211 case IB_CM_REQ_SENT:
3212 if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) !=
3213 CM_MSG_RESPONSE_REQ ||
3214 ib_modify_mad(cm_id_priv->av.port->mad_agent,
3215 cm_id_priv->msg, timeout))
3216 goto out;
3217 cm_id_priv->id.state = IB_CM_MRA_REQ_RCVD;
3218 break;
3219 case IB_CM_REP_SENT:
3220 if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) !=
3221 CM_MSG_RESPONSE_REP ||
3222 ib_modify_mad(cm_id_priv->av.port->mad_agent,
3223 cm_id_priv->msg, timeout))
3224 goto out;
3225 cm_id_priv->id.state = IB_CM_MRA_REP_RCVD;
3226 break;
3227 case IB_CM_ESTABLISHED:
3228 if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) !=
3229 CM_MSG_RESPONSE_OTHER ||
3230 cm_id_priv->id.lap_state != IB_CM_LAP_SENT ||
3231 ib_modify_mad(cm_id_priv->av.port->mad_agent,
3232 cm_id_priv->msg, timeout)) {
3233 if (cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD)
3234 atomic_long_inc(&work->port->
3235 counter_group[CM_RECV_DUPLICATES].
3236 counter[CM_MRA_COUNTER]);
3237 goto out;
3238 }
3239 cm_id_priv->id.lap_state = IB_CM_MRA_LAP_RCVD;
3240 break;
3241 case IB_CM_MRA_REQ_RCVD:
3242 case IB_CM_MRA_REP_RCVD:
3243 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
3244 counter[CM_MRA_COUNTER]);
3245 /* fall through */
3246 default:
3247 pr_debug("%s local_id %d, cm_id_priv->id.state: %d\n",
3248 __func__, be32_to_cpu(cm_id_priv->id.local_id),
3249 cm_id_priv->id.state);
3250 goto out;
3251 }
3252
3253 cm_id_priv->msg->context[1] = (void *) (unsigned long)
3254 cm_id_priv->id.state;
3255 ret = atomic_inc_and_test(&cm_id_priv->work_count);
3256 if (!ret)
3257 list_add_tail(&work->list, &cm_id_priv->work_list);
3258 spin_unlock_irq(&cm_id_priv->lock);
3259
3260 if (ret)
3261 cm_process_work(cm_id_priv, work);
3262 else
3263 cm_deref_id(cm_id_priv);
3264 return 0;
3265 out:
3266 spin_unlock_irq(&cm_id_priv->lock);
3267 cm_deref_id(cm_id_priv);
3268 return -EINVAL;
3269 }
3270
3271 static void cm_format_path_lid_from_lap(struct cm_lap_msg *lap_msg,
3272 struct sa_path_rec *path)
3273 {
3274 u32 lid;
3275
3276 if (path->rec_type != SA_PATH_REC_TYPE_OPA) {
3277 sa_path_set_dlid(path, IBA_GET(CM_LAP_ALTERNATE_LOCAL_PORT_LID,
3278 lap_msg));
3279 sa_path_set_slid(path, IBA_GET(CM_LAP_ALTERNATE_REMOTE_PORT_LID,
3280 lap_msg));
3281 } else {
3282 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
3283 CM_LAP_ALTERNATE_LOCAL_PORT_GID, lap_msg));
3284 sa_path_set_dlid(path, lid);
3285
3286 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
3287 CM_LAP_ALTERNATE_REMOTE_PORT_GID, lap_msg));
3288 sa_path_set_slid(path, lid);
3289 }
3290 }
3291
3292 static void cm_format_path_from_lap(struct cm_id_private *cm_id_priv,
3293 struct sa_path_rec *path,
3294 struct cm_lap_msg *lap_msg)
3295 {
3296 path->dgid = *IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_LOCAL_PORT_GID, lap_msg);
3297 path->sgid =
3298 *IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_REMOTE_PORT_GID, lap_msg);
3299 path->flow_label =
3300 cpu_to_be32(IBA_GET(CM_LAP_ALTERNATE_FLOW_LABEL, lap_msg));
3301 path->hop_limit = IBA_GET(CM_LAP_ALTERNATE_HOP_LIMIT, lap_msg);
3302 path->traffic_class = IBA_GET(CM_LAP_ALTERNATE_TRAFFIC_CLASS, lap_msg);
3303 path->reversible = 1;
3304 path->pkey = cm_id_priv->pkey;
3305 path->sl = IBA_GET(CM_LAP_ALTERNATE_SL, lap_msg);
3306 path->mtu_selector = IB_SA_EQ;
3307 path->mtu = cm_id_priv->path_mtu;
3308 path->rate_selector = IB_SA_EQ;
3309 path->rate = IBA_GET(CM_LAP_ALTERNATE_PACKET_RATE, lap_msg);
3310 path->packet_life_time_selector = IB_SA_EQ;
3311 path->packet_life_time =
3312 IBA_GET(CM_LAP_ALTERNATE_LOCAL_ACK_TIMEOUT, lap_msg);
3313 path->packet_life_time -= (path->packet_life_time > 0);
3314 cm_format_path_lid_from_lap(lap_msg, path);
3315 }
3316
3317 static int cm_lap_handler(struct cm_work *work)
3318 {
3319 struct cm_id_private *cm_id_priv;
3320 struct cm_lap_msg *lap_msg;
3321 struct ib_cm_lap_event_param *param;
3322 struct ib_mad_send_buf *msg = NULL;
3323 int ret;
3324
3325 /* Currently Alternate path messages are not supported for
3326 * RoCE link layer.
3327 */
3328 if (rdma_protocol_roce(work->port->cm_dev->ib_device,
3329 work->port->port_num))
3330 return -EINVAL;
3331
3332 /* todo: verify LAP request and send reject APR if invalid. */
3333 lap_msg = (struct cm_lap_msg *)work->mad_recv_wc->recv_buf.mad;
3334 cm_id_priv = cm_acquire_id(
3335 cpu_to_be32(IBA_GET(CM_LAP_REMOTE_COMM_ID, lap_msg)),
3336 cpu_to_be32(IBA_GET(CM_LAP_LOCAL_COMM_ID, lap_msg)));
3337 if (!cm_id_priv)
3338 return -EINVAL;
3339
3340 param = &work->cm_event.param.lap_rcvd;
3341 memset(&work->path[0], 0, sizeof(work->path[1]));
3342 cm_path_set_rec_type(work->port->cm_dev->ib_device,
3343 work->port->port_num, &work->path[0],
3344 IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_LOCAL_PORT_GID,
3345 lap_msg));
3346 param->alternate_path = &work->path[0];
3347 cm_format_path_from_lap(cm_id_priv, param->alternate_path, lap_msg);
3348 work->cm_event.private_data =
3349 IBA_GET_MEM_PTR(CM_LAP_PRIVATE_DATA, lap_msg);
3350
3351 spin_lock_irq(&cm_id_priv->lock);
3352 if (cm_id_priv->id.state != IB_CM_ESTABLISHED)
3353 goto unlock;
3354
3355 switch (cm_id_priv->id.lap_state) {
3356 case IB_CM_LAP_UNINIT:
3357 case IB_CM_LAP_IDLE:
3358 break;
3359 case IB_CM_MRA_LAP_SENT:
3360 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
3361 counter[CM_LAP_COUNTER]);
3362 msg = cm_alloc_response_msg_no_ah(work->port, work->mad_recv_wc);
3363 if (IS_ERR(msg))
3364 goto unlock;
3365
3366 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
3367 CM_MSG_RESPONSE_OTHER,
3368 cm_id_priv->service_timeout,
3369 cm_id_priv->private_data,
3370 cm_id_priv->private_data_len);
3371 spin_unlock_irq(&cm_id_priv->lock);
3372
3373 if (cm_create_response_msg_ah(work->port, work->mad_recv_wc, msg) ||
3374 ib_post_send_mad(msg, NULL))
3375 cm_free_msg(msg);
3376 goto deref;
3377 case IB_CM_LAP_RCVD:
3378 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
3379 counter[CM_LAP_COUNTER]);
3380 goto unlock;
3381 default:
3382 goto unlock;
3383 }
3384
3385 ret = cm_init_av_for_lap(work->port, work->mad_recv_wc->wc,
3386 work->mad_recv_wc->recv_buf.grh,
3387 &cm_id_priv->av);
3388 if (ret)
3389 goto unlock;
3390
3391 ret = cm_init_av_by_path(param->alternate_path, NULL,
3392 &cm_id_priv->alt_av, cm_id_priv);
3393 if (ret)
3394 goto unlock;
3395
3396 cm_id_priv->id.lap_state = IB_CM_LAP_RCVD;
3397 cm_id_priv->tid = lap_msg->hdr.tid;
3398 ret = atomic_inc_and_test(&cm_id_priv->work_count);
3399 if (!ret)
3400 list_add_tail(&work->list, &cm_id_priv->work_list);
3401 spin_unlock_irq(&cm_id_priv->lock);
3402
3403 if (ret)
3404 cm_process_work(cm_id_priv, work);
3405 else
3406 cm_deref_id(cm_id_priv);
3407 return 0;
3408
3409 unlock: spin_unlock_irq(&cm_id_priv->lock);
3410 deref: cm_deref_id(cm_id_priv);
3411 return -EINVAL;
3412 }
3413
3414 static int cm_apr_handler(struct cm_work *work)
3415 {
3416 struct cm_id_private *cm_id_priv;
3417 struct cm_apr_msg *apr_msg;
3418 int ret;
3419
3420 /* Currently Alternate path messages are not supported for
3421 * RoCE link layer.
3422 */
3423 if (rdma_protocol_roce(work->port->cm_dev->ib_device,
3424 work->port->port_num))
3425 return -EINVAL;
3426
3427 apr_msg = (struct cm_apr_msg *)work->mad_recv_wc->recv_buf.mad;
3428 cm_id_priv = cm_acquire_id(
3429 cpu_to_be32(IBA_GET(CM_APR_REMOTE_COMM_ID, apr_msg)),
3430 cpu_to_be32(IBA_GET(CM_APR_LOCAL_COMM_ID, apr_msg)));
3431 if (!cm_id_priv)
3432 return -EINVAL; /* Unmatched reply. */
3433
3434 work->cm_event.param.apr_rcvd.ap_status =
3435 IBA_GET(CM_APR_AR_STATUS, apr_msg);
3436 work->cm_event.param.apr_rcvd.apr_info =
3437 IBA_GET_MEM_PTR(CM_APR_ADDITIONAL_INFORMATION, apr_msg);
3438 work->cm_event.param.apr_rcvd.info_len =
3439 IBA_GET(CM_APR_ADDITIONAL_INFORMATION_LENGTH, apr_msg);
3440 work->cm_event.private_data =
3441 IBA_GET_MEM_PTR(CM_APR_PRIVATE_DATA, apr_msg);
3442
3443 spin_lock_irq(&cm_id_priv->lock);
3444 if (cm_id_priv->id.state != IB_CM_ESTABLISHED ||
3445 (cm_id_priv->id.lap_state != IB_CM_LAP_SENT &&
3446 cm_id_priv->id.lap_state != IB_CM_MRA_LAP_RCVD)) {
3447 spin_unlock_irq(&cm_id_priv->lock);
3448 goto out;
3449 }
3450 cm_id_priv->id.lap_state = IB_CM_LAP_IDLE;
3451 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
3452 cm_id_priv->msg = NULL;
3453
3454 ret = atomic_inc_and_test(&cm_id_priv->work_count);
3455 if (!ret)
3456 list_add_tail(&work->list, &cm_id_priv->work_list);
3457 spin_unlock_irq(&cm_id_priv->lock);
3458
3459 if (ret)
3460 cm_process_work(cm_id_priv, work);
3461 else
3462 cm_deref_id(cm_id_priv);
3463 return 0;
3464 out:
3465 cm_deref_id(cm_id_priv);
3466 return -EINVAL;
3467 }
3468
3469 static int cm_timewait_handler(struct cm_work *work)
3470 {
3471 struct cm_timewait_info *timewait_info;
3472 struct cm_id_private *cm_id_priv;
3473 int ret;
3474
3475 timewait_info = container_of(work, struct cm_timewait_info, work);
3476 spin_lock_irq(&cm.lock);
3477 list_del(&timewait_info->list);
3478 spin_unlock_irq(&cm.lock);
3479
3480 cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
3481 timewait_info->work.remote_id);
3482 if (!cm_id_priv)
3483 return -EINVAL;
3484
3485 spin_lock_irq(&cm_id_priv->lock);
3486 if (cm_id_priv->id.state != IB_CM_TIMEWAIT ||
3487 cm_id_priv->remote_qpn != timewait_info->remote_qpn) {
3488 spin_unlock_irq(&cm_id_priv->lock);
3489 goto out;
3490 }
3491 cm_id_priv->id.state = IB_CM_IDLE;
3492 ret = atomic_inc_and_test(&cm_id_priv->work_count);
3493 if (!ret)
3494 list_add_tail(&work->list, &cm_id_priv->work_list);
3495 spin_unlock_irq(&cm_id_priv->lock);
3496
3497 if (ret)
3498 cm_process_work(cm_id_priv, work);
3499 else
3500 cm_deref_id(cm_id_priv);
3501 return 0;
3502 out:
3503 cm_deref_id(cm_id_priv);
3504 return -EINVAL;
3505 }
3506
3507 static void cm_format_sidr_req(struct cm_sidr_req_msg *sidr_req_msg,
3508 struct cm_id_private *cm_id_priv,
3509 struct ib_cm_sidr_req_param *param)
3510 {
3511 cm_format_mad_hdr(&sidr_req_msg->hdr, CM_SIDR_REQ_ATTR_ID,
3512 cm_form_tid(cm_id_priv));
3513 IBA_SET(CM_SIDR_REQ_REQUESTID, sidr_req_msg,
3514 be32_to_cpu(cm_id_priv->id.local_id));
3515 IBA_SET(CM_SIDR_REQ_PARTITION_KEY, sidr_req_msg,
3516 be16_to_cpu(param->path->pkey));
3517 IBA_SET(CM_SIDR_REQ_SERVICEID, sidr_req_msg,
3518 be64_to_cpu(param->service_id));
3519
3520 if (param->private_data && param->private_data_len)
3521 IBA_SET_MEM(CM_SIDR_REQ_PRIVATE_DATA, sidr_req_msg,
3522 param->private_data, param->private_data_len);
3523 }
3524
3525 int ib_send_cm_sidr_req(struct ib_cm_id *cm_id,
3526 struct ib_cm_sidr_req_param *param)
3527 {
3528 struct cm_id_private *cm_id_priv;
3529 struct ib_mad_send_buf *msg;
3530 unsigned long flags;
3531 int ret;
3532
3533 if (!param->path || (param->private_data &&
3534 param->private_data_len > IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE))
3535 return -EINVAL;
3536
3537 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3538 ret = cm_init_av_by_path(param->path, param->sgid_attr,
3539 &cm_id_priv->av,
3540 cm_id_priv);
3541 if (ret)
3542 goto out;
3543
3544 cm_id->service_id = param->service_id;
3545 cm_id->service_mask = ~cpu_to_be64(0);
3546 cm_id_priv->timeout_ms = param->timeout_ms;
3547 cm_id_priv->max_cm_retries = param->max_cm_retries;
3548 ret = cm_alloc_msg(cm_id_priv, &msg);
3549 if (ret)
3550 goto out;
3551
3552 cm_format_sidr_req((struct cm_sidr_req_msg *) msg->mad, cm_id_priv,
3553 param);
3554 msg->timeout_ms = cm_id_priv->timeout_ms;
3555 msg->context[1] = (void *) (unsigned long) IB_CM_SIDR_REQ_SENT;
3556
3557 spin_lock_irqsave(&cm_id_priv->lock, flags);
3558 if (cm_id->state == IB_CM_IDLE)
3559 ret = ib_post_send_mad(msg, NULL);
3560 else
3561 ret = -EINVAL;
3562
3563 if (ret) {
3564 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3565 cm_free_msg(msg);
3566 goto out;
3567 }
3568 cm_id->state = IB_CM_SIDR_REQ_SENT;
3569 cm_id_priv->msg = msg;
3570 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3571 out:
3572 return ret;
3573 }
3574 EXPORT_SYMBOL(ib_send_cm_sidr_req);
3575
3576 static void cm_format_sidr_req_event(struct cm_work *work,
3577 const struct cm_id_private *rx_cm_id,
3578 struct ib_cm_id *listen_id)
3579 {
3580 struct cm_sidr_req_msg *sidr_req_msg;
3581 struct ib_cm_sidr_req_event_param *param;
3582
3583 sidr_req_msg = (struct cm_sidr_req_msg *)
3584 work->mad_recv_wc->recv_buf.mad;
3585 param = &work->cm_event.param.sidr_req_rcvd;
3586 param->pkey = IBA_GET(CM_SIDR_REQ_PARTITION_KEY, sidr_req_msg);
3587 param->listen_id = listen_id;
3588 param->service_id =
3589 cpu_to_be64(IBA_GET(CM_SIDR_REQ_SERVICEID, sidr_req_msg));
3590 param->bth_pkey = cm_get_bth_pkey(work);
3591 param->port = work->port->port_num;
3592 param->sgid_attr = rx_cm_id->av.ah_attr.grh.sgid_attr;
3593 work->cm_event.private_data =
3594 IBA_GET_MEM_PTR(CM_SIDR_REQ_PRIVATE_DATA, sidr_req_msg);
3595 }
3596
3597 static int cm_sidr_req_handler(struct cm_work *work)
3598 {
3599 struct cm_id_private *cm_id_priv, *listen_cm_id_priv;
3600 struct cm_sidr_req_msg *sidr_req_msg;
3601 struct ib_wc *wc;
3602 int ret;
3603
3604 cm_id_priv =
3605 cm_alloc_id_priv(work->port->cm_dev->ib_device, NULL, NULL);
3606 if (IS_ERR(cm_id_priv))
3607 return PTR_ERR(cm_id_priv);
3608
3609 /* Record SGID/SLID and request ID for lookup. */
3610 sidr_req_msg = (struct cm_sidr_req_msg *)
3611 work->mad_recv_wc->recv_buf.mad;
3612
3613 cm_id_priv->id.remote_id =
3614 cpu_to_be32(IBA_GET(CM_SIDR_REQ_REQUESTID, sidr_req_msg));
3615 cm_id_priv->id.service_id =
3616 cpu_to_be64(IBA_GET(CM_SIDR_REQ_SERVICEID, sidr_req_msg));
3617 cm_id_priv->id.service_mask = ~cpu_to_be64(0);
3618 cm_id_priv->tid = sidr_req_msg->hdr.tid;
3619
3620 wc = work->mad_recv_wc->wc;
3621 cm_id_priv->av.dgid.global.subnet_prefix = cpu_to_be64(wc->slid);
3622 cm_id_priv->av.dgid.global.interface_id = 0;
3623 ret = cm_init_av_for_response(work->port, work->mad_recv_wc->wc,
3624 work->mad_recv_wc->recv_buf.grh,
3625 &cm_id_priv->av);
3626 if (ret)
3627 goto out;
3628
3629 spin_lock_irq(&cm.lock);
3630 listen_cm_id_priv = cm_insert_remote_sidr(cm_id_priv);
3631 if (listen_cm_id_priv) {
3632 spin_unlock_irq(&cm.lock);
3633 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
3634 counter[CM_SIDR_REQ_COUNTER]);
3635 goto out; /* Duplicate message. */
3636 }
3637 cm_id_priv->id.state = IB_CM_SIDR_REQ_RCVD;
3638 listen_cm_id_priv = cm_find_listen(cm_id_priv->id.device,
3639 cm_id_priv->id.service_id);
3640 if (!listen_cm_id_priv) {
3641 spin_unlock_irq(&cm.lock);
3642 ib_send_cm_sidr_rep(&cm_id_priv->id,
3643 &(struct ib_cm_sidr_rep_param){
3644 .status = IB_SIDR_UNSUPPORTED });
3645 goto out; /* No match. */
3646 }
3647 refcount_inc(&listen_cm_id_priv->refcount);
3648 spin_unlock_irq(&cm.lock);
3649
3650 cm_id_priv->id.cm_handler = listen_cm_id_priv->id.cm_handler;
3651 cm_id_priv->id.context = listen_cm_id_priv->id.context;
3652
3653 /*
3654 * A SIDR ID does not need to be in the xarray since it does not receive
3655 * mads, is not placed in the remote_id or remote_qpn rbtree, and does
3656 * not enter timewait.
3657 */
3658
3659 cm_format_sidr_req_event(work, cm_id_priv, &listen_cm_id_priv->id);
3660 ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &work->cm_event);
3661 cm_free_work(work);
3662 /*
3663 * A pointer to the listen_cm_id is held in the event, so this deref
3664 * must be after the event is delivered above.
3665 */
3666 cm_deref_id(listen_cm_id_priv);
3667 if (ret)
3668 cm_destroy_id(&cm_id_priv->id, ret);
3669 return 0;
3670 out:
3671 ib_destroy_cm_id(&cm_id_priv->id);
3672 return -EINVAL;
3673 }
3674
3675 static void cm_format_sidr_rep(struct cm_sidr_rep_msg *sidr_rep_msg,
3676 struct cm_id_private *cm_id_priv,
3677 struct ib_cm_sidr_rep_param *param)
3678 {
3679 cm_format_mad_hdr(&sidr_rep_msg->hdr, CM_SIDR_REP_ATTR_ID,
3680 cm_id_priv->tid);
3681 IBA_SET(CM_SIDR_REP_REQUESTID, sidr_rep_msg,
3682 be32_to_cpu(cm_id_priv->id.remote_id));
3683 IBA_SET(CM_SIDR_REP_STATUS, sidr_rep_msg, param->status);
3684 IBA_SET(CM_SIDR_REP_QPN, sidr_rep_msg, param->qp_num);
3685 IBA_SET(CM_SIDR_REP_SERVICEID, sidr_rep_msg,
3686 be64_to_cpu(cm_id_priv->id.service_id));
3687 IBA_SET(CM_SIDR_REP_Q_KEY, sidr_rep_msg, param->qkey);
3688
3689 if (param->info && param->info_length)
3690 IBA_SET_MEM(CM_SIDR_REP_ADDITIONAL_INFORMATION, sidr_rep_msg,
3691 param->info, param->info_length);
3692
3693 if (param->private_data && param->private_data_len)
3694 IBA_SET_MEM(CM_SIDR_REP_PRIVATE_DATA, sidr_rep_msg,
3695 param->private_data, param->private_data_len);
3696 }
3697
3698 static int cm_send_sidr_rep_locked(struct cm_id_private *cm_id_priv,
3699 struct ib_cm_sidr_rep_param *param)
3700 {
3701 struct ib_mad_send_buf *msg;
3702 int ret;
3703
3704 lockdep_assert_held(&cm_id_priv->lock);
3705
3706 if ((param->info && param->info_length > IB_CM_SIDR_REP_INFO_LENGTH) ||
3707 (param->private_data &&
3708 param->private_data_len > IB_CM_SIDR_REP_PRIVATE_DATA_SIZE))
3709 return -EINVAL;
3710
3711 if (cm_id_priv->id.state != IB_CM_SIDR_REQ_RCVD)
3712 return -EINVAL;
3713
3714 ret = cm_alloc_msg(cm_id_priv, &msg);
3715 if (ret)
3716 return ret;
3717
3718 cm_format_sidr_rep((struct cm_sidr_rep_msg *) msg->mad, cm_id_priv,
3719 param);
3720 ret = ib_post_send_mad(msg, NULL);
3721 if (ret) {
3722 cm_free_msg(msg);
3723 return ret;
3724 }
3725 cm_id_priv->id.state = IB_CM_IDLE;
3726 if (!RB_EMPTY_NODE(&cm_id_priv->sidr_id_node)) {
3727 rb_erase(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table);
3728 RB_CLEAR_NODE(&cm_id_priv->sidr_id_node);
3729 }
3730 return 0;
3731 }
3732
3733 int ib_send_cm_sidr_rep(struct ib_cm_id *cm_id,
3734 struct ib_cm_sidr_rep_param *param)
3735 {
3736 struct cm_id_private *cm_id_priv =
3737 container_of(cm_id, struct cm_id_private, id);
3738 unsigned long flags;
3739 int ret;
3740
3741 spin_lock_irqsave(&cm_id_priv->lock, flags);
3742 ret = cm_send_sidr_rep_locked(cm_id_priv, param);
3743 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3744 return ret;
3745 }
3746 EXPORT_SYMBOL(ib_send_cm_sidr_rep);
3747
3748 static void cm_format_sidr_rep_event(struct cm_work *work,
3749 const struct cm_id_private *cm_id_priv)
3750 {
3751 struct cm_sidr_rep_msg *sidr_rep_msg;
3752 struct ib_cm_sidr_rep_event_param *param;
3753
3754 sidr_rep_msg = (struct cm_sidr_rep_msg *)
3755 work->mad_recv_wc->recv_buf.mad;
3756 param = &work->cm_event.param.sidr_rep_rcvd;
3757 param->status = IBA_GET(CM_SIDR_REP_STATUS, sidr_rep_msg);
3758 param->qkey = IBA_GET(CM_SIDR_REP_Q_KEY, sidr_rep_msg);
3759 param->qpn = IBA_GET(CM_SIDR_REP_QPN, sidr_rep_msg);
3760 param->info = IBA_GET_MEM_PTR(CM_SIDR_REP_ADDITIONAL_INFORMATION,
3761 sidr_rep_msg);
3762 param->info_len = IBA_GET(CM_SIDR_REP_ADDITIONAL_INFORMATION_LENGTH,
3763 sidr_rep_msg);
3764 param->sgid_attr = cm_id_priv->av.ah_attr.grh.sgid_attr;
3765 work->cm_event.private_data =
3766 IBA_GET_MEM_PTR(CM_SIDR_REP_PRIVATE_DATA, sidr_rep_msg);
3767 }
3768
3769 static int cm_sidr_rep_handler(struct cm_work *work)
3770 {
3771 struct cm_sidr_rep_msg *sidr_rep_msg;
3772 struct cm_id_private *cm_id_priv;
3773
3774 sidr_rep_msg = (struct cm_sidr_rep_msg *)
3775 work->mad_recv_wc->recv_buf.mad;
3776 cm_id_priv = cm_acquire_id(
3777 cpu_to_be32(IBA_GET(CM_SIDR_REP_REQUESTID, sidr_rep_msg)), 0);
3778 if (!cm_id_priv)
3779 return -EINVAL; /* Unmatched reply. */
3780
3781 spin_lock_irq(&cm_id_priv->lock);
3782 if (cm_id_priv->id.state != IB_CM_SIDR_REQ_SENT) {
3783 spin_unlock_irq(&cm_id_priv->lock);
3784 goto out;
3785 }
3786 cm_id_priv->id.state = IB_CM_IDLE;
3787 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
3788 spin_unlock_irq(&cm_id_priv->lock);
3789
3790 cm_format_sidr_rep_event(work, cm_id_priv);
3791 cm_process_work(cm_id_priv, work);
3792 return 0;
3793 out:
3794 cm_deref_id(cm_id_priv);
3795 return -EINVAL;
3796 }
3797
3798 static void cm_process_send_error(struct ib_mad_send_buf *msg,
3799 enum ib_wc_status wc_status)
3800 {
3801 struct cm_id_private *cm_id_priv;
3802 struct ib_cm_event cm_event;
3803 enum ib_cm_state state;
3804 int ret;
3805
3806 memset(&cm_event, 0, sizeof cm_event);
3807 cm_id_priv = msg->context[0];
3808
3809 /* Discard old sends or ones without a response. */
3810 spin_lock_irq(&cm_id_priv->lock);
3811 state = (enum ib_cm_state) (unsigned long) msg->context[1];
3812 if (msg != cm_id_priv->msg || state != cm_id_priv->id.state)
3813 goto discard;
3814
3815 pr_debug_ratelimited("CM: failed sending MAD in state %d. (%s)\n",
3816 state, ib_wc_status_msg(wc_status));
3817 switch (state) {
3818 case IB_CM_REQ_SENT:
3819 case IB_CM_MRA_REQ_RCVD:
3820 cm_reset_to_idle(cm_id_priv);
3821 cm_event.event = IB_CM_REQ_ERROR;
3822 break;
3823 case IB_CM_REP_SENT:
3824 case IB_CM_MRA_REP_RCVD:
3825 cm_reset_to_idle(cm_id_priv);
3826 cm_event.event = IB_CM_REP_ERROR;
3827 break;
3828 case IB_CM_DREQ_SENT:
3829 cm_enter_timewait(cm_id_priv);
3830 cm_event.event = IB_CM_DREQ_ERROR;
3831 break;
3832 case IB_CM_SIDR_REQ_SENT:
3833 cm_id_priv->id.state = IB_CM_IDLE;
3834 cm_event.event = IB_CM_SIDR_REQ_ERROR;
3835 break;
3836 default:
3837 goto discard;
3838 }
3839 spin_unlock_irq(&cm_id_priv->lock);
3840 cm_event.param.send_status = wc_status;
3841
3842 /* No other events can occur on the cm_id at this point. */
3843 ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &cm_event);
3844 cm_free_msg(msg);
3845 if (ret)
3846 ib_destroy_cm_id(&cm_id_priv->id);
3847 return;
3848 discard:
3849 spin_unlock_irq(&cm_id_priv->lock);
3850 cm_free_msg(msg);
3851 }
3852
3853 static void cm_send_handler(struct ib_mad_agent *mad_agent,
3854 struct ib_mad_send_wc *mad_send_wc)
3855 {
3856 struct ib_mad_send_buf *msg = mad_send_wc->send_buf;
3857 struct cm_port *port;
3858 u16 attr_index;
3859
3860 port = mad_agent->context;
3861 attr_index = be16_to_cpu(((struct ib_mad_hdr *)
3862 msg->mad)->attr_id) - CM_ATTR_ID_OFFSET;
3863
3864 /*
3865 * If the send was in response to a received message (context[0] is not
3866 * set to a cm_id), and is not a REJ, then it is a send that was
3867 * manually retried.
3868 */
3869 if (!msg->context[0] && (attr_index != CM_REJ_COUNTER))
3870 msg->retries = 1;
3871
3872 atomic_long_add(1 + msg->retries,
3873 &port->counter_group[CM_XMIT].counter[attr_index]);
3874 if (msg->retries)
3875 atomic_long_add(msg->retries,
3876 &port->counter_group[CM_XMIT_RETRIES].
3877 counter[attr_index]);
3878
3879 switch (mad_send_wc->status) {
3880 case IB_WC_SUCCESS:
3881 case IB_WC_WR_FLUSH_ERR:
3882 cm_free_msg(msg);
3883 break;
3884 default:
3885 if (msg->context[0] && msg->context[1])
3886 cm_process_send_error(msg, mad_send_wc->status);
3887 else
3888 cm_free_msg(msg);
3889 break;
3890 }
3891 }
3892
3893 static void cm_work_handler(struct work_struct *_work)
3894 {
3895 struct cm_work *work = container_of(_work, struct cm_work, work.work);
3896 int ret;
3897
3898 switch (work->cm_event.event) {
3899 case IB_CM_REQ_RECEIVED:
3900 ret = cm_req_handler(work);
3901 break;
3902 case IB_CM_MRA_RECEIVED:
3903 ret = cm_mra_handler(work);
3904 break;
3905 case IB_CM_REJ_RECEIVED:
3906 ret = cm_rej_handler(work);
3907 break;
3908 case IB_CM_REP_RECEIVED:
3909 ret = cm_rep_handler(work);
3910 break;
3911 case IB_CM_RTU_RECEIVED:
3912 ret = cm_rtu_handler(work);
3913 break;
3914 case IB_CM_USER_ESTABLISHED:
3915 ret = cm_establish_handler(work);
3916 break;
3917 case IB_CM_DREQ_RECEIVED:
3918 ret = cm_dreq_handler(work);
3919 break;
3920 case IB_CM_DREP_RECEIVED:
3921 ret = cm_drep_handler(work);
3922 break;
3923 case IB_CM_SIDR_REQ_RECEIVED:
3924 ret = cm_sidr_req_handler(work);
3925 break;
3926 case IB_CM_SIDR_REP_RECEIVED:
3927 ret = cm_sidr_rep_handler(work);
3928 break;
3929 case IB_CM_LAP_RECEIVED:
3930 ret = cm_lap_handler(work);
3931 break;
3932 case IB_CM_APR_RECEIVED:
3933 ret = cm_apr_handler(work);
3934 break;
3935 case IB_CM_TIMEWAIT_EXIT:
3936 ret = cm_timewait_handler(work);
3937 break;
3938 default:
3939 pr_debug("cm_event.event: 0x%x\n", work->cm_event.event);
3940 ret = -EINVAL;
3941 break;
3942 }
3943 if (ret)
3944 cm_free_work(work);
3945 }
3946
3947 static int cm_establish(struct ib_cm_id *cm_id)
3948 {
3949 struct cm_id_private *cm_id_priv;
3950 struct cm_work *work;
3951 unsigned long flags;
3952 int ret = 0;
3953 struct cm_device *cm_dev;
3954
3955 cm_dev = ib_get_client_data(cm_id->device, &cm_client);
3956 if (!cm_dev)
3957 return -ENODEV;
3958
3959 work = kmalloc(sizeof *work, GFP_ATOMIC);
3960 if (!work)
3961 return -ENOMEM;
3962
3963 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3964 spin_lock_irqsave(&cm_id_priv->lock, flags);
3965 switch (cm_id->state)
3966 {
3967 case IB_CM_REP_SENT:
3968 case IB_CM_MRA_REP_RCVD:
3969 cm_id->state = IB_CM_ESTABLISHED;
3970 break;
3971 case IB_CM_ESTABLISHED:
3972 ret = -EISCONN;
3973 break;
3974 default:
3975 pr_debug("%s: local_id %d, cm_id->state: %d\n", __func__,
3976 be32_to_cpu(cm_id->local_id), cm_id->state);
3977 ret = -EINVAL;
3978 break;
3979 }
3980 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3981
3982 if (ret) {
3983 kfree(work);
3984 goto out;
3985 }
3986
3987 /*
3988 * The CM worker thread may try to destroy the cm_id before it
3989 * can execute this work item. To prevent potential deadlock,
3990 * we need to find the cm_id once we're in the context of the
3991 * worker thread, rather than holding a reference on it.
3992 */
3993 INIT_DELAYED_WORK(&work->work, cm_work_handler);
3994 work->local_id = cm_id->local_id;
3995 work->remote_id = cm_id->remote_id;
3996 work->mad_recv_wc = NULL;
3997 work->cm_event.event = IB_CM_USER_ESTABLISHED;
3998
3999 /* Check if the device started its remove_one */
4000 spin_lock_irqsave(&cm.lock, flags);
4001 if (!cm_dev->going_down) {
4002 queue_delayed_work(cm.wq, &work->work, 0);
4003 } else {
4004 kfree(work);
4005 ret = -ENODEV;
4006 }
4007 spin_unlock_irqrestore(&cm.lock, flags);
4008
4009 out:
4010 return ret;
4011 }
4012
4013 static int cm_migrate(struct ib_cm_id *cm_id)
4014 {
4015 struct cm_id_private *cm_id_priv;
4016 struct cm_av tmp_av;
4017 unsigned long flags;
4018 int tmp_send_port_not_ready;
4019 int ret = 0;
4020
4021 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
4022 spin_lock_irqsave(&cm_id_priv->lock, flags);
4023 if (cm_id->state == IB_CM_ESTABLISHED &&
4024 (cm_id->lap_state == IB_CM_LAP_UNINIT ||
4025 cm_id->lap_state == IB_CM_LAP_IDLE)) {
4026 cm_id->lap_state = IB_CM_LAP_IDLE;
4027 /* Swap address vector */
4028 tmp_av = cm_id_priv->av;
4029 cm_id_priv->av = cm_id_priv->alt_av;
4030 cm_id_priv->alt_av = tmp_av;
4031 /* Swap port send ready state */
4032 tmp_send_port_not_ready = cm_id_priv->prim_send_port_not_ready;
4033 cm_id_priv->prim_send_port_not_ready = cm_id_priv->altr_send_port_not_ready;
4034 cm_id_priv->altr_send_port_not_ready = tmp_send_port_not_ready;
4035 } else
4036 ret = -EINVAL;
4037 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
4038
4039 return ret;
4040 }
4041
4042 int ib_cm_notify(struct ib_cm_id *cm_id, enum ib_event_type event)
4043 {
4044 int ret;
4045
4046 switch (event) {
4047 case IB_EVENT_COMM_EST:
4048 ret = cm_establish(cm_id);
4049 break;
4050 case IB_EVENT_PATH_MIG:
4051 ret = cm_migrate(cm_id);
4052 break;
4053 default:
4054 ret = -EINVAL;
4055 }
4056 return ret;
4057 }
4058 EXPORT_SYMBOL(ib_cm_notify);
4059
4060 static void cm_recv_handler(struct ib_mad_agent *mad_agent,
4061 struct ib_mad_send_buf *send_buf,
4062 struct ib_mad_recv_wc *mad_recv_wc)
4063 {
4064 struct cm_port *port = mad_agent->context;
4065 struct cm_work *work;
4066 enum ib_cm_event_type event;
4067 bool alt_path = false;
4068 u16 attr_id;
4069 int paths = 0;
4070 int going_down = 0;
4071
4072 switch (mad_recv_wc->recv_buf.mad->mad_hdr.attr_id) {
4073 case CM_REQ_ATTR_ID:
4074 alt_path = cm_req_has_alt_path((struct cm_req_msg *)
4075 mad_recv_wc->recv_buf.mad);
4076 paths = 1 + (alt_path != 0);
4077 event = IB_CM_REQ_RECEIVED;
4078 break;
4079 case CM_MRA_ATTR_ID:
4080 event = IB_CM_MRA_RECEIVED;
4081 break;
4082 case CM_REJ_ATTR_ID:
4083 event = IB_CM_REJ_RECEIVED;
4084 break;
4085 case CM_REP_ATTR_ID:
4086 event = IB_CM_REP_RECEIVED;
4087 break;
4088 case CM_RTU_ATTR_ID:
4089 event = IB_CM_RTU_RECEIVED;
4090 break;
4091 case CM_DREQ_ATTR_ID:
4092 event = IB_CM_DREQ_RECEIVED;
4093 break;
4094 case CM_DREP_ATTR_ID:
4095 event = IB_CM_DREP_RECEIVED;
4096 break;
4097 case CM_SIDR_REQ_ATTR_ID:
4098 event = IB_CM_SIDR_REQ_RECEIVED;
4099 break;
4100 case CM_SIDR_REP_ATTR_ID:
4101 event = IB_CM_SIDR_REP_RECEIVED;
4102 break;
4103 case CM_LAP_ATTR_ID:
4104 paths = 1;
4105 event = IB_CM_LAP_RECEIVED;
4106 break;
4107 case CM_APR_ATTR_ID:
4108 event = IB_CM_APR_RECEIVED;
4109 break;
4110 default:
4111 ib_free_recv_mad(mad_recv_wc);
4112 return;
4113 }
4114
4115 attr_id = be16_to_cpu(mad_recv_wc->recv_buf.mad->mad_hdr.attr_id);
4116 atomic_long_inc(&port->counter_group[CM_RECV].
4117 counter[attr_id - CM_ATTR_ID_OFFSET]);
4118
4119 work = kmalloc(struct_size(work, path, paths), GFP_KERNEL);
4120 if (!work) {
4121 ib_free_recv_mad(mad_recv_wc);
4122 return;
4123 }
4124
4125 INIT_DELAYED_WORK(&work->work, cm_work_handler);
4126 work->cm_event.event = event;
4127 work->mad_recv_wc = mad_recv_wc;
4128 work->port = port;
4129
4130 /* Check if the device started its remove_one */
4131 spin_lock_irq(&cm.lock);
4132 if (!port->cm_dev->going_down)
4133 queue_delayed_work(cm.wq, &work->work, 0);
4134 else
4135 going_down = 1;
4136 spin_unlock_irq(&cm.lock);
4137
4138 if (going_down) {
4139 kfree(work);
4140 ib_free_recv_mad(mad_recv_wc);
4141 }
4142 }
4143
4144 static int cm_init_qp_init_attr(struct cm_id_private *cm_id_priv,
4145 struct ib_qp_attr *qp_attr,
4146 int *qp_attr_mask)
4147 {
4148 unsigned long flags;
4149 int ret;
4150
4151 spin_lock_irqsave(&cm_id_priv->lock, flags);
4152 switch (cm_id_priv->id.state) {
4153 case IB_CM_REQ_SENT:
4154 case IB_CM_MRA_REQ_RCVD:
4155 case IB_CM_REQ_RCVD:
4156 case IB_CM_MRA_REQ_SENT:
4157 case IB_CM_REP_RCVD:
4158 case IB_CM_MRA_REP_SENT:
4159 case IB_CM_REP_SENT:
4160 case IB_CM_MRA_REP_RCVD:
4161 case IB_CM_ESTABLISHED:
4162 *qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS |
4163 IB_QP_PKEY_INDEX | IB_QP_PORT;
4164 qp_attr->qp_access_flags = IB_ACCESS_REMOTE_WRITE;
4165 if (cm_id_priv->responder_resources)
4166 qp_attr->qp_access_flags |= IB_ACCESS_REMOTE_READ |
4167 IB_ACCESS_REMOTE_ATOMIC;
4168 qp_attr->pkey_index = cm_id_priv->av.pkey_index;
4169 qp_attr->port_num = cm_id_priv->av.port->port_num;
4170 ret = 0;
4171 break;
4172 default:
4173 pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n",
4174 __func__, be32_to_cpu(cm_id_priv->id.local_id),
4175 cm_id_priv->id.state);
4176 ret = -EINVAL;
4177 break;
4178 }
4179 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
4180 return ret;
4181 }
4182
4183 static int cm_init_qp_rtr_attr(struct cm_id_private *cm_id_priv,
4184 struct ib_qp_attr *qp_attr,
4185 int *qp_attr_mask)
4186 {
4187 unsigned long flags;
4188 int ret;
4189
4190 spin_lock_irqsave(&cm_id_priv->lock, flags);
4191 switch (cm_id_priv->id.state) {
4192 case IB_CM_REQ_RCVD:
4193 case IB_CM_MRA_REQ_SENT:
4194 case IB_CM_REP_RCVD:
4195 case IB_CM_MRA_REP_SENT:
4196 case IB_CM_REP_SENT:
4197 case IB_CM_MRA_REP_RCVD:
4198 case IB_CM_ESTABLISHED:
4199 *qp_attr_mask = IB_QP_STATE | IB_QP_AV | IB_QP_PATH_MTU |
4200 IB_QP_DEST_QPN | IB_QP_RQ_PSN;
4201 qp_attr->ah_attr = cm_id_priv->av.ah_attr;
4202 qp_attr->path_mtu = cm_id_priv->path_mtu;
4203 qp_attr->dest_qp_num = be32_to_cpu(cm_id_priv->remote_qpn);
4204 qp_attr->rq_psn = be32_to_cpu(cm_id_priv->rq_psn);
4205 if (cm_id_priv->qp_type == IB_QPT_RC ||
4206 cm_id_priv->qp_type == IB_QPT_XRC_TGT) {
4207 *qp_attr_mask |= IB_QP_MAX_DEST_RD_ATOMIC |
4208 IB_QP_MIN_RNR_TIMER;
4209 qp_attr->max_dest_rd_atomic =
4210 cm_id_priv->responder_resources;
4211 qp_attr->min_rnr_timer = 0;
4212 }
4213 if (rdma_ah_get_dlid(&cm_id_priv->alt_av.ah_attr)) {
4214 *qp_attr_mask |= IB_QP_ALT_PATH;
4215 qp_attr->alt_port_num = cm_id_priv->alt_av.port->port_num;
4216 qp_attr->alt_pkey_index = cm_id_priv->alt_av.pkey_index;
4217 qp_attr->alt_timeout = cm_id_priv->alt_av.timeout;
4218 qp_attr->alt_ah_attr = cm_id_priv->alt_av.ah_attr;
4219 }
4220 ret = 0;
4221 break;
4222 default:
4223 pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n",
4224 __func__, be32_to_cpu(cm_id_priv->id.local_id),
4225 cm_id_priv->id.state);
4226 ret = -EINVAL;
4227 break;
4228 }
4229 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
4230 return ret;
4231 }
4232
4233 static int cm_init_qp_rts_attr(struct cm_id_private *cm_id_priv,
4234 struct ib_qp_attr *qp_attr,
4235 int *qp_attr_mask)
4236 {
4237 unsigned long flags;
4238 int ret;
4239
4240 spin_lock_irqsave(&cm_id_priv->lock, flags);
4241 switch (cm_id_priv->id.state) {
4242 /* Allow transition to RTS before sending REP */
4243 case IB_CM_REQ_RCVD:
4244 case IB_CM_MRA_REQ_SENT:
4245
4246 case IB_CM_REP_RCVD:
4247 case IB_CM_MRA_REP_SENT:
4248 case IB_CM_REP_SENT:
4249 case IB_CM_MRA_REP_RCVD:
4250 case IB_CM_ESTABLISHED:
4251 if (cm_id_priv->id.lap_state == IB_CM_LAP_UNINIT) {
4252 *qp_attr_mask = IB_QP_STATE | IB_QP_SQ_PSN;
4253 qp_attr->sq_psn = be32_to_cpu(cm_id_priv->sq_psn);
4254 switch (cm_id_priv->qp_type) {
4255 case IB_QPT_RC:
4256 case IB_QPT_XRC_INI:
4257 *qp_attr_mask |= IB_QP_RETRY_CNT | IB_QP_RNR_RETRY |
4258 IB_QP_MAX_QP_RD_ATOMIC;
4259 qp_attr->retry_cnt = cm_id_priv->retry_count;
4260 qp_attr->rnr_retry = cm_id_priv->rnr_retry_count;
4261 qp_attr->max_rd_atomic = cm_id_priv->initiator_depth;
4262 /* fall through */
4263 case IB_QPT_XRC_TGT:
4264 *qp_attr_mask |= IB_QP_TIMEOUT;
4265 qp_attr->timeout = cm_id_priv->av.timeout;
4266 break;
4267 default:
4268 break;
4269 }
4270 if (rdma_ah_get_dlid(&cm_id_priv->alt_av.ah_attr)) {
4271 *qp_attr_mask |= IB_QP_PATH_MIG_STATE;
4272 qp_attr->path_mig_state = IB_MIG_REARM;
4273 }
4274 } else {
4275 *qp_attr_mask = IB_QP_ALT_PATH | IB_QP_PATH_MIG_STATE;
4276 qp_attr->alt_port_num = cm_id_priv->alt_av.port->port_num;
4277 qp_attr->alt_pkey_index = cm_id_priv->alt_av.pkey_index;
4278 qp_attr->alt_timeout = cm_id_priv->alt_av.timeout;
4279 qp_attr->alt_ah_attr = cm_id_priv->alt_av.ah_attr;
4280 qp_attr->path_mig_state = IB_MIG_REARM;
4281 }
4282 ret = 0;
4283 break;
4284 default:
4285 pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n",
4286 __func__, be32_to_cpu(cm_id_priv->id.local_id),
4287 cm_id_priv->id.state);
4288 ret = -EINVAL;
4289 break;
4290 }
4291 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
4292 return ret;
4293 }
4294
4295 int ib_cm_init_qp_attr(struct ib_cm_id *cm_id,
4296 struct ib_qp_attr *qp_attr,
4297 int *qp_attr_mask)
4298 {
4299 struct cm_id_private *cm_id_priv;
4300 int ret;
4301
4302 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
4303 switch (qp_attr->qp_state) {
4304 case IB_QPS_INIT:
4305 ret = cm_init_qp_init_attr(cm_id_priv, qp_attr, qp_attr_mask);
4306 break;
4307 case IB_QPS_RTR:
4308 ret = cm_init_qp_rtr_attr(cm_id_priv, qp_attr, qp_attr_mask);
4309 break;
4310 case IB_QPS_RTS:
4311 ret = cm_init_qp_rts_attr(cm_id_priv, qp_attr, qp_attr_mask);
4312 break;
4313 default:
4314 ret = -EINVAL;
4315 break;
4316 }
4317 return ret;
4318 }
4319 EXPORT_SYMBOL(ib_cm_init_qp_attr);
4320
4321 static ssize_t cm_show_counter(struct kobject *obj, struct attribute *attr,
4322 char *buf)
4323 {
4324 struct cm_counter_group *group;
4325 struct cm_counter_attribute *cm_attr;
4326
4327 group = container_of(obj, struct cm_counter_group, obj);
4328 cm_attr = container_of(attr, struct cm_counter_attribute, attr);
4329
4330 return sprintf(buf, "%ld\n",
4331 atomic_long_read(&group->counter[cm_attr->index]));
4332 }
4333
4334 static const struct sysfs_ops cm_counter_ops = {
4335 .show = cm_show_counter
4336 };
4337
4338 static struct kobj_type cm_counter_obj_type = {
4339 .sysfs_ops = &cm_counter_ops,
4340 .default_attrs = cm_counter_default_attrs
4341 };
4342
4343 static char *cm_devnode(struct device *dev, umode_t *mode)
4344 {
4345 if (mode)
4346 *mode = 0666;
4347 return kasprintf(GFP_KERNEL, "infiniband/%s", dev_name(dev));
4348 }
4349
4350 struct class cm_class = {
4351 .owner = THIS_MODULE,
4352 .name = "infiniband_cm",
4353 .devnode = cm_devnode,
4354 };
4355 EXPORT_SYMBOL(cm_class);
4356
4357 static int cm_create_port_fs(struct cm_port *port)
4358 {
4359 int i, ret;
4360
4361 for (i = 0; i < CM_COUNTER_GROUPS; i++) {
4362 ret = ib_port_register_module_stat(port->cm_dev->ib_device,
4363 port->port_num,
4364 &port->counter_group[i].obj,
4365 &cm_counter_obj_type,
4366 counter_group_names[i]);
4367 if (ret)
4368 goto error;
4369 }
4370
4371 return 0;
4372
4373 error:
4374 while (i--)
4375 ib_port_unregister_module_stat(&port->counter_group[i].obj);
4376 return ret;
4377
4378 }
4379
4380 static void cm_remove_port_fs(struct cm_port *port)
4381 {
4382 int i;
4383
4384 for (i = 0; i < CM_COUNTER_GROUPS; i++)
4385 ib_port_unregister_module_stat(&port->counter_group[i].obj);
4386
4387 }
4388
4389 static void cm_add_one(struct ib_device *ib_device)
4390 {
4391 struct cm_device *cm_dev;
4392 struct cm_port *port;
4393 struct ib_mad_reg_req reg_req = {
4394 .mgmt_class = IB_MGMT_CLASS_CM,
4395 .mgmt_class_version = IB_CM_CLASS_VERSION,
4396 };
4397 struct ib_port_modify port_modify = {
4398 .set_port_cap_mask = IB_PORT_CM_SUP
4399 };
4400 unsigned long flags;
4401 int ret;
4402 int count = 0;
4403 u8 i;
4404
4405 cm_dev = kzalloc(struct_size(cm_dev, port, ib_device->phys_port_cnt),
4406 GFP_KERNEL);
4407 if (!cm_dev)
4408 return;
4409
4410 cm_dev->ib_device = ib_device;
4411 cm_dev->ack_delay = ib_device->attrs.local_ca_ack_delay;
4412 cm_dev->going_down = 0;
4413
4414 set_bit(IB_MGMT_METHOD_SEND, reg_req.method_mask);
4415 for (i = 1; i <= ib_device->phys_port_cnt; i++) {
4416 if (!rdma_cap_ib_cm(ib_device, i))
4417 continue;
4418
4419 port = kzalloc(sizeof *port, GFP_KERNEL);
4420 if (!port)
4421 goto error1;
4422
4423 cm_dev->port[i-1] = port;
4424 port->cm_dev = cm_dev;
4425 port->port_num = i;
4426
4427 INIT_LIST_HEAD(&port->cm_priv_prim_list);
4428 INIT_LIST_HEAD(&port->cm_priv_altr_list);
4429
4430 ret = cm_create_port_fs(port);
4431 if (ret)
4432 goto error1;
4433
4434 port->mad_agent = ib_register_mad_agent(ib_device, i,
4435 IB_QPT_GSI,
4436 &reg_req,
4437 0,
4438 cm_send_handler,
4439 cm_recv_handler,
4440 port,
4441 0);
4442 if (IS_ERR(port->mad_agent))
4443 goto error2;
4444
4445 ret = ib_modify_port(ib_device, i, 0, &port_modify);
4446 if (ret)
4447 goto error3;
4448
4449 count++;
4450 }
4451
4452 if (!count)
4453 goto free;
4454
4455 ib_set_client_data(ib_device, &cm_client, cm_dev);
4456
4457 write_lock_irqsave(&cm.device_lock, flags);
4458 list_add_tail(&cm_dev->list, &cm.device_list);
4459 write_unlock_irqrestore(&cm.device_lock, flags);
4460 return;
4461
4462 error3:
4463 ib_unregister_mad_agent(port->mad_agent);
4464 error2:
4465 cm_remove_port_fs(port);
4466 error1:
4467 port_modify.set_port_cap_mask = 0;
4468 port_modify.clr_port_cap_mask = IB_PORT_CM_SUP;
4469 kfree(port);
4470 while (--i) {
4471 if (!rdma_cap_ib_cm(ib_device, i))
4472 continue;
4473
4474 port = cm_dev->port[i-1];
4475 ib_modify_port(ib_device, port->port_num, 0, &port_modify);
4476 ib_unregister_mad_agent(port->mad_agent);
4477 cm_remove_port_fs(port);
4478 kfree(port);
4479 }
4480 free:
4481 kfree(cm_dev);
4482 }
4483
4484 static void cm_remove_one(struct ib_device *ib_device, void *client_data)
4485 {
4486 struct cm_device *cm_dev = client_data;
4487 struct cm_port *port;
4488 struct cm_id_private *cm_id_priv;
4489 struct ib_mad_agent *cur_mad_agent;
4490 struct ib_port_modify port_modify = {
4491 .clr_port_cap_mask = IB_PORT_CM_SUP
4492 };
4493 unsigned long flags;
4494 int i;
4495
4496 if (!cm_dev)
4497 return;
4498
4499 write_lock_irqsave(&cm.device_lock, flags);
4500 list_del(&cm_dev->list);
4501 write_unlock_irqrestore(&cm.device_lock, flags);
4502
4503 spin_lock_irq(&cm.lock);
4504 cm_dev->going_down = 1;
4505 spin_unlock_irq(&cm.lock);
4506
4507 for (i = 1; i <= ib_device->phys_port_cnt; i++) {
4508 if (!rdma_cap_ib_cm(ib_device, i))
4509 continue;
4510
4511 port = cm_dev->port[i-1];
4512 ib_modify_port(ib_device, port->port_num, 0, &port_modify);
4513 /* Mark all the cm_id's as not valid */
4514 spin_lock_irq(&cm.lock);
4515 list_for_each_entry(cm_id_priv, &port->cm_priv_altr_list, altr_list)
4516 cm_id_priv->altr_send_port_not_ready = 1;
4517 list_for_each_entry(cm_id_priv, &port->cm_priv_prim_list, prim_list)
4518 cm_id_priv->prim_send_port_not_ready = 1;
4519 spin_unlock_irq(&cm.lock);
4520 /*
4521 * We flush the queue here after the going_down set, this
4522 * verify that no new works will be queued in the recv handler,
4523 * after that we can call the unregister_mad_agent
4524 */
4525 flush_workqueue(cm.wq);
4526 spin_lock_irq(&cm.state_lock);
4527 cur_mad_agent = port->mad_agent;
4528 port->mad_agent = NULL;
4529 spin_unlock_irq(&cm.state_lock);
4530 ib_unregister_mad_agent(cur_mad_agent);
4531 cm_remove_port_fs(port);
4532 kfree(port);
4533 }
4534
4535 kfree(cm_dev);
4536 }
4537
4538 static int __init ib_cm_init(void)
4539 {
4540 int ret;
4541
4542 INIT_LIST_HEAD(&cm.device_list);
4543 rwlock_init(&cm.device_lock);
4544 spin_lock_init(&cm.lock);
4545 spin_lock_init(&cm.state_lock);
4546 cm.listen_service_table = RB_ROOT;
4547 cm.listen_service_id = be64_to_cpu(IB_CM_ASSIGN_SERVICE_ID);
4548 cm.remote_id_table = RB_ROOT;
4549 cm.remote_qp_table = RB_ROOT;
4550 cm.remote_sidr_table = RB_ROOT;
4551 xa_init_flags(&cm.local_id_table, XA_FLAGS_ALLOC | XA_FLAGS_LOCK_IRQ);
4552 get_random_bytes(&cm.random_id_operand, sizeof cm.random_id_operand);
4553 INIT_LIST_HEAD(&cm.timewait_list);
4554
4555 ret = class_register(&cm_class);
4556 if (ret) {
4557 ret = -ENOMEM;
4558 goto error1;
4559 }
4560
4561 cm.wq = alloc_workqueue("ib_cm", 0, 1);
4562 if (!cm.wq) {
4563 ret = -ENOMEM;
4564 goto error2;
4565 }
4566
4567 ret = ib_register_client(&cm_client);
4568 if (ret)
4569 goto error3;
4570
4571 return 0;
4572 error3:
4573 destroy_workqueue(cm.wq);
4574 error2:
4575 class_unregister(&cm_class);
4576 error1:
4577 return ret;
4578 }
4579
4580 static void __exit ib_cm_cleanup(void)
4581 {
4582 struct cm_timewait_info *timewait_info, *tmp;
4583
4584 spin_lock_irq(&cm.lock);
4585 list_for_each_entry(timewait_info, &cm.timewait_list, list)
4586 cancel_delayed_work(&timewait_info->work.work);
4587 spin_unlock_irq(&cm.lock);
4588
4589 ib_unregister_client(&cm_client);
4590 destroy_workqueue(cm.wq);
4591
4592 list_for_each_entry_safe(timewait_info, tmp, &cm.timewait_list, list) {
4593 list_del(&timewait_info->list);
4594 kfree(timewait_info);
4595 }
4596
4597 class_unregister(&cm_class);
4598 WARN_ON(!xa_empty(&cm.local_id_table));
4599 }
4600
4601 module_init(ib_cm_init);
4602 module_exit(ib_cm_cleanup);
Linux with added FPC-III support