?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
perf tools: Don't clone maps from parent when synthesizing forks
[linux/fpc-iii.git] / net / smc / af_smc.c
blob80e2119f1c7010ead3c493c60c511c125d6c7149
1 /*
2 * Shared Memory Communications over RDMA (SMC-R) and RoCE
3 *
4 * AF_SMC protocol family socket handler keeping the AF_INET sock address type
5 * applies to SOCK_STREAM sockets only
6 * offers an alternative communication option for TCP-protocol sockets
7 * applicable with RoCE-cards only
8 *
9 * Initial restrictions:
10 * - support for alternate links postponed
11 *
12 * Copyright IBM Corp. 2016, 2018
13 *
14 * Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com>
15 * based on prototype from Frank Blaschka
16 */
17
18 #define KMSG_COMPONENT "smc"
19 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
20
21 #include <linux/module.h>
22 #include <linux/socket.h>
23 #include <linux/workqueue.h>
24 #include <linux/in.h>
25 #include <linux/sched/signal.h>
26 #include <linux/if_vlan.h>
27
28 #include <net/sock.h>
29 #include <net/tcp.h>
30 #include <net/smc.h>
31 #include <asm/ioctls.h>
32
33 #include "smc.h"
34 #include "smc_clc.h"
35 #include "smc_llc.h"
36 #include "smc_cdc.h"
37 #include "smc_core.h"
38 #include "smc_ib.h"
39 #include "smc_ism.h"
40 #include "smc_pnet.h"
41 #include "smc_tx.h"
42 #include "smc_rx.h"
43 #include "smc_close.h"
44
45 static DEFINE_MUTEX(smc_create_lgr_pending); /* serialize link group
46 * creation
47 */
48
49 static void smc_tcp_listen_work(struct work_struct *);
50 static void smc_connect_work(struct work_struct *);
51
52 static void smc_set_keepalive(struct sock *sk, int val)
53 {
54 struct smc_sock *smc = smc_sk(sk);
55
56 smc->clcsock->sk->sk_prot->keepalive(smc->clcsock->sk, val);
57 }
58
59 static struct smc_hashinfo smc_v4_hashinfo = {
60 .lock = __RW_LOCK_UNLOCKED(smc_v4_hashinfo.lock),
61 };
62
63 static struct smc_hashinfo smc_v6_hashinfo = {
64 .lock = __RW_LOCK_UNLOCKED(smc_v6_hashinfo.lock),
65 };
66
67 int smc_hash_sk(struct sock *sk)
68 {
69 struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
70 struct hlist_head *head;
71
72 head = &h->ht;
73
74 write_lock_bh(&h->lock);
75 sk_add_node(sk, head);
76 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
77 write_unlock_bh(&h->lock);
78
79 return 0;
80 }
81 EXPORT_SYMBOL_GPL(smc_hash_sk);
82
83 void smc_unhash_sk(struct sock *sk)
84 {
85 struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
86
87 write_lock_bh(&h->lock);
88 if (sk_del_node_init(sk))
89 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
90 write_unlock_bh(&h->lock);
91 }
92 EXPORT_SYMBOL_GPL(smc_unhash_sk);
93
94 struct proto smc_proto = {
95 .name = "SMC",
96 .owner = THIS_MODULE,
97 .keepalive = smc_set_keepalive,
98 .hash = smc_hash_sk,
99 .unhash = smc_unhash_sk,
100 .obj_size = sizeof(struct smc_sock),
101 .h.smc_hash = &smc_v4_hashinfo,
102 .slab_flags = SLAB_TYPESAFE_BY_RCU,
103 };
104 EXPORT_SYMBOL_GPL(smc_proto);
105
106 struct proto smc_proto6 = {
107 .name = "SMC6",
108 .owner = THIS_MODULE,
109 .keepalive = smc_set_keepalive,
110 .hash = smc_hash_sk,
111 .unhash = smc_unhash_sk,
112 .obj_size = sizeof(struct smc_sock),
113 .h.smc_hash = &smc_v6_hashinfo,
114 .slab_flags = SLAB_TYPESAFE_BY_RCU,
115 };
116 EXPORT_SYMBOL_GPL(smc_proto6);
117
118 static int smc_release(struct socket *sock)
119 {
120 struct sock *sk = sock->sk;
121 struct smc_sock *smc;
122 int rc = 0;
123
124 if (!sk)
125 goto out;
126
127 smc = smc_sk(sk);
128
129 /* cleanup for a dangling non-blocking connect */
130 flush_work(&smc->connect_work);
131 kfree(smc->connect_info);
132 smc->connect_info = NULL;
133
134 if (sk->sk_state == SMC_LISTEN)
135 /* smc_close_non_accepted() is called and acquires
136 * sock lock for child sockets again
137 */
138 lock_sock_nested(sk, SINGLE_DEPTH_NESTING);
139 else
140 lock_sock(sk);
141
142 if (!smc->use_fallback) {
143 rc = smc_close_active(smc);
144 sock_set_flag(sk, SOCK_DEAD);
145 sk->sk_shutdown |= SHUTDOWN_MASK;
146 }
147 if (smc->clcsock) {
148 sock_release(smc->clcsock);
149 smc->clcsock = NULL;
150 }
151 if (smc->use_fallback) {
152 if (sk->sk_state != SMC_LISTEN && sk->sk_state != SMC_INIT)
153 sock_put(sk); /* passive closing */
154 sk->sk_state = SMC_CLOSED;
155 sk->sk_state_change(sk);
156 }
157
158 /* detach socket */
159 sock_orphan(sk);
160 sock->sk = NULL;
161 if (!smc->use_fallback && sk->sk_state == SMC_CLOSED)
162 smc_conn_free(&smc->conn);
163 release_sock(sk);
164
165 sk->sk_prot->unhash(sk);
166 sock_put(sk); /* final sock_put */
167 out:
168 return rc;
169 }
170
171 static void smc_destruct(struct sock *sk)
172 {
173 if (sk->sk_state != SMC_CLOSED)
174 return;
175 if (!sock_flag(sk, SOCK_DEAD))
176 return;
177
178 sk_refcnt_debug_dec(sk);
179 }
180
181 static struct sock *smc_sock_alloc(struct net *net, struct socket *sock,
182 int protocol)
183 {
184 struct smc_sock *smc;
185 struct proto *prot;
186 struct sock *sk;
187
188 prot = (protocol == SMCPROTO_SMC6) ? &smc_proto6 : &smc_proto;
189 sk = sk_alloc(net, PF_SMC, GFP_KERNEL, prot, 0);
190 if (!sk)
191 return NULL;
192
193 sock_init_data(sock, sk); /* sets sk_refcnt to 1 */
194 sk->sk_state = SMC_INIT;
195 sk->sk_destruct = smc_destruct;
196 sk->sk_protocol = protocol;
197 smc = smc_sk(sk);
198 INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work);
199 INIT_WORK(&smc->connect_work, smc_connect_work);
200 INIT_DELAYED_WORK(&smc->conn.tx_work, smc_tx_work);
201 INIT_LIST_HEAD(&smc->accept_q);
202 spin_lock_init(&smc->accept_q_lock);
203 spin_lock_init(&smc->conn.send_lock);
204 sk->sk_prot->hash(sk);
205 sk_refcnt_debug_inc(sk);
206
207 return sk;
208 }
209
210 static int smc_bind(struct socket *sock, struct sockaddr *uaddr,
211 int addr_len)
212 {
213 struct sockaddr_in *addr = (struct sockaddr_in *)uaddr;
214 struct sock *sk = sock->sk;
215 struct smc_sock *smc;
216 int rc;
217
218 smc = smc_sk(sk);
219
220 /* replicate tests from inet_bind(), to be safe wrt. future changes */
221 rc = -EINVAL;
222 if (addr_len < sizeof(struct sockaddr_in))
223 goto out;
224
225 rc = -EAFNOSUPPORT;
226 if (addr->sin_family != AF_INET &&
227 addr->sin_family != AF_INET6 &&
228 addr->sin_family != AF_UNSPEC)
229 goto out;
230 /* accept AF_UNSPEC (mapped to AF_INET) only if s_addr is INADDR_ANY */
231 if (addr->sin_family == AF_UNSPEC &&
232 addr->sin_addr.s_addr != htonl(INADDR_ANY))
233 goto out;
234
235 lock_sock(sk);
236
237 /* Check if socket is already active */
238 rc = -EINVAL;
239 if (sk->sk_state != SMC_INIT)
240 goto out_rel;
241
242 smc->clcsock->sk->sk_reuse = sk->sk_reuse;
243 rc = kernel_bind(smc->clcsock, uaddr, addr_len);
244
245 out_rel:
246 release_sock(sk);
247 out:
248 return rc;
249 }
250
251 static void smc_copy_sock_settings(struct sock *nsk, struct sock *osk,
252 unsigned long mask)
253 {
254 /* options we don't get control via setsockopt for */
255 nsk->sk_type = osk->sk_type;
256 nsk->sk_sndbuf = osk->sk_sndbuf;
257 nsk->sk_rcvbuf = osk->sk_rcvbuf;
258 nsk->sk_sndtimeo = osk->sk_sndtimeo;
259 nsk->sk_rcvtimeo = osk->sk_rcvtimeo;
260 nsk->sk_mark = osk->sk_mark;
261 nsk->sk_priority = osk->sk_priority;
262 nsk->sk_rcvlowat = osk->sk_rcvlowat;
263 nsk->sk_bound_dev_if = osk->sk_bound_dev_if;
264 nsk->sk_err = osk->sk_err;
265
266 nsk->sk_flags &= ~mask;
267 nsk->sk_flags |= osk->sk_flags & mask;
268 }
269
270 #define SK_FLAGS_SMC_TO_CLC ((1UL << SOCK_URGINLINE) | \
271 (1UL << SOCK_KEEPOPEN) | \
272 (1UL << SOCK_LINGER) | \
273 (1UL << SOCK_BROADCAST) | \
274 (1UL << SOCK_TIMESTAMP) | \
275 (1UL << SOCK_DBG) | \
276 (1UL << SOCK_RCVTSTAMP) | \
277 (1UL << SOCK_RCVTSTAMPNS) | \
278 (1UL << SOCK_LOCALROUTE) | \
279 (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE) | \
280 (1UL << SOCK_RXQ_OVFL) | \
281 (1UL << SOCK_WIFI_STATUS) | \
282 (1UL << SOCK_NOFCS) | \
283 (1UL << SOCK_FILTER_LOCKED))
284 /* copy only relevant settings and flags of SOL_SOCKET level from smc to
285 * clc socket (since smc is not called for these options from net/core)
286 */
287 static void smc_copy_sock_settings_to_clc(struct smc_sock *smc)
288 {
289 smc_copy_sock_settings(smc->clcsock->sk, &smc->sk, SK_FLAGS_SMC_TO_CLC);
290 }
291
292 #define SK_FLAGS_CLC_TO_SMC ((1UL << SOCK_URGINLINE) | \
293 (1UL << SOCK_KEEPOPEN) | \
294 (1UL << SOCK_LINGER) | \
295 (1UL << SOCK_DBG))
296 /* copy only settings and flags relevant for smc from clc to smc socket */
297 static void smc_copy_sock_settings_to_smc(struct smc_sock *smc)
298 {
299 smc_copy_sock_settings(&smc->sk, smc->clcsock->sk, SK_FLAGS_CLC_TO_SMC);
300 }
301
302 /* register a new rmb, optionally send confirm_rkey msg to register with peer */
303 static int smc_reg_rmb(struct smc_link *link, struct smc_buf_desc *rmb_desc,
304 bool conf_rkey)
305 {
306 /* register memory region for new rmb */
307 if (smc_wr_reg_send(link, rmb_desc->mr_rx[SMC_SINGLE_LINK])) {
308 rmb_desc->regerr = 1;
309 return -EFAULT;
310 }
311 if (!conf_rkey)
312 return 0;
313 /* exchange confirm_rkey msg with peer */
314 if (smc_llc_do_confirm_rkey(link, rmb_desc)) {
315 rmb_desc->regerr = 1;
316 return -EFAULT;
317 }
318 return 0;
319 }
320
321 static int smc_clnt_conf_first_link(struct smc_sock *smc)
322 {
323 struct net *net = sock_net(smc->clcsock->sk);
324 struct smc_link_group *lgr = smc->conn.lgr;
325 struct smc_link *link;
326 int rest;
327 int rc;
328
329 link = &lgr->lnk[SMC_SINGLE_LINK];
330 /* receive CONFIRM LINK request from server over RoCE fabric */
331 rest = wait_for_completion_interruptible_timeout(
332 &link->llc_confirm,
333 SMC_LLC_WAIT_FIRST_TIME);
334 if (rest <= 0) {
335 struct smc_clc_msg_decline dclc;
336
337 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
338 SMC_CLC_DECLINE);
339 return rc;
340 }
341
342 if (link->llc_confirm_rc)
343 return SMC_CLC_DECL_RMBE_EC;
344
345 rc = smc_ib_modify_qp_rts(link);
346 if (rc)
347 return SMC_CLC_DECL_ERR_RDYLNK;
348
349 smc_wr_remember_qp_attr(link);
350
351 if (smc_reg_rmb(link, smc->conn.rmb_desc, false))
352 return SMC_CLC_DECL_ERR_REGRMB;
353
354 /* send CONFIRM LINK response over RoCE fabric */
355 rc = smc_llc_send_confirm_link(link, SMC_LLC_RESP);
356 if (rc < 0)
357 return SMC_CLC_DECL_TIMEOUT_CL;
358
359 /* receive ADD LINK request from server over RoCE fabric */
360 rest = wait_for_completion_interruptible_timeout(&link->llc_add,
361 SMC_LLC_WAIT_TIME);
362 if (rest <= 0) {
363 struct smc_clc_msg_decline dclc;
364
365 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
366 SMC_CLC_DECLINE);
367 return rc;
368 }
369
370 /* send add link reject message, only one link supported for now */
371 rc = smc_llc_send_add_link(link,
372 link->smcibdev->mac[link->ibport - 1],
373 link->gid, SMC_LLC_RESP);
374 if (rc < 0)
375 return SMC_CLC_DECL_TIMEOUT_AL;
376
377 smc_llc_link_active(link, net->ipv4.sysctl_tcp_keepalive_time);
378
379 return 0;
380 }
381
382 static void smcr_conn_save_peer_info(struct smc_sock *smc,
383 struct smc_clc_msg_accept_confirm *clc)
384 {
385 int bufsize = smc_uncompress_bufsize(clc->rmbe_size);
386
387 smc->conn.peer_rmbe_idx = clc->rmbe_idx;
388 smc->conn.local_tx_ctrl.token = ntohl(clc->rmbe_alert_token);
389 smc->conn.peer_rmbe_size = bufsize;
390 atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
391 smc->conn.tx_off = bufsize * (smc->conn.peer_rmbe_idx - 1);
392 }
393
394 static void smcd_conn_save_peer_info(struct smc_sock *smc,
395 struct smc_clc_msg_accept_confirm *clc)
396 {
397 int bufsize = smc_uncompress_bufsize(clc->dmbe_size);
398
399 smc->conn.peer_rmbe_idx = clc->dmbe_idx;
400 smc->conn.peer_token = clc->token;
401 /* msg header takes up space in the buffer */
402 smc->conn.peer_rmbe_size = bufsize - sizeof(struct smcd_cdc_msg);
403 atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
404 smc->conn.tx_off = bufsize * smc->conn.peer_rmbe_idx;
405 }
406
407 static void smc_conn_save_peer_info(struct smc_sock *smc,
408 struct smc_clc_msg_accept_confirm *clc)
409 {
410 if (smc->conn.lgr->is_smcd)
411 smcd_conn_save_peer_info(smc, clc);
412 else
413 smcr_conn_save_peer_info(smc, clc);
414 }
415
416 static void smc_link_save_peer_info(struct smc_link *link,
417 struct smc_clc_msg_accept_confirm *clc)
418 {
419 link->peer_qpn = ntoh24(clc->qpn);
420 memcpy(link->peer_gid, clc->lcl.gid, SMC_GID_SIZE);
421 memcpy(link->peer_mac, clc->lcl.mac, sizeof(link->peer_mac));
422 link->peer_psn = ntoh24(clc->psn);
423 link->peer_mtu = clc->qp_mtu;
424 }
425
426 /* fall back during connect */
427 static int smc_connect_fallback(struct smc_sock *smc, int reason_code)
428 {
429 smc->use_fallback = true;
430 smc->fallback_rsn = reason_code;
431 smc_copy_sock_settings_to_clc(smc);
432 if (smc->sk.sk_state == SMC_INIT)
433 smc->sk.sk_state = SMC_ACTIVE;
434 return 0;
435 }
436
437 /* decline and fall back during connect */
438 static int smc_connect_decline_fallback(struct smc_sock *smc, int reason_code)
439 {
440 int rc;
441
442 if (reason_code < 0) { /* error, fallback is not possible */
443 if (smc->sk.sk_state == SMC_INIT)
444 sock_put(&smc->sk); /* passive closing */
445 return reason_code;
446 }
447 if (reason_code != SMC_CLC_DECL_PEERDECL) {
448 rc = smc_clc_send_decline(smc, reason_code);
449 if (rc < 0) {
450 if (smc->sk.sk_state == SMC_INIT)
451 sock_put(&smc->sk); /* passive closing */
452 return rc;
453 }
454 }
455 return smc_connect_fallback(smc, reason_code);
456 }
457
458 /* abort connecting */
459 static int smc_connect_abort(struct smc_sock *smc, int reason_code,
460 int local_contact)
461 {
462 if (local_contact == SMC_FIRST_CONTACT)
463 smc_lgr_forget(smc->conn.lgr);
464 mutex_unlock(&smc_create_lgr_pending);
465 smc_conn_free(&smc->conn);
466 return reason_code;
467 }
468
469 /* check if there is a rdma device available for this connection. */
470 /* called for connect and listen */
471 static int smc_check_rdma(struct smc_sock *smc, struct smc_ib_device **ibdev,
472 u8 *ibport, unsigned short vlan_id, u8 gid[])
473 {
474 int reason_code = 0;
475
476 /* PNET table look up: search active ib_device and port
477 * within same PNETID that also contains the ethernet device
478 * used for the internal TCP socket
479 */
480 smc_pnet_find_roce_resource(smc->clcsock->sk, ibdev, ibport, vlan_id,
481 gid);
482 if (!(*ibdev))
483 reason_code = SMC_CLC_DECL_CNFERR; /* configuration error */
484
485 return reason_code;
486 }
487
488 /* check if there is an ISM device available for this connection. */
489 /* called for connect and listen */
490 static int smc_check_ism(struct smc_sock *smc, struct smcd_dev **ismdev)
491 {
492 /* Find ISM device with same PNETID as connecting interface */
493 smc_pnet_find_ism_resource(smc->clcsock->sk, ismdev);
494 if (!(*ismdev))
495 return SMC_CLC_DECL_CNFERR; /* configuration error */
496 return 0;
497 }
498
499 /* Check for VLAN ID and register it on ISM device just for CLC handshake */
500 static int smc_connect_ism_vlan_setup(struct smc_sock *smc,
501 struct smcd_dev *ismdev,
502 unsigned short vlan_id)
503 {
504 if (vlan_id && smc_ism_get_vlan(ismdev, vlan_id))
505 return SMC_CLC_DECL_CNFERR;
506 return 0;
507 }
508
509 /* cleanup temporary VLAN ID registration used for CLC handshake. If ISM is
510 * used, the VLAN ID will be registered again during the connection setup.
511 */
512 static int smc_connect_ism_vlan_cleanup(struct smc_sock *smc, bool is_smcd,
513 struct smcd_dev *ismdev,
514 unsigned short vlan_id)
515 {
516 if (!is_smcd)
517 return 0;
518 if (vlan_id && smc_ism_put_vlan(ismdev, vlan_id))
519 return SMC_CLC_DECL_CNFERR;
520 return 0;
521 }
522
523 /* CLC handshake during connect */
524 static int smc_connect_clc(struct smc_sock *smc, int smc_type,
525 struct smc_clc_msg_accept_confirm *aclc,
526 struct smc_ib_device *ibdev, u8 ibport,
527 u8 gid[], struct smcd_dev *ismdev)
528 {
529 int rc = 0;
530
531 /* do inband token exchange */
532 rc = smc_clc_send_proposal(smc, smc_type, ibdev, ibport, gid, ismdev);
533 if (rc)
534 return rc;
535 /* receive SMC Accept CLC message */
536 return smc_clc_wait_msg(smc, aclc, sizeof(*aclc), SMC_CLC_ACCEPT);
537 }
538
539 /* setup for RDMA connection of client */
540 static int smc_connect_rdma(struct smc_sock *smc,
541 struct smc_clc_msg_accept_confirm *aclc,
542 struct smc_ib_device *ibdev, u8 ibport)
543 {
544 int local_contact = SMC_FIRST_CONTACT;
545 struct smc_link *link;
546 int reason_code = 0;
547
548 mutex_lock(&smc_create_lgr_pending);
549 local_contact = smc_conn_create(smc, false, aclc->hdr.flag, ibdev,
550 ibport, &aclc->lcl, NULL, 0);
551 if (local_contact < 0) {
552 if (local_contact == -ENOMEM)
553 reason_code = SMC_CLC_DECL_MEM;/* insufficient memory*/
554 else if (local_contact == -ENOLINK)
555 reason_code = SMC_CLC_DECL_SYNCERR; /* synchr. error */
556 else
557 reason_code = SMC_CLC_DECL_INTERR; /* other error */
558 return smc_connect_abort(smc, reason_code, 0);
559 }
560 link = &smc->conn.lgr->lnk[SMC_SINGLE_LINK];
561
562 smc_conn_save_peer_info(smc, aclc);
563
564 /* create send buffer and rmb */
565 if (smc_buf_create(smc, false))
566 return smc_connect_abort(smc, SMC_CLC_DECL_MEM, local_contact);
567
568 if (local_contact == SMC_FIRST_CONTACT)
569 smc_link_save_peer_info(link, aclc);
570
571 if (smc_rmb_rtoken_handling(&smc->conn, aclc))
572 return smc_connect_abort(smc, SMC_CLC_DECL_ERR_RTOK,
573 local_contact);
574
575 smc_close_init(smc);
576 smc_rx_init(smc);
577
578 if (local_contact == SMC_FIRST_CONTACT) {
579 if (smc_ib_ready_link(link))
580 return smc_connect_abort(smc, SMC_CLC_DECL_ERR_RDYLNK,
581 local_contact);
582 } else {
583 if (!smc->conn.rmb_desc->reused &&
584 smc_reg_rmb(link, smc->conn.rmb_desc, true))
585 return smc_connect_abort(smc, SMC_CLC_DECL_ERR_REGRMB,
586 local_contact);
587 }
588 smc_rmb_sync_sg_for_device(&smc->conn);
589
590 reason_code = smc_clc_send_confirm(smc);
591 if (reason_code)
592 return smc_connect_abort(smc, reason_code, local_contact);
593
594 smc_tx_init(smc);
595
596 if (local_contact == SMC_FIRST_CONTACT) {
597 /* QP confirmation over RoCE fabric */
598 reason_code = smc_clnt_conf_first_link(smc);
599 if (reason_code)
600 return smc_connect_abort(smc, reason_code,
601 local_contact);
602 }
603 mutex_unlock(&smc_create_lgr_pending);
604
605 smc_copy_sock_settings_to_clc(smc);
606 if (smc->sk.sk_state == SMC_INIT)
607 smc->sk.sk_state = SMC_ACTIVE;
608
609 return 0;
610 }
611
612 /* setup for ISM connection of client */
613 static int smc_connect_ism(struct smc_sock *smc,
614 struct smc_clc_msg_accept_confirm *aclc,
615 struct smcd_dev *ismdev)
616 {
617 int local_contact = SMC_FIRST_CONTACT;
618 int rc = 0;
619
620 mutex_lock(&smc_create_lgr_pending);
621 local_contact = smc_conn_create(smc, true, aclc->hdr.flag, NULL, 0,
622 NULL, ismdev, aclc->gid);
623 if (local_contact < 0)
624 return smc_connect_abort(smc, SMC_CLC_DECL_MEM, 0);
625
626 /* Create send and receive buffers */
627 if (smc_buf_create(smc, true))
628 return smc_connect_abort(smc, SMC_CLC_DECL_MEM, local_contact);
629
630 smc_conn_save_peer_info(smc, aclc);
631 smc_close_init(smc);
632 smc_rx_init(smc);
633 smc_tx_init(smc);
634
635 rc = smc_clc_send_confirm(smc);
636 if (rc)
637 return smc_connect_abort(smc, rc, local_contact);
638 mutex_unlock(&smc_create_lgr_pending);
639
640 smc_copy_sock_settings_to_clc(smc);
641 if (smc->sk.sk_state == SMC_INIT)
642 smc->sk.sk_state = SMC_ACTIVE;
643
644 return 0;
645 }
646
647 /* perform steps before actually connecting */
648 static int __smc_connect(struct smc_sock *smc)
649 {
650 bool ism_supported = false, rdma_supported = false;
651 struct smc_clc_msg_accept_confirm aclc;
652 struct smc_ib_device *ibdev;
653 struct smcd_dev *ismdev;
654 u8 gid[SMC_GID_SIZE];
655 unsigned short vlan;
656 int smc_type;
657 int rc = 0;
658 u8 ibport;
659
660 sock_hold(&smc->sk); /* sock put in passive closing */
661
662 if (smc->use_fallback)
663 return smc_connect_fallback(smc, smc->fallback_rsn);
664
665 /* if peer has not signalled SMC-capability, fall back */
666 if (!tcp_sk(smc->clcsock->sk)->syn_smc)
667 return smc_connect_fallback(smc, SMC_CLC_DECL_PEERNOSMC);
668
669 /* IPSec connections opt out of SMC-R optimizations */
670 if (using_ipsec(smc))
671 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_IPSEC);
672
673 /* check for VLAN ID */
674 if (smc_vlan_by_tcpsk(smc->clcsock, &vlan))
675 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_CNFERR);
676
677 /* check if there is an ism device available */
678 if (!smc_check_ism(smc, &ismdev) &&
679 !smc_connect_ism_vlan_setup(smc, ismdev, vlan)) {
680 /* ISM is supported for this connection */
681 ism_supported = true;
682 smc_type = SMC_TYPE_D;
683 }
684
685 /* check if there is a rdma device available */
686 if (!smc_check_rdma(smc, &ibdev, &ibport, vlan, gid)) {
687 /* RDMA is supported for this connection */
688 rdma_supported = true;
689 if (ism_supported)
690 smc_type = SMC_TYPE_B; /* both */
691 else
692 smc_type = SMC_TYPE_R; /* only RDMA */
693 }
694
695 /* if neither ISM nor RDMA are supported, fallback */
696 if (!rdma_supported && !ism_supported)
697 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_NOSMCDEV);
698
699 /* perform CLC handshake */
700 rc = smc_connect_clc(smc, smc_type, &aclc, ibdev, ibport, gid, ismdev);
701 if (rc) {
702 smc_connect_ism_vlan_cleanup(smc, ism_supported, ismdev, vlan);
703 return smc_connect_decline_fallback(smc, rc);
704 }
705
706 /* depending on previous steps, connect using rdma or ism */
707 if (rdma_supported && aclc.hdr.path == SMC_TYPE_R)
708 rc = smc_connect_rdma(smc, &aclc, ibdev, ibport);
709 else if (ism_supported && aclc.hdr.path == SMC_TYPE_D)
710 rc = smc_connect_ism(smc, &aclc, ismdev);
711 else
712 rc = SMC_CLC_DECL_MODEUNSUPP;
713 if (rc) {
714 smc_connect_ism_vlan_cleanup(smc, ism_supported, ismdev, vlan);
715 return smc_connect_decline_fallback(smc, rc);
716 }
717
718 smc_connect_ism_vlan_cleanup(smc, ism_supported, ismdev, vlan);
719 return 0;
720 }
721
722 static void smc_connect_work(struct work_struct *work)
723 {
724 struct smc_sock *smc = container_of(work, struct smc_sock,
725 connect_work);
726 int rc;
727
728 lock_sock(&smc->sk);
729 rc = kernel_connect(smc->clcsock, &smc->connect_info->addr,
730 smc->connect_info->alen, smc->connect_info->flags);
731 if (smc->clcsock->sk->sk_err) {
732 smc->sk.sk_err = smc->clcsock->sk->sk_err;
733 goto out;
734 }
735 if (rc < 0) {
736 smc->sk.sk_err = -rc;
737 goto out;
738 }
739
740 rc = __smc_connect(smc);
741 if (rc < 0)
742 smc->sk.sk_err = -rc;
743
744 out:
745 if (smc->sk.sk_err)
746 smc->sk.sk_state_change(&smc->sk);
747 else
748 smc->sk.sk_write_space(&smc->sk);
749 kfree(smc->connect_info);
750 smc->connect_info = NULL;
751 release_sock(&smc->sk);
752 }
753
754 static int smc_connect(struct socket *sock, struct sockaddr *addr,
755 int alen, int flags)
756 {
757 struct sock *sk = sock->sk;
758 struct smc_sock *smc;
759 int rc = -EINVAL;
760
761 smc = smc_sk(sk);
762
763 /* separate smc parameter checking to be safe */
764 if (alen < sizeof(addr->sa_family))
765 goto out_err;
766 if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6)
767 goto out_err;
768
769 lock_sock(sk);
770 switch (sk->sk_state) {
771 default:
772 goto out;
773 case SMC_ACTIVE:
774 rc = -EISCONN;
775 goto out;
776 case SMC_INIT:
777 rc = 0;
778 break;
779 }
780
781 smc_copy_sock_settings_to_clc(smc);
782 tcp_sk(smc->clcsock->sk)->syn_smc = 1;
783 if (flags & O_NONBLOCK) {
784 if (smc->connect_info) {
785 rc = -EALREADY;
786 goto out;
787 }
788 smc->connect_info = kzalloc(alen + 2 * sizeof(int), GFP_KERNEL);
789 if (!smc->connect_info) {
790 rc = -ENOMEM;
791 goto out;
792 }
793 smc->connect_info->alen = alen;
794 smc->connect_info->flags = flags ^ O_NONBLOCK;
795 memcpy(&smc->connect_info->addr, addr, alen);
796 schedule_work(&smc->connect_work);
797 rc = -EINPROGRESS;
798 } else {
799 rc = kernel_connect(smc->clcsock, addr, alen, flags);
800 if (rc)
801 goto out;
802
803 rc = __smc_connect(smc);
804 if (rc < 0)
805 goto out;
806 else
807 rc = 0; /* success cases including fallback */
808 }
809
810 out:
811 release_sock(sk);
812 out_err:
813 return rc;
814 }
815
816 static int smc_clcsock_accept(struct smc_sock *lsmc, struct smc_sock **new_smc)
817 {
818 struct socket *new_clcsock = NULL;
819 struct sock *lsk = &lsmc->sk;
820 struct sock *new_sk;
821 int rc;
822
823 release_sock(lsk);
824 new_sk = smc_sock_alloc(sock_net(lsk), NULL, lsk->sk_protocol);
825 if (!new_sk) {
826 rc = -ENOMEM;
827 lsk->sk_err = ENOMEM;
828 *new_smc = NULL;
829 lock_sock(lsk);
830 goto out;
831 }
832 *new_smc = smc_sk(new_sk);
833
834 rc = kernel_accept(lsmc->clcsock, &new_clcsock, 0);
835 lock_sock(lsk);
836 if (rc < 0)
837 lsk->sk_err = -rc;
838 if (rc < 0 || lsk->sk_state == SMC_CLOSED) {
839 if (new_clcsock)
840 sock_release(new_clcsock);
841 new_sk->sk_state = SMC_CLOSED;
842 sock_set_flag(new_sk, SOCK_DEAD);
843 new_sk->sk_prot->unhash(new_sk);
844 sock_put(new_sk); /* final */
845 *new_smc = NULL;
846 goto out;
847 }
848
849 (*new_smc)->clcsock = new_clcsock;
850 out:
851 return rc;
852 }
853
854 /* add a just created sock to the accept queue of the listen sock as
855 * candidate for a following socket accept call from user space
856 */
857 static void smc_accept_enqueue(struct sock *parent, struct sock *sk)
858 {
859 struct smc_sock *par = smc_sk(parent);
860
861 sock_hold(sk); /* sock_put in smc_accept_unlink () */
862 spin_lock(&par->accept_q_lock);
863 list_add_tail(&smc_sk(sk)->accept_q, &par->accept_q);
864 spin_unlock(&par->accept_q_lock);
865 sk_acceptq_added(parent);
866 }
867
868 /* remove a socket from the accept queue of its parental listening socket */
869 static void smc_accept_unlink(struct sock *sk)
870 {
871 struct smc_sock *par = smc_sk(sk)->listen_smc;
872
873 spin_lock(&par->accept_q_lock);
874 list_del_init(&smc_sk(sk)->accept_q);
875 spin_unlock(&par->accept_q_lock);
876 sk_acceptq_removed(&smc_sk(sk)->listen_smc->sk);
877 sock_put(sk); /* sock_hold in smc_accept_enqueue */
878 }
879
880 /* remove a sock from the accept queue to bind it to a new socket created
881 * for a socket accept call from user space
882 */
883 struct sock *smc_accept_dequeue(struct sock *parent,
884 struct socket *new_sock)
885 {
886 struct smc_sock *isk, *n;
887 struct sock *new_sk;
888
889 list_for_each_entry_safe(isk, n, &smc_sk(parent)->accept_q, accept_q) {
890 new_sk = (struct sock *)isk;
891
892 smc_accept_unlink(new_sk);
893 if (new_sk->sk_state == SMC_CLOSED) {
894 if (isk->clcsock) {
895 sock_release(isk->clcsock);
896 isk->clcsock = NULL;
897 }
898 new_sk->sk_prot->unhash(new_sk);
899 sock_put(new_sk); /* final */
900 continue;
901 }
902 if (new_sock)
903 sock_graft(new_sk, new_sock);
904 return new_sk;
905 }
906 return NULL;
907 }
908
909 /* clean up for a created but never accepted sock */
910 void smc_close_non_accepted(struct sock *sk)
911 {
912 struct smc_sock *smc = smc_sk(sk);
913
914 lock_sock(sk);
915 if (!sk->sk_lingertime)
916 /* wait for peer closing */
917 sk->sk_lingertime = SMC_MAX_STREAM_WAIT_TIMEOUT;
918 if (!smc->use_fallback) {
919 smc_close_active(smc);
920 sock_set_flag(sk, SOCK_DEAD);
921 sk->sk_shutdown |= SHUTDOWN_MASK;
922 }
923 if (smc->clcsock) {
924 struct socket *tcp;
925
926 tcp = smc->clcsock;
927 smc->clcsock = NULL;
928 sock_release(tcp);
929 }
930 if (smc->use_fallback) {
931 sock_put(sk); /* passive closing */
932 sk->sk_state = SMC_CLOSED;
933 } else {
934 if (sk->sk_state == SMC_CLOSED)
935 smc_conn_free(&smc->conn);
936 }
937 release_sock(sk);
938 sk->sk_prot->unhash(sk);
939 sock_put(sk); /* final sock_put */
940 }
941
942 static int smc_serv_conf_first_link(struct smc_sock *smc)
943 {
944 struct net *net = sock_net(smc->clcsock->sk);
945 struct smc_link_group *lgr = smc->conn.lgr;
946 struct smc_link *link;
947 int rest;
948 int rc;
949
950 link = &lgr->lnk[SMC_SINGLE_LINK];
951
952 if (smc_reg_rmb(link, smc->conn.rmb_desc, false))
953 return SMC_CLC_DECL_ERR_REGRMB;
954
955 /* send CONFIRM LINK request to client over the RoCE fabric */
956 rc = smc_llc_send_confirm_link(link, SMC_LLC_REQ);
957 if (rc < 0)
958 return SMC_CLC_DECL_TIMEOUT_CL;
959
960 /* receive CONFIRM LINK response from client over the RoCE fabric */
961 rest = wait_for_completion_interruptible_timeout(
962 &link->llc_confirm_resp,
963 SMC_LLC_WAIT_FIRST_TIME);
964 if (rest <= 0) {
965 struct smc_clc_msg_decline dclc;
966
967 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
968 SMC_CLC_DECLINE);
969 return rc;
970 }
971
972 if (link->llc_confirm_resp_rc)
973 return SMC_CLC_DECL_RMBE_EC;
974
975 /* send ADD LINK request to client over the RoCE fabric */
976 rc = smc_llc_send_add_link(link,
977 link->smcibdev->mac[link->ibport - 1],
978 link->gid, SMC_LLC_REQ);
979 if (rc < 0)
980 return SMC_CLC_DECL_TIMEOUT_AL;
981
982 /* receive ADD LINK response from client over the RoCE fabric */
983 rest = wait_for_completion_interruptible_timeout(&link->llc_add_resp,
984 SMC_LLC_WAIT_TIME);
985 if (rest <= 0) {
986 struct smc_clc_msg_decline dclc;
987
988 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
989 SMC_CLC_DECLINE);
990 return rc;
991 }
992
993 smc_llc_link_active(link, net->ipv4.sysctl_tcp_keepalive_time);
994
995 return 0;
996 }
997
998 /* listen worker: finish */
999 static void smc_listen_out(struct smc_sock *new_smc)
1000 {
1001 struct smc_sock *lsmc = new_smc->listen_smc;
1002 struct sock *newsmcsk = &new_smc->sk;
1003
1004 lock_sock_nested(&lsmc->sk, SINGLE_DEPTH_NESTING);
1005 if (lsmc->sk.sk_state == SMC_LISTEN) {
1006 smc_accept_enqueue(&lsmc->sk, newsmcsk);
1007 } else { /* no longer listening */
1008 smc_close_non_accepted(newsmcsk);
1009 }
1010 release_sock(&lsmc->sk);
1011
1012 /* Wake up accept */
1013 lsmc->sk.sk_data_ready(&lsmc->sk);
1014 sock_put(&lsmc->sk); /* sock_hold in smc_tcp_listen_work */
1015 }
1016
1017 /* listen worker: finish in state connected */
1018 static void smc_listen_out_connected(struct smc_sock *new_smc)
1019 {
1020 struct sock *newsmcsk = &new_smc->sk;
1021
1022 sk_refcnt_debug_inc(newsmcsk);
1023 if (newsmcsk->sk_state == SMC_INIT)
1024 newsmcsk->sk_state = SMC_ACTIVE;
1025
1026 smc_listen_out(new_smc);
1027 }
1028
1029 /* listen worker: finish in error state */
1030 static void smc_listen_out_err(struct smc_sock *new_smc)
1031 {
1032 struct sock *newsmcsk = &new_smc->sk;
1033
1034 if (newsmcsk->sk_state == SMC_INIT)
1035 sock_put(&new_smc->sk); /* passive closing */
1036 newsmcsk->sk_state = SMC_CLOSED;
1037 smc_conn_free(&new_smc->conn);
1038
1039 smc_listen_out(new_smc);
1040 }
1041
1042 /* listen worker: decline and fall back if possible */
1043 static void smc_listen_decline(struct smc_sock *new_smc, int reason_code,
1044 int local_contact)
1045 {
1046 /* RDMA setup failed, switch back to TCP */
1047 if (local_contact == SMC_FIRST_CONTACT)
1048 smc_lgr_forget(new_smc->conn.lgr);
1049 if (reason_code < 0) { /* error, no fallback possible */
1050 smc_listen_out_err(new_smc);
1051 return;
1052 }
1053 smc_conn_free(&new_smc->conn);
1054 new_smc->use_fallback = true;
1055 new_smc->fallback_rsn = reason_code;
1056 if (reason_code && reason_code != SMC_CLC_DECL_PEERDECL) {
1057 if (smc_clc_send_decline(new_smc, reason_code) < 0) {
1058 smc_listen_out_err(new_smc);
1059 return;
1060 }
1061 }
1062 smc_listen_out_connected(new_smc);
1063 }
1064
1065 /* listen worker: check prefixes */
1066 static int smc_listen_rdma_check(struct smc_sock *new_smc,
1067 struct smc_clc_msg_proposal *pclc)
1068 {
1069 struct smc_clc_msg_proposal_prefix *pclc_prfx;
1070 struct socket *newclcsock = new_smc->clcsock;
1071
1072 pclc_prfx = smc_clc_proposal_get_prefix(pclc);
1073 if (smc_clc_prfx_match(newclcsock, pclc_prfx))
1074 return SMC_CLC_DECL_CNFERR;
1075
1076 return 0;
1077 }
1078
1079 /* listen worker: initialize connection and buffers */
1080 static int smc_listen_rdma_init(struct smc_sock *new_smc,
1081 struct smc_clc_msg_proposal *pclc,
1082 struct smc_ib_device *ibdev, u8 ibport,
1083 int *local_contact)
1084 {
1085 /* allocate connection / link group */
1086 *local_contact = smc_conn_create(new_smc, false, 0, ibdev, ibport,
1087 &pclc->lcl, NULL, 0);
1088 if (*local_contact < 0) {
1089 if (*local_contact == -ENOMEM)
1090 return SMC_CLC_DECL_MEM;/* insufficient memory*/
1091 return SMC_CLC_DECL_INTERR; /* other error */
1092 }
1093
1094 /* create send buffer and rmb */
1095 if (smc_buf_create(new_smc, false))
1096 return SMC_CLC_DECL_MEM;
1097
1098 return 0;
1099 }
1100
1101 /* listen worker: initialize connection and buffers for SMC-D */
1102 static int smc_listen_ism_init(struct smc_sock *new_smc,
1103 struct smc_clc_msg_proposal *pclc,
1104 struct smcd_dev *ismdev,
1105 int *local_contact)
1106 {
1107 struct smc_clc_msg_smcd *pclc_smcd;
1108
1109 pclc_smcd = smc_get_clc_msg_smcd(pclc);
1110 *local_contact = smc_conn_create(new_smc, true, 0, NULL, 0, NULL,
1111 ismdev, pclc_smcd->gid);
1112 if (*local_contact < 0) {
1113 if (*local_contact == -ENOMEM)
1114 return SMC_CLC_DECL_MEM;/* insufficient memory*/
1115 return SMC_CLC_DECL_INTERR; /* other error */
1116 }
1117
1118 /* Check if peer can be reached via ISM device */
1119 if (smc_ism_cantalk(new_smc->conn.lgr->peer_gid,
1120 new_smc->conn.lgr->vlan_id,
1121 new_smc->conn.lgr->smcd)) {
1122 if (*local_contact == SMC_FIRST_CONTACT)
1123 smc_lgr_forget(new_smc->conn.lgr);
1124 smc_conn_free(&new_smc->conn);
1125 return SMC_CLC_DECL_CNFERR;
1126 }
1127
1128 /* Create send and receive buffers */
1129 if (smc_buf_create(new_smc, true)) {
1130 if (*local_contact == SMC_FIRST_CONTACT)
1131 smc_lgr_forget(new_smc->conn.lgr);
1132 smc_conn_free(&new_smc->conn);
1133 return SMC_CLC_DECL_MEM;
1134 }
1135
1136 return 0;
1137 }
1138
1139 /* listen worker: register buffers */
1140 static int smc_listen_rdma_reg(struct smc_sock *new_smc, int local_contact)
1141 {
1142 struct smc_link *link = &new_smc->conn.lgr->lnk[SMC_SINGLE_LINK];
1143
1144 if (local_contact != SMC_FIRST_CONTACT) {
1145 if (!new_smc->conn.rmb_desc->reused) {
1146 if (smc_reg_rmb(link, new_smc->conn.rmb_desc, true))
1147 return SMC_CLC_DECL_ERR_REGRMB;
1148 }
1149 }
1150 smc_rmb_sync_sg_for_device(&new_smc->conn);
1151
1152 return 0;
1153 }
1154
1155 /* listen worker: finish RDMA setup */
1156 static int smc_listen_rdma_finish(struct smc_sock *new_smc,
1157 struct smc_clc_msg_accept_confirm *cclc,
1158 int local_contact)
1159 {
1160 struct smc_link *link = &new_smc->conn.lgr->lnk[SMC_SINGLE_LINK];
1161 int reason_code = 0;
1162
1163 if (local_contact == SMC_FIRST_CONTACT)
1164 smc_link_save_peer_info(link, cclc);
1165
1166 if (smc_rmb_rtoken_handling(&new_smc->conn, cclc)) {
1167 reason_code = SMC_CLC_DECL_ERR_RTOK;
1168 goto decline;
1169 }
1170
1171 if (local_contact == SMC_FIRST_CONTACT) {
1172 if (smc_ib_ready_link(link)) {
1173 reason_code = SMC_CLC_DECL_ERR_RDYLNK;
1174 goto decline;
1175 }
1176 /* QP confirmation over RoCE fabric */
1177 reason_code = smc_serv_conf_first_link(new_smc);
1178 if (reason_code)
1179 goto decline;
1180 }
1181 return 0;
1182
1183 decline:
1184 mutex_unlock(&smc_create_lgr_pending);
1185 smc_listen_decline(new_smc, reason_code, local_contact);
1186 return reason_code;
1187 }
1188
1189 /* setup for RDMA connection of server */
1190 static void smc_listen_work(struct work_struct *work)
1191 {
1192 struct smc_sock *new_smc = container_of(work, struct smc_sock,
1193 smc_listen_work);
1194 struct socket *newclcsock = new_smc->clcsock;
1195 struct smc_clc_msg_accept_confirm cclc;
1196 struct smc_clc_msg_proposal *pclc;
1197 struct smc_ib_device *ibdev;
1198 bool ism_supported = false;
1199 struct smcd_dev *ismdev;
1200 u8 buf[SMC_CLC_MAX_LEN];
1201 int local_contact = 0;
1202 unsigned short vlan;
1203 int reason_code = 0;
1204 int rc = 0;
1205 u8 ibport;
1206
1207 if (new_smc->use_fallback) {
1208 smc_listen_out_connected(new_smc);
1209 return;
1210 }
1211
1212 /* check if peer is smc capable */
1213 if (!tcp_sk(newclcsock->sk)->syn_smc) {
1214 new_smc->use_fallback = true;
1215 new_smc->fallback_rsn = SMC_CLC_DECL_PEERNOSMC;
1216 smc_listen_out_connected(new_smc);
1217 return;
1218 }
1219
1220 /* do inband token exchange -
1221 * wait for and receive SMC Proposal CLC message
1222 */
1223 pclc = (struct smc_clc_msg_proposal *)&buf;
1224 reason_code = smc_clc_wait_msg(new_smc, pclc, SMC_CLC_MAX_LEN,
1225 SMC_CLC_PROPOSAL);
1226 if (reason_code) {
1227 smc_listen_decline(new_smc, reason_code, 0);
1228 return;
1229 }
1230
1231 /* IPSec connections opt out of SMC-R optimizations */
1232 if (using_ipsec(new_smc)) {
1233 smc_listen_decline(new_smc, SMC_CLC_DECL_IPSEC, 0);
1234 return;
1235 }
1236
1237 mutex_lock(&smc_create_lgr_pending);
1238 smc_close_init(new_smc);
1239 smc_rx_init(new_smc);
1240 smc_tx_init(new_smc);
1241
1242 /* check if ISM is available */
1243 if ((pclc->hdr.path == SMC_TYPE_D || pclc->hdr.path == SMC_TYPE_B) &&
1244 !smc_check_ism(new_smc, &ismdev) &&
1245 !smc_listen_ism_init(new_smc, pclc, ismdev, &local_contact)) {
1246 ism_supported = true;
1247 }
1248
1249 /* check if RDMA is available */
1250 if (!ism_supported &&
1251 ((pclc->hdr.path != SMC_TYPE_R && pclc->hdr.path != SMC_TYPE_B) ||
1252 smc_vlan_by_tcpsk(new_smc->clcsock, &vlan) ||
1253 smc_check_rdma(new_smc, &ibdev, &ibport, vlan, NULL) ||
1254 smc_listen_rdma_check(new_smc, pclc) ||
1255 smc_listen_rdma_init(new_smc, pclc, ibdev, ibport,
1256 &local_contact) ||
1257 smc_listen_rdma_reg(new_smc, local_contact))) {
1258 /* SMC not supported, decline */
1259 mutex_unlock(&smc_create_lgr_pending);
1260 smc_listen_decline(new_smc, SMC_CLC_DECL_MODEUNSUPP,
1261 local_contact);
1262 return;
1263 }
1264
1265 /* send SMC Accept CLC message */
1266 rc = smc_clc_send_accept(new_smc, local_contact);
1267 if (rc) {
1268 mutex_unlock(&smc_create_lgr_pending);
1269 smc_listen_decline(new_smc, rc, local_contact);
1270 return;
1271 }
1272
1273 /* receive SMC Confirm CLC message */
1274 reason_code = smc_clc_wait_msg(new_smc, &cclc, sizeof(cclc),
1275 SMC_CLC_CONFIRM);
1276 if (reason_code) {
1277 mutex_unlock(&smc_create_lgr_pending);
1278 smc_listen_decline(new_smc, reason_code, local_contact);
1279 return;
1280 }
1281
1282 /* finish worker */
1283 if (!ism_supported) {
1284 if (smc_listen_rdma_finish(new_smc, &cclc, local_contact))
1285 return;
1286 }
1287 smc_conn_save_peer_info(new_smc, &cclc);
1288 mutex_unlock(&smc_create_lgr_pending);
1289 smc_listen_out_connected(new_smc);
1290 }
1291
1292 static void smc_tcp_listen_work(struct work_struct *work)
1293 {
1294 struct smc_sock *lsmc = container_of(work, struct smc_sock,
1295 tcp_listen_work);
1296 struct sock *lsk = &lsmc->sk;
1297 struct smc_sock *new_smc;
1298 int rc = 0;
1299
1300 lock_sock(lsk);
1301 while (lsk->sk_state == SMC_LISTEN) {
1302 rc = smc_clcsock_accept(lsmc, &new_smc);
1303 if (rc)
1304 goto out;
1305 if (!new_smc)
1306 continue;
1307
1308 new_smc->listen_smc = lsmc;
1309 new_smc->use_fallback = lsmc->use_fallback;
1310 new_smc->fallback_rsn = lsmc->fallback_rsn;
1311 sock_hold(lsk); /* sock_put in smc_listen_work */
1312 INIT_WORK(&new_smc->smc_listen_work, smc_listen_work);
1313 smc_copy_sock_settings_to_smc(new_smc);
1314 new_smc->sk.sk_sndbuf = lsmc->sk.sk_sndbuf;
1315 new_smc->sk.sk_rcvbuf = lsmc->sk.sk_rcvbuf;
1316 sock_hold(&new_smc->sk); /* sock_put in passive closing */
1317 if (!schedule_work(&new_smc->smc_listen_work))
1318 sock_put(&new_smc->sk);
1319 }
1320
1321 out:
1322 release_sock(lsk);
1323 sock_put(&lsmc->sk); /* sock_hold in smc_listen */
1324 }
1325
1326 static int smc_listen(struct socket *sock, int backlog)
1327 {
1328 struct sock *sk = sock->sk;
1329 struct smc_sock *smc;
1330 int rc;
1331
1332 smc = smc_sk(sk);
1333 lock_sock(sk);
1334
1335 rc = -EINVAL;
1336 if ((sk->sk_state != SMC_INIT) && (sk->sk_state != SMC_LISTEN))
1337 goto out;
1338
1339 rc = 0;
1340 if (sk->sk_state == SMC_LISTEN) {
1341 sk->sk_max_ack_backlog = backlog;
1342 goto out;
1343 }
1344 /* some socket options are handled in core, so we could not apply
1345 * them to the clc socket -- copy smc socket options to clc socket
1346 */
1347 smc_copy_sock_settings_to_clc(smc);
1348 if (!smc->use_fallback)
1349 tcp_sk(smc->clcsock->sk)->syn_smc = 1;
1350
1351 rc = kernel_listen(smc->clcsock, backlog);
1352 if (rc)
1353 goto out;
1354 sk->sk_max_ack_backlog = backlog;
1355 sk->sk_ack_backlog = 0;
1356 sk->sk_state = SMC_LISTEN;
1357 INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work);
1358 sock_hold(sk); /* sock_hold in tcp_listen_worker */
1359 if (!schedule_work(&smc->tcp_listen_work))
1360 sock_put(sk);
1361
1362 out:
1363 release_sock(sk);
1364 return rc;
1365 }
1366
1367 static int smc_accept(struct socket *sock, struct socket *new_sock,
1368 int flags, bool kern)
1369 {
1370 struct sock *sk = sock->sk, *nsk;
1371 DECLARE_WAITQUEUE(wait, current);
1372 struct smc_sock *lsmc;
1373 long timeo;
1374 int rc = 0;
1375
1376 lsmc = smc_sk(sk);
1377 sock_hold(sk); /* sock_put below */
1378 lock_sock(sk);
1379
1380 if (lsmc->sk.sk_state != SMC_LISTEN) {
1381 rc = -EINVAL;
1382 release_sock(sk);
1383 goto out;
1384 }
1385
1386 /* Wait for an incoming connection */
1387 timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK);
1388 add_wait_queue_exclusive(sk_sleep(sk), &wait);
1389 while (!(nsk = smc_accept_dequeue(sk, new_sock))) {
1390 set_current_state(TASK_INTERRUPTIBLE);
1391 if (!timeo) {
1392 rc = -EAGAIN;
1393 break;
1394 }
1395 release_sock(sk);
1396 timeo = schedule_timeout(timeo);
1397 /* wakeup by sk_data_ready in smc_listen_work() */
1398 sched_annotate_sleep();
1399 lock_sock(sk);
1400 if (signal_pending(current)) {
1401 rc = sock_intr_errno(timeo);
1402 break;
1403 }
1404 }
1405 set_current_state(TASK_RUNNING);
1406 remove_wait_queue(sk_sleep(sk), &wait);
1407
1408 if (!rc)
1409 rc = sock_error(nsk);
1410 release_sock(sk);
1411 if (rc)
1412 goto out;
1413
1414 if (lsmc->sockopt_defer_accept && !(flags & O_NONBLOCK)) {
1415 /* wait till data arrives on the socket */
1416 timeo = msecs_to_jiffies(lsmc->sockopt_defer_accept *
1417 MSEC_PER_SEC);
1418 if (smc_sk(nsk)->use_fallback) {
1419 struct sock *clcsk = smc_sk(nsk)->clcsock->sk;
1420
1421 lock_sock(clcsk);
1422 if (skb_queue_empty(&clcsk->sk_receive_queue))
1423 sk_wait_data(clcsk, &timeo, NULL);
1424 release_sock(clcsk);
1425 } else if (!atomic_read(&smc_sk(nsk)->conn.bytes_to_rcv)) {
1426 lock_sock(nsk);
1427 smc_rx_wait(smc_sk(nsk), &timeo, smc_rx_data_available);
1428 release_sock(nsk);
1429 }
1430 }
1431
1432 out:
1433 sock_put(sk); /* sock_hold above */
1434 return rc;
1435 }
1436
1437 static int smc_getname(struct socket *sock, struct sockaddr *addr,
1438 int peer)
1439 {
1440 struct smc_sock *smc;
1441
1442 if (peer && (sock->sk->sk_state != SMC_ACTIVE) &&
1443 (sock->sk->sk_state != SMC_APPCLOSEWAIT1))
1444 return -ENOTCONN;
1445
1446 smc = smc_sk(sock->sk);
1447
1448 return smc->clcsock->ops->getname(smc->clcsock, addr, peer);
1449 }
1450
1451 static int smc_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
1452 {
1453 struct sock *sk = sock->sk;
1454 struct smc_sock *smc;
1455 int rc = -EPIPE;
1456
1457 smc = smc_sk(sk);
1458 lock_sock(sk);
1459 if ((sk->sk_state != SMC_ACTIVE) &&
1460 (sk->sk_state != SMC_APPCLOSEWAIT1) &&
1461 (sk->sk_state != SMC_INIT))
1462 goto out;
1463
1464 if (msg->msg_flags & MSG_FASTOPEN) {
1465 if (sk->sk_state == SMC_INIT) {
1466 smc->use_fallback = true;
1467 smc->fallback_rsn = SMC_CLC_DECL_OPTUNSUPP;
1468 } else {
1469 rc = -EINVAL;
1470 goto out;
1471 }
1472 }
1473
1474 if (smc->use_fallback)
1475 rc = smc->clcsock->ops->sendmsg(smc->clcsock, msg, len);
1476 else
1477 rc = smc_tx_sendmsg(smc, msg, len);
1478 out:
1479 release_sock(sk);
1480 return rc;
1481 }
1482
1483 static int smc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
1484 int flags)
1485 {
1486 struct sock *sk = sock->sk;
1487 struct smc_sock *smc;
1488 int rc = -ENOTCONN;
1489
1490 smc = smc_sk(sk);
1491 lock_sock(sk);
1492 if ((sk->sk_state == SMC_INIT) ||
1493 (sk->sk_state == SMC_LISTEN) ||
1494 (sk->sk_state == SMC_CLOSED))
1495 goto out;
1496
1497 if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
1498 rc = 0;
1499 goto out;
1500 }
1501
1502 if (smc->use_fallback) {
1503 rc = smc->clcsock->ops->recvmsg(smc->clcsock, msg, len, flags);
1504 } else {
1505 msg->msg_namelen = 0;
1506 rc = smc_rx_recvmsg(smc, msg, NULL, len, flags);
1507 }
1508
1509 out:
1510 release_sock(sk);
1511 return rc;
1512 }
1513
1514 static __poll_t smc_accept_poll(struct sock *parent)
1515 {
1516 struct smc_sock *isk = smc_sk(parent);
1517 __poll_t mask = 0;
1518
1519 spin_lock(&isk->accept_q_lock);
1520 if (!list_empty(&isk->accept_q))
1521 mask = EPOLLIN | EPOLLRDNORM;
1522 spin_unlock(&isk->accept_q_lock);
1523
1524 return mask;
1525 }
1526
1527 static __poll_t smc_poll(struct file *file, struct socket *sock,
1528 poll_table *wait)
1529 {
1530 struct sock *sk = sock->sk;
1531 __poll_t mask = 0;
1532 struct smc_sock *smc;
1533
1534 if (!sk)
1535 return EPOLLNVAL;
1536
1537 smc = smc_sk(sock->sk);
1538 if (smc->use_fallback) {
1539 /* delegate to CLC child sock */
1540 mask = smc->clcsock->ops->poll(file, smc->clcsock, wait);
1541 sk->sk_err = smc->clcsock->sk->sk_err;
1542 if (sk->sk_err)
1543 mask |= EPOLLERR;
1544 } else {
1545 if (sk->sk_state != SMC_CLOSED)
1546 sock_poll_wait(file, sock, wait);
1547 if (sk->sk_err)
1548 mask |= EPOLLERR;
1549 if ((sk->sk_shutdown == SHUTDOWN_MASK) ||
1550 (sk->sk_state == SMC_CLOSED))
1551 mask |= EPOLLHUP;
1552 if (sk->sk_state == SMC_LISTEN) {
1553 /* woken up by sk_data_ready in smc_listen_work() */
1554 mask = smc_accept_poll(sk);
1555 } else {
1556 if (atomic_read(&smc->conn.sndbuf_space) ||
1557 sk->sk_shutdown & SEND_SHUTDOWN) {
1558 mask |= EPOLLOUT | EPOLLWRNORM;
1559 } else {
1560 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
1561 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1562 }
1563 if (atomic_read(&smc->conn.bytes_to_rcv))
1564 mask |= EPOLLIN | EPOLLRDNORM;
1565 if (sk->sk_shutdown & RCV_SHUTDOWN)
1566 mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
1567 if (sk->sk_state == SMC_APPCLOSEWAIT1)
1568 mask |= EPOLLIN;
1569 if (smc->conn.urg_state == SMC_URG_VALID)
1570 mask |= EPOLLPRI;
1571 }
1572 }
1573
1574 return mask;
1575 }
1576
1577 static int smc_shutdown(struct socket *sock, int how)
1578 {
1579 struct sock *sk = sock->sk;
1580 struct smc_sock *smc;
1581 int rc = -EINVAL;
1582 int rc1 = 0;
1583
1584 smc = smc_sk(sk);
1585
1586 if ((how < SHUT_RD) || (how > SHUT_RDWR))
1587 return rc;
1588
1589 lock_sock(sk);
1590
1591 rc = -ENOTCONN;
1592 if ((sk->sk_state != SMC_ACTIVE) &&
1593 (sk->sk_state != SMC_PEERCLOSEWAIT1) &&
1594 (sk->sk_state != SMC_PEERCLOSEWAIT2) &&
1595 (sk->sk_state != SMC_APPCLOSEWAIT1) &&
1596 (sk->sk_state != SMC_APPCLOSEWAIT2) &&
1597 (sk->sk_state != SMC_APPFINCLOSEWAIT))
1598 goto out;
1599 if (smc->use_fallback) {
1600 rc = kernel_sock_shutdown(smc->clcsock, how);
1601 sk->sk_shutdown = smc->clcsock->sk->sk_shutdown;
1602 if (sk->sk_shutdown == SHUTDOWN_MASK)
1603 sk->sk_state = SMC_CLOSED;
1604 goto out;
1605 }
1606 switch (how) {
1607 case SHUT_RDWR: /* shutdown in both directions */
1608 rc = smc_close_active(smc);
1609 break;
1610 case SHUT_WR:
1611 rc = smc_close_shutdown_write(smc);
1612 break;
1613 case SHUT_RD:
1614 rc = 0;
1615 /* nothing more to do because peer is not involved */
1616 break;
1617 }
1618 if (smc->clcsock)
1619 rc1 = kernel_sock_shutdown(smc->clcsock, how);
1620 /* map sock_shutdown_cmd constants to sk_shutdown value range */
1621 sk->sk_shutdown |= how + 1;
1622
1623 out:
1624 release_sock(sk);
1625 return rc ? rc : rc1;
1626 }
1627
1628 static int smc_setsockopt(struct socket *sock, int level, int optname,
1629 char __user *optval, unsigned int optlen)
1630 {
1631 struct sock *sk = sock->sk;
1632 struct smc_sock *smc;
1633 int val, rc;
1634
1635 smc = smc_sk(sk);
1636
1637 /* generic setsockopts reaching us here always apply to the
1638 * CLC socket
1639 */
1640 rc = smc->clcsock->ops->setsockopt(smc->clcsock, level, optname,
1641 optval, optlen);
1642 if (smc->clcsock->sk->sk_err) {
1643 sk->sk_err = smc->clcsock->sk->sk_err;
1644 sk->sk_error_report(sk);
1645 }
1646 if (rc)
1647 return rc;
1648
1649 if (optlen < sizeof(int))
1650 return -EINVAL;
1651 if (get_user(val, (int __user *)optval))
1652 return -EFAULT;
1653
1654 lock_sock(sk);
1655 switch (optname) {
1656 case TCP_ULP:
1657 case TCP_FASTOPEN:
1658 case TCP_FASTOPEN_CONNECT:
1659 case TCP_FASTOPEN_KEY:
1660 case TCP_FASTOPEN_NO_COOKIE:
1661 /* option not supported by SMC */
1662 if (sk->sk_state == SMC_INIT) {
1663 smc->use_fallback = true;
1664 smc->fallback_rsn = SMC_CLC_DECL_OPTUNSUPP;
1665 } else {
1666 if (!smc->use_fallback)
1667 rc = -EINVAL;
1668 }
1669 break;
1670 case TCP_NODELAY:
1671 if (sk->sk_state != SMC_INIT && sk->sk_state != SMC_LISTEN) {
1672 if (val && !smc->use_fallback)
1673 mod_delayed_work(system_wq, &smc->conn.tx_work,
1674 0);
1675 }
1676 break;
1677 case TCP_CORK:
1678 if (sk->sk_state != SMC_INIT && sk->sk_state != SMC_LISTEN) {
1679 if (!val && !smc->use_fallback)
1680 mod_delayed_work(system_wq, &smc->conn.tx_work,
1681 0);
1682 }
1683 break;
1684 case TCP_DEFER_ACCEPT:
1685 smc->sockopt_defer_accept = val;
1686 break;
1687 default:
1688 break;
1689 }
1690 release_sock(sk);
1691
1692 return rc;
1693 }
1694
1695 static int smc_getsockopt(struct socket *sock, int level, int optname,
1696 char __user *optval, int __user *optlen)
1697 {
1698 struct smc_sock *smc;
1699
1700 smc = smc_sk(sock->sk);
1701 /* socket options apply to the CLC socket */
1702 return smc->clcsock->ops->getsockopt(smc->clcsock, level, optname,
1703 optval, optlen);
1704 }
1705
1706 static int smc_ioctl(struct socket *sock, unsigned int cmd,
1707 unsigned long arg)
1708 {
1709 union smc_host_cursor cons, urg;
1710 struct smc_connection *conn;
1711 struct smc_sock *smc;
1712 int answ;
1713
1714 smc = smc_sk(sock->sk);
1715 conn = &smc->conn;
1716 lock_sock(&smc->sk);
1717 if (smc->use_fallback) {
1718 if (!smc->clcsock) {
1719 release_sock(&smc->sk);
1720 return -EBADF;
1721 }
1722 answ = smc->clcsock->ops->ioctl(smc->clcsock, cmd, arg);
1723 release_sock(&smc->sk);
1724 return answ;
1725 }
1726 switch (cmd) {
1727 case SIOCINQ: /* same as FIONREAD */
1728 if (smc->sk.sk_state == SMC_LISTEN) {
1729 release_sock(&smc->sk);
1730 return -EINVAL;
1731 }
1732 if (smc->sk.sk_state == SMC_INIT ||
1733 smc->sk.sk_state == SMC_CLOSED)
1734 answ = 0;
1735 else
1736 answ = atomic_read(&smc->conn.bytes_to_rcv);
1737 break;
1738 case SIOCOUTQ:
1739 /* output queue size (not send + not acked) */
1740 if (smc->sk.sk_state == SMC_LISTEN) {
1741 release_sock(&smc->sk);
1742 return -EINVAL;
1743 }
1744 if (smc->sk.sk_state == SMC_INIT ||
1745 smc->sk.sk_state == SMC_CLOSED)
1746 answ = 0;
1747 else
1748 answ = smc->conn.sndbuf_desc->len -
1749 atomic_read(&smc->conn.sndbuf_space);
1750 break;
1751 case SIOCOUTQNSD:
1752 /* output queue size (not send only) */
1753 if (smc->sk.sk_state == SMC_LISTEN) {
1754 release_sock(&smc->sk);
1755 return -EINVAL;
1756 }
1757 if (smc->sk.sk_state == SMC_INIT ||
1758 smc->sk.sk_state == SMC_CLOSED)
1759 answ = 0;
1760 else
1761 answ = smc_tx_prepared_sends(&smc->conn);
1762 break;
1763 case SIOCATMARK:
1764 if (smc->sk.sk_state == SMC_LISTEN) {
1765 release_sock(&smc->sk);
1766 return -EINVAL;
1767 }
1768 if (smc->sk.sk_state == SMC_INIT ||
1769 smc->sk.sk_state == SMC_CLOSED) {
1770 answ = 0;
1771 } else {
1772 smc_curs_copy(&cons, &conn->local_tx_ctrl.cons, conn);
1773 smc_curs_copy(&urg, &conn->urg_curs, conn);
1774 answ = smc_curs_diff(conn->rmb_desc->len,
1775 &cons, &urg) == 1;
1776 }
1777 break;
1778 default:
1779 release_sock(&smc->sk);
1780 return -ENOIOCTLCMD;
1781 }
1782 release_sock(&smc->sk);
1783
1784 return put_user(answ, (int __user *)arg);
1785 }
1786
1787 static ssize_t smc_sendpage(struct socket *sock, struct page *page,
1788 int offset, size_t size, int flags)
1789 {
1790 struct sock *sk = sock->sk;
1791 struct smc_sock *smc;
1792 int rc = -EPIPE;
1793
1794 smc = smc_sk(sk);
1795 lock_sock(sk);
1796 if (sk->sk_state != SMC_ACTIVE) {
1797 release_sock(sk);
1798 goto out;
1799 }
1800 release_sock(sk);
1801 if (smc->use_fallback)
1802 rc = kernel_sendpage(smc->clcsock, page, offset,
1803 size, flags);
1804 else
1805 rc = sock_no_sendpage(sock, page, offset, size, flags);
1806
1807 out:
1808 return rc;
1809 }
1810
1811 /* Map the affected portions of the rmbe into an spd, note the number of bytes
1812 * to splice in conn->splice_pending, and press 'go'. Delays consumer cursor
1813 * updates till whenever a respective page has been fully processed.
1814 * Note that subsequent recv() calls have to wait till all splice() processing
1815 * completed.
1816 */
1817 static ssize_t smc_splice_read(struct socket *sock, loff_t *ppos,
1818 struct pipe_inode_info *pipe, size_t len,
1819 unsigned int flags)
1820 {
1821 struct sock *sk = sock->sk;
1822 struct smc_sock *smc;
1823 int rc = -ENOTCONN;
1824
1825 smc = smc_sk(sk);
1826 lock_sock(sk);
1827
1828 if (sk->sk_state == SMC_INIT ||
1829 sk->sk_state == SMC_LISTEN ||
1830 sk->sk_state == SMC_CLOSED)
1831 goto out;
1832
1833 if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
1834 rc = 0;
1835 goto out;
1836 }
1837
1838 if (smc->use_fallback) {
1839 rc = smc->clcsock->ops->splice_read(smc->clcsock, ppos,
1840 pipe, len, flags);
1841 } else {
1842 if (*ppos) {
1843 rc = -ESPIPE;
1844 goto out;
1845 }
1846 if (flags & SPLICE_F_NONBLOCK)
1847 flags = MSG_DONTWAIT;
1848 else
1849 flags = 0;
1850 rc = smc_rx_recvmsg(smc, NULL, pipe, len, flags);
1851 }
1852 out:
1853 release_sock(sk);
1854
1855 return rc;
1856 }
1857
1858 /* must look like tcp */
1859 static const struct proto_ops smc_sock_ops = {
1860 .family = PF_SMC,
1861 .owner = THIS_MODULE,
1862 .release = smc_release,
1863 .bind = smc_bind,
1864 .connect = smc_connect,
1865 .socketpair = sock_no_socketpair,
1866 .accept = smc_accept,
1867 .getname = smc_getname,
1868 .poll = smc_poll,
1869 .ioctl = smc_ioctl,
1870 .listen = smc_listen,
1871 .shutdown = smc_shutdown,
1872 .setsockopt = smc_setsockopt,
1873 .getsockopt = smc_getsockopt,
1874 .sendmsg = smc_sendmsg,
1875 .recvmsg = smc_recvmsg,
1876 .mmap = sock_no_mmap,
1877 .sendpage = smc_sendpage,
1878 .splice_read = smc_splice_read,
1879 };
1880
1881 static int smc_create(struct net *net, struct socket *sock, int protocol,
1882 int kern)
1883 {
1884 int family = (protocol == SMCPROTO_SMC6) ? PF_INET6 : PF_INET;
1885 struct smc_sock *smc;
1886 struct sock *sk;
1887 int rc;
1888
1889 rc = -ESOCKTNOSUPPORT;
1890 if (sock->type != SOCK_STREAM)
1891 goto out;
1892
1893 rc = -EPROTONOSUPPORT;
1894 if (protocol != SMCPROTO_SMC && protocol != SMCPROTO_SMC6)
1895 goto out;
1896
1897 rc = -ENOBUFS;
1898 sock->ops = &smc_sock_ops;
1899 sk = smc_sock_alloc(net, sock, protocol);
1900 if (!sk)
1901 goto out;
1902
1903 /* create internal TCP socket for CLC handshake and fallback */
1904 smc = smc_sk(sk);
1905 smc->use_fallback = false; /* assume rdma capability first */
1906 smc->fallback_rsn = 0;
1907 rc = sock_create_kern(net, family, SOCK_STREAM, IPPROTO_TCP,
1908 &smc->clcsock);
1909 if (rc) {
1910 sk_common_release(sk);
1911 goto out;
1912 }
1913 smc->sk.sk_sndbuf = max(smc->clcsock->sk->sk_sndbuf, SMC_BUF_MIN_SIZE);
1914 smc->sk.sk_rcvbuf = max(smc->clcsock->sk->sk_rcvbuf, SMC_BUF_MIN_SIZE);
1915
1916 out:
1917 return rc;
1918 }
1919
1920 static const struct net_proto_family smc_sock_family_ops = {
1921 .family = PF_SMC,
1922 .owner = THIS_MODULE,
1923 .create = smc_create,
1924 };
1925
1926 static int __init smc_init(void)
1927 {
1928 int rc;
1929
1930 rc = smc_pnet_init();
1931 if (rc)
1932 return rc;
1933
1934 rc = smc_llc_init();
1935 if (rc) {
1936 pr_err("%s: smc_llc_init fails with %d\n", __func__, rc);
1937 goto out_pnet;
1938 }
1939
1940 rc = smc_cdc_init();
1941 if (rc) {
1942 pr_err("%s: smc_cdc_init fails with %d\n", __func__, rc);
1943 goto out_pnet;
1944 }
1945
1946 rc = proto_register(&smc_proto, 1);
1947 if (rc) {
1948 pr_err("%s: proto_register(v4) fails with %d\n", __func__, rc);
1949 goto out_pnet;
1950 }
1951
1952 rc = proto_register(&smc_proto6, 1);
1953 if (rc) {
1954 pr_err("%s: proto_register(v6) fails with %d\n", __func__, rc);
1955 goto out_proto;
1956 }
1957
1958 rc = sock_register(&smc_sock_family_ops);
1959 if (rc) {
1960 pr_err("%s: sock_register fails with %d\n", __func__, rc);
1961 goto out_proto6;
1962 }
1963 INIT_HLIST_HEAD(&smc_v4_hashinfo.ht);
1964 INIT_HLIST_HEAD(&smc_v6_hashinfo.ht);
1965
1966 rc = smc_ib_register_client();
1967 if (rc) {
1968 pr_err("%s: ib_register fails with %d\n", __func__, rc);
1969 goto out_sock;
1970 }
1971
1972 static_branch_enable(&tcp_have_smc);
1973 return 0;
1974
1975 out_sock:
1976 sock_unregister(PF_SMC);
1977 out_proto6:
1978 proto_unregister(&smc_proto6);
1979 out_proto:
1980 proto_unregister(&smc_proto);
1981 out_pnet:
1982 smc_pnet_exit();
1983 return rc;
1984 }
1985
1986 static void __exit smc_exit(void)
1987 {
1988 smc_core_exit();
1989 static_branch_disable(&tcp_have_smc);
1990 smc_ib_unregister_client();
1991 sock_unregister(PF_SMC);
1992 proto_unregister(&smc_proto6);
1993 proto_unregister(&smc_proto);
1994 smc_pnet_exit();
1995 }
1996
1997 module_init(smc_init);
1998 module_exit(smc_exit);
1999
2000 MODULE_AUTHOR("Ursula Braun <ubraun@linux.vnet.ibm.com>");
2001 MODULE_DESCRIPTION("smc socket address family");
2002 MODULE_LICENSE("GPL");
2003 MODULE_ALIAS_NETPROTO(PF_SMC);
Linux with added FPC-III support