?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Linux 5.7.7
[linux/fpc-iii.git] / net / smc / af_smc.c
blob6fd44bdb0fc3ebb8d9b32e5aebbb494267d71b27
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Shared Memory Communications over RDMA (SMC-R) and RoCE
4 *
5 * AF_SMC protocol family socket handler keeping the AF_INET sock address type
6 * applies to SOCK_STREAM sockets only
7 * offers an alternative communication option for TCP-protocol sockets
8 * applicable with RoCE-cards only
9 *
10 * Initial restrictions:
11 * - support for alternate links postponed
12 *
13 * Copyright IBM Corp. 2016, 2018
14 *
15 * Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com>
16 * based on prototype from Frank Blaschka
17 */
18
19 #define KMSG_COMPONENT "smc"
20 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
21
22 #include <linux/module.h>
23 #include <linux/socket.h>
24 #include <linux/workqueue.h>
25 #include <linux/in.h>
26 #include <linux/sched/signal.h>
27 #include <linux/if_vlan.h>
28 #include <linux/rcupdate_wait.h>
29
30 #include <net/sock.h>
31 #include <net/tcp.h>
32 #include <net/smc.h>
33 #include <asm/ioctls.h>
34
35 #include <net/net_namespace.h>
36 #include <net/netns/generic.h>
37 #include "smc_netns.h"
38
39 #include "smc.h"
40 #include "smc_clc.h"
41 #include "smc_llc.h"
42 #include "smc_cdc.h"
43 #include "smc_core.h"
44 #include "smc_ib.h"
45 #include "smc_ism.h"
46 #include "smc_pnet.h"
47 #include "smc_tx.h"
48 #include "smc_rx.h"
49 #include "smc_close.h"
50
51 static DEFINE_MUTEX(smc_server_lgr_pending); /* serialize link group
52 * creation on server
53 */
54 static DEFINE_MUTEX(smc_client_lgr_pending); /* serialize link group
55 * creation on client
56 */
57
58 static void smc_tcp_listen_work(struct work_struct *);
59 static void smc_connect_work(struct work_struct *);
60
61 static void smc_set_keepalive(struct sock *sk, int val)
62 {
63 struct smc_sock *smc = smc_sk(sk);
64
65 smc->clcsock->sk->sk_prot->keepalive(smc->clcsock->sk, val);
66 }
67
68 static struct smc_hashinfo smc_v4_hashinfo = {
69 .lock = __RW_LOCK_UNLOCKED(smc_v4_hashinfo.lock),
70 };
71
72 static struct smc_hashinfo smc_v6_hashinfo = {
73 .lock = __RW_LOCK_UNLOCKED(smc_v6_hashinfo.lock),
74 };
75
76 int smc_hash_sk(struct sock *sk)
77 {
78 struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
79 struct hlist_head *head;
80
81 head = &h->ht;
82
83 write_lock_bh(&h->lock);
84 sk_add_node(sk, head);
85 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
86 write_unlock_bh(&h->lock);
87
88 return 0;
89 }
90 EXPORT_SYMBOL_GPL(smc_hash_sk);
91
92 void smc_unhash_sk(struct sock *sk)
93 {
94 struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
95
96 write_lock_bh(&h->lock);
97 if (sk_del_node_init(sk))
98 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
99 write_unlock_bh(&h->lock);
100 }
101 EXPORT_SYMBOL_GPL(smc_unhash_sk);
102
103 struct proto smc_proto = {
104 .name = "SMC",
105 .owner = THIS_MODULE,
106 .keepalive = smc_set_keepalive,
107 .hash = smc_hash_sk,
108 .unhash = smc_unhash_sk,
109 .obj_size = sizeof(struct smc_sock),
110 .h.smc_hash = &smc_v4_hashinfo,
111 .slab_flags = SLAB_TYPESAFE_BY_RCU,
112 };
113 EXPORT_SYMBOL_GPL(smc_proto);
114
115 struct proto smc_proto6 = {
116 .name = "SMC6",
117 .owner = THIS_MODULE,
118 .keepalive = smc_set_keepalive,
119 .hash = smc_hash_sk,
120 .unhash = smc_unhash_sk,
121 .obj_size = sizeof(struct smc_sock),
122 .h.smc_hash = &smc_v6_hashinfo,
123 .slab_flags = SLAB_TYPESAFE_BY_RCU,
124 };
125 EXPORT_SYMBOL_GPL(smc_proto6);
126
127 static void smc_restore_fallback_changes(struct smc_sock *smc)
128 {
129 smc->clcsock->file->private_data = smc->sk.sk_socket;
130 smc->clcsock->file = NULL;
131 }
132
133 static int __smc_release(struct smc_sock *smc)
134 {
135 struct sock *sk = &smc->sk;
136 int rc = 0;
137
138 if (!smc->use_fallback) {
139 rc = smc_close_active(smc);
140 sock_set_flag(sk, SOCK_DEAD);
141 sk->sk_shutdown |= SHUTDOWN_MASK;
142 } else {
143 if (sk->sk_state != SMC_LISTEN && sk->sk_state != SMC_INIT)
144 sock_put(sk); /* passive closing */
145 if (sk->sk_state == SMC_LISTEN) {
146 /* wake up clcsock accept */
147 rc = kernel_sock_shutdown(smc->clcsock, SHUT_RDWR);
148 }
149 sk->sk_state = SMC_CLOSED;
150 sk->sk_state_change(sk);
151 smc_restore_fallback_changes(smc);
152 }
153
154 sk->sk_prot->unhash(sk);
155
156 if (sk->sk_state == SMC_CLOSED) {
157 if (smc->clcsock) {
158 release_sock(sk);
159 smc_clcsock_release(smc);
160 lock_sock(sk);
161 }
162 if (!smc->use_fallback)
163 smc_conn_free(&smc->conn);
164 }
165
166 return rc;
167 }
168
169 static int smc_release(struct socket *sock)
170 {
171 struct sock *sk = sock->sk;
172 struct smc_sock *smc;
173 int rc = 0;
174
175 if (!sk)
176 goto out;
177
178 sock_hold(sk); /* sock_put below */
179 smc = smc_sk(sk);
180
181 /* cleanup for a dangling non-blocking connect */
182 if (smc->connect_nonblock && sk->sk_state == SMC_INIT)
183 tcp_abort(smc->clcsock->sk, ECONNABORTED);
184 flush_work(&smc->connect_work);
185
186 if (sk->sk_state == SMC_LISTEN)
187 /* smc_close_non_accepted() is called and acquires
188 * sock lock for child sockets again
189 */
190 lock_sock_nested(sk, SINGLE_DEPTH_NESTING);
191 else
192 lock_sock(sk);
193
194 rc = __smc_release(smc);
195
196 /* detach socket */
197 sock_orphan(sk);
198 sock->sk = NULL;
199 release_sock(sk);
200
201 sock_put(sk); /* sock_hold above */
202 sock_put(sk); /* final sock_put */
203 out:
204 return rc;
205 }
206
207 static void smc_destruct(struct sock *sk)
208 {
209 if (sk->sk_state != SMC_CLOSED)
210 return;
211 if (!sock_flag(sk, SOCK_DEAD))
212 return;
213
214 sk_refcnt_debug_dec(sk);
215 }
216
217 static struct sock *smc_sock_alloc(struct net *net, struct socket *sock,
218 int protocol)
219 {
220 struct smc_sock *smc;
221 struct proto *prot;
222 struct sock *sk;
223
224 prot = (protocol == SMCPROTO_SMC6) ? &smc_proto6 : &smc_proto;
225 sk = sk_alloc(net, PF_SMC, GFP_KERNEL, prot, 0);
226 if (!sk)
227 return NULL;
228
229 sock_init_data(sock, sk); /* sets sk_refcnt to 1 */
230 sk->sk_state = SMC_INIT;
231 sk->sk_destruct = smc_destruct;
232 sk->sk_protocol = protocol;
233 smc = smc_sk(sk);
234 INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work);
235 INIT_WORK(&smc->connect_work, smc_connect_work);
236 INIT_DELAYED_WORK(&smc->conn.tx_work, smc_tx_work);
237 INIT_LIST_HEAD(&smc->accept_q);
238 spin_lock_init(&smc->accept_q_lock);
239 spin_lock_init(&smc->conn.send_lock);
240 sk->sk_prot->hash(sk);
241 sk_refcnt_debug_inc(sk);
242 mutex_init(&smc->clcsock_release_lock);
243
244 return sk;
245 }
246
247 static int smc_bind(struct socket *sock, struct sockaddr *uaddr,
248 int addr_len)
249 {
250 struct sockaddr_in *addr = (struct sockaddr_in *)uaddr;
251 struct sock *sk = sock->sk;
252 struct smc_sock *smc;
253 int rc;
254
255 smc = smc_sk(sk);
256
257 /* replicate tests from inet_bind(), to be safe wrt. future changes */
258 rc = -EINVAL;
259 if (addr_len < sizeof(struct sockaddr_in))
260 goto out;
261
262 rc = -EAFNOSUPPORT;
263 if (addr->sin_family != AF_INET &&
264 addr->sin_family != AF_INET6 &&
265 addr->sin_family != AF_UNSPEC)
266 goto out;
267 /* accept AF_UNSPEC (mapped to AF_INET) only if s_addr is INADDR_ANY */
268 if (addr->sin_family == AF_UNSPEC &&
269 addr->sin_addr.s_addr != htonl(INADDR_ANY))
270 goto out;
271
272 lock_sock(sk);
273
274 /* Check if socket is already active */
275 rc = -EINVAL;
276 if (sk->sk_state != SMC_INIT || smc->connect_nonblock)
277 goto out_rel;
278
279 smc->clcsock->sk->sk_reuse = sk->sk_reuse;
280 rc = kernel_bind(smc->clcsock, uaddr, addr_len);
281
282 out_rel:
283 release_sock(sk);
284 out:
285 return rc;
286 }
287
288 static void smc_copy_sock_settings(struct sock *nsk, struct sock *osk,
289 unsigned long mask)
290 {
291 /* options we don't get control via setsockopt for */
292 nsk->sk_type = osk->sk_type;
293 nsk->sk_sndbuf = osk->sk_sndbuf;
294 nsk->sk_rcvbuf = osk->sk_rcvbuf;
295 nsk->sk_sndtimeo = osk->sk_sndtimeo;
296 nsk->sk_rcvtimeo = osk->sk_rcvtimeo;
297 nsk->sk_mark = osk->sk_mark;
298 nsk->sk_priority = osk->sk_priority;
299 nsk->sk_rcvlowat = osk->sk_rcvlowat;
300 nsk->sk_bound_dev_if = osk->sk_bound_dev_if;
301 nsk->sk_err = osk->sk_err;
302
303 nsk->sk_flags &= ~mask;
304 nsk->sk_flags |= osk->sk_flags & mask;
305 }
306
307 #define SK_FLAGS_SMC_TO_CLC ((1UL << SOCK_URGINLINE) | \
308 (1UL << SOCK_KEEPOPEN) | \
309 (1UL << SOCK_LINGER) | \
310 (1UL << SOCK_BROADCAST) | \
311 (1UL << SOCK_TIMESTAMP) | \
312 (1UL << SOCK_DBG) | \
313 (1UL << SOCK_RCVTSTAMP) | \
314 (1UL << SOCK_RCVTSTAMPNS) | \
315 (1UL << SOCK_LOCALROUTE) | \
316 (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE) | \
317 (1UL << SOCK_RXQ_OVFL) | \
318 (1UL << SOCK_WIFI_STATUS) | \
319 (1UL << SOCK_NOFCS) | \
320 (1UL << SOCK_FILTER_LOCKED) | \
321 (1UL << SOCK_TSTAMP_NEW))
322 /* copy only relevant settings and flags of SOL_SOCKET level from smc to
323 * clc socket (since smc is not called for these options from net/core)
324 */
325 static void smc_copy_sock_settings_to_clc(struct smc_sock *smc)
326 {
327 smc_copy_sock_settings(smc->clcsock->sk, &smc->sk, SK_FLAGS_SMC_TO_CLC);
328 }
329
330 #define SK_FLAGS_CLC_TO_SMC ((1UL << SOCK_URGINLINE) | \
331 (1UL << SOCK_KEEPOPEN) | \
332 (1UL << SOCK_LINGER) | \
333 (1UL << SOCK_DBG))
334 /* copy only settings and flags relevant for smc from clc to smc socket */
335 static void smc_copy_sock_settings_to_smc(struct smc_sock *smc)
336 {
337 smc_copy_sock_settings(&smc->sk, smc->clcsock->sk, SK_FLAGS_CLC_TO_SMC);
338 }
339
340 /* register a new rmb, send confirm_rkey msg to register with peer */
341 static int smc_reg_rmb(struct smc_link *link, struct smc_buf_desc *rmb_desc,
342 bool conf_rkey)
343 {
344 if (!rmb_desc->wr_reg) {
345 /* register memory region for new rmb */
346 if (smc_wr_reg_send(link, rmb_desc->mr_rx[SMC_SINGLE_LINK])) {
347 rmb_desc->regerr = 1;
348 return -EFAULT;
349 }
350 rmb_desc->wr_reg = 1;
351 }
352 if (!conf_rkey)
353 return 0;
354 /* exchange confirm_rkey msg with peer */
355 if (smc_llc_do_confirm_rkey(link, rmb_desc)) {
356 rmb_desc->regerr = 1;
357 return -EFAULT;
358 }
359 return 0;
360 }
361
362 static int smc_clnt_conf_first_link(struct smc_sock *smc)
363 {
364 struct net *net = sock_net(smc->clcsock->sk);
365 struct smc_link_group *lgr = smc->conn.lgr;
366 struct smc_link *link;
367 int rest;
368 int rc;
369
370 link = &lgr->lnk[SMC_SINGLE_LINK];
371 /* receive CONFIRM LINK request from server over RoCE fabric */
372 rest = wait_for_completion_interruptible_timeout(
373 &link->llc_confirm,
374 SMC_LLC_WAIT_FIRST_TIME);
375 if (rest <= 0) {
376 struct smc_clc_msg_decline dclc;
377
378 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
379 SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
380 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc;
381 }
382
383 if (link->llc_confirm_rc)
384 return SMC_CLC_DECL_RMBE_EC;
385
386 rc = smc_ib_modify_qp_rts(link);
387 if (rc)
388 return SMC_CLC_DECL_ERR_RDYLNK;
389
390 smc_wr_remember_qp_attr(link);
391
392 if (smc_reg_rmb(link, smc->conn.rmb_desc, false))
393 return SMC_CLC_DECL_ERR_REGRMB;
394
395 /* send CONFIRM LINK response over RoCE fabric */
396 rc = smc_llc_send_confirm_link(link, SMC_LLC_RESP);
397 if (rc < 0)
398 return SMC_CLC_DECL_TIMEOUT_CL;
399
400 /* receive ADD LINK request from server over RoCE fabric */
401 rest = wait_for_completion_interruptible_timeout(&link->llc_add,
402 SMC_LLC_WAIT_TIME);
403 if (rest <= 0) {
404 struct smc_clc_msg_decline dclc;
405
406 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
407 SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
408 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_AL : rc;
409 }
410
411 /* send add link reject message, only one link supported for now */
412 rc = smc_llc_send_add_link(link,
413 link->smcibdev->mac[link->ibport - 1],
414 link->gid, SMC_LLC_RESP);
415 if (rc < 0)
416 return SMC_CLC_DECL_TIMEOUT_AL;
417
418 smc_llc_link_active(link, net->ipv4.sysctl_tcp_keepalive_time);
419
420 return 0;
421 }
422
423 static void smcr_conn_save_peer_info(struct smc_sock *smc,
424 struct smc_clc_msg_accept_confirm *clc)
425 {
426 int bufsize = smc_uncompress_bufsize(clc->rmbe_size);
427
428 smc->conn.peer_rmbe_idx = clc->rmbe_idx;
429 smc->conn.local_tx_ctrl.token = ntohl(clc->rmbe_alert_token);
430 smc->conn.peer_rmbe_size = bufsize;
431 atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
432 smc->conn.tx_off = bufsize * (smc->conn.peer_rmbe_idx - 1);
433 }
434
435 static void smcd_conn_save_peer_info(struct smc_sock *smc,
436 struct smc_clc_msg_accept_confirm *clc)
437 {
438 int bufsize = smc_uncompress_bufsize(clc->dmbe_size);
439
440 smc->conn.peer_rmbe_idx = clc->dmbe_idx;
441 smc->conn.peer_token = clc->token;
442 /* msg header takes up space in the buffer */
443 smc->conn.peer_rmbe_size = bufsize - sizeof(struct smcd_cdc_msg);
444 atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
445 smc->conn.tx_off = bufsize * smc->conn.peer_rmbe_idx;
446 }
447
448 static void smc_conn_save_peer_info(struct smc_sock *smc,
449 struct smc_clc_msg_accept_confirm *clc)
450 {
451 if (smc->conn.lgr->is_smcd)
452 smcd_conn_save_peer_info(smc, clc);
453 else
454 smcr_conn_save_peer_info(smc, clc);
455 }
456
457 static void smc_link_save_peer_info(struct smc_link *link,
458 struct smc_clc_msg_accept_confirm *clc)
459 {
460 link->peer_qpn = ntoh24(clc->qpn);
461 memcpy(link->peer_gid, clc->lcl.gid, SMC_GID_SIZE);
462 memcpy(link->peer_mac, clc->lcl.mac, sizeof(link->peer_mac));
463 link->peer_psn = ntoh24(clc->psn);
464 link->peer_mtu = clc->qp_mtu;
465 }
466
467 static void smc_switch_to_fallback(struct smc_sock *smc)
468 {
469 smc->use_fallback = true;
470 if (smc->sk.sk_socket && smc->sk.sk_socket->file) {
471 smc->clcsock->file = smc->sk.sk_socket->file;
472 smc->clcsock->file->private_data = smc->clcsock;
473 smc->clcsock->wq.fasync_list =
474 smc->sk.sk_socket->wq.fasync_list;
475 }
476 }
477
478 /* fall back during connect */
479 static int smc_connect_fallback(struct smc_sock *smc, int reason_code)
480 {
481 smc_switch_to_fallback(smc);
482 smc->fallback_rsn = reason_code;
483 smc_copy_sock_settings_to_clc(smc);
484 smc->connect_nonblock = 0;
485 if (smc->sk.sk_state == SMC_INIT)
486 smc->sk.sk_state = SMC_ACTIVE;
487 return 0;
488 }
489
490 /* decline and fall back during connect */
491 static int smc_connect_decline_fallback(struct smc_sock *smc, int reason_code)
492 {
493 int rc;
494
495 if (reason_code < 0) { /* error, fallback is not possible */
496 if (smc->sk.sk_state == SMC_INIT)
497 sock_put(&smc->sk); /* passive closing */
498 return reason_code;
499 }
500 if (reason_code != SMC_CLC_DECL_PEERDECL) {
501 rc = smc_clc_send_decline(smc, reason_code);
502 if (rc < 0) {
503 if (smc->sk.sk_state == SMC_INIT)
504 sock_put(&smc->sk); /* passive closing */
505 return rc;
506 }
507 }
508 return smc_connect_fallback(smc, reason_code);
509 }
510
511 /* abort connecting */
512 static int smc_connect_abort(struct smc_sock *smc, int reason_code,
513 int local_contact)
514 {
515 bool is_smcd = smc->conn.lgr->is_smcd;
516
517 if (local_contact == SMC_FIRST_CONTACT)
518 smc_lgr_cleanup_early(&smc->conn);
519 else
520 smc_conn_free(&smc->conn);
521 if (is_smcd)
522 /* there is only one lgr role for SMC-D; use server lock */
523 mutex_unlock(&smc_server_lgr_pending);
524 else
525 mutex_unlock(&smc_client_lgr_pending);
526
527 smc->connect_nonblock = 0;
528 return reason_code;
529 }
530
531 /* check if there is a rdma device available for this connection. */
532 /* called for connect and listen */
533 static int smc_find_rdma_device(struct smc_sock *smc, struct smc_init_info *ini)
534 {
535 /* PNET table look up: search active ib_device and port
536 * within same PNETID that also contains the ethernet device
537 * used for the internal TCP socket
538 */
539 smc_pnet_find_roce_resource(smc->clcsock->sk, ini);
540 if (!ini->ib_dev)
541 return SMC_CLC_DECL_NOSMCRDEV;
542 return 0;
543 }
544
545 /* check if there is an ISM device available for this connection. */
546 /* called for connect and listen */
547 static int smc_find_ism_device(struct smc_sock *smc, struct smc_init_info *ini)
548 {
549 /* Find ISM device with same PNETID as connecting interface */
550 smc_pnet_find_ism_resource(smc->clcsock->sk, ini);
551 if (!ini->ism_dev)
552 return SMC_CLC_DECL_NOSMCDDEV;
553 return 0;
554 }
555
556 /* Check for VLAN ID and register it on ISM device just for CLC handshake */
557 static int smc_connect_ism_vlan_setup(struct smc_sock *smc,
558 struct smc_init_info *ini)
559 {
560 if (ini->vlan_id && smc_ism_get_vlan(ini->ism_dev, ini->vlan_id))
561 return SMC_CLC_DECL_ISMVLANERR;
562 return 0;
563 }
564
565 /* cleanup temporary VLAN ID registration used for CLC handshake. If ISM is
566 * used, the VLAN ID will be registered again during the connection setup.
567 */
568 static int smc_connect_ism_vlan_cleanup(struct smc_sock *smc, bool is_smcd,
569 struct smc_init_info *ini)
570 {
571 if (!is_smcd)
572 return 0;
573 if (ini->vlan_id && smc_ism_put_vlan(ini->ism_dev, ini->vlan_id))
574 return SMC_CLC_DECL_CNFERR;
575 return 0;
576 }
577
578 /* CLC handshake during connect */
579 static int smc_connect_clc(struct smc_sock *smc, int smc_type,
580 struct smc_clc_msg_accept_confirm *aclc,
581 struct smc_init_info *ini)
582 {
583 int rc = 0;
584
585 /* do inband token exchange */
586 rc = smc_clc_send_proposal(smc, smc_type, ini);
587 if (rc)
588 return rc;
589 /* receive SMC Accept CLC message */
590 return smc_clc_wait_msg(smc, aclc, sizeof(*aclc), SMC_CLC_ACCEPT,
591 CLC_WAIT_TIME);
592 }
593
594 /* setup for RDMA connection of client */
595 static int smc_connect_rdma(struct smc_sock *smc,
596 struct smc_clc_msg_accept_confirm *aclc,
597 struct smc_init_info *ini)
598 {
599 struct smc_link *link;
600 int reason_code = 0;
601
602 ini->is_smcd = false;
603 ini->ib_lcl = &aclc->lcl;
604 ini->ib_clcqpn = ntoh24(aclc->qpn);
605 ini->srv_first_contact = aclc->hdr.flag;
606
607 mutex_lock(&smc_client_lgr_pending);
608 reason_code = smc_conn_create(smc, ini);
609 if (reason_code) {
610 mutex_unlock(&smc_client_lgr_pending);
611 return reason_code;
612 }
613 link = &smc->conn.lgr->lnk[SMC_SINGLE_LINK];
614
615 smc_conn_save_peer_info(smc, aclc);
616
617 /* create send buffer and rmb */
618 if (smc_buf_create(smc, false))
619 return smc_connect_abort(smc, SMC_CLC_DECL_MEM,
620 ini->cln_first_contact);
621
622 if (ini->cln_first_contact == SMC_FIRST_CONTACT)
623 smc_link_save_peer_info(link, aclc);
624
625 if (smc_rmb_rtoken_handling(&smc->conn, aclc))
626 return smc_connect_abort(smc, SMC_CLC_DECL_ERR_RTOK,
627 ini->cln_first_contact);
628
629 smc_close_init(smc);
630 smc_rx_init(smc);
631
632 if (ini->cln_first_contact == SMC_FIRST_CONTACT) {
633 if (smc_ib_ready_link(link))
634 return smc_connect_abort(smc, SMC_CLC_DECL_ERR_RDYLNK,
635 ini->cln_first_contact);
636 } else {
637 if (smc_reg_rmb(link, smc->conn.rmb_desc, true))
638 return smc_connect_abort(smc, SMC_CLC_DECL_ERR_REGRMB,
639 ini->cln_first_contact);
640 }
641 smc_rmb_sync_sg_for_device(&smc->conn);
642
643 reason_code = smc_clc_send_confirm(smc);
644 if (reason_code)
645 return smc_connect_abort(smc, reason_code,
646 ini->cln_first_contact);
647
648 smc_tx_init(smc);
649
650 if (ini->cln_first_contact == SMC_FIRST_CONTACT) {
651 /* QP confirmation over RoCE fabric */
652 reason_code = smc_clnt_conf_first_link(smc);
653 if (reason_code)
654 return smc_connect_abort(smc, reason_code,
655 ini->cln_first_contact);
656 }
657 mutex_unlock(&smc_client_lgr_pending);
658
659 smc_copy_sock_settings_to_clc(smc);
660 smc->connect_nonblock = 0;
661 if (smc->sk.sk_state == SMC_INIT)
662 smc->sk.sk_state = SMC_ACTIVE;
663
664 return 0;
665 }
666
667 /* setup for ISM connection of client */
668 static int smc_connect_ism(struct smc_sock *smc,
669 struct smc_clc_msg_accept_confirm *aclc,
670 struct smc_init_info *ini)
671 {
672 int rc = 0;
673
674 ini->is_smcd = true;
675 ini->ism_gid = aclc->gid;
676 ini->srv_first_contact = aclc->hdr.flag;
677
678 /* there is only one lgr role for SMC-D; use server lock */
679 mutex_lock(&smc_server_lgr_pending);
680 rc = smc_conn_create(smc, ini);
681 if (rc) {
682 mutex_unlock(&smc_server_lgr_pending);
683 return rc;
684 }
685
686 /* Create send and receive buffers */
687 if (smc_buf_create(smc, true))
688 return smc_connect_abort(smc, SMC_CLC_DECL_MEM,
689 ini->cln_first_contact);
690
691 smc_conn_save_peer_info(smc, aclc);
692 smc_close_init(smc);
693 smc_rx_init(smc);
694 smc_tx_init(smc);
695
696 rc = smc_clc_send_confirm(smc);
697 if (rc)
698 return smc_connect_abort(smc, rc, ini->cln_first_contact);
699 mutex_unlock(&smc_server_lgr_pending);
700
701 smc_copy_sock_settings_to_clc(smc);
702 smc->connect_nonblock = 0;
703 if (smc->sk.sk_state == SMC_INIT)
704 smc->sk.sk_state = SMC_ACTIVE;
705
706 return 0;
707 }
708
709 /* perform steps before actually connecting */
710 static int __smc_connect(struct smc_sock *smc)
711 {
712 bool ism_supported = false, rdma_supported = false;
713 struct smc_clc_msg_accept_confirm aclc;
714 struct smc_init_info ini = {0};
715 int smc_type;
716 int rc = 0;
717
718 if (smc->use_fallback)
719 return smc_connect_fallback(smc, smc->fallback_rsn);
720
721 /* if peer has not signalled SMC-capability, fall back */
722 if (!tcp_sk(smc->clcsock->sk)->syn_smc)
723 return smc_connect_fallback(smc, SMC_CLC_DECL_PEERNOSMC);
724
725 /* IPSec connections opt out of SMC-R optimizations */
726 if (using_ipsec(smc))
727 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_IPSEC);
728
729 /* get vlan id from IP device */
730 if (smc_vlan_by_tcpsk(smc->clcsock, &ini))
731 return smc_connect_decline_fallback(smc,
732 SMC_CLC_DECL_GETVLANERR);
733
734 /* check if there is an ism device available */
735 if (!smc_find_ism_device(smc, &ini) &&
736 !smc_connect_ism_vlan_setup(smc, &ini)) {
737 /* ISM is supported for this connection */
738 ism_supported = true;
739 smc_type = SMC_TYPE_D;
740 }
741
742 /* check if there is a rdma device available */
743 if (!smc_find_rdma_device(smc, &ini)) {
744 /* RDMA is supported for this connection */
745 rdma_supported = true;
746 if (ism_supported)
747 smc_type = SMC_TYPE_B; /* both */
748 else
749 smc_type = SMC_TYPE_R; /* only RDMA */
750 }
751
752 /* if neither ISM nor RDMA are supported, fallback */
753 if (!rdma_supported && !ism_supported)
754 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_NOSMCDEV);
755
756 /* perform CLC handshake */
757 rc = smc_connect_clc(smc, smc_type, &aclc, &ini);
758 if (rc) {
759 smc_connect_ism_vlan_cleanup(smc, ism_supported, &ini);
760 return smc_connect_decline_fallback(smc, rc);
761 }
762
763 /* depending on previous steps, connect using rdma or ism */
764 if (rdma_supported && aclc.hdr.path == SMC_TYPE_R)
765 rc = smc_connect_rdma(smc, &aclc, &ini);
766 else if (ism_supported && aclc.hdr.path == SMC_TYPE_D)
767 rc = smc_connect_ism(smc, &aclc, &ini);
768 else
769 rc = SMC_CLC_DECL_MODEUNSUPP;
770 if (rc) {
771 smc_connect_ism_vlan_cleanup(smc, ism_supported, &ini);
772 return smc_connect_decline_fallback(smc, rc);
773 }
774
775 smc_connect_ism_vlan_cleanup(smc, ism_supported, &ini);
776 return 0;
777 }
778
779 static void smc_connect_work(struct work_struct *work)
780 {
781 struct smc_sock *smc = container_of(work, struct smc_sock,
782 connect_work);
783 long timeo = smc->sk.sk_sndtimeo;
784 int rc = 0;
785
786 if (!timeo)
787 timeo = MAX_SCHEDULE_TIMEOUT;
788 lock_sock(smc->clcsock->sk);
789 if (smc->clcsock->sk->sk_err) {
790 smc->sk.sk_err = smc->clcsock->sk->sk_err;
791 } else if ((1 << smc->clcsock->sk->sk_state) &
792 (TCPF_SYN_SENT | TCP_SYN_RECV)) {
793 rc = sk_stream_wait_connect(smc->clcsock->sk, &timeo);
794 if ((rc == -EPIPE) &&
795 ((1 << smc->clcsock->sk->sk_state) &
796 (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)))
797 rc = 0;
798 }
799 release_sock(smc->clcsock->sk);
800 lock_sock(&smc->sk);
801 if (rc != 0 || smc->sk.sk_err) {
802 smc->sk.sk_state = SMC_CLOSED;
803 if (rc == -EPIPE || rc == -EAGAIN)
804 smc->sk.sk_err = EPIPE;
805 else if (signal_pending(current))
806 smc->sk.sk_err = -sock_intr_errno(timeo);
807 sock_put(&smc->sk); /* passive closing */
808 goto out;
809 }
810
811 rc = __smc_connect(smc);
812 if (rc < 0)
813 smc->sk.sk_err = -rc;
814
815 out:
816 if (!sock_flag(&smc->sk, SOCK_DEAD)) {
817 if (smc->sk.sk_err) {
818 smc->sk.sk_state_change(&smc->sk);
819 } else { /* allow polling before and after fallback decision */
820 smc->clcsock->sk->sk_write_space(smc->clcsock->sk);
821 smc->sk.sk_write_space(&smc->sk);
822 }
823 }
824 release_sock(&smc->sk);
825 }
826
827 static int smc_connect(struct socket *sock, struct sockaddr *addr,
828 int alen, int flags)
829 {
830 struct sock *sk = sock->sk;
831 struct smc_sock *smc;
832 int rc = -EINVAL;
833
834 smc = smc_sk(sk);
835
836 /* separate smc parameter checking to be safe */
837 if (alen < sizeof(addr->sa_family))
838 goto out_err;
839 if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6)
840 goto out_err;
841
842 lock_sock(sk);
843 switch (sk->sk_state) {
844 default:
845 goto out;
846 case SMC_ACTIVE:
847 rc = -EISCONN;
848 goto out;
849 case SMC_INIT:
850 rc = 0;
851 break;
852 }
853
854 smc_copy_sock_settings_to_clc(smc);
855 tcp_sk(smc->clcsock->sk)->syn_smc = 1;
856 if (smc->connect_nonblock) {
857 rc = -EALREADY;
858 goto out;
859 }
860 rc = kernel_connect(smc->clcsock, addr, alen, flags);
861 if (rc && rc != -EINPROGRESS)
862 goto out;
863
864 sock_hold(&smc->sk); /* sock put in passive closing */
865 if (smc->use_fallback)
866 goto out;
867 if (flags & O_NONBLOCK) {
868 if (schedule_work(&smc->connect_work))
869 smc->connect_nonblock = 1;
870 rc = -EINPROGRESS;
871 } else {
872 rc = __smc_connect(smc);
873 if (rc < 0)
874 goto out;
875 else
876 rc = 0; /* success cases including fallback */
877 }
878
879 out:
880 release_sock(sk);
881 out_err:
882 return rc;
883 }
884
885 static int smc_clcsock_accept(struct smc_sock *lsmc, struct smc_sock **new_smc)
886 {
887 struct socket *new_clcsock = NULL;
888 struct sock *lsk = &lsmc->sk;
889 struct sock *new_sk;
890 int rc = -EINVAL;
891
892 release_sock(lsk);
893 new_sk = smc_sock_alloc(sock_net(lsk), NULL, lsk->sk_protocol);
894 if (!new_sk) {
895 rc = -ENOMEM;
896 lsk->sk_err = ENOMEM;
897 *new_smc = NULL;
898 lock_sock(lsk);
899 goto out;
900 }
901 *new_smc = smc_sk(new_sk);
902
903 mutex_lock(&lsmc->clcsock_release_lock);
904 if (lsmc->clcsock)
905 rc = kernel_accept(lsmc->clcsock, &new_clcsock, 0);
906 mutex_unlock(&lsmc->clcsock_release_lock);
907 lock_sock(lsk);
908 if (rc < 0)
909 lsk->sk_err = -rc;
910 if (rc < 0 || lsk->sk_state == SMC_CLOSED) {
911 new_sk->sk_prot->unhash(new_sk);
912 if (new_clcsock)
913 sock_release(new_clcsock);
914 new_sk->sk_state = SMC_CLOSED;
915 sock_set_flag(new_sk, SOCK_DEAD);
916 sock_put(new_sk); /* final */
917 *new_smc = NULL;
918 goto out;
919 }
920
921 (*new_smc)->clcsock = new_clcsock;
922 out:
923 return rc;
924 }
925
926 /* add a just created sock to the accept queue of the listen sock as
927 * candidate for a following socket accept call from user space
928 */
929 static void smc_accept_enqueue(struct sock *parent, struct sock *sk)
930 {
931 struct smc_sock *par = smc_sk(parent);
932
933 sock_hold(sk); /* sock_put in smc_accept_unlink () */
934 spin_lock(&par->accept_q_lock);
935 list_add_tail(&smc_sk(sk)->accept_q, &par->accept_q);
936 spin_unlock(&par->accept_q_lock);
937 sk_acceptq_added(parent);
938 }
939
940 /* remove a socket from the accept queue of its parental listening socket */
941 static void smc_accept_unlink(struct sock *sk)
942 {
943 struct smc_sock *par = smc_sk(sk)->listen_smc;
944
945 spin_lock(&par->accept_q_lock);
946 list_del_init(&smc_sk(sk)->accept_q);
947 spin_unlock(&par->accept_q_lock);
948 sk_acceptq_removed(&smc_sk(sk)->listen_smc->sk);
949 sock_put(sk); /* sock_hold in smc_accept_enqueue */
950 }
951
952 /* remove a sock from the accept queue to bind it to a new socket created
953 * for a socket accept call from user space
954 */
955 struct sock *smc_accept_dequeue(struct sock *parent,
956 struct socket *new_sock)
957 {
958 struct smc_sock *isk, *n;
959 struct sock *new_sk;
960
961 list_for_each_entry_safe(isk, n, &smc_sk(parent)->accept_q, accept_q) {
962 new_sk = (struct sock *)isk;
963
964 smc_accept_unlink(new_sk);
965 if (new_sk->sk_state == SMC_CLOSED) {
966 new_sk->sk_prot->unhash(new_sk);
967 if (isk->clcsock) {
968 sock_release(isk->clcsock);
969 isk->clcsock = NULL;
970 }
971 sock_put(new_sk); /* final */
972 continue;
973 }
974 if (new_sock) {
975 sock_graft(new_sk, new_sock);
976 if (isk->use_fallback) {
977 smc_sk(new_sk)->clcsock->file = new_sock->file;
978 isk->clcsock->file->private_data = isk->clcsock;
979 }
980 }
981 return new_sk;
982 }
983 return NULL;
984 }
985
986 /* clean up for a created but never accepted sock */
987 void smc_close_non_accepted(struct sock *sk)
988 {
989 struct smc_sock *smc = smc_sk(sk);
990
991 sock_hold(sk); /* sock_put below */
992 lock_sock(sk);
993 if (!sk->sk_lingertime)
994 /* wait for peer closing */
995 sk->sk_lingertime = SMC_MAX_STREAM_WAIT_TIMEOUT;
996 __smc_release(smc);
997 release_sock(sk);
998 sock_put(sk); /* sock_hold above */
999 sock_put(sk); /* final sock_put */
1000 }
1001
1002 static int smc_serv_conf_first_link(struct smc_sock *smc)
1003 {
1004 struct net *net = sock_net(smc->clcsock->sk);
1005 struct smc_link_group *lgr = smc->conn.lgr;
1006 struct smc_link *link;
1007 int rest;
1008 int rc;
1009
1010 link = &lgr->lnk[SMC_SINGLE_LINK];
1011
1012 if (smc_reg_rmb(link, smc->conn.rmb_desc, false))
1013 return SMC_CLC_DECL_ERR_REGRMB;
1014
1015 /* send CONFIRM LINK request to client over the RoCE fabric */
1016 rc = smc_llc_send_confirm_link(link, SMC_LLC_REQ);
1017 if (rc < 0)
1018 return SMC_CLC_DECL_TIMEOUT_CL;
1019
1020 /* receive CONFIRM LINK response from client over the RoCE fabric */
1021 rest = wait_for_completion_interruptible_timeout(
1022 &link->llc_confirm_resp,
1023 SMC_LLC_WAIT_FIRST_TIME);
1024 if (rest <= 0) {
1025 struct smc_clc_msg_decline dclc;
1026
1027 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
1028 SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
1029 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc;
1030 }
1031
1032 if (link->llc_confirm_resp_rc)
1033 return SMC_CLC_DECL_RMBE_EC;
1034
1035 /* send ADD LINK request to client over the RoCE fabric */
1036 rc = smc_llc_send_add_link(link,
1037 link->smcibdev->mac[link->ibport - 1],
1038 link->gid, SMC_LLC_REQ);
1039 if (rc < 0)
1040 return SMC_CLC_DECL_TIMEOUT_AL;
1041
1042 /* receive ADD LINK response from client over the RoCE fabric */
1043 rest = wait_for_completion_interruptible_timeout(&link->llc_add_resp,
1044 SMC_LLC_WAIT_TIME);
1045 if (rest <= 0) {
1046 struct smc_clc_msg_decline dclc;
1047
1048 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
1049 SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
1050 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_AL : rc;
1051 }
1052
1053 smc_llc_link_active(link, net->ipv4.sysctl_tcp_keepalive_time);
1054
1055 return 0;
1056 }
1057
1058 /* listen worker: finish */
1059 static void smc_listen_out(struct smc_sock *new_smc)
1060 {
1061 struct smc_sock *lsmc = new_smc->listen_smc;
1062 struct sock *newsmcsk = &new_smc->sk;
1063
1064 if (lsmc->sk.sk_state == SMC_LISTEN) {
1065 lock_sock_nested(&lsmc->sk, SINGLE_DEPTH_NESTING);
1066 smc_accept_enqueue(&lsmc->sk, newsmcsk);
1067 release_sock(&lsmc->sk);
1068 } else { /* no longer listening */
1069 smc_close_non_accepted(newsmcsk);
1070 }
1071
1072 /* Wake up accept */
1073 lsmc->sk.sk_data_ready(&lsmc->sk);
1074 sock_put(&lsmc->sk); /* sock_hold in smc_tcp_listen_work */
1075 }
1076
1077 /* listen worker: finish in state connected */
1078 static void smc_listen_out_connected(struct smc_sock *new_smc)
1079 {
1080 struct sock *newsmcsk = &new_smc->sk;
1081
1082 sk_refcnt_debug_inc(newsmcsk);
1083 if (newsmcsk->sk_state == SMC_INIT)
1084 newsmcsk->sk_state = SMC_ACTIVE;
1085
1086 smc_listen_out(new_smc);
1087 }
1088
1089 /* listen worker: finish in error state */
1090 static void smc_listen_out_err(struct smc_sock *new_smc)
1091 {
1092 struct sock *newsmcsk = &new_smc->sk;
1093
1094 if (newsmcsk->sk_state == SMC_INIT)
1095 sock_put(&new_smc->sk); /* passive closing */
1096 newsmcsk->sk_state = SMC_CLOSED;
1097
1098 smc_listen_out(new_smc);
1099 }
1100
1101 /* listen worker: decline and fall back if possible */
1102 static void smc_listen_decline(struct smc_sock *new_smc, int reason_code,
1103 int local_contact)
1104 {
1105 /* RDMA setup failed, switch back to TCP */
1106 if (local_contact == SMC_FIRST_CONTACT)
1107 smc_lgr_cleanup_early(&new_smc->conn);
1108 else
1109 smc_conn_free(&new_smc->conn);
1110 if (reason_code < 0) { /* error, no fallback possible */
1111 smc_listen_out_err(new_smc);
1112 return;
1113 }
1114 smc_switch_to_fallback(new_smc);
1115 new_smc->fallback_rsn = reason_code;
1116 if (reason_code && reason_code != SMC_CLC_DECL_PEERDECL) {
1117 if (smc_clc_send_decline(new_smc, reason_code) < 0) {
1118 smc_listen_out_err(new_smc);
1119 return;
1120 }
1121 }
1122 smc_listen_out_connected(new_smc);
1123 }
1124
1125 /* listen worker: check prefixes */
1126 static int smc_listen_prfx_check(struct smc_sock *new_smc,
1127 struct smc_clc_msg_proposal *pclc)
1128 {
1129 struct smc_clc_msg_proposal_prefix *pclc_prfx;
1130 struct socket *newclcsock = new_smc->clcsock;
1131
1132 pclc_prfx = smc_clc_proposal_get_prefix(pclc);
1133 if (smc_clc_prfx_match(newclcsock, pclc_prfx))
1134 return SMC_CLC_DECL_DIFFPREFIX;
1135
1136 return 0;
1137 }
1138
1139 /* listen worker: initialize connection and buffers */
1140 static int smc_listen_rdma_init(struct smc_sock *new_smc,
1141 struct smc_init_info *ini)
1142 {
1143 int rc;
1144
1145 /* allocate connection / link group */
1146 rc = smc_conn_create(new_smc, ini);
1147 if (rc)
1148 return rc;
1149
1150 /* create send buffer and rmb */
1151 if (smc_buf_create(new_smc, false))
1152 return SMC_CLC_DECL_MEM;
1153
1154 return 0;
1155 }
1156
1157 /* listen worker: initialize connection and buffers for SMC-D */
1158 static int smc_listen_ism_init(struct smc_sock *new_smc,
1159 struct smc_clc_msg_proposal *pclc,
1160 struct smc_init_info *ini)
1161 {
1162 struct smc_clc_msg_smcd *pclc_smcd;
1163 int rc;
1164
1165 pclc_smcd = smc_get_clc_msg_smcd(pclc);
1166 ini->ism_gid = pclc_smcd->gid;
1167 rc = smc_conn_create(new_smc, ini);
1168 if (rc)
1169 return rc;
1170
1171 /* Check if peer can be reached via ISM device */
1172 if (smc_ism_cantalk(new_smc->conn.lgr->peer_gid,
1173 new_smc->conn.lgr->vlan_id,
1174 new_smc->conn.lgr->smcd)) {
1175 if (ini->cln_first_contact == SMC_FIRST_CONTACT)
1176 smc_lgr_cleanup_early(&new_smc->conn);
1177 else
1178 smc_conn_free(&new_smc->conn);
1179 return SMC_CLC_DECL_SMCDNOTALK;
1180 }
1181
1182 /* Create send and receive buffers */
1183 if (smc_buf_create(new_smc, true)) {
1184 if (ini->cln_first_contact == SMC_FIRST_CONTACT)
1185 smc_lgr_cleanup_early(&new_smc->conn);
1186 else
1187 smc_conn_free(&new_smc->conn);
1188 return SMC_CLC_DECL_MEM;
1189 }
1190
1191 return 0;
1192 }
1193
1194 /* listen worker: register buffers */
1195 static int smc_listen_rdma_reg(struct smc_sock *new_smc, int local_contact)
1196 {
1197 struct smc_link *link = &new_smc->conn.lgr->lnk[SMC_SINGLE_LINK];
1198
1199 if (local_contact != SMC_FIRST_CONTACT) {
1200 if (smc_reg_rmb(link, new_smc->conn.rmb_desc, true))
1201 return SMC_CLC_DECL_ERR_REGRMB;
1202 }
1203 smc_rmb_sync_sg_for_device(&new_smc->conn);
1204
1205 return 0;
1206 }
1207
1208 /* listen worker: finish RDMA setup */
1209 static int smc_listen_rdma_finish(struct smc_sock *new_smc,
1210 struct smc_clc_msg_accept_confirm *cclc,
1211 int local_contact)
1212 {
1213 struct smc_link *link = &new_smc->conn.lgr->lnk[SMC_SINGLE_LINK];
1214 int reason_code = 0;
1215
1216 if (local_contact == SMC_FIRST_CONTACT)
1217 smc_link_save_peer_info(link, cclc);
1218
1219 if (smc_rmb_rtoken_handling(&new_smc->conn, cclc)) {
1220 reason_code = SMC_CLC_DECL_ERR_RTOK;
1221 goto decline;
1222 }
1223
1224 if (local_contact == SMC_FIRST_CONTACT) {
1225 if (smc_ib_ready_link(link)) {
1226 reason_code = SMC_CLC_DECL_ERR_RDYLNK;
1227 goto decline;
1228 }
1229 /* QP confirmation over RoCE fabric */
1230 reason_code = smc_serv_conf_first_link(new_smc);
1231 if (reason_code)
1232 goto decline;
1233 }
1234 return 0;
1235
1236 decline:
1237 smc_listen_decline(new_smc, reason_code, local_contact);
1238 return reason_code;
1239 }
1240
1241 /* setup for RDMA connection of server */
1242 static void smc_listen_work(struct work_struct *work)
1243 {
1244 struct smc_sock *new_smc = container_of(work, struct smc_sock,
1245 smc_listen_work);
1246 struct socket *newclcsock = new_smc->clcsock;
1247 struct smc_clc_msg_accept_confirm cclc;
1248 struct smc_clc_msg_proposal *pclc;
1249 struct smc_init_info ini = {0};
1250 bool ism_supported = false;
1251 u8 buf[SMC_CLC_MAX_LEN];
1252 int rc = 0;
1253
1254 if (new_smc->listen_smc->sk.sk_state != SMC_LISTEN)
1255 return smc_listen_out_err(new_smc);
1256
1257 if (new_smc->use_fallback) {
1258 smc_listen_out_connected(new_smc);
1259 return;
1260 }
1261
1262 /* check if peer is smc capable */
1263 if (!tcp_sk(newclcsock->sk)->syn_smc) {
1264 smc_switch_to_fallback(new_smc);
1265 new_smc->fallback_rsn = SMC_CLC_DECL_PEERNOSMC;
1266 smc_listen_out_connected(new_smc);
1267 return;
1268 }
1269
1270 /* do inband token exchange -
1271 * wait for and receive SMC Proposal CLC message
1272 */
1273 pclc = (struct smc_clc_msg_proposal *)&buf;
1274 rc = smc_clc_wait_msg(new_smc, pclc, SMC_CLC_MAX_LEN,
1275 SMC_CLC_PROPOSAL, CLC_WAIT_TIME);
1276 if (rc)
1277 goto out_decl;
1278
1279 /* IPSec connections opt out of SMC-R optimizations */
1280 if (using_ipsec(new_smc)) {
1281 rc = SMC_CLC_DECL_IPSEC;
1282 goto out_decl;
1283 }
1284
1285 /* check for matching IP prefix and subnet length */
1286 rc = smc_listen_prfx_check(new_smc, pclc);
1287 if (rc)
1288 goto out_decl;
1289
1290 /* get vlan id from IP device */
1291 if (smc_vlan_by_tcpsk(new_smc->clcsock, &ini)) {
1292 rc = SMC_CLC_DECL_GETVLANERR;
1293 goto out_decl;
1294 }
1295
1296 mutex_lock(&smc_server_lgr_pending);
1297 smc_close_init(new_smc);
1298 smc_rx_init(new_smc);
1299 smc_tx_init(new_smc);
1300
1301 /* check if ISM is available */
1302 if (pclc->hdr.path == SMC_TYPE_D || pclc->hdr.path == SMC_TYPE_B) {
1303 ini.is_smcd = true; /* prepare ISM check */
1304 rc = smc_find_ism_device(new_smc, &ini);
1305 if (!rc)
1306 rc = smc_listen_ism_init(new_smc, pclc, &ini);
1307 if (!rc)
1308 ism_supported = true;
1309 else if (pclc->hdr.path == SMC_TYPE_D)
1310 goto out_unlock; /* skip RDMA and decline */
1311 }
1312
1313 /* check if RDMA is available */
1314 if (!ism_supported) { /* SMC_TYPE_R or SMC_TYPE_B */
1315 /* prepare RDMA check */
1316 ini.is_smcd = false;
1317 ini.ism_dev = NULL;
1318 ini.ib_lcl = &pclc->lcl;
1319 rc = smc_find_rdma_device(new_smc, &ini);
1320 if (rc) {
1321 /* no RDMA device found */
1322 if (pclc->hdr.path == SMC_TYPE_B)
1323 /* neither ISM nor RDMA device found */
1324 rc = SMC_CLC_DECL_NOSMCDEV;
1325 goto out_unlock;
1326 }
1327 rc = smc_listen_rdma_init(new_smc, &ini);
1328 if (rc)
1329 goto out_unlock;
1330 rc = smc_listen_rdma_reg(new_smc, ini.cln_first_contact);
1331 if (rc)
1332 goto out_unlock;
1333 }
1334
1335 /* send SMC Accept CLC message */
1336 rc = smc_clc_send_accept(new_smc, ini.cln_first_contact);
1337 if (rc)
1338 goto out_unlock;
1339
1340 /* SMC-D does not need this lock any more */
1341 if (ism_supported)
1342 mutex_unlock(&smc_server_lgr_pending);
1343
1344 /* receive SMC Confirm CLC message */
1345 rc = smc_clc_wait_msg(new_smc, &cclc, sizeof(cclc),
1346 SMC_CLC_CONFIRM, CLC_WAIT_TIME);
1347 if (rc) {
1348 if (!ism_supported)
1349 goto out_unlock;
1350 goto out_decl;
1351 }
1352
1353 /* finish worker */
1354 if (!ism_supported) {
1355 rc = smc_listen_rdma_finish(new_smc, &cclc,
1356 ini.cln_first_contact);
1357 mutex_unlock(&smc_server_lgr_pending);
1358 if (rc)
1359 return;
1360 }
1361 smc_conn_save_peer_info(new_smc, &cclc);
1362 smc_listen_out_connected(new_smc);
1363 return;
1364
1365 out_unlock:
1366 mutex_unlock(&smc_server_lgr_pending);
1367 out_decl:
1368 smc_listen_decline(new_smc, rc, ini.cln_first_contact);
1369 }
1370
1371 static void smc_tcp_listen_work(struct work_struct *work)
1372 {
1373 struct smc_sock *lsmc = container_of(work, struct smc_sock,
1374 tcp_listen_work);
1375 struct sock *lsk = &lsmc->sk;
1376 struct smc_sock *new_smc;
1377 int rc = 0;
1378
1379 lock_sock(lsk);
1380 while (lsk->sk_state == SMC_LISTEN) {
1381 rc = smc_clcsock_accept(lsmc, &new_smc);
1382 if (rc)
1383 goto out;
1384 if (!new_smc)
1385 continue;
1386
1387 new_smc->listen_smc = lsmc;
1388 new_smc->use_fallback = lsmc->use_fallback;
1389 new_smc->fallback_rsn = lsmc->fallback_rsn;
1390 sock_hold(lsk); /* sock_put in smc_listen_work */
1391 INIT_WORK(&new_smc->smc_listen_work, smc_listen_work);
1392 smc_copy_sock_settings_to_smc(new_smc);
1393 new_smc->sk.sk_sndbuf = lsmc->sk.sk_sndbuf;
1394 new_smc->sk.sk_rcvbuf = lsmc->sk.sk_rcvbuf;
1395 sock_hold(&new_smc->sk); /* sock_put in passive closing */
1396 if (!schedule_work(&new_smc->smc_listen_work))
1397 sock_put(&new_smc->sk);
1398 }
1399
1400 out:
1401 release_sock(lsk);
1402 sock_put(&lsmc->sk); /* sock_hold in smc_listen */
1403 }
1404
1405 static int smc_listen(struct socket *sock, int backlog)
1406 {
1407 struct sock *sk = sock->sk;
1408 struct smc_sock *smc;
1409 int rc;
1410
1411 smc = smc_sk(sk);
1412 lock_sock(sk);
1413
1414 rc = -EINVAL;
1415 if ((sk->sk_state != SMC_INIT && sk->sk_state != SMC_LISTEN) ||
1416 smc->connect_nonblock)
1417 goto out;
1418
1419 rc = 0;
1420 if (sk->sk_state == SMC_LISTEN) {
1421 sk->sk_max_ack_backlog = backlog;
1422 goto out;
1423 }
1424 /* some socket options are handled in core, so we could not apply
1425 * them to the clc socket -- copy smc socket options to clc socket
1426 */
1427 smc_copy_sock_settings_to_clc(smc);
1428 if (!smc->use_fallback)
1429 tcp_sk(smc->clcsock->sk)->syn_smc = 1;
1430
1431 rc = kernel_listen(smc->clcsock, backlog);
1432 if (rc)
1433 goto out;
1434 sk->sk_max_ack_backlog = backlog;
1435 sk->sk_ack_backlog = 0;
1436 sk->sk_state = SMC_LISTEN;
1437 sock_hold(sk); /* sock_hold in tcp_listen_worker */
1438 if (!schedule_work(&smc->tcp_listen_work))
1439 sock_put(sk);
1440
1441 out:
1442 release_sock(sk);
1443 return rc;
1444 }
1445
1446 static int smc_accept(struct socket *sock, struct socket *new_sock,
1447 int flags, bool kern)
1448 {
1449 struct sock *sk = sock->sk, *nsk;
1450 DECLARE_WAITQUEUE(wait, current);
1451 struct smc_sock *lsmc;
1452 long timeo;
1453 int rc = 0;
1454
1455 lsmc = smc_sk(sk);
1456 sock_hold(sk); /* sock_put below */
1457 lock_sock(sk);
1458
1459 if (lsmc->sk.sk_state != SMC_LISTEN) {
1460 rc = -EINVAL;
1461 release_sock(sk);
1462 goto out;
1463 }
1464
1465 /* Wait for an incoming connection */
1466 timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK);
1467 add_wait_queue_exclusive(sk_sleep(sk), &wait);
1468 while (!(nsk = smc_accept_dequeue(sk, new_sock))) {
1469 set_current_state(TASK_INTERRUPTIBLE);
1470 if (!timeo) {
1471 rc = -EAGAIN;
1472 break;
1473 }
1474 release_sock(sk);
1475 timeo = schedule_timeout(timeo);
1476 /* wakeup by sk_data_ready in smc_listen_work() */
1477 sched_annotate_sleep();
1478 lock_sock(sk);
1479 if (signal_pending(current)) {
1480 rc = sock_intr_errno(timeo);
1481 break;
1482 }
1483 }
1484 set_current_state(TASK_RUNNING);
1485 remove_wait_queue(sk_sleep(sk), &wait);
1486
1487 if (!rc)
1488 rc = sock_error(nsk);
1489 release_sock(sk);
1490 if (rc)
1491 goto out;
1492
1493 if (lsmc->sockopt_defer_accept && !(flags & O_NONBLOCK)) {
1494 /* wait till data arrives on the socket */
1495 timeo = msecs_to_jiffies(lsmc->sockopt_defer_accept *
1496 MSEC_PER_SEC);
1497 if (smc_sk(nsk)->use_fallback) {
1498 struct sock *clcsk = smc_sk(nsk)->clcsock->sk;
1499
1500 lock_sock(clcsk);
1501 if (skb_queue_empty(&clcsk->sk_receive_queue))
1502 sk_wait_data(clcsk, &timeo, NULL);
1503 release_sock(clcsk);
1504 } else if (!atomic_read(&smc_sk(nsk)->conn.bytes_to_rcv)) {
1505 lock_sock(nsk);
1506 smc_rx_wait(smc_sk(nsk), &timeo, smc_rx_data_available);
1507 release_sock(nsk);
1508 }
1509 }
1510
1511 out:
1512 sock_put(sk); /* sock_hold above */
1513 return rc;
1514 }
1515
1516 static int smc_getname(struct socket *sock, struct sockaddr *addr,
1517 int peer)
1518 {
1519 struct smc_sock *smc;
1520
1521 if (peer && (sock->sk->sk_state != SMC_ACTIVE) &&
1522 (sock->sk->sk_state != SMC_APPCLOSEWAIT1))
1523 return -ENOTCONN;
1524
1525 smc = smc_sk(sock->sk);
1526
1527 return smc->clcsock->ops->getname(smc->clcsock, addr, peer);
1528 }
1529
1530 static int smc_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
1531 {
1532 struct sock *sk = sock->sk;
1533 struct smc_sock *smc;
1534 int rc = -EPIPE;
1535
1536 smc = smc_sk(sk);
1537 lock_sock(sk);
1538 if ((sk->sk_state != SMC_ACTIVE) &&
1539 (sk->sk_state != SMC_APPCLOSEWAIT1) &&
1540 (sk->sk_state != SMC_INIT))
1541 goto out;
1542
1543 if (msg->msg_flags & MSG_FASTOPEN) {
1544 if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) {
1545 smc_switch_to_fallback(smc);
1546 smc->fallback_rsn = SMC_CLC_DECL_OPTUNSUPP;
1547 } else {
1548 rc = -EINVAL;
1549 goto out;
1550 }
1551 }
1552
1553 if (smc->use_fallback)
1554 rc = smc->clcsock->ops->sendmsg(smc->clcsock, msg, len);
1555 else
1556 rc = smc_tx_sendmsg(smc, msg, len);
1557 out:
1558 release_sock(sk);
1559 return rc;
1560 }
1561
1562 static int smc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
1563 int flags)
1564 {
1565 struct sock *sk = sock->sk;
1566 struct smc_sock *smc;
1567 int rc = -ENOTCONN;
1568
1569 smc = smc_sk(sk);
1570 lock_sock(sk);
1571 if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) {
1572 /* socket was connected before, no more data to read */
1573 rc = 0;
1574 goto out;
1575 }
1576 if ((sk->sk_state == SMC_INIT) ||
1577 (sk->sk_state == SMC_LISTEN) ||
1578 (sk->sk_state == SMC_CLOSED))
1579 goto out;
1580
1581 if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
1582 rc = 0;
1583 goto out;
1584 }
1585
1586 if (smc->use_fallback) {
1587 rc = smc->clcsock->ops->recvmsg(smc->clcsock, msg, len, flags);
1588 } else {
1589 msg->msg_namelen = 0;
1590 rc = smc_rx_recvmsg(smc, msg, NULL, len, flags);
1591 }
1592
1593 out:
1594 release_sock(sk);
1595 return rc;
1596 }
1597
1598 static __poll_t smc_accept_poll(struct sock *parent)
1599 {
1600 struct smc_sock *isk = smc_sk(parent);
1601 __poll_t mask = 0;
1602
1603 spin_lock(&isk->accept_q_lock);
1604 if (!list_empty(&isk->accept_q))
1605 mask = EPOLLIN | EPOLLRDNORM;
1606 spin_unlock(&isk->accept_q_lock);
1607
1608 return mask;
1609 }
1610
1611 static __poll_t smc_poll(struct file *file, struct socket *sock,
1612 poll_table *wait)
1613 {
1614 struct sock *sk = sock->sk;
1615 struct smc_sock *smc;
1616 __poll_t mask = 0;
1617
1618 if (!sk)
1619 return EPOLLNVAL;
1620
1621 smc = smc_sk(sock->sk);
1622 if (smc->use_fallback) {
1623 /* delegate to CLC child sock */
1624 mask = smc->clcsock->ops->poll(file, smc->clcsock, wait);
1625 sk->sk_err = smc->clcsock->sk->sk_err;
1626 } else {
1627 if (sk->sk_state != SMC_CLOSED)
1628 sock_poll_wait(file, sock, wait);
1629 if (sk->sk_err)
1630 mask |= EPOLLERR;
1631 if ((sk->sk_shutdown == SHUTDOWN_MASK) ||
1632 (sk->sk_state == SMC_CLOSED))
1633 mask |= EPOLLHUP;
1634 if (sk->sk_state == SMC_LISTEN) {
1635 /* woken up by sk_data_ready in smc_listen_work() */
1636 mask |= smc_accept_poll(sk);
1637 } else if (smc->use_fallback) { /* as result of connect_work()*/
1638 mask |= smc->clcsock->ops->poll(file, smc->clcsock,
1639 wait);
1640 sk->sk_err = smc->clcsock->sk->sk_err;
1641 } else {
1642 if ((sk->sk_state != SMC_INIT &&
1643 atomic_read(&smc->conn.sndbuf_space)) ||
1644 sk->sk_shutdown & SEND_SHUTDOWN) {
1645 mask |= EPOLLOUT | EPOLLWRNORM;
1646 } else {
1647 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
1648 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1649 }
1650 if (atomic_read(&smc->conn.bytes_to_rcv))
1651 mask |= EPOLLIN | EPOLLRDNORM;
1652 if (sk->sk_shutdown & RCV_SHUTDOWN)
1653 mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
1654 if (sk->sk_state == SMC_APPCLOSEWAIT1)
1655 mask |= EPOLLIN;
1656 if (smc->conn.urg_state == SMC_URG_VALID)
1657 mask |= EPOLLPRI;
1658 }
1659 }
1660
1661 return mask;
1662 }
1663
1664 static int smc_shutdown(struct socket *sock, int how)
1665 {
1666 struct sock *sk = sock->sk;
1667 struct smc_sock *smc;
1668 int rc = -EINVAL;
1669 int rc1 = 0;
1670
1671 smc = smc_sk(sk);
1672
1673 if ((how < SHUT_RD) || (how > SHUT_RDWR))
1674 return rc;
1675
1676 lock_sock(sk);
1677
1678 rc = -ENOTCONN;
1679 if ((sk->sk_state != SMC_ACTIVE) &&
1680 (sk->sk_state != SMC_PEERCLOSEWAIT1) &&
1681 (sk->sk_state != SMC_PEERCLOSEWAIT2) &&
1682 (sk->sk_state != SMC_APPCLOSEWAIT1) &&
1683 (sk->sk_state != SMC_APPCLOSEWAIT2) &&
1684 (sk->sk_state != SMC_APPFINCLOSEWAIT))
1685 goto out;
1686 if (smc->use_fallback) {
1687 rc = kernel_sock_shutdown(smc->clcsock, how);
1688 sk->sk_shutdown = smc->clcsock->sk->sk_shutdown;
1689 if (sk->sk_shutdown == SHUTDOWN_MASK)
1690 sk->sk_state = SMC_CLOSED;
1691 goto out;
1692 }
1693 switch (how) {
1694 case SHUT_RDWR: /* shutdown in both directions */
1695 rc = smc_close_active(smc);
1696 break;
1697 case SHUT_WR:
1698 rc = smc_close_shutdown_write(smc);
1699 break;
1700 case SHUT_RD:
1701 rc = 0;
1702 /* nothing more to do because peer is not involved */
1703 break;
1704 }
1705 if (smc->clcsock)
1706 rc1 = kernel_sock_shutdown(smc->clcsock, how);
1707 /* map sock_shutdown_cmd constants to sk_shutdown value range */
1708 sk->sk_shutdown |= how + 1;
1709
1710 out:
1711 release_sock(sk);
1712 return rc ? rc : rc1;
1713 }
1714
1715 static int smc_setsockopt(struct socket *sock, int level, int optname,
1716 char __user *optval, unsigned int optlen)
1717 {
1718 struct sock *sk = sock->sk;
1719 struct smc_sock *smc;
1720 int val, rc;
1721
1722 smc = smc_sk(sk);
1723
1724 /* generic setsockopts reaching us here always apply to the
1725 * CLC socket
1726 */
1727 rc = smc->clcsock->ops->setsockopt(smc->clcsock, level, optname,
1728 optval, optlen);
1729 if (smc->clcsock->sk->sk_err) {
1730 sk->sk_err = smc->clcsock->sk->sk_err;
1731 sk->sk_error_report(sk);
1732 }
1733
1734 if (optlen < sizeof(int))
1735 return -EINVAL;
1736 if (get_user(val, (int __user *)optval))
1737 return -EFAULT;
1738
1739 lock_sock(sk);
1740 if (rc || smc->use_fallback)
1741 goto out;
1742 switch (optname) {
1743 case TCP_ULP:
1744 case TCP_FASTOPEN:
1745 case TCP_FASTOPEN_CONNECT:
1746 case TCP_FASTOPEN_KEY:
1747 case TCP_FASTOPEN_NO_COOKIE:
1748 /* option not supported by SMC */
1749 if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) {
1750 smc_switch_to_fallback(smc);
1751 smc->fallback_rsn = SMC_CLC_DECL_OPTUNSUPP;
1752 } else {
1753 rc = -EINVAL;
1754 }
1755 break;
1756 case TCP_NODELAY:
1757 if (sk->sk_state != SMC_INIT &&
1758 sk->sk_state != SMC_LISTEN &&
1759 sk->sk_state != SMC_CLOSED) {
1760 if (val)
1761 mod_delayed_work(system_wq, &smc->conn.tx_work,
1762 0);
1763 }
1764 break;
1765 case TCP_CORK:
1766 if (sk->sk_state != SMC_INIT &&
1767 sk->sk_state != SMC_LISTEN &&
1768 sk->sk_state != SMC_CLOSED) {
1769 if (!val)
1770 mod_delayed_work(system_wq, &smc->conn.tx_work,
1771 0);
1772 }
1773 break;
1774 case TCP_DEFER_ACCEPT:
1775 smc->sockopt_defer_accept = val;
1776 break;
1777 default:
1778 break;
1779 }
1780 out:
1781 release_sock(sk);
1782
1783 return rc;
1784 }
1785
1786 static int smc_getsockopt(struct socket *sock, int level, int optname,
1787 char __user *optval, int __user *optlen)
1788 {
1789 struct smc_sock *smc;
1790
1791 smc = smc_sk(sock->sk);
1792 /* socket options apply to the CLC socket */
1793 return smc->clcsock->ops->getsockopt(smc->clcsock, level, optname,
1794 optval, optlen);
1795 }
1796
1797 static int smc_ioctl(struct socket *sock, unsigned int cmd,
1798 unsigned long arg)
1799 {
1800 union smc_host_cursor cons, urg;
1801 struct smc_connection *conn;
1802 struct smc_sock *smc;
1803 int answ;
1804
1805 smc = smc_sk(sock->sk);
1806 conn = &smc->conn;
1807 lock_sock(&smc->sk);
1808 if (smc->use_fallback) {
1809 if (!smc->clcsock) {
1810 release_sock(&smc->sk);
1811 return -EBADF;
1812 }
1813 answ = smc->clcsock->ops->ioctl(smc->clcsock, cmd, arg);
1814 release_sock(&smc->sk);
1815 return answ;
1816 }
1817 switch (cmd) {
1818 case SIOCINQ: /* same as FIONREAD */
1819 if (smc->sk.sk_state == SMC_LISTEN) {
1820 release_sock(&smc->sk);
1821 return -EINVAL;
1822 }
1823 if (smc->sk.sk_state == SMC_INIT ||
1824 smc->sk.sk_state == SMC_CLOSED)
1825 answ = 0;
1826 else
1827 answ = atomic_read(&smc->conn.bytes_to_rcv);
1828 break;
1829 case SIOCOUTQ:
1830 /* output queue size (not send + not acked) */
1831 if (smc->sk.sk_state == SMC_LISTEN) {
1832 release_sock(&smc->sk);
1833 return -EINVAL;
1834 }
1835 if (smc->sk.sk_state == SMC_INIT ||
1836 smc->sk.sk_state == SMC_CLOSED)
1837 answ = 0;
1838 else
1839 answ = smc->conn.sndbuf_desc->len -
1840 atomic_read(&smc->conn.sndbuf_space);
1841 break;
1842 case SIOCOUTQNSD:
1843 /* output queue size (not send only) */
1844 if (smc->sk.sk_state == SMC_LISTEN) {
1845 release_sock(&smc->sk);
1846 return -EINVAL;
1847 }
1848 if (smc->sk.sk_state == SMC_INIT ||
1849 smc->sk.sk_state == SMC_CLOSED)
1850 answ = 0;
1851 else
1852 answ = smc_tx_prepared_sends(&smc->conn);
1853 break;
1854 case SIOCATMARK:
1855 if (smc->sk.sk_state == SMC_LISTEN) {
1856 release_sock(&smc->sk);
1857 return -EINVAL;
1858 }
1859 if (smc->sk.sk_state == SMC_INIT ||
1860 smc->sk.sk_state == SMC_CLOSED) {
1861 answ = 0;
1862 } else {
1863 smc_curs_copy(&cons, &conn->local_tx_ctrl.cons, conn);
1864 smc_curs_copy(&urg, &conn->urg_curs, conn);
1865 answ = smc_curs_diff(conn->rmb_desc->len,
1866 &cons, &urg) == 1;
1867 }
1868 break;
1869 default:
1870 release_sock(&smc->sk);
1871 return -ENOIOCTLCMD;
1872 }
1873 release_sock(&smc->sk);
1874
1875 return put_user(answ, (int __user *)arg);
1876 }
1877
1878 static ssize_t smc_sendpage(struct socket *sock, struct page *page,
1879 int offset, size_t size, int flags)
1880 {
1881 struct sock *sk = sock->sk;
1882 struct smc_sock *smc;
1883 int rc = -EPIPE;
1884
1885 smc = smc_sk(sk);
1886 lock_sock(sk);
1887 if (sk->sk_state != SMC_ACTIVE) {
1888 release_sock(sk);
1889 goto out;
1890 }
1891 release_sock(sk);
1892 if (smc->use_fallback)
1893 rc = kernel_sendpage(smc->clcsock, page, offset,
1894 size, flags);
1895 else
1896 rc = sock_no_sendpage(sock, page, offset, size, flags);
1897
1898 out:
1899 return rc;
1900 }
1901
1902 /* Map the affected portions of the rmbe into an spd, note the number of bytes
1903 * to splice in conn->splice_pending, and press 'go'. Delays consumer cursor
1904 * updates till whenever a respective page has been fully processed.
1905 * Note that subsequent recv() calls have to wait till all splice() processing
1906 * completed.
1907 */
1908 static ssize_t smc_splice_read(struct socket *sock, loff_t *ppos,
1909 struct pipe_inode_info *pipe, size_t len,
1910 unsigned int flags)
1911 {
1912 struct sock *sk = sock->sk;
1913 struct smc_sock *smc;
1914 int rc = -ENOTCONN;
1915
1916 smc = smc_sk(sk);
1917 lock_sock(sk);
1918 if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) {
1919 /* socket was connected before, no more data to read */
1920 rc = 0;
1921 goto out;
1922 }
1923 if (sk->sk_state == SMC_INIT ||
1924 sk->sk_state == SMC_LISTEN ||
1925 sk->sk_state == SMC_CLOSED)
1926 goto out;
1927
1928 if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
1929 rc = 0;
1930 goto out;
1931 }
1932
1933 if (smc->use_fallback) {
1934 rc = smc->clcsock->ops->splice_read(smc->clcsock, ppos,
1935 pipe, len, flags);
1936 } else {
1937 if (*ppos) {
1938 rc = -ESPIPE;
1939 goto out;
1940 }
1941 if (flags & SPLICE_F_NONBLOCK)
1942 flags = MSG_DONTWAIT;
1943 else
1944 flags = 0;
1945 rc = smc_rx_recvmsg(smc, NULL, pipe, len, flags);
1946 }
1947 out:
1948 release_sock(sk);
1949
1950 return rc;
1951 }
1952
1953 /* must look like tcp */
1954 static const struct proto_ops smc_sock_ops = {
1955 .family = PF_SMC,
1956 .owner = THIS_MODULE,
1957 .release = smc_release,
1958 .bind = smc_bind,
1959 .connect = smc_connect,
1960 .socketpair = sock_no_socketpair,
1961 .accept = smc_accept,
1962 .getname = smc_getname,
1963 .poll = smc_poll,
1964 .ioctl = smc_ioctl,
1965 .listen = smc_listen,
1966 .shutdown = smc_shutdown,
1967 .setsockopt = smc_setsockopt,
1968 .getsockopt = smc_getsockopt,
1969 .sendmsg = smc_sendmsg,
1970 .recvmsg = smc_recvmsg,
1971 .mmap = sock_no_mmap,
1972 .sendpage = smc_sendpage,
1973 .splice_read = smc_splice_read,
1974 };
1975
1976 static int smc_create(struct net *net, struct socket *sock, int protocol,
1977 int kern)
1978 {
1979 int family = (protocol == SMCPROTO_SMC6) ? PF_INET6 : PF_INET;
1980 struct smc_sock *smc;
1981 struct sock *sk;
1982 int rc;
1983
1984 rc = -ESOCKTNOSUPPORT;
1985 if (sock->type != SOCK_STREAM)
1986 goto out;
1987
1988 rc = -EPROTONOSUPPORT;
1989 if (protocol != SMCPROTO_SMC && protocol != SMCPROTO_SMC6)
1990 goto out;
1991
1992 rc = -ENOBUFS;
1993 sock->ops = &smc_sock_ops;
1994 sk = smc_sock_alloc(net, sock, protocol);
1995 if (!sk)
1996 goto out;
1997
1998 /* create internal TCP socket for CLC handshake and fallback */
1999 smc = smc_sk(sk);
2000 smc->use_fallback = false; /* assume rdma capability first */
2001 smc->fallback_rsn = 0;
2002 rc = sock_create_kern(net, family, SOCK_STREAM, IPPROTO_TCP,
2003 &smc->clcsock);
2004 if (rc) {
2005 sk_common_release(sk);
2006 goto out;
2007 }
2008 smc->sk.sk_sndbuf = max(smc->clcsock->sk->sk_sndbuf, SMC_BUF_MIN_SIZE);
2009 smc->sk.sk_rcvbuf = max(smc->clcsock->sk->sk_rcvbuf, SMC_BUF_MIN_SIZE);
2010
2011 out:
2012 return rc;
2013 }
2014
2015 static const struct net_proto_family smc_sock_family_ops = {
2016 .family = PF_SMC,
2017 .owner = THIS_MODULE,
2018 .create = smc_create,
2019 };
2020
2021 unsigned int smc_net_id;
2022
2023 static __net_init int smc_net_init(struct net *net)
2024 {
2025 return smc_pnet_net_init(net);
2026 }
2027
2028 static void __net_exit smc_net_exit(struct net *net)
2029 {
2030 smc_pnet_net_exit(net);
2031 }
2032
2033 static struct pernet_operations smc_net_ops = {
2034 .init = smc_net_init,
2035 .exit = smc_net_exit,
2036 .id = &smc_net_id,
2037 .size = sizeof(struct smc_net),
2038 };
2039
2040 static int __init smc_init(void)
2041 {
2042 int rc;
2043
2044 rc = register_pernet_subsys(&smc_net_ops);
2045 if (rc)
2046 return rc;
2047
2048 rc = smc_pnet_init();
2049 if (rc)
2050 goto out_pernet_subsys;
2051
2052 rc = smc_core_init();
2053 if (rc) {
2054 pr_err("%s: smc_core_init fails with %d\n", __func__, rc);
2055 goto out_pnet;
2056 }
2057
2058 rc = smc_llc_init();
2059 if (rc) {
2060 pr_err("%s: smc_llc_init fails with %d\n", __func__, rc);
2061 goto out_core;
2062 }
2063
2064 rc = smc_cdc_init();
2065 if (rc) {
2066 pr_err("%s: smc_cdc_init fails with %d\n", __func__, rc);
2067 goto out_core;
2068 }
2069
2070 rc = proto_register(&smc_proto, 1);
2071 if (rc) {
2072 pr_err("%s: proto_register(v4) fails with %d\n", __func__, rc);
2073 goto out_core;
2074 }
2075
2076 rc = proto_register(&smc_proto6, 1);
2077 if (rc) {
2078 pr_err("%s: proto_register(v6) fails with %d\n", __func__, rc);
2079 goto out_proto;
2080 }
2081
2082 rc = sock_register(&smc_sock_family_ops);
2083 if (rc) {
2084 pr_err("%s: sock_register fails with %d\n", __func__, rc);
2085 goto out_proto6;
2086 }
2087 INIT_HLIST_HEAD(&smc_v4_hashinfo.ht);
2088 INIT_HLIST_HEAD(&smc_v6_hashinfo.ht);
2089
2090 rc = smc_ib_register_client();
2091 if (rc) {
2092 pr_err("%s: ib_register fails with %d\n", __func__, rc);
2093 goto out_sock;
2094 }
2095
2096 static_branch_enable(&tcp_have_smc);
2097 return 0;
2098
2099 out_sock:
2100 sock_unregister(PF_SMC);
2101 out_proto6:
2102 proto_unregister(&smc_proto6);
2103 out_proto:
2104 proto_unregister(&smc_proto);
2105 out_core:
2106 smc_core_exit();
2107 out_pnet:
2108 smc_pnet_exit();
2109 out_pernet_subsys:
2110 unregister_pernet_subsys(&smc_net_ops);
2111
2112 return rc;
2113 }
2114
2115 static void __exit smc_exit(void)
2116 {
2117 static_branch_disable(&tcp_have_smc);
2118 sock_unregister(PF_SMC);
2119 smc_core_exit();
2120 smc_ib_unregister_client();
2121 proto_unregister(&smc_proto6);
2122 proto_unregister(&smc_proto);
2123 smc_pnet_exit();
2124 unregister_pernet_subsys(&smc_net_ops);
2125 rcu_barrier();
2126 }
2127
2128 module_init(smc_init);
2129 module_exit(smc_exit);
2130
2131 MODULE_AUTHOR("Ursula Braun <ubraun@linux.vnet.ibm.com>");
2132 MODULE_DESCRIPTION("smc socket address family");
2133 MODULE_LICENSE("GPL");
2134 MODULE_ALIAS_NETPROTO(PF_SMC);
Linux with added FPC-III support