?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
gpio: rcar: Fix runtime PM imbalance on error
[linux/fpc-iii.git] / net / mptcp / subflow.c
blob50a8bea987c6e859303941d1a41165c651806bfb
1 // SPDX-License-Identifier: GPL-2.0
2 /* Multipath TCP
3 *
4 * Copyright (c) 2017 - 2019, Intel Corporation.
5 */
6
7 #define pr_fmt(fmt) "MPTCP: " fmt
8
9 #include <linux/kernel.h>
10 #include <linux/module.h>
11 #include <linux/netdevice.h>
12 #include <crypto/algapi.h>
13 #include <net/sock.h>
14 #include <net/inet_common.h>
15 #include <net/inet_hashtables.h>
16 #include <net/protocol.h>
17 #include <net/tcp.h>
18 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
19 #include <net/ip6_route.h>
20 #endif
21 #include <net/mptcp.h>
22 #include "protocol.h"
23 #include "mib.h"
24
25 static void SUBFLOW_REQ_INC_STATS(struct request_sock *req,
26 enum linux_mptcp_mib_field field)
27 {
28 MPTCP_INC_STATS(sock_net(req_to_sk(req)), field);
29 }
30
31 static int subflow_rebuild_header(struct sock *sk)
32 {
33 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
34 int local_id, err = 0;
35
36 if (subflow->request_mptcp && !subflow->token) {
37 pr_debug("subflow=%p", sk);
38 err = mptcp_token_new_connect(sk);
39 } else if (subflow->request_join && !subflow->local_nonce) {
40 struct mptcp_sock *msk = (struct mptcp_sock *)subflow->conn;
41
42 pr_debug("subflow=%p", sk);
43
44 do {
45 get_random_bytes(&subflow->local_nonce, sizeof(u32));
46 } while (!subflow->local_nonce);
47
48 if (subflow->local_id)
49 goto out;
50
51 local_id = mptcp_pm_get_local_id(msk, (struct sock_common *)sk);
52 if (local_id < 0)
53 return -EINVAL;
54
55 subflow->local_id = local_id;
56 }
57
58 out:
59 if (err)
60 return err;
61
62 return subflow->icsk_af_ops->rebuild_header(sk);
63 }
64
65 static void subflow_req_destructor(struct request_sock *req)
66 {
67 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
68
69 pr_debug("subflow_req=%p", subflow_req);
70
71 if (subflow_req->mp_capable)
72 mptcp_token_destroy_request(subflow_req->token);
73 tcp_request_sock_ops.destructor(req);
74 }
75
76 static void subflow_generate_hmac(u64 key1, u64 key2, u32 nonce1, u32 nonce2,
77 void *hmac)
78 {
79 u8 msg[8];
80
81 put_unaligned_be32(nonce1, &msg[0]);
82 put_unaligned_be32(nonce2, &msg[4]);
83
84 mptcp_crypto_hmac_sha(key1, key2, msg, 8, hmac);
85 }
86
87 /* validate received token and create truncated hmac and nonce for SYN-ACK */
88 static bool subflow_token_join_request(struct request_sock *req,
89 const struct sk_buff *skb)
90 {
91 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
92 u8 hmac[MPTCPOPT_HMAC_LEN];
93 struct mptcp_sock *msk;
94 int local_id;
95
96 msk = mptcp_token_get_sock(subflow_req->token);
97 if (!msk) {
98 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINNOTOKEN);
99 return false;
100 }
101
102 local_id = mptcp_pm_get_local_id(msk, (struct sock_common *)req);
103 if (local_id < 0) {
104 sock_put((struct sock *)msk);
105 return false;
106 }
107 subflow_req->local_id = local_id;
108
109 get_random_bytes(&subflow_req->local_nonce, sizeof(u32));
110
111 subflow_generate_hmac(msk->local_key, msk->remote_key,
112 subflow_req->local_nonce,
113 subflow_req->remote_nonce, hmac);
114
115 subflow_req->thmac = get_unaligned_be64(hmac);
116
117 sock_put((struct sock *)msk);
118 return true;
119 }
120
121 static void subflow_init_req(struct request_sock *req,
122 const struct sock *sk_listener,
123 struct sk_buff *skb)
124 {
125 struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk_listener);
126 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
127 struct tcp_options_received rx_opt;
128
129 pr_debug("subflow_req=%p, listener=%p", subflow_req, listener);
130
131 memset(&rx_opt.mptcp, 0, sizeof(rx_opt.mptcp));
132 mptcp_get_options(skb, &rx_opt);
133
134 subflow_req->mp_capable = 0;
135 subflow_req->mp_join = 0;
136 subflow_req->remote_key_valid = 0;
137
138 #ifdef CONFIG_TCP_MD5SIG
139 /* no MPTCP if MD5SIG is enabled on this socket or we may run out of
140 * TCP option space.
141 */
142 if (rcu_access_pointer(tcp_sk(sk_listener)->md5sig_info))
143 return;
144 #endif
145
146 if (rx_opt.mptcp.mp_capable) {
147 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MPCAPABLEPASSIVE);
148
149 if (rx_opt.mptcp.mp_join)
150 return;
151 } else if (rx_opt.mptcp.mp_join) {
152 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINSYNRX);
153 }
154
155 if (rx_opt.mptcp.mp_capable && listener->request_mptcp) {
156 int err;
157
158 err = mptcp_token_new_request(req);
159 if (err == 0)
160 subflow_req->mp_capable = 1;
161
162 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq;
163 } else if (rx_opt.mptcp.mp_join && listener->request_mptcp) {
164 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq;
165 subflow_req->mp_join = 1;
166 subflow_req->backup = rx_opt.mptcp.backup;
167 subflow_req->remote_id = rx_opt.mptcp.join_id;
168 subflow_req->token = rx_opt.mptcp.token;
169 subflow_req->remote_nonce = rx_opt.mptcp.nonce;
170 pr_debug("token=%u, remote_nonce=%u", subflow_req->token,
171 subflow_req->remote_nonce);
172 if (!subflow_token_join_request(req, skb)) {
173 subflow_req->mp_join = 0;
174 // @@ need to trigger RST
175 }
176 }
177 }
178
179 static void subflow_v4_init_req(struct request_sock *req,
180 const struct sock *sk_listener,
181 struct sk_buff *skb)
182 {
183 tcp_rsk(req)->is_mptcp = 1;
184
185 tcp_request_sock_ipv4_ops.init_req(req, sk_listener, skb);
186
187 subflow_init_req(req, sk_listener, skb);
188 }
189
190 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
191 static void subflow_v6_init_req(struct request_sock *req,
192 const struct sock *sk_listener,
193 struct sk_buff *skb)
194 {
195 tcp_rsk(req)->is_mptcp = 1;
196
197 tcp_request_sock_ipv6_ops.init_req(req, sk_listener, skb);
198
199 subflow_init_req(req, sk_listener, skb);
200 }
201 #endif
202
203 /* validate received truncated hmac and create hmac for third ACK */
204 static bool subflow_thmac_valid(struct mptcp_subflow_context *subflow)
205 {
206 u8 hmac[MPTCPOPT_HMAC_LEN];
207 u64 thmac;
208
209 subflow_generate_hmac(subflow->remote_key, subflow->local_key,
210 subflow->remote_nonce, subflow->local_nonce,
211 hmac);
212
213 thmac = get_unaligned_be64(hmac);
214 pr_debug("subflow=%p, token=%u, thmac=%llu, subflow->thmac=%llu\n",
215 subflow, subflow->token,
216 (unsigned long long)thmac,
217 (unsigned long long)subflow->thmac);
218
219 return thmac == subflow->thmac;
220 }
221
222 static void subflow_finish_connect(struct sock *sk, const struct sk_buff *skb)
223 {
224 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
225 struct sock *parent = subflow->conn;
226
227 subflow->icsk_af_ops->sk_rx_dst_set(sk, skb);
228
229 if (inet_sk_state_load(parent) != TCP_ESTABLISHED) {
230 inet_sk_state_store(parent, TCP_ESTABLISHED);
231 parent->sk_state_change(parent);
232 }
233
234 if (subflow->conn_finished || !tcp_sk(sk)->is_mptcp)
235 return;
236
237 if (subflow->mp_capable) {
238 pr_debug("subflow=%p, remote_key=%llu", mptcp_subflow_ctx(sk),
239 subflow->remote_key);
240 mptcp_finish_connect(sk);
241 subflow->conn_finished = 1;
242
243 if (skb) {
244 pr_debug("synack seq=%u", TCP_SKB_CB(skb)->seq);
245 subflow->ssn_offset = TCP_SKB_CB(skb)->seq;
246 }
247 } else if (subflow->mp_join) {
248 pr_debug("subflow=%p, thmac=%llu, remote_nonce=%u",
249 subflow, subflow->thmac,
250 subflow->remote_nonce);
251 if (!subflow_thmac_valid(subflow)) {
252 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINACKMAC);
253 subflow->mp_join = 0;
254 goto do_reset;
255 }
256
257 subflow_generate_hmac(subflow->local_key, subflow->remote_key,
258 subflow->local_nonce,
259 subflow->remote_nonce,
260 subflow->hmac);
261
262 if (skb)
263 subflow->ssn_offset = TCP_SKB_CB(skb)->seq;
264
265 if (!mptcp_finish_join(sk))
266 goto do_reset;
267
268 subflow->conn_finished = 1;
269 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINSYNACKRX);
270 } else {
271 do_reset:
272 tcp_send_active_reset(sk, GFP_ATOMIC);
273 tcp_done(sk);
274 }
275 }
276
277 static struct request_sock_ops subflow_request_sock_ops;
278 static struct tcp_request_sock_ops subflow_request_sock_ipv4_ops;
279
280 static int subflow_v4_conn_request(struct sock *sk, struct sk_buff *skb)
281 {
282 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
283
284 pr_debug("subflow=%p", subflow);
285
286 /* Never answer to SYNs sent to broadcast or multicast */
287 if (skb_rtable(skb)->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST))
288 goto drop;
289
290 return tcp_conn_request(&subflow_request_sock_ops,
291 &subflow_request_sock_ipv4_ops,
292 sk, skb);
293 drop:
294 tcp_listendrop(sk);
295 return 0;
296 }
297
298 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
299 static struct tcp_request_sock_ops subflow_request_sock_ipv6_ops;
300 static struct inet_connection_sock_af_ops subflow_v6_specific;
301 static struct inet_connection_sock_af_ops subflow_v6m_specific;
302
303 static int subflow_v6_conn_request(struct sock *sk, struct sk_buff *skb)
304 {
305 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
306
307 pr_debug("subflow=%p", subflow);
308
309 if (skb->protocol == htons(ETH_P_IP))
310 return subflow_v4_conn_request(sk, skb);
311
312 if (!ipv6_unicast_destination(skb))
313 goto drop;
314
315 return tcp_conn_request(&subflow_request_sock_ops,
316 &subflow_request_sock_ipv6_ops, sk, skb);
317
318 drop:
319 tcp_listendrop(sk);
320 return 0; /* don't send reset */
321 }
322 #endif
323
324 /* validate hmac received in third ACK */
325 static bool subflow_hmac_valid(const struct request_sock *req,
326 const struct tcp_options_received *rx_opt)
327 {
328 const struct mptcp_subflow_request_sock *subflow_req;
329 u8 hmac[MPTCPOPT_HMAC_LEN];
330 struct mptcp_sock *msk;
331 bool ret;
332
333 subflow_req = mptcp_subflow_rsk(req);
334 msk = mptcp_token_get_sock(subflow_req->token);
335 if (!msk)
336 return false;
337
338 subflow_generate_hmac(msk->remote_key, msk->local_key,
339 subflow_req->remote_nonce,
340 subflow_req->local_nonce, hmac);
341
342 ret = true;
343 if (crypto_memneq(hmac, rx_opt->mptcp.hmac, sizeof(hmac)))
344 ret = false;
345
346 sock_put((struct sock *)msk);
347 return ret;
348 }
349
350 static struct sock *subflow_syn_recv_sock(const struct sock *sk,
351 struct sk_buff *skb,
352 struct request_sock *req,
353 struct dst_entry *dst,
354 struct request_sock *req_unhash,
355 bool *own_req)
356 {
357 struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk);
358 struct mptcp_subflow_request_sock *subflow_req;
359 struct tcp_options_received opt_rx;
360 bool fallback_is_fatal = false;
361 struct sock *new_msk = NULL;
362 struct sock *child;
363
364 pr_debug("listener=%p, req=%p, conn=%p", listener, req, listener->conn);
365
366 if (tcp_rsk(req)->is_mptcp == 0)
367 goto create_child;
368
369 /* if the sk is MP_CAPABLE, we try to fetch the client key */
370 subflow_req = mptcp_subflow_rsk(req);
371 if (subflow_req->mp_capable) {
372 if (TCP_SKB_CB(skb)->seq != subflow_req->ssn_offset + 1) {
373 /* here we can receive and accept an in-window,
374 * out-of-order pkt, which will not carry the MP_CAPABLE
375 * opt even on mptcp enabled paths
376 */
377 goto create_msk;
378 }
379
380 opt_rx.mptcp.mp_capable = 0;
381 mptcp_get_options(skb, &opt_rx);
382 if (opt_rx.mptcp.mp_capable) {
383 subflow_req->remote_key = opt_rx.mptcp.sndr_key;
384 subflow_req->remote_key_valid = 1;
385 } else {
386 subflow_req->mp_capable = 0;
387 goto create_child;
388 }
389
390 create_msk:
391 new_msk = mptcp_sk_clone(listener->conn, req);
392 if (!new_msk)
393 subflow_req->mp_capable = 0;
394 } else if (subflow_req->mp_join) {
395 fallback_is_fatal = true;
396 opt_rx.mptcp.mp_join = 0;
397 mptcp_get_options(skb, &opt_rx);
398 if (!opt_rx.mptcp.mp_join ||
399 !subflow_hmac_valid(req, &opt_rx)) {
400 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINACKMAC);
401 return NULL;
402 }
403 }
404
405 create_child:
406 child = listener->icsk_af_ops->syn_recv_sock(sk, skb, req, dst,
407 req_unhash, own_req);
408
409 if (child && *own_req) {
410 struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(child);
411
412 /* we have null ctx on TCP fallback, which is fatal on
413 * MPJ handshake
414 */
415 if (!ctx) {
416 if (fallback_is_fatal)
417 goto close_child;
418 goto out;
419 }
420
421 if (ctx->mp_capable) {
422 /* new mpc subflow takes ownership of the newly
423 * created mptcp socket
424 */
425 inet_sk_state_store(new_msk, TCP_ESTABLISHED);
426 mptcp_pm_new_connection(mptcp_sk(new_msk), 1);
427 ctx->conn = new_msk;
428 new_msk = NULL;
429 } else if (ctx->mp_join) {
430 struct mptcp_sock *owner;
431
432 owner = mptcp_token_get_sock(ctx->token);
433 if (!owner)
434 goto close_child;
435
436 ctx->conn = (struct sock *)owner;
437 if (!mptcp_finish_join(child))
438 goto close_child;
439
440 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINACKRX);
441 }
442 }
443
444 out:
445 /* dispose of the left over mptcp master, if any */
446 if (unlikely(new_msk))
447 sock_put(new_msk);
448 return child;
449
450 close_child:
451 tcp_send_active_reset(child, GFP_ATOMIC);
452 inet_csk_prepare_forced_close(child);
453 tcp_done(child);
454 return NULL;
455 }
456
457 static struct inet_connection_sock_af_ops subflow_specific;
458
459 enum mapping_status {
460 MAPPING_OK,
461 MAPPING_INVALID,
462 MAPPING_EMPTY,
463 MAPPING_DATA_FIN
464 };
465
466 static u64 expand_seq(u64 old_seq, u16 old_data_len, u64 seq)
467 {
468 if ((u32)seq == (u32)old_seq)
469 return old_seq;
470
471 /* Assume map covers data not mapped yet. */
472 return seq | ((old_seq + old_data_len + 1) & GENMASK_ULL(63, 32));
473 }
474
475 static void warn_bad_map(struct mptcp_subflow_context *subflow, u32 ssn)
476 {
477 WARN_ONCE(1, "Bad mapping: ssn=%d map_seq=%d map_data_len=%d",
478 ssn, subflow->map_subflow_seq, subflow->map_data_len);
479 }
480
481 static bool skb_is_fully_mapped(struct sock *ssk, struct sk_buff *skb)
482 {
483 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
484 unsigned int skb_consumed;
485
486 skb_consumed = tcp_sk(ssk)->copied_seq - TCP_SKB_CB(skb)->seq;
487 if (WARN_ON_ONCE(skb_consumed >= skb->len))
488 return true;
489
490 return skb->len - skb_consumed <= subflow->map_data_len -
491 mptcp_subflow_get_map_offset(subflow);
492 }
493
494 static bool validate_mapping(struct sock *ssk, struct sk_buff *skb)
495 {
496 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
497 u32 ssn = tcp_sk(ssk)->copied_seq - subflow->ssn_offset;
498
499 if (unlikely(before(ssn, subflow->map_subflow_seq))) {
500 /* Mapping covers data later in the subflow stream,
501 * currently unsupported.
502 */
503 warn_bad_map(subflow, ssn);
504 return false;
505 }
506 if (unlikely(!before(ssn, subflow->map_subflow_seq +
507 subflow->map_data_len))) {
508 /* Mapping does covers past subflow data, invalid */
509 warn_bad_map(subflow, ssn + skb->len);
510 return false;
511 }
512 return true;
513 }
514
515 static enum mapping_status get_mapping_status(struct sock *ssk)
516 {
517 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
518 struct mptcp_ext *mpext;
519 struct sk_buff *skb;
520 u16 data_len;
521 u64 map_seq;
522
523 skb = skb_peek(&ssk->sk_receive_queue);
524 if (!skb)
525 return MAPPING_EMPTY;
526
527 mpext = mptcp_get_ext(skb);
528 if (!mpext || !mpext->use_map) {
529 if (!subflow->map_valid && !skb->len) {
530 /* the TCP stack deliver 0 len FIN pkt to the receive
531 * queue, that is the only 0len pkts ever expected here,
532 * and we can admit no mapping only for 0 len pkts
533 */
534 if (!(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN))
535 WARN_ONCE(1, "0len seq %d:%d flags %x",
536 TCP_SKB_CB(skb)->seq,
537 TCP_SKB_CB(skb)->end_seq,
538 TCP_SKB_CB(skb)->tcp_flags);
539 sk_eat_skb(ssk, skb);
540 return MAPPING_EMPTY;
541 }
542
543 if (!subflow->map_valid)
544 return MAPPING_INVALID;
545
546 goto validate_seq;
547 }
548
549 pr_debug("seq=%llu is64=%d ssn=%u data_len=%u data_fin=%d",
550 mpext->data_seq, mpext->dsn64, mpext->subflow_seq,
551 mpext->data_len, mpext->data_fin);
552
553 data_len = mpext->data_len;
554 if (data_len == 0) {
555 pr_err("Infinite mapping not handled");
556 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_INFINITEMAPRX);
557 return MAPPING_INVALID;
558 }
559
560 if (mpext->data_fin == 1) {
561 if (data_len == 1) {
562 pr_debug("DATA_FIN with no payload");
563 if (subflow->map_valid) {
564 /* A DATA_FIN might arrive in a DSS
565 * option before the previous mapping
566 * has been fully consumed. Continue
567 * handling the existing mapping.
568 */
569 skb_ext_del(skb, SKB_EXT_MPTCP);
570 return MAPPING_OK;
571 } else {
572 return MAPPING_DATA_FIN;
573 }
574 }
575
576 /* Adjust for DATA_FIN using 1 byte of sequence space */
577 data_len--;
578 }
579
580 if (!mpext->dsn64) {
581 map_seq = expand_seq(subflow->map_seq, subflow->map_data_len,
582 mpext->data_seq);
583 pr_debug("expanded seq=%llu", subflow->map_seq);
584 } else {
585 map_seq = mpext->data_seq;
586 }
587
588 if (subflow->map_valid) {
589 /* Allow replacing only with an identical map */
590 if (subflow->map_seq == map_seq &&
591 subflow->map_subflow_seq == mpext->subflow_seq &&
592 subflow->map_data_len == data_len) {
593 skb_ext_del(skb, SKB_EXT_MPTCP);
594 return MAPPING_OK;
595 }
596
597 /* If this skb data are fully covered by the current mapping,
598 * the new map would need caching, which is not supported
599 */
600 if (skb_is_fully_mapped(ssk, skb)) {
601 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DSSNOMATCH);
602 return MAPPING_INVALID;
603 }
604
605 /* will validate the next map after consuming the current one */
606 return MAPPING_OK;
607 }
608
609 subflow->map_seq = map_seq;
610 subflow->map_subflow_seq = mpext->subflow_seq;
611 subflow->map_data_len = data_len;
612 subflow->map_valid = 1;
613 subflow->mpc_map = mpext->mpc_map;
614 pr_debug("new map seq=%llu subflow_seq=%u data_len=%u",
615 subflow->map_seq, subflow->map_subflow_seq,
616 subflow->map_data_len);
617
618 validate_seq:
619 /* we revalidate valid mapping on new skb, because we must ensure
620 * the current skb is completely covered by the available mapping
621 */
622 if (!validate_mapping(ssk, skb))
623 return MAPPING_INVALID;
624
625 skb_ext_del(skb, SKB_EXT_MPTCP);
626 return MAPPING_OK;
627 }
628
629 static int subflow_read_actor(read_descriptor_t *desc,
630 struct sk_buff *skb,
631 unsigned int offset, size_t len)
632 {
633 size_t copy_len = min(desc->count, len);
634
635 desc->count -= copy_len;
636
637 pr_debug("flushed %zu bytes, %zu left", copy_len, desc->count);
638 return copy_len;
639 }
640
641 static bool subflow_check_data_avail(struct sock *ssk)
642 {
643 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
644 enum mapping_status status;
645 struct mptcp_sock *msk;
646 struct sk_buff *skb;
647
648 pr_debug("msk=%p ssk=%p data_avail=%d skb=%p", subflow->conn, ssk,
649 subflow->data_avail, skb_peek(&ssk->sk_receive_queue));
650 if (subflow->data_avail)
651 return true;
652
653 msk = mptcp_sk(subflow->conn);
654 for (;;) {
655 u32 map_remaining;
656 size_t delta;
657 u64 ack_seq;
658 u64 old_ack;
659
660 status = get_mapping_status(ssk);
661 pr_debug("msk=%p ssk=%p status=%d", msk, ssk, status);
662 if (status == MAPPING_INVALID) {
663 ssk->sk_err = EBADMSG;
664 goto fatal;
665 }
666
667 if (status != MAPPING_OK)
668 return false;
669
670 skb = skb_peek(&ssk->sk_receive_queue);
671 if (WARN_ON_ONCE(!skb))
672 return false;
673
674 /* if msk lacks the remote key, this subflow must provide an
675 * MP_CAPABLE-based mapping
676 */
677 if (unlikely(!READ_ONCE(msk->can_ack))) {
678 if (!subflow->mpc_map) {
679 ssk->sk_err = EBADMSG;
680 goto fatal;
681 }
682 WRITE_ONCE(msk->remote_key, subflow->remote_key);
683 WRITE_ONCE(msk->ack_seq, subflow->map_seq);
684 WRITE_ONCE(msk->can_ack, true);
685 }
686
687 old_ack = READ_ONCE(msk->ack_seq);
688 ack_seq = mptcp_subflow_get_mapped_dsn(subflow);
689 pr_debug("msk ack_seq=%llx subflow ack_seq=%llx", old_ack,
690 ack_seq);
691 if (ack_seq == old_ack)
692 break;
693
694 /* only accept in-sequence mapping. Old values are spurious
695 * retransmission; we can hit "future" values on active backup
696 * subflow switch, we relay on retransmissions to get
697 * in-sequence data.
698 * Cuncurrent subflows support will require subflow data
699 * reordering
700 */
701 map_remaining = subflow->map_data_len -
702 mptcp_subflow_get_map_offset(subflow);
703 if (before64(ack_seq, old_ack))
704 delta = min_t(size_t, old_ack - ack_seq, map_remaining);
705 else
706 delta = min_t(size_t, ack_seq - old_ack, map_remaining);
707
708 /* discard mapped data */
709 pr_debug("discarding %zu bytes, current map len=%d", delta,
710 map_remaining);
711 if (delta) {
712 read_descriptor_t desc = {
713 .count = delta,
714 };
715 int ret;
716
717 ret = tcp_read_sock(ssk, &desc, subflow_read_actor);
718 if (ret < 0) {
719 ssk->sk_err = -ret;
720 goto fatal;
721 }
722 if (ret < delta)
723 return false;
724 if (delta == map_remaining)
725 subflow->map_valid = 0;
726 }
727 }
728 return true;
729
730 fatal:
731 /* fatal protocol error, close the socket */
732 /* This barrier is coupled with smp_rmb() in tcp_poll() */
733 smp_wmb();
734 ssk->sk_error_report(ssk);
735 tcp_set_state(ssk, TCP_CLOSE);
736 tcp_send_active_reset(ssk, GFP_ATOMIC);
737 return false;
738 }
739
740 bool mptcp_subflow_data_available(struct sock *sk)
741 {
742 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
743 struct sk_buff *skb;
744
745 /* check if current mapping is still valid */
746 if (subflow->map_valid &&
747 mptcp_subflow_get_map_offset(subflow) >= subflow->map_data_len) {
748 subflow->map_valid = 0;
749 subflow->data_avail = 0;
750
751 pr_debug("Done with mapping: seq=%u data_len=%u",
752 subflow->map_subflow_seq,
753 subflow->map_data_len);
754 }
755
756 if (!subflow_check_data_avail(sk)) {
757 subflow->data_avail = 0;
758 return false;
759 }
760
761 skb = skb_peek(&sk->sk_receive_queue);
762 subflow->data_avail = skb &&
763 before(tcp_sk(sk)->copied_seq, TCP_SKB_CB(skb)->end_seq);
764 return subflow->data_avail;
765 }
766
767 static void subflow_data_ready(struct sock *sk)
768 {
769 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
770 struct sock *parent = subflow->conn;
771
772 if (!subflow->mp_capable && !subflow->mp_join) {
773 subflow->tcp_data_ready(sk);
774
775 parent->sk_data_ready(parent);
776 return;
777 }
778
779 if (mptcp_subflow_data_available(sk))
780 mptcp_data_ready(parent, sk);
781 }
782
783 static void subflow_write_space(struct sock *sk)
784 {
785 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
786 struct sock *parent = subflow->conn;
787
788 sk_stream_write_space(sk);
789 if (sk_stream_is_writeable(sk)) {
790 set_bit(MPTCP_SEND_SPACE, &mptcp_sk(parent)->flags);
791 smp_mb__after_atomic();
792 /* set SEND_SPACE before sk_stream_write_space clears NOSPACE */
793 sk_stream_write_space(parent);
794 }
795 }
796
797 static struct inet_connection_sock_af_ops *
798 subflow_default_af_ops(struct sock *sk)
799 {
800 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
801 if (sk->sk_family == AF_INET6)
802 return &subflow_v6_specific;
803 #endif
804 return &subflow_specific;
805 }
806
807 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
808 void mptcpv6_handle_mapped(struct sock *sk, bool mapped)
809 {
810 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
811 struct inet_connection_sock *icsk = inet_csk(sk);
812 struct inet_connection_sock_af_ops *target;
813
814 target = mapped ? &subflow_v6m_specific : subflow_default_af_ops(sk);
815
816 pr_debug("subflow=%p family=%d ops=%p target=%p mapped=%d",
817 subflow, sk->sk_family, icsk->icsk_af_ops, target, mapped);
818
819 if (likely(icsk->icsk_af_ops == target))
820 return;
821
822 subflow->icsk_af_ops = icsk->icsk_af_ops;
823 icsk->icsk_af_ops = target;
824 }
825 #endif
826
827 static void mptcp_info2sockaddr(const struct mptcp_addr_info *info,
828 struct sockaddr_storage *addr)
829 {
830 memset(addr, 0, sizeof(*addr));
831 addr->ss_family = info->family;
832 if (addr->ss_family == AF_INET) {
833 struct sockaddr_in *in_addr = (struct sockaddr_in *)addr;
834
835 in_addr->sin_addr = info->addr;
836 in_addr->sin_port = info->port;
837 }
838 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
839 else if (addr->ss_family == AF_INET6) {
840 struct sockaddr_in6 *in6_addr = (struct sockaddr_in6 *)addr;
841
842 in6_addr->sin6_addr = info->addr6;
843 in6_addr->sin6_port = info->port;
844 }
845 #endif
846 }
847
848 int __mptcp_subflow_connect(struct sock *sk, int ifindex,
849 const struct mptcp_addr_info *loc,
850 const struct mptcp_addr_info *remote)
851 {
852 struct mptcp_sock *msk = mptcp_sk(sk);
853 struct mptcp_subflow_context *subflow;
854 struct sockaddr_storage addr;
855 struct socket *sf;
856 u32 remote_token;
857 int addrlen;
858 int err;
859
860 if (sk->sk_state != TCP_ESTABLISHED)
861 return -ENOTCONN;
862
863 err = mptcp_subflow_create_socket(sk, &sf);
864 if (err)
865 return err;
866
867 subflow = mptcp_subflow_ctx(sf->sk);
868 subflow->remote_key = msk->remote_key;
869 subflow->local_key = msk->local_key;
870 subflow->token = msk->token;
871 mptcp_info2sockaddr(loc, &addr);
872
873 addrlen = sizeof(struct sockaddr_in);
874 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
875 if (loc->family == AF_INET6)
876 addrlen = sizeof(struct sockaddr_in6);
877 #endif
878 sf->sk->sk_bound_dev_if = ifindex;
879 err = kernel_bind(sf, (struct sockaddr *)&addr, addrlen);
880 if (err)
881 goto failed;
882
883 mptcp_crypto_key_sha(subflow->remote_key, &remote_token, NULL);
884 pr_debug("msk=%p remote_token=%u", msk, remote_token);
885 subflow->remote_token = remote_token;
886 subflow->local_id = loc->id;
887 subflow->request_join = 1;
888 subflow->request_bkup = 1;
889 mptcp_info2sockaddr(remote, &addr);
890
891 err = kernel_connect(sf, (struct sockaddr *)&addr, addrlen, O_NONBLOCK);
892 if (err && err != -EINPROGRESS)
893 goto failed;
894
895 spin_lock_bh(&msk->join_list_lock);
896 list_add_tail(&subflow->node, &msk->join_list);
897 spin_unlock_bh(&msk->join_list_lock);
898
899 return err;
900
901 failed:
902 sock_release(sf);
903 return err;
904 }
905
906 int mptcp_subflow_create_socket(struct sock *sk, struct socket **new_sock)
907 {
908 struct mptcp_subflow_context *subflow;
909 struct net *net = sock_net(sk);
910 struct socket *sf;
911 int err;
912
913 err = sock_create_kern(net, sk->sk_family, SOCK_STREAM, IPPROTO_TCP,
914 &sf);
915 if (err)
916 return err;
917
918 lock_sock(sf->sk);
919
920 /* kernel sockets do not by default acquire net ref, but TCP timer
921 * needs it.
922 */
923 sf->sk->sk_net_refcnt = 1;
924 get_net(net);
925 #ifdef CONFIG_PROC_FS
926 this_cpu_add(*net->core.sock_inuse, 1);
927 #endif
928 err = tcp_set_ulp(sf->sk, "mptcp");
929 release_sock(sf->sk);
930
931 if (err)
932 return err;
933
934 subflow = mptcp_subflow_ctx(sf->sk);
935 pr_debug("subflow=%p", subflow);
936
937 *new_sock = sf;
938 sock_hold(sk);
939 subflow->conn = sk;
940
941 return 0;
942 }
943
944 static struct mptcp_subflow_context *subflow_create_ctx(struct sock *sk,
945 gfp_t priority)
946 {
947 struct inet_connection_sock *icsk = inet_csk(sk);
948 struct mptcp_subflow_context *ctx;
949
950 ctx = kzalloc(sizeof(*ctx), priority);
951 if (!ctx)
952 return NULL;
953
954 rcu_assign_pointer(icsk->icsk_ulp_data, ctx);
955 INIT_LIST_HEAD(&ctx->node);
956
957 pr_debug("subflow=%p", ctx);
958
959 ctx->tcp_sock = sk;
960
961 return ctx;
962 }
963
964 static void __subflow_state_change(struct sock *sk)
965 {
966 struct socket_wq *wq;
967
968 rcu_read_lock();
969 wq = rcu_dereference(sk->sk_wq);
970 if (skwq_has_sleeper(wq))
971 wake_up_interruptible_all(&wq->wait);
972 rcu_read_unlock();
973 }
974
975 static bool subflow_is_done(const struct sock *sk)
976 {
977 return sk->sk_shutdown & RCV_SHUTDOWN || sk->sk_state == TCP_CLOSE;
978 }
979
980 static void subflow_state_change(struct sock *sk)
981 {
982 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
983 struct sock *parent = subflow->conn;
984
985 __subflow_state_change(sk);
986
987 /* as recvmsg() does not acquire the subflow socket for ssk selection
988 * a fin packet carrying a DSS can be unnoticed if we don't trigger
989 * the data available machinery here.
990 */
991 if (subflow->mp_capable && mptcp_subflow_data_available(sk))
992 mptcp_data_ready(parent, sk);
993
994 if (!(parent->sk_shutdown & RCV_SHUTDOWN) &&
995 !subflow->rx_eof && subflow_is_done(sk)) {
996 subflow->rx_eof = 1;
997 mptcp_subflow_eof(parent);
998 }
999 }
1000
1001 static int subflow_ulp_init(struct sock *sk)
1002 {
1003 struct inet_connection_sock *icsk = inet_csk(sk);
1004 struct mptcp_subflow_context *ctx;
1005 struct tcp_sock *tp = tcp_sk(sk);
1006 int err = 0;
1007
1008 /* disallow attaching ULP to a socket unless it has been
1009 * created with sock_create_kern()
1010 */
1011 if (!sk->sk_kern_sock) {
1012 err = -EOPNOTSUPP;
1013 goto out;
1014 }
1015
1016 ctx = subflow_create_ctx(sk, GFP_KERNEL);
1017 if (!ctx) {
1018 err = -ENOMEM;
1019 goto out;
1020 }
1021
1022 pr_debug("subflow=%p, family=%d", ctx, sk->sk_family);
1023
1024 tp->is_mptcp = 1;
1025 ctx->icsk_af_ops = icsk->icsk_af_ops;
1026 icsk->icsk_af_ops = subflow_default_af_ops(sk);
1027 ctx->tcp_data_ready = sk->sk_data_ready;
1028 ctx->tcp_state_change = sk->sk_state_change;
1029 ctx->tcp_write_space = sk->sk_write_space;
1030 sk->sk_data_ready = subflow_data_ready;
1031 sk->sk_write_space = subflow_write_space;
1032 sk->sk_state_change = subflow_state_change;
1033 out:
1034 return err;
1035 }
1036
1037 static void subflow_ulp_release(struct sock *sk)
1038 {
1039 struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(sk);
1040
1041 if (!ctx)
1042 return;
1043
1044 if (ctx->conn)
1045 sock_put(ctx->conn);
1046
1047 kfree_rcu(ctx, rcu);
1048 }
1049
1050 static void subflow_ulp_fallback(struct sock *sk,
1051 struct mptcp_subflow_context *old_ctx)
1052 {
1053 struct inet_connection_sock *icsk = inet_csk(sk);
1054
1055 mptcp_subflow_tcp_fallback(sk, old_ctx);
1056 icsk->icsk_ulp_ops = NULL;
1057 rcu_assign_pointer(icsk->icsk_ulp_data, NULL);
1058 tcp_sk(sk)->is_mptcp = 0;
1059 }
1060
1061 static void subflow_ulp_clone(const struct request_sock *req,
1062 struct sock *newsk,
1063 const gfp_t priority)
1064 {
1065 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
1066 struct mptcp_subflow_context *old_ctx = mptcp_subflow_ctx(newsk);
1067 struct mptcp_subflow_context *new_ctx;
1068
1069 if (!tcp_rsk(req)->is_mptcp ||
1070 (!subflow_req->mp_capable && !subflow_req->mp_join)) {
1071 subflow_ulp_fallback(newsk, old_ctx);
1072 return;
1073 }
1074
1075 new_ctx = subflow_create_ctx(newsk, priority);
1076 if (!new_ctx) {
1077 subflow_ulp_fallback(newsk, old_ctx);
1078 return;
1079 }
1080
1081 new_ctx->conn_finished = 1;
1082 new_ctx->icsk_af_ops = old_ctx->icsk_af_ops;
1083 new_ctx->tcp_data_ready = old_ctx->tcp_data_ready;
1084 new_ctx->tcp_state_change = old_ctx->tcp_state_change;
1085 new_ctx->tcp_write_space = old_ctx->tcp_write_space;
1086 new_ctx->rel_write_seq = 1;
1087 new_ctx->tcp_sock = newsk;
1088
1089 if (subflow_req->mp_capable) {
1090 /* see comments in subflow_syn_recv_sock(), MPTCP connection
1091 * is fully established only after we receive the remote key
1092 */
1093 new_ctx->mp_capable = 1;
1094 new_ctx->fully_established = subflow_req->remote_key_valid;
1095 new_ctx->can_ack = subflow_req->remote_key_valid;
1096 new_ctx->remote_key = subflow_req->remote_key;
1097 new_ctx->local_key = subflow_req->local_key;
1098 new_ctx->token = subflow_req->token;
1099 new_ctx->ssn_offset = subflow_req->ssn_offset;
1100 new_ctx->idsn = subflow_req->idsn;
1101 } else if (subflow_req->mp_join) {
1102 new_ctx->ssn_offset = subflow_req->ssn_offset;
1103 new_ctx->mp_join = 1;
1104 new_ctx->fully_established = 1;
1105 new_ctx->backup = subflow_req->backup;
1106 new_ctx->local_id = subflow_req->local_id;
1107 new_ctx->token = subflow_req->token;
1108 new_ctx->thmac = subflow_req->thmac;
1109 }
1110 }
1111
1112 static struct tcp_ulp_ops subflow_ulp_ops __read_mostly = {
1113 .name = "mptcp",
1114 .owner = THIS_MODULE,
1115 .init = subflow_ulp_init,
1116 .release = subflow_ulp_release,
1117 .clone = subflow_ulp_clone,
1118 };
1119
1120 static int subflow_ops_init(struct request_sock_ops *subflow_ops)
1121 {
1122 subflow_ops->obj_size = sizeof(struct mptcp_subflow_request_sock);
1123 subflow_ops->slab_name = "request_sock_subflow";
1124
1125 subflow_ops->slab = kmem_cache_create(subflow_ops->slab_name,
1126 subflow_ops->obj_size, 0,
1127 SLAB_ACCOUNT |
1128 SLAB_TYPESAFE_BY_RCU,
1129 NULL);
1130 if (!subflow_ops->slab)
1131 return -ENOMEM;
1132
1133 subflow_ops->destructor = subflow_req_destructor;
1134
1135 return 0;
1136 }
1137
1138 void mptcp_subflow_init(void)
1139 {
1140 subflow_request_sock_ops = tcp_request_sock_ops;
1141 if (subflow_ops_init(&subflow_request_sock_ops) != 0)
1142 panic("MPTCP: failed to init subflow request sock ops\n");
1143
1144 subflow_request_sock_ipv4_ops = tcp_request_sock_ipv4_ops;
1145 subflow_request_sock_ipv4_ops.init_req = subflow_v4_init_req;
1146
1147 subflow_specific = ipv4_specific;
1148 subflow_specific.conn_request = subflow_v4_conn_request;
1149 subflow_specific.syn_recv_sock = subflow_syn_recv_sock;
1150 subflow_specific.sk_rx_dst_set = subflow_finish_connect;
1151 subflow_specific.rebuild_header = subflow_rebuild_header;
1152
1153 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
1154 subflow_request_sock_ipv6_ops = tcp_request_sock_ipv6_ops;
1155 subflow_request_sock_ipv6_ops.init_req = subflow_v6_init_req;
1156
1157 subflow_v6_specific = ipv6_specific;
1158 subflow_v6_specific.conn_request = subflow_v6_conn_request;
1159 subflow_v6_specific.syn_recv_sock = subflow_syn_recv_sock;
1160 subflow_v6_specific.sk_rx_dst_set = subflow_finish_connect;
1161 subflow_v6_specific.rebuild_header = subflow_rebuild_header;
1162
1163 subflow_v6m_specific = subflow_v6_specific;
1164 subflow_v6m_specific.queue_xmit = ipv4_specific.queue_xmit;
1165 subflow_v6m_specific.send_check = ipv4_specific.send_check;
1166 subflow_v6m_specific.net_header_len = ipv4_specific.net_header_len;
1167 subflow_v6m_specific.mtu_reduced = ipv4_specific.mtu_reduced;
1168 subflow_v6m_specific.net_frag_header_len = 0;
1169 #endif
1170
1171 mptcp_diag_subflow_init(&subflow_ulp_ops);
1172
1173 if (tcp_register_ulp(&subflow_ulp_ops) != 0)
1174 panic("MPTCP: failed to register subflows to ULP\n");
1175 }
Linux with added FPC-III support