?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
bus: mhi: core: Fix some error return code
[linux/fpc-iii.git] / net / mptcp / subflow.c
blob67a4e35d48384ee0fd54b3d9cafe0a0bf25e034a
1 // SPDX-License-Identifier: GPL-2.0
2 /* Multipath TCP
3 *
4 * Copyright (c) 2017 - 2019, Intel Corporation.
5 */
6
7 #define pr_fmt(fmt) "MPTCP: " fmt
8
9 #include <linux/kernel.h>
10 #include <linux/module.h>
11 #include <linux/netdevice.h>
12 #include <crypto/algapi.h>
13 #include <net/sock.h>
14 #include <net/inet_common.h>
15 #include <net/inet_hashtables.h>
16 #include <net/protocol.h>
17 #include <net/tcp.h>
18 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
19 #include <net/ip6_route.h>
20 #endif
21 #include <net/mptcp.h>
22 #include "protocol.h"
23 #include "mib.h"
24
25 static void SUBFLOW_REQ_INC_STATS(struct request_sock *req,
26 enum linux_mptcp_mib_field field)
27 {
28 MPTCP_INC_STATS(sock_net(req_to_sk(req)), field);
29 }
30
31 static int subflow_rebuild_header(struct sock *sk)
32 {
33 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
34 int local_id, err = 0;
35
36 if (subflow->request_mptcp && !subflow->token) {
37 pr_debug("subflow=%p", sk);
38 err = mptcp_token_new_connect(sk);
39 } else if (subflow->request_join && !subflow->local_nonce) {
40 struct mptcp_sock *msk = (struct mptcp_sock *)subflow->conn;
41
42 pr_debug("subflow=%p", sk);
43
44 do {
45 get_random_bytes(&subflow->local_nonce, sizeof(u32));
46 } while (!subflow->local_nonce);
47
48 if (subflow->local_id)
49 goto out;
50
51 local_id = mptcp_pm_get_local_id(msk, (struct sock_common *)sk);
52 if (local_id < 0)
53 return -EINVAL;
54
55 subflow->local_id = local_id;
56 }
57
58 out:
59 if (err)
60 return err;
61
62 return subflow->icsk_af_ops->rebuild_header(sk);
63 }
64
65 static void subflow_req_destructor(struct request_sock *req)
66 {
67 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
68
69 pr_debug("subflow_req=%p", subflow_req);
70
71 if (subflow_req->mp_capable)
72 mptcp_token_destroy_request(subflow_req->token);
73 tcp_request_sock_ops.destructor(req);
74 }
75
76 static void subflow_generate_hmac(u64 key1, u64 key2, u32 nonce1, u32 nonce2,
77 void *hmac)
78 {
79 u8 msg[8];
80
81 put_unaligned_be32(nonce1, &msg[0]);
82 put_unaligned_be32(nonce2, &msg[4]);
83
84 mptcp_crypto_hmac_sha(key1, key2, msg, 8, hmac);
85 }
86
87 /* validate received token and create truncated hmac and nonce for SYN-ACK */
88 static bool subflow_token_join_request(struct request_sock *req,
89 const struct sk_buff *skb)
90 {
91 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
92 u8 hmac[MPTCPOPT_HMAC_LEN];
93 struct mptcp_sock *msk;
94 int local_id;
95
96 msk = mptcp_token_get_sock(subflow_req->token);
97 if (!msk) {
98 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINNOTOKEN);
99 return false;
100 }
101
102 local_id = mptcp_pm_get_local_id(msk, (struct sock_common *)req);
103 if (local_id < 0) {
104 sock_put((struct sock *)msk);
105 return false;
106 }
107 subflow_req->local_id = local_id;
108
109 get_random_bytes(&subflow_req->local_nonce, sizeof(u32));
110
111 subflow_generate_hmac(msk->local_key, msk->remote_key,
112 subflow_req->local_nonce,
113 subflow_req->remote_nonce, hmac);
114
115 subflow_req->thmac = get_unaligned_be64(hmac);
116
117 sock_put((struct sock *)msk);
118 return true;
119 }
120
121 static void subflow_init_req(struct request_sock *req,
122 const struct sock *sk_listener,
123 struct sk_buff *skb)
124 {
125 struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk_listener);
126 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
127 struct mptcp_options_received mp_opt;
128
129 pr_debug("subflow_req=%p, listener=%p", subflow_req, listener);
130
131 mptcp_get_options(skb, &mp_opt);
132
133 subflow_req->mp_capable = 0;
134 subflow_req->mp_join = 0;
135
136 #ifdef CONFIG_TCP_MD5SIG
137 /* no MPTCP if MD5SIG is enabled on this socket or we may run out of
138 * TCP option space.
139 */
140 if (rcu_access_pointer(tcp_sk(sk_listener)->md5sig_info))
141 return;
142 #endif
143
144 if (mp_opt.mp_capable) {
145 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MPCAPABLEPASSIVE);
146
147 if (mp_opt.mp_join)
148 return;
149 } else if (mp_opt.mp_join) {
150 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINSYNRX);
151 }
152
153 if (mp_opt.mp_capable && listener->request_mptcp) {
154 int err;
155
156 err = mptcp_token_new_request(req);
157 if (err == 0)
158 subflow_req->mp_capable = 1;
159
160 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq;
161 } else if (mp_opt.mp_join && listener->request_mptcp) {
162 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq;
163 subflow_req->mp_join = 1;
164 subflow_req->backup = mp_opt.backup;
165 subflow_req->remote_id = mp_opt.join_id;
166 subflow_req->token = mp_opt.token;
167 subflow_req->remote_nonce = mp_opt.nonce;
168 pr_debug("token=%u, remote_nonce=%u", subflow_req->token,
169 subflow_req->remote_nonce);
170 if (!subflow_token_join_request(req, skb)) {
171 subflow_req->mp_join = 0;
172 // @@ need to trigger RST
173 }
174 }
175 }
176
177 static void subflow_v4_init_req(struct request_sock *req,
178 const struct sock *sk_listener,
179 struct sk_buff *skb)
180 {
181 tcp_rsk(req)->is_mptcp = 1;
182
183 tcp_request_sock_ipv4_ops.init_req(req, sk_listener, skb);
184
185 subflow_init_req(req, sk_listener, skb);
186 }
187
188 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
189 static void subflow_v6_init_req(struct request_sock *req,
190 const struct sock *sk_listener,
191 struct sk_buff *skb)
192 {
193 tcp_rsk(req)->is_mptcp = 1;
194
195 tcp_request_sock_ipv6_ops.init_req(req, sk_listener, skb);
196
197 subflow_init_req(req, sk_listener, skb);
198 }
199 #endif
200
201 /* validate received truncated hmac and create hmac for third ACK */
202 static bool subflow_thmac_valid(struct mptcp_subflow_context *subflow)
203 {
204 u8 hmac[MPTCPOPT_HMAC_LEN];
205 u64 thmac;
206
207 subflow_generate_hmac(subflow->remote_key, subflow->local_key,
208 subflow->remote_nonce, subflow->local_nonce,
209 hmac);
210
211 thmac = get_unaligned_be64(hmac);
212 pr_debug("subflow=%p, token=%u, thmac=%llu, subflow->thmac=%llu\n",
213 subflow, subflow->token,
214 (unsigned long long)thmac,
215 (unsigned long long)subflow->thmac);
216
217 return thmac == subflow->thmac;
218 }
219
220 static void subflow_finish_connect(struct sock *sk, const struct sk_buff *skb)
221 {
222 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
223 struct mptcp_options_received mp_opt;
224 struct sock *parent = subflow->conn;
225 struct tcp_sock *tp = tcp_sk(sk);
226
227 subflow->icsk_af_ops->sk_rx_dst_set(sk, skb);
228
229 if (inet_sk_state_load(parent) == TCP_SYN_SENT) {
230 inet_sk_state_store(parent, TCP_ESTABLISHED);
231 parent->sk_state_change(parent);
232 }
233
234 /* be sure no special action on any packet other than syn-ack */
235 if (subflow->conn_finished)
236 return;
237
238 subflow->conn_finished = 1;
239
240 mptcp_get_options(skb, &mp_opt);
241 if (subflow->request_mptcp && mp_opt.mp_capable) {
242 subflow->mp_capable = 1;
243 subflow->can_ack = 1;
244 subflow->remote_key = mp_opt.sndr_key;
245 pr_debug("subflow=%p, remote_key=%llu", subflow,
246 subflow->remote_key);
247 } else if (subflow->request_join && mp_opt.mp_join) {
248 subflow->mp_join = 1;
249 subflow->thmac = mp_opt.thmac;
250 subflow->remote_nonce = mp_opt.nonce;
251 pr_debug("subflow=%p, thmac=%llu, remote_nonce=%u", subflow,
252 subflow->thmac, subflow->remote_nonce);
253 } else if (subflow->request_mptcp) {
254 tp->is_mptcp = 0;
255 }
256
257 if (!tp->is_mptcp)
258 return;
259
260 if (subflow->mp_capable) {
261 pr_debug("subflow=%p, remote_key=%llu", mptcp_subflow_ctx(sk),
262 subflow->remote_key);
263 mptcp_finish_connect(sk);
264
265 if (skb) {
266 pr_debug("synack seq=%u", TCP_SKB_CB(skb)->seq);
267 subflow->ssn_offset = TCP_SKB_CB(skb)->seq;
268 }
269 } else if (subflow->mp_join) {
270 pr_debug("subflow=%p, thmac=%llu, remote_nonce=%u",
271 subflow, subflow->thmac,
272 subflow->remote_nonce);
273 if (!subflow_thmac_valid(subflow)) {
274 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINACKMAC);
275 subflow->mp_join = 0;
276 goto do_reset;
277 }
278
279 subflow_generate_hmac(subflow->local_key, subflow->remote_key,
280 subflow->local_nonce,
281 subflow->remote_nonce,
282 subflow->hmac);
283
284 if (skb)
285 subflow->ssn_offset = TCP_SKB_CB(skb)->seq;
286
287 if (!mptcp_finish_join(sk))
288 goto do_reset;
289
290 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINSYNACKRX);
291 } else {
292 do_reset:
293 tcp_send_active_reset(sk, GFP_ATOMIC);
294 tcp_done(sk);
295 }
296 }
297
298 static struct request_sock_ops subflow_request_sock_ops;
299 static struct tcp_request_sock_ops subflow_request_sock_ipv4_ops;
300
301 static int subflow_v4_conn_request(struct sock *sk, struct sk_buff *skb)
302 {
303 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
304
305 pr_debug("subflow=%p", subflow);
306
307 /* Never answer to SYNs sent to broadcast or multicast */
308 if (skb_rtable(skb)->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST))
309 goto drop;
310
311 return tcp_conn_request(&subflow_request_sock_ops,
312 &subflow_request_sock_ipv4_ops,
313 sk, skb);
314 drop:
315 tcp_listendrop(sk);
316 return 0;
317 }
318
319 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
320 static struct tcp_request_sock_ops subflow_request_sock_ipv6_ops;
321 static struct inet_connection_sock_af_ops subflow_v6_specific;
322 static struct inet_connection_sock_af_ops subflow_v6m_specific;
323
324 static int subflow_v6_conn_request(struct sock *sk, struct sk_buff *skb)
325 {
326 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
327
328 pr_debug("subflow=%p", subflow);
329
330 if (skb->protocol == htons(ETH_P_IP))
331 return subflow_v4_conn_request(sk, skb);
332
333 if (!ipv6_unicast_destination(skb))
334 goto drop;
335
336 return tcp_conn_request(&subflow_request_sock_ops,
337 &subflow_request_sock_ipv6_ops, sk, skb);
338
339 drop:
340 tcp_listendrop(sk);
341 return 0; /* don't send reset */
342 }
343 #endif
344
345 /* validate hmac received in third ACK */
346 static bool subflow_hmac_valid(const struct request_sock *req,
347 const struct mptcp_options_received *mp_opt)
348 {
349 const struct mptcp_subflow_request_sock *subflow_req;
350 u8 hmac[MPTCPOPT_HMAC_LEN];
351 struct mptcp_sock *msk;
352 bool ret;
353
354 subflow_req = mptcp_subflow_rsk(req);
355 msk = mptcp_token_get_sock(subflow_req->token);
356 if (!msk)
357 return false;
358
359 subflow_generate_hmac(msk->remote_key, msk->local_key,
360 subflow_req->remote_nonce,
361 subflow_req->local_nonce, hmac);
362
363 ret = true;
364 if (crypto_memneq(hmac, mp_opt->hmac, sizeof(hmac)))
365 ret = false;
366
367 sock_put((struct sock *)msk);
368 return ret;
369 }
370
371 static void mptcp_sock_destruct(struct sock *sk)
372 {
373 /* if new mptcp socket isn't accepted, it is free'd
374 * from the tcp listener sockets request queue, linked
375 * from req->sk. The tcp socket is released.
376 * This calls the ULP release function which will
377 * also remove the mptcp socket, via
378 * sock_put(ctx->conn).
379 *
380 * Problem is that the mptcp socket will not be in
381 * SYN_RECV state and doesn't have SOCK_DEAD flag.
382 * Both result in warnings from inet_sock_destruct.
383 */
384
385 if (sk->sk_state == TCP_SYN_RECV) {
386 sk->sk_state = TCP_CLOSE;
387 WARN_ON_ONCE(sk->sk_socket);
388 sock_orphan(sk);
389 }
390
391 inet_sock_destruct(sk);
392 }
393
394 static void mptcp_force_close(struct sock *sk)
395 {
396 inet_sk_state_store(sk, TCP_CLOSE);
397 sk_common_release(sk);
398 }
399
400 static void subflow_ulp_fallback(struct sock *sk,
401 struct mptcp_subflow_context *old_ctx)
402 {
403 struct inet_connection_sock *icsk = inet_csk(sk);
404
405 mptcp_subflow_tcp_fallback(sk, old_ctx);
406 icsk->icsk_ulp_ops = NULL;
407 rcu_assign_pointer(icsk->icsk_ulp_data, NULL);
408 tcp_sk(sk)->is_mptcp = 0;
409 }
410
411 static struct sock *subflow_syn_recv_sock(const struct sock *sk,
412 struct sk_buff *skb,
413 struct request_sock *req,
414 struct dst_entry *dst,
415 struct request_sock *req_unhash,
416 bool *own_req)
417 {
418 struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk);
419 struct mptcp_subflow_request_sock *subflow_req;
420 struct mptcp_options_received mp_opt;
421 bool fallback_is_fatal = false;
422 struct sock *new_msk = NULL;
423 bool fallback = false;
424 struct sock *child;
425
426 pr_debug("listener=%p, req=%p, conn=%p", listener, req, listener->conn);
427
428 /* we need later a valid 'mp_capable' value even when options are not
429 * parsed
430 */
431 mp_opt.mp_capable = 0;
432 if (tcp_rsk(req)->is_mptcp == 0)
433 goto create_child;
434
435 /* if the sk is MP_CAPABLE, we try to fetch the client key */
436 subflow_req = mptcp_subflow_rsk(req);
437 if (subflow_req->mp_capable) {
438 if (TCP_SKB_CB(skb)->seq != subflow_req->ssn_offset + 1) {
439 /* here we can receive and accept an in-window,
440 * out-of-order pkt, which will not carry the MP_CAPABLE
441 * opt even on mptcp enabled paths
442 */
443 goto create_msk;
444 }
445
446 mptcp_get_options(skb, &mp_opt);
447 if (!mp_opt.mp_capable) {
448 fallback = true;
449 goto create_child;
450 }
451
452 create_msk:
453 new_msk = mptcp_sk_clone(listener->conn, &mp_opt, req);
454 if (!new_msk)
455 fallback = true;
456 } else if (subflow_req->mp_join) {
457 fallback_is_fatal = true;
458 mptcp_get_options(skb, &mp_opt);
459 if (!mp_opt.mp_join ||
460 !subflow_hmac_valid(req, &mp_opt)) {
461 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINACKMAC);
462 return NULL;
463 }
464 }
465
466 create_child:
467 child = listener->icsk_af_ops->syn_recv_sock(sk, skb, req, dst,
468 req_unhash, own_req);
469
470 if (child && *own_req) {
471 struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(child);
472
473 /* we need to fallback on ctx allocation failure and on pre-reqs
474 * checking above. In the latter scenario we additionally need
475 * to reset the context to non MPTCP status.
476 */
477 if (!ctx || fallback) {
478 if (fallback_is_fatal)
479 goto close_child;
480
481 if (ctx) {
482 subflow_ulp_fallback(child, ctx);
483 kfree_rcu(ctx, rcu);
484 }
485 goto out;
486 }
487
488 if (ctx->mp_capable) {
489 /* new mpc subflow takes ownership of the newly
490 * created mptcp socket
491 */
492 new_msk->sk_destruct = mptcp_sock_destruct;
493 mptcp_pm_new_connection(mptcp_sk(new_msk), 1);
494 ctx->conn = new_msk;
495 new_msk = NULL;
496
497 /* with OoO packets we can reach here without ingress
498 * mpc option
499 */
500 ctx->remote_key = mp_opt.sndr_key;
501 ctx->fully_established = mp_opt.mp_capable;
502 ctx->can_ack = mp_opt.mp_capable;
503 } else if (ctx->mp_join) {
504 struct mptcp_sock *owner;
505
506 owner = mptcp_token_get_sock(ctx->token);
507 if (!owner)
508 goto close_child;
509
510 ctx->conn = (struct sock *)owner;
511 if (!mptcp_finish_join(child))
512 goto close_child;
513
514 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINACKRX);
515 }
516 }
517
518 out:
519 /* dispose of the left over mptcp master, if any */
520 if (unlikely(new_msk))
521 mptcp_force_close(new_msk);
522
523 /* check for expected invariant - should never trigger, just help
524 * catching eariler subtle bugs
525 */
526 WARN_ON_ONCE(child && *own_req && tcp_sk(child)->is_mptcp &&
527 (!mptcp_subflow_ctx(child) ||
528 !mptcp_subflow_ctx(child)->conn));
529 return child;
530
531 close_child:
532 tcp_send_active_reset(child, GFP_ATOMIC);
533 inet_csk_prepare_forced_close(child);
534 tcp_done(child);
535 return NULL;
536 }
537
538 static struct inet_connection_sock_af_ops subflow_specific;
539
540 enum mapping_status {
541 MAPPING_OK,
542 MAPPING_INVALID,
543 MAPPING_EMPTY,
544 MAPPING_DATA_FIN
545 };
546
547 static u64 expand_seq(u64 old_seq, u16 old_data_len, u64 seq)
548 {
549 if ((u32)seq == (u32)old_seq)
550 return old_seq;
551
552 /* Assume map covers data not mapped yet. */
553 return seq | ((old_seq + old_data_len + 1) & GENMASK_ULL(63, 32));
554 }
555
556 static void warn_bad_map(struct mptcp_subflow_context *subflow, u32 ssn)
557 {
558 WARN_ONCE(1, "Bad mapping: ssn=%d map_seq=%d map_data_len=%d",
559 ssn, subflow->map_subflow_seq, subflow->map_data_len);
560 }
561
562 static bool skb_is_fully_mapped(struct sock *ssk, struct sk_buff *skb)
563 {
564 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
565 unsigned int skb_consumed;
566
567 skb_consumed = tcp_sk(ssk)->copied_seq - TCP_SKB_CB(skb)->seq;
568 if (WARN_ON_ONCE(skb_consumed >= skb->len))
569 return true;
570
571 return skb->len - skb_consumed <= subflow->map_data_len -
572 mptcp_subflow_get_map_offset(subflow);
573 }
574
575 static bool validate_mapping(struct sock *ssk, struct sk_buff *skb)
576 {
577 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
578 u32 ssn = tcp_sk(ssk)->copied_seq - subflow->ssn_offset;
579
580 if (unlikely(before(ssn, subflow->map_subflow_seq))) {
581 /* Mapping covers data later in the subflow stream,
582 * currently unsupported.
583 */
584 warn_bad_map(subflow, ssn);
585 return false;
586 }
587 if (unlikely(!before(ssn, subflow->map_subflow_seq +
588 subflow->map_data_len))) {
589 /* Mapping does covers past subflow data, invalid */
590 warn_bad_map(subflow, ssn + skb->len);
591 return false;
592 }
593 return true;
594 }
595
596 static enum mapping_status get_mapping_status(struct sock *ssk)
597 {
598 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
599 struct mptcp_ext *mpext;
600 struct sk_buff *skb;
601 u16 data_len;
602 u64 map_seq;
603
604 skb = skb_peek(&ssk->sk_receive_queue);
605 if (!skb)
606 return MAPPING_EMPTY;
607
608 mpext = mptcp_get_ext(skb);
609 if (!mpext || !mpext->use_map) {
610 if (!subflow->map_valid && !skb->len) {
611 /* the TCP stack deliver 0 len FIN pkt to the receive
612 * queue, that is the only 0len pkts ever expected here,
613 * and we can admit no mapping only for 0 len pkts
614 */
615 if (!(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN))
616 WARN_ONCE(1, "0len seq %d:%d flags %x",
617 TCP_SKB_CB(skb)->seq,
618 TCP_SKB_CB(skb)->end_seq,
619 TCP_SKB_CB(skb)->tcp_flags);
620 sk_eat_skb(ssk, skb);
621 return MAPPING_EMPTY;
622 }
623
624 if (!subflow->map_valid)
625 return MAPPING_INVALID;
626
627 goto validate_seq;
628 }
629
630 pr_debug("seq=%llu is64=%d ssn=%u data_len=%u data_fin=%d",
631 mpext->data_seq, mpext->dsn64, mpext->subflow_seq,
632 mpext->data_len, mpext->data_fin);
633
634 data_len = mpext->data_len;
635 if (data_len == 0) {
636 pr_err("Infinite mapping not handled");
637 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_INFINITEMAPRX);
638 return MAPPING_INVALID;
639 }
640
641 if (mpext->data_fin == 1) {
642 if (data_len == 1) {
643 pr_debug("DATA_FIN with no payload");
644 if (subflow->map_valid) {
645 /* A DATA_FIN might arrive in a DSS
646 * option before the previous mapping
647 * has been fully consumed. Continue
648 * handling the existing mapping.
649 */
650 skb_ext_del(skb, SKB_EXT_MPTCP);
651 return MAPPING_OK;
652 } else {
653 return MAPPING_DATA_FIN;
654 }
655 }
656
657 /* Adjust for DATA_FIN using 1 byte of sequence space */
658 data_len--;
659 }
660
661 if (!mpext->dsn64) {
662 map_seq = expand_seq(subflow->map_seq, subflow->map_data_len,
663 mpext->data_seq);
664 pr_debug("expanded seq=%llu", subflow->map_seq);
665 } else {
666 map_seq = mpext->data_seq;
667 }
668
669 if (subflow->map_valid) {
670 /* Allow replacing only with an identical map */
671 if (subflow->map_seq == map_seq &&
672 subflow->map_subflow_seq == mpext->subflow_seq &&
673 subflow->map_data_len == data_len) {
674 skb_ext_del(skb, SKB_EXT_MPTCP);
675 return MAPPING_OK;
676 }
677
678 /* If this skb data are fully covered by the current mapping,
679 * the new map would need caching, which is not supported
680 */
681 if (skb_is_fully_mapped(ssk, skb)) {
682 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DSSNOMATCH);
683 return MAPPING_INVALID;
684 }
685
686 /* will validate the next map after consuming the current one */
687 return MAPPING_OK;
688 }
689
690 subflow->map_seq = map_seq;
691 subflow->map_subflow_seq = mpext->subflow_seq;
692 subflow->map_data_len = data_len;
693 subflow->map_valid = 1;
694 subflow->mpc_map = mpext->mpc_map;
695 pr_debug("new map seq=%llu subflow_seq=%u data_len=%u",
696 subflow->map_seq, subflow->map_subflow_seq,
697 subflow->map_data_len);
698
699 validate_seq:
700 /* we revalidate valid mapping on new skb, because we must ensure
701 * the current skb is completely covered by the available mapping
702 */
703 if (!validate_mapping(ssk, skb))
704 return MAPPING_INVALID;
705
706 skb_ext_del(skb, SKB_EXT_MPTCP);
707 return MAPPING_OK;
708 }
709
710 static int subflow_read_actor(read_descriptor_t *desc,
711 struct sk_buff *skb,
712 unsigned int offset, size_t len)
713 {
714 size_t copy_len = min(desc->count, len);
715
716 desc->count -= copy_len;
717
718 pr_debug("flushed %zu bytes, %zu left", copy_len, desc->count);
719 return copy_len;
720 }
721
722 static bool subflow_check_data_avail(struct sock *ssk)
723 {
724 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
725 enum mapping_status status;
726 struct mptcp_sock *msk;
727 struct sk_buff *skb;
728
729 pr_debug("msk=%p ssk=%p data_avail=%d skb=%p", subflow->conn, ssk,
730 subflow->data_avail, skb_peek(&ssk->sk_receive_queue));
731 if (subflow->data_avail)
732 return true;
733
734 msk = mptcp_sk(subflow->conn);
735 for (;;) {
736 u32 map_remaining;
737 size_t delta;
738 u64 ack_seq;
739 u64 old_ack;
740
741 status = get_mapping_status(ssk);
742 pr_debug("msk=%p ssk=%p status=%d", msk, ssk, status);
743 if (status == MAPPING_INVALID) {
744 ssk->sk_err = EBADMSG;
745 goto fatal;
746 }
747
748 if (status != MAPPING_OK)
749 return false;
750
751 skb = skb_peek(&ssk->sk_receive_queue);
752 if (WARN_ON_ONCE(!skb))
753 return false;
754
755 /* if msk lacks the remote key, this subflow must provide an
756 * MP_CAPABLE-based mapping
757 */
758 if (unlikely(!READ_ONCE(msk->can_ack))) {
759 if (!subflow->mpc_map) {
760 ssk->sk_err = EBADMSG;
761 goto fatal;
762 }
763 WRITE_ONCE(msk->remote_key, subflow->remote_key);
764 WRITE_ONCE(msk->ack_seq, subflow->map_seq);
765 WRITE_ONCE(msk->can_ack, true);
766 }
767
768 old_ack = READ_ONCE(msk->ack_seq);
769 ack_seq = mptcp_subflow_get_mapped_dsn(subflow);
770 pr_debug("msk ack_seq=%llx subflow ack_seq=%llx", old_ack,
771 ack_seq);
772 if (ack_seq == old_ack)
773 break;
774
775 /* only accept in-sequence mapping. Old values are spurious
776 * retransmission; we can hit "future" values on active backup
777 * subflow switch, we relay on retransmissions to get
778 * in-sequence data.
779 * Cuncurrent subflows support will require subflow data
780 * reordering
781 */
782 map_remaining = subflow->map_data_len -
783 mptcp_subflow_get_map_offset(subflow);
784 if (before64(ack_seq, old_ack))
785 delta = min_t(size_t, old_ack - ack_seq, map_remaining);
786 else
787 delta = min_t(size_t, ack_seq - old_ack, map_remaining);
788
789 /* discard mapped data */
790 pr_debug("discarding %zu bytes, current map len=%d", delta,
791 map_remaining);
792 if (delta) {
793 read_descriptor_t desc = {
794 .count = delta,
795 };
796 int ret;
797
798 ret = tcp_read_sock(ssk, &desc, subflow_read_actor);
799 if (ret < 0) {
800 ssk->sk_err = -ret;
801 goto fatal;
802 }
803 if (ret < delta)
804 return false;
805 if (delta == map_remaining)
806 subflow->map_valid = 0;
807 }
808 }
809 return true;
810
811 fatal:
812 /* fatal protocol error, close the socket */
813 /* This barrier is coupled with smp_rmb() in tcp_poll() */
814 smp_wmb();
815 ssk->sk_error_report(ssk);
816 tcp_set_state(ssk, TCP_CLOSE);
817 tcp_send_active_reset(ssk, GFP_ATOMIC);
818 return false;
819 }
820
821 bool mptcp_subflow_data_available(struct sock *sk)
822 {
823 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
824 struct sk_buff *skb;
825
826 /* check if current mapping is still valid */
827 if (subflow->map_valid &&
828 mptcp_subflow_get_map_offset(subflow) >= subflow->map_data_len) {
829 subflow->map_valid = 0;
830 subflow->data_avail = 0;
831
832 pr_debug("Done with mapping: seq=%u data_len=%u",
833 subflow->map_subflow_seq,
834 subflow->map_data_len);
835 }
836
837 if (!subflow_check_data_avail(sk)) {
838 subflow->data_avail = 0;
839 return false;
840 }
841
842 skb = skb_peek(&sk->sk_receive_queue);
843 subflow->data_avail = skb &&
844 before(tcp_sk(sk)->copied_seq, TCP_SKB_CB(skb)->end_seq);
845 return subflow->data_avail;
846 }
847
848 static void subflow_data_ready(struct sock *sk)
849 {
850 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
851 struct sock *parent = subflow->conn;
852
853 if (!subflow->mp_capable && !subflow->mp_join) {
854 subflow->tcp_data_ready(sk);
855
856 parent->sk_data_ready(parent);
857 return;
858 }
859
860 if (mptcp_subflow_data_available(sk))
861 mptcp_data_ready(parent, sk);
862 }
863
864 static void subflow_write_space(struct sock *sk)
865 {
866 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
867 struct sock *parent = subflow->conn;
868
869 sk_stream_write_space(sk);
870 if (sk_stream_is_writeable(sk)) {
871 set_bit(MPTCP_SEND_SPACE, &mptcp_sk(parent)->flags);
872 smp_mb__after_atomic();
873 /* set SEND_SPACE before sk_stream_write_space clears NOSPACE */
874 sk_stream_write_space(parent);
875 }
876 }
877
878 static struct inet_connection_sock_af_ops *
879 subflow_default_af_ops(struct sock *sk)
880 {
881 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
882 if (sk->sk_family == AF_INET6)
883 return &subflow_v6_specific;
884 #endif
885 return &subflow_specific;
886 }
887
888 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
889 void mptcpv6_handle_mapped(struct sock *sk, bool mapped)
890 {
891 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
892 struct inet_connection_sock *icsk = inet_csk(sk);
893 struct inet_connection_sock_af_ops *target;
894
895 target = mapped ? &subflow_v6m_specific : subflow_default_af_ops(sk);
896
897 pr_debug("subflow=%p family=%d ops=%p target=%p mapped=%d",
898 subflow, sk->sk_family, icsk->icsk_af_ops, target, mapped);
899
900 if (likely(icsk->icsk_af_ops == target))
901 return;
902
903 subflow->icsk_af_ops = icsk->icsk_af_ops;
904 icsk->icsk_af_ops = target;
905 }
906 #endif
907
908 static void mptcp_info2sockaddr(const struct mptcp_addr_info *info,
909 struct sockaddr_storage *addr)
910 {
911 memset(addr, 0, sizeof(*addr));
912 addr->ss_family = info->family;
913 if (addr->ss_family == AF_INET) {
914 struct sockaddr_in *in_addr = (struct sockaddr_in *)addr;
915
916 in_addr->sin_addr = info->addr;
917 in_addr->sin_port = info->port;
918 }
919 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
920 else if (addr->ss_family == AF_INET6) {
921 struct sockaddr_in6 *in6_addr = (struct sockaddr_in6 *)addr;
922
923 in6_addr->sin6_addr = info->addr6;
924 in6_addr->sin6_port = info->port;
925 }
926 #endif
927 }
928
929 int __mptcp_subflow_connect(struct sock *sk, int ifindex,
930 const struct mptcp_addr_info *loc,
931 const struct mptcp_addr_info *remote)
932 {
933 struct mptcp_sock *msk = mptcp_sk(sk);
934 struct mptcp_subflow_context *subflow;
935 struct sockaddr_storage addr;
936 struct socket *sf;
937 u32 remote_token;
938 int addrlen;
939 int err;
940
941 if (sk->sk_state != TCP_ESTABLISHED)
942 return -ENOTCONN;
943
944 err = mptcp_subflow_create_socket(sk, &sf);
945 if (err)
946 return err;
947
948 subflow = mptcp_subflow_ctx(sf->sk);
949 subflow->remote_key = msk->remote_key;
950 subflow->local_key = msk->local_key;
951 subflow->token = msk->token;
952 mptcp_info2sockaddr(loc, &addr);
953
954 addrlen = sizeof(struct sockaddr_in);
955 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
956 if (loc->family == AF_INET6)
957 addrlen = sizeof(struct sockaddr_in6);
958 #endif
959 sf->sk->sk_bound_dev_if = ifindex;
960 err = kernel_bind(sf, (struct sockaddr *)&addr, addrlen);
961 if (err)
962 goto failed;
963
964 mptcp_crypto_key_sha(subflow->remote_key, &remote_token, NULL);
965 pr_debug("msk=%p remote_token=%u", msk, remote_token);
966 subflow->remote_token = remote_token;
967 subflow->local_id = loc->id;
968 subflow->request_join = 1;
969 subflow->request_bkup = 1;
970 mptcp_info2sockaddr(remote, &addr);
971
972 err = kernel_connect(sf, (struct sockaddr *)&addr, addrlen, O_NONBLOCK);
973 if (err && err != -EINPROGRESS)
974 goto failed;
975
976 spin_lock_bh(&msk->join_list_lock);
977 list_add_tail(&subflow->node, &msk->join_list);
978 spin_unlock_bh(&msk->join_list_lock);
979
980 return err;
981
982 failed:
983 sock_release(sf);
984 return err;
985 }
986
987 int mptcp_subflow_create_socket(struct sock *sk, struct socket **new_sock)
988 {
989 struct mptcp_subflow_context *subflow;
990 struct net *net = sock_net(sk);
991 struct socket *sf;
992 int err;
993
994 err = sock_create_kern(net, sk->sk_family, SOCK_STREAM, IPPROTO_TCP,
995 &sf);
996 if (err)
997 return err;
998
999 lock_sock(sf->sk);
1000
1001 /* kernel sockets do not by default acquire net ref, but TCP timer
1002 * needs it.
1003 */
1004 sf->sk->sk_net_refcnt = 1;
1005 get_net(net);
1006 #ifdef CONFIG_PROC_FS
1007 this_cpu_add(*net->core.sock_inuse, 1);
1008 #endif
1009 err = tcp_set_ulp(sf->sk, "mptcp");
1010 release_sock(sf->sk);
1011
1012 if (err)
1013 return err;
1014
1015 subflow = mptcp_subflow_ctx(sf->sk);
1016 pr_debug("subflow=%p", subflow);
1017
1018 *new_sock = sf;
1019 sock_hold(sk);
1020 subflow->conn = sk;
1021
1022 return 0;
1023 }
1024
1025 static struct mptcp_subflow_context *subflow_create_ctx(struct sock *sk,
1026 gfp_t priority)
1027 {
1028 struct inet_connection_sock *icsk = inet_csk(sk);
1029 struct mptcp_subflow_context *ctx;
1030
1031 ctx = kzalloc(sizeof(*ctx), priority);
1032 if (!ctx)
1033 return NULL;
1034
1035 rcu_assign_pointer(icsk->icsk_ulp_data, ctx);
1036 INIT_LIST_HEAD(&ctx->node);
1037
1038 pr_debug("subflow=%p", ctx);
1039
1040 ctx->tcp_sock = sk;
1041
1042 return ctx;
1043 }
1044
1045 static void __subflow_state_change(struct sock *sk)
1046 {
1047 struct socket_wq *wq;
1048
1049 rcu_read_lock();
1050 wq = rcu_dereference(sk->sk_wq);
1051 if (skwq_has_sleeper(wq))
1052 wake_up_interruptible_all(&wq->wait);
1053 rcu_read_unlock();
1054 }
1055
1056 static bool subflow_is_done(const struct sock *sk)
1057 {
1058 return sk->sk_shutdown & RCV_SHUTDOWN || sk->sk_state == TCP_CLOSE;
1059 }
1060
1061 static void subflow_state_change(struct sock *sk)
1062 {
1063 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
1064 struct sock *parent = subflow->conn;
1065
1066 __subflow_state_change(sk);
1067
1068 /* as recvmsg() does not acquire the subflow socket for ssk selection
1069 * a fin packet carrying a DSS can be unnoticed if we don't trigger
1070 * the data available machinery here.
1071 */
1072 if (subflow->mp_capable && mptcp_subflow_data_available(sk))
1073 mptcp_data_ready(parent, sk);
1074
1075 if (!(parent->sk_shutdown & RCV_SHUTDOWN) &&
1076 !subflow->rx_eof && subflow_is_done(sk)) {
1077 subflow->rx_eof = 1;
1078 mptcp_subflow_eof(parent);
1079 }
1080 }
1081
1082 static int subflow_ulp_init(struct sock *sk)
1083 {
1084 struct inet_connection_sock *icsk = inet_csk(sk);
1085 struct mptcp_subflow_context *ctx;
1086 struct tcp_sock *tp = tcp_sk(sk);
1087 int err = 0;
1088
1089 /* disallow attaching ULP to a socket unless it has been
1090 * created with sock_create_kern()
1091 */
1092 if (!sk->sk_kern_sock) {
1093 err = -EOPNOTSUPP;
1094 goto out;
1095 }
1096
1097 ctx = subflow_create_ctx(sk, GFP_KERNEL);
1098 if (!ctx) {
1099 err = -ENOMEM;
1100 goto out;
1101 }
1102
1103 pr_debug("subflow=%p, family=%d", ctx, sk->sk_family);
1104
1105 tp->is_mptcp = 1;
1106 ctx->icsk_af_ops = icsk->icsk_af_ops;
1107 icsk->icsk_af_ops = subflow_default_af_ops(sk);
1108 ctx->tcp_data_ready = sk->sk_data_ready;
1109 ctx->tcp_state_change = sk->sk_state_change;
1110 ctx->tcp_write_space = sk->sk_write_space;
1111 sk->sk_data_ready = subflow_data_ready;
1112 sk->sk_write_space = subflow_write_space;
1113 sk->sk_state_change = subflow_state_change;
1114 out:
1115 return err;
1116 }
1117
1118 static void subflow_ulp_release(struct sock *sk)
1119 {
1120 struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(sk);
1121
1122 if (!ctx)
1123 return;
1124
1125 if (ctx->conn)
1126 sock_put(ctx->conn);
1127
1128 kfree_rcu(ctx, rcu);
1129 }
1130
1131 static void subflow_ulp_clone(const struct request_sock *req,
1132 struct sock *newsk,
1133 const gfp_t priority)
1134 {
1135 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
1136 struct mptcp_subflow_context *old_ctx = mptcp_subflow_ctx(newsk);
1137 struct mptcp_subflow_context *new_ctx;
1138
1139 if (!tcp_rsk(req)->is_mptcp ||
1140 (!subflow_req->mp_capable && !subflow_req->mp_join)) {
1141 subflow_ulp_fallback(newsk, old_ctx);
1142 return;
1143 }
1144
1145 new_ctx = subflow_create_ctx(newsk, priority);
1146 if (!new_ctx) {
1147 subflow_ulp_fallback(newsk, old_ctx);
1148 return;
1149 }
1150
1151 new_ctx->conn_finished = 1;
1152 new_ctx->icsk_af_ops = old_ctx->icsk_af_ops;
1153 new_ctx->tcp_data_ready = old_ctx->tcp_data_ready;
1154 new_ctx->tcp_state_change = old_ctx->tcp_state_change;
1155 new_ctx->tcp_write_space = old_ctx->tcp_write_space;
1156 new_ctx->rel_write_seq = 1;
1157 new_ctx->tcp_sock = newsk;
1158
1159 if (subflow_req->mp_capable) {
1160 /* see comments in subflow_syn_recv_sock(), MPTCP connection
1161 * is fully established only after we receive the remote key
1162 */
1163 new_ctx->mp_capable = 1;
1164 new_ctx->local_key = subflow_req->local_key;
1165 new_ctx->token = subflow_req->token;
1166 new_ctx->ssn_offset = subflow_req->ssn_offset;
1167 new_ctx->idsn = subflow_req->idsn;
1168 } else if (subflow_req->mp_join) {
1169 new_ctx->ssn_offset = subflow_req->ssn_offset;
1170 new_ctx->mp_join = 1;
1171 new_ctx->fully_established = 1;
1172 new_ctx->backup = subflow_req->backup;
1173 new_ctx->local_id = subflow_req->local_id;
1174 new_ctx->token = subflow_req->token;
1175 new_ctx->thmac = subflow_req->thmac;
1176 }
1177 }
1178
1179 static struct tcp_ulp_ops subflow_ulp_ops __read_mostly = {
1180 .name = "mptcp",
1181 .owner = THIS_MODULE,
1182 .init = subflow_ulp_init,
1183 .release = subflow_ulp_release,
1184 .clone = subflow_ulp_clone,
1185 };
1186
1187 static int subflow_ops_init(struct request_sock_ops *subflow_ops)
1188 {
1189 subflow_ops->obj_size = sizeof(struct mptcp_subflow_request_sock);
1190 subflow_ops->slab_name = "request_sock_subflow";
1191
1192 subflow_ops->slab = kmem_cache_create(subflow_ops->slab_name,
1193 subflow_ops->obj_size, 0,
1194 SLAB_ACCOUNT |
1195 SLAB_TYPESAFE_BY_RCU,
1196 NULL);
1197 if (!subflow_ops->slab)
1198 return -ENOMEM;
1199
1200 subflow_ops->destructor = subflow_req_destructor;
1201
1202 return 0;
1203 }
1204
1205 void mptcp_subflow_init(void)
1206 {
1207 subflow_request_sock_ops = tcp_request_sock_ops;
1208 if (subflow_ops_init(&subflow_request_sock_ops) != 0)
1209 panic("MPTCP: failed to init subflow request sock ops\n");
1210
1211 subflow_request_sock_ipv4_ops = tcp_request_sock_ipv4_ops;
1212 subflow_request_sock_ipv4_ops.init_req = subflow_v4_init_req;
1213
1214 subflow_specific = ipv4_specific;
1215 subflow_specific.conn_request = subflow_v4_conn_request;
1216 subflow_specific.syn_recv_sock = subflow_syn_recv_sock;
1217 subflow_specific.sk_rx_dst_set = subflow_finish_connect;
1218 subflow_specific.rebuild_header = subflow_rebuild_header;
1219
1220 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
1221 subflow_request_sock_ipv6_ops = tcp_request_sock_ipv6_ops;
1222 subflow_request_sock_ipv6_ops.init_req = subflow_v6_init_req;
1223
1224 subflow_v6_specific = ipv6_specific;
1225 subflow_v6_specific.conn_request = subflow_v6_conn_request;
1226 subflow_v6_specific.syn_recv_sock = subflow_syn_recv_sock;
1227 subflow_v6_specific.sk_rx_dst_set = subflow_finish_connect;
1228 subflow_v6_specific.rebuild_header = subflow_rebuild_header;
1229
1230 subflow_v6m_specific = subflow_v6_specific;
1231 subflow_v6m_specific.queue_xmit = ipv4_specific.queue_xmit;
1232 subflow_v6m_specific.send_check = ipv4_specific.send_check;
1233 subflow_v6m_specific.net_header_len = ipv4_specific.net_header_len;
1234 subflow_v6m_specific.mtu_reduced = ipv4_specific.mtu_reduced;
1235 subflow_v6m_specific.net_frag_header_len = 0;
1236 #endif
1237
1238 mptcp_diag_subflow_init(&subflow_ulp_ops);
1239
1240 if (tcp_register_ulp(&subflow_ulp_ops) != 0)
1241 panic("MPTCP: failed to register subflows to ULP\n");
1242 }
Linux with added FPC-III support