?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge branch 'bnxt_en-next'
[linux/fpc-iii.git] / net / ipv4 / tcp_ipv4.c
blob32b048e524d6773538918eca175b3f422f9c2aa7
1 /*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * Implementation of the Transmission Control Protocol(TCP).
7 *
8 * IPv4 specific functions
9 *
10 *
11 * code split from:
12 * linux/ipv4/tcp.c
13 * linux/ipv4/tcp_input.c
14 * linux/ipv4/tcp_output.c
15 *
16 * See tcp.c for author information
17 *
18 * This program is free software; you can redistribute it and/or
19 * modify it under the terms of the GNU General Public License
20 * as published by the Free Software Foundation; either version
21 * 2 of the License, or (at your option) any later version.
22 */
23
24 /*
25 * Changes:
26 * David S. Miller : New socket lookup architecture.
27 * This code is dedicated to John Dyson.
28 * David S. Miller : Change semantics of established hash,
29 * half is devoted to TIME_WAIT sockets
30 * and the rest go in the other half.
31 * Andi Kleen : Add support for syncookies and fixed
32 * some bugs: ip options weren't passed to
33 * the TCP layer, missed a check for an
34 * ACK bit.
35 * Andi Kleen : Implemented fast path mtu discovery.
36 * Fixed many serious bugs in the
37 * request_sock handling and moved
38 * most of it into the af independent code.
39 * Added tail drop and some other bugfixes.
40 * Added new listen semantics.
41 * Mike McLagan : Routing by source
42 * Juan Jose Ciarlante: ip_dynaddr bits
43 * Andi Kleen: various fixes.
44 * Vitaly E. Lavrov : Transparent proxy revived after year
45 * coma.
46 * Andi Kleen : Fix new listen.
47 * Andi Kleen : Fix accept error reporting.
48 * YOSHIFUJI Hideaki @USAGI and: Support IPV6_V6ONLY socket option, which
49 * Alexey Kuznetsov allow both IPv4 and IPv6 sockets to bind
50 * a single port at the same time.
51 */
52
53 #define pr_fmt(fmt) "TCP: " fmt
54
55 #include <linux/bottom_half.h>
56 #include <linux/types.h>
57 #include <linux/fcntl.h>
58 #include <linux/module.h>
59 #include <linux/random.h>
60 #include <linux/cache.h>
61 #include <linux/jhash.h>
62 #include <linux/init.h>
63 #include <linux/times.h>
64 #include <linux/slab.h>
65
66 #include <net/net_namespace.h>
67 #include <net/icmp.h>
68 #include <net/inet_hashtables.h>
69 #include <net/tcp.h>
70 #include <net/transp_v6.h>
71 #include <net/ipv6.h>
72 #include <net/inet_common.h>
73 #include <net/timewait_sock.h>
74 #include <net/xfrm.h>
75 #include <net/secure_seq.h>
76 #include <net/busy_poll.h>
77
78 #include <linux/inet.h>
79 #include <linux/ipv6.h>
80 #include <linux/stddef.h>
81 #include <linux/proc_fs.h>
82 #include <linux/seq_file.h>
83
84 #include <crypto/hash.h>
85 #include <linux/scatterlist.h>
86
87 int sysctl_tcp_tw_reuse __read_mostly;
88 int sysctl_tcp_low_latency __read_mostly;
89 EXPORT_SYMBOL(sysctl_tcp_low_latency);
90
91 #ifdef CONFIG_TCP_MD5SIG
92 static int tcp_v4_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key,
93 __be32 daddr, __be32 saddr, const struct tcphdr *th);
94 #endif
95
96 struct inet_hashinfo tcp_hashinfo;
97 EXPORT_SYMBOL(tcp_hashinfo);
98
99 static __u32 tcp_v4_init_sequence(const struct sk_buff *skb)
100 {
101 return secure_tcp_sequence_number(ip_hdr(skb)->daddr,
102 ip_hdr(skb)->saddr,
103 tcp_hdr(skb)->dest,
104 tcp_hdr(skb)->source);
105 }
106
107 int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp)
108 {
109 const struct tcp_timewait_sock *tcptw = tcp_twsk(sktw);
110 struct tcp_sock *tp = tcp_sk(sk);
111
112 /* With PAWS, it is safe from the viewpoint
113 of data integrity. Even without PAWS it is safe provided sequence
114 spaces do not overlap i.e. at data rates <= 80Mbit/sec.
115
116 Actually, the idea is close to VJ's one, only timestamp cache is
117 held not per host, but per port pair and TW bucket is used as state
118 holder.
119
120 If TW bucket has been already destroyed we fall back to VJ's scheme
121 and use initial timestamp retrieved from peer table.
122 */
123 if (tcptw->tw_ts_recent_stamp &&
124 (!twp || (sysctl_tcp_tw_reuse &&
125 get_seconds() - tcptw->tw_ts_recent_stamp > 1))) {
126 tp->write_seq = tcptw->tw_snd_nxt + 65535 + 2;
127 if (tp->write_seq == 0)
128 tp->write_seq = 1;
129 tp->rx_opt.ts_recent = tcptw->tw_ts_recent;
130 tp->rx_opt.ts_recent_stamp = tcptw->tw_ts_recent_stamp;
131 sock_hold(sktw);
132 return 1;
133 }
134
135 return 0;
136 }
137 EXPORT_SYMBOL_GPL(tcp_twsk_unique);
138
139 /* This will initiate an outgoing connection. */
140 int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
141 {
142 struct sockaddr_in *usin = (struct sockaddr_in *)uaddr;
143 struct inet_sock *inet = inet_sk(sk);
144 struct tcp_sock *tp = tcp_sk(sk);
145 __be16 orig_sport, orig_dport;
146 __be32 daddr, nexthop;
147 struct flowi4 *fl4;
148 struct rtable *rt;
149 int err;
150 struct ip_options_rcu *inet_opt;
151
152 if (addr_len < sizeof(struct sockaddr_in))
153 return -EINVAL;
154
155 if (usin->sin_family != AF_INET)
156 return -EAFNOSUPPORT;
157
158 nexthop = daddr = usin->sin_addr.s_addr;
159 inet_opt = rcu_dereference_protected(inet->inet_opt,
160 lockdep_sock_is_held(sk));
161 if (inet_opt && inet_opt->opt.srr) {
162 if (!daddr)
163 return -EINVAL;
164 nexthop = inet_opt->opt.faddr;
165 }
166
167 orig_sport = inet->inet_sport;
168 orig_dport = usin->sin_port;
169 fl4 = &inet->cork.fl.u.ip4;
170 rt = ip_route_connect(fl4, nexthop, inet->inet_saddr,
171 RT_CONN_FLAGS(sk), sk->sk_bound_dev_if,
172 IPPROTO_TCP,
173 orig_sport, orig_dport, sk);
174 if (IS_ERR(rt)) {
175 err = PTR_ERR(rt);
176 if (err == -ENETUNREACH)
177 IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTNOROUTES);
178 return err;
179 }
180
181 if (rt->rt_flags & (RTCF_MULTICAST | RTCF_BROADCAST)) {
182 ip_rt_put(rt);
183 return -ENETUNREACH;
184 }
185
186 if (!inet_opt || !inet_opt->opt.srr)
187 daddr = fl4->daddr;
188
189 if (!inet->inet_saddr)
190 inet->inet_saddr = fl4->saddr;
191 sk_rcv_saddr_set(sk, inet->inet_saddr);
192
193 if (tp->rx_opt.ts_recent_stamp && inet->inet_daddr != daddr) {
194 /* Reset inherited state */
195 tp->rx_opt.ts_recent = 0;
196 tp->rx_opt.ts_recent_stamp = 0;
197 if (likely(!tp->repair))
198 tp->write_seq = 0;
199 }
200
201 if (tcp_death_row.sysctl_tw_recycle &&
202 !tp->rx_opt.ts_recent_stamp && fl4->daddr == daddr)
203 tcp_fetch_timewait_stamp(sk, &rt->dst);
204
205 inet->inet_dport = usin->sin_port;
206 sk_daddr_set(sk, daddr);
207
208 inet_csk(sk)->icsk_ext_hdr_len = 0;
209 if (inet_opt)
210 inet_csk(sk)->icsk_ext_hdr_len = inet_opt->opt.optlen;
211
212 tp->rx_opt.mss_clamp = TCP_MSS_DEFAULT;
213
214 /* Socket identity is still unknown (sport may be zero).
215 * However we set state to SYN-SENT and not releasing socket
216 * lock select source port, enter ourselves into the hash tables and
217 * complete initialization after this.
218 */
219 tcp_set_state(sk, TCP_SYN_SENT);
220 err = inet_hash_connect(&tcp_death_row, sk);
221 if (err)
222 goto failure;
223
224 sk_set_txhash(sk);
225
226 rt = ip_route_newports(fl4, rt, orig_sport, orig_dport,
227 inet->inet_sport, inet->inet_dport, sk);
228 if (IS_ERR(rt)) {
229 err = PTR_ERR(rt);
230 rt = NULL;
231 goto failure;
232 }
233 /* OK, now commit destination to socket. */
234 sk->sk_gso_type = SKB_GSO_TCPV4;
235 sk_setup_caps(sk, &rt->dst);
236
237 if (!tp->write_seq && likely(!tp->repair))
238 tp->write_seq = secure_tcp_sequence_number(inet->inet_saddr,
239 inet->inet_daddr,
240 inet->inet_sport,
241 usin->sin_port);
242
243 inet->inet_id = tp->write_seq ^ jiffies;
244
245 err = tcp_connect(sk);
246
247 rt = NULL;
248 if (err)
249 goto failure;
250
251 return 0;
252
253 failure:
254 /*
255 * This unhashes the socket and releases the local port,
256 * if necessary.
257 */
258 tcp_set_state(sk, TCP_CLOSE);
259 ip_rt_put(rt);
260 sk->sk_route_caps = 0;
261 inet->inet_dport = 0;
262 return err;
263 }
264 EXPORT_SYMBOL(tcp_v4_connect);
265
266 /*
267 * This routine reacts to ICMP_FRAG_NEEDED mtu indications as defined in RFC1191.
268 * It can be called through tcp_release_cb() if socket was owned by user
269 * at the time tcp_v4_err() was called to handle ICMP message.
270 */
271 void tcp_v4_mtu_reduced(struct sock *sk)
272 {
273 struct dst_entry *dst;
274 struct inet_sock *inet = inet_sk(sk);
275 u32 mtu = tcp_sk(sk)->mtu_info;
276
277 dst = inet_csk_update_pmtu(sk, mtu);
278 if (!dst)
279 return;
280
281 /* Something is about to be wrong... Remember soft error
282 * for the case, if this connection will not able to recover.
283 */
284 if (mtu < dst_mtu(dst) && ip_dont_fragment(sk, dst))
285 sk->sk_err_soft = EMSGSIZE;
286
287 mtu = dst_mtu(dst);
288
289 if (inet->pmtudisc != IP_PMTUDISC_DONT &&
290 ip_sk_accept_pmtu(sk) &&
291 inet_csk(sk)->icsk_pmtu_cookie > mtu) {
292 tcp_sync_mss(sk, mtu);
293
294 /* Resend the TCP packet because it's
295 * clear that the old packet has been
296 * dropped. This is the new "fast" path mtu
297 * discovery.
298 */
299 tcp_simple_retransmit(sk);
300 } /* else let the usual retransmit timer handle it */
301 }
302 EXPORT_SYMBOL(tcp_v4_mtu_reduced);
303
304 static void do_redirect(struct sk_buff *skb, struct sock *sk)
305 {
306 struct dst_entry *dst = __sk_dst_check(sk, 0);
307
308 if (dst)
309 dst->ops->redirect(dst, sk, skb);
310 }
311
312
313 /* handle ICMP messages on TCP_NEW_SYN_RECV request sockets */
314 void tcp_req_err(struct sock *sk, u32 seq, bool abort)
315 {
316 struct request_sock *req = inet_reqsk(sk);
317 struct net *net = sock_net(sk);
318
319 /* ICMPs are not backlogged, hence we cannot get
320 * an established socket here.
321 */
322 if (seq != tcp_rsk(req)->snt_isn) {
323 __NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS);
324 } else if (abort) {
325 /*
326 * Still in SYN_RECV, just remove it silently.
327 * There is no good way to pass the error to the newly
328 * created socket, and POSIX does not want network
329 * errors returned from accept().
330 */
331 inet_csk_reqsk_queue_drop(req->rsk_listener, req);
332 tcp_listendrop(req->rsk_listener);
333 }
334 reqsk_put(req);
335 }
336 EXPORT_SYMBOL(tcp_req_err);
337
338 /*
339 * This routine is called by the ICMP module when it gets some
340 * sort of error condition. If err < 0 then the socket should
341 * be closed and the error returned to the user. If err > 0
342 * it's just the icmp type << 8 | icmp code. After adjustment
343 * header points to the first 8 bytes of the tcp header. We need
344 * to find the appropriate port.
345 *
346 * The locking strategy used here is very "optimistic". When
347 * someone else accesses the socket the ICMP is just dropped
348 * and for some paths there is no check at all.
349 * A more general error queue to queue errors for later handling
350 * is probably better.
351 *
352 */
353
354 void tcp_v4_err(struct sk_buff *icmp_skb, u32 info)
355 {
356 const struct iphdr *iph = (const struct iphdr *)icmp_skb->data;
357 struct tcphdr *th = (struct tcphdr *)(icmp_skb->data + (iph->ihl << 2));
358 struct inet_connection_sock *icsk;
359 struct tcp_sock *tp;
360 struct inet_sock *inet;
361 const int type = icmp_hdr(icmp_skb)->type;
362 const int code = icmp_hdr(icmp_skb)->code;
363 struct sock *sk;
364 struct sk_buff *skb;
365 struct request_sock *fastopen;
366 __u32 seq, snd_una;
367 __u32 remaining;
368 int err;
369 struct net *net = dev_net(icmp_skb->dev);
370
371 sk = __inet_lookup_established(net, &tcp_hashinfo, iph->daddr,
372 th->dest, iph->saddr, ntohs(th->source),
373 inet_iif(icmp_skb));
374 if (!sk) {
375 __ICMP_INC_STATS(net, ICMP_MIB_INERRORS);
376 return;
377 }
378 if (sk->sk_state == TCP_TIME_WAIT) {
379 inet_twsk_put(inet_twsk(sk));
380 return;
381 }
382 seq = ntohl(th->seq);
383 if (sk->sk_state == TCP_NEW_SYN_RECV)
384 return tcp_req_err(sk, seq,
385 type == ICMP_PARAMETERPROB ||
386 type == ICMP_TIME_EXCEEDED ||
387 (type == ICMP_DEST_UNREACH &&
388 (code == ICMP_NET_UNREACH ||
389 code == ICMP_HOST_UNREACH)));
390
391 bh_lock_sock(sk);
392 /* If too many ICMPs get dropped on busy
393 * servers this needs to be solved differently.
394 * We do take care of PMTU discovery (RFC1191) special case :
395 * we can receive locally generated ICMP messages while socket is held.
396 */
397 if (sock_owned_by_user(sk)) {
398 if (!(type == ICMP_DEST_UNREACH && code == ICMP_FRAG_NEEDED))
399 __NET_INC_STATS(net, LINUX_MIB_LOCKDROPPEDICMPS);
400 }
401 if (sk->sk_state == TCP_CLOSE)
402 goto out;
403
404 if (unlikely(iph->ttl < inet_sk(sk)->min_ttl)) {
405 __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
406 goto out;
407 }
408
409 icsk = inet_csk(sk);
410 tp = tcp_sk(sk);
411 /* XXX (TFO) - tp->snd_una should be ISN (tcp_create_openreq_child() */
412 fastopen = tp->fastopen_rsk;
413 snd_una = fastopen ? tcp_rsk(fastopen)->snt_isn : tp->snd_una;
414 if (sk->sk_state != TCP_LISTEN &&
415 !between(seq, snd_una, tp->snd_nxt)) {
416 __NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS);
417 goto out;
418 }
419
420 switch (type) {
421 case ICMP_REDIRECT:
422 do_redirect(icmp_skb, sk);
423 goto out;
424 case ICMP_SOURCE_QUENCH:
425 /* Just silently ignore these. */
426 goto out;
427 case ICMP_PARAMETERPROB:
428 err = EPROTO;
429 break;
430 case ICMP_DEST_UNREACH:
431 if (code > NR_ICMP_UNREACH)
432 goto out;
433
434 if (code == ICMP_FRAG_NEEDED) { /* PMTU discovery (RFC1191) */
435 /* We are not interested in TCP_LISTEN and open_requests
436 * (SYN-ACKs send out by Linux are always <576bytes so
437 * they should go through unfragmented).
438 */
439 if (sk->sk_state == TCP_LISTEN)
440 goto out;
441
442 tp->mtu_info = info;
443 if (!sock_owned_by_user(sk)) {
444 tcp_v4_mtu_reduced(sk);
445 } else {
446 if (!test_and_set_bit(TCP_MTU_REDUCED_DEFERRED, &tp->tsq_flags))
447 sock_hold(sk);
448 }
449 goto out;
450 }
451
452 err = icmp_err_convert[code].errno;
453 /* check if icmp_skb allows revert of backoff
454 * (see draft-zimmermann-tcp-lcd) */
455 if (code != ICMP_NET_UNREACH && code != ICMP_HOST_UNREACH)
456 break;
457 if (seq != tp->snd_una || !icsk->icsk_retransmits ||
458 !icsk->icsk_backoff || fastopen)
459 break;
460
461 if (sock_owned_by_user(sk))
462 break;
463
464 icsk->icsk_backoff--;
465 icsk->icsk_rto = tp->srtt_us ? __tcp_set_rto(tp) :
466 TCP_TIMEOUT_INIT;
467 icsk->icsk_rto = inet_csk_rto_backoff(icsk, TCP_RTO_MAX);
468
469 skb = tcp_write_queue_head(sk);
470 BUG_ON(!skb);
471
472 remaining = icsk->icsk_rto -
473 min(icsk->icsk_rto,
474 tcp_time_stamp - tcp_skb_timestamp(skb));
475
476 if (remaining) {
477 inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
478 remaining, TCP_RTO_MAX);
479 } else {
480 /* RTO revert clocked out retransmission.
481 * Will retransmit now */
482 tcp_retransmit_timer(sk);
483 }
484
485 break;
486 case ICMP_TIME_EXCEEDED:
487 err = EHOSTUNREACH;
488 break;
489 default:
490 goto out;
491 }
492
493 switch (sk->sk_state) {
494 case TCP_SYN_SENT:
495 case TCP_SYN_RECV:
496 /* Only in fast or simultaneous open. If a fast open socket is
497 * is already accepted it is treated as a connected one below.
498 */
499 if (fastopen && !fastopen->sk)
500 break;
501
502 if (!sock_owned_by_user(sk)) {
503 sk->sk_err = err;
504
505 sk->sk_error_report(sk);
506
507 tcp_done(sk);
508 } else {
509 sk->sk_err_soft = err;
510 }
511 goto out;
512 }
513
514 /* If we've already connected we will keep trying
515 * until we time out, or the user gives up.
516 *
517 * rfc1122 4.2.3.9 allows to consider as hard errors
518 * only PROTO_UNREACH and PORT_UNREACH (well, FRAG_FAILED too,
519 * but it is obsoleted by pmtu discovery).
520 *
521 * Note, that in modern internet, where routing is unreliable
522 * and in each dark corner broken firewalls sit, sending random
523 * errors ordered by their masters even this two messages finally lose
524 * their original sense (even Linux sends invalid PORT_UNREACHs)
525 *
526 * Now we are in compliance with RFCs.
527 * --ANK (980905)
528 */
529
530 inet = inet_sk(sk);
531 if (!sock_owned_by_user(sk) && inet->recverr) {
532 sk->sk_err = err;
533 sk->sk_error_report(sk);
534 } else { /* Only an error on timeout */
535 sk->sk_err_soft = err;
536 }
537
538 out:
539 bh_unlock_sock(sk);
540 sock_put(sk);
541 }
542
543 void __tcp_v4_send_check(struct sk_buff *skb, __be32 saddr, __be32 daddr)
544 {
545 struct tcphdr *th = tcp_hdr(skb);
546
547 if (skb->ip_summed == CHECKSUM_PARTIAL) {
548 th->check = ~tcp_v4_check(skb->len, saddr, daddr, 0);
549 skb->csum_start = skb_transport_header(skb) - skb->head;
550 skb->csum_offset = offsetof(struct tcphdr, check);
551 } else {
552 th->check = tcp_v4_check(skb->len, saddr, daddr,
553 csum_partial(th,
554 th->doff << 2,
555 skb->csum));
556 }
557 }
558
559 /* This routine computes an IPv4 TCP checksum. */
560 void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb)
561 {
562 const struct inet_sock *inet = inet_sk(sk);
563
564 __tcp_v4_send_check(skb, inet->inet_saddr, inet->inet_daddr);
565 }
566 EXPORT_SYMBOL(tcp_v4_send_check);
567
568 /*
569 * This routine will send an RST to the other tcp.
570 *
571 * Someone asks: why I NEVER use socket parameters (TOS, TTL etc.)
572 * for reset.
573 * Answer: if a packet caused RST, it is not for a socket
574 * existing in our system, if it is matched to a socket,
575 * it is just duplicate segment or bug in other side's TCP.
576 * So that we build reply only basing on parameters
577 * arrived with segment.
578 * Exception: precedence violation. We do not implement it in any case.
579 */
580
581 static void tcp_v4_send_reset(const struct sock *sk, struct sk_buff *skb)
582 {
583 const struct tcphdr *th = tcp_hdr(skb);
584 struct {
585 struct tcphdr th;
586 #ifdef CONFIG_TCP_MD5SIG
587 __be32 opt[(TCPOLEN_MD5SIG_ALIGNED >> 2)];
588 #endif
589 } rep;
590 struct ip_reply_arg arg;
591 #ifdef CONFIG_TCP_MD5SIG
592 struct tcp_md5sig_key *key = NULL;
593 const __u8 *hash_location = NULL;
594 unsigned char newhash[16];
595 int genhash;
596 struct sock *sk1 = NULL;
597 #endif
598 struct net *net;
599
600 /* Never send a reset in response to a reset. */
601 if (th->rst)
602 return;
603
604 /* If sk not NULL, it means we did a successful lookup and incoming
605 * route had to be correct. prequeue might have dropped our dst.
606 */
607 if (!sk && skb_rtable(skb)->rt_type != RTN_LOCAL)
608 return;
609
610 /* Swap the send and the receive. */
611 memset(&rep, 0, sizeof(rep));
612 rep.th.dest = th->source;
613 rep.th.source = th->dest;
614 rep.th.doff = sizeof(struct tcphdr) / 4;
615 rep.th.rst = 1;
616
617 if (th->ack) {
618 rep.th.seq = th->ack_seq;
619 } else {
620 rep.th.ack = 1;
621 rep.th.ack_seq = htonl(ntohl(th->seq) + th->syn + th->fin +
622 skb->len - (th->doff << 2));
623 }
624
625 memset(&arg, 0, sizeof(arg));
626 arg.iov[0].iov_base = (unsigned char *)&rep;
627 arg.iov[0].iov_len = sizeof(rep.th);
628
629 net = sk ? sock_net(sk) : dev_net(skb_dst(skb)->dev);
630 #ifdef CONFIG_TCP_MD5SIG
631 rcu_read_lock();
632 hash_location = tcp_parse_md5sig_option(th);
633 if (sk && sk_fullsock(sk)) {
634 key = tcp_md5_do_lookup(sk, (union tcp_md5_addr *)
635 &ip_hdr(skb)->saddr, AF_INET);
636 } else if (hash_location) {
637 /*
638 * active side is lost. Try to find listening socket through
639 * source port, and then find md5 key through listening socket.
640 * we are not loose security here:
641 * Incoming packet is checked with md5 hash with finding key,
642 * no RST generated if md5 hash doesn't match.
643 */
644 sk1 = __inet_lookup_listener(net, &tcp_hashinfo, NULL, 0,
645 ip_hdr(skb)->saddr,
646 th->source, ip_hdr(skb)->daddr,
647 ntohs(th->source), inet_iif(skb));
648 /* don't send rst if it can't find key */
649 if (!sk1)
650 goto out;
651
652 key = tcp_md5_do_lookup(sk1, (union tcp_md5_addr *)
653 &ip_hdr(skb)->saddr, AF_INET);
654 if (!key)
655 goto out;
656
657
658 genhash = tcp_v4_md5_hash_skb(newhash, key, NULL, skb);
659 if (genhash || memcmp(hash_location, newhash, 16) != 0)
660 goto out;
661
662 }
663
664 if (key) {
665 rep.opt[0] = htonl((TCPOPT_NOP << 24) |
666 (TCPOPT_NOP << 16) |
667 (TCPOPT_MD5SIG << 8) |
668 TCPOLEN_MD5SIG);
669 /* Update length and the length the header thinks exists */
670 arg.iov[0].iov_len += TCPOLEN_MD5SIG_ALIGNED;
671 rep.th.doff = arg.iov[0].iov_len / 4;
672
673 tcp_v4_md5_hash_hdr((__u8 *) &rep.opt[1],
674 key, ip_hdr(skb)->saddr,
675 ip_hdr(skb)->daddr, &rep.th);
676 }
677 #endif
678 arg.csum = csum_tcpudp_nofold(ip_hdr(skb)->daddr,
679 ip_hdr(skb)->saddr, /* XXX */
680 arg.iov[0].iov_len, IPPROTO_TCP, 0);
681 arg.csumoffset = offsetof(struct tcphdr, check) / 2;
682 arg.flags = (sk && inet_sk_transparent(sk)) ? IP_REPLY_ARG_NOSRCCHECK : 0;
683
684 /* When socket is gone, all binding information is lost.
685 * routing might fail in this case. No choice here, if we choose to force
686 * input interface, we will misroute in case of asymmetric route.
687 */
688 if (sk)
689 arg.bound_dev_if = sk->sk_bound_dev_if;
690
691 BUILD_BUG_ON(offsetof(struct sock, sk_bound_dev_if) !=
692 offsetof(struct inet_timewait_sock, tw_bound_dev_if));
693
694 arg.tos = ip_hdr(skb)->tos;
695 local_bh_disable();
696 ip_send_unicast_reply(*this_cpu_ptr(net->ipv4.tcp_sk),
697 skb, &TCP_SKB_CB(skb)->header.h4.opt,
698 ip_hdr(skb)->saddr, ip_hdr(skb)->daddr,
699 &arg, arg.iov[0].iov_len);
700
701 __TCP_INC_STATS(net, TCP_MIB_OUTSEGS);
702 __TCP_INC_STATS(net, TCP_MIB_OUTRSTS);
703 local_bh_enable();
704
705 #ifdef CONFIG_TCP_MD5SIG
706 out:
707 rcu_read_unlock();
708 #endif
709 }
710
711 /* The code following below sending ACKs in SYN-RECV and TIME-WAIT states
712 outside socket context is ugly, certainly. What can I do?
713 */
714
715 static void tcp_v4_send_ack(struct net *net,
716 struct sk_buff *skb, u32 seq, u32 ack,
717 u32 win, u32 tsval, u32 tsecr, int oif,
718 struct tcp_md5sig_key *key,
719 int reply_flags, u8 tos)
720 {
721 const struct tcphdr *th = tcp_hdr(skb);
722 struct {
723 struct tcphdr th;
724 __be32 opt[(TCPOLEN_TSTAMP_ALIGNED >> 2)
725 #ifdef CONFIG_TCP_MD5SIG
726 + (TCPOLEN_MD5SIG_ALIGNED >> 2)
727 #endif
728 ];
729 } rep;
730 struct ip_reply_arg arg;
731
732 memset(&rep.th, 0, sizeof(struct tcphdr));
733 memset(&arg, 0, sizeof(arg));
734
735 arg.iov[0].iov_base = (unsigned char *)&rep;
736 arg.iov[0].iov_len = sizeof(rep.th);
737 if (tsecr) {
738 rep.opt[0] = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
739 (TCPOPT_TIMESTAMP << 8) |
740 TCPOLEN_TIMESTAMP);
741 rep.opt[1] = htonl(tsval);
742 rep.opt[2] = htonl(tsecr);
743 arg.iov[0].iov_len += TCPOLEN_TSTAMP_ALIGNED;
744 }
745
746 /* Swap the send and the receive. */
747 rep.th.dest = th->source;
748 rep.th.source = th->dest;
749 rep.th.doff = arg.iov[0].iov_len / 4;
750 rep.th.seq = htonl(seq);
751 rep.th.ack_seq = htonl(ack);
752 rep.th.ack = 1;
753 rep.th.window = htons(win);
754
755 #ifdef CONFIG_TCP_MD5SIG
756 if (key) {
757 int offset = (tsecr) ? 3 : 0;
758
759 rep.opt[offset++] = htonl((TCPOPT_NOP << 24) |
760 (TCPOPT_NOP << 16) |
761 (TCPOPT_MD5SIG << 8) |
762 TCPOLEN_MD5SIG);
763 arg.iov[0].iov_len += TCPOLEN_MD5SIG_ALIGNED;
764 rep.th.doff = arg.iov[0].iov_len/4;
765
766 tcp_v4_md5_hash_hdr((__u8 *) &rep.opt[offset],
767 key, ip_hdr(skb)->saddr,
768 ip_hdr(skb)->daddr, &rep.th);
769 }
770 #endif
771 arg.flags = reply_flags;
772 arg.csum = csum_tcpudp_nofold(ip_hdr(skb)->daddr,
773 ip_hdr(skb)->saddr, /* XXX */
774 arg.iov[0].iov_len, IPPROTO_TCP, 0);
775 arg.csumoffset = offsetof(struct tcphdr, check) / 2;
776 if (oif)
777 arg.bound_dev_if = oif;
778 arg.tos = tos;
779 local_bh_disable();
780 ip_send_unicast_reply(*this_cpu_ptr(net->ipv4.tcp_sk),
781 skb, &TCP_SKB_CB(skb)->header.h4.opt,
782 ip_hdr(skb)->saddr, ip_hdr(skb)->daddr,
783 &arg, arg.iov[0].iov_len);
784
785 __TCP_INC_STATS(net, TCP_MIB_OUTSEGS);
786 local_bh_enable();
787 }
788
789 static void tcp_v4_timewait_ack(struct sock *sk, struct sk_buff *skb)
790 {
791 struct inet_timewait_sock *tw = inet_twsk(sk);
792 struct tcp_timewait_sock *tcptw = tcp_twsk(sk);
793
794 tcp_v4_send_ack(sock_net(sk), skb,
795 tcptw->tw_snd_nxt, tcptw->tw_rcv_nxt,
796 tcptw->tw_rcv_wnd >> tw->tw_rcv_wscale,
797 tcp_time_stamp + tcptw->tw_ts_offset,
798 tcptw->tw_ts_recent,
799 tw->tw_bound_dev_if,
800 tcp_twsk_md5_key(tcptw),
801 tw->tw_transparent ? IP_REPLY_ARG_NOSRCCHECK : 0,
802 tw->tw_tos
803 );
804
805 inet_twsk_put(tw);
806 }
807
808 static void tcp_v4_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb,
809 struct request_sock *req)
810 {
811 /* sk->sk_state == TCP_LISTEN -> for regular TCP_SYN_RECV
812 * sk->sk_state == TCP_SYN_RECV -> for Fast Open.
813 */
814 u32 seq = (sk->sk_state == TCP_LISTEN) ? tcp_rsk(req)->snt_isn + 1 :
815 tcp_sk(sk)->snd_nxt;
816
817 tcp_v4_send_ack(sock_net(sk), skb, seq,
818 tcp_rsk(req)->rcv_nxt, req->rsk_rcv_wnd,
819 tcp_time_stamp,
820 req->ts_recent,
821 0,
822 tcp_md5_do_lookup(sk, (union tcp_md5_addr *)&ip_hdr(skb)->daddr,
823 AF_INET),
824 inet_rsk(req)->no_srccheck ? IP_REPLY_ARG_NOSRCCHECK : 0,
825 ip_hdr(skb)->tos);
826 }
827
828 /*
829 * Send a SYN-ACK after having received a SYN.
830 * This still operates on a request_sock only, not on a big
831 * socket.
832 */
833 static int tcp_v4_send_synack(const struct sock *sk, struct dst_entry *dst,
834 struct flowi *fl,
835 struct request_sock *req,
836 struct tcp_fastopen_cookie *foc,
837 enum tcp_synack_type synack_type)
838 {
839 const struct inet_request_sock *ireq = inet_rsk(req);
840 struct flowi4 fl4;
841 int err = -1;
842 struct sk_buff *skb;
843
844 /* First, grab a route. */
845 if (!dst && (dst = inet_csk_route_req(sk, &fl4, req)) == NULL)
846 return -1;
847
848 skb = tcp_make_synack(sk, dst, req, foc, synack_type);
849
850 if (skb) {
851 __tcp_v4_send_check(skb, ireq->ir_loc_addr, ireq->ir_rmt_addr);
852
853 err = ip_build_and_send_pkt(skb, sk, ireq->ir_loc_addr,
854 ireq->ir_rmt_addr,
855 ireq->opt);
856 err = net_xmit_eval(err);
857 }
858
859 return err;
860 }
861
862 /*
863 * IPv4 request_sock destructor.
864 */
865 static void tcp_v4_reqsk_destructor(struct request_sock *req)
866 {
867 kfree(inet_rsk(req)->opt);
868 }
869
870 #ifdef CONFIG_TCP_MD5SIG
871 /*
872 * RFC2385 MD5 checksumming requires a mapping of
873 * IP address->MD5 Key.
874 * We need to maintain these in the sk structure.
875 */
876
877 /* Find the Key structure for an address. */
878 struct tcp_md5sig_key *tcp_md5_do_lookup(const struct sock *sk,
879 const union tcp_md5_addr *addr,
880 int family)
881 {
882 const struct tcp_sock *tp = tcp_sk(sk);
883 struct tcp_md5sig_key *key;
884 unsigned int size = sizeof(struct in_addr);
885 const struct tcp_md5sig_info *md5sig;
886
887 /* caller either holds rcu_read_lock() or socket lock */
888 md5sig = rcu_dereference_check(tp->md5sig_info,
889 lockdep_sock_is_held(sk));
890 if (!md5sig)
891 return NULL;
892 #if IS_ENABLED(CONFIG_IPV6)
893 if (family == AF_INET6)
894 size = sizeof(struct in6_addr);
895 #endif
896 hlist_for_each_entry_rcu(key, &md5sig->head, node) {
897 if (key->family != family)
898 continue;
899 if (!memcmp(&key->addr, addr, size))
900 return key;
901 }
902 return NULL;
903 }
904 EXPORT_SYMBOL(tcp_md5_do_lookup);
905
906 struct tcp_md5sig_key *tcp_v4_md5_lookup(const struct sock *sk,
907 const struct sock *addr_sk)
908 {
909 const union tcp_md5_addr *addr;
910
911 addr = (const union tcp_md5_addr *)&addr_sk->sk_daddr;
912 return tcp_md5_do_lookup(sk, addr, AF_INET);
913 }
914 EXPORT_SYMBOL(tcp_v4_md5_lookup);
915
916 /* This can be called on a newly created socket, from other files */
917 int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr,
918 int family, const u8 *newkey, u8 newkeylen, gfp_t gfp)
919 {
920 /* Add Key to the list */
921 struct tcp_md5sig_key *key;
922 struct tcp_sock *tp = tcp_sk(sk);
923 struct tcp_md5sig_info *md5sig;
924
925 key = tcp_md5_do_lookup(sk, addr, family);
926 if (key) {
927 /* Pre-existing entry - just update that one. */
928 memcpy(key->key, newkey, newkeylen);
929 key->keylen = newkeylen;
930 return 0;
931 }
932
933 md5sig = rcu_dereference_protected(tp->md5sig_info,
934 lockdep_sock_is_held(sk));
935 if (!md5sig) {
936 md5sig = kmalloc(sizeof(*md5sig), gfp);
937 if (!md5sig)
938 return -ENOMEM;
939
940 sk_nocaps_add(sk, NETIF_F_GSO_MASK);
941 INIT_HLIST_HEAD(&md5sig->head);
942 rcu_assign_pointer(tp->md5sig_info, md5sig);
943 }
944
945 key = sock_kmalloc(sk, sizeof(*key), gfp);
946 if (!key)
947 return -ENOMEM;
948 if (!tcp_alloc_md5sig_pool()) {
949 sock_kfree_s(sk, key, sizeof(*key));
950 return -ENOMEM;
951 }
952
953 memcpy(key->key, newkey, newkeylen);
954 key->keylen = newkeylen;
955 key->family = family;
956 memcpy(&key->addr, addr,
957 (family == AF_INET6) ? sizeof(struct in6_addr) :
958 sizeof(struct in_addr));
959 hlist_add_head_rcu(&key->node, &md5sig->head);
960 return 0;
961 }
962 EXPORT_SYMBOL(tcp_md5_do_add);
963
964 int tcp_md5_do_del(struct sock *sk, const union tcp_md5_addr *addr, int family)
965 {
966 struct tcp_md5sig_key *key;
967
968 key = tcp_md5_do_lookup(sk, addr, family);
969 if (!key)
970 return -ENOENT;
971 hlist_del_rcu(&key->node);
972 atomic_sub(sizeof(*key), &sk->sk_omem_alloc);
973 kfree_rcu(key, rcu);
974 return 0;
975 }
976 EXPORT_SYMBOL(tcp_md5_do_del);
977
978 static void tcp_clear_md5_list(struct sock *sk)
979 {
980 struct tcp_sock *tp = tcp_sk(sk);
981 struct tcp_md5sig_key *key;
982 struct hlist_node *n;
983 struct tcp_md5sig_info *md5sig;
984
985 md5sig = rcu_dereference_protected(tp->md5sig_info, 1);
986
987 hlist_for_each_entry_safe(key, n, &md5sig->head, node) {
988 hlist_del_rcu(&key->node);
989 atomic_sub(sizeof(*key), &sk->sk_omem_alloc);
990 kfree_rcu(key, rcu);
991 }
992 }
993
994 static int tcp_v4_parse_md5_keys(struct sock *sk, char __user *optval,
995 int optlen)
996 {
997 struct tcp_md5sig cmd;
998 struct sockaddr_in *sin = (struct sockaddr_in *)&cmd.tcpm_addr;
999
1000 if (optlen < sizeof(cmd))
1001 return -EINVAL;
1002
1003 if (copy_from_user(&cmd, optval, sizeof(cmd)))
1004 return -EFAULT;
1005
1006 if (sin->sin_family != AF_INET)
1007 return -EINVAL;
1008
1009 if (!cmd.tcpm_keylen)
1010 return tcp_md5_do_del(sk, (union tcp_md5_addr *)&sin->sin_addr.s_addr,
1011 AF_INET);
1012
1013 if (cmd.tcpm_keylen > TCP_MD5SIG_MAXKEYLEN)
1014 return -EINVAL;
1015
1016 return tcp_md5_do_add(sk, (union tcp_md5_addr *)&sin->sin_addr.s_addr,
1017 AF_INET, cmd.tcpm_key, cmd.tcpm_keylen,
1018 GFP_KERNEL);
1019 }
1020
1021 static int tcp_v4_md5_hash_headers(struct tcp_md5sig_pool *hp,
1022 __be32 daddr, __be32 saddr,
1023 const struct tcphdr *th, int nbytes)
1024 {
1025 struct tcp4_pseudohdr *bp;
1026 struct scatterlist sg;
1027 struct tcphdr *_th;
1028
1029 bp = hp->scratch;
1030 bp->saddr = saddr;
1031 bp->daddr = daddr;
1032 bp->pad = 0;
1033 bp->protocol = IPPROTO_TCP;
1034 bp->len = cpu_to_be16(nbytes);
1035
1036 _th = (struct tcphdr *)(bp + 1);
1037 memcpy(_th, th, sizeof(*th));
1038 _th->check = 0;
1039
1040 sg_init_one(&sg, bp, sizeof(*bp) + sizeof(*th));
1041 ahash_request_set_crypt(hp->md5_req, &sg, NULL,
1042 sizeof(*bp) + sizeof(*th));
1043 return crypto_ahash_update(hp->md5_req);
1044 }
1045
1046 static int tcp_v4_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key,
1047 __be32 daddr, __be32 saddr, const struct tcphdr *th)
1048 {
1049 struct tcp_md5sig_pool *hp;
1050 struct ahash_request *req;
1051
1052 hp = tcp_get_md5sig_pool();
1053 if (!hp)
1054 goto clear_hash_noput;
1055 req = hp->md5_req;
1056
1057 if (crypto_ahash_init(req))
1058 goto clear_hash;
1059 if (tcp_v4_md5_hash_headers(hp, daddr, saddr, th, th->doff << 2))
1060 goto clear_hash;
1061 if (tcp_md5_hash_key(hp, key))
1062 goto clear_hash;
1063 ahash_request_set_crypt(req, NULL, md5_hash, 0);
1064 if (crypto_ahash_final(req))
1065 goto clear_hash;
1066
1067 tcp_put_md5sig_pool();
1068 return 0;
1069
1070 clear_hash:
1071 tcp_put_md5sig_pool();
1072 clear_hash_noput:
1073 memset(md5_hash, 0, 16);
1074 return 1;
1075 }
1076
1077 int tcp_v4_md5_hash_skb(char *md5_hash, const struct tcp_md5sig_key *key,
1078 const struct sock *sk,
1079 const struct sk_buff *skb)
1080 {
1081 struct tcp_md5sig_pool *hp;
1082 struct ahash_request *req;
1083 const struct tcphdr *th = tcp_hdr(skb);
1084 __be32 saddr, daddr;
1085
1086 if (sk) { /* valid for establish/request sockets */
1087 saddr = sk->sk_rcv_saddr;
1088 daddr = sk->sk_daddr;
1089 } else {
1090 const struct iphdr *iph = ip_hdr(skb);
1091 saddr = iph->saddr;
1092 daddr = iph->daddr;
1093 }
1094
1095 hp = tcp_get_md5sig_pool();
1096 if (!hp)
1097 goto clear_hash_noput;
1098 req = hp->md5_req;
1099
1100 if (crypto_ahash_init(req))
1101 goto clear_hash;
1102
1103 if (tcp_v4_md5_hash_headers(hp, daddr, saddr, th, skb->len))
1104 goto clear_hash;
1105 if (tcp_md5_hash_skb_data(hp, skb, th->doff << 2))
1106 goto clear_hash;
1107 if (tcp_md5_hash_key(hp, key))
1108 goto clear_hash;
1109 ahash_request_set_crypt(req, NULL, md5_hash, 0);
1110 if (crypto_ahash_final(req))
1111 goto clear_hash;
1112
1113 tcp_put_md5sig_pool();
1114 return 0;
1115
1116 clear_hash:
1117 tcp_put_md5sig_pool();
1118 clear_hash_noput:
1119 memset(md5_hash, 0, 16);
1120 return 1;
1121 }
1122 EXPORT_SYMBOL(tcp_v4_md5_hash_skb);
1123
1124 #endif
1125
1126 /* Called with rcu_read_lock() */
1127 static bool tcp_v4_inbound_md5_hash(const struct sock *sk,
1128 const struct sk_buff *skb)
1129 {
1130 #ifdef CONFIG_TCP_MD5SIG
1131 /*
1132 * This gets called for each TCP segment that arrives
1133 * so we want to be efficient.
1134 * We have 3 drop cases:
1135 * o No MD5 hash and one expected.
1136 * o MD5 hash and we're not expecting one.
1137 * o MD5 hash and its wrong.
1138 */
1139 const __u8 *hash_location = NULL;
1140 struct tcp_md5sig_key *hash_expected;
1141 const struct iphdr *iph = ip_hdr(skb);
1142 const struct tcphdr *th = tcp_hdr(skb);
1143 int genhash;
1144 unsigned char newhash[16];
1145
1146 hash_expected = tcp_md5_do_lookup(sk, (union tcp_md5_addr *)&iph->saddr,
1147 AF_INET);
1148 hash_location = tcp_parse_md5sig_option(th);
1149
1150 /* We've parsed the options - do we have a hash? */
1151 if (!hash_expected && !hash_location)
1152 return false;
1153
1154 if (hash_expected && !hash_location) {
1155 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5NOTFOUND);
1156 return true;
1157 }
1158
1159 if (!hash_expected && hash_location) {
1160 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5UNEXPECTED);
1161 return true;
1162 }
1163
1164 /* Okay, so this is hash_expected and hash_location -
1165 * so we need to calculate the checksum.
1166 */
1167 genhash = tcp_v4_md5_hash_skb(newhash,
1168 hash_expected,
1169 NULL, skb);
1170
1171 if (genhash || memcmp(hash_location, newhash, 16) != 0) {
1172 net_info_ratelimited("MD5 Hash failed for (%pI4, %d)->(%pI4, %d)%s\n",
1173 &iph->saddr, ntohs(th->source),
1174 &iph->daddr, ntohs(th->dest),
1175 genhash ? " tcp_v4_calc_md5_hash failed"
1176 : "");
1177 return true;
1178 }
1179 return false;
1180 #endif
1181 return false;
1182 }
1183
1184 static void tcp_v4_init_req(struct request_sock *req,
1185 const struct sock *sk_listener,
1186 struct sk_buff *skb)
1187 {
1188 struct inet_request_sock *ireq = inet_rsk(req);
1189
1190 sk_rcv_saddr_set(req_to_sk(req), ip_hdr(skb)->daddr);
1191 sk_daddr_set(req_to_sk(req), ip_hdr(skb)->saddr);
1192 ireq->no_srccheck = inet_sk(sk_listener)->transparent;
1193 ireq->opt = tcp_v4_save_options(skb);
1194 }
1195
1196 static struct dst_entry *tcp_v4_route_req(const struct sock *sk,
1197 struct flowi *fl,
1198 const struct request_sock *req,
1199 bool *strict)
1200 {
1201 struct dst_entry *dst = inet_csk_route_req(sk, &fl->u.ip4, req);
1202
1203 if (strict) {
1204 if (fl->u.ip4.daddr == inet_rsk(req)->ir_rmt_addr)
1205 *strict = true;
1206 else
1207 *strict = false;
1208 }
1209
1210 return dst;
1211 }
1212
1213 struct request_sock_ops tcp_request_sock_ops __read_mostly = {
1214 .family = PF_INET,
1215 .obj_size = sizeof(struct tcp_request_sock),
1216 .rtx_syn_ack = tcp_rtx_synack,
1217 .send_ack = tcp_v4_reqsk_send_ack,
1218 .destructor = tcp_v4_reqsk_destructor,
1219 .send_reset = tcp_v4_send_reset,
1220 .syn_ack_timeout = tcp_syn_ack_timeout,
1221 };
1222
1223 static const struct tcp_request_sock_ops tcp_request_sock_ipv4_ops = {
1224 .mss_clamp = TCP_MSS_DEFAULT,
1225 #ifdef CONFIG_TCP_MD5SIG
1226 .req_md5_lookup = tcp_v4_md5_lookup,
1227 .calc_md5_hash = tcp_v4_md5_hash_skb,
1228 #endif
1229 .init_req = tcp_v4_init_req,
1230 #ifdef CONFIG_SYN_COOKIES
1231 .cookie_init_seq = cookie_v4_init_sequence,
1232 #endif
1233 .route_req = tcp_v4_route_req,
1234 .init_seq = tcp_v4_init_sequence,
1235 .send_synack = tcp_v4_send_synack,
1236 };
1237
1238 int tcp_v4_conn_request(struct sock *sk, struct sk_buff *skb)
1239 {
1240 /* Never answer to SYNs send to broadcast or multicast */
1241 if (skb_rtable(skb)->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST))
1242 goto drop;
1243
1244 return tcp_conn_request(&tcp_request_sock_ops,
1245 &tcp_request_sock_ipv4_ops, sk, skb);
1246
1247 drop:
1248 tcp_listendrop(sk);
1249 return 0;
1250 }
1251 EXPORT_SYMBOL(tcp_v4_conn_request);
1252
1253
1254 /*
1255 * The three way handshake has completed - we got a valid synack -
1256 * now create the new socket.
1257 */
1258 struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb,
1259 struct request_sock *req,
1260 struct dst_entry *dst,
1261 struct request_sock *req_unhash,
1262 bool *own_req)
1263 {
1264 struct inet_request_sock *ireq;
1265 struct inet_sock *newinet;
1266 struct tcp_sock *newtp;
1267 struct sock *newsk;
1268 #ifdef CONFIG_TCP_MD5SIG
1269 struct tcp_md5sig_key *key;
1270 #endif
1271 struct ip_options_rcu *inet_opt;
1272
1273 if (sk_acceptq_is_full(sk))
1274 goto exit_overflow;
1275
1276 newsk = tcp_create_openreq_child(sk, req, skb);
1277 if (!newsk)
1278 goto exit_nonewsk;
1279
1280 newsk->sk_gso_type = SKB_GSO_TCPV4;
1281 inet_sk_rx_dst_set(newsk, skb);
1282
1283 newtp = tcp_sk(newsk);
1284 newinet = inet_sk(newsk);
1285 ireq = inet_rsk(req);
1286 sk_daddr_set(newsk, ireq->ir_rmt_addr);
1287 sk_rcv_saddr_set(newsk, ireq->ir_loc_addr);
1288 newsk->sk_bound_dev_if = ireq->ir_iif;
1289 newinet->inet_saddr = ireq->ir_loc_addr;
1290 inet_opt = ireq->opt;
1291 rcu_assign_pointer(newinet->inet_opt, inet_opt);
1292 ireq->opt = NULL;
1293 newinet->mc_index = inet_iif(skb);
1294 newinet->mc_ttl = ip_hdr(skb)->ttl;
1295 newinet->rcv_tos = ip_hdr(skb)->tos;
1296 inet_csk(newsk)->icsk_ext_hdr_len = 0;
1297 if (inet_opt)
1298 inet_csk(newsk)->icsk_ext_hdr_len = inet_opt->opt.optlen;
1299 newinet->inet_id = newtp->write_seq ^ jiffies;
1300
1301 if (!dst) {
1302 dst = inet_csk_route_child_sock(sk, newsk, req);
1303 if (!dst)
1304 goto put_and_exit;
1305 } else {
1306 /* syncookie case : see end of cookie_v4_check() */
1307 }
1308 sk_setup_caps(newsk, dst);
1309
1310 tcp_ca_openreq_child(newsk, dst);
1311
1312 tcp_sync_mss(newsk, dst_mtu(dst));
1313 newtp->advmss = dst_metric_advmss(dst);
1314 if (tcp_sk(sk)->rx_opt.user_mss &&
1315 tcp_sk(sk)->rx_opt.user_mss < newtp->advmss)
1316 newtp->advmss = tcp_sk(sk)->rx_opt.user_mss;
1317
1318 tcp_initialize_rcv_mss(newsk);
1319
1320 #ifdef CONFIG_TCP_MD5SIG
1321 /* Copy over the MD5 key from the original socket */
1322 key = tcp_md5_do_lookup(sk, (union tcp_md5_addr *)&newinet->inet_daddr,
1323 AF_INET);
1324 if (key) {
1325 /*
1326 * We're using one, so create a matching key
1327 * on the newsk structure. If we fail to get
1328 * memory, then we end up not copying the key
1329 * across. Shucks.
1330 */
1331 tcp_md5_do_add(newsk, (union tcp_md5_addr *)&newinet->inet_daddr,
1332 AF_INET, key->key, key->keylen, GFP_ATOMIC);
1333 sk_nocaps_add(newsk, NETIF_F_GSO_MASK);
1334 }
1335 #endif
1336
1337 if (__inet_inherit_port(sk, newsk) < 0)
1338 goto put_and_exit;
1339 *own_req = inet_ehash_nolisten(newsk, req_to_sk(req_unhash));
1340 if (*own_req)
1341 tcp_move_syn(newtp, req);
1342
1343 return newsk;
1344
1345 exit_overflow:
1346 NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS);
1347 exit_nonewsk:
1348 dst_release(dst);
1349 exit:
1350 tcp_listendrop(sk);
1351 return NULL;
1352 put_and_exit:
1353 inet_csk_prepare_forced_close(newsk);
1354 tcp_done(newsk);
1355 goto exit;
1356 }
1357 EXPORT_SYMBOL(tcp_v4_syn_recv_sock);
1358
1359 static struct sock *tcp_v4_cookie_check(struct sock *sk, struct sk_buff *skb)
1360 {
1361 #ifdef CONFIG_SYN_COOKIES
1362 const struct tcphdr *th = tcp_hdr(skb);
1363
1364 if (!th->syn)
1365 sk = cookie_v4_check(sk, skb);
1366 #endif
1367 return sk;
1368 }
1369
1370 /* The socket must have it's spinlock held when we get
1371 * here, unless it is a TCP_LISTEN socket.
1372 *
1373 * We have a potential double-lock case here, so even when
1374 * doing backlog processing we use the BH locking scheme.
1375 * This is because we cannot sleep with the original spinlock
1376 * held.
1377 */
1378 int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb)
1379 {
1380 struct sock *rsk;
1381
1382 if (sk->sk_state == TCP_ESTABLISHED) { /* Fast path */
1383 struct dst_entry *dst = sk->sk_rx_dst;
1384
1385 sock_rps_save_rxhash(sk, skb);
1386 sk_mark_napi_id(sk, skb);
1387 if (dst) {
1388 if (inet_sk(sk)->rx_dst_ifindex != skb->skb_iif ||
1389 !dst->ops->check(dst, 0)) {
1390 dst_release(dst);
1391 sk->sk_rx_dst = NULL;
1392 }
1393 }
1394 tcp_rcv_established(sk, skb, tcp_hdr(skb), skb->len);
1395 return 0;
1396 }
1397
1398 if (tcp_checksum_complete(skb))
1399 goto csum_err;
1400
1401 if (sk->sk_state == TCP_LISTEN) {
1402 struct sock *nsk = tcp_v4_cookie_check(sk, skb);
1403
1404 if (!nsk)
1405 goto discard;
1406 if (nsk != sk) {
1407 sock_rps_save_rxhash(nsk, skb);
1408 sk_mark_napi_id(nsk, skb);
1409 if (tcp_child_process(sk, nsk, skb)) {
1410 rsk = nsk;
1411 goto reset;
1412 }
1413 return 0;
1414 }
1415 } else
1416 sock_rps_save_rxhash(sk, skb);
1417
1418 if (tcp_rcv_state_process(sk, skb)) {
1419 rsk = sk;
1420 goto reset;
1421 }
1422 return 0;
1423
1424 reset:
1425 tcp_v4_send_reset(rsk, skb);
1426 discard:
1427 kfree_skb(skb);
1428 /* Be careful here. If this function gets more complicated and
1429 * gcc suffers from register pressure on the x86, sk (in %ebx)
1430 * might be destroyed here. This current version compiles correctly,
1431 * but you have been warned.
1432 */
1433 return 0;
1434
1435 csum_err:
1436 TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
1437 TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
1438 goto discard;
1439 }
1440 EXPORT_SYMBOL(tcp_v4_do_rcv);
1441
1442 void tcp_v4_early_demux(struct sk_buff *skb)
1443 {
1444 const struct iphdr *iph;
1445 const struct tcphdr *th;
1446 struct sock *sk;
1447
1448 if (skb->pkt_type != PACKET_HOST)
1449 return;
1450
1451 if (!pskb_may_pull(skb, skb_transport_offset(skb) + sizeof(struct tcphdr)))
1452 return;
1453
1454 iph = ip_hdr(skb);
1455 th = tcp_hdr(skb);
1456
1457 if (th->doff < sizeof(struct tcphdr) / 4)
1458 return;
1459
1460 sk = __inet_lookup_established(dev_net(skb->dev), &tcp_hashinfo,
1461 iph->saddr, th->source,
1462 iph->daddr, ntohs(th->dest),
1463 skb->skb_iif);
1464 if (sk) {
1465 skb->sk = sk;
1466 skb->destructor = sock_edemux;
1467 if (sk_fullsock(sk)) {
1468 struct dst_entry *dst = READ_ONCE(sk->sk_rx_dst);
1469
1470 if (dst)
1471 dst = dst_check(dst, 0);
1472 if (dst &&
1473 inet_sk(sk)->rx_dst_ifindex == skb->skb_iif)
1474 skb_dst_set_noref(skb, dst);
1475 }
1476 }
1477 }
1478
1479 /* Packet is added to VJ-style prequeue for processing in process
1480 * context, if a reader task is waiting. Apparently, this exciting
1481 * idea (VJ's mail "Re: query about TCP header on tcp-ip" of 07 Sep 93)
1482 * failed somewhere. Latency? Burstiness? Well, at least now we will
1483 * see, why it failed. 8)8) --ANK
1484 *
1485 */
1486 bool tcp_prequeue(struct sock *sk, struct sk_buff *skb)
1487 {
1488 struct tcp_sock *tp = tcp_sk(sk);
1489
1490 if (sysctl_tcp_low_latency || !tp->ucopy.task)
1491 return false;
1492
1493 if (skb->len <= tcp_hdrlen(skb) &&
1494 skb_queue_len(&tp->ucopy.prequeue) == 0)
1495 return false;
1496
1497 /* Before escaping RCU protected region, we need to take care of skb
1498 * dst. Prequeue is only enabled for established sockets.
1499 * For such sockets, we might need the skb dst only to set sk->sk_rx_dst
1500 * Instead of doing full sk_rx_dst validity here, let's perform
1501 * an optimistic check.
1502 */
1503 if (likely(sk->sk_rx_dst))
1504 skb_dst_drop(skb);
1505 else
1506 skb_dst_force_safe(skb);
1507
1508 __skb_queue_tail(&tp->ucopy.prequeue, skb);
1509 tp->ucopy.memory += skb->truesize;
1510 if (skb_queue_len(&tp->ucopy.prequeue) >= 32 ||
1511 tp->ucopy.memory + atomic_read(&sk->sk_rmem_alloc) > sk->sk_rcvbuf) {
1512 struct sk_buff *skb1;
1513
1514 BUG_ON(sock_owned_by_user(sk));
1515 __NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPPREQUEUEDROPPED,
1516 skb_queue_len(&tp->ucopy.prequeue));
1517
1518 while ((skb1 = __skb_dequeue(&tp->ucopy.prequeue)) != NULL)
1519 sk_backlog_rcv(sk, skb1);
1520
1521 tp->ucopy.memory = 0;
1522 } else if (skb_queue_len(&tp->ucopy.prequeue) == 1) {
1523 wake_up_interruptible_sync_poll(sk_sleep(sk),
1524 POLLIN | POLLRDNORM | POLLRDBAND);
1525 if (!inet_csk_ack_scheduled(sk))
1526 inet_csk_reset_xmit_timer(sk, ICSK_TIME_DACK,
1527 (3 * tcp_rto_min(sk)) / 4,
1528 TCP_RTO_MAX);
1529 }
1530 return true;
1531 }
1532 EXPORT_SYMBOL(tcp_prequeue);
1533
1534 /*
1535 * From tcp_input.c
1536 */
1537
1538 int tcp_v4_rcv(struct sk_buff *skb)
1539 {
1540 struct net *net = dev_net(skb->dev);
1541 const struct iphdr *iph;
1542 const struct tcphdr *th;
1543 bool refcounted;
1544 struct sock *sk;
1545 int ret;
1546
1547 if (skb->pkt_type != PACKET_HOST)
1548 goto discard_it;
1549
1550 /* Count it even if it's bad */
1551 __TCP_INC_STATS(net, TCP_MIB_INSEGS);
1552
1553 if (!pskb_may_pull(skb, sizeof(struct tcphdr)))
1554 goto discard_it;
1555
1556 th = (const struct tcphdr *)skb->data;
1557
1558 if (unlikely(th->doff < sizeof(struct tcphdr) / 4))
1559 goto bad_packet;
1560 if (!pskb_may_pull(skb, th->doff * 4))
1561 goto discard_it;
1562
1563 /* An explanation is required here, I think.
1564 * Packet length and doff are validated by header prediction,
1565 * provided case of th->doff==0 is eliminated.
1566 * So, we defer the checks. */
1567
1568 if (skb_checksum_init(skb, IPPROTO_TCP, inet_compute_pseudo))
1569 goto csum_error;
1570
1571 th = (const struct tcphdr *)skb->data;
1572 iph = ip_hdr(skb);
1573 /* This is tricky : We move IPCB at its correct location into TCP_SKB_CB()
1574 * barrier() makes sure compiler wont play fool^Waliasing games.
1575 */
1576 memmove(&TCP_SKB_CB(skb)->header.h4, IPCB(skb),
1577 sizeof(struct inet_skb_parm));
1578 barrier();
1579
1580 TCP_SKB_CB(skb)->seq = ntohl(th->seq);
1581 TCP_SKB_CB(skb)->end_seq = (TCP_SKB_CB(skb)->seq + th->syn + th->fin +
1582 skb->len - th->doff * 4);
1583 TCP_SKB_CB(skb)->ack_seq = ntohl(th->ack_seq);
1584 TCP_SKB_CB(skb)->tcp_flags = tcp_flag_byte(th);
1585 TCP_SKB_CB(skb)->tcp_tw_isn = 0;
1586 TCP_SKB_CB(skb)->ip_dsfield = ipv4_get_dsfield(iph);
1587 TCP_SKB_CB(skb)->sacked = 0;
1588
1589 lookup:
1590 sk = __inet_lookup_skb(&tcp_hashinfo, skb, __tcp_hdrlen(th), th->source,
1591 th->dest, &refcounted);
1592 if (!sk)
1593 goto no_tcp_socket;
1594
1595 process:
1596 if (sk->sk_state == TCP_TIME_WAIT)
1597 goto do_time_wait;
1598
1599 if (sk->sk_state == TCP_NEW_SYN_RECV) {
1600 struct request_sock *req = inet_reqsk(sk);
1601 struct sock *nsk;
1602
1603 sk = req->rsk_listener;
1604 if (unlikely(tcp_v4_inbound_md5_hash(sk, skb))) {
1605 reqsk_put(req);
1606 goto discard_it;
1607 }
1608 if (unlikely(sk->sk_state != TCP_LISTEN)) {
1609 inet_csk_reqsk_queue_drop_and_put(sk, req);
1610 goto lookup;
1611 }
1612 /* We own a reference on the listener, increase it again
1613 * as we might lose it too soon.
1614 */
1615 sock_hold(sk);
1616 refcounted = true;
1617 nsk = tcp_check_req(sk, skb, req, false);
1618 if (!nsk) {
1619 reqsk_put(req);
1620 goto discard_and_relse;
1621 }
1622 if (nsk == sk) {
1623 reqsk_put(req);
1624 } else if (tcp_child_process(sk, nsk, skb)) {
1625 tcp_v4_send_reset(nsk, skb);
1626 goto discard_and_relse;
1627 } else {
1628 sock_put(sk);
1629 return 0;
1630 }
1631 }
1632 if (unlikely(iph->ttl < inet_sk(sk)->min_ttl)) {
1633 __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
1634 goto discard_and_relse;
1635 }
1636
1637 if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb))
1638 goto discard_and_relse;
1639
1640 if (tcp_v4_inbound_md5_hash(sk, skb))
1641 goto discard_and_relse;
1642
1643 nf_reset(skb);
1644
1645 if (sk_filter(sk, skb))
1646 goto discard_and_relse;
1647
1648 skb->dev = NULL;
1649
1650 if (sk->sk_state == TCP_LISTEN) {
1651 ret = tcp_v4_do_rcv(sk, skb);
1652 goto put_and_return;
1653 }
1654
1655 sk_incoming_cpu_update(sk);
1656
1657 bh_lock_sock_nested(sk);
1658 tcp_segs_in(tcp_sk(sk), skb);
1659 ret = 0;
1660 if (!sock_owned_by_user(sk)) {
1661 if (!tcp_prequeue(sk, skb))
1662 ret = tcp_v4_do_rcv(sk, skb);
1663 } else if (unlikely(sk_add_backlog(sk, skb,
1664 sk->sk_rcvbuf + sk->sk_sndbuf))) {
1665 bh_unlock_sock(sk);
1666 __NET_INC_STATS(net, LINUX_MIB_TCPBACKLOGDROP);
1667 goto discard_and_relse;
1668 }
1669 bh_unlock_sock(sk);
1670
1671 put_and_return:
1672 if (refcounted)
1673 sock_put(sk);
1674
1675 return ret;
1676
1677 no_tcp_socket:
1678 if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb))
1679 goto discard_it;
1680
1681 if (tcp_checksum_complete(skb)) {
1682 csum_error:
1683 __TCP_INC_STATS(net, TCP_MIB_CSUMERRORS);
1684 bad_packet:
1685 __TCP_INC_STATS(net, TCP_MIB_INERRS);
1686 } else {
1687 tcp_v4_send_reset(NULL, skb);
1688 }
1689
1690 discard_it:
1691 /* Discard frame. */
1692 kfree_skb(skb);
1693 return 0;
1694
1695 discard_and_relse:
1696 sk_drops_add(sk, skb);
1697 if (refcounted)
1698 sock_put(sk);
1699 goto discard_it;
1700
1701 do_time_wait:
1702 if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb)) {
1703 inet_twsk_put(inet_twsk(sk));
1704 goto discard_it;
1705 }
1706
1707 if (tcp_checksum_complete(skb)) {
1708 inet_twsk_put(inet_twsk(sk));
1709 goto csum_error;
1710 }
1711 switch (tcp_timewait_state_process(inet_twsk(sk), skb, th)) {
1712 case TCP_TW_SYN: {
1713 struct sock *sk2 = inet_lookup_listener(dev_net(skb->dev),
1714 &tcp_hashinfo, skb,
1715 __tcp_hdrlen(th),
1716 iph->saddr, th->source,
1717 iph->daddr, th->dest,
1718 inet_iif(skb));
1719 if (sk2) {
1720 inet_twsk_deschedule_put(inet_twsk(sk));
1721 sk = sk2;
1722 refcounted = false;
1723 goto process;
1724 }
1725 /* Fall through to ACK */
1726 }
1727 case TCP_TW_ACK:
1728 tcp_v4_timewait_ack(sk, skb);
1729 break;
1730 case TCP_TW_RST:
1731 tcp_v4_send_reset(sk, skb);
1732 inet_twsk_deschedule_put(inet_twsk(sk));
1733 goto discard_it;
1734 case TCP_TW_SUCCESS:;
1735 }
1736 goto discard_it;
1737 }
1738
1739 static struct timewait_sock_ops tcp_timewait_sock_ops = {
1740 .twsk_obj_size = sizeof(struct tcp_timewait_sock),
1741 .twsk_unique = tcp_twsk_unique,
1742 .twsk_destructor= tcp_twsk_destructor,
1743 };
1744
1745 void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb)
1746 {
1747 struct dst_entry *dst = skb_dst(skb);
1748
1749 if (dst && dst_hold_safe(dst)) {
1750 sk->sk_rx_dst = dst;
1751 inet_sk(sk)->rx_dst_ifindex = skb->skb_iif;
1752 }
1753 }
1754 EXPORT_SYMBOL(inet_sk_rx_dst_set);
1755
1756 const struct inet_connection_sock_af_ops ipv4_specific = {
1757 .queue_xmit = ip_queue_xmit,
1758 .send_check = tcp_v4_send_check,
1759 .rebuild_header = inet_sk_rebuild_header,
1760 .sk_rx_dst_set = inet_sk_rx_dst_set,
1761 .conn_request = tcp_v4_conn_request,
1762 .syn_recv_sock = tcp_v4_syn_recv_sock,
1763 .net_header_len = sizeof(struct iphdr),
1764 .setsockopt = ip_setsockopt,
1765 .getsockopt = ip_getsockopt,
1766 .addr2sockaddr = inet_csk_addr2sockaddr,
1767 .sockaddr_len = sizeof(struct sockaddr_in),
1768 .bind_conflict = inet_csk_bind_conflict,
1769 #ifdef CONFIG_COMPAT
1770 .compat_setsockopt = compat_ip_setsockopt,
1771 .compat_getsockopt = compat_ip_getsockopt,
1772 #endif
1773 .mtu_reduced = tcp_v4_mtu_reduced,
1774 };
1775 EXPORT_SYMBOL(ipv4_specific);
1776
1777 #ifdef CONFIG_TCP_MD5SIG
1778 static const struct tcp_sock_af_ops tcp_sock_ipv4_specific = {
1779 .md5_lookup = tcp_v4_md5_lookup,
1780 .calc_md5_hash = tcp_v4_md5_hash_skb,
1781 .md5_parse = tcp_v4_parse_md5_keys,
1782 };
1783 #endif
1784
1785 /* NOTE: A lot of things set to zero explicitly by call to
1786 * sk_alloc() so need not be done here.
1787 */
1788 static int tcp_v4_init_sock(struct sock *sk)
1789 {
1790 struct inet_connection_sock *icsk = inet_csk(sk);
1791
1792 tcp_init_sock(sk);
1793
1794 icsk->icsk_af_ops = &ipv4_specific;
1795
1796 #ifdef CONFIG_TCP_MD5SIG
1797 tcp_sk(sk)->af_specific = &tcp_sock_ipv4_specific;
1798 #endif
1799
1800 return 0;
1801 }
1802
1803 void tcp_v4_destroy_sock(struct sock *sk)
1804 {
1805 struct tcp_sock *tp = tcp_sk(sk);
1806
1807 tcp_clear_xmit_timers(sk);
1808
1809 tcp_cleanup_congestion_control(sk);
1810
1811 /* Cleanup up the write buffer. */
1812 tcp_write_queue_purge(sk);
1813
1814 /* Cleans up our, hopefully empty, out_of_order_queue. */
1815 __skb_queue_purge(&tp->out_of_order_queue);
1816
1817 #ifdef CONFIG_TCP_MD5SIG
1818 /* Clean up the MD5 key list, if any */
1819 if (tp->md5sig_info) {
1820 tcp_clear_md5_list(sk);
1821 kfree_rcu(tp->md5sig_info, rcu);
1822 tp->md5sig_info = NULL;
1823 }
1824 #endif
1825
1826 /* Clean prequeue, it must be empty really */
1827 __skb_queue_purge(&tp->ucopy.prequeue);
1828
1829 /* Clean up a referenced TCP bind bucket. */
1830 if (inet_csk(sk)->icsk_bind_hash)
1831 inet_put_port(sk);
1832
1833 BUG_ON(tp->fastopen_rsk);
1834
1835 /* If socket is aborted during connect operation */
1836 tcp_free_fastopen_req(tp);
1837 tcp_saved_syn_free(tp);
1838
1839 local_bh_disable();
1840 sk_sockets_allocated_dec(sk);
1841 local_bh_enable();
1842
1843 if (mem_cgroup_sockets_enabled && sk->sk_memcg)
1844 sock_release_memcg(sk);
1845 }
1846 EXPORT_SYMBOL(tcp_v4_destroy_sock);
1847
1848 #ifdef CONFIG_PROC_FS
1849 /* Proc filesystem TCP sock list dumping. */
1850
1851 /*
1852 * Get next listener socket follow cur. If cur is NULL, get first socket
1853 * starting from bucket given in st->bucket; when st->bucket is zero the
1854 * very first socket in the hash table is returned.
1855 */
1856 static void *listening_get_next(struct seq_file *seq, void *cur)
1857 {
1858 struct tcp_iter_state *st = seq->private;
1859 struct net *net = seq_file_net(seq);
1860 struct inet_listen_hashbucket *ilb;
1861 struct inet_connection_sock *icsk;
1862 struct sock *sk = cur;
1863
1864 if (!sk) {
1865 get_head:
1866 ilb = &tcp_hashinfo.listening_hash[st->bucket];
1867 spin_lock_bh(&ilb->lock);
1868 sk = sk_head(&ilb->head);
1869 st->offset = 0;
1870 goto get_sk;
1871 }
1872 ilb = &tcp_hashinfo.listening_hash[st->bucket];
1873 ++st->num;
1874 ++st->offset;
1875
1876 sk = sk_next(sk);
1877 get_sk:
1878 sk_for_each_from(sk) {
1879 if (!net_eq(sock_net(sk), net))
1880 continue;
1881 if (sk->sk_family == st->family)
1882 return sk;
1883 icsk = inet_csk(sk);
1884 }
1885 spin_unlock_bh(&ilb->lock);
1886 st->offset = 0;
1887 if (++st->bucket < INET_LHTABLE_SIZE)
1888 goto get_head;
1889 return NULL;
1890 }
1891
1892 static void *listening_get_idx(struct seq_file *seq, loff_t *pos)
1893 {
1894 struct tcp_iter_state *st = seq->private;
1895 void *rc;
1896
1897 st->bucket = 0;
1898 st->offset = 0;
1899 rc = listening_get_next(seq, NULL);
1900
1901 while (rc && *pos) {
1902 rc = listening_get_next(seq, rc);
1903 --*pos;
1904 }
1905 return rc;
1906 }
1907
1908 static inline bool empty_bucket(const struct tcp_iter_state *st)
1909 {
1910 return hlist_nulls_empty(&tcp_hashinfo.ehash[st->bucket].chain);
1911 }
1912
1913 /*
1914 * Get first established socket starting from bucket given in st->bucket.
1915 * If st->bucket is zero, the very first socket in the hash is returned.
1916 */
1917 static void *established_get_first(struct seq_file *seq)
1918 {
1919 struct tcp_iter_state *st = seq->private;
1920 struct net *net = seq_file_net(seq);
1921 void *rc = NULL;
1922
1923 st->offset = 0;
1924 for (; st->bucket <= tcp_hashinfo.ehash_mask; ++st->bucket) {
1925 struct sock *sk;
1926 struct hlist_nulls_node *node;
1927 spinlock_t *lock = inet_ehash_lockp(&tcp_hashinfo, st->bucket);
1928
1929 /* Lockless fast path for the common case of empty buckets */
1930 if (empty_bucket(st))
1931 continue;
1932
1933 spin_lock_bh(lock);
1934 sk_nulls_for_each(sk, node, &tcp_hashinfo.ehash[st->bucket].chain) {
1935 if (sk->sk_family != st->family ||
1936 !net_eq(sock_net(sk), net)) {
1937 continue;
1938 }
1939 rc = sk;
1940 goto out;
1941 }
1942 spin_unlock_bh(lock);
1943 }
1944 out:
1945 return rc;
1946 }
1947
1948 static void *established_get_next(struct seq_file *seq, void *cur)
1949 {
1950 struct sock *sk = cur;
1951 struct hlist_nulls_node *node;
1952 struct tcp_iter_state *st = seq->private;
1953 struct net *net = seq_file_net(seq);
1954
1955 ++st->num;
1956 ++st->offset;
1957
1958 sk = sk_nulls_next(sk);
1959
1960 sk_nulls_for_each_from(sk, node) {
1961 if (sk->sk_family == st->family && net_eq(sock_net(sk), net))
1962 return sk;
1963 }
1964
1965 spin_unlock_bh(inet_ehash_lockp(&tcp_hashinfo, st->bucket));
1966 ++st->bucket;
1967 return established_get_first(seq);
1968 }
1969
1970 static void *established_get_idx(struct seq_file *seq, loff_t pos)
1971 {
1972 struct tcp_iter_state *st = seq->private;
1973 void *rc;
1974
1975 st->bucket = 0;
1976 rc = established_get_first(seq);
1977
1978 while (rc && pos) {
1979 rc = established_get_next(seq, rc);
1980 --pos;
1981 }
1982 return rc;
1983 }
1984
1985 static void *tcp_get_idx(struct seq_file *seq, loff_t pos)
1986 {
1987 void *rc;
1988 struct tcp_iter_state *st = seq->private;
1989
1990 st->state = TCP_SEQ_STATE_LISTENING;
1991 rc = listening_get_idx(seq, &pos);
1992
1993 if (!rc) {
1994 st->state = TCP_SEQ_STATE_ESTABLISHED;
1995 rc = established_get_idx(seq, pos);
1996 }
1997
1998 return rc;
1999 }
2000
2001 static void *tcp_seek_last_pos(struct seq_file *seq)
2002 {
2003 struct tcp_iter_state *st = seq->private;
2004 int offset = st->offset;
2005 int orig_num = st->num;
2006 void *rc = NULL;
2007
2008 switch (st->state) {
2009 case TCP_SEQ_STATE_LISTENING:
2010 if (st->bucket >= INET_LHTABLE_SIZE)
2011 break;
2012 st->state = TCP_SEQ_STATE_LISTENING;
2013 rc = listening_get_next(seq, NULL);
2014 while (offset-- && rc)
2015 rc = listening_get_next(seq, rc);
2016 if (rc)
2017 break;
2018 st->bucket = 0;
2019 st->state = TCP_SEQ_STATE_ESTABLISHED;
2020 /* Fallthrough */
2021 case TCP_SEQ_STATE_ESTABLISHED:
2022 if (st->bucket > tcp_hashinfo.ehash_mask)
2023 break;
2024 rc = established_get_first(seq);
2025 while (offset-- && rc)
2026 rc = established_get_next(seq, rc);
2027 }
2028
2029 st->num = orig_num;
2030
2031 return rc;
2032 }
2033
2034 static void *tcp_seq_start(struct seq_file *seq, loff_t *pos)
2035 {
2036 struct tcp_iter_state *st = seq->private;
2037 void *rc;
2038
2039 if (*pos && *pos == st->last_pos) {
2040 rc = tcp_seek_last_pos(seq);
2041 if (rc)
2042 goto out;
2043 }
2044
2045 st->state = TCP_SEQ_STATE_LISTENING;
2046 st->num = 0;
2047 st->bucket = 0;
2048 st->offset = 0;
2049 rc = *pos ? tcp_get_idx(seq, *pos - 1) : SEQ_START_TOKEN;
2050
2051 out:
2052 st->last_pos = *pos;
2053 return rc;
2054 }
2055
2056 static void *tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
2057 {
2058 struct tcp_iter_state *st = seq->private;
2059 void *rc = NULL;
2060
2061 if (v == SEQ_START_TOKEN) {
2062 rc = tcp_get_idx(seq, 0);
2063 goto out;
2064 }
2065
2066 switch (st->state) {
2067 case TCP_SEQ_STATE_LISTENING:
2068 rc = listening_get_next(seq, v);
2069 if (!rc) {
2070 st->state = TCP_SEQ_STATE_ESTABLISHED;
2071 st->bucket = 0;
2072 st->offset = 0;
2073 rc = established_get_first(seq);
2074 }
2075 break;
2076 case TCP_SEQ_STATE_ESTABLISHED:
2077 rc = established_get_next(seq, v);
2078 break;
2079 }
2080 out:
2081 ++*pos;
2082 st->last_pos = *pos;
2083 return rc;
2084 }
2085
2086 static void tcp_seq_stop(struct seq_file *seq, void *v)
2087 {
2088 struct tcp_iter_state *st = seq->private;
2089
2090 switch (st->state) {
2091 case TCP_SEQ_STATE_LISTENING:
2092 if (v != SEQ_START_TOKEN)
2093 spin_unlock_bh(&tcp_hashinfo.listening_hash[st->bucket].lock);
2094 break;
2095 case TCP_SEQ_STATE_ESTABLISHED:
2096 if (v)
2097 spin_unlock_bh(inet_ehash_lockp(&tcp_hashinfo, st->bucket));
2098 break;
2099 }
2100 }
2101
2102 int tcp_seq_open(struct inode *inode, struct file *file)
2103 {
2104 struct tcp_seq_afinfo *afinfo = PDE_DATA(inode);
2105 struct tcp_iter_state *s;
2106 int err;
2107
2108 err = seq_open_net(inode, file, &afinfo->seq_ops,
2109 sizeof(struct tcp_iter_state));
2110 if (err < 0)
2111 return err;
2112
2113 s = ((struct seq_file *)file->private_data)->private;
2114 s->family = afinfo->family;
2115 s->last_pos = 0;
2116 return 0;
2117 }
2118 EXPORT_SYMBOL(tcp_seq_open);
2119
2120 int tcp_proc_register(struct net *net, struct tcp_seq_afinfo *afinfo)
2121 {
2122 int rc = 0;
2123 struct proc_dir_entry *p;
2124
2125 afinfo->seq_ops.start = tcp_seq_start;
2126 afinfo->seq_ops.next = tcp_seq_next;
2127 afinfo->seq_ops.stop = tcp_seq_stop;
2128
2129 p = proc_create_data(afinfo->name, S_IRUGO, net->proc_net,
2130 afinfo->seq_fops, afinfo);
2131 if (!p)
2132 rc = -ENOMEM;
2133 return rc;
2134 }
2135 EXPORT_SYMBOL(tcp_proc_register);
2136
2137 void tcp_proc_unregister(struct net *net, struct tcp_seq_afinfo *afinfo)
2138 {
2139 remove_proc_entry(afinfo->name, net->proc_net);
2140 }
2141 EXPORT_SYMBOL(tcp_proc_unregister);
2142
2143 static void get_openreq4(const struct request_sock *req,
2144 struct seq_file *f, int i)
2145 {
2146 const struct inet_request_sock *ireq = inet_rsk(req);
2147 long delta = req->rsk_timer.expires - jiffies;
2148
2149 seq_printf(f, "%4d: %08X:%04X %08X:%04X"
2150 " %02X %08X:%08X %02X:%08lX %08X %5u %8d %u %d %pK",
2151 i,
2152 ireq->ir_loc_addr,
2153 ireq->ir_num,
2154 ireq->ir_rmt_addr,
2155 ntohs(ireq->ir_rmt_port),
2156 TCP_SYN_RECV,
2157 0, 0, /* could print option size, but that is af dependent. */
2158 1, /* timers active (only the expire timer) */
2159 jiffies_delta_to_clock_t(delta),
2160 req->num_timeout,
2161 from_kuid_munged(seq_user_ns(f),
2162 sock_i_uid(req->rsk_listener)),
2163 0, /* non standard timer */
2164 0, /* open_requests have no inode */
2165 0,
2166 req);
2167 }
2168
2169 static void get_tcp4_sock(struct sock *sk, struct seq_file *f, int i)
2170 {
2171 int timer_active;
2172 unsigned long timer_expires;
2173 const struct tcp_sock *tp = tcp_sk(sk);
2174 const struct inet_connection_sock *icsk = inet_csk(sk);
2175 const struct inet_sock *inet = inet_sk(sk);
2176 const struct fastopen_queue *fastopenq = &icsk->icsk_accept_queue.fastopenq;
2177 __be32 dest = inet->inet_daddr;
2178 __be32 src = inet->inet_rcv_saddr;
2179 __u16 destp = ntohs(inet->inet_dport);
2180 __u16 srcp = ntohs(inet->inet_sport);
2181 int rx_queue;
2182 int state;
2183
2184 if (icsk->icsk_pending == ICSK_TIME_RETRANS ||
2185 icsk->icsk_pending == ICSK_TIME_EARLY_RETRANS ||
2186 icsk->icsk_pending == ICSK_TIME_LOSS_PROBE) {
2187 timer_active = 1;
2188 timer_expires = icsk->icsk_timeout;
2189 } else if (icsk->icsk_pending == ICSK_TIME_PROBE0) {
2190 timer_active = 4;
2191 timer_expires = icsk->icsk_timeout;
2192 } else if (timer_pending(&sk->sk_timer)) {
2193 timer_active = 2;
2194 timer_expires = sk->sk_timer.expires;
2195 } else {
2196 timer_active = 0;
2197 timer_expires = jiffies;
2198 }
2199
2200 state = sk_state_load(sk);
2201 if (state == TCP_LISTEN)
2202 rx_queue = sk->sk_ack_backlog;
2203 else
2204 /* Because we don't lock the socket,
2205 * we might find a transient negative value.
2206 */
2207 rx_queue = max_t(int, tp->rcv_nxt - tp->copied_seq, 0);
2208
2209 seq_printf(f, "%4d: %08X:%04X %08X:%04X %02X %08X:%08X %02X:%08lX "
2210 "%08X %5u %8d %lu %d %pK %lu %lu %u %u %d",
2211 i, src, srcp, dest, destp, state,
2212 tp->write_seq - tp->snd_una,
2213 rx_queue,
2214 timer_active,
2215 jiffies_delta_to_clock_t(timer_expires - jiffies),
2216 icsk->icsk_retransmits,
2217 from_kuid_munged(seq_user_ns(f), sock_i_uid(sk)),
2218 icsk->icsk_probes_out,
2219 sock_i_ino(sk),
2220 atomic_read(&sk->sk_refcnt), sk,
2221 jiffies_to_clock_t(icsk->icsk_rto),
2222 jiffies_to_clock_t(icsk->icsk_ack.ato),
2223 (icsk->icsk_ack.quick << 1) | icsk->icsk_ack.pingpong,
2224 tp->snd_cwnd,
2225 state == TCP_LISTEN ?
2226 fastopenq->max_qlen :
2227 (tcp_in_initial_slowstart(tp) ? -1 : tp->snd_ssthresh));
2228 }
2229
2230 static void get_timewait4_sock(const struct inet_timewait_sock *tw,
2231 struct seq_file *f, int i)
2232 {
2233 long delta = tw->tw_timer.expires - jiffies;
2234 __be32 dest, src;
2235 __u16 destp, srcp;
2236
2237 dest = tw->tw_daddr;
2238 src = tw->tw_rcv_saddr;
2239 destp = ntohs(tw->tw_dport);
2240 srcp = ntohs(tw->tw_sport);
2241
2242 seq_printf(f, "%4d: %08X:%04X %08X:%04X"
2243 " %02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK",
2244 i, src, srcp, dest, destp, tw->tw_substate, 0, 0,
2245 3, jiffies_delta_to_clock_t(delta), 0, 0, 0, 0,
2246 atomic_read(&tw->tw_refcnt), tw);
2247 }
2248
2249 #define TMPSZ 150
2250
2251 static int tcp4_seq_show(struct seq_file *seq, void *v)
2252 {
2253 struct tcp_iter_state *st;
2254 struct sock *sk = v;
2255
2256 seq_setwidth(seq, TMPSZ - 1);
2257 if (v == SEQ_START_TOKEN) {
2258 seq_puts(seq, " sl local_address rem_address st tx_queue "
2259 "rx_queue tr tm->when retrnsmt uid timeout "
2260 "inode");
2261 goto out;
2262 }
2263 st = seq->private;
2264
2265 if (sk->sk_state == TCP_TIME_WAIT)
2266 get_timewait4_sock(v, seq, st->num);
2267 else if (sk->sk_state == TCP_NEW_SYN_RECV)
2268 get_openreq4(v, seq, st->num);
2269 else
2270 get_tcp4_sock(v, seq, st->num);
2271 out:
2272 seq_pad(seq, '\n');
2273 return 0;
2274 }
2275
2276 static const struct file_operations tcp_afinfo_seq_fops = {
2277 .owner = THIS_MODULE,
2278 .open = tcp_seq_open,
2279 .read = seq_read,
2280 .llseek = seq_lseek,
2281 .release = seq_release_net
2282 };
2283
2284 static struct tcp_seq_afinfo tcp4_seq_afinfo = {
2285 .name = "tcp",
2286 .family = AF_INET,
2287 .seq_fops = &tcp_afinfo_seq_fops,
2288 .seq_ops = {
2289 .show = tcp4_seq_show,
2290 },
2291 };
2292
2293 static int __net_init tcp4_proc_init_net(struct net *net)
2294 {
2295 return tcp_proc_register(net, &tcp4_seq_afinfo);
2296 }
2297
2298 static void __net_exit tcp4_proc_exit_net(struct net *net)
2299 {
2300 tcp_proc_unregister(net, &tcp4_seq_afinfo);
2301 }
2302
2303 static struct pernet_operations tcp4_net_ops = {
2304 .init = tcp4_proc_init_net,
2305 .exit = tcp4_proc_exit_net,
2306 };
2307
2308 int __init tcp4_proc_init(void)
2309 {
2310 return register_pernet_subsys(&tcp4_net_ops);
2311 }
2312
2313 void tcp4_proc_exit(void)
2314 {
2315 unregister_pernet_subsys(&tcp4_net_ops);
2316 }
2317 #endif /* CONFIG_PROC_FS */
2318
2319 struct proto tcp_prot = {
2320 .name = "TCP",
2321 .owner = THIS_MODULE,
2322 .close = tcp_close,
2323 .connect = tcp_v4_connect,
2324 .disconnect = tcp_disconnect,
2325 .accept = inet_csk_accept,
2326 .ioctl = tcp_ioctl,
2327 .init = tcp_v4_init_sock,
2328 .destroy = tcp_v4_destroy_sock,
2329 .shutdown = tcp_shutdown,
2330 .setsockopt = tcp_setsockopt,
2331 .getsockopt = tcp_getsockopt,
2332 .recvmsg = tcp_recvmsg,
2333 .sendmsg = tcp_sendmsg,
2334 .sendpage = tcp_sendpage,
2335 .backlog_rcv = tcp_v4_do_rcv,
2336 .release_cb = tcp_release_cb,
2337 .hash = inet_hash,
2338 .unhash = inet_unhash,
2339 .get_port = inet_csk_get_port,
2340 .enter_memory_pressure = tcp_enter_memory_pressure,
2341 .stream_memory_free = tcp_stream_memory_free,
2342 .sockets_allocated = &tcp_sockets_allocated,
2343 .orphan_count = &tcp_orphan_count,
2344 .memory_allocated = &tcp_memory_allocated,
2345 .memory_pressure = &tcp_memory_pressure,
2346 .sysctl_mem = sysctl_tcp_mem,
2347 .sysctl_wmem = sysctl_tcp_wmem,
2348 .sysctl_rmem = sysctl_tcp_rmem,
2349 .max_header = MAX_TCP_HEADER,
2350 .obj_size = sizeof(struct tcp_sock),
2351 .slab_flags = SLAB_DESTROY_BY_RCU,
2352 .twsk_prot = &tcp_timewait_sock_ops,
2353 .rsk_prot = &tcp_request_sock_ops,
2354 .h.hashinfo = &tcp_hashinfo,
2355 .no_autobind = true,
2356 #ifdef CONFIG_COMPAT
2357 .compat_setsockopt = compat_tcp_setsockopt,
2358 .compat_getsockopt = compat_tcp_getsockopt,
2359 #endif
2360 .diag_destroy = tcp_abort,
2361 };
2362 EXPORT_SYMBOL(tcp_prot);
2363
2364 static void __net_exit tcp_sk_exit(struct net *net)
2365 {
2366 int cpu;
2367
2368 for_each_possible_cpu(cpu)
2369 inet_ctl_sock_destroy(*per_cpu_ptr(net->ipv4.tcp_sk, cpu));
2370 free_percpu(net->ipv4.tcp_sk);
2371 }
2372
2373 static int __net_init tcp_sk_init(struct net *net)
2374 {
2375 int res, cpu;
2376
2377 net->ipv4.tcp_sk = alloc_percpu(struct sock *);
2378 if (!net->ipv4.tcp_sk)
2379 return -ENOMEM;
2380
2381 for_each_possible_cpu(cpu) {
2382 struct sock *sk;
2383
2384 res = inet_ctl_sock_create(&sk, PF_INET, SOCK_RAW,
2385 IPPROTO_TCP, net);
2386 if (res)
2387 goto fail;
2388 sock_set_flag(sk, SOCK_USE_WRITE_QUEUE);
2389 *per_cpu_ptr(net->ipv4.tcp_sk, cpu) = sk;
2390 }
2391
2392 net->ipv4.sysctl_tcp_ecn = 2;
2393 net->ipv4.sysctl_tcp_ecn_fallback = 1;
2394
2395 net->ipv4.sysctl_tcp_base_mss = TCP_BASE_MSS;
2396 net->ipv4.sysctl_tcp_probe_threshold = TCP_PROBE_THRESHOLD;
2397 net->ipv4.sysctl_tcp_probe_interval = TCP_PROBE_INTERVAL;
2398
2399 net->ipv4.sysctl_tcp_keepalive_time = TCP_KEEPALIVE_TIME;
2400 net->ipv4.sysctl_tcp_keepalive_probes = TCP_KEEPALIVE_PROBES;
2401 net->ipv4.sysctl_tcp_keepalive_intvl = TCP_KEEPALIVE_INTVL;
2402
2403 net->ipv4.sysctl_tcp_syn_retries = TCP_SYN_RETRIES;
2404 net->ipv4.sysctl_tcp_synack_retries = TCP_SYNACK_RETRIES;
2405 net->ipv4.sysctl_tcp_syncookies = 1;
2406 net->ipv4.sysctl_tcp_reordering = TCP_FASTRETRANS_THRESH;
2407 net->ipv4.sysctl_tcp_retries1 = TCP_RETR1;
2408 net->ipv4.sysctl_tcp_retries2 = TCP_RETR2;
2409 net->ipv4.sysctl_tcp_orphan_retries = 0;
2410 net->ipv4.sysctl_tcp_fin_timeout = TCP_FIN_TIMEOUT;
2411 net->ipv4.sysctl_tcp_notsent_lowat = UINT_MAX;
2412
2413 return 0;
2414 fail:
2415 tcp_sk_exit(net);
2416
2417 return res;
2418 }
2419
2420 static void __net_exit tcp_sk_exit_batch(struct list_head *net_exit_list)
2421 {
2422 inet_twsk_purge(&tcp_hashinfo, &tcp_death_row, AF_INET);
2423 }
2424
2425 static struct pernet_operations __net_initdata tcp_sk_ops = {
2426 .init = tcp_sk_init,
2427 .exit = tcp_sk_exit,
2428 .exit_batch = tcp_sk_exit_batch,
2429 };
2430
2431 void __init tcp_v4_init(void)
2432 {
2433 inet_hashinfo_init(&tcp_hashinfo);
2434 if (register_pernet_subsys(&tcp_sk_ops))
2435 panic("Failed to create the TCP control socket.\n");
2436 }
Linux with added FPC-III support