?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'io_uring-5.11-2021-01-16' of git://git.kernel.dk/linux-block
[linux/fpc-iii.git] / net / ipv4 / tcp_ipv4.c
blob58207c7769d05693b650e3c93e4ef405a5d4b23a
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * INET An implementation of the TCP/IP protocol suite for the LINUX
4 * operating system. INET is implemented using the BSD Socket
5 * interface as the means of communication with the user level.
6 *
7 * Implementation of the Transmission Control Protocol(TCP).
8 *
9 * IPv4 specific functions
10 *
11 * code split from:
12 * linux/ipv4/tcp.c
13 * linux/ipv4/tcp_input.c
14 * linux/ipv4/tcp_output.c
15 *
16 * See tcp.c for author information
17 */
18
19 /*
20 * Changes:
21 * David S. Miller : New socket lookup architecture.
22 * This code is dedicated to John Dyson.
23 * David S. Miller : Change semantics of established hash,
24 * half is devoted to TIME_WAIT sockets
25 * and the rest go in the other half.
26 * Andi Kleen : Add support for syncookies and fixed
27 * some bugs: ip options weren't passed to
28 * the TCP layer, missed a check for an
29 * ACK bit.
30 * Andi Kleen : Implemented fast path mtu discovery.
31 * Fixed many serious bugs in the
32 * request_sock handling and moved
33 * most of it into the af independent code.
34 * Added tail drop and some other bugfixes.
35 * Added new listen semantics.
36 * Mike McLagan : Routing by source
37 * Juan Jose Ciarlante: ip_dynaddr bits
38 * Andi Kleen: various fixes.
39 * Vitaly E. Lavrov : Transparent proxy revived after year
40 * coma.
41 * Andi Kleen : Fix new listen.
42 * Andi Kleen : Fix accept error reporting.
43 * YOSHIFUJI Hideaki @USAGI and: Support IPV6_V6ONLY socket option, which
44 * Alexey Kuznetsov allow both IPv4 and IPv6 sockets to bind
45 * a single port at the same time.
46 */
47
48 #define pr_fmt(fmt) "TCP: " fmt
49
50 #include <linux/bottom_half.h>
51 #include <linux/types.h>
52 #include <linux/fcntl.h>
53 #include <linux/module.h>
54 #include <linux/random.h>
55 #include <linux/cache.h>
56 #include <linux/jhash.h>
57 #include <linux/init.h>
58 #include <linux/times.h>
59 #include <linux/slab.h>
60
61 #include <net/net_namespace.h>
62 #include <net/icmp.h>
63 #include <net/inet_hashtables.h>
64 #include <net/tcp.h>
65 #include <net/transp_v6.h>
66 #include <net/ipv6.h>
67 #include <net/inet_common.h>
68 #include <net/timewait_sock.h>
69 #include <net/xfrm.h>
70 #include <net/secure_seq.h>
71 #include <net/busy_poll.h>
72
73 #include <linux/inet.h>
74 #include <linux/ipv6.h>
75 #include <linux/stddef.h>
76 #include <linux/proc_fs.h>
77 #include <linux/seq_file.h>
78 #include <linux/inetdevice.h>
79 #include <linux/btf_ids.h>
80
81 #include <crypto/hash.h>
82 #include <linux/scatterlist.h>
83
84 #include <trace/events/tcp.h>
85
86 #ifdef CONFIG_TCP_MD5SIG
87 static int tcp_v4_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key,
88 __be32 daddr, __be32 saddr, const struct tcphdr *th);
89 #endif
90
91 struct inet_hashinfo tcp_hashinfo;
92 EXPORT_SYMBOL(tcp_hashinfo);
93
94 static u32 tcp_v4_init_seq(const struct sk_buff *skb)
95 {
96 return secure_tcp_seq(ip_hdr(skb)->daddr,
97 ip_hdr(skb)->saddr,
98 tcp_hdr(skb)->dest,
99 tcp_hdr(skb)->source);
100 }
101
102 static u32 tcp_v4_init_ts_off(const struct net *net, const struct sk_buff *skb)
103 {
104 return secure_tcp_ts_off(net, ip_hdr(skb)->daddr, ip_hdr(skb)->saddr);
105 }
106
107 int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp)
108 {
109 const struct inet_timewait_sock *tw = inet_twsk(sktw);
110 const struct tcp_timewait_sock *tcptw = tcp_twsk(sktw);
111 struct tcp_sock *tp = tcp_sk(sk);
112 int reuse = sock_net(sk)->ipv4.sysctl_tcp_tw_reuse;
113
114 if (reuse == 2) {
115 /* Still does not detect *everything* that goes through
116 * lo, since we require a loopback src or dst address
117 * or direct binding to 'lo' interface.
118 */
119 bool loopback = false;
120 if (tw->tw_bound_dev_if == LOOPBACK_IFINDEX)
121 loopback = true;
122 #if IS_ENABLED(CONFIG_IPV6)
123 if (tw->tw_family == AF_INET6) {
124 if (ipv6_addr_loopback(&tw->tw_v6_daddr) ||
125 ipv6_addr_v4mapped_loopback(&tw->tw_v6_daddr) ||
126 ipv6_addr_loopback(&tw->tw_v6_rcv_saddr) ||
127 ipv6_addr_v4mapped_loopback(&tw->tw_v6_rcv_saddr))
128 loopback = true;
129 } else
130 #endif
131 {
132 if (ipv4_is_loopback(tw->tw_daddr) ||
133 ipv4_is_loopback(tw->tw_rcv_saddr))
134 loopback = true;
135 }
136 if (!loopback)
137 reuse = 0;
138 }
139
140 /* With PAWS, it is safe from the viewpoint
141 of data integrity. Even without PAWS it is safe provided sequence
142 spaces do not overlap i.e. at data rates <= 80Mbit/sec.
143
144 Actually, the idea is close to VJ's one, only timestamp cache is
145 held not per host, but per port pair and TW bucket is used as state
146 holder.
147
148 If TW bucket has been already destroyed we fall back to VJ's scheme
149 and use initial timestamp retrieved from peer table.
150 */
151 if (tcptw->tw_ts_recent_stamp &&
152 (!twp || (reuse && time_after32(ktime_get_seconds(),
153 tcptw->tw_ts_recent_stamp)))) {
154 /* In case of repair and re-using TIME-WAIT sockets we still
155 * want to be sure that it is safe as above but honor the
156 * sequence numbers and time stamps set as part of the repair
157 * process.
158 *
159 * Without this check re-using a TIME-WAIT socket with TCP
160 * repair would accumulate a -1 on the repair assigned
161 * sequence number. The first time it is reused the sequence
162 * is -1, the second time -2, etc. This fixes that issue
163 * without appearing to create any others.
164 */
165 if (likely(!tp->repair)) {
166 u32 seq = tcptw->tw_snd_nxt + 65535 + 2;
167
168 if (!seq)
169 seq = 1;
170 WRITE_ONCE(tp->write_seq, seq);
171 tp->rx_opt.ts_recent = tcptw->tw_ts_recent;
172 tp->rx_opt.ts_recent_stamp = tcptw->tw_ts_recent_stamp;
173 }
174 sock_hold(sktw);
175 return 1;
176 }
177
178 return 0;
179 }
180 EXPORT_SYMBOL_GPL(tcp_twsk_unique);
181
182 static int tcp_v4_pre_connect(struct sock *sk, struct sockaddr *uaddr,
183 int addr_len)
184 {
185 /* This check is replicated from tcp_v4_connect() and intended to
186 * prevent BPF program called below from accessing bytes that are out
187 * of the bound specified by user in addr_len.
188 */
189 if (addr_len < sizeof(struct sockaddr_in))
190 return -EINVAL;
191
192 sock_owned_by_me(sk);
193
194 return BPF_CGROUP_RUN_PROG_INET4_CONNECT(sk, uaddr);
195 }
196
197 /* This will initiate an outgoing connection. */
198 int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
199 {
200 struct sockaddr_in *usin = (struct sockaddr_in *)uaddr;
201 struct inet_sock *inet = inet_sk(sk);
202 struct tcp_sock *tp = tcp_sk(sk);
203 __be16 orig_sport, orig_dport;
204 __be32 daddr, nexthop;
205 struct flowi4 *fl4;
206 struct rtable *rt;
207 int err;
208 struct ip_options_rcu *inet_opt;
209 struct inet_timewait_death_row *tcp_death_row = &sock_net(sk)->ipv4.tcp_death_row;
210
211 if (addr_len < sizeof(struct sockaddr_in))
212 return -EINVAL;
213
214 if (usin->sin_family != AF_INET)
215 return -EAFNOSUPPORT;
216
217 nexthop = daddr = usin->sin_addr.s_addr;
218 inet_opt = rcu_dereference_protected(inet->inet_opt,
219 lockdep_sock_is_held(sk));
220 if (inet_opt && inet_opt->opt.srr) {
221 if (!daddr)
222 return -EINVAL;
223 nexthop = inet_opt->opt.faddr;
224 }
225
226 orig_sport = inet->inet_sport;
227 orig_dport = usin->sin_port;
228 fl4 = &inet->cork.fl.u.ip4;
229 rt = ip_route_connect(fl4, nexthop, inet->inet_saddr,
230 RT_CONN_FLAGS(sk), sk->sk_bound_dev_if,
231 IPPROTO_TCP,
232 orig_sport, orig_dport, sk);
233 if (IS_ERR(rt)) {
234 err = PTR_ERR(rt);
235 if (err == -ENETUNREACH)
236 IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTNOROUTES);
237 return err;
238 }
239
240 if (rt->rt_flags & (RTCF_MULTICAST | RTCF_BROADCAST)) {
241 ip_rt_put(rt);
242 return -ENETUNREACH;
243 }
244
245 if (!inet_opt || !inet_opt->opt.srr)
246 daddr = fl4->daddr;
247
248 if (!inet->inet_saddr)
249 inet->inet_saddr = fl4->saddr;
250 sk_rcv_saddr_set(sk, inet->inet_saddr);
251
252 if (tp->rx_opt.ts_recent_stamp && inet->inet_daddr != daddr) {
253 /* Reset inherited state */
254 tp->rx_opt.ts_recent = 0;
255 tp->rx_opt.ts_recent_stamp = 0;
256 if (likely(!tp->repair))
257 WRITE_ONCE(tp->write_seq, 0);
258 }
259
260 inet->inet_dport = usin->sin_port;
261 sk_daddr_set(sk, daddr);
262
263 inet_csk(sk)->icsk_ext_hdr_len = 0;
264 if (inet_opt)
265 inet_csk(sk)->icsk_ext_hdr_len = inet_opt->opt.optlen;
266
267 tp->rx_opt.mss_clamp = TCP_MSS_DEFAULT;
268
269 /* Socket identity is still unknown (sport may be zero).
270 * However we set state to SYN-SENT and not releasing socket
271 * lock select source port, enter ourselves into the hash tables and
272 * complete initialization after this.
273 */
274 tcp_set_state(sk, TCP_SYN_SENT);
275 err = inet_hash_connect(tcp_death_row, sk);
276 if (err)
277 goto failure;
278
279 sk_set_txhash(sk);
280
281 rt = ip_route_newports(fl4, rt, orig_sport, orig_dport,
282 inet->inet_sport, inet->inet_dport, sk);
283 if (IS_ERR(rt)) {
284 err = PTR_ERR(rt);
285 rt = NULL;
286 goto failure;
287 }
288 /* OK, now commit destination to socket. */
289 sk->sk_gso_type = SKB_GSO_TCPV4;
290 sk_setup_caps(sk, &rt->dst);
291 rt = NULL;
292
293 if (likely(!tp->repair)) {
294 if (!tp->write_seq)
295 WRITE_ONCE(tp->write_seq,
296 secure_tcp_seq(inet->inet_saddr,
297 inet->inet_daddr,
298 inet->inet_sport,
299 usin->sin_port));
300 tp->tsoffset = secure_tcp_ts_off(sock_net(sk),
301 inet->inet_saddr,
302 inet->inet_daddr);
303 }
304
305 inet->inet_id = prandom_u32();
306
307 if (tcp_fastopen_defer_connect(sk, &err))
308 return err;
309 if (err)
310 goto failure;
311
312 err = tcp_connect(sk);
313
314 if (err)
315 goto failure;
316
317 return 0;
318
319 failure:
320 /*
321 * This unhashes the socket and releases the local port,
322 * if necessary.
323 */
324 tcp_set_state(sk, TCP_CLOSE);
325 ip_rt_put(rt);
326 sk->sk_route_caps = 0;
327 inet->inet_dport = 0;
328 return err;
329 }
330 EXPORT_SYMBOL(tcp_v4_connect);
331
332 /*
333 * This routine reacts to ICMP_FRAG_NEEDED mtu indications as defined in RFC1191.
334 * It can be called through tcp_release_cb() if socket was owned by user
335 * at the time tcp_v4_err() was called to handle ICMP message.
336 */
337 void tcp_v4_mtu_reduced(struct sock *sk)
338 {
339 struct inet_sock *inet = inet_sk(sk);
340 struct dst_entry *dst;
341 u32 mtu;
342
343 if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE))
344 return;
345 mtu = tcp_sk(sk)->mtu_info;
346 dst = inet_csk_update_pmtu(sk, mtu);
347 if (!dst)
348 return;
349
350 /* Something is about to be wrong... Remember soft error
351 * for the case, if this connection will not able to recover.
352 */
353 if (mtu < dst_mtu(dst) && ip_dont_fragment(sk, dst))
354 sk->sk_err_soft = EMSGSIZE;
355
356 mtu = dst_mtu(dst);
357
358 if (inet->pmtudisc != IP_PMTUDISC_DONT &&
359 ip_sk_accept_pmtu(sk) &&
360 inet_csk(sk)->icsk_pmtu_cookie > mtu) {
361 tcp_sync_mss(sk, mtu);
362
363 /* Resend the TCP packet because it's
364 * clear that the old packet has been
365 * dropped. This is the new "fast" path mtu
366 * discovery.
367 */
368 tcp_simple_retransmit(sk);
369 } /* else let the usual retransmit timer handle it */
370 }
371 EXPORT_SYMBOL(tcp_v4_mtu_reduced);
372
373 static void do_redirect(struct sk_buff *skb, struct sock *sk)
374 {
375 struct dst_entry *dst = __sk_dst_check(sk, 0);
376
377 if (dst)
378 dst->ops->redirect(dst, sk, skb);
379 }
380
381
382 /* handle ICMP messages on TCP_NEW_SYN_RECV request sockets */
383 void tcp_req_err(struct sock *sk, u32 seq, bool abort)
384 {
385 struct request_sock *req = inet_reqsk(sk);
386 struct net *net = sock_net(sk);
387
388 /* ICMPs are not backlogged, hence we cannot get
389 * an established socket here.
390 */
391 if (seq != tcp_rsk(req)->snt_isn) {
392 __NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS);
393 } else if (abort) {
394 /*
395 * Still in SYN_RECV, just remove it silently.
396 * There is no good way to pass the error to the newly
397 * created socket, and POSIX does not want network
398 * errors returned from accept().
399 */
400 inet_csk_reqsk_queue_drop(req->rsk_listener, req);
401 tcp_listendrop(req->rsk_listener);
402 }
403 reqsk_put(req);
404 }
405 EXPORT_SYMBOL(tcp_req_err);
406
407 /* TCP-LD (RFC 6069) logic */
408 void tcp_ld_RTO_revert(struct sock *sk, u32 seq)
409 {
410 struct inet_connection_sock *icsk = inet_csk(sk);
411 struct tcp_sock *tp = tcp_sk(sk);
412 struct sk_buff *skb;
413 s32 remaining;
414 u32 delta_us;
415
416 if (sock_owned_by_user(sk))
417 return;
418
419 if (seq != tp->snd_una || !icsk->icsk_retransmits ||
420 !icsk->icsk_backoff)
421 return;
422
423 skb = tcp_rtx_queue_head(sk);
424 if (WARN_ON_ONCE(!skb))
425 return;
426
427 icsk->icsk_backoff--;
428 icsk->icsk_rto = tp->srtt_us ? __tcp_set_rto(tp) : TCP_TIMEOUT_INIT;
429 icsk->icsk_rto = inet_csk_rto_backoff(icsk, TCP_RTO_MAX);
430
431 tcp_mstamp_refresh(tp);
432 delta_us = (u32)(tp->tcp_mstamp - tcp_skb_timestamp_us(skb));
433 remaining = icsk->icsk_rto - usecs_to_jiffies(delta_us);
434
435 if (remaining > 0) {
436 inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
437 remaining, TCP_RTO_MAX);
438 } else {
439 /* RTO revert clocked out retransmission.
440 * Will retransmit now.
441 */
442 tcp_retransmit_timer(sk);
443 }
444 }
445 EXPORT_SYMBOL(tcp_ld_RTO_revert);
446
447 /*
448 * This routine is called by the ICMP module when it gets some
449 * sort of error condition. If err < 0 then the socket should
450 * be closed and the error returned to the user. If err > 0
451 * it's just the icmp type << 8 | icmp code. After adjustment
452 * header points to the first 8 bytes of the tcp header. We need
453 * to find the appropriate port.
454 *
455 * The locking strategy used here is very "optimistic". When
456 * someone else accesses the socket the ICMP is just dropped
457 * and for some paths there is no check at all.
458 * A more general error queue to queue errors for later handling
459 * is probably better.
460 *
461 */
462
463 int tcp_v4_err(struct sk_buff *skb, u32 info)
464 {
465 const struct iphdr *iph = (const struct iphdr *)skb->data;
466 struct tcphdr *th = (struct tcphdr *)(skb->data + (iph->ihl << 2));
467 struct tcp_sock *tp;
468 struct inet_sock *inet;
469 const int type = icmp_hdr(skb)->type;
470 const int code = icmp_hdr(skb)->code;
471 struct sock *sk;
472 struct request_sock *fastopen;
473 u32 seq, snd_una;
474 int err;
475 struct net *net = dev_net(skb->dev);
476
477 sk = __inet_lookup_established(net, &tcp_hashinfo, iph->daddr,
478 th->dest, iph->saddr, ntohs(th->source),
479 inet_iif(skb), 0);
480 if (!sk) {
481 __ICMP_INC_STATS(net, ICMP_MIB_INERRORS);
482 return -ENOENT;
483 }
484 if (sk->sk_state == TCP_TIME_WAIT) {
485 inet_twsk_put(inet_twsk(sk));
486 return 0;
487 }
488 seq = ntohl(th->seq);
489 if (sk->sk_state == TCP_NEW_SYN_RECV) {
490 tcp_req_err(sk, seq, type == ICMP_PARAMETERPROB ||
491 type == ICMP_TIME_EXCEEDED ||
492 (type == ICMP_DEST_UNREACH &&
493 (code == ICMP_NET_UNREACH ||
494 code == ICMP_HOST_UNREACH)));
495 return 0;
496 }
497
498 bh_lock_sock(sk);
499 /* If too many ICMPs get dropped on busy
500 * servers this needs to be solved differently.
501 * We do take care of PMTU discovery (RFC1191) special case :
502 * we can receive locally generated ICMP messages while socket is held.
503 */
504 if (sock_owned_by_user(sk)) {
505 if (!(type == ICMP_DEST_UNREACH && code == ICMP_FRAG_NEEDED))
506 __NET_INC_STATS(net, LINUX_MIB_LOCKDROPPEDICMPS);
507 }
508 if (sk->sk_state == TCP_CLOSE)
509 goto out;
510
511 if (unlikely(iph->ttl < inet_sk(sk)->min_ttl)) {
512 __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
513 goto out;
514 }
515
516 tp = tcp_sk(sk);
517 /* XXX (TFO) - tp->snd_una should be ISN (tcp_create_openreq_child() */
518 fastopen = rcu_dereference(tp->fastopen_rsk);
519 snd_una = fastopen ? tcp_rsk(fastopen)->snt_isn : tp->snd_una;
520 if (sk->sk_state != TCP_LISTEN &&
521 !between(seq, snd_una, tp->snd_nxt)) {
522 __NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS);
523 goto out;
524 }
525
526 switch (type) {
527 case ICMP_REDIRECT:
528 if (!sock_owned_by_user(sk))
529 do_redirect(skb, sk);
530 goto out;
531 case ICMP_SOURCE_QUENCH:
532 /* Just silently ignore these. */
533 goto out;
534 case ICMP_PARAMETERPROB:
535 err = EPROTO;
536 break;
537 case ICMP_DEST_UNREACH:
538 if (code > NR_ICMP_UNREACH)
539 goto out;
540
541 if (code == ICMP_FRAG_NEEDED) { /* PMTU discovery (RFC1191) */
542 /* We are not interested in TCP_LISTEN and open_requests
543 * (SYN-ACKs send out by Linux are always <576bytes so
544 * they should go through unfragmented).
545 */
546 if (sk->sk_state == TCP_LISTEN)
547 goto out;
548
549 tp->mtu_info = info;
550 if (!sock_owned_by_user(sk)) {
551 tcp_v4_mtu_reduced(sk);
552 } else {
553 if (!test_and_set_bit(TCP_MTU_REDUCED_DEFERRED, &sk->sk_tsq_flags))
554 sock_hold(sk);
555 }
556 goto out;
557 }
558
559 err = icmp_err_convert[code].errno;
560 /* check if this ICMP message allows revert of backoff.
561 * (see RFC 6069)
562 */
563 if (!fastopen &&
564 (code == ICMP_NET_UNREACH || code == ICMP_HOST_UNREACH))
565 tcp_ld_RTO_revert(sk, seq);
566 break;
567 case ICMP_TIME_EXCEEDED:
568 err = EHOSTUNREACH;
569 break;
570 default:
571 goto out;
572 }
573
574 switch (sk->sk_state) {
575 case TCP_SYN_SENT:
576 case TCP_SYN_RECV:
577 /* Only in fast or simultaneous open. If a fast open socket is
578 * already accepted it is treated as a connected one below.
579 */
580 if (fastopen && !fastopen->sk)
581 break;
582
583 ip_icmp_error(sk, skb, err, th->dest, info, (u8 *)th);
584
585 if (!sock_owned_by_user(sk)) {
586 sk->sk_err = err;
587
588 sk->sk_error_report(sk);
589
590 tcp_done(sk);
591 } else {
592 sk->sk_err_soft = err;
593 }
594 goto out;
595 }
596
597 /* If we've already connected we will keep trying
598 * until we time out, or the user gives up.
599 *
600 * rfc1122 4.2.3.9 allows to consider as hard errors
601 * only PROTO_UNREACH and PORT_UNREACH (well, FRAG_FAILED too,
602 * but it is obsoleted by pmtu discovery).
603 *
604 * Note, that in modern internet, where routing is unreliable
605 * and in each dark corner broken firewalls sit, sending random
606 * errors ordered by their masters even this two messages finally lose
607 * their original sense (even Linux sends invalid PORT_UNREACHs)
608 *
609 * Now we are in compliance with RFCs.
610 * --ANK (980905)
611 */
612
613 inet = inet_sk(sk);
614 if (!sock_owned_by_user(sk) && inet->recverr) {
615 sk->sk_err = err;
616 sk->sk_error_report(sk);
617 } else { /* Only an error on timeout */
618 sk->sk_err_soft = err;
619 }
620
621 out:
622 bh_unlock_sock(sk);
623 sock_put(sk);
624 return 0;
625 }
626
627 void __tcp_v4_send_check(struct sk_buff *skb, __be32 saddr, __be32 daddr)
628 {
629 struct tcphdr *th = tcp_hdr(skb);
630
631 th->check = ~tcp_v4_check(skb->len, saddr, daddr, 0);
632 skb->csum_start = skb_transport_header(skb) - skb->head;
633 skb->csum_offset = offsetof(struct tcphdr, check);
634 }
635
636 /* This routine computes an IPv4 TCP checksum. */
637 void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb)
638 {
639 const struct inet_sock *inet = inet_sk(sk);
640
641 __tcp_v4_send_check(skb, inet->inet_saddr, inet->inet_daddr);
642 }
643 EXPORT_SYMBOL(tcp_v4_send_check);
644
645 /*
646 * This routine will send an RST to the other tcp.
647 *
648 * Someone asks: why I NEVER use socket parameters (TOS, TTL etc.)
649 * for reset.
650 * Answer: if a packet caused RST, it is not for a socket
651 * existing in our system, if it is matched to a socket,
652 * it is just duplicate segment or bug in other side's TCP.
653 * So that we build reply only basing on parameters
654 * arrived with segment.
655 * Exception: precedence violation. We do not implement it in any case.
656 */
657
658 static void tcp_v4_send_reset(const struct sock *sk, struct sk_buff *skb)
659 {
660 const struct tcphdr *th = tcp_hdr(skb);
661 struct {
662 struct tcphdr th;
663 #ifdef CONFIG_TCP_MD5SIG
664 __be32 opt[(TCPOLEN_MD5SIG_ALIGNED >> 2)];
665 #endif
666 } rep;
667 struct ip_reply_arg arg;
668 #ifdef CONFIG_TCP_MD5SIG
669 struct tcp_md5sig_key *key = NULL;
670 const __u8 *hash_location = NULL;
671 unsigned char newhash[16];
672 int genhash;
673 struct sock *sk1 = NULL;
674 #endif
675 u64 transmit_time = 0;
676 struct sock *ctl_sk;
677 struct net *net;
678
679 /* Never send a reset in response to a reset. */
680 if (th->rst)
681 return;
682
683 /* If sk not NULL, it means we did a successful lookup and incoming
684 * route had to be correct. prequeue might have dropped our dst.
685 */
686 if (!sk && skb_rtable(skb)->rt_type != RTN_LOCAL)
687 return;
688
689 /* Swap the send and the receive. */
690 memset(&rep, 0, sizeof(rep));
691 rep.th.dest = th->source;
692 rep.th.source = th->dest;
693 rep.th.doff = sizeof(struct tcphdr) / 4;
694 rep.th.rst = 1;
695
696 if (th->ack) {
697 rep.th.seq = th->ack_seq;
698 } else {
699 rep.th.ack = 1;
700 rep.th.ack_seq = htonl(ntohl(th->seq) + th->syn + th->fin +
701 skb->len - (th->doff << 2));
702 }
703
704 memset(&arg, 0, sizeof(arg));
705 arg.iov[0].iov_base = (unsigned char *)&rep;
706 arg.iov[0].iov_len = sizeof(rep.th);
707
708 net = sk ? sock_net(sk) : dev_net(skb_dst(skb)->dev);
709 #ifdef CONFIG_TCP_MD5SIG
710 rcu_read_lock();
711 hash_location = tcp_parse_md5sig_option(th);
712 if (sk && sk_fullsock(sk)) {
713 const union tcp_md5_addr *addr;
714 int l3index;
715
716 /* sdif set, means packet ingressed via a device
717 * in an L3 domain and inet_iif is set to it.
718 */
719 l3index = tcp_v4_sdif(skb) ? inet_iif(skb) : 0;
720 addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr;
721 key = tcp_md5_do_lookup(sk, l3index, addr, AF_INET);
722 } else if (hash_location) {
723 const union tcp_md5_addr *addr;
724 int sdif = tcp_v4_sdif(skb);
725 int dif = inet_iif(skb);
726 int l3index;
727
728 /*
729 * active side is lost. Try to find listening socket through
730 * source port, and then find md5 key through listening socket.
731 * we are not loose security here:
732 * Incoming packet is checked with md5 hash with finding key,
733 * no RST generated if md5 hash doesn't match.
734 */
735 sk1 = __inet_lookup_listener(net, &tcp_hashinfo, NULL, 0,
736 ip_hdr(skb)->saddr,
737 th->source, ip_hdr(skb)->daddr,
738 ntohs(th->source), dif, sdif);
739 /* don't send rst if it can't find key */
740 if (!sk1)
741 goto out;
742
743 /* sdif set, means packet ingressed via a device
744 * in an L3 domain and dif is set to it.
745 */
746 l3index = sdif ? dif : 0;
747 addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr;
748 key = tcp_md5_do_lookup(sk1, l3index, addr, AF_INET);
749 if (!key)
750 goto out;
751
752
753 genhash = tcp_v4_md5_hash_skb(newhash, key, NULL, skb);
754 if (genhash || memcmp(hash_location, newhash, 16) != 0)
755 goto out;
756
757 }
758
759 if (key) {
760 rep.opt[0] = htonl((TCPOPT_NOP << 24) |
761 (TCPOPT_NOP << 16) |
762 (TCPOPT_MD5SIG << 8) |
763 TCPOLEN_MD5SIG);
764 /* Update length and the length the header thinks exists */
765 arg.iov[0].iov_len += TCPOLEN_MD5SIG_ALIGNED;
766 rep.th.doff = arg.iov[0].iov_len / 4;
767
768 tcp_v4_md5_hash_hdr((__u8 *) &rep.opt[1],
769 key, ip_hdr(skb)->saddr,
770 ip_hdr(skb)->daddr, &rep.th);
771 }
772 #endif
773 arg.csum = csum_tcpudp_nofold(ip_hdr(skb)->daddr,
774 ip_hdr(skb)->saddr, /* XXX */
775 arg.iov[0].iov_len, IPPROTO_TCP, 0);
776 arg.csumoffset = offsetof(struct tcphdr, check) / 2;
777 arg.flags = (sk && inet_sk_transparent(sk)) ? IP_REPLY_ARG_NOSRCCHECK : 0;
778
779 /* When socket is gone, all binding information is lost.
780 * routing might fail in this case. No choice here, if we choose to force
781 * input interface, we will misroute in case of asymmetric route.
782 */
783 if (sk) {
784 arg.bound_dev_if = sk->sk_bound_dev_if;
785 if (sk_fullsock(sk))
786 trace_tcp_send_reset(sk, skb);
787 }
788
789 BUILD_BUG_ON(offsetof(struct sock, sk_bound_dev_if) !=
790 offsetof(struct inet_timewait_sock, tw_bound_dev_if));
791
792 arg.tos = ip_hdr(skb)->tos;
793 arg.uid = sock_net_uid(net, sk && sk_fullsock(sk) ? sk : NULL);
794 local_bh_disable();
795 ctl_sk = this_cpu_read(*net->ipv4.tcp_sk);
796 if (sk) {
797 ctl_sk->sk_mark = (sk->sk_state == TCP_TIME_WAIT) ?
798 inet_twsk(sk)->tw_mark : sk->sk_mark;
799 ctl_sk->sk_priority = (sk->sk_state == TCP_TIME_WAIT) ?
800 inet_twsk(sk)->tw_priority : sk->sk_priority;
801 transmit_time = tcp_transmit_time(sk);
802 }
803 ip_send_unicast_reply(ctl_sk,
804 skb, &TCP_SKB_CB(skb)->header.h4.opt,
805 ip_hdr(skb)->saddr, ip_hdr(skb)->daddr,
806 &arg, arg.iov[0].iov_len,
807 transmit_time);
808
809 ctl_sk->sk_mark = 0;
810 __TCP_INC_STATS(net, TCP_MIB_OUTSEGS);
811 __TCP_INC_STATS(net, TCP_MIB_OUTRSTS);
812 local_bh_enable();
813
814 #ifdef CONFIG_TCP_MD5SIG
815 out:
816 rcu_read_unlock();
817 #endif
818 }
819
820 /* The code following below sending ACKs in SYN-RECV and TIME-WAIT states
821 outside socket context is ugly, certainly. What can I do?
822 */
823
824 static void tcp_v4_send_ack(const struct sock *sk,
825 struct sk_buff *skb, u32 seq, u32 ack,
826 u32 win, u32 tsval, u32 tsecr, int oif,
827 struct tcp_md5sig_key *key,
828 int reply_flags, u8 tos)
829 {
830 const struct tcphdr *th = tcp_hdr(skb);
831 struct {
832 struct tcphdr th;
833 __be32 opt[(TCPOLEN_TSTAMP_ALIGNED >> 2)
834 #ifdef CONFIG_TCP_MD5SIG
835 + (TCPOLEN_MD5SIG_ALIGNED >> 2)
836 #endif
837 ];
838 } rep;
839 struct net *net = sock_net(sk);
840 struct ip_reply_arg arg;
841 struct sock *ctl_sk;
842 u64 transmit_time;
843
844 memset(&rep.th, 0, sizeof(struct tcphdr));
845 memset(&arg, 0, sizeof(arg));
846
847 arg.iov[0].iov_base = (unsigned char *)&rep;
848 arg.iov[0].iov_len = sizeof(rep.th);
849 if (tsecr) {
850 rep.opt[0] = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
851 (TCPOPT_TIMESTAMP << 8) |
852 TCPOLEN_TIMESTAMP);
853 rep.opt[1] = htonl(tsval);
854 rep.opt[2] = htonl(tsecr);
855 arg.iov[0].iov_len += TCPOLEN_TSTAMP_ALIGNED;
856 }
857
858 /* Swap the send and the receive. */
859 rep.th.dest = th->source;
860 rep.th.source = th->dest;
861 rep.th.doff = arg.iov[0].iov_len / 4;
862 rep.th.seq = htonl(seq);
863 rep.th.ack_seq = htonl(ack);
864 rep.th.ack = 1;
865 rep.th.window = htons(win);
866
867 #ifdef CONFIG_TCP_MD5SIG
868 if (key) {
869 int offset = (tsecr) ? 3 : 0;
870
871 rep.opt[offset++] = htonl((TCPOPT_NOP << 24) |
872 (TCPOPT_NOP << 16) |
873 (TCPOPT_MD5SIG << 8) |
874 TCPOLEN_MD5SIG);
875 arg.iov[0].iov_len += TCPOLEN_MD5SIG_ALIGNED;
876 rep.th.doff = arg.iov[0].iov_len/4;
877
878 tcp_v4_md5_hash_hdr((__u8 *) &rep.opt[offset],
879 key, ip_hdr(skb)->saddr,
880 ip_hdr(skb)->daddr, &rep.th);
881 }
882 #endif
883 arg.flags = reply_flags;
884 arg.csum = csum_tcpudp_nofold(ip_hdr(skb)->daddr,
885 ip_hdr(skb)->saddr, /* XXX */
886 arg.iov[0].iov_len, IPPROTO_TCP, 0);
887 arg.csumoffset = offsetof(struct tcphdr, check) / 2;
888 if (oif)
889 arg.bound_dev_if = oif;
890 arg.tos = tos;
891 arg.uid = sock_net_uid(net, sk_fullsock(sk) ? sk : NULL);
892 local_bh_disable();
893 ctl_sk = this_cpu_read(*net->ipv4.tcp_sk);
894 ctl_sk->sk_mark = (sk->sk_state == TCP_TIME_WAIT) ?
895 inet_twsk(sk)->tw_mark : sk->sk_mark;
896 ctl_sk->sk_priority = (sk->sk_state == TCP_TIME_WAIT) ?
897 inet_twsk(sk)->tw_priority : sk->sk_priority;
898 transmit_time = tcp_transmit_time(sk);
899 ip_send_unicast_reply(ctl_sk,
900 skb, &TCP_SKB_CB(skb)->header.h4.opt,
901 ip_hdr(skb)->saddr, ip_hdr(skb)->daddr,
902 &arg, arg.iov[0].iov_len,
903 transmit_time);
904
905 ctl_sk->sk_mark = 0;
906 __TCP_INC_STATS(net, TCP_MIB_OUTSEGS);
907 local_bh_enable();
908 }
909
910 static void tcp_v4_timewait_ack(struct sock *sk, struct sk_buff *skb)
911 {
912 struct inet_timewait_sock *tw = inet_twsk(sk);
913 struct tcp_timewait_sock *tcptw = tcp_twsk(sk);
914
915 tcp_v4_send_ack(sk, skb,
916 tcptw->tw_snd_nxt, tcptw->tw_rcv_nxt,
917 tcptw->tw_rcv_wnd >> tw->tw_rcv_wscale,
918 tcp_time_stamp_raw() + tcptw->tw_ts_offset,
919 tcptw->tw_ts_recent,
920 tw->tw_bound_dev_if,
921 tcp_twsk_md5_key(tcptw),
922 tw->tw_transparent ? IP_REPLY_ARG_NOSRCCHECK : 0,
923 tw->tw_tos
924 );
925
926 inet_twsk_put(tw);
927 }
928
929 static void tcp_v4_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb,
930 struct request_sock *req)
931 {
932 const union tcp_md5_addr *addr;
933 int l3index;
934
935 /* sk->sk_state == TCP_LISTEN -> for regular TCP_SYN_RECV
936 * sk->sk_state == TCP_SYN_RECV -> for Fast Open.
937 */
938 u32 seq = (sk->sk_state == TCP_LISTEN) ? tcp_rsk(req)->snt_isn + 1 :
939 tcp_sk(sk)->snd_nxt;
940
941 /* RFC 7323 2.3
942 * The window field (SEG.WND) of every outgoing segment, with the
943 * exception of <SYN> segments, MUST be right-shifted by
944 * Rcv.Wind.Shift bits:
945 */
946 addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr;
947 l3index = tcp_v4_sdif(skb) ? inet_iif(skb) : 0;
948 tcp_v4_send_ack(sk, skb, seq,
949 tcp_rsk(req)->rcv_nxt,
950 req->rsk_rcv_wnd >> inet_rsk(req)->rcv_wscale,
951 tcp_time_stamp_raw() + tcp_rsk(req)->ts_off,
952 req->ts_recent,
953 0,
954 tcp_md5_do_lookup(sk, l3index, addr, AF_INET),
955 inet_rsk(req)->no_srccheck ? IP_REPLY_ARG_NOSRCCHECK : 0,
956 ip_hdr(skb)->tos);
957 }
958
959 /*
960 * Send a SYN-ACK after having received a SYN.
961 * This still operates on a request_sock only, not on a big
962 * socket.
963 */
964 static int tcp_v4_send_synack(const struct sock *sk, struct dst_entry *dst,
965 struct flowi *fl,
966 struct request_sock *req,
967 struct tcp_fastopen_cookie *foc,
968 enum tcp_synack_type synack_type,
969 struct sk_buff *syn_skb)
970 {
971 const struct inet_request_sock *ireq = inet_rsk(req);
972 struct flowi4 fl4;
973 int err = -1;
974 struct sk_buff *skb;
975 u8 tos;
976
977 /* First, grab a route. */
978 if (!dst && (dst = inet_csk_route_req(sk, &fl4, req)) == NULL)
979 return -1;
980
981 skb = tcp_make_synack(sk, dst, req, foc, synack_type, syn_skb);
982
983 if (skb) {
984 __tcp_v4_send_check(skb, ireq->ir_loc_addr, ireq->ir_rmt_addr);
985
986 tos = sock_net(sk)->ipv4.sysctl_tcp_reflect_tos ?
987 (tcp_rsk(req)->syn_tos & ~INET_ECN_MASK) |
988 (inet_sk(sk)->tos & INET_ECN_MASK) :
989 inet_sk(sk)->tos;
990
991 if (!INET_ECN_is_capable(tos) &&
992 tcp_bpf_ca_needs_ecn((struct sock *)req))
993 tos |= INET_ECN_ECT_0;
994
995 rcu_read_lock();
996 err = ip_build_and_send_pkt(skb, sk, ireq->ir_loc_addr,
997 ireq->ir_rmt_addr,
998 rcu_dereference(ireq->ireq_opt),
999 tos);
1000 rcu_read_unlock();
1001 err = net_xmit_eval(err);
1002 }
1003
1004 return err;
1005 }
1006
1007 /*
1008 * IPv4 request_sock destructor.
1009 */
1010 static void tcp_v4_reqsk_destructor(struct request_sock *req)
1011 {
1012 kfree(rcu_dereference_protected(inet_rsk(req)->ireq_opt, 1));
1013 }
1014
1015 #ifdef CONFIG_TCP_MD5SIG
1016 /*
1017 * RFC2385 MD5 checksumming requires a mapping of
1018 * IP address->MD5 Key.
1019 * We need to maintain these in the sk structure.
1020 */
1021
1022 DEFINE_STATIC_KEY_FALSE(tcp_md5_needed);
1023 EXPORT_SYMBOL(tcp_md5_needed);
1024
1025 /* Find the Key structure for an address. */
1026 struct tcp_md5sig_key *__tcp_md5_do_lookup(const struct sock *sk, int l3index,
1027 const union tcp_md5_addr *addr,
1028 int family)
1029 {
1030 const struct tcp_sock *tp = tcp_sk(sk);
1031 struct tcp_md5sig_key *key;
1032 const struct tcp_md5sig_info *md5sig;
1033 __be32 mask;
1034 struct tcp_md5sig_key *best_match = NULL;
1035 bool match;
1036
1037 /* caller either holds rcu_read_lock() or socket lock */
1038 md5sig = rcu_dereference_check(tp->md5sig_info,
1039 lockdep_sock_is_held(sk));
1040 if (!md5sig)
1041 return NULL;
1042
1043 hlist_for_each_entry_rcu(key, &md5sig->head, node,
1044 lockdep_sock_is_held(sk)) {
1045 if (key->family != family)
1046 continue;
1047 if (key->l3index && key->l3index != l3index)
1048 continue;
1049 if (family == AF_INET) {
1050 mask = inet_make_mask(key->prefixlen);
1051 match = (key->addr.a4.s_addr & mask) ==
1052 (addr->a4.s_addr & mask);
1053 #if IS_ENABLED(CONFIG_IPV6)
1054 } else if (family == AF_INET6) {
1055 match = ipv6_prefix_equal(&key->addr.a6, &addr->a6,
1056 key->prefixlen);
1057 #endif
1058 } else {
1059 match = false;
1060 }
1061
1062 if (match && (!best_match ||
1063 key->prefixlen > best_match->prefixlen))
1064 best_match = key;
1065 }
1066 return best_match;
1067 }
1068 EXPORT_SYMBOL(__tcp_md5_do_lookup);
1069
1070 static struct tcp_md5sig_key *tcp_md5_do_lookup_exact(const struct sock *sk,
1071 const union tcp_md5_addr *addr,
1072 int family, u8 prefixlen,
1073 int l3index)
1074 {
1075 const struct tcp_sock *tp = tcp_sk(sk);
1076 struct tcp_md5sig_key *key;
1077 unsigned int size = sizeof(struct in_addr);
1078 const struct tcp_md5sig_info *md5sig;
1079
1080 /* caller either holds rcu_read_lock() or socket lock */
1081 md5sig = rcu_dereference_check(tp->md5sig_info,
1082 lockdep_sock_is_held(sk));
1083 if (!md5sig)
1084 return NULL;
1085 #if IS_ENABLED(CONFIG_IPV6)
1086 if (family == AF_INET6)
1087 size = sizeof(struct in6_addr);
1088 #endif
1089 hlist_for_each_entry_rcu(key, &md5sig->head, node,
1090 lockdep_sock_is_held(sk)) {
1091 if (key->family != family)
1092 continue;
1093 if (key->l3index && key->l3index != l3index)
1094 continue;
1095 if (!memcmp(&key->addr, addr, size) &&
1096 key->prefixlen == prefixlen)
1097 return key;
1098 }
1099 return NULL;
1100 }
1101
1102 struct tcp_md5sig_key *tcp_v4_md5_lookup(const struct sock *sk,
1103 const struct sock *addr_sk)
1104 {
1105 const union tcp_md5_addr *addr;
1106 int l3index;
1107
1108 l3index = l3mdev_master_ifindex_by_index(sock_net(sk),
1109 addr_sk->sk_bound_dev_if);
1110 addr = (const union tcp_md5_addr *)&addr_sk->sk_daddr;
1111 return tcp_md5_do_lookup(sk, l3index, addr, AF_INET);
1112 }
1113 EXPORT_SYMBOL(tcp_v4_md5_lookup);
1114
1115 /* This can be called on a newly created socket, from other files */
1116 int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr,
1117 int family, u8 prefixlen, int l3index,
1118 const u8 *newkey, u8 newkeylen, gfp_t gfp)
1119 {
1120 /* Add Key to the list */
1121 struct tcp_md5sig_key *key;
1122 struct tcp_sock *tp = tcp_sk(sk);
1123 struct tcp_md5sig_info *md5sig;
1124
1125 key = tcp_md5_do_lookup_exact(sk, addr, family, prefixlen, l3index);
1126 if (key) {
1127 /* Pre-existing entry - just update that one.
1128 * Note that the key might be used concurrently.
1129 * data_race() is telling kcsan that we do not care of
1130 * key mismatches, since changing MD5 key on live flows
1131 * can lead to packet drops.
1132 */
1133 data_race(memcpy(key->key, newkey, newkeylen));
1134
1135 /* Pairs with READ_ONCE() in tcp_md5_hash_key().
1136 * Also note that a reader could catch new key->keylen value
1137 * but old key->key[], this is the reason we use __GFP_ZERO
1138 * at sock_kmalloc() time below these lines.
1139 */
1140 WRITE_ONCE(key->keylen, newkeylen);
1141
1142 return 0;
1143 }
1144
1145 md5sig = rcu_dereference_protected(tp->md5sig_info,
1146 lockdep_sock_is_held(sk));
1147 if (!md5sig) {
1148 md5sig = kmalloc(sizeof(*md5sig), gfp);
1149 if (!md5sig)
1150 return -ENOMEM;
1151
1152 sk_nocaps_add(sk, NETIF_F_GSO_MASK);
1153 INIT_HLIST_HEAD(&md5sig->head);
1154 rcu_assign_pointer(tp->md5sig_info, md5sig);
1155 }
1156
1157 key = sock_kmalloc(sk, sizeof(*key), gfp | __GFP_ZERO);
1158 if (!key)
1159 return -ENOMEM;
1160 if (!tcp_alloc_md5sig_pool()) {
1161 sock_kfree_s(sk, key, sizeof(*key));
1162 return -ENOMEM;
1163 }
1164
1165 memcpy(key->key, newkey, newkeylen);
1166 key->keylen = newkeylen;
1167 key->family = family;
1168 key->prefixlen = prefixlen;
1169 key->l3index = l3index;
1170 memcpy(&key->addr, addr,
1171 (family == AF_INET6) ? sizeof(struct in6_addr) :
1172 sizeof(struct in_addr));
1173 hlist_add_head_rcu(&key->node, &md5sig->head);
1174 return 0;
1175 }
1176 EXPORT_SYMBOL(tcp_md5_do_add);
1177
1178 int tcp_md5_do_del(struct sock *sk, const union tcp_md5_addr *addr, int family,
1179 u8 prefixlen, int l3index)
1180 {
1181 struct tcp_md5sig_key *key;
1182
1183 key = tcp_md5_do_lookup_exact(sk, addr, family, prefixlen, l3index);
1184 if (!key)
1185 return -ENOENT;
1186 hlist_del_rcu(&key->node);
1187 atomic_sub(sizeof(*key), &sk->sk_omem_alloc);
1188 kfree_rcu(key, rcu);
1189 return 0;
1190 }
1191 EXPORT_SYMBOL(tcp_md5_do_del);
1192
1193 static void tcp_clear_md5_list(struct sock *sk)
1194 {
1195 struct tcp_sock *tp = tcp_sk(sk);
1196 struct tcp_md5sig_key *key;
1197 struct hlist_node *n;
1198 struct tcp_md5sig_info *md5sig;
1199
1200 md5sig = rcu_dereference_protected(tp->md5sig_info, 1);
1201
1202 hlist_for_each_entry_safe(key, n, &md5sig->head, node) {
1203 hlist_del_rcu(&key->node);
1204 atomic_sub(sizeof(*key), &sk->sk_omem_alloc);
1205 kfree_rcu(key, rcu);
1206 }
1207 }
1208
1209 static int tcp_v4_parse_md5_keys(struct sock *sk, int optname,
1210 sockptr_t optval, int optlen)
1211 {
1212 struct tcp_md5sig cmd;
1213 struct sockaddr_in *sin = (struct sockaddr_in *)&cmd.tcpm_addr;
1214 const union tcp_md5_addr *addr;
1215 u8 prefixlen = 32;
1216 int l3index = 0;
1217
1218 if (optlen < sizeof(cmd))
1219 return -EINVAL;
1220
1221 if (copy_from_sockptr(&cmd, optval, sizeof(cmd)))
1222 return -EFAULT;
1223
1224 if (sin->sin_family != AF_INET)
1225 return -EINVAL;
1226
1227 if (optname == TCP_MD5SIG_EXT &&
1228 cmd.tcpm_flags & TCP_MD5SIG_FLAG_PREFIX) {
1229 prefixlen = cmd.tcpm_prefixlen;
1230 if (prefixlen > 32)
1231 return -EINVAL;
1232 }
1233
1234 if (optname == TCP_MD5SIG_EXT &&
1235 cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX) {
1236 struct net_device *dev;
1237
1238 rcu_read_lock();
1239 dev = dev_get_by_index_rcu(sock_net(sk), cmd.tcpm_ifindex);
1240 if (dev && netif_is_l3_master(dev))
1241 l3index = dev->ifindex;
1242
1243 rcu_read_unlock();
1244
1245 /* ok to reference set/not set outside of rcu;
1246 * right now device MUST be an L3 master
1247 */
1248 if (!dev || !l3index)
1249 return -EINVAL;
1250 }
1251
1252 addr = (union tcp_md5_addr *)&sin->sin_addr.s_addr;
1253
1254 if (!cmd.tcpm_keylen)
1255 return tcp_md5_do_del(sk, addr, AF_INET, prefixlen, l3index);
1256
1257 if (cmd.tcpm_keylen > TCP_MD5SIG_MAXKEYLEN)
1258 return -EINVAL;
1259
1260 return tcp_md5_do_add(sk, addr, AF_INET, prefixlen, l3index,
1261 cmd.tcpm_key, cmd.tcpm_keylen, GFP_KERNEL);
1262 }
1263
1264 static int tcp_v4_md5_hash_headers(struct tcp_md5sig_pool *hp,
1265 __be32 daddr, __be32 saddr,
1266 const struct tcphdr *th, int nbytes)
1267 {
1268 struct tcp4_pseudohdr *bp;
1269 struct scatterlist sg;
1270 struct tcphdr *_th;
1271
1272 bp = hp->scratch;
1273 bp->saddr = saddr;
1274 bp->daddr = daddr;
1275 bp->pad = 0;
1276 bp->protocol = IPPROTO_TCP;
1277 bp->len = cpu_to_be16(nbytes);
1278
1279 _th = (struct tcphdr *)(bp + 1);
1280 memcpy(_th, th, sizeof(*th));
1281 _th->check = 0;
1282
1283 sg_init_one(&sg, bp, sizeof(*bp) + sizeof(*th));
1284 ahash_request_set_crypt(hp->md5_req, &sg, NULL,
1285 sizeof(*bp) + sizeof(*th));
1286 return crypto_ahash_update(hp->md5_req);
1287 }
1288
1289 static int tcp_v4_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key,
1290 __be32 daddr, __be32 saddr, const struct tcphdr *th)
1291 {
1292 struct tcp_md5sig_pool *hp;
1293 struct ahash_request *req;
1294
1295 hp = tcp_get_md5sig_pool();
1296 if (!hp)
1297 goto clear_hash_noput;
1298 req = hp->md5_req;
1299
1300 if (crypto_ahash_init(req))
1301 goto clear_hash;
1302 if (tcp_v4_md5_hash_headers(hp, daddr, saddr, th, th->doff << 2))
1303 goto clear_hash;
1304 if (tcp_md5_hash_key(hp, key))
1305 goto clear_hash;
1306 ahash_request_set_crypt(req, NULL, md5_hash, 0);
1307 if (crypto_ahash_final(req))
1308 goto clear_hash;
1309
1310 tcp_put_md5sig_pool();
1311 return 0;
1312
1313 clear_hash:
1314 tcp_put_md5sig_pool();
1315 clear_hash_noput:
1316 memset(md5_hash, 0, 16);
1317 return 1;
1318 }
1319
1320 int tcp_v4_md5_hash_skb(char *md5_hash, const struct tcp_md5sig_key *key,
1321 const struct sock *sk,
1322 const struct sk_buff *skb)
1323 {
1324 struct tcp_md5sig_pool *hp;
1325 struct ahash_request *req;
1326 const struct tcphdr *th = tcp_hdr(skb);
1327 __be32 saddr, daddr;
1328
1329 if (sk) { /* valid for establish/request sockets */
1330 saddr = sk->sk_rcv_saddr;
1331 daddr = sk->sk_daddr;
1332 } else {
1333 const struct iphdr *iph = ip_hdr(skb);
1334 saddr = iph->saddr;
1335 daddr = iph->daddr;
1336 }
1337
1338 hp = tcp_get_md5sig_pool();
1339 if (!hp)
1340 goto clear_hash_noput;
1341 req = hp->md5_req;
1342
1343 if (crypto_ahash_init(req))
1344 goto clear_hash;
1345
1346 if (tcp_v4_md5_hash_headers(hp, daddr, saddr, th, skb->len))
1347 goto clear_hash;
1348 if (tcp_md5_hash_skb_data(hp, skb, th->doff << 2))
1349 goto clear_hash;
1350 if (tcp_md5_hash_key(hp, key))
1351 goto clear_hash;
1352 ahash_request_set_crypt(req, NULL, md5_hash, 0);
1353 if (crypto_ahash_final(req))
1354 goto clear_hash;
1355
1356 tcp_put_md5sig_pool();
1357 return 0;
1358
1359 clear_hash:
1360 tcp_put_md5sig_pool();
1361 clear_hash_noput:
1362 memset(md5_hash, 0, 16);
1363 return 1;
1364 }
1365 EXPORT_SYMBOL(tcp_v4_md5_hash_skb);
1366
1367 #endif
1368
1369 /* Called with rcu_read_lock() */
1370 static bool tcp_v4_inbound_md5_hash(const struct sock *sk,
1371 const struct sk_buff *skb,
1372 int dif, int sdif)
1373 {
1374 #ifdef CONFIG_TCP_MD5SIG
1375 /*
1376 * This gets called for each TCP segment that arrives
1377 * so we want to be efficient.
1378 * We have 3 drop cases:
1379 * o No MD5 hash and one expected.
1380 * o MD5 hash and we're not expecting one.
1381 * o MD5 hash and its wrong.
1382 */
1383 const __u8 *hash_location = NULL;
1384 struct tcp_md5sig_key *hash_expected;
1385 const struct iphdr *iph = ip_hdr(skb);
1386 const struct tcphdr *th = tcp_hdr(skb);
1387 const union tcp_md5_addr *addr;
1388 unsigned char newhash[16];
1389 int genhash, l3index;
1390
1391 /* sdif set, means packet ingressed via a device
1392 * in an L3 domain and dif is set to the l3mdev
1393 */
1394 l3index = sdif ? dif : 0;
1395
1396 addr = (union tcp_md5_addr *)&iph->saddr;
1397 hash_expected = tcp_md5_do_lookup(sk, l3index, addr, AF_INET);
1398 hash_location = tcp_parse_md5sig_option(th);
1399
1400 /* We've parsed the options - do we have a hash? */
1401 if (!hash_expected && !hash_location)
1402 return false;
1403
1404 if (hash_expected && !hash_location) {
1405 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5NOTFOUND);
1406 return true;
1407 }
1408
1409 if (!hash_expected && hash_location) {
1410 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5UNEXPECTED);
1411 return true;
1412 }
1413
1414 /* Okay, so this is hash_expected and hash_location -
1415 * so we need to calculate the checksum.
1416 */
1417 genhash = tcp_v4_md5_hash_skb(newhash,
1418 hash_expected,
1419 NULL, skb);
1420
1421 if (genhash || memcmp(hash_location, newhash, 16) != 0) {
1422 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5FAILURE);
1423 net_info_ratelimited("MD5 Hash failed for (%pI4, %d)->(%pI4, %d)%s L3 index %d\n",
1424 &iph->saddr, ntohs(th->source),
1425 &iph->daddr, ntohs(th->dest),
1426 genhash ? " tcp_v4_calc_md5_hash failed"
1427 : "", l3index);
1428 return true;
1429 }
1430 return false;
1431 #endif
1432 return false;
1433 }
1434
1435 static void tcp_v4_init_req(struct request_sock *req,
1436 const struct sock *sk_listener,
1437 struct sk_buff *skb)
1438 {
1439 struct inet_request_sock *ireq = inet_rsk(req);
1440 struct net *net = sock_net(sk_listener);
1441
1442 sk_rcv_saddr_set(req_to_sk(req), ip_hdr(skb)->daddr);
1443 sk_daddr_set(req_to_sk(req), ip_hdr(skb)->saddr);
1444 RCU_INIT_POINTER(ireq->ireq_opt, tcp_v4_save_options(net, skb));
1445 }
1446
1447 static struct dst_entry *tcp_v4_route_req(const struct sock *sk,
1448 struct sk_buff *skb,
1449 struct flowi *fl,
1450 struct request_sock *req)
1451 {
1452 tcp_v4_init_req(req, sk, skb);
1453
1454 if (security_inet_conn_request(sk, skb, req))
1455 return NULL;
1456
1457 return inet_csk_route_req(sk, &fl->u.ip4, req);
1458 }
1459
1460 struct request_sock_ops tcp_request_sock_ops __read_mostly = {
1461 .family = PF_INET,
1462 .obj_size = sizeof(struct tcp_request_sock),
1463 .rtx_syn_ack = tcp_rtx_synack,
1464 .send_ack = tcp_v4_reqsk_send_ack,
1465 .destructor = tcp_v4_reqsk_destructor,
1466 .send_reset = tcp_v4_send_reset,
1467 .syn_ack_timeout = tcp_syn_ack_timeout,
1468 };
1469
1470 const struct tcp_request_sock_ops tcp_request_sock_ipv4_ops = {
1471 .mss_clamp = TCP_MSS_DEFAULT,
1472 #ifdef CONFIG_TCP_MD5SIG
1473 .req_md5_lookup = tcp_v4_md5_lookup,
1474 .calc_md5_hash = tcp_v4_md5_hash_skb,
1475 #endif
1476 #ifdef CONFIG_SYN_COOKIES
1477 .cookie_init_seq = cookie_v4_init_sequence,
1478 #endif
1479 .route_req = tcp_v4_route_req,
1480 .init_seq = tcp_v4_init_seq,
1481 .init_ts_off = tcp_v4_init_ts_off,
1482 .send_synack = tcp_v4_send_synack,
1483 };
1484
1485 int tcp_v4_conn_request(struct sock *sk, struct sk_buff *skb)
1486 {
1487 /* Never answer to SYNs send to broadcast or multicast */
1488 if (skb_rtable(skb)->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST))
1489 goto drop;
1490
1491 return tcp_conn_request(&tcp_request_sock_ops,
1492 &tcp_request_sock_ipv4_ops, sk, skb);
1493
1494 drop:
1495 tcp_listendrop(sk);
1496 return 0;
1497 }
1498 EXPORT_SYMBOL(tcp_v4_conn_request);
1499
1500
1501 /*
1502 * The three way handshake has completed - we got a valid synack -
1503 * now create the new socket.
1504 */
1505 struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb,
1506 struct request_sock *req,
1507 struct dst_entry *dst,
1508 struct request_sock *req_unhash,
1509 bool *own_req)
1510 {
1511 struct inet_request_sock *ireq;
1512 bool found_dup_sk = false;
1513 struct inet_sock *newinet;
1514 struct tcp_sock *newtp;
1515 struct sock *newsk;
1516 #ifdef CONFIG_TCP_MD5SIG
1517 const union tcp_md5_addr *addr;
1518 struct tcp_md5sig_key *key;
1519 int l3index;
1520 #endif
1521 struct ip_options_rcu *inet_opt;
1522
1523 if (sk_acceptq_is_full(sk))
1524 goto exit_overflow;
1525
1526 newsk = tcp_create_openreq_child(sk, req, skb);
1527 if (!newsk)
1528 goto exit_nonewsk;
1529
1530 newsk->sk_gso_type = SKB_GSO_TCPV4;
1531 inet_sk_rx_dst_set(newsk, skb);
1532
1533 newtp = tcp_sk(newsk);
1534 newinet = inet_sk(newsk);
1535 ireq = inet_rsk(req);
1536 sk_daddr_set(newsk, ireq->ir_rmt_addr);
1537 sk_rcv_saddr_set(newsk, ireq->ir_loc_addr);
1538 newsk->sk_bound_dev_if = ireq->ir_iif;
1539 newinet->inet_saddr = ireq->ir_loc_addr;
1540 inet_opt = rcu_dereference(ireq->ireq_opt);
1541 RCU_INIT_POINTER(newinet->inet_opt, inet_opt);
1542 newinet->mc_index = inet_iif(skb);
1543 newinet->mc_ttl = ip_hdr(skb)->ttl;
1544 newinet->rcv_tos = ip_hdr(skb)->tos;
1545 inet_csk(newsk)->icsk_ext_hdr_len = 0;
1546 if (inet_opt)
1547 inet_csk(newsk)->icsk_ext_hdr_len = inet_opt->opt.optlen;
1548 newinet->inet_id = prandom_u32();
1549
1550 /* Set ToS of the new socket based upon the value of incoming SYN.
1551 * ECT bits are set later in tcp_init_transfer().
1552 */
1553 if (sock_net(sk)->ipv4.sysctl_tcp_reflect_tos)
1554 newinet->tos = tcp_rsk(req)->syn_tos & ~INET_ECN_MASK;
1555
1556 if (!dst) {
1557 dst = inet_csk_route_child_sock(sk, newsk, req);
1558 if (!dst)
1559 goto put_and_exit;
1560 } else {
1561 /* syncookie case : see end of cookie_v4_check() */
1562 }
1563 sk_setup_caps(newsk, dst);
1564
1565 tcp_ca_openreq_child(newsk, dst);
1566
1567 tcp_sync_mss(newsk, dst_mtu(dst));
1568 newtp->advmss = tcp_mss_clamp(tcp_sk(sk), dst_metric_advmss(dst));
1569
1570 tcp_initialize_rcv_mss(newsk);
1571
1572 #ifdef CONFIG_TCP_MD5SIG
1573 l3index = l3mdev_master_ifindex_by_index(sock_net(sk), ireq->ir_iif);
1574 /* Copy over the MD5 key from the original socket */
1575 addr = (union tcp_md5_addr *)&newinet->inet_daddr;
1576 key = tcp_md5_do_lookup(sk, l3index, addr, AF_INET);
1577 if (key) {
1578 /*
1579 * We're using one, so create a matching key
1580 * on the newsk structure. If we fail to get
1581 * memory, then we end up not copying the key
1582 * across. Shucks.
1583 */
1584 tcp_md5_do_add(newsk, addr, AF_INET, 32, l3index,
1585 key->key, key->keylen, GFP_ATOMIC);
1586 sk_nocaps_add(newsk, NETIF_F_GSO_MASK);
1587 }
1588 #endif
1589
1590 if (__inet_inherit_port(sk, newsk) < 0)
1591 goto put_and_exit;
1592 *own_req = inet_ehash_nolisten(newsk, req_to_sk(req_unhash),
1593 &found_dup_sk);
1594 if (likely(*own_req)) {
1595 tcp_move_syn(newtp, req);
1596 ireq->ireq_opt = NULL;
1597 } else {
1598 if (!req_unhash && found_dup_sk) {
1599 /* This code path should only be executed in the
1600 * syncookie case only
1601 */
1602 bh_unlock_sock(newsk);
1603 sock_put(newsk);
1604 newsk = NULL;
1605 } else {
1606 newinet->inet_opt = NULL;
1607 }
1608 }
1609 return newsk;
1610
1611 exit_overflow:
1612 NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS);
1613 exit_nonewsk:
1614 dst_release(dst);
1615 exit:
1616 tcp_listendrop(sk);
1617 return NULL;
1618 put_and_exit:
1619 newinet->inet_opt = NULL;
1620 inet_csk_prepare_forced_close(newsk);
1621 tcp_done(newsk);
1622 goto exit;
1623 }
1624 EXPORT_SYMBOL(tcp_v4_syn_recv_sock);
1625
1626 static struct sock *tcp_v4_cookie_check(struct sock *sk, struct sk_buff *skb)
1627 {
1628 #ifdef CONFIG_SYN_COOKIES
1629 const struct tcphdr *th = tcp_hdr(skb);
1630
1631 if (!th->syn)
1632 sk = cookie_v4_check(sk, skb);
1633 #endif
1634 return sk;
1635 }
1636
1637 u16 tcp_v4_get_syncookie(struct sock *sk, struct iphdr *iph,
1638 struct tcphdr *th, u32 *cookie)
1639 {
1640 u16 mss = 0;
1641 #ifdef CONFIG_SYN_COOKIES
1642 mss = tcp_get_syncookie_mss(&tcp_request_sock_ops,
1643 &tcp_request_sock_ipv4_ops, sk, th);
1644 if (mss) {
1645 *cookie = __cookie_v4_init_sequence(iph, th, &mss);
1646 tcp_synq_overflow(sk);
1647 }
1648 #endif
1649 return mss;
1650 }
1651
1652 /* The socket must have it's spinlock held when we get
1653 * here, unless it is a TCP_LISTEN socket.
1654 *
1655 * We have a potential double-lock case here, so even when
1656 * doing backlog processing we use the BH locking scheme.
1657 * This is because we cannot sleep with the original spinlock
1658 * held.
1659 */
1660 int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb)
1661 {
1662 struct sock *rsk;
1663
1664 if (sk->sk_state == TCP_ESTABLISHED) { /* Fast path */
1665 struct dst_entry *dst = sk->sk_rx_dst;
1666
1667 sock_rps_save_rxhash(sk, skb);
1668 sk_mark_napi_id(sk, skb);
1669 if (dst) {
1670 if (inet_sk(sk)->rx_dst_ifindex != skb->skb_iif ||
1671 !dst->ops->check(dst, 0)) {
1672 dst_release(dst);
1673 sk->sk_rx_dst = NULL;
1674 }
1675 }
1676 tcp_rcv_established(sk, skb);
1677 return 0;
1678 }
1679
1680 if (tcp_checksum_complete(skb))
1681 goto csum_err;
1682
1683 if (sk->sk_state == TCP_LISTEN) {
1684 struct sock *nsk = tcp_v4_cookie_check(sk, skb);
1685
1686 if (!nsk)
1687 goto discard;
1688 if (nsk != sk) {
1689 if (tcp_child_process(sk, nsk, skb)) {
1690 rsk = nsk;
1691 goto reset;
1692 }
1693 return 0;
1694 }
1695 } else
1696 sock_rps_save_rxhash(sk, skb);
1697
1698 if (tcp_rcv_state_process(sk, skb)) {
1699 rsk = sk;
1700 goto reset;
1701 }
1702 return 0;
1703
1704 reset:
1705 tcp_v4_send_reset(rsk, skb);
1706 discard:
1707 kfree_skb(skb);
1708 /* Be careful here. If this function gets more complicated and
1709 * gcc suffers from register pressure on the x86, sk (in %ebx)
1710 * might be destroyed here. This current version compiles correctly,
1711 * but you have been warned.
1712 */
1713 return 0;
1714
1715 csum_err:
1716 TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
1717 TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
1718 goto discard;
1719 }
1720 EXPORT_SYMBOL(tcp_v4_do_rcv);
1721
1722 int tcp_v4_early_demux(struct sk_buff *skb)
1723 {
1724 const struct iphdr *iph;
1725 const struct tcphdr *th;
1726 struct sock *sk;
1727
1728 if (skb->pkt_type != PACKET_HOST)
1729 return 0;
1730
1731 if (!pskb_may_pull(skb, skb_transport_offset(skb) + sizeof(struct tcphdr)))
1732 return 0;
1733
1734 iph = ip_hdr(skb);
1735 th = tcp_hdr(skb);
1736
1737 if (th->doff < sizeof(struct tcphdr) / 4)
1738 return 0;
1739
1740 sk = __inet_lookup_established(dev_net(skb->dev), &tcp_hashinfo,
1741 iph->saddr, th->source,
1742 iph->daddr, ntohs(th->dest),
1743 skb->skb_iif, inet_sdif(skb));
1744 if (sk) {
1745 skb->sk = sk;
1746 skb->destructor = sock_edemux;
1747 if (sk_fullsock(sk)) {
1748 struct dst_entry *dst = READ_ONCE(sk->sk_rx_dst);
1749
1750 if (dst)
1751 dst = dst_check(dst, 0);
1752 if (dst &&
1753 inet_sk(sk)->rx_dst_ifindex == skb->skb_iif)
1754 skb_dst_set_noref(skb, dst);
1755 }
1756 }
1757 return 0;
1758 }
1759
1760 bool tcp_add_backlog(struct sock *sk, struct sk_buff *skb)
1761 {
1762 u32 limit = READ_ONCE(sk->sk_rcvbuf) + READ_ONCE(sk->sk_sndbuf);
1763 struct skb_shared_info *shinfo;
1764 const struct tcphdr *th;
1765 struct tcphdr *thtail;
1766 struct sk_buff *tail;
1767 unsigned int hdrlen;
1768 bool fragstolen;
1769 u32 gso_segs;
1770 int delta;
1771
1772 /* In case all data was pulled from skb frags (in __pskb_pull_tail()),
1773 * we can fix skb->truesize to its real value to avoid future drops.
1774 * This is valid because skb is not yet charged to the socket.
1775 * It has been noticed pure SACK packets were sometimes dropped
1776 * (if cooked by drivers without copybreak feature).
1777 */
1778 skb_condense(skb);
1779
1780 skb_dst_drop(skb);
1781
1782 if (unlikely(tcp_checksum_complete(skb))) {
1783 bh_unlock_sock(sk);
1784 __TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
1785 __TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
1786 return true;
1787 }
1788
1789 /* Attempt coalescing to last skb in backlog, even if we are
1790 * above the limits.
1791 * This is okay because skb capacity is limited to MAX_SKB_FRAGS.
1792 */
1793 th = (const struct tcphdr *)skb->data;
1794 hdrlen = th->doff * 4;
1795 shinfo = skb_shinfo(skb);
1796
1797 if (!shinfo->gso_size)
1798 shinfo->gso_size = skb->len - hdrlen;
1799
1800 if (!shinfo->gso_segs)
1801 shinfo->gso_segs = 1;
1802
1803 tail = sk->sk_backlog.tail;
1804 if (!tail)
1805 goto no_coalesce;
1806 thtail = (struct tcphdr *)tail->data;
1807
1808 if (TCP_SKB_CB(tail)->end_seq != TCP_SKB_CB(skb)->seq ||
1809 TCP_SKB_CB(tail)->ip_dsfield != TCP_SKB_CB(skb)->ip_dsfield ||
1810 ((TCP_SKB_CB(tail)->tcp_flags |
1811 TCP_SKB_CB(skb)->tcp_flags) & (TCPHDR_SYN | TCPHDR_RST | TCPHDR_URG)) ||
1812 !((TCP_SKB_CB(tail)->tcp_flags &
1813 TCP_SKB_CB(skb)->tcp_flags) & TCPHDR_ACK) ||
1814 ((TCP_SKB_CB(tail)->tcp_flags ^
1815 TCP_SKB_CB(skb)->tcp_flags) & (TCPHDR_ECE | TCPHDR_CWR)) ||
1816 #ifdef CONFIG_TLS_DEVICE
1817 tail->decrypted != skb->decrypted ||
1818 #endif
1819 thtail->doff != th->doff ||
1820 memcmp(thtail + 1, th + 1, hdrlen - sizeof(*th)))
1821 goto no_coalesce;
1822
1823 __skb_pull(skb, hdrlen);
1824 if (skb_try_coalesce(tail, skb, &fragstolen, &delta)) {
1825 TCP_SKB_CB(tail)->end_seq = TCP_SKB_CB(skb)->end_seq;
1826
1827 if (likely(!before(TCP_SKB_CB(skb)->ack_seq, TCP_SKB_CB(tail)->ack_seq))) {
1828 TCP_SKB_CB(tail)->ack_seq = TCP_SKB_CB(skb)->ack_seq;
1829 thtail->window = th->window;
1830 }
1831
1832 /* We have to update both TCP_SKB_CB(tail)->tcp_flags and
1833 * thtail->fin, so that the fast path in tcp_rcv_established()
1834 * is not entered if we append a packet with a FIN.
1835 * SYN, RST, URG are not present.
1836 * ACK is set on both packets.
1837 * PSH : we do not really care in TCP stack,
1838 * at least for 'GRO' packets.
1839 */
1840 thtail->fin |= th->fin;
1841 TCP_SKB_CB(tail)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags;
1842
1843 if (TCP_SKB_CB(skb)->has_rxtstamp) {
1844 TCP_SKB_CB(tail)->has_rxtstamp = true;
1845 tail->tstamp = skb->tstamp;
1846 skb_hwtstamps(tail)->hwtstamp = skb_hwtstamps(skb)->hwtstamp;
1847 }
1848
1849 /* Not as strict as GRO. We only need to carry mss max value */
1850 skb_shinfo(tail)->gso_size = max(shinfo->gso_size,
1851 skb_shinfo(tail)->gso_size);
1852
1853 gso_segs = skb_shinfo(tail)->gso_segs + shinfo->gso_segs;
1854 skb_shinfo(tail)->gso_segs = min_t(u32, gso_segs, 0xFFFF);
1855
1856 sk->sk_backlog.len += delta;
1857 __NET_INC_STATS(sock_net(sk),
1858 LINUX_MIB_TCPBACKLOGCOALESCE);
1859 kfree_skb_partial(skb, fragstolen);
1860 return false;
1861 }
1862 __skb_push(skb, hdrlen);
1863
1864 no_coalesce:
1865 /* Only socket owner can try to collapse/prune rx queues
1866 * to reduce memory overhead, so add a little headroom here.
1867 * Few sockets backlog are possibly concurrently non empty.
1868 */
1869 limit += 64*1024;
1870
1871 if (unlikely(sk_add_backlog(sk, skb, limit))) {
1872 bh_unlock_sock(sk);
1873 __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPBACKLOGDROP);
1874 return true;
1875 }
1876 return false;
1877 }
1878 EXPORT_SYMBOL(tcp_add_backlog);
1879
1880 int tcp_filter(struct sock *sk, struct sk_buff *skb)
1881 {
1882 struct tcphdr *th = (struct tcphdr *)skb->data;
1883
1884 return sk_filter_trim_cap(sk, skb, th->doff * 4);
1885 }
1886 EXPORT_SYMBOL(tcp_filter);
1887
1888 static void tcp_v4_restore_cb(struct sk_buff *skb)
1889 {
1890 memmove(IPCB(skb), &TCP_SKB_CB(skb)->header.h4,
1891 sizeof(struct inet_skb_parm));
1892 }
1893
1894 static void tcp_v4_fill_cb(struct sk_buff *skb, const struct iphdr *iph,
1895 const struct tcphdr *th)
1896 {
1897 /* This is tricky : We move IPCB at its correct location into TCP_SKB_CB()
1898 * barrier() makes sure compiler wont play fool^Waliasing games.
1899 */
1900 memmove(&TCP_SKB_CB(skb)->header.h4, IPCB(skb),
1901 sizeof(struct inet_skb_parm));
1902 barrier();
1903
1904 TCP_SKB_CB(skb)->seq = ntohl(th->seq);
1905 TCP_SKB_CB(skb)->end_seq = (TCP_SKB_CB(skb)->seq + th->syn + th->fin +
1906 skb->len - th->doff * 4);
1907 TCP_SKB_CB(skb)->ack_seq = ntohl(th->ack_seq);
1908 TCP_SKB_CB(skb)->tcp_flags = tcp_flag_byte(th);
1909 TCP_SKB_CB(skb)->tcp_tw_isn = 0;
1910 TCP_SKB_CB(skb)->ip_dsfield = ipv4_get_dsfield(iph);
1911 TCP_SKB_CB(skb)->sacked = 0;
1912 TCP_SKB_CB(skb)->has_rxtstamp =
1913 skb->tstamp || skb_hwtstamps(skb)->hwtstamp;
1914 }
1915
1916 /*
1917 * From tcp_input.c
1918 */
1919
1920 int tcp_v4_rcv(struct sk_buff *skb)
1921 {
1922 struct net *net = dev_net(skb->dev);
1923 struct sk_buff *skb_to_free;
1924 int sdif = inet_sdif(skb);
1925 int dif = inet_iif(skb);
1926 const struct iphdr *iph;
1927 const struct tcphdr *th;
1928 bool refcounted;
1929 struct sock *sk;
1930 int ret;
1931
1932 if (skb->pkt_type != PACKET_HOST)
1933 goto discard_it;
1934
1935 /* Count it even if it's bad */
1936 __TCP_INC_STATS(net, TCP_MIB_INSEGS);
1937
1938 if (!pskb_may_pull(skb, sizeof(struct tcphdr)))
1939 goto discard_it;
1940
1941 th = (const struct tcphdr *)skb->data;
1942
1943 if (unlikely(th->doff < sizeof(struct tcphdr) / 4))
1944 goto bad_packet;
1945 if (!pskb_may_pull(skb, th->doff * 4))
1946 goto discard_it;
1947
1948 /* An explanation is required here, I think.
1949 * Packet length and doff are validated by header prediction,
1950 * provided case of th->doff==0 is eliminated.
1951 * So, we defer the checks. */
1952
1953 if (skb_checksum_init(skb, IPPROTO_TCP, inet_compute_pseudo))
1954 goto csum_error;
1955
1956 th = (const struct tcphdr *)skb->data;
1957 iph = ip_hdr(skb);
1958 lookup:
1959 sk = __inet_lookup_skb(&tcp_hashinfo, skb, __tcp_hdrlen(th), th->source,
1960 th->dest, sdif, &refcounted);
1961 if (!sk)
1962 goto no_tcp_socket;
1963
1964 process:
1965 if (sk->sk_state == TCP_TIME_WAIT)
1966 goto do_time_wait;
1967
1968 if (sk->sk_state == TCP_NEW_SYN_RECV) {
1969 struct request_sock *req = inet_reqsk(sk);
1970 bool req_stolen = false;
1971 struct sock *nsk;
1972
1973 sk = req->rsk_listener;
1974 if (unlikely(tcp_v4_inbound_md5_hash(sk, skb, dif, sdif))) {
1975 sk_drops_add(sk, skb);
1976 reqsk_put(req);
1977 goto discard_it;
1978 }
1979 if (tcp_checksum_complete(skb)) {
1980 reqsk_put(req);
1981 goto csum_error;
1982 }
1983 if (unlikely(sk->sk_state != TCP_LISTEN)) {
1984 inet_csk_reqsk_queue_drop_and_put(sk, req);
1985 goto lookup;
1986 }
1987 /* We own a reference on the listener, increase it again
1988 * as we might lose it too soon.
1989 */
1990 sock_hold(sk);
1991 refcounted = true;
1992 nsk = NULL;
1993 if (!tcp_filter(sk, skb)) {
1994 th = (const struct tcphdr *)skb->data;
1995 iph = ip_hdr(skb);
1996 tcp_v4_fill_cb(skb, iph, th);
1997 nsk = tcp_check_req(sk, skb, req, false, &req_stolen);
1998 }
1999 if (!nsk) {
2000 reqsk_put(req);
2001 if (req_stolen) {
2002 /* Another cpu got exclusive access to req
2003 * and created a full blown socket.
2004 * Try to feed this packet to this socket
2005 * instead of discarding it.
2006 */
2007 tcp_v4_restore_cb(skb);
2008 sock_put(sk);
2009 goto lookup;
2010 }
2011 goto discard_and_relse;
2012 }
2013 if (nsk == sk) {
2014 reqsk_put(req);
2015 tcp_v4_restore_cb(skb);
2016 } else if (tcp_child_process(sk, nsk, skb)) {
2017 tcp_v4_send_reset(nsk, skb);
2018 goto discard_and_relse;
2019 } else {
2020 sock_put(sk);
2021 return 0;
2022 }
2023 }
2024 if (unlikely(iph->ttl < inet_sk(sk)->min_ttl)) {
2025 __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
2026 goto discard_and_relse;
2027 }
2028
2029 if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb))
2030 goto discard_and_relse;
2031
2032 if (tcp_v4_inbound_md5_hash(sk, skb, dif, sdif))
2033 goto discard_and_relse;
2034
2035 nf_reset_ct(skb);
2036
2037 if (tcp_filter(sk, skb))
2038 goto discard_and_relse;
2039 th = (const struct tcphdr *)skb->data;
2040 iph = ip_hdr(skb);
2041 tcp_v4_fill_cb(skb, iph, th);
2042
2043 skb->dev = NULL;
2044
2045 if (sk->sk_state == TCP_LISTEN) {
2046 ret = tcp_v4_do_rcv(sk, skb);
2047 goto put_and_return;
2048 }
2049
2050 sk_incoming_cpu_update(sk);
2051
2052 bh_lock_sock_nested(sk);
2053 tcp_segs_in(tcp_sk(sk), skb);
2054 ret = 0;
2055 if (!sock_owned_by_user(sk)) {
2056 skb_to_free = sk->sk_rx_skb_cache;
2057 sk->sk_rx_skb_cache = NULL;
2058 ret = tcp_v4_do_rcv(sk, skb);
2059 } else {
2060 if (tcp_add_backlog(sk, skb))
2061 goto discard_and_relse;
2062 skb_to_free = NULL;
2063 }
2064 bh_unlock_sock(sk);
2065 if (skb_to_free)
2066 __kfree_skb(skb_to_free);
2067
2068 put_and_return:
2069 if (refcounted)
2070 sock_put(sk);
2071
2072 return ret;
2073
2074 no_tcp_socket:
2075 if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb))
2076 goto discard_it;
2077
2078 tcp_v4_fill_cb(skb, iph, th);
2079
2080 if (tcp_checksum_complete(skb)) {
2081 csum_error:
2082 __TCP_INC_STATS(net, TCP_MIB_CSUMERRORS);
2083 bad_packet:
2084 __TCP_INC_STATS(net, TCP_MIB_INERRS);
2085 } else {
2086 tcp_v4_send_reset(NULL, skb);
2087 }
2088
2089 discard_it:
2090 /* Discard frame. */
2091 kfree_skb(skb);
2092 return 0;
2093
2094 discard_and_relse:
2095 sk_drops_add(sk, skb);
2096 if (refcounted)
2097 sock_put(sk);
2098 goto discard_it;
2099
2100 do_time_wait:
2101 if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb)) {
2102 inet_twsk_put(inet_twsk(sk));
2103 goto discard_it;
2104 }
2105
2106 tcp_v4_fill_cb(skb, iph, th);
2107
2108 if (tcp_checksum_complete(skb)) {
2109 inet_twsk_put(inet_twsk(sk));
2110 goto csum_error;
2111 }
2112 switch (tcp_timewait_state_process(inet_twsk(sk), skb, th)) {
2113 case TCP_TW_SYN: {
2114 struct sock *sk2 = inet_lookup_listener(dev_net(skb->dev),
2115 &tcp_hashinfo, skb,
2116 __tcp_hdrlen(th),
2117 iph->saddr, th->source,
2118 iph->daddr, th->dest,
2119 inet_iif(skb),
2120 sdif);
2121 if (sk2) {
2122 inet_twsk_deschedule_put(inet_twsk(sk));
2123 sk = sk2;
2124 tcp_v4_restore_cb(skb);
2125 refcounted = false;
2126 goto process;
2127 }
2128 }
2129 /* to ACK */
2130 fallthrough;
2131 case TCP_TW_ACK:
2132 tcp_v4_timewait_ack(sk, skb);
2133 break;
2134 case TCP_TW_RST:
2135 tcp_v4_send_reset(sk, skb);
2136 inet_twsk_deschedule_put(inet_twsk(sk));
2137 goto discard_it;
2138 case TCP_TW_SUCCESS:;
2139 }
2140 goto discard_it;
2141 }
2142
2143 static struct timewait_sock_ops tcp_timewait_sock_ops = {
2144 .twsk_obj_size = sizeof(struct tcp_timewait_sock),
2145 .twsk_unique = tcp_twsk_unique,
2146 .twsk_destructor= tcp_twsk_destructor,
2147 };
2148
2149 void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb)
2150 {
2151 struct dst_entry *dst = skb_dst(skb);
2152
2153 if (dst && dst_hold_safe(dst)) {
2154 sk->sk_rx_dst = dst;
2155 inet_sk(sk)->rx_dst_ifindex = skb->skb_iif;
2156 }
2157 }
2158 EXPORT_SYMBOL(inet_sk_rx_dst_set);
2159
2160 const struct inet_connection_sock_af_ops ipv4_specific = {
2161 .queue_xmit = ip_queue_xmit,
2162 .send_check = tcp_v4_send_check,
2163 .rebuild_header = inet_sk_rebuild_header,
2164 .sk_rx_dst_set = inet_sk_rx_dst_set,
2165 .conn_request = tcp_v4_conn_request,
2166 .syn_recv_sock = tcp_v4_syn_recv_sock,
2167 .net_header_len = sizeof(struct iphdr),
2168 .setsockopt = ip_setsockopt,
2169 .getsockopt = ip_getsockopt,
2170 .addr2sockaddr = inet_csk_addr2sockaddr,
2171 .sockaddr_len = sizeof(struct sockaddr_in),
2172 .mtu_reduced = tcp_v4_mtu_reduced,
2173 };
2174 EXPORT_SYMBOL(ipv4_specific);
2175
2176 #ifdef CONFIG_TCP_MD5SIG
2177 static const struct tcp_sock_af_ops tcp_sock_ipv4_specific = {
2178 .md5_lookup = tcp_v4_md5_lookup,
2179 .calc_md5_hash = tcp_v4_md5_hash_skb,
2180 .md5_parse = tcp_v4_parse_md5_keys,
2181 };
2182 #endif
2183
2184 /* NOTE: A lot of things set to zero explicitly by call to
2185 * sk_alloc() so need not be done here.
2186 */
2187 static int tcp_v4_init_sock(struct sock *sk)
2188 {
2189 struct inet_connection_sock *icsk = inet_csk(sk);
2190
2191 tcp_init_sock(sk);
2192
2193 icsk->icsk_af_ops = &ipv4_specific;
2194
2195 #ifdef CONFIG_TCP_MD5SIG
2196 tcp_sk(sk)->af_specific = &tcp_sock_ipv4_specific;
2197 #endif
2198
2199 return 0;
2200 }
2201
2202 void tcp_v4_destroy_sock(struct sock *sk)
2203 {
2204 struct tcp_sock *tp = tcp_sk(sk);
2205
2206 trace_tcp_destroy_sock(sk);
2207
2208 tcp_clear_xmit_timers(sk);
2209
2210 tcp_cleanup_congestion_control(sk);
2211
2212 tcp_cleanup_ulp(sk);
2213
2214 /* Cleanup up the write buffer. */
2215 tcp_write_queue_purge(sk);
2216
2217 /* Check if we want to disable active TFO */
2218 tcp_fastopen_active_disable_ofo_check(sk);
2219
2220 /* Cleans up our, hopefully empty, out_of_order_queue. */
2221 skb_rbtree_purge(&tp->out_of_order_queue);
2222
2223 #ifdef CONFIG_TCP_MD5SIG
2224 /* Clean up the MD5 key list, if any */
2225 if (tp->md5sig_info) {
2226 tcp_clear_md5_list(sk);
2227 kfree_rcu(rcu_dereference_protected(tp->md5sig_info, 1), rcu);
2228 tp->md5sig_info = NULL;
2229 }
2230 #endif
2231
2232 /* Clean up a referenced TCP bind bucket. */
2233 if (inet_csk(sk)->icsk_bind_hash)
2234 inet_put_port(sk);
2235
2236 BUG_ON(rcu_access_pointer(tp->fastopen_rsk));
2237
2238 /* If socket is aborted during connect operation */
2239 tcp_free_fastopen_req(tp);
2240 tcp_fastopen_destroy_cipher(sk);
2241 tcp_saved_syn_free(tp);
2242
2243 sk_sockets_allocated_dec(sk);
2244 }
2245 EXPORT_SYMBOL(tcp_v4_destroy_sock);
2246
2247 #ifdef CONFIG_PROC_FS
2248 /* Proc filesystem TCP sock list dumping. */
2249
2250 /*
2251 * Get next listener socket follow cur. If cur is NULL, get first socket
2252 * starting from bucket given in st->bucket; when st->bucket is zero the
2253 * very first socket in the hash table is returned.
2254 */
2255 static void *listening_get_next(struct seq_file *seq, void *cur)
2256 {
2257 struct tcp_seq_afinfo *afinfo;
2258 struct tcp_iter_state *st = seq->private;
2259 struct net *net = seq_file_net(seq);
2260 struct inet_listen_hashbucket *ilb;
2261 struct hlist_nulls_node *node;
2262 struct sock *sk = cur;
2263
2264 if (st->bpf_seq_afinfo)
2265 afinfo = st->bpf_seq_afinfo;
2266 else
2267 afinfo = PDE_DATA(file_inode(seq->file));
2268
2269 if (!sk) {
2270 get_head:
2271 ilb = &tcp_hashinfo.listening_hash[st->bucket];
2272 spin_lock(&ilb->lock);
2273 sk = sk_nulls_head(&ilb->nulls_head);
2274 st->offset = 0;
2275 goto get_sk;
2276 }
2277 ilb = &tcp_hashinfo.listening_hash[st->bucket];
2278 ++st->num;
2279 ++st->offset;
2280
2281 sk = sk_nulls_next(sk);
2282 get_sk:
2283 sk_nulls_for_each_from(sk, node) {
2284 if (!net_eq(sock_net(sk), net))
2285 continue;
2286 if (afinfo->family == AF_UNSPEC ||
2287 sk->sk_family == afinfo->family)
2288 return sk;
2289 }
2290 spin_unlock(&ilb->lock);
2291 st->offset = 0;
2292 if (++st->bucket < INET_LHTABLE_SIZE)
2293 goto get_head;
2294 return NULL;
2295 }
2296
2297 static void *listening_get_idx(struct seq_file *seq, loff_t *pos)
2298 {
2299 struct tcp_iter_state *st = seq->private;
2300 void *rc;
2301
2302 st->bucket = 0;
2303 st->offset = 0;
2304 rc = listening_get_next(seq, NULL);
2305
2306 while (rc && *pos) {
2307 rc = listening_get_next(seq, rc);
2308 --*pos;
2309 }
2310 return rc;
2311 }
2312
2313 static inline bool empty_bucket(const struct tcp_iter_state *st)
2314 {
2315 return hlist_nulls_empty(&tcp_hashinfo.ehash[st->bucket].chain);
2316 }
2317
2318 /*
2319 * Get first established socket starting from bucket given in st->bucket.
2320 * If st->bucket is zero, the very first socket in the hash is returned.
2321 */
2322 static void *established_get_first(struct seq_file *seq)
2323 {
2324 struct tcp_seq_afinfo *afinfo;
2325 struct tcp_iter_state *st = seq->private;
2326 struct net *net = seq_file_net(seq);
2327 void *rc = NULL;
2328
2329 if (st->bpf_seq_afinfo)
2330 afinfo = st->bpf_seq_afinfo;
2331 else
2332 afinfo = PDE_DATA(file_inode(seq->file));
2333
2334 st->offset = 0;
2335 for (; st->bucket <= tcp_hashinfo.ehash_mask; ++st->bucket) {
2336 struct sock *sk;
2337 struct hlist_nulls_node *node;
2338 spinlock_t *lock = inet_ehash_lockp(&tcp_hashinfo, st->bucket);
2339
2340 /* Lockless fast path for the common case of empty buckets */
2341 if (empty_bucket(st))
2342 continue;
2343
2344 spin_lock_bh(lock);
2345 sk_nulls_for_each(sk, node, &tcp_hashinfo.ehash[st->bucket].chain) {
2346 if ((afinfo->family != AF_UNSPEC &&
2347 sk->sk_family != afinfo->family) ||
2348 !net_eq(sock_net(sk), net)) {
2349 continue;
2350 }
2351 rc = sk;
2352 goto out;
2353 }
2354 spin_unlock_bh(lock);
2355 }
2356 out:
2357 return rc;
2358 }
2359
2360 static void *established_get_next(struct seq_file *seq, void *cur)
2361 {
2362 struct tcp_seq_afinfo *afinfo;
2363 struct sock *sk = cur;
2364 struct hlist_nulls_node *node;
2365 struct tcp_iter_state *st = seq->private;
2366 struct net *net = seq_file_net(seq);
2367
2368 if (st->bpf_seq_afinfo)
2369 afinfo = st->bpf_seq_afinfo;
2370 else
2371 afinfo = PDE_DATA(file_inode(seq->file));
2372
2373 ++st->num;
2374 ++st->offset;
2375
2376 sk = sk_nulls_next(sk);
2377
2378 sk_nulls_for_each_from(sk, node) {
2379 if ((afinfo->family == AF_UNSPEC ||
2380 sk->sk_family == afinfo->family) &&
2381 net_eq(sock_net(sk), net))
2382 return sk;
2383 }
2384
2385 spin_unlock_bh(inet_ehash_lockp(&tcp_hashinfo, st->bucket));
2386 ++st->bucket;
2387 return established_get_first(seq);
2388 }
2389
2390 static void *established_get_idx(struct seq_file *seq, loff_t pos)
2391 {
2392 struct tcp_iter_state *st = seq->private;
2393 void *rc;
2394
2395 st->bucket = 0;
2396 rc = established_get_first(seq);
2397
2398 while (rc && pos) {
2399 rc = established_get_next(seq, rc);
2400 --pos;
2401 }
2402 return rc;
2403 }
2404
2405 static void *tcp_get_idx(struct seq_file *seq, loff_t pos)
2406 {
2407 void *rc;
2408 struct tcp_iter_state *st = seq->private;
2409
2410 st->state = TCP_SEQ_STATE_LISTENING;
2411 rc = listening_get_idx(seq, &pos);
2412
2413 if (!rc) {
2414 st->state = TCP_SEQ_STATE_ESTABLISHED;
2415 rc = established_get_idx(seq, pos);
2416 }
2417
2418 return rc;
2419 }
2420
2421 static void *tcp_seek_last_pos(struct seq_file *seq)
2422 {
2423 struct tcp_iter_state *st = seq->private;
2424 int offset = st->offset;
2425 int orig_num = st->num;
2426 void *rc = NULL;
2427
2428 switch (st->state) {
2429 case TCP_SEQ_STATE_LISTENING:
2430 if (st->bucket >= INET_LHTABLE_SIZE)
2431 break;
2432 st->state = TCP_SEQ_STATE_LISTENING;
2433 rc = listening_get_next(seq, NULL);
2434 while (offset-- && rc)
2435 rc = listening_get_next(seq, rc);
2436 if (rc)
2437 break;
2438 st->bucket = 0;
2439 st->state = TCP_SEQ_STATE_ESTABLISHED;
2440 fallthrough;
2441 case TCP_SEQ_STATE_ESTABLISHED:
2442 if (st->bucket > tcp_hashinfo.ehash_mask)
2443 break;
2444 rc = established_get_first(seq);
2445 while (offset-- && rc)
2446 rc = established_get_next(seq, rc);
2447 }
2448
2449 st->num = orig_num;
2450
2451 return rc;
2452 }
2453
2454 void *tcp_seq_start(struct seq_file *seq, loff_t *pos)
2455 {
2456 struct tcp_iter_state *st = seq->private;
2457 void *rc;
2458
2459 if (*pos && *pos == st->last_pos) {
2460 rc = tcp_seek_last_pos(seq);
2461 if (rc)
2462 goto out;
2463 }
2464
2465 st->state = TCP_SEQ_STATE_LISTENING;
2466 st->num = 0;
2467 st->bucket = 0;
2468 st->offset = 0;
2469 rc = *pos ? tcp_get_idx(seq, *pos - 1) : SEQ_START_TOKEN;
2470
2471 out:
2472 st->last_pos = *pos;
2473 return rc;
2474 }
2475 EXPORT_SYMBOL(tcp_seq_start);
2476
2477 void *tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
2478 {
2479 struct tcp_iter_state *st = seq->private;
2480 void *rc = NULL;
2481
2482 if (v == SEQ_START_TOKEN) {
2483 rc = tcp_get_idx(seq, 0);
2484 goto out;
2485 }
2486
2487 switch (st->state) {
2488 case TCP_SEQ_STATE_LISTENING:
2489 rc = listening_get_next(seq, v);
2490 if (!rc) {
2491 st->state = TCP_SEQ_STATE_ESTABLISHED;
2492 st->bucket = 0;
2493 st->offset = 0;
2494 rc = established_get_first(seq);
2495 }
2496 break;
2497 case TCP_SEQ_STATE_ESTABLISHED:
2498 rc = established_get_next(seq, v);
2499 break;
2500 }
2501 out:
2502 ++*pos;
2503 st->last_pos = *pos;
2504 return rc;
2505 }
2506 EXPORT_SYMBOL(tcp_seq_next);
2507
2508 void tcp_seq_stop(struct seq_file *seq, void *v)
2509 {
2510 struct tcp_iter_state *st = seq->private;
2511
2512 switch (st->state) {
2513 case TCP_SEQ_STATE_LISTENING:
2514 if (v != SEQ_START_TOKEN)
2515 spin_unlock(&tcp_hashinfo.listening_hash[st->bucket].lock);
2516 break;
2517 case TCP_SEQ_STATE_ESTABLISHED:
2518 if (v)
2519 spin_unlock_bh(inet_ehash_lockp(&tcp_hashinfo, st->bucket));
2520 break;
2521 }
2522 }
2523 EXPORT_SYMBOL(tcp_seq_stop);
2524
2525 static void get_openreq4(const struct request_sock *req,
2526 struct seq_file *f, int i)
2527 {
2528 const struct inet_request_sock *ireq = inet_rsk(req);
2529 long delta = req->rsk_timer.expires - jiffies;
2530
2531 seq_printf(f, "%4d: %08X:%04X %08X:%04X"
2532 " %02X %08X:%08X %02X:%08lX %08X %5u %8d %u %d %pK",
2533 i,
2534 ireq->ir_loc_addr,
2535 ireq->ir_num,
2536 ireq->ir_rmt_addr,
2537 ntohs(ireq->ir_rmt_port),
2538 TCP_SYN_RECV,
2539 0, 0, /* could print option size, but that is af dependent. */
2540 1, /* timers active (only the expire timer) */
2541 jiffies_delta_to_clock_t(delta),
2542 req->num_timeout,
2543 from_kuid_munged(seq_user_ns(f),
2544 sock_i_uid(req->rsk_listener)),
2545 0, /* non standard timer */
2546 0, /* open_requests have no inode */
2547 0,
2548 req);
2549 }
2550
2551 static void get_tcp4_sock(struct sock *sk, struct seq_file *f, int i)
2552 {
2553 int timer_active;
2554 unsigned long timer_expires;
2555 const struct tcp_sock *tp = tcp_sk(sk);
2556 const struct inet_connection_sock *icsk = inet_csk(sk);
2557 const struct inet_sock *inet = inet_sk(sk);
2558 const struct fastopen_queue *fastopenq = &icsk->icsk_accept_queue.fastopenq;
2559 __be32 dest = inet->inet_daddr;
2560 __be32 src = inet->inet_rcv_saddr;
2561 __u16 destp = ntohs(inet->inet_dport);
2562 __u16 srcp = ntohs(inet->inet_sport);
2563 int rx_queue;
2564 int state;
2565
2566 if (icsk->icsk_pending == ICSK_TIME_RETRANS ||
2567 icsk->icsk_pending == ICSK_TIME_REO_TIMEOUT ||
2568 icsk->icsk_pending == ICSK_TIME_LOSS_PROBE) {
2569 timer_active = 1;
2570 timer_expires = icsk->icsk_timeout;
2571 } else if (icsk->icsk_pending == ICSK_TIME_PROBE0) {
2572 timer_active = 4;
2573 timer_expires = icsk->icsk_timeout;
2574 } else if (timer_pending(&sk->sk_timer)) {
2575 timer_active = 2;
2576 timer_expires = sk->sk_timer.expires;
2577 } else {
2578 timer_active = 0;
2579 timer_expires = jiffies;
2580 }
2581
2582 state = inet_sk_state_load(sk);
2583 if (state == TCP_LISTEN)
2584 rx_queue = READ_ONCE(sk->sk_ack_backlog);
2585 else
2586 /* Because we don't lock the socket,
2587 * we might find a transient negative value.
2588 */
2589 rx_queue = max_t(int, READ_ONCE(tp->rcv_nxt) -
2590 READ_ONCE(tp->copied_seq), 0);
2591
2592 seq_printf(f, "%4d: %08X:%04X %08X:%04X %02X %08X:%08X %02X:%08lX "
2593 "%08X %5u %8d %lu %d %pK %lu %lu %u %u %d",
2594 i, src, srcp, dest, destp, state,
2595 READ_ONCE(tp->write_seq) - tp->snd_una,
2596 rx_queue,
2597 timer_active,
2598 jiffies_delta_to_clock_t(timer_expires - jiffies),
2599 icsk->icsk_retransmits,
2600 from_kuid_munged(seq_user_ns(f), sock_i_uid(sk)),
2601 icsk->icsk_probes_out,
2602 sock_i_ino(sk),
2603 refcount_read(&sk->sk_refcnt), sk,
2604 jiffies_to_clock_t(icsk->icsk_rto),
2605 jiffies_to_clock_t(icsk->icsk_ack.ato),
2606 (icsk->icsk_ack.quick << 1) | inet_csk_in_pingpong_mode(sk),
2607 tp->snd_cwnd,
2608 state == TCP_LISTEN ?
2609 fastopenq->max_qlen :
2610 (tcp_in_initial_slowstart(tp) ? -1 : tp->snd_ssthresh));
2611 }
2612
2613 static void get_timewait4_sock(const struct inet_timewait_sock *tw,
2614 struct seq_file *f, int i)
2615 {
2616 long delta = tw->tw_timer.expires - jiffies;
2617 __be32 dest, src;
2618 __u16 destp, srcp;
2619
2620 dest = tw->tw_daddr;
2621 src = tw->tw_rcv_saddr;
2622 destp = ntohs(tw->tw_dport);
2623 srcp = ntohs(tw->tw_sport);
2624
2625 seq_printf(f, "%4d: %08X:%04X %08X:%04X"
2626 " %02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK",
2627 i, src, srcp, dest, destp, tw->tw_substate, 0, 0,
2628 3, jiffies_delta_to_clock_t(delta), 0, 0, 0, 0,
2629 refcount_read(&tw->tw_refcnt), tw);
2630 }
2631
2632 #define TMPSZ 150
2633
2634 static int tcp4_seq_show(struct seq_file *seq, void *v)
2635 {
2636 struct tcp_iter_state *st;
2637 struct sock *sk = v;
2638
2639 seq_setwidth(seq, TMPSZ - 1);
2640 if (v == SEQ_START_TOKEN) {
2641 seq_puts(seq, " sl local_address rem_address st tx_queue "
2642 "rx_queue tr tm->when retrnsmt uid timeout "
2643 "inode");
2644 goto out;
2645 }
2646 st = seq->private;
2647
2648 if (sk->sk_state == TCP_TIME_WAIT)
2649 get_timewait4_sock(v, seq, st->num);
2650 else if (sk->sk_state == TCP_NEW_SYN_RECV)
2651 get_openreq4(v, seq, st->num);
2652 else
2653 get_tcp4_sock(v, seq, st->num);
2654 out:
2655 seq_pad(seq, '\n');
2656 return 0;
2657 }
2658
2659 #ifdef CONFIG_BPF_SYSCALL
2660 struct bpf_iter__tcp {
2661 __bpf_md_ptr(struct bpf_iter_meta *, meta);
2662 __bpf_md_ptr(struct sock_common *, sk_common);
2663 uid_t uid __aligned(8);
2664 };
2665
2666 static int tcp_prog_seq_show(struct bpf_prog *prog, struct bpf_iter_meta *meta,
2667 struct sock_common *sk_common, uid_t uid)
2668 {
2669 struct bpf_iter__tcp ctx;
2670
2671 meta->seq_num--; /* skip SEQ_START_TOKEN */
2672 ctx.meta = meta;
2673 ctx.sk_common = sk_common;
2674 ctx.uid = uid;
2675 return bpf_iter_run_prog(prog, &ctx);
2676 }
2677
2678 static int bpf_iter_tcp_seq_show(struct seq_file *seq, void *v)
2679 {
2680 struct bpf_iter_meta meta;
2681 struct bpf_prog *prog;
2682 struct sock *sk = v;
2683 uid_t uid;
2684
2685 if (v == SEQ_START_TOKEN)
2686 return 0;
2687
2688 if (sk->sk_state == TCP_TIME_WAIT) {
2689 uid = 0;
2690 } else if (sk->sk_state == TCP_NEW_SYN_RECV) {
2691 const struct request_sock *req = v;
2692
2693 uid = from_kuid_munged(seq_user_ns(seq),
2694 sock_i_uid(req->rsk_listener));
2695 } else {
2696 uid = from_kuid_munged(seq_user_ns(seq), sock_i_uid(sk));
2697 }
2698
2699 meta.seq = seq;
2700 prog = bpf_iter_get_info(&meta, false);
2701 return tcp_prog_seq_show(prog, &meta, v, uid);
2702 }
2703
2704 static void bpf_iter_tcp_seq_stop(struct seq_file *seq, void *v)
2705 {
2706 struct bpf_iter_meta meta;
2707 struct bpf_prog *prog;
2708
2709 if (!v) {
2710 meta.seq = seq;
2711 prog = bpf_iter_get_info(&meta, true);
2712 if (prog)
2713 (void)tcp_prog_seq_show(prog, &meta, v, 0);
2714 }
2715
2716 tcp_seq_stop(seq, v);
2717 }
2718
2719 static const struct seq_operations bpf_iter_tcp_seq_ops = {
2720 .show = bpf_iter_tcp_seq_show,
2721 .start = tcp_seq_start,
2722 .next = tcp_seq_next,
2723 .stop = bpf_iter_tcp_seq_stop,
2724 };
2725 #endif
2726
2727 static const struct seq_operations tcp4_seq_ops = {
2728 .show = tcp4_seq_show,
2729 .start = tcp_seq_start,
2730 .next = tcp_seq_next,
2731 .stop = tcp_seq_stop,
2732 };
2733
2734 static struct tcp_seq_afinfo tcp4_seq_afinfo = {
2735 .family = AF_INET,
2736 };
2737
2738 static int __net_init tcp4_proc_init_net(struct net *net)
2739 {
2740 if (!proc_create_net_data("tcp", 0444, net->proc_net, &tcp4_seq_ops,
2741 sizeof(struct tcp_iter_state), &tcp4_seq_afinfo))
2742 return -ENOMEM;
2743 return 0;
2744 }
2745
2746 static void __net_exit tcp4_proc_exit_net(struct net *net)
2747 {
2748 remove_proc_entry("tcp", net->proc_net);
2749 }
2750
2751 static struct pernet_operations tcp4_net_ops = {
2752 .init = tcp4_proc_init_net,
2753 .exit = tcp4_proc_exit_net,
2754 };
2755
2756 int __init tcp4_proc_init(void)
2757 {
2758 return register_pernet_subsys(&tcp4_net_ops);
2759 }
2760
2761 void tcp4_proc_exit(void)
2762 {
2763 unregister_pernet_subsys(&tcp4_net_ops);
2764 }
2765 #endif /* CONFIG_PROC_FS */
2766
2767 /* @wake is one when sk_stream_write_space() calls us.
2768 * This sends EPOLLOUT only if notsent_bytes is half the limit.
2769 * This mimics the strategy used in sock_def_write_space().
2770 */
2771 bool tcp_stream_memory_free(const struct sock *sk, int wake)
2772 {
2773 const struct tcp_sock *tp = tcp_sk(sk);
2774 u32 notsent_bytes = READ_ONCE(tp->write_seq) -
2775 READ_ONCE(tp->snd_nxt);
2776
2777 return (notsent_bytes << wake) < tcp_notsent_lowat(tp);
2778 }
2779 EXPORT_SYMBOL(tcp_stream_memory_free);
2780
2781 struct proto tcp_prot = {
2782 .name = "TCP",
2783 .owner = THIS_MODULE,
2784 .close = tcp_close,
2785 .pre_connect = tcp_v4_pre_connect,
2786 .connect = tcp_v4_connect,
2787 .disconnect = tcp_disconnect,
2788 .accept = inet_csk_accept,
2789 .ioctl = tcp_ioctl,
2790 .init = tcp_v4_init_sock,
2791 .destroy = tcp_v4_destroy_sock,
2792 .shutdown = tcp_shutdown,
2793 .setsockopt = tcp_setsockopt,
2794 .getsockopt = tcp_getsockopt,
2795 .keepalive = tcp_set_keepalive,
2796 .recvmsg = tcp_recvmsg,
2797 .sendmsg = tcp_sendmsg,
2798 .sendpage = tcp_sendpage,
2799 .backlog_rcv = tcp_v4_do_rcv,
2800 .release_cb = tcp_release_cb,
2801 .hash = inet_hash,
2802 .unhash = inet_unhash,
2803 .get_port = inet_csk_get_port,
2804 .enter_memory_pressure = tcp_enter_memory_pressure,
2805 .leave_memory_pressure = tcp_leave_memory_pressure,
2806 .stream_memory_free = tcp_stream_memory_free,
2807 .sockets_allocated = &tcp_sockets_allocated,
2808 .orphan_count = &tcp_orphan_count,
2809 .memory_allocated = &tcp_memory_allocated,
2810 .memory_pressure = &tcp_memory_pressure,
2811 .sysctl_mem = sysctl_tcp_mem,
2812 .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_tcp_wmem),
2813 .sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_tcp_rmem),
2814 .max_header = MAX_TCP_HEADER,
2815 .obj_size = sizeof(struct tcp_sock),
2816 .slab_flags = SLAB_TYPESAFE_BY_RCU,
2817 .twsk_prot = &tcp_timewait_sock_ops,
2818 .rsk_prot = &tcp_request_sock_ops,
2819 .h.hashinfo = &tcp_hashinfo,
2820 .no_autobind = true,
2821 .diag_destroy = tcp_abort,
2822 };
2823 EXPORT_SYMBOL(tcp_prot);
2824
2825 static void __net_exit tcp_sk_exit(struct net *net)
2826 {
2827 int cpu;
2828
2829 if (net->ipv4.tcp_congestion_control)
2830 bpf_module_put(net->ipv4.tcp_congestion_control,
2831 net->ipv4.tcp_congestion_control->owner);
2832
2833 for_each_possible_cpu(cpu)
2834 inet_ctl_sock_destroy(*per_cpu_ptr(net->ipv4.tcp_sk, cpu));
2835 free_percpu(net->ipv4.tcp_sk);
2836 }
2837
2838 static int __net_init tcp_sk_init(struct net *net)
2839 {
2840 int res, cpu, cnt;
2841
2842 net->ipv4.tcp_sk = alloc_percpu(struct sock *);
2843 if (!net->ipv4.tcp_sk)
2844 return -ENOMEM;
2845
2846 for_each_possible_cpu(cpu) {
2847 struct sock *sk;
2848
2849 res = inet_ctl_sock_create(&sk, PF_INET, SOCK_RAW,
2850 IPPROTO_TCP, net);
2851 if (res)
2852 goto fail;
2853 sock_set_flag(sk, SOCK_USE_WRITE_QUEUE);
2854
2855 /* Please enforce IP_DF and IPID==0 for RST and
2856 * ACK sent in SYN-RECV and TIME-WAIT state.
2857 */
2858 inet_sk(sk)->pmtudisc = IP_PMTUDISC_DO;
2859
2860 *per_cpu_ptr(net->ipv4.tcp_sk, cpu) = sk;
2861 }
2862
2863 net->ipv4.sysctl_tcp_ecn = 2;
2864 net->ipv4.sysctl_tcp_ecn_fallback = 1;
2865
2866 net->ipv4.sysctl_tcp_base_mss = TCP_BASE_MSS;
2867 net->ipv4.sysctl_tcp_min_snd_mss = TCP_MIN_SND_MSS;
2868 net->ipv4.sysctl_tcp_probe_threshold = TCP_PROBE_THRESHOLD;
2869 net->ipv4.sysctl_tcp_probe_interval = TCP_PROBE_INTERVAL;
2870 net->ipv4.sysctl_tcp_mtu_probe_floor = TCP_MIN_SND_MSS;
2871
2872 net->ipv4.sysctl_tcp_keepalive_time = TCP_KEEPALIVE_TIME;
2873 net->ipv4.sysctl_tcp_keepalive_probes = TCP_KEEPALIVE_PROBES;
2874 net->ipv4.sysctl_tcp_keepalive_intvl = TCP_KEEPALIVE_INTVL;
2875
2876 net->ipv4.sysctl_tcp_syn_retries = TCP_SYN_RETRIES;
2877 net->ipv4.sysctl_tcp_synack_retries = TCP_SYNACK_RETRIES;
2878 net->ipv4.sysctl_tcp_syncookies = 1;
2879 net->ipv4.sysctl_tcp_reordering = TCP_FASTRETRANS_THRESH;
2880 net->ipv4.sysctl_tcp_retries1 = TCP_RETR1;
2881 net->ipv4.sysctl_tcp_retries2 = TCP_RETR2;
2882 net->ipv4.sysctl_tcp_orphan_retries = 0;
2883 net->ipv4.sysctl_tcp_fin_timeout = TCP_FIN_TIMEOUT;
2884 net->ipv4.sysctl_tcp_notsent_lowat = UINT_MAX;
2885 net->ipv4.sysctl_tcp_tw_reuse = 2;
2886 net->ipv4.sysctl_tcp_no_ssthresh_metrics_save = 1;
2887
2888 cnt = tcp_hashinfo.ehash_mask + 1;
2889 net->ipv4.tcp_death_row.sysctl_max_tw_buckets = cnt / 2;
2890 net->ipv4.tcp_death_row.hashinfo = &tcp_hashinfo;
2891
2892 net->ipv4.sysctl_max_syn_backlog = max(128, cnt / 128);
2893 net->ipv4.sysctl_tcp_sack = 1;
2894 net->ipv4.sysctl_tcp_window_scaling = 1;
2895 net->ipv4.sysctl_tcp_timestamps = 1;
2896 net->ipv4.sysctl_tcp_early_retrans = 3;
2897 net->ipv4.sysctl_tcp_recovery = TCP_RACK_LOSS_DETECTION;
2898 net->ipv4.sysctl_tcp_slow_start_after_idle = 1; /* By default, RFC2861 behavior. */
2899 net->ipv4.sysctl_tcp_retrans_collapse = 1;
2900 net->ipv4.sysctl_tcp_max_reordering = 300;
2901 net->ipv4.sysctl_tcp_dsack = 1;
2902 net->ipv4.sysctl_tcp_app_win = 31;
2903 net->ipv4.sysctl_tcp_adv_win_scale = 1;
2904 net->ipv4.sysctl_tcp_frto = 2;
2905 net->ipv4.sysctl_tcp_moderate_rcvbuf = 1;
2906 /* This limits the percentage of the congestion window which we
2907 * will allow a single TSO frame to consume. Building TSO frames
2908 * which are too large can cause TCP streams to be bursty.
2909 */
2910 net->ipv4.sysctl_tcp_tso_win_divisor = 3;
2911 /* Default TSQ limit of 16 TSO segments */
2912 net->ipv4.sysctl_tcp_limit_output_bytes = 16 * 65536;
2913 /* rfc5961 challenge ack rate limiting */
2914 net->ipv4.sysctl_tcp_challenge_ack_limit = 1000;
2915 net->ipv4.sysctl_tcp_min_tso_segs = 2;
2916 net->ipv4.sysctl_tcp_min_rtt_wlen = 300;
2917 net->ipv4.sysctl_tcp_autocorking = 1;
2918 net->ipv4.sysctl_tcp_invalid_ratelimit = HZ/2;
2919 net->ipv4.sysctl_tcp_pacing_ss_ratio = 200;
2920 net->ipv4.sysctl_tcp_pacing_ca_ratio = 120;
2921 if (net != &init_net) {
2922 memcpy(net->ipv4.sysctl_tcp_rmem,
2923 init_net.ipv4.sysctl_tcp_rmem,
2924 sizeof(init_net.ipv4.sysctl_tcp_rmem));
2925 memcpy(net->ipv4.sysctl_tcp_wmem,
2926 init_net.ipv4.sysctl_tcp_wmem,
2927 sizeof(init_net.ipv4.sysctl_tcp_wmem));
2928 }
2929 net->ipv4.sysctl_tcp_comp_sack_delay_ns = NSEC_PER_MSEC;
2930 net->ipv4.sysctl_tcp_comp_sack_slack_ns = 100 * NSEC_PER_USEC;
2931 net->ipv4.sysctl_tcp_comp_sack_nr = 44;
2932 net->ipv4.sysctl_tcp_fastopen = TFO_CLIENT_ENABLE;
2933 spin_lock_init(&net->ipv4.tcp_fastopen_ctx_lock);
2934 net->ipv4.sysctl_tcp_fastopen_blackhole_timeout = 60 * 60;
2935 atomic_set(&net->ipv4.tfo_active_disable_times, 0);
2936
2937 /* Reno is always built in */
2938 if (!net_eq(net, &init_net) &&
2939 bpf_try_module_get(init_net.ipv4.tcp_congestion_control,
2940 init_net.ipv4.tcp_congestion_control->owner))
2941 net->ipv4.tcp_congestion_control = init_net.ipv4.tcp_congestion_control;
2942 else
2943 net->ipv4.tcp_congestion_control = &tcp_reno;
2944
2945 return 0;
2946 fail:
2947 tcp_sk_exit(net);
2948
2949 return res;
2950 }
2951
2952 static void __net_exit tcp_sk_exit_batch(struct list_head *net_exit_list)
2953 {
2954 struct net *net;
2955
2956 inet_twsk_purge(&tcp_hashinfo, AF_INET);
2957
2958 list_for_each_entry(net, net_exit_list, exit_list)
2959 tcp_fastopen_ctx_destroy(net);
2960 }
2961
2962 static struct pernet_operations __net_initdata tcp_sk_ops = {
2963 .init = tcp_sk_init,
2964 .exit = tcp_sk_exit,
2965 .exit_batch = tcp_sk_exit_batch,
2966 };
2967
2968 #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
2969 DEFINE_BPF_ITER_FUNC(tcp, struct bpf_iter_meta *meta,
2970 struct sock_common *sk_common, uid_t uid)
2971
2972 static int bpf_iter_init_tcp(void *priv_data, struct bpf_iter_aux_info *aux)
2973 {
2974 struct tcp_iter_state *st = priv_data;
2975 struct tcp_seq_afinfo *afinfo;
2976 int ret;
2977
2978 afinfo = kmalloc(sizeof(*afinfo), GFP_USER | __GFP_NOWARN);
2979 if (!afinfo)
2980 return -ENOMEM;
2981
2982 afinfo->family = AF_UNSPEC;
2983 st->bpf_seq_afinfo = afinfo;
2984 ret = bpf_iter_init_seq_net(priv_data, aux);
2985 if (ret)
2986 kfree(afinfo);
2987 return ret;
2988 }
2989
2990 static void bpf_iter_fini_tcp(void *priv_data)
2991 {
2992 struct tcp_iter_state *st = priv_data;
2993
2994 kfree(st->bpf_seq_afinfo);
2995 bpf_iter_fini_seq_net(priv_data);
2996 }
2997
2998 static const struct bpf_iter_seq_info tcp_seq_info = {
2999 .seq_ops = &bpf_iter_tcp_seq_ops,
3000 .init_seq_private = bpf_iter_init_tcp,
3001 .fini_seq_private = bpf_iter_fini_tcp,
3002 .seq_priv_size = sizeof(struct tcp_iter_state),
3003 };
3004
3005 static struct bpf_iter_reg tcp_reg_info = {
3006 .target = "tcp",
3007 .ctx_arg_info_size = 1,
3008 .ctx_arg_info = {
3009 { offsetof(struct bpf_iter__tcp, sk_common),
3010 PTR_TO_BTF_ID_OR_NULL },
3011 },
3012 .seq_info = &tcp_seq_info,
3013 };
3014
3015 static void __init bpf_iter_register(void)
3016 {
3017 tcp_reg_info.ctx_arg_info[0].btf_id = btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON];
3018 if (bpf_iter_reg_target(&tcp_reg_info))
3019 pr_warn("Warning: could not register bpf iterator tcp\n");
3020 }
3021
3022 #endif
3023
3024 void __init tcp_v4_init(void)
3025 {
3026 if (register_pernet_subsys(&tcp_sk_ops))
3027 panic("Failed to create the TCP control socket.\n");
3028
3029 #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
3030 bpf_iter_register();
3031 #endif
3032 }
Linux with added FPC-III support