2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
6 * Generic INET transport hashtables
8 * Authors: Lotsa people, from code originally in tcp
10 * This program is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU General Public License
12 * as published by the Free Software Foundation; either version
13 * 2 of the License, or (at your option) any later version.
16 #include <linux/module.h>
17 #include <linux/random.h>
18 #include <linux/sched.h>
19 #include <linux/slab.h>
20 #include <linux/wait.h>
21 #include <linux/vmalloc.h>
22 #include <linux/bootmem.h>
24 #include <net/addrconf.h>
25 #include <net/inet_connection_sock.h>
26 #include <net/inet_hashtables.h>
27 #include <net/secure_seq.h>
30 #include <net/sock_reuseport.h>
32 static u32
inet_ehashfn(const struct net
*net
, const __be32 laddr
,
33 const __u16 lport
, const __be32 faddr
,
36 static u32 inet_ehash_secret __read_mostly
;
38 net_get_random_once(&inet_ehash_secret
, sizeof(inet_ehash_secret
));
40 return __inet_ehashfn(laddr
, lport
, faddr
, fport
,
41 inet_ehash_secret
+ net_hash_mix(net
));
44 /* This function handles inet_sock, but also timewait and request sockets
47 static u32
sk_ehashfn(const struct sock
*sk
)
49 #if IS_ENABLED(CONFIG_IPV6)
50 if (sk
->sk_family
== AF_INET6
&&
51 !ipv6_addr_v4mapped(&sk
->sk_v6_daddr
))
52 return inet6_ehashfn(sock_net(sk
),
53 &sk
->sk_v6_rcv_saddr
, sk
->sk_num
,
54 &sk
->sk_v6_daddr
, sk
->sk_dport
);
56 return inet_ehashfn(sock_net(sk
),
57 sk
->sk_rcv_saddr
, sk
->sk_num
,
58 sk
->sk_daddr
, sk
->sk_dport
);
62 * Allocate and initialize a new local port bind bucket.
63 * The bindhash mutex for snum's hash chain must be held here.
65 struct inet_bind_bucket
*inet_bind_bucket_create(struct kmem_cache
*cachep
,
67 struct inet_bind_hashbucket
*head
,
68 const unsigned short snum
)
70 struct inet_bind_bucket
*tb
= kmem_cache_alloc(cachep
, GFP_ATOMIC
);
73 write_pnet(&tb
->ib_net
, net
);
76 tb
->fastreuseport
= 0;
77 INIT_HLIST_HEAD(&tb
->owners
);
78 hlist_add_head(&tb
->node
, &head
->chain
);
84 * Caller must hold hashbucket lock for this tb with local BH disabled
86 void inet_bind_bucket_destroy(struct kmem_cache
*cachep
, struct inet_bind_bucket
*tb
)
88 if (hlist_empty(&tb
->owners
)) {
89 __hlist_del(&tb
->node
);
90 kmem_cache_free(cachep
, tb
);
94 void inet_bind_hash(struct sock
*sk
, struct inet_bind_bucket
*tb
,
95 const unsigned short snum
)
97 inet_sk(sk
)->inet_num
= snum
;
98 sk_add_bind_node(sk
, &tb
->owners
);
99 inet_csk(sk
)->icsk_bind_hash
= tb
;
103 * Get rid of any references to a local port held by the given sock.
105 static void __inet_put_port(struct sock
*sk
)
107 struct inet_hashinfo
*hashinfo
= sk
->sk_prot
->h
.hashinfo
;
108 const int bhash
= inet_bhashfn(sock_net(sk
), inet_sk(sk
)->inet_num
,
109 hashinfo
->bhash_size
);
110 struct inet_bind_hashbucket
*head
= &hashinfo
->bhash
[bhash
];
111 struct inet_bind_bucket
*tb
;
113 spin_lock(&head
->lock
);
114 tb
= inet_csk(sk
)->icsk_bind_hash
;
115 __sk_del_bind_node(sk
);
116 inet_csk(sk
)->icsk_bind_hash
= NULL
;
117 inet_sk(sk
)->inet_num
= 0;
118 inet_bind_bucket_destroy(hashinfo
->bind_bucket_cachep
, tb
);
119 spin_unlock(&head
->lock
);
122 void inet_put_port(struct sock
*sk
)
128 EXPORT_SYMBOL(inet_put_port
);
130 int __inet_inherit_port(const struct sock
*sk
, struct sock
*child
)
132 struct inet_hashinfo
*table
= sk
->sk_prot
->h
.hashinfo
;
133 unsigned short port
= inet_sk(child
)->inet_num
;
134 const int bhash
= inet_bhashfn(sock_net(sk
), port
,
136 struct inet_bind_hashbucket
*head
= &table
->bhash
[bhash
];
137 struct inet_bind_bucket
*tb
;
139 spin_lock(&head
->lock
);
140 tb
= inet_csk(sk
)->icsk_bind_hash
;
142 spin_unlock(&head
->lock
);
145 if (tb
->port
!= port
) {
146 /* NOTE: using tproxy and redirecting skbs to a proxy
147 * on a different listener port breaks the assumption
148 * that the listener socket's icsk_bind_hash is the same
149 * as that of the child socket. We have to look up or
150 * create a new bind bucket for the child here. */
151 inet_bind_bucket_for_each(tb
, &head
->chain
) {
152 if (net_eq(ib_net(tb
), sock_net(sk
)) &&
157 tb
= inet_bind_bucket_create(table
->bind_bucket_cachep
,
158 sock_net(sk
), head
, port
);
160 spin_unlock(&head
->lock
);
165 inet_bind_hash(child
, tb
, port
);
166 spin_unlock(&head
->lock
);
170 EXPORT_SYMBOL_GPL(__inet_inherit_port
);
172 static struct inet_listen_hashbucket
*
173 inet_lhash2_bucket_sk(struct inet_hashinfo
*h
, struct sock
*sk
)
177 #if IS_ENABLED(CONFIG_IPV6)
178 if (sk
->sk_family
== AF_INET6
)
179 hash
= ipv6_portaddr_hash(sock_net(sk
),
180 &sk
->sk_v6_rcv_saddr
,
181 inet_sk(sk
)->inet_num
);
184 hash
= ipv4_portaddr_hash(sock_net(sk
),
185 inet_sk(sk
)->inet_rcv_saddr
,
186 inet_sk(sk
)->inet_num
);
187 return inet_lhash2_bucket(h
, hash
);
190 static void inet_hash2(struct inet_hashinfo
*h
, struct sock
*sk
)
192 struct inet_listen_hashbucket
*ilb2
;
197 ilb2
= inet_lhash2_bucket_sk(h
, sk
);
199 spin_lock(&ilb2
->lock
);
200 if (sk
->sk_reuseport
&& sk
->sk_family
== AF_INET6
)
201 hlist_add_tail_rcu(&inet_csk(sk
)->icsk_listen_portaddr_node
,
204 hlist_add_head_rcu(&inet_csk(sk
)->icsk_listen_portaddr_node
,
207 spin_unlock(&ilb2
->lock
);
210 static void inet_unhash2(struct inet_hashinfo
*h
, struct sock
*sk
)
212 struct inet_listen_hashbucket
*ilb2
;
215 WARN_ON_ONCE(hlist_unhashed(&inet_csk(sk
)->icsk_listen_portaddr_node
)))
218 ilb2
= inet_lhash2_bucket_sk(h
, sk
);
220 spin_lock(&ilb2
->lock
);
221 hlist_del_init_rcu(&inet_csk(sk
)->icsk_listen_portaddr_node
);
223 spin_unlock(&ilb2
->lock
);
226 static inline int compute_score(struct sock
*sk
, struct net
*net
,
227 const unsigned short hnum
, const __be32 daddr
,
228 const int dif
, const int sdif
, bool exact_dif
)
231 struct inet_sock
*inet
= inet_sk(sk
);
233 if (net_eq(sock_net(sk
), net
) && inet
->inet_num
== hnum
&&
234 !ipv6_only_sock(sk
)) {
235 __be32 rcv_saddr
= inet
->inet_rcv_saddr
;
236 score
= sk
->sk_family
== PF_INET
? 2 : 1;
238 if (rcv_saddr
!= daddr
)
242 if (sk
->sk_bound_dev_if
|| exact_dif
) {
243 bool dev_match
= (sk
->sk_bound_dev_if
== dif
||
244 sk
->sk_bound_dev_if
== sdif
);
248 if (sk
->sk_bound_dev_if
)
251 if (sk
->sk_incoming_cpu
== raw_smp_processor_id())
258 * Here are some nice properties to exploit here. The BSD API
259 * does not allow a listening sock to specify the remote port nor the
260 * remote address for the connection. So always assume those are both
261 * wildcarded during the search since they can never be otherwise.
264 /* called with rcu_read_lock() : No refcount taken on the socket */
265 static struct sock
*inet_lhash2_lookup(struct net
*net
,
266 struct inet_listen_hashbucket
*ilb2
,
267 struct sk_buff
*skb
, int doff
,
268 const __be32 saddr
, __be16 sport
,
269 const __be32 daddr
, const unsigned short hnum
,
270 const int dif
, const int sdif
)
272 bool exact_dif
= inet_exact_dif_match(net
, skb
);
273 struct inet_connection_sock
*icsk
;
274 struct sock
*sk
, *result
= NULL
;
275 int score
, hiscore
= 0;
278 inet_lhash2_for_each_icsk_rcu(icsk
, &ilb2
->head
) {
279 sk
= (struct sock
*)icsk
;
280 score
= compute_score(sk
, net
, hnum
, daddr
,
281 dif
, sdif
, exact_dif
);
282 if (score
> hiscore
) {
283 if (sk
->sk_reuseport
) {
284 phash
= inet_ehashfn(net
, daddr
, hnum
,
286 result
= reuseport_select_sock(sk
, phash
,
299 struct sock
*__inet_lookup_listener(struct net
*net
,
300 struct inet_hashinfo
*hashinfo
,
301 struct sk_buff
*skb
, int doff
,
302 const __be32 saddr
, __be16 sport
,
303 const __be32 daddr
, const unsigned short hnum
,
304 const int dif
, const int sdif
)
306 unsigned int hash
= inet_lhashfn(net
, hnum
);
307 struct inet_listen_hashbucket
*ilb
= &hashinfo
->listening_hash
[hash
];
308 bool exact_dif
= inet_exact_dif_match(net
, skb
);
309 struct inet_listen_hashbucket
*ilb2
;
310 struct sock
*sk
, *result
= NULL
;
311 int score
, hiscore
= 0;
315 if (ilb
->count
<= 10 || !hashinfo
->lhash2
)
318 /* Too many sk in the ilb bucket (which is hashed by port alone).
319 * Try lhash2 (which is hashed by port and addr) instead.
322 hash2
= ipv4_portaddr_hash(net
, daddr
, hnum
);
323 ilb2
= inet_lhash2_bucket(hashinfo
, hash2
);
324 if (ilb2
->count
> ilb
->count
)
327 result
= inet_lhash2_lookup(net
, ilb2
, skb
, doff
,
328 saddr
, sport
, daddr
, hnum
,
333 /* Lookup lhash2 with INADDR_ANY */
335 hash2
= ipv4_portaddr_hash(net
, htonl(INADDR_ANY
), hnum
);
336 ilb2
= inet_lhash2_bucket(hashinfo
, hash2
);
337 if (ilb2
->count
> ilb
->count
)
340 result
= inet_lhash2_lookup(net
, ilb2
, skb
, doff
,
341 saddr
, sport
, daddr
, hnum
,
346 sk_for_each_rcu(sk
, &ilb
->head
) {
347 score
= compute_score(sk
, net
, hnum
, daddr
,
348 dif
, sdif
, exact_dif
);
349 if (score
> hiscore
) {
350 if (sk
->sk_reuseport
) {
351 phash
= inet_ehashfn(net
, daddr
, hnum
,
353 result
= reuseport_select_sock(sk
, phash
,
363 if (unlikely(IS_ERR(result
)))
367 EXPORT_SYMBOL_GPL(__inet_lookup_listener
);
369 /* All sockets share common refcount, but have different destructors */
370 void sock_gen_put(struct sock
*sk
)
372 if (!refcount_dec_and_test(&sk
->sk_refcnt
))
375 if (sk
->sk_state
== TCP_TIME_WAIT
)
376 inet_twsk_free(inet_twsk(sk
));
377 else if (sk
->sk_state
== TCP_NEW_SYN_RECV
)
378 reqsk_free(inet_reqsk(sk
));
382 EXPORT_SYMBOL_GPL(sock_gen_put
);
384 void sock_edemux(struct sk_buff
*skb
)
386 sock_gen_put(skb
->sk
);
388 EXPORT_SYMBOL(sock_edemux
);
390 struct sock
*__inet_lookup_established(struct net
*net
,
391 struct inet_hashinfo
*hashinfo
,
392 const __be32 saddr
, const __be16 sport
,
393 const __be32 daddr
, const u16 hnum
,
394 const int dif
, const int sdif
)
396 INET_ADDR_COOKIE(acookie
, saddr
, daddr
);
397 const __portpair ports
= INET_COMBINED_PORTS(sport
, hnum
);
399 const struct hlist_nulls_node
*node
;
400 /* Optimize here for direct hit, only listening connections can
401 * have wildcards anyways.
403 unsigned int hash
= inet_ehashfn(net
, daddr
, hnum
, saddr
, sport
);
404 unsigned int slot
= hash
& hashinfo
->ehash_mask
;
405 struct inet_ehash_bucket
*head
= &hashinfo
->ehash
[slot
];
408 sk_nulls_for_each_rcu(sk
, node
, &head
->chain
) {
409 if (sk
->sk_hash
!= hash
)
411 if (likely(INET_MATCH(sk
, net
, acookie
,
412 saddr
, daddr
, ports
, dif
, sdif
))) {
413 if (unlikely(!refcount_inc_not_zero(&sk
->sk_refcnt
)))
415 if (unlikely(!INET_MATCH(sk
, net
, acookie
,
425 * if the nulls value we got at the end of this lookup is
426 * not the expected one, we must restart lookup.
427 * We probably met an item that was moved to another chain.
429 if (get_nulls_value(node
) != slot
)
436 EXPORT_SYMBOL_GPL(__inet_lookup_established
);
438 /* called with local bh disabled */
439 static int __inet_check_established(struct inet_timewait_death_row
*death_row
,
440 struct sock
*sk
, __u16 lport
,
441 struct inet_timewait_sock
**twp
)
443 struct inet_hashinfo
*hinfo
= death_row
->hashinfo
;
444 struct inet_sock
*inet
= inet_sk(sk
);
445 __be32 daddr
= inet
->inet_rcv_saddr
;
446 __be32 saddr
= inet
->inet_daddr
;
447 int dif
= sk
->sk_bound_dev_if
;
448 struct net
*net
= sock_net(sk
);
449 int sdif
= l3mdev_master_ifindex_by_index(net
, dif
);
450 INET_ADDR_COOKIE(acookie
, saddr
, daddr
);
451 const __portpair ports
= INET_COMBINED_PORTS(inet
->inet_dport
, lport
);
452 unsigned int hash
= inet_ehashfn(net
, daddr
, lport
,
453 saddr
, inet
->inet_dport
);
454 struct inet_ehash_bucket
*head
= inet_ehash_bucket(hinfo
, hash
);
455 spinlock_t
*lock
= inet_ehash_lockp(hinfo
, hash
);
457 const struct hlist_nulls_node
*node
;
458 struct inet_timewait_sock
*tw
= NULL
;
462 sk_nulls_for_each(sk2
, node
, &head
->chain
) {
463 if (sk2
->sk_hash
!= hash
)
466 if (likely(INET_MATCH(sk2
, net
, acookie
,
467 saddr
, daddr
, ports
, dif
, sdif
))) {
468 if (sk2
->sk_state
== TCP_TIME_WAIT
) {
470 if (twsk_unique(sk
, sk2
, twp
))
477 /* Must record num and sport now. Otherwise we will see
478 * in hash table socket with a funny identity.
480 inet
->inet_num
= lport
;
481 inet
->inet_sport
= htons(lport
);
483 WARN_ON(!sk_unhashed(sk
));
484 __sk_nulls_add_node_rcu(sk
, &head
->chain
);
486 sk_nulls_del_node_init_rcu((struct sock
*)tw
);
487 __NET_INC_STATS(net
, LINUX_MIB_TIMEWAITRECYCLED
);
490 sock_prot_inuse_add(sock_net(sk
), sk
->sk_prot
, 1);
495 /* Silly. Should hash-dance instead... */
496 inet_twsk_deschedule_put(tw
);
502 return -EADDRNOTAVAIL
;
505 static u32
inet_sk_port_offset(const struct sock
*sk
)
507 const struct inet_sock
*inet
= inet_sk(sk
);
509 return secure_ipv4_port_ephemeral(inet
->inet_rcv_saddr
,
514 /* insert a socket into ehash, and eventually remove another one
515 * (The another one can be a SYN_RECV or TIMEWAIT
517 bool inet_ehash_insert(struct sock
*sk
, struct sock
*osk
)
519 struct inet_hashinfo
*hashinfo
= sk
->sk_prot
->h
.hashinfo
;
520 struct hlist_nulls_head
*list
;
521 struct inet_ehash_bucket
*head
;
525 WARN_ON_ONCE(!sk_unhashed(sk
));
527 sk
->sk_hash
= sk_ehashfn(sk
);
528 head
= inet_ehash_bucket(hashinfo
, sk
->sk_hash
);
530 lock
= inet_ehash_lockp(hashinfo
, sk
->sk_hash
);
534 WARN_ON_ONCE(sk
->sk_hash
!= osk
->sk_hash
);
535 ret
= sk_nulls_del_node_init_rcu(osk
);
538 __sk_nulls_add_node_rcu(sk
, list
);
543 bool inet_ehash_nolisten(struct sock
*sk
, struct sock
*osk
)
545 bool ok
= inet_ehash_insert(sk
, osk
);
548 sock_prot_inuse_add(sock_net(sk
), sk
->sk_prot
, 1);
550 percpu_counter_inc(sk
->sk_prot
->orphan_count
);
551 inet_sk_set_state(sk
, TCP_CLOSE
);
552 sock_set_flag(sk
, SOCK_DEAD
);
553 inet_csk_destroy_sock(sk
);
557 EXPORT_SYMBOL_GPL(inet_ehash_nolisten
);
559 static int inet_reuseport_add_sock(struct sock
*sk
,
560 struct inet_listen_hashbucket
*ilb
)
562 struct inet_bind_bucket
*tb
= inet_csk(sk
)->icsk_bind_hash
;
564 kuid_t uid
= sock_i_uid(sk
);
566 sk_for_each_rcu(sk2
, &ilb
->head
) {
568 sk2
->sk_family
== sk
->sk_family
&&
569 ipv6_only_sock(sk2
) == ipv6_only_sock(sk
) &&
570 sk2
->sk_bound_dev_if
== sk
->sk_bound_dev_if
&&
571 inet_csk(sk2
)->icsk_bind_hash
== tb
&&
572 sk2
->sk_reuseport
&& uid_eq(uid
, sock_i_uid(sk2
)) &&
573 inet_rcv_saddr_equal(sk
, sk2
, false))
574 return reuseport_add_sock(sk
, sk2
,
575 inet_rcv_saddr_any(sk
));
578 return reuseport_alloc(sk
, inet_rcv_saddr_any(sk
));
581 int __inet_hash(struct sock
*sk
, struct sock
*osk
)
583 struct inet_hashinfo
*hashinfo
= sk
->sk_prot
->h
.hashinfo
;
584 struct inet_listen_hashbucket
*ilb
;
587 if (sk
->sk_state
!= TCP_LISTEN
) {
588 inet_ehash_nolisten(sk
, osk
);
591 WARN_ON(!sk_unhashed(sk
));
592 ilb
= &hashinfo
->listening_hash
[inet_sk_listen_hashfn(sk
)];
594 spin_lock(&ilb
->lock
);
595 if (sk
->sk_reuseport
) {
596 err
= inet_reuseport_add_sock(sk
, ilb
);
600 if (IS_ENABLED(CONFIG_IPV6
) && sk
->sk_reuseport
&&
601 sk
->sk_family
== AF_INET6
)
602 hlist_add_tail_rcu(&sk
->sk_node
, &ilb
->head
);
604 hlist_add_head_rcu(&sk
->sk_node
, &ilb
->head
);
605 inet_hash2(hashinfo
, sk
);
607 sock_set_flag(sk
, SOCK_RCU_FREE
);
608 sock_prot_inuse_add(sock_net(sk
), sk
->sk_prot
, 1);
610 spin_unlock(&ilb
->lock
);
614 EXPORT_SYMBOL(__inet_hash
);
616 int inet_hash(struct sock
*sk
)
620 if (sk
->sk_state
!= TCP_CLOSE
) {
622 err
= __inet_hash(sk
, NULL
);
628 EXPORT_SYMBOL_GPL(inet_hash
);
630 void inet_unhash(struct sock
*sk
)
632 struct inet_hashinfo
*hashinfo
= sk
->sk_prot
->h
.hashinfo
;
633 struct inet_listen_hashbucket
*ilb
= NULL
;
639 if (sk
->sk_state
== TCP_LISTEN
) {
640 ilb
= &hashinfo
->listening_hash
[inet_sk_listen_hashfn(sk
)];
643 lock
= inet_ehash_lockp(hashinfo
, sk
->sk_hash
);
649 if (rcu_access_pointer(sk
->sk_reuseport_cb
))
650 reuseport_detach_sock(sk
);
652 inet_unhash2(hashinfo
, sk
);
653 __sk_del_node_init(sk
);
656 __sk_nulls_del_node_init_rcu(sk
);
658 sock_prot_inuse_add(sock_net(sk
), sk
->sk_prot
, -1);
660 spin_unlock_bh(lock
);
662 EXPORT_SYMBOL_GPL(inet_unhash
);
664 int __inet_hash_connect(struct inet_timewait_death_row
*death_row
,
665 struct sock
*sk
, u32 port_offset
,
666 int (*check_established
)(struct inet_timewait_death_row
*,
667 struct sock
*, __u16
, struct inet_timewait_sock
**))
669 struct inet_hashinfo
*hinfo
= death_row
->hashinfo
;
670 struct inet_timewait_sock
*tw
= NULL
;
671 struct inet_bind_hashbucket
*head
;
672 int port
= inet_sk(sk
)->inet_num
;
673 struct net
*net
= sock_net(sk
);
674 struct inet_bind_bucket
*tb
;
675 u32 remaining
, offset
;
676 int ret
, i
, low
, high
;
680 head
= &hinfo
->bhash
[inet_bhashfn(net
, port
,
682 tb
= inet_csk(sk
)->icsk_bind_hash
;
683 spin_lock_bh(&head
->lock
);
684 if (sk_head(&tb
->owners
) == sk
&& !sk
->sk_bind_node
.next
) {
685 inet_ehash_nolisten(sk
, NULL
);
686 spin_unlock_bh(&head
->lock
);
689 spin_unlock(&head
->lock
);
690 /* No definite answer... Walk to established hash table */
691 ret
= check_established(death_row
, sk
, port
, NULL
);
696 inet_get_local_port_range(net
, &low
, &high
);
697 high
++; /* [32768, 60999] -> [32768, 61000[ */
698 remaining
= high
- low
;
699 if (likely(remaining
> 1))
702 offset
= (hint
+ port_offset
) % remaining
;
703 /* In first pass we try ports of @low parity.
704 * inet_csk_get_port() does the opposite choice.
709 for (i
= 0; i
< remaining
; i
+= 2, port
+= 2) {
710 if (unlikely(port
>= high
))
712 if (inet_is_local_reserved_port(net
, port
))
714 head
= &hinfo
->bhash
[inet_bhashfn(net
, port
,
716 spin_lock_bh(&head
->lock
);
718 /* Does not bother with rcv_saddr checks, because
719 * the established check is already unique enough.
721 inet_bind_bucket_for_each(tb
, &head
->chain
) {
722 if (net_eq(ib_net(tb
), net
) && tb
->port
== port
) {
723 if (tb
->fastreuse
>= 0 ||
724 tb
->fastreuseport
>= 0)
726 WARN_ON(hlist_empty(&tb
->owners
));
727 if (!check_established(death_row
, sk
,
734 tb
= inet_bind_bucket_create(hinfo
->bind_bucket_cachep
,
737 spin_unlock_bh(&head
->lock
);
741 tb
->fastreuseport
= -1;
744 spin_unlock_bh(&head
->lock
);
749 if ((offset
& 1) && remaining
> 1)
750 goto other_parity_scan
;
752 return -EADDRNOTAVAIL
;
757 /* Head lock still held and bh's disabled */
758 inet_bind_hash(sk
, tb
, port
);
759 if (sk_unhashed(sk
)) {
760 inet_sk(sk
)->inet_sport
= htons(port
);
761 inet_ehash_nolisten(sk
, (struct sock
*)tw
);
764 inet_twsk_bind_unhash(tw
, hinfo
);
765 spin_unlock(&head
->lock
);
767 inet_twsk_deschedule_put(tw
);
773 * Bind a port for a connect operation and hash it.
775 int inet_hash_connect(struct inet_timewait_death_row
*death_row
,
780 if (!inet_sk(sk
)->inet_num
)
781 port_offset
= inet_sk_port_offset(sk
);
782 return __inet_hash_connect(death_row
, sk
, port_offset
,
783 __inet_check_established
);
785 EXPORT_SYMBOL_GPL(inet_hash_connect
);
787 void inet_hashinfo_init(struct inet_hashinfo
*h
)
791 for (i
= 0; i
< INET_LHTABLE_SIZE
; i
++) {
792 spin_lock_init(&h
->listening_hash
[i
].lock
);
793 INIT_HLIST_HEAD(&h
->listening_hash
[i
].head
);
794 h
->listening_hash
[i
].count
= 0;
799 EXPORT_SYMBOL_GPL(inet_hashinfo_init
);
801 void __init
inet_hashinfo2_init(struct inet_hashinfo
*h
, const char *name
,
802 unsigned long numentries
, int scale
,
803 unsigned long low_limit
,
804 unsigned long high_limit
)
808 h
->lhash2
= alloc_large_system_hash(name
,
818 for (i
= 0; i
<= h
->lhash2_mask
; i
++) {
819 spin_lock_init(&h
->lhash2
[i
].lock
);
820 INIT_HLIST_HEAD(&h
->lhash2
[i
].head
);
821 h
->lhash2
[i
].count
= 0;
825 int inet_ehash_locks_alloc(struct inet_hashinfo
*hashinfo
)
827 unsigned int locksz
= sizeof(spinlock_t
);
828 unsigned int i
, nblocks
= 1;
831 /* allocate 2 cache lines or at least one spinlock per cpu */
832 nblocks
= max(2U * L1_CACHE_BYTES
/ locksz
, 1U);
833 nblocks
= roundup_pow_of_two(nblocks
* num_possible_cpus());
835 /* no more locks than number of hash buckets */
836 nblocks
= min(nblocks
, hashinfo
->ehash_mask
+ 1);
838 hashinfo
->ehash_locks
= kvmalloc_array(nblocks
, locksz
, GFP_KERNEL
);
839 if (!hashinfo
->ehash_locks
)
842 for (i
= 0; i
< nblocks
; i
++)
843 spin_lock_init(&hashinfo
->ehash_locks
[i
]);
845 hashinfo
->ehash_locks_mask
= nblocks
- 1;
848 EXPORT_SYMBOL_GPL(inet_ehash_locks_alloc
);