1 // SPDX-License-Identifier: GPL-2.0
3 * To speed up listener socket lookup, create an array to store all sockets
4 * listening on the same port. This allows a decision to be made after finding
5 * the first socket. An optional BPF program can also be configured for
6 * selecting the socket index from the array of available sockets.
10 #include <net/sock_reuseport.h>
11 #include <linux/bpf.h>
12 #include <linux/idr.h>
13 #include <linux/filter.h>
14 #include <linux/rcupdate.h>
16 #define INIT_SOCKS 128
18 DEFINE_SPINLOCK(reuseport_lock
);
20 static DEFINE_IDA(reuseport_ida
);
21 static int reuseport_resurrect(struct sock
*sk
, struct sock_reuseport
*old_reuse
,
22 struct sock_reuseport
*reuse
, bool bind_inany
);
24 void reuseport_has_conns_set(struct sock
*sk
)
26 struct sock_reuseport
*reuse
;
28 if (!rcu_access_pointer(sk
->sk_reuseport_cb
))
31 spin_lock_bh(&reuseport_lock
);
32 reuse
= rcu_dereference_protected(sk
->sk_reuseport_cb
,
33 lockdep_is_held(&reuseport_lock
));
36 spin_unlock_bh(&reuseport_lock
);
38 EXPORT_SYMBOL(reuseport_has_conns_set
);
40 static void __reuseport_get_incoming_cpu(struct sock_reuseport
*reuse
)
42 /* Paired with READ_ONCE() in reuseport_select_sock_by_hash(). */
43 WRITE_ONCE(reuse
->incoming_cpu
, reuse
->incoming_cpu
+ 1);
46 static void __reuseport_put_incoming_cpu(struct sock_reuseport
*reuse
)
48 /* Paired with READ_ONCE() in reuseport_select_sock_by_hash(). */
49 WRITE_ONCE(reuse
->incoming_cpu
, reuse
->incoming_cpu
- 1);
52 static void reuseport_get_incoming_cpu(struct sock
*sk
, struct sock_reuseport
*reuse
)
54 if (sk
->sk_incoming_cpu
>= 0)
55 __reuseport_get_incoming_cpu(reuse
);
58 static void reuseport_put_incoming_cpu(struct sock
*sk
, struct sock_reuseport
*reuse
)
60 if (sk
->sk_incoming_cpu
>= 0)
61 __reuseport_put_incoming_cpu(reuse
);
64 void reuseport_update_incoming_cpu(struct sock
*sk
, int val
)
66 struct sock_reuseport
*reuse
;
67 int old_sk_incoming_cpu
;
69 if (unlikely(!rcu_access_pointer(sk
->sk_reuseport_cb
))) {
70 /* Paired with REAE_ONCE() in sk_incoming_cpu_update()
71 * and compute_score().
73 WRITE_ONCE(sk
->sk_incoming_cpu
, val
);
77 spin_lock_bh(&reuseport_lock
);
79 /* This must be done under reuseport_lock to avoid a race with
80 * reuseport_grow(), which accesses sk->sk_incoming_cpu without
81 * lock_sock() when detaching a shutdown()ed sk.
83 * Paired with READ_ONCE() in reuseport_select_sock_by_hash().
85 old_sk_incoming_cpu
= sk
->sk_incoming_cpu
;
86 WRITE_ONCE(sk
->sk_incoming_cpu
, val
);
88 reuse
= rcu_dereference_protected(sk
->sk_reuseport_cb
,
89 lockdep_is_held(&reuseport_lock
));
91 /* reuseport_grow() has detached a closed sk. */
95 if (old_sk_incoming_cpu
< 0 && val
>= 0)
96 __reuseport_get_incoming_cpu(reuse
);
97 else if (old_sk_incoming_cpu
>= 0 && val
< 0)
98 __reuseport_put_incoming_cpu(reuse
);
101 spin_unlock_bh(&reuseport_lock
);
104 static int reuseport_sock_index(struct sock
*sk
,
105 const struct sock_reuseport
*reuse
,
112 right
= reuse
->num_socks
;
114 left
= reuse
->max_socks
- reuse
->num_closed_socks
;
115 right
= reuse
->max_socks
;
118 for (; left
< right
; left
++)
119 if (reuse
->socks
[left
] == sk
)
124 static void __reuseport_add_sock(struct sock
*sk
,
125 struct sock_reuseport
*reuse
)
127 reuse
->socks
[reuse
->num_socks
] = sk
;
128 /* paired with smp_rmb() in reuseport_(select|migrate)_sock() */
131 reuseport_get_incoming_cpu(sk
, reuse
);
134 static bool __reuseport_detach_sock(struct sock
*sk
,
135 struct sock_reuseport
*reuse
)
137 int i
= reuseport_sock_index(sk
, reuse
, false);
142 reuse
->socks
[i
] = reuse
->socks
[reuse
->num_socks
- 1];
144 reuseport_put_incoming_cpu(sk
, reuse
);
149 static void __reuseport_add_closed_sock(struct sock
*sk
,
150 struct sock_reuseport
*reuse
)
152 reuse
->socks
[reuse
->max_socks
- reuse
->num_closed_socks
- 1] = sk
;
153 /* paired with READ_ONCE() in inet_csk_bind_conflict() */
154 WRITE_ONCE(reuse
->num_closed_socks
, reuse
->num_closed_socks
+ 1);
155 reuseport_get_incoming_cpu(sk
, reuse
);
158 static bool __reuseport_detach_closed_sock(struct sock
*sk
,
159 struct sock_reuseport
*reuse
)
161 int i
= reuseport_sock_index(sk
, reuse
, true);
166 reuse
->socks
[i
] = reuse
->socks
[reuse
->max_socks
- reuse
->num_closed_socks
];
167 /* paired with READ_ONCE() in inet_csk_bind_conflict() */
168 WRITE_ONCE(reuse
->num_closed_socks
, reuse
->num_closed_socks
- 1);
169 reuseport_put_incoming_cpu(sk
, reuse
);
174 static struct sock_reuseport
*__reuseport_alloc(unsigned int max_socks
)
176 struct sock_reuseport
*reuse
;
178 reuse
= kzalloc(struct_size(reuse
, socks
, max_socks
), GFP_ATOMIC
);
182 reuse
->max_socks
= max_socks
;
184 RCU_INIT_POINTER(reuse
->prog
, NULL
);
188 int reuseport_alloc(struct sock
*sk
, bool bind_inany
)
190 struct sock_reuseport
*reuse
;
193 /* bh lock used since this function call may precede hlist lock in
194 * soft irq of receive path or setsockopt from process context
196 spin_lock_bh(&reuseport_lock
);
198 /* Allocation attempts can occur concurrently via the setsockopt path
199 * and the bind/hash path. Nothing to do when we lose the race.
201 reuse
= rcu_dereference_protected(sk
->sk_reuseport_cb
,
202 lockdep_is_held(&reuseport_lock
));
204 if (reuse
->num_closed_socks
) {
205 /* sk was shutdown()ed before */
206 ret
= reuseport_resurrect(sk
, reuse
, NULL
, bind_inany
);
210 /* Only set reuse->bind_inany if the bind_inany is true.
211 * Otherwise, it will overwrite the reuse->bind_inany
212 * which was set by the bind/hash path.
215 reuse
->bind_inany
= bind_inany
;
219 reuse
= __reuseport_alloc(INIT_SOCKS
);
225 id
= ida_alloc(&reuseport_ida
, GFP_ATOMIC
);
232 reuse
->reuseport_id
= id
;
233 reuse
->bind_inany
= bind_inany
;
234 reuse
->socks
[0] = sk
;
235 reuse
->num_socks
= 1;
236 reuseport_get_incoming_cpu(sk
, reuse
);
237 rcu_assign_pointer(sk
->sk_reuseport_cb
, reuse
);
240 spin_unlock_bh(&reuseport_lock
);
244 EXPORT_SYMBOL(reuseport_alloc
);
246 static struct sock_reuseport
*reuseport_grow(struct sock_reuseport
*reuse
)
248 struct sock_reuseport
*more_reuse
;
249 u32 more_socks_size
, i
;
251 more_socks_size
= reuse
->max_socks
* 2U;
252 if (more_socks_size
> U16_MAX
) {
253 if (reuse
->num_closed_socks
) {
254 /* Make room by removing a closed sk.
255 * The child has already been migrated.
256 * Only reqsk left at this point.
260 sk
= reuse
->socks
[reuse
->max_socks
- reuse
->num_closed_socks
];
261 RCU_INIT_POINTER(sk
->sk_reuseport_cb
, NULL
);
262 __reuseport_detach_closed_sock(sk
, reuse
);
270 more_reuse
= __reuseport_alloc(more_socks_size
);
274 more_reuse
->num_socks
= reuse
->num_socks
;
275 more_reuse
->num_closed_socks
= reuse
->num_closed_socks
;
276 more_reuse
->prog
= reuse
->prog
;
277 more_reuse
->reuseport_id
= reuse
->reuseport_id
;
278 more_reuse
->bind_inany
= reuse
->bind_inany
;
279 more_reuse
->has_conns
= reuse
->has_conns
;
280 more_reuse
->incoming_cpu
= reuse
->incoming_cpu
;
282 memcpy(more_reuse
->socks
, reuse
->socks
,
283 reuse
->num_socks
* sizeof(struct sock
*));
284 memcpy(more_reuse
->socks
+
285 (more_reuse
->max_socks
- more_reuse
->num_closed_socks
),
286 reuse
->socks
+ (reuse
->max_socks
- reuse
->num_closed_socks
),
287 reuse
->num_closed_socks
* sizeof(struct sock
*));
288 more_reuse
->synq_overflow_ts
= READ_ONCE(reuse
->synq_overflow_ts
);
290 for (i
= 0; i
< reuse
->max_socks
; ++i
)
291 rcu_assign_pointer(reuse
->socks
[i
]->sk_reuseport_cb
,
294 /* Note: we use kfree_rcu here instead of reuseport_free_rcu so
295 * that reuse and more_reuse can temporarily share a reference
298 kfree_rcu(reuse
, rcu
);
302 static void reuseport_free_rcu(struct rcu_head
*head
)
304 struct sock_reuseport
*reuse
;
306 reuse
= container_of(head
, struct sock_reuseport
, rcu
);
307 sk_reuseport_prog_free(rcu_dereference_protected(reuse
->prog
, 1));
308 ida_free(&reuseport_ida
, reuse
->reuseport_id
);
313 * reuseport_add_sock - Add a socket to the reuseport group of another.
314 * @sk: New socket to add to the group.
315 * @sk2: Socket belonging to the existing reuseport group.
316 * @bind_inany: Whether or not the group is bound to a local INANY address.
318 * May return ENOMEM and not add socket to group under memory pressure.
320 int reuseport_add_sock(struct sock
*sk
, struct sock
*sk2
, bool bind_inany
)
322 struct sock_reuseport
*old_reuse
, *reuse
;
324 if (!rcu_access_pointer(sk2
->sk_reuseport_cb
)) {
325 int err
= reuseport_alloc(sk2
, bind_inany
);
331 spin_lock_bh(&reuseport_lock
);
332 reuse
= rcu_dereference_protected(sk2
->sk_reuseport_cb
,
333 lockdep_is_held(&reuseport_lock
));
334 old_reuse
= rcu_dereference_protected(sk
->sk_reuseport_cb
,
335 lockdep_is_held(&reuseport_lock
));
336 if (old_reuse
&& old_reuse
->num_closed_socks
) {
337 /* sk was shutdown()ed before */
338 int err
= reuseport_resurrect(sk
, old_reuse
, reuse
, reuse
->bind_inany
);
340 spin_unlock_bh(&reuseport_lock
);
344 if (old_reuse
&& old_reuse
->num_socks
!= 1) {
345 spin_unlock_bh(&reuseport_lock
);
349 if (reuse
->num_socks
+ reuse
->num_closed_socks
== reuse
->max_socks
) {
350 reuse
= reuseport_grow(reuse
);
352 spin_unlock_bh(&reuseport_lock
);
357 __reuseport_add_sock(sk
, reuse
);
358 rcu_assign_pointer(sk
->sk_reuseport_cb
, reuse
);
360 spin_unlock_bh(&reuseport_lock
);
363 call_rcu(&old_reuse
->rcu
, reuseport_free_rcu
);
366 EXPORT_SYMBOL(reuseport_add_sock
);
368 static int reuseport_resurrect(struct sock
*sk
, struct sock_reuseport
*old_reuse
,
369 struct sock_reuseport
*reuse
, bool bind_inany
)
371 if (old_reuse
== reuse
) {
372 /* If sk was in the same reuseport group, just pop sk out of
373 * the closed section and push sk into the listening section.
375 __reuseport_detach_closed_sock(sk
, old_reuse
);
376 __reuseport_add_sock(sk
, old_reuse
);
381 /* In bind()/listen() path, we cannot carry over the eBPF prog
382 * for the shutdown()ed socket. In setsockopt() path, we should
383 * not change the eBPF prog of listening sockets by attaching a
384 * prog to the shutdown()ed socket. Thus, we will allocate a new
385 * reuseport group and detach sk from the old group.
389 reuse
= __reuseport_alloc(INIT_SOCKS
);
393 id
= ida_alloc(&reuseport_ida
, GFP_ATOMIC
);
399 reuse
->reuseport_id
= id
;
400 reuse
->bind_inany
= bind_inany
;
402 /* Move sk from the old group to the new one if
403 * - all the other listeners in the old group were close()d or
404 * shutdown()ed, and then sk2 has listen()ed on the same port
406 * - sk listen()ed without bind() (or with autobind), was
407 * shutdown()ed, and then listen()s on another port which
410 if (reuse
->num_socks
+ reuse
->num_closed_socks
== reuse
->max_socks
) {
411 reuse
= reuseport_grow(reuse
);
417 __reuseport_detach_closed_sock(sk
, old_reuse
);
418 __reuseport_add_sock(sk
, reuse
);
419 rcu_assign_pointer(sk
->sk_reuseport_cb
, reuse
);
421 if (old_reuse
->num_socks
+ old_reuse
->num_closed_socks
== 0)
422 call_rcu(&old_reuse
->rcu
, reuseport_free_rcu
);
427 void reuseport_detach_sock(struct sock
*sk
)
429 struct sock_reuseport
*reuse
;
431 spin_lock_bh(&reuseport_lock
);
432 reuse
= rcu_dereference_protected(sk
->sk_reuseport_cb
,
433 lockdep_is_held(&reuseport_lock
));
435 /* reuseport_grow() has detached a closed sk */
439 /* Notify the bpf side. The sk may be added to a sockarray
440 * map. If so, sockarray logic will remove it from the map.
442 * Other bpf map types that work with reuseport, like sockmap,
443 * don't need an explicit callback from here. They override sk
444 * unhash/close ops to remove the sk from the map before we
447 bpf_sk_reuseport_detach(sk
);
449 rcu_assign_pointer(sk
->sk_reuseport_cb
, NULL
);
451 if (!__reuseport_detach_closed_sock(sk
, reuse
))
452 __reuseport_detach_sock(sk
, reuse
);
454 if (reuse
->num_socks
+ reuse
->num_closed_socks
== 0)
455 call_rcu(&reuse
->rcu
, reuseport_free_rcu
);
458 spin_unlock_bh(&reuseport_lock
);
460 EXPORT_SYMBOL(reuseport_detach_sock
);
462 void reuseport_stop_listen_sock(struct sock
*sk
)
464 if (sk
->sk_protocol
== IPPROTO_TCP
) {
465 struct sock_reuseport
*reuse
;
466 struct bpf_prog
*prog
;
468 spin_lock_bh(&reuseport_lock
);
470 reuse
= rcu_dereference_protected(sk
->sk_reuseport_cb
,
471 lockdep_is_held(&reuseport_lock
));
472 prog
= rcu_dereference_protected(reuse
->prog
,
473 lockdep_is_held(&reuseport_lock
));
475 if (READ_ONCE(sock_net(sk
)->ipv4
.sysctl_tcp_migrate_req
) ||
476 (prog
&& prog
->expected_attach_type
== BPF_SK_REUSEPORT_SELECT_OR_MIGRATE
)) {
477 /* Migration capable, move sk from the listening section
478 * to the closed section.
480 bpf_sk_reuseport_detach(sk
);
482 __reuseport_detach_sock(sk
, reuse
);
483 __reuseport_add_closed_sock(sk
, reuse
);
485 spin_unlock_bh(&reuseport_lock
);
489 spin_unlock_bh(&reuseport_lock
);
492 /* Not capable to do migration, detach immediately */
493 reuseport_detach_sock(sk
);
495 EXPORT_SYMBOL(reuseport_stop_listen_sock
);
497 static struct sock
*run_bpf_filter(struct sock_reuseport
*reuse
, u16 socks
,
498 struct bpf_prog
*prog
, struct sk_buff
*skb
,
501 struct sk_buff
*nskb
= NULL
;
504 if (skb_shared(skb
)) {
505 nskb
= skb_clone(skb
, GFP_ATOMIC
);
511 /* temporarily advance data past protocol header */
512 if (!pskb_pull(skb
, hdr_len
)) {
516 index
= bpf_prog_run_save_cb(prog
, skb
);
517 __skb_push(skb
, hdr_len
);
524 return reuse
->socks
[index
];
527 static struct sock
*reuseport_select_sock_by_hash(struct sock_reuseport
*reuse
,
528 u32 hash
, u16 num_socks
)
530 struct sock
*first_valid_sk
= NULL
;
533 i
= j
= reciprocal_scale(hash
, num_socks
);
535 struct sock
*sk
= reuse
->socks
[i
];
537 if (sk
->sk_state
!= TCP_ESTABLISHED
) {
538 /* Paired with WRITE_ONCE() in __reuseport_(get|put)_incoming_cpu(). */
539 if (!READ_ONCE(reuse
->incoming_cpu
))
542 /* Paired with WRITE_ONCE() in reuseport_update_incoming_cpu(). */
543 if (READ_ONCE(sk
->sk_incoming_cpu
) == raw_smp_processor_id())
555 return first_valid_sk
;
559 * reuseport_select_sock - Select a socket from an SO_REUSEPORT group.
560 * @sk: First socket in the group.
561 * @hash: When no BPF filter is available, use this hash to select.
562 * @skb: skb to run through BPF filter.
563 * @hdr_len: BPF filter expects skb data pointer at payload data. If
564 * the skb does not yet point at the payload, this parameter represents
565 * how far the pointer needs to advance to reach the payload.
566 * Returns a socket that should receive the packet (or NULL on error).
568 struct sock
*reuseport_select_sock(struct sock
*sk
,
573 struct sock_reuseport
*reuse
;
574 struct bpf_prog
*prog
;
575 struct sock
*sk2
= NULL
;
579 reuse
= rcu_dereference(sk
->sk_reuseport_cb
);
581 /* if memory allocation failed or add call is not yet complete */
585 prog
= rcu_dereference(reuse
->prog
);
586 socks
= READ_ONCE(reuse
->num_socks
);
588 /* paired with smp_wmb() in __reuseport_add_sock() */
594 if (prog
->type
== BPF_PROG_TYPE_SK_REUSEPORT
)
595 sk2
= bpf_run_sk_reuseport(reuse
, sk
, prog
, skb
, NULL
, hash
);
597 sk2
= run_bpf_filter(reuse
, socks
, prog
, skb
, hdr_len
);
600 /* no bpf or invalid bpf result: fall back to hash usage */
602 sk2
= reuseport_select_sock_by_hash(reuse
, hash
, socks
);
609 EXPORT_SYMBOL(reuseport_select_sock
);
612 * reuseport_migrate_sock - Select a socket from an SO_REUSEPORT group.
613 * @sk: close()ed or shutdown()ed socket in the group.
614 * @migrating_sk: ESTABLISHED/SYN_RECV full socket in the accept queue or
615 * NEW_SYN_RECV request socket during 3WHS.
616 * @skb: skb to run through BPF filter.
617 * Returns a socket (with sk_refcnt +1) that should accept the child socket
618 * (or NULL on error).
620 struct sock
*reuseport_migrate_sock(struct sock
*sk
,
621 struct sock
*migrating_sk
,
624 struct sock_reuseport
*reuse
;
625 struct sock
*nsk
= NULL
;
626 bool allocated
= false;
627 struct bpf_prog
*prog
;
633 reuse
= rcu_dereference(sk
->sk_reuseport_cb
);
637 socks
= READ_ONCE(reuse
->num_socks
);
638 if (unlikely(!socks
))
641 /* paired with smp_wmb() in __reuseport_add_sock() */
644 hash
= migrating_sk
->sk_hash
;
645 prog
= rcu_dereference(reuse
->prog
);
646 if (!prog
|| prog
->expected_attach_type
!= BPF_SK_REUSEPORT_SELECT_OR_MIGRATE
) {
647 if (READ_ONCE(sock_net(sk
)->ipv4
.sysctl_tcp_migrate_req
))
653 skb
= alloc_skb(0, GFP_ATOMIC
);
659 nsk
= bpf_run_sk_reuseport(reuse
, sk
, prog
, skb
, migrating_sk
, hash
);
666 nsk
= reuseport_select_sock_by_hash(reuse
, hash
, socks
);
668 if (IS_ERR_OR_NULL(nsk
) || unlikely(!refcount_inc_not_zero(&nsk
->sk_refcnt
))) {
678 __NET_INC_STATS(sock_net(sk
), LINUX_MIB_TCPMIGRATEREQFAILURE
);
681 EXPORT_SYMBOL(reuseport_migrate_sock
);
683 int reuseport_attach_prog(struct sock
*sk
, struct bpf_prog
*prog
)
685 struct sock_reuseport
*reuse
;
686 struct bpf_prog
*old_prog
;
688 if (sk_unhashed(sk
)) {
691 if (!sk
->sk_reuseport
)
694 err
= reuseport_alloc(sk
, false);
697 } else if (!rcu_access_pointer(sk
->sk_reuseport_cb
)) {
698 /* The socket wasn't bound with SO_REUSEPORT */
702 spin_lock_bh(&reuseport_lock
);
703 reuse
= rcu_dereference_protected(sk
->sk_reuseport_cb
,
704 lockdep_is_held(&reuseport_lock
));
705 old_prog
= rcu_dereference_protected(reuse
->prog
,
706 lockdep_is_held(&reuseport_lock
));
707 rcu_assign_pointer(reuse
->prog
, prog
);
708 spin_unlock_bh(&reuseport_lock
);
710 sk_reuseport_prog_free(old_prog
);
713 EXPORT_SYMBOL(reuseport_attach_prog
);
715 int reuseport_detach_prog(struct sock
*sk
)
717 struct sock_reuseport
*reuse
;
718 struct bpf_prog
*old_prog
;
721 spin_lock_bh(&reuseport_lock
);
722 reuse
= rcu_dereference_protected(sk
->sk_reuseport_cb
,
723 lockdep_is_held(&reuseport_lock
));
725 /* reuse must be checked after acquiring the reuseport_lock
726 * because reuseport_grow() can detach a closed sk.
729 spin_unlock_bh(&reuseport_lock
);
730 return sk
->sk_reuseport
? -ENOENT
: -EINVAL
;
733 if (sk_unhashed(sk
) && reuse
->num_closed_socks
) {
734 spin_unlock_bh(&reuseport_lock
);
738 old_prog
= rcu_replace_pointer(reuse
->prog
, old_prog
,
739 lockdep_is_held(&reuseport_lock
));
740 spin_unlock_bh(&reuseport_lock
);
745 sk_reuseport_prog_free(old_prog
);
748 EXPORT_SYMBOL(reuseport_detach_prog
);