?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'iommu-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux
[linux/fpc-iii.git] / net / core / bpf_sk_storage.c
blob4edd033e899c004eff91994d7a6509084c76679b
1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2019 Facebook */
3 #include <linux/rculist.h>
4 #include <linux/list.h>
5 #include <linux/hash.h>
6 #include <linux/types.h>
7 #include <linux/spinlock.h>
8 #include <linux/bpf.h>
9 #include <linux/btf.h>
10 #include <linux/btf_ids.h>
11 #include <linux/bpf_local_storage.h>
12 #include <net/bpf_sk_storage.h>
13 #include <net/sock.h>
14 #include <uapi/linux/sock_diag.h>
15 #include <uapi/linux/btf.h>
16
17 DEFINE_BPF_STORAGE_CACHE(sk_cache);
18
19 static struct bpf_local_storage_data *
20 bpf_sk_storage_lookup(struct sock *sk, struct bpf_map *map, bool cacheit_lockit)
21 {
22 struct bpf_local_storage *sk_storage;
23 struct bpf_local_storage_map *smap;
24
25 sk_storage = rcu_dereference(sk->sk_bpf_storage);
26 if (!sk_storage)
27 return NULL;
28
29 smap = (struct bpf_local_storage_map *)map;
30 return bpf_local_storage_lookup(sk_storage, smap, cacheit_lockit);
31 }
32
33 static int bpf_sk_storage_del(struct sock *sk, struct bpf_map *map)
34 {
35 struct bpf_local_storage_data *sdata;
36
37 sdata = bpf_sk_storage_lookup(sk, map, false);
38 if (!sdata)
39 return -ENOENT;
40
41 bpf_selem_unlink(SELEM(sdata));
42
43 return 0;
44 }
45
46 /* Called by __sk_destruct() & bpf_sk_storage_clone() */
47 void bpf_sk_storage_free(struct sock *sk)
48 {
49 struct bpf_local_storage_elem *selem;
50 struct bpf_local_storage *sk_storage;
51 bool free_sk_storage = false;
52 struct hlist_node *n;
53
54 rcu_read_lock();
55 sk_storage = rcu_dereference(sk->sk_bpf_storage);
56 if (!sk_storage) {
57 rcu_read_unlock();
58 return;
59 }
60
61 /* Netiher the bpf_prog nor the bpf-map's syscall
62 * could be modifying the sk_storage->list now.
63 * Thus, no elem can be added-to or deleted-from the
64 * sk_storage->list by the bpf_prog or by the bpf-map's syscall.
65 *
66 * It is racing with bpf_local_storage_map_free() alone
67 * when unlinking elem from the sk_storage->list and
68 * the map's bucket->list.
69 */
70 raw_spin_lock_bh(&sk_storage->lock);
71 hlist_for_each_entry_safe(selem, n, &sk_storage->list, snode) {
72 /* Always unlink from map before unlinking from
73 * sk_storage.
74 */
75 bpf_selem_unlink_map(selem);
76 free_sk_storage = bpf_selem_unlink_storage_nolock(sk_storage,
77 selem, true);
78 }
79 raw_spin_unlock_bh(&sk_storage->lock);
80 rcu_read_unlock();
81
82 if (free_sk_storage)
83 kfree_rcu(sk_storage, rcu);
84 }
85
86 static void bpf_sk_storage_map_free(struct bpf_map *map)
87 {
88 struct bpf_local_storage_map *smap;
89
90 smap = (struct bpf_local_storage_map *)map;
91 bpf_local_storage_cache_idx_free(&sk_cache, smap->cache_idx);
92 bpf_local_storage_map_free(smap);
93 }
94
95 static struct bpf_map *bpf_sk_storage_map_alloc(union bpf_attr *attr)
96 {
97 struct bpf_local_storage_map *smap;
98
99 smap = bpf_local_storage_map_alloc(attr);
100 if (IS_ERR(smap))
101 return ERR_CAST(smap);
102
103 smap->cache_idx = bpf_local_storage_cache_idx_get(&sk_cache);
104 return &smap->map;
105 }
106
107 static int notsupp_get_next_key(struct bpf_map *map, void *key,
108 void *next_key)
109 {
110 return -ENOTSUPP;
111 }
112
113 static void *bpf_fd_sk_storage_lookup_elem(struct bpf_map *map, void *key)
114 {
115 struct bpf_local_storage_data *sdata;
116 struct socket *sock;
117 int fd, err;
118
119 fd = *(int *)key;
120 sock = sockfd_lookup(fd, &err);
121 if (sock) {
122 sdata = bpf_sk_storage_lookup(sock->sk, map, true);
123 sockfd_put(sock);
124 return sdata ? sdata->data : NULL;
125 }
126
127 return ERR_PTR(err);
128 }
129
130 static int bpf_fd_sk_storage_update_elem(struct bpf_map *map, void *key,
131 void *value, u64 map_flags)
132 {
133 struct bpf_local_storage_data *sdata;
134 struct socket *sock;
135 int fd, err;
136
137 fd = *(int *)key;
138 sock = sockfd_lookup(fd, &err);
139 if (sock) {
140 sdata = bpf_local_storage_update(
141 sock->sk, (struct bpf_local_storage_map *)map, value,
142 map_flags);
143 sockfd_put(sock);
144 return PTR_ERR_OR_ZERO(sdata);
145 }
146
147 return err;
148 }
149
150 static int bpf_fd_sk_storage_delete_elem(struct bpf_map *map, void *key)
151 {
152 struct socket *sock;
153 int fd, err;
154
155 fd = *(int *)key;
156 sock = sockfd_lookup(fd, &err);
157 if (sock) {
158 err = bpf_sk_storage_del(sock->sk, map);
159 sockfd_put(sock);
160 return err;
161 }
162
163 return err;
164 }
165
166 static struct bpf_local_storage_elem *
167 bpf_sk_storage_clone_elem(struct sock *newsk,
168 struct bpf_local_storage_map *smap,
169 struct bpf_local_storage_elem *selem)
170 {
171 struct bpf_local_storage_elem *copy_selem;
172
173 copy_selem = bpf_selem_alloc(smap, newsk, NULL, true);
174 if (!copy_selem)
175 return NULL;
176
177 if (map_value_has_spin_lock(&smap->map))
178 copy_map_value_locked(&smap->map, SDATA(copy_selem)->data,
179 SDATA(selem)->data, true);
180 else
181 copy_map_value(&smap->map, SDATA(copy_selem)->data,
182 SDATA(selem)->data);
183
184 return copy_selem;
185 }
186
187 int bpf_sk_storage_clone(const struct sock *sk, struct sock *newsk)
188 {
189 struct bpf_local_storage *new_sk_storage = NULL;
190 struct bpf_local_storage *sk_storage;
191 struct bpf_local_storage_elem *selem;
192 int ret = 0;
193
194 RCU_INIT_POINTER(newsk->sk_bpf_storage, NULL);
195
196 rcu_read_lock();
197 sk_storage = rcu_dereference(sk->sk_bpf_storage);
198
199 if (!sk_storage || hlist_empty(&sk_storage->list))
200 goto out;
201
202 hlist_for_each_entry_rcu(selem, &sk_storage->list, snode) {
203 struct bpf_local_storage_elem *copy_selem;
204 struct bpf_local_storage_map *smap;
205 struct bpf_map *map;
206
207 smap = rcu_dereference(SDATA(selem)->smap);
208 if (!(smap->map.map_flags & BPF_F_CLONE))
209 continue;
210
211 /* Note that for lockless listeners adding new element
212 * here can race with cleanup in bpf_local_storage_map_free.
213 * Try to grab map refcnt to make sure that it's still
214 * alive and prevent concurrent removal.
215 */
216 map = bpf_map_inc_not_zero(&smap->map);
217 if (IS_ERR(map))
218 continue;
219
220 copy_selem = bpf_sk_storage_clone_elem(newsk, smap, selem);
221 if (!copy_selem) {
222 ret = -ENOMEM;
223 bpf_map_put(map);
224 goto out;
225 }
226
227 if (new_sk_storage) {
228 bpf_selem_link_map(smap, copy_selem);
229 bpf_selem_link_storage_nolock(new_sk_storage, copy_selem);
230 } else {
231 ret = bpf_local_storage_alloc(newsk, smap, copy_selem);
232 if (ret) {
233 kfree(copy_selem);
234 atomic_sub(smap->elem_size,
235 &newsk->sk_omem_alloc);
236 bpf_map_put(map);
237 goto out;
238 }
239
240 new_sk_storage =
241 rcu_dereference(copy_selem->local_storage);
242 }
243 bpf_map_put(map);
244 }
245
246 out:
247 rcu_read_unlock();
248
249 /* In case of an error, don't free anything explicitly here, the
250 * caller is responsible to call bpf_sk_storage_free.
251 */
252
253 return ret;
254 }
255
256 BPF_CALL_4(bpf_sk_storage_get, struct bpf_map *, map, struct sock *, sk,
257 void *, value, u64, flags)
258 {
259 struct bpf_local_storage_data *sdata;
260
261 if (!sk || !sk_fullsock(sk) || flags > BPF_SK_STORAGE_GET_F_CREATE)
262 return (unsigned long)NULL;
263
264 sdata = bpf_sk_storage_lookup(sk, map, true);
265 if (sdata)
266 return (unsigned long)sdata->data;
267
268 if (flags == BPF_SK_STORAGE_GET_F_CREATE &&
269 /* Cannot add new elem to a going away sk.
270 * Otherwise, the new elem may become a leak
271 * (and also other memory issues during map
272 * destruction).
273 */
274 refcount_inc_not_zero(&sk->sk_refcnt)) {
275 sdata = bpf_local_storage_update(
276 sk, (struct bpf_local_storage_map *)map, value,
277 BPF_NOEXIST);
278 /* sk must be a fullsock (guaranteed by verifier),
279 * so sock_gen_put() is unnecessary.
280 */
281 sock_put(sk);
282 return IS_ERR(sdata) ?
283 (unsigned long)NULL : (unsigned long)sdata->data;
284 }
285
286 return (unsigned long)NULL;
287 }
288
289 BPF_CALL_2(bpf_sk_storage_delete, struct bpf_map *, map, struct sock *, sk)
290 {
291 if (!sk || !sk_fullsock(sk))
292 return -EINVAL;
293
294 if (refcount_inc_not_zero(&sk->sk_refcnt)) {
295 int err;
296
297 err = bpf_sk_storage_del(sk, map);
298 sock_put(sk);
299 return err;
300 }
301
302 return -ENOENT;
303 }
304
305 static int bpf_sk_storage_charge(struct bpf_local_storage_map *smap,
306 void *owner, u32 size)
307 {
308 struct sock *sk = (struct sock *)owner;
309
310 /* same check as in sock_kmalloc() */
311 if (size <= sysctl_optmem_max &&
312 atomic_read(&sk->sk_omem_alloc) + size < sysctl_optmem_max) {
313 atomic_add(size, &sk->sk_omem_alloc);
314 return 0;
315 }
316
317 return -ENOMEM;
318 }
319
320 static void bpf_sk_storage_uncharge(struct bpf_local_storage_map *smap,
321 void *owner, u32 size)
322 {
323 struct sock *sk = owner;
324
325 atomic_sub(size, &sk->sk_omem_alloc);
326 }
327
328 static struct bpf_local_storage __rcu **
329 bpf_sk_storage_ptr(void *owner)
330 {
331 struct sock *sk = owner;
332
333 return &sk->sk_bpf_storage;
334 }
335
336 static int sk_storage_map_btf_id;
337 const struct bpf_map_ops sk_storage_map_ops = {
338 .map_meta_equal = bpf_map_meta_equal,
339 .map_alloc_check = bpf_local_storage_map_alloc_check,
340 .map_alloc = bpf_sk_storage_map_alloc,
341 .map_free = bpf_sk_storage_map_free,
342 .map_get_next_key = notsupp_get_next_key,
343 .map_lookup_elem = bpf_fd_sk_storage_lookup_elem,
344 .map_update_elem = bpf_fd_sk_storage_update_elem,
345 .map_delete_elem = bpf_fd_sk_storage_delete_elem,
346 .map_check_btf = bpf_local_storage_map_check_btf,
347 .map_btf_name = "bpf_local_storage_map",
348 .map_btf_id = &sk_storage_map_btf_id,
349 .map_local_storage_charge = bpf_sk_storage_charge,
350 .map_local_storage_uncharge = bpf_sk_storage_uncharge,
351 .map_owner_storage_ptr = bpf_sk_storage_ptr,
352 };
353
354 const struct bpf_func_proto bpf_sk_storage_get_proto = {
355 .func = bpf_sk_storage_get,
356 .gpl_only = false,
357 .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL,
358 .arg1_type = ARG_CONST_MAP_PTR,
359 .arg2_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
360 .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL,
361 .arg4_type = ARG_ANYTHING,
362 };
363
364 const struct bpf_func_proto bpf_sk_storage_get_cg_sock_proto = {
365 .func = bpf_sk_storage_get,
366 .gpl_only = false,
367 .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL,
368 .arg1_type = ARG_CONST_MAP_PTR,
369 .arg2_type = ARG_PTR_TO_CTX, /* context is 'struct sock' */
370 .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL,
371 .arg4_type = ARG_ANYTHING,
372 };
373
374 const struct bpf_func_proto bpf_sk_storage_delete_proto = {
375 .func = bpf_sk_storage_delete,
376 .gpl_only = false,
377 .ret_type = RET_INTEGER,
378 .arg1_type = ARG_CONST_MAP_PTR,
379 .arg2_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
380 };
381
382 static bool bpf_sk_storage_tracing_allowed(const struct bpf_prog *prog)
383 {
384 const struct btf *btf_vmlinux;
385 const struct btf_type *t;
386 const char *tname;
387 u32 btf_id;
388
389 if (prog->aux->dst_prog)
390 return false;
391
392 /* Ensure the tracing program is not tracing
393 * any bpf_sk_storage*() function and also
394 * use the bpf_sk_storage_(get|delete) helper.
395 */
396 switch (prog->expected_attach_type) {
397 case BPF_TRACE_ITER:
398 case BPF_TRACE_RAW_TP:
399 /* bpf_sk_storage has no trace point */
400 return true;
401 case BPF_TRACE_FENTRY:
402 case BPF_TRACE_FEXIT:
403 btf_vmlinux = bpf_get_btf_vmlinux();
404 btf_id = prog->aux->attach_btf_id;
405 t = btf_type_by_id(btf_vmlinux, btf_id);
406 tname = btf_name_by_offset(btf_vmlinux, t->name_off);
407 return !!strncmp(tname, "bpf_sk_storage",
408 strlen("bpf_sk_storage"));
409 default:
410 return false;
411 }
412
413 return false;
414 }
415
416 BPF_CALL_4(bpf_sk_storage_get_tracing, struct bpf_map *, map, struct sock *, sk,
417 void *, value, u64, flags)
418 {
419 if (in_irq() || in_nmi())
420 return (unsigned long)NULL;
421
422 return (unsigned long)____bpf_sk_storage_get(map, sk, value, flags);
423 }
424
425 BPF_CALL_2(bpf_sk_storage_delete_tracing, struct bpf_map *, map,
426 struct sock *, sk)
427 {
428 if (in_irq() || in_nmi())
429 return -EPERM;
430
431 return ____bpf_sk_storage_delete(map, sk);
432 }
433
434 const struct bpf_func_proto bpf_sk_storage_get_tracing_proto = {
435 .func = bpf_sk_storage_get_tracing,
436 .gpl_only = false,
437 .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL,
438 .arg1_type = ARG_CONST_MAP_PTR,
439 .arg2_type = ARG_PTR_TO_BTF_ID,
440 .arg2_btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],
441 .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL,
442 .arg4_type = ARG_ANYTHING,
443 .allowed = bpf_sk_storage_tracing_allowed,
444 };
445
446 const struct bpf_func_proto bpf_sk_storage_delete_tracing_proto = {
447 .func = bpf_sk_storage_delete_tracing,
448 .gpl_only = false,
449 .ret_type = RET_INTEGER,
450 .arg1_type = ARG_CONST_MAP_PTR,
451 .arg2_type = ARG_PTR_TO_BTF_ID,
452 .arg2_btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],
453 .allowed = bpf_sk_storage_tracing_allowed,
454 };
455
456 struct bpf_sk_storage_diag {
457 u32 nr_maps;
458 struct bpf_map *maps[];
459 };
460
461 /* The reply will be like:
462 * INET_DIAG_BPF_SK_STORAGES (nla_nest)
463 * SK_DIAG_BPF_STORAGE (nla_nest)
464 * SK_DIAG_BPF_STORAGE_MAP_ID (nla_put_u32)
465 * SK_DIAG_BPF_STORAGE_MAP_VALUE (nla_reserve_64bit)
466 * SK_DIAG_BPF_STORAGE (nla_nest)
467 * SK_DIAG_BPF_STORAGE_MAP_ID (nla_put_u32)
468 * SK_DIAG_BPF_STORAGE_MAP_VALUE (nla_reserve_64bit)
469 * ....
470 */
471 static int nla_value_size(u32 value_size)
472 {
473 /* SK_DIAG_BPF_STORAGE (nla_nest)
474 * SK_DIAG_BPF_STORAGE_MAP_ID (nla_put_u32)
475 * SK_DIAG_BPF_STORAGE_MAP_VALUE (nla_reserve_64bit)
476 */
477 return nla_total_size(0) + nla_total_size(sizeof(u32)) +
478 nla_total_size_64bit(value_size);
479 }
480
481 void bpf_sk_storage_diag_free(struct bpf_sk_storage_diag *diag)
482 {
483 u32 i;
484
485 if (!diag)
486 return;
487
488 for (i = 0; i < diag->nr_maps; i++)
489 bpf_map_put(diag->maps[i]);
490
491 kfree(diag);
492 }
493 EXPORT_SYMBOL_GPL(bpf_sk_storage_diag_free);
494
495 static bool diag_check_dup(const struct bpf_sk_storage_diag *diag,
496 const struct bpf_map *map)
497 {
498 u32 i;
499
500 for (i = 0; i < diag->nr_maps; i++) {
501 if (diag->maps[i] == map)
502 return true;
503 }
504
505 return false;
506 }
507
508 struct bpf_sk_storage_diag *
509 bpf_sk_storage_diag_alloc(const struct nlattr *nla_stgs)
510 {
511 struct bpf_sk_storage_diag *diag;
512 struct nlattr *nla;
513 u32 nr_maps = 0;
514 int rem, err;
515
516 /* bpf_local_storage_map is currently limited to CAP_SYS_ADMIN as
517 * the map_alloc_check() side also does.
518 */
519 if (!bpf_capable())
520 return ERR_PTR(-EPERM);
521
522 nla_for_each_nested(nla, nla_stgs, rem) {
523 if (nla_type(nla) == SK_DIAG_BPF_STORAGE_REQ_MAP_FD)
524 nr_maps++;
525 }
526
527 diag = kzalloc(sizeof(*diag) + sizeof(diag->maps[0]) * nr_maps,
528 GFP_KERNEL);
529 if (!diag)
530 return ERR_PTR(-ENOMEM);
531
532 nla_for_each_nested(nla, nla_stgs, rem) {
533 struct bpf_map *map;
534 int map_fd;
535
536 if (nla_type(nla) != SK_DIAG_BPF_STORAGE_REQ_MAP_FD)
537 continue;
538
539 map_fd = nla_get_u32(nla);
540 map = bpf_map_get(map_fd);
541 if (IS_ERR(map)) {
542 err = PTR_ERR(map);
543 goto err_free;
544 }
545 if (map->map_type != BPF_MAP_TYPE_SK_STORAGE) {
546 bpf_map_put(map);
547 err = -EINVAL;
548 goto err_free;
549 }
550 if (diag_check_dup(diag, map)) {
551 bpf_map_put(map);
552 err = -EEXIST;
553 goto err_free;
554 }
555 diag->maps[diag->nr_maps++] = map;
556 }
557
558 return diag;
559
560 err_free:
561 bpf_sk_storage_diag_free(diag);
562 return ERR_PTR(err);
563 }
564 EXPORT_SYMBOL_GPL(bpf_sk_storage_diag_alloc);
565
566 static int diag_get(struct bpf_local_storage_data *sdata, struct sk_buff *skb)
567 {
568 struct nlattr *nla_stg, *nla_value;
569 struct bpf_local_storage_map *smap;
570
571 /* It cannot exceed max nlattr's payload */
572 BUILD_BUG_ON(U16_MAX - NLA_HDRLEN < BPF_LOCAL_STORAGE_MAX_VALUE_SIZE);
573
574 nla_stg = nla_nest_start(skb, SK_DIAG_BPF_STORAGE);
575 if (!nla_stg)
576 return -EMSGSIZE;
577
578 smap = rcu_dereference(sdata->smap);
579 if (nla_put_u32(skb, SK_DIAG_BPF_STORAGE_MAP_ID, smap->map.id))
580 goto errout;
581
582 nla_value = nla_reserve_64bit(skb, SK_DIAG_BPF_STORAGE_MAP_VALUE,
583 smap->map.value_size,
584 SK_DIAG_BPF_STORAGE_PAD);
585 if (!nla_value)
586 goto errout;
587
588 if (map_value_has_spin_lock(&smap->map))
589 copy_map_value_locked(&smap->map, nla_data(nla_value),
590 sdata->data, true);
591 else
592 copy_map_value(&smap->map, nla_data(nla_value), sdata->data);
593
594 nla_nest_end(skb, nla_stg);
595 return 0;
596
597 errout:
598 nla_nest_cancel(skb, nla_stg);
599 return -EMSGSIZE;
600 }
601
602 static int bpf_sk_storage_diag_put_all(struct sock *sk, struct sk_buff *skb,
603 int stg_array_type,
604 unsigned int *res_diag_size)
605 {
606 /* stg_array_type (e.g. INET_DIAG_BPF_SK_STORAGES) */
607 unsigned int diag_size = nla_total_size(0);
608 struct bpf_local_storage *sk_storage;
609 struct bpf_local_storage_elem *selem;
610 struct bpf_local_storage_map *smap;
611 struct nlattr *nla_stgs;
612 unsigned int saved_len;
613 int err = 0;
614
615 rcu_read_lock();
616
617 sk_storage = rcu_dereference(sk->sk_bpf_storage);
618 if (!sk_storage || hlist_empty(&sk_storage->list)) {
619 rcu_read_unlock();
620 return 0;
621 }
622
623 nla_stgs = nla_nest_start(skb, stg_array_type);
624 if (!nla_stgs)
625 /* Continue to learn diag_size */
626 err = -EMSGSIZE;
627
628 saved_len = skb->len;
629 hlist_for_each_entry_rcu(selem, &sk_storage->list, snode) {
630 smap = rcu_dereference(SDATA(selem)->smap);
631 diag_size += nla_value_size(smap->map.value_size);
632
633 if (nla_stgs && diag_get(SDATA(selem), skb))
634 /* Continue to learn diag_size */
635 err = -EMSGSIZE;
636 }
637
638 rcu_read_unlock();
639
640 if (nla_stgs) {
641 if (saved_len == skb->len)
642 nla_nest_cancel(skb, nla_stgs);
643 else
644 nla_nest_end(skb, nla_stgs);
645 }
646
647 if (diag_size == nla_total_size(0)) {
648 *res_diag_size = 0;
649 return 0;
650 }
651
652 *res_diag_size = diag_size;
653 return err;
654 }
655
656 int bpf_sk_storage_diag_put(struct bpf_sk_storage_diag *diag,
657 struct sock *sk, struct sk_buff *skb,
658 int stg_array_type,
659 unsigned int *res_diag_size)
660 {
661 /* stg_array_type (e.g. INET_DIAG_BPF_SK_STORAGES) */
662 unsigned int diag_size = nla_total_size(0);
663 struct bpf_local_storage *sk_storage;
664 struct bpf_local_storage_data *sdata;
665 struct nlattr *nla_stgs;
666 unsigned int saved_len;
667 int err = 0;
668 u32 i;
669
670 *res_diag_size = 0;
671
672 /* No map has been specified. Dump all. */
673 if (!diag->nr_maps)
674 return bpf_sk_storage_diag_put_all(sk, skb, stg_array_type,
675 res_diag_size);
676
677 rcu_read_lock();
678 sk_storage = rcu_dereference(sk->sk_bpf_storage);
679 if (!sk_storage || hlist_empty(&sk_storage->list)) {
680 rcu_read_unlock();
681 return 0;
682 }
683
684 nla_stgs = nla_nest_start(skb, stg_array_type);
685 if (!nla_stgs)
686 /* Continue to learn diag_size */
687 err = -EMSGSIZE;
688
689 saved_len = skb->len;
690 for (i = 0; i < diag->nr_maps; i++) {
691 sdata = bpf_local_storage_lookup(sk_storage,
692 (struct bpf_local_storage_map *)diag->maps[i],
693 false);
694
695 if (!sdata)
696 continue;
697
698 diag_size += nla_value_size(diag->maps[i]->value_size);
699
700 if (nla_stgs && diag_get(sdata, skb))
701 /* Continue to learn diag_size */
702 err = -EMSGSIZE;
703 }
704 rcu_read_unlock();
705
706 if (nla_stgs) {
707 if (saved_len == skb->len)
708 nla_nest_cancel(skb, nla_stgs);
709 else
710 nla_nest_end(skb, nla_stgs);
711 }
712
713 if (diag_size == nla_total_size(0)) {
714 *res_diag_size = 0;
715 return 0;
716 }
717
718 *res_diag_size = diag_size;
719 return err;
720 }
721 EXPORT_SYMBOL_GPL(bpf_sk_storage_diag_put);
722
723 struct bpf_iter_seq_sk_storage_map_info {
724 struct bpf_map *map;
725 unsigned int bucket_id;
726 unsigned skip_elems;
727 };
728
729 static struct bpf_local_storage_elem *
730 bpf_sk_storage_map_seq_find_next(struct bpf_iter_seq_sk_storage_map_info *info,
731 struct bpf_local_storage_elem *prev_selem)
732 __acquires(RCU) __releases(RCU)
733 {
734 struct bpf_local_storage *sk_storage;
735 struct bpf_local_storage_elem *selem;
736 u32 skip_elems = info->skip_elems;
737 struct bpf_local_storage_map *smap;
738 u32 bucket_id = info->bucket_id;
739 u32 i, count, n_buckets;
740 struct bpf_local_storage_map_bucket *b;
741
742 smap = (struct bpf_local_storage_map *)info->map;
743 n_buckets = 1U << smap->bucket_log;
744 if (bucket_id >= n_buckets)
745 return NULL;
746
747 /* try to find next selem in the same bucket */
748 selem = prev_selem;
749 count = 0;
750 while (selem) {
751 selem = hlist_entry_safe(rcu_dereference(hlist_next_rcu(&selem->map_node)),
752 struct bpf_local_storage_elem, map_node);
753 if (!selem) {
754 /* not found, unlock and go to the next bucket */
755 b = &smap->buckets[bucket_id++];
756 rcu_read_unlock();
757 skip_elems = 0;
758 break;
759 }
760 sk_storage = rcu_dereference(selem->local_storage);
761 if (sk_storage) {
762 info->skip_elems = skip_elems + count;
763 return selem;
764 }
765 count++;
766 }
767
768 for (i = bucket_id; i < (1U << smap->bucket_log); i++) {
769 b = &smap->buckets[i];
770 rcu_read_lock();
771 count = 0;
772 hlist_for_each_entry_rcu(selem, &b->list, map_node) {
773 sk_storage = rcu_dereference(selem->local_storage);
774 if (sk_storage && count >= skip_elems) {
775 info->bucket_id = i;
776 info->skip_elems = count;
777 return selem;
778 }
779 count++;
780 }
781 rcu_read_unlock();
782 skip_elems = 0;
783 }
784
785 info->bucket_id = i;
786 info->skip_elems = 0;
787 return NULL;
788 }
789
790 static void *bpf_sk_storage_map_seq_start(struct seq_file *seq, loff_t *pos)
791 {
792 struct bpf_local_storage_elem *selem;
793
794 selem = bpf_sk_storage_map_seq_find_next(seq->private, NULL);
795 if (!selem)
796 return NULL;
797
798 if (*pos == 0)
799 ++*pos;
800 return selem;
801 }
802
803 static void *bpf_sk_storage_map_seq_next(struct seq_file *seq, void *v,
804 loff_t *pos)
805 {
806 struct bpf_iter_seq_sk_storage_map_info *info = seq->private;
807
808 ++*pos;
809 ++info->skip_elems;
810 return bpf_sk_storage_map_seq_find_next(seq->private, v);
811 }
812
813 struct bpf_iter__bpf_sk_storage_map {
814 __bpf_md_ptr(struct bpf_iter_meta *, meta);
815 __bpf_md_ptr(struct bpf_map *, map);
816 __bpf_md_ptr(struct sock *, sk);
817 __bpf_md_ptr(void *, value);
818 };
819
820 DEFINE_BPF_ITER_FUNC(bpf_sk_storage_map, struct bpf_iter_meta *meta,
821 struct bpf_map *map, struct sock *sk,
822 void *value)
823
824 static int __bpf_sk_storage_map_seq_show(struct seq_file *seq,
825 struct bpf_local_storage_elem *selem)
826 {
827 struct bpf_iter_seq_sk_storage_map_info *info = seq->private;
828 struct bpf_iter__bpf_sk_storage_map ctx = {};
829 struct bpf_local_storage *sk_storage;
830 struct bpf_iter_meta meta;
831 struct bpf_prog *prog;
832 int ret = 0;
833
834 meta.seq = seq;
835 prog = bpf_iter_get_info(&meta, selem == NULL);
836 if (prog) {
837 ctx.meta = &meta;
838 ctx.map = info->map;
839 if (selem) {
840 sk_storage = rcu_dereference(selem->local_storage);
841 ctx.sk = sk_storage->owner;
842 ctx.value = SDATA(selem)->data;
843 }
844 ret = bpf_iter_run_prog(prog, &ctx);
845 }
846
847 return ret;
848 }
849
850 static int bpf_sk_storage_map_seq_show(struct seq_file *seq, void *v)
851 {
852 return __bpf_sk_storage_map_seq_show(seq, v);
853 }
854
855 static void bpf_sk_storage_map_seq_stop(struct seq_file *seq, void *v)
856 __releases(RCU)
857 {
858 if (!v)
859 (void)__bpf_sk_storage_map_seq_show(seq, v);
860 else
861 rcu_read_unlock();
862 }
863
864 static int bpf_iter_init_sk_storage_map(void *priv_data,
865 struct bpf_iter_aux_info *aux)
866 {
867 struct bpf_iter_seq_sk_storage_map_info *seq_info = priv_data;
868
869 seq_info->map = aux->map;
870 return 0;
871 }
872
873 static int bpf_iter_attach_map(struct bpf_prog *prog,
874 union bpf_iter_link_info *linfo,
875 struct bpf_iter_aux_info *aux)
876 {
877 struct bpf_map *map;
878 int err = -EINVAL;
879
880 if (!linfo->map.map_fd)
881 return -EBADF;
882
883 map = bpf_map_get_with_uref(linfo->map.map_fd);
884 if (IS_ERR(map))
885 return PTR_ERR(map);
886
887 if (map->map_type != BPF_MAP_TYPE_SK_STORAGE)
888 goto put_map;
889
890 if (prog->aux->max_rdonly_access > map->value_size) {
891 err = -EACCES;
892 goto put_map;
893 }
894
895 aux->map = map;
896 return 0;
897
898 put_map:
899 bpf_map_put_with_uref(map);
900 return err;
901 }
902
903 static void bpf_iter_detach_map(struct bpf_iter_aux_info *aux)
904 {
905 bpf_map_put_with_uref(aux->map);
906 }
907
908 static const struct seq_operations bpf_sk_storage_map_seq_ops = {
909 .start = bpf_sk_storage_map_seq_start,
910 .next = bpf_sk_storage_map_seq_next,
911 .stop = bpf_sk_storage_map_seq_stop,
912 .show = bpf_sk_storage_map_seq_show,
913 };
914
915 static const struct bpf_iter_seq_info iter_seq_info = {
916 .seq_ops = &bpf_sk_storage_map_seq_ops,
917 .init_seq_private = bpf_iter_init_sk_storage_map,
918 .fini_seq_private = NULL,
919 .seq_priv_size = sizeof(struct bpf_iter_seq_sk_storage_map_info),
920 };
921
922 static struct bpf_iter_reg bpf_sk_storage_map_reg_info = {
923 .target = "bpf_sk_storage_map",
924 .attach_target = bpf_iter_attach_map,
925 .detach_target = bpf_iter_detach_map,
926 .show_fdinfo = bpf_iter_map_show_fdinfo,
927 .fill_link_info = bpf_iter_map_fill_link_info,
928 .ctx_arg_info_size = 2,
929 .ctx_arg_info = {
930 { offsetof(struct bpf_iter__bpf_sk_storage_map, sk),
931 PTR_TO_BTF_ID_OR_NULL },
932 { offsetof(struct bpf_iter__bpf_sk_storage_map, value),
933 PTR_TO_RDWR_BUF_OR_NULL },
934 },
935 .seq_info = &iter_seq_info,
936 };
937
938 static int __init bpf_sk_storage_map_iter_init(void)
939 {
940 bpf_sk_storage_map_reg_info.ctx_arg_info[0].btf_id =
941 btf_sock_ids[BTF_SOCK_TYPE_SOCK];
942 return bpf_iter_reg_target(&bpf_sk_storage_map_reg_info);
943 }
944 late_initcall(bpf_sk_storage_map_iter_init);
Linux with added FPC-III support