?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
mm/slab: sanity-check page type when looking up cache
[linux/fpc-iii.git] / kernel / bpf / syscall.c
blob5d141f16f6fa95dd21fbacdc6000521d26a394c0
1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
3 */
4 #include <linux/bpf.h>
5 #include <linux/bpf_trace.h>
6 #include <linux/bpf_lirc.h>
7 #include <linux/btf.h>
8 #include <linux/syscalls.h>
9 #include <linux/slab.h>
10 #include <linux/sched/signal.h>
11 #include <linux/vmalloc.h>
12 #include <linux/mmzone.h>
13 #include <linux/anon_inodes.h>
14 #include <linux/fdtable.h>
15 #include <linux/file.h>
16 #include <linux/fs.h>
17 #include <linux/license.h>
18 #include <linux/filter.h>
19 #include <linux/version.h>
20 #include <linux/kernel.h>
21 #include <linux/idr.h>
22 #include <linux/cred.h>
23 #include <linux/timekeeping.h>
24 #include <linux/ctype.h>
25 #include <linux/nospec.h>
26
27 #define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY || \
28 (map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \
29 (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \
30 (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)
31 #define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS)
32 #define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_HASH(map))
33
34 #define BPF_OBJ_FLAG_MASK (BPF_F_RDONLY | BPF_F_WRONLY)
35
36 DEFINE_PER_CPU(int, bpf_prog_active);
37 static DEFINE_IDR(prog_idr);
38 static DEFINE_SPINLOCK(prog_idr_lock);
39 static DEFINE_IDR(map_idr);
40 static DEFINE_SPINLOCK(map_idr_lock);
41
42 int sysctl_unprivileged_bpf_disabled __read_mostly;
43
44 static const struct bpf_map_ops * const bpf_map_types[] = {
45 #define BPF_PROG_TYPE(_id, _ops)
46 #define BPF_MAP_TYPE(_id, _ops) \
47 [_id] = &_ops,
48 #include <linux/bpf_types.h>
49 #undef BPF_PROG_TYPE
50 #undef BPF_MAP_TYPE
51 };
52
53 /*
54 * If we're handed a bigger struct than we know of, ensure all the unknown bits
55 * are 0 - i.e. new user-space does not rely on any kernel feature extensions
56 * we don't know about yet.
57 *
58 * There is a ToCToU between this function call and the following
59 * copy_from_user() call. However, this is not a concern since this function is
60 * meant to be a future-proofing of bits.
61 */
62 int bpf_check_uarg_tail_zero(void __user *uaddr,
63 size_t expected_size,
64 size_t actual_size)
65 {
66 unsigned char __user *addr;
67 unsigned char __user *end;
68 unsigned char val;
69 int err;
70
71 if (unlikely(actual_size > PAGE_SIZE)) /* silly large */
72 return -E2BIG;
73
74 if (unlikely(!access_ok(uaddr, actual_size)))
75 return -EFAULT;
76
77 if (actual_size <= expected_size)
78 return 0;
79
80 addr = uaddr + expected_size;
81 end = uaddr + actual_size;
82
83 for (; addr < end; addr++) {
84 err = get_user(val, addr);
85 if (err)
86 return err;
87 if (val)
88 return -E2BIG;
89 }
90
91 return 0;
92 }
93
94 const struct bpf_map_ops bpf_map_offload_ops = {
95 .map_alloc = bpf_map_offload_map_alloc,
96 .map_free = bpf_map_offload_map_free,
97 .map_check_btf = map_check_no_btf,
98 };
99
100 static struct bpf_map *find_and_alloc_map(union bpf_attr *attr)
101 {
102 const struct bpf_map_ops *ops;
103 u32 type = attr->map_type;
104 struct bpf_map *map;
105 int err;
106
107 if (type >= ARRAY_SIZE(bpf_map_types))
108 return ERR_PTR(-EINVAL);
109 type = array_index_nospec(type, ARRAY_SIZE(bpf_map_types));
110 ops = bpf_map_types[type];
111 if (!ops)
112 return ERR_PTR(-EINVAL);
113
114 if (ops->map_alloc_check) {
115 err = ops->map_alloc_check(attr);
116 if (err)
117 return ERR_PTR(err);
118 }
119 if (attr->map_ifindex)
120 ops = &bpf_map_offload_ops;
121 map = ops->map_alloc(attr);
122 if (IS_ERR(map))
123 return map;
124 map->ops = ops;
125 map->map_type = type;
126 return map;
127 }
128
129 void *bpf_map_area_alloc(size_t size, int numa_node)
130 {
131 /* We really just want to fail instead of triggering OOM killer
132 * under memory pressure, therefore we set __GFP_NORETRY to kmalloc,
133 * which is used for lower order allocation requests.
134 *
135 * It has been observed that higher order allocation requests done by
136 * vmalloc with __GFP_NORETRY being set might fail due to not trying
137 * to reclaim memory from the page cache, thus we set
138 * __GFP_RETRY_MAYFAIL to avoid such situations.
139 */
140
141 const gfp_t flags = __GFP_NOWARN | __GFP_ZERO;
142 void *area;
143
144 if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) {
145 area = kmalloc_node(size, GFP_USER | __GFP_NORETRY | flags,
146 numa_node);
147 if (area != NULL)
148 return area;
149 }
150
151 return __vmalloc_node_flags_caller(size, numa_node,
152 GFP_KERNEL | __GFP_RETRY_MAYFAIL |
153 flags, __builtin_return_address(0));
154 }
155
156 void bpf_map_area_free(void *area)
157 {
158 kvfree(area);
159 }
160
161 static u32 bpf_map_flags_retain_permanent(u32 flags)
162 {
163 /* Some map creation flags are not tied to the map object but
164 * rather to the map fd instead, so they have no meaning upon
165 * map object inspection since multiple file descriptors with
166 * different (access) properties can exist here. Thus, given
167 * this has zero meaning for the map itself, lets clear these
168 * from here.
169 */
170 return flags & ~(BPF_F_RDONLY | BPF_F_WRONLY);
171 }
172
173 void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr)
174 {
175 map->map_type = attr->map_type;
176 map->key_size = attr->key_size;
177 map->value_size = attr->value_size;
178 map->max_entries = attr->max_entries;
179 map->map_flags = bpf_map_flags_retain_permanent(attr->map_flags);
180 map->numa_node = bpf_map_attr_numa_node(attr);
181 }
182
183 static int bpf_charge_memlock(struct user_struct *user, u32 pages)
184 {
185 unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
186
187 if (atomic_long_add_return(pages, &user->locked_vm) > memlock_limit) {
188 atomic_long_sub(pages, &user->locked_vm);
189 return -EPERM;
190 }
191 return 0;
192 }
193
194 static void bpf_uncharge_memlock(struct user_struct *user, u32 pages)
195 {
196 if (user)
197 atomic_long_sub(pages, &user->locked_vm);
198 }
199
200 int bpf_map_charge_init(struct bpf_map_memory *mem, size_t size)
201 {
202 u32 pages = round_up(size, PAGE_SIZE) >> PAGE_SHIFT;
203 struct user_struct *user;
204 int ret;
205
206 if (size >= U32_MAX - PAGE_SIZE)
207 return -E2BIG;
208
209 user = get_current_user();
210 ret = bpf_charge_memlock(user, pages);
211 if (ret) {
212 free_uid(user);
213 return ret;
214 }
215
216 mem->pages = pages;
217 mem->user = user;
218
219 return 0;
220 }
221
222 void bpf_map_charge_finish(struct bpf_map_memory *mem)
223 {
224 bpf_uncharge_memlock(mem->user, mem->pages);
225 free_uid(mem->user);
226 }
227
228 void bpf_map_charge_move(struct bpf_map_memory *dst,
229 struct bpf_map_memory *src)
230 {
231 *dst = *src;
232
233 /* Make sure src will not be used for the redundant uncharging. */
234 memset(src, 0, sizeof(struct bpf_map_memory));
235 }
236
237 int bpf_map_charge_memlock(struct bpf_map *map, u32 pages)
238 {
239 int ret;
240
241 ret = bpf_charge_memlock(map->memory.user, pages);
242 if (ret)
243 return ret;
244 map->memory.pages += pages;
245 return ret;
246 }
247
248 void bpf_map_uncharge_memlock(struct bpf_map *map, u32 pages)
249 {
250 bpf_uncharge_memlock(map->memory.user, pages);
251 map->memory.pages -= pages;
252 }
253
254 static int bpf_map_alloc_id(struct bpf_map *map)
255 {
256 int id;
257
258 idr_preload(GFP_KERNEL);
259 spin_lock_bh(&map_idr_lock);
260 id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC);
261 if (id > 0)
262 map->id = id;
263 spin_unlock_bh(&map_idr_lock);
264 idr_preload_end();
265
266 if (WARN_ON_ONCE(!id))
267 return -ENOSPC;
268
269 return id > 0 ? 0 : id;
270 }
271
272 void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock)
273 {
274 unsigned long flags;
275
276 /* Offloaded maps are removed from the IDR store when their device
277 * disappears - even if someone holds an fd to them they are unusable,
278 * the memory is gone, all ops will fail; they are simply waiting for
279 * refcnt to drop to be freed.
280 */
281 if (!map->id)
282 return;
283
284 if (do_idr_lock)
285 spin_lock_irqsave(&map_idr_lock, flags);
286 else
287 __acquire(&map_idr_lock);
288
289 idr_remove(&map_idr, map->id);
290 map->id = 0;
291
292 if (do_idr_lock)
293 spin_unlock_irqrestore(&map_idr_lock, flags);
294 else
295 __release(&map_idr_lock);
296 }
297
298 /* called from workqueue */
299 static void bpf_map_free_deferred(struct work_struct *work)
300 {
301 struct bpf_map *map = container_of(work, struct bpf_map, work);
302 struct bpf_map_memory mem;
303
304 bpf_map_charge_move(&mem, &map->memory);
305 security_bpf_map_free(map);
306 /* implementation dependent freeing */
307 map->ops->map_free(map);
308 bpf_map_charge_finish(&mem);
309 }
310
311 static void bpf_map_put_uref(struct bpf_map *map)
312 {
313 if (atomic_dec_and_test(&map->usercnt)) {
314 if (map->ops->map_release_uref)
315 map->ops->map_release_uref(map);
316 }
317 }
318
319 /* decrement map refcnt and schedule it for freeing via workqueue
320 * (unrelying map implementation ops->map_free() might sleep)
321 */
322 static void __bpf_map_put(struct bpf_map *map, bool do_idr_lock)
323 {
324 if (atomic_dec_and_test(&map->refcnt)) {
325 /* bpf_map_free_id() must be called first */
326 bpf_map_free_id(map, do_idr_lock);
327 btf_put(map->btf);
328 INIT_WORK(&map->work, bpf_map_free_deferred);
329 schedule_work(&map->work);
330 }
331 }
332
333 void bpf_map_put(struct bpf_map *map)
334 {
335 __bpf_map_put(map, true);
336 }
337 EXPORT_SYMBOL_GPL(bpf_map_put);
338
339 void bpf_map_put_with_uref(struct bpf_map *map)
340 {
341 bpf_map_put_uref(map);
342 bpf_map_put(map);
343 }
344
345 static int bpf_map_release(struct inode *inode, struct file *filp)
346 {
347 struct bpf_map *map = filp->private_data;
348
349 if (map->ops->map_release)
350 map->ops->map_release(map, filp);
351
352 bpf_map_put_with_uref(map);
353 return 0;
354 }
355
356 static fmode_t map_get_sys_perms(struct bpf_map *map, struct fd f)
357 {
358 fmode_t mode = f.file->f_mode;
359
360 /* Our file permissions may have been overridden by global
361 * map permissions facing syscall side.
362 */
363 if (READ_ONCE(map->frozen))
364 mode &= ~FMODE_CAN_WRITE;
365 return mode;
366 }
367
368 #ifdef CONFIG_PROC_FS
369 static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp)
370 {
371 const struct bpf_map *map = filp->private_data;
372 const struct bpf_array *array;
373 u32 owner_prog_type = 0;
374 u32 owner_jited = 0;
375
376 if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) {
377 array = container_of(map, struct bpf_array, map);
378 owner_prog_type = array->owner_prog_type;
379 owner_jited = array->owner_jited;
380 }
381
382 seq_printf(m,
383 "map_type:\t%u\n"
384 "key_size:\t%u\n"
385 "value_size:\t%u\n"
386 "max_entries:\t%u\n"
387 "map_flags:\t%#x\n"
388 "memlock:\t%llu\n"
389 "map_id:\t%u\n"
390 "frozen:\t%u\n",
391 map->map_type,
392 map->key_size,
393 map->value_size,
394 map->max_entries,
395 map->map_flags,
396 map->memory.pages * 1ULL << PAGE_SHIFT,
397 map->id,
398 READ_ONCE(map->frozen));
399
400 if (owner_prog_type) {
401 seq_printf(m, "owner_prog_type:\t%u\n",
402 owner_prog_type);
403 seq_printf(m, "owner_jited:\t%u\n",
404 owner_jited);
405 }
406 }
407 #endif
408
409 static ssize_t bpf_dummy_read(struct file *filp, char __user *buf, size_t siz,
410 loff_t *ppos)
411 {
412 /* We need this handler such that alloc_file() enables
413 * f_mode with FMODE_CAN_READ.
414 */
415 return -EINVAL;
416 }
417
418 static ssize_t bpf_dummy_write(struct file *filp, const char __user *buf,
419 size_t siz, loff_t *ppos)
420 {
421 /* We need this handler such that alloc_file() enables
422 * f_mode with FMODE_CAN_WRITE.
423 */
424 return -EINVAL;
425 }
426
427 const struct file_operations bpf_map_fops = {
428 #ifdef CONFIG_PROC_FS
429 .show_fdinfo = bpf_map_show_fdinfo,
430 #endif
431 .release = bpf_map_release,
432 .read = bpf_dummy_read,
433 .write = bpf_dummy_write,
434 };
435
436 int bpf_map_new_fd(struct bpf_map *map, int flags)
437 {
438 int ret;
439
440 ret = security_bpf_map(map, OPEN_FMODE(flags));
441 if (ret < 0)
442 return ret;
443
444 return anon_inode_getfd("bpf-map", &bpf_map_fops, map,
445 flags | O_CLOEXEC);
446 }
447
448 int bpf_get_file_flag(int flags)
449 {
450 if ((flags & BPF_F_RDONLY) && (flags & BPF_F_WRONLY))
451 return -EINVAL;
452 if (flags & BPF_F_RDONLY)
453 return O_RDONLY;
454 if (flags & BPF_F_WRONLY)
455 return O_WRONLY;
456 return O_RDWR;
457 }
458
459 /* helper macro to check that unused fields 'union bpf_attr' are zero */
460 #define CHECK_ATTR(CMD) \
461 memchr_inv((void *) &attr->CMD##_LAST_FIELD + \
462 sizeof(attr->CMD##_LAST_FIELD), 0, \
463 sizeof(*attr) - \
464 offsetof(union bpf_attr, CMD##_LAST_FIELD) - \
465 sizeof(attr->CMD##_LAST_FIELD)) != NULL
466
467 /* dst and src must have at least BPF_OBJ_NAME_LEN number of bytes.
468 * Return 0 on success and < 0 on error.
469 */
470 static int bpf_obj_name_cpy(char *dst, const char *src)
471 {
472 const char *end = src + BPF_OBJ_NAME_LEN;
473
474 memset(dst, 0, BPF_OBJ_NAME_LEN);
475 /* Copy all isalnum(), '_' and '.' chars. */
476 while (src < end && *src) {
477 if (!isalnum(*src) &&
478 *src != '_' && *src != '.')
479 return -EINVAL;
480 *dst++ = *src++;
481 }
482
483 /* No '\0' found in BPF_OBJ_NAME_LEN number of bytes */
484 if (src == end)
485 return -EINVAL;
486
487 return 0;
488 }
489
490 int map_check_no_btf(const struct bpf_map *map,
491 const struct btf *btf,
492 const struct btf_type *key_type,
493 const struct btf_type *value_type)
494 {
495 return -ENOTSUPP;
496 }
497
498 static int map_check_btf(struct bpf_map *map, const struct btf *btf,
499 u32 btf_key_id, u32 btf_value_id)
500 {
501 const struct btf_type *key_type, *value_type;
502 u32 key_size, value_size;
503 int ret = 0;
504
505 /* Some maps allow key to be unspecified. */
506 if (btf_key_id) {
507 key_type = btf_type_id_size(btf, &btf_key_id, &key_size);
508 if (!key_type || key_size != map->key_size)
509 return -EINVAL;
510 } else {
511 key_type = btf_type_by_id(btf, 0);
512 if (!map->ops->map_check_btf)
513 return -EINVAL;
514 }
515
516 value_type = btf_type_id_size(btf, &btf_value_id, &value_size);
517 if (!value_type || value_size != map->value_size)
518 return -EINVAL;
519
520 map->spin_lock_off = btf_find_spin_lock(btf, value_type);
521
522 if (map_value_has_spin_lock(map)) {
523 if (map->map_flags & BPF_F_RDONLY_PROG)
524 return -EACCES;
525 if (map->map_type != BPF_MAP_TYPE_HASH &&
526 map->map_type != BPF_MAP_TYPE_ARRAY &&
527 map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE &&
528 map->map_type != BPF_MAP_TYPE_SK_STORAGE)
529 return -ENOTSUPP;
530 if (map->spin_lock_off + sizeof(struct bpf_spin_lock) >
531 map->value_size) {
532 WARN_ONCE(1,
533 "verifier bug spin_lock_off %d value_size %d\n",
534 map->spin_lock_off, map->value_size);
535 return -EFAULT;
536 }
537 }
538
539 if (map->ops->map_check_btf)
540 ret = map->ops->map_check_btf(map, btf, key_type, value_type);
541
542 return ret;
543 }
544
545 #define BPF_MAP_CREATE_LAST_FIELD btf_value_type_id
546 /* called via syscall */
547 static int map_create(union bpf_attr *attr)
548 {
549 int numa_node = bpf_map_attr_numa_node(attr);
550 struct bpf_map_memory mem;
551 struct bpf_map *map;
552 int f_flags;
553 int err;
554
555 err = CHECK_ATTR(BPF_MAP_CREATE);
556 if (err)
557 return -EINVAL;
558
559 f_flags = bpf_get_file_flag(attr->map_flags);
560 if (f_flags < 0)
561 return f_flags;
562
563 if (numa_node != NUMA_NO_NODE &&
564 ((unsigned int)numa_node >= nr_node_ids ||
565 !node_online(numa_node)))
566 return -EINVAL;
567
568 /* find map type and init map: hashtable vs rbtree vs bloom vs ... */
569 map = find_and_alloc_map(attr);
570 if (IS_ERR(map))
571 return PTR_ERR(map);
572
573 err = bpf_obj_name_cpy(map->name, attr->map_name);
574 if (err)
575 goto free_map;
576
577 atomic_set(&map->refcnt, 1);
578 atomic_set(&map->usercnt, 1);
579
580 if (attr->btf_key_type_id || attr->btf_value_type_id) {
581 struct btf *btf;
582
583 if (!attr->btf_value_type_id) {
584 err = -EINVAL;
585 goto free_map;
586 }
587
588 btf = btf_get_by_fd(attr->btf_fd);
589 if (IS_ERR(btf)) {
590 err = PTR_ERR(btf);
591 goto free_map;
592 }
593
594 err = map_check_btf(map, btf, attr->btf_key_type_id,
595 attr->btf_value_type_id);
596 if (err) {
597 btf_put(btf);
598 goto free_map;
599 }
600
601 map->btf = btf;
602 map->btf_key_type_id = attr->btf_key_type_id;
603 map->btf_value_type_id = attr->btf_value_type_id;
604 } else {
605 map->spin_lock_off = -EINVAL;
606 }
607
608 err = security_bpf_map_alloc(map);
609 if (err)
610 goto free_map;
611
612 err = bpf_map_alloc_id(map);
613 if (err)
614 goto free_map_sec;
615
616 err = bpf_map_new_fd(map, f_flags);
617 if (err < 0) {
618 /* failed to allocate fd.
619 * bpf_map_put_with_uref() is needed because the above
620 * bpf_map_alloc_id() has published the map
621 * to the userspace and the userspace may
622 * have refcnt-ed it through BPF_MAP_GET_FD_BY_ID.
623 */
624 bpf_map_put_with_uref(map);
625 return err;
626 }
627
628 return err;
629
630 free_map_sec:
631 security_bpf_map_free(map);
632 free_map:
633 btf_put(map->btf);
634 bpf_map_charge_move(&mem, &map->memory);
635 map->ops->map_free(map);
636 bpf_map_charge_finish(&mem);
637 return err;
638 }
639
640 /* if error is returned, fd is released.
641 * On success caller should complete fd access with matching fdput()
642 */
643 struct bpf_map *__bpf_map_get(struct fd f)
644 {
645 if (!f.file)
646 return ERR_PTR(-EBADF);
647 if (f.file->f_op != &bpf_map_fops) {
648 fdput(f);
649 return ERR_PTR(-EINVAL);
650 }
651
652 return f.file->private_data;
653 }
654
655 /* prog's and map's refcnt limit */
656 #define BPF_MAX_REFCNT 32768
657
658 struct bpf_map *bpf_map_inc(struct bpf_map *map, bool uref)
659 {
660 if (atomic_inc_return(&map->refcnt) > BPF_MAX_REFCNT) {
661 atomic_dec(&map->refcnt);
662 return ERR_PTR(-EBUSY);
663 }
664 if (uref)
665 atomic_inc(&map->usercnt);
666 return map;
667 }
668 EXPORT_SYMBOL_GPL(bpf_map_inc);
669
670 struct bpf_map *bpf_map_get_with_uref(u32 ufd)
671 {
672 struct fd f = fdget(ufd);
673 struct bpf_map *map;
674
675 map = __bpf_map_get(f);
676 if (IS_ERR(map))
677 return map;
678
679 map = bpf_map_inc(map, true);
680 fdput(f);
681
682 return map;
683 }
684
685 /* map_idr_lock should have been held */
686 static struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map,
687 bool uref)
688 {
689 int refold;
690
691 refold = atomic_fetch_add_unless(&map->refcnt, 1, 0);
692
693 if (refold >= BPF_MAX_REFCNT) {
694 __bpf_map_put(map, false);
695 return ERR_PTR(-EBUSY);
696 }
697
698 if (!refold)
699 return ERR_PTR(-ENOENT);
700
701 if (uref)
702 atomic_inc(&map->usercnt);
703
704 return map;
705 }
706
707 int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value)
708 {
709 return -ENOTSUPP;
710 }
711
712 static void *__bpf_copy_key(void __user *ukey, u64 key_size)
713 {
714 if (key_size)
715 return memdup_user(ukey, key_size);
716
717 if (ukey)
718 return ERR_PTR(-EINVAL);
719
720 return NULL;
721 }
722
723 /* last field in 'union bpf_attr' used by this command */
724 #define BPF_MAP_LOOKUP_ELEM_LAST_FIELD flags
725
726 static int map_lookup_elem(union bpf_attr *attr)
727 {
728 void __user *ukey = u64_to_user_ptr(attr->key);
729 void __user *uvalue = u64_to_user_ptr(attr->value);
730 int ufd = attr->map_fd;
731 struct bpf_map *map;
732 void *key, *value, *ptr;
733 u32 value_size;
734 struct fd f;
735 int err;
736
737 if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM))
738 return -EINVAL;
739
740 if (attr->flags & ~BPF_F_LOCK)
741 return -EINVAL;
742
743 f = fdget(ufd);
744 map = __bpf_map_get(f);
745 if (IS_ERR(map))
746 return PTR_ERR(map);
747 if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) {
748 err = -EPERM;
749 goto err_put;
750 }
751
752 if ((attr->flags & BPF_F_LOCK) &&
753 !map_value_has_spin_lock(map)) {
754 err = -EINVAL;
755 goto err_put;
756 }
757
758 key = __bpf_copy_key(ukey, map->key_size);
759 if (IS_ERR(key)) {
760 err = PTR_ERR(key);
761 goto err_put;
762 }
763
764 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
765 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
766 map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY ||
767 map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
768 value_size = round_up(map->value_size, 8) * num_possible_cpus();
769 else if (IS_FD_MAP(map))
770 value_size = sizeof(u32);
771 else
772 value_size = map->value_size;
773
774 err = -ENOMEM;
775 value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
776 if (!value)
777 goto free_key;
778
779 if (bpf_map_is_dev_bound(map)) {
780 err = bpf_map_offload_lookup_elem(map, key, value);
781 goto done;
782 }
783
784 preempt_disable();
785 this_cpu_inc(bpf_prog_active);
786 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
787 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
788 err = bpf_percpu_hash_copy(map, key, value);
789 } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
790 err = bpf_percpu_array_copy(map, key, value);
791 } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {
792 err = bpf_percpu_cgroup_storage_copy(map, key, value);
793 } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) {
794 err = bpf_stackmap_copy(map, key, value);
795 } else if (IS_FD_ARRAY(map)) {
796 err = bpf_fd_array_map_lookup_elem(map, key, value);
797 } else if (IS_FD_HASH(map)) {
798 err = bpf_fd_htab_map_lookup_elem(map, key, value);
799 } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {
800 err = bpf_fd_reuseport_array_lookup_elem(map, key, value);
801 } else if (map->map_type == BPF_MAP_TYPE_QUEUE ||
802 map->map_type == BPF_MAP_TYPE_STACK) {
803 err = map->ops->map_peek_elem(map, value);
804 } else {
805 rcu_read_lock();
806 if (map->ops->map_lookup_elem_sys_only)
807 ptr = map->ops->map_lookup_elem_sys_only(map, key);
808 else
809 ptr = map->ops->map_lookup_elem(map, key);
810 if (IS_ERR(ptr)) {
811 err = PTR_ERR(ptr);
812 } else if (!ptr) {
813 err = -ENOENT;
814 } else {
815 err = 0;
816 if (attr->flags & BPF_F_LOCK)
817 /* lock 'ptr' and copy everything but lock */
818 copy_map_value_locked(map, value, ptr, true);
819 else
820 copy_map_value(map, value, ptr);
821 /* mask lock, since value wasn't zero inited */
822 check_and_init_map_lock(map, value);
823 }
824 rcu_read_unlock();
825 }
826 this_cpu_dec(bpf_prog_active);
827 preempt_enable();
828
829 done:
830 if (err)
831 goto free_value;
832
833 err = -EFAULT;
834 if (copy_to_user(uvalue, value, value_size) != 0)
835 goto free_value;
836
837 err = 0;
838
839 free_value:
840 kfree(value);
841 free_key:
842 kfree(key);
843 err_put:
844 fdput(f);
845 return err;
846 }
847
848 static void maybe_wait_bpf_programs(struct bpf_map *map)
849 {
850 /* Wait for any running BPF programs to complete so that
851 * userspace, when we return to it, knows that all programs
852 * that could be running use the new map value.
853 */
854 if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS ||
855 map->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)
856 synchronize_rcu();
857 }
858
859 #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags
860
861 static int map_update_elem(union bpf_attr *attr)
862 {
863 void __user *ukey = u64_to_user_ptr(attr->key);
864 void __user *uvalue = u64_to_user_ptr(attr->value);
865 int ufd = attr->map_fd;
866 struct bpf_map *map;
867 void *key, *value;
868 u32 value_size;
869 struct fd f;
870 int err;
871
872 if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM))
873 return -EINVAL;
874
875 f = fdget(ufd);
876 map = __bpf_map_get(f);
877 if (IS_ERR(map))
878 return PTR_ERR(map);
879 if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
880 err = -EPERM;
881 goto err_put;
882 }
883
884 if ((attr->flags & BPF_F_LOCK) &&
885 !map_value_has_spin_lock(map)) {
886 err = -EINVAL;
887 goto err_put;
888 }
889
890 key = __bpf_copy_key(ukey, map->key_size);
891 if (IS_ERR(key)) {
892 err = PTR_ERR(key);
893 goto err_put;
894 }
895
896 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
897 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
898 map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY ||
899 map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
900 value_size = round_up(map->value_size, 8) * num_possible_cpus();
901 else
902 value_size = map->value_size;
903
904 err = -ENOMEM;
905 value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
906 if (!value)
907 goto free_key;
908
909 err = -EFAULT;
910 if (copy_from_user(value, uvalue, value_size) != 0)
911 goto free_value;
912
913 /* Need to create a kthread, thus must support schedule */
914 if (bpf_map_is_dev_bound(map)) {
915 err = bpf_map_offload_update_elem(map, key, value, attr->flags);
916 goto out;
917 } else if (map->map_type == BPF_MAP_TYPE_CPUMAP ||
918 map->map_type == BPF_MAP_TYPE_SOCKHASH ||
919 map->map_type == BPF_MAP_TYPE_SOCKMAP) {
920 err = map->ops->map_update_elem(map, key, value, attr->flags);
921 goto out;
922 }
923
924 /* must increment bpf_prog_active to avoid kprobe+bpf triggering from
925 * inside bpf map update or delete otherwise deadlocks are possible
926 */
927 preempt_disable();
928 __this_cpu_inc(bpf_prog_active);
929 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
930 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
931 err = bpf_percpu_hash_update(map, key, value, attr->flags);
932 } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
933 err = bpf_percpu_array_update(map, key, value, attr->flags);
934 } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {
935 err = bpf_percpu_cgroup_storage_update(map, key, value,
936 attr->flags);
937 } else if (IS_FD_ARRAY(map)) {
938 rcu_read_lock();
939 err = bpf_fd_array_map_update_elem(map, f.file, key, value,
940 attr->flags);
941 rcu_read_unlock();
942 } else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) {
943 rcu_read_lock();
944 err = bpf_fd_htab_map_update_elem(map, f.file, key, value,
945 attr->flags);
946 rcu_read_unlock();
947 } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {
948 /* rcu_read_lock() is not needed */
949 err = bpf_fd_reuseport_array_update_elem(map, key, value,
950 attr->flags);
951 } else if (map->map_type == BPF_MAP_TYPE_QUEUE ||
952 map->map_type == BPF_MAP_TYPE_STACK) {
953 err = map->ops->map_push_elem(map, value, attr->flags);
954 } else {
955 rcu_read_lock();
956 err = map->ops->map_update_elem(map, key, value, attr->flags);
957 rcu_read_unlock();
958 }
959 __this_cpu_dec(bpf_prog_active);
960 preempt_enable();
961 maybe_wait_bpf_programs(map);
962 out:
963 free_value:
964 kfree(value);
965 free_key:
966 kfree(key);
967 err_put:
968 fdput(f);
969 return err;
970 }
971
972 #define BPF_MAP_DELETE_ELEM_LAST_FIELD key
973
974 static int map_delete_elem(union bpf_attr *attr)
975 {
976 void __user *ukey = u64_to_user_ptr(attr->key);
977 int ufd = attr->map_fd;
978 struct bpf_map *map;
979 struct fd f;
980 void *key;
981 int err;
982
983 if (CHECK_ATTR(BPF_MAP_DELETE_ELEM))
984 return -EINVAL;
985
986 f = fdget(ufd);
987 map = __bpf_map_get(f);
988 if (IS_ERR(map))
989 return PTR_ERR(map);
990 if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
991 err = -EPERM;
992 goto err_put;
993 }
994
995 key = __bpf_copy_key(ukey, map->key_size);
996 if (IS_ERR(key)) {
997 err = PTR_ERR(key);
998 goto err_put;
999 }
1000
1001 if (bpf_map_is_dev_bound(map)) {
1002 err = bpf_map_offload_delete_elem(map, key);
1003 goto out;
1004 }
1005
1006 preempt_disable();
1007 __this_cpu_inc(bpf_prog_active);
1008 rcu_read_lock();
1009 err = map->ops->map_delete_elem(map, key);
1010 rcu_read_unlock();
1011 __this_cpu_dec(bpf_prog_active);
1012 preempt_enable();
1013 maybe_wait_bpf_programs(map);
1014 out:
1015 kfree(key);
1016 err_put:
1017 fdput(f);
1018 return err;
1019 }
1020
1021 /* last field in 'union bpf_attr' used by this command */
1022 #define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key
1023
1024 static int map_get_next_key(union bpf_attr *attr)
1025 {
1026 void __user *ukey = u64_to_user_ptr(attr->key);
1027 void __user *unext_key = u64_to_user_ptr(attr->next_key);
1028 int ufd = attr->map_fd;
1029 struct bpf_map *map;
1030 void *key, *next_key;
1031 struct fd f;
1032 int err;
1033
1034 if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY))
1035 return -EINVAL;
1036
1037 f = fdget(ufd);
1038 map = __bpf_map_get(f);
1039 if (IS_ERR(map))
1040 return PTR_ERR(map);
1041 if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) {
1042 err = -EPERM;
1043 goto err_put;
1044 }
1045
1046 if (ukey) {
1047 key = __bpf_copy_key(ukey, map->key_size);
1048 if (IS_ERR(key)) {
1049 err = PTR_ERR(key);
1050 goto err_put;
1051 }
1052 } else {
1053 key = NULL;
1054 }
1055
1056 err = -ENOMEM;
1057 next_key = kmalloc(map->key_size, GFP_USER);
1058 if (!next_key)
1059 goto free_key;
1060
1061 if (bpf_map_is_dev_bound(map)) {
1062 err = bpf_map_offload_get_next_key(map, key, next_key);
1063 goto out;
1064 }
1065
1066 rcu_read_lock();
1067 err = map->ops->map_get_next_key(map, key, next_key);
1068 rcu_read_unlock();
1069 out:
1070 if (err)
1071 goto free_next_key;
1072
1073 err = -EFAULT;
1074 if (copy_to_user(unext_key, next_key, map->key_size) != 0)
1075 goto free_next_key;
1076
1077 err = 0;
1078
1079 free_next_key:
1080 kfree(next_key);
1081 free_key:
1082 kfree(key);
1083 err_put:
1084 fdput(f);
1085 return err;
1086 }
1087
1088 #define BPF_MAP_LOOKUP_AND_DELETE_ELEM_LAST_FIELD value
1089
1090 static int map_lookup_and_delete_elem(union bpf_attr *attr)
1091 {
1092 void __user *ukey = u64_to_user_ptr(attr->key);
1093 void __user *uvalue = u64_to_user_ptr(attr->value);
1094 int ufd = attr->map_fd;
1095 struct bpf_map *map;
1096 void *key, *value;
1097 u32 value_size;
1098 struct fd f;
1099 int err;
1100
1101 if (CHECK_ATTR(BPF_MAP_LOOKUP_AND_DELETE_ELEM))
1102 return -EINVAL;
1103
1104 f = fdget(ufd);
1105 map = __bpf_map_get(f);
1106 if (IS_ERR(map))
1107 return PTR_ERR(map);
1108 if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
1109 err = -EPERM;
1110 goto err_put;
1111 }
1112
1113 key = __bpf_copy_key(ukey, map->key_size);
1114 if (IS_ERR(key)) {
1115 err = PTR_ERR(key);
1116 goto err_put;
1117 }
1118
1119 value_size = map->value_size;
1120
1121 err = -ENOMEM;
1122 value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
1123 if (!value)
1124 goto free_key;
1125
1126 if (map->map_type == BPF_MAP_TYPE_QUEUE ||
1127 map->map_type == BPF_MAP_TYPE_STACK) {
1128 err = map->ops->map_pop_elem(map, value);
1129 } else {
1130 err = -ENOTSUPP;
1131 }
1132
1133 if (err)
1134 goto free_value;
1135
1136 if (copy_to_user(uvalue, value, value_size) != 0)
1137 goto free_value;
1138
1139 err = 0;
1140
1141 free_value:
1142 kfree(value);
1143 free_key:
1144 kfree(key);
1145 err_put:
1146 fdput(f);
1147 return err;
1148 }
1149
1150 #define BPF_MAP_FREEZE_LAST_FIELD map_fd
1151
1152 static int map_freeze(const union bpf_attr *attr)
1153 {
1154 int err = 0, ufd = attr->map_fd;
1155 struct bpf_map *map;
1156 struct fd f;
1157
1158 if (CHECK_ATTR(BPF_MAP_FREEZE))
1159 return -EINVAL;
1160
1161 f = fdget(ufd);
1162 map = __bpf_map_get(f);
1163 if (IS_ERR(map))
1164 return PTR_ERR(map);
1165 if (READ_ONCE(map->frozen)) {
1166 err = -EBUSY;
1167 goto err_put;
1168 }
1169 if (!capable(CAP_SYS_ADMIN)) {
1170 err = -EPERM;
1171 goto err_put;
1172 }
1173
1174 WRITE_ONCE(map->frozen, true);
1175 err_put:
1176 fdput(f);
1177 return err;
1178 }
1179
1180 static const struct bpf_prog_ops * const bpf_prog_types[] = {
1181 #define BPF_PROG_TYPE(_id, _name) \
1182 [_id] = & _name ## _prog_ops,
1183 #define BPF_MAP_TYPE(_id, _ops)
1184 #include <linux/bpf_types.h>
1185 #undef BPF_PROG_TYPE
1186 #undef BPF_MAP_TYPE
1187 };
1188
1189 static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog)
1190 {
1191 const struct bpf_prog_ops *ops;
1192
1193 if (type >= ARRAY_SIZE(bpf_prog_types))
1194 return -EINVAL;
1195 type = array_index_nospec(type, ARRAY_SIZE(bpf_prog_types));
1196 ops = bpf_prog_types[type];
1197 if (!ops)
1198 return -EINVAL;
1199
1200 if (!bpf_prog_is_dev_bound(prog->aux))
1201 prog->aux->ops = ops;
1202 else
1203 prog->aux->ops = &bpf_offload_prog_ops;
1204 prog->type = type;
1205 return 0;
1206 }
1207
1208 /* drop refcnt on maps used by eBPF program and free auxilary data */
1209 static void free_used_maps(struct bpf_prog_aux *aux)
1210 {
1211 enum bpf_cgroup_storage_type stype;
1212 int i;
1213
1214 for_each_cgroup_storage_type(stype) {
1215 if (!aux->cgroup_storage[stype])
1216 continue;
1217 bpf_cgroup_storage_release(aux->prog,
1218 aux->cgroup_storage[stype]);
1219 }
1220
1221 for (i = 0; i < aux->used_map_cnt; i++)
1222 bpf_map_put(aux->used_maps[i]);
1223
1224 kfree(aux->used_maps);
1225 }
1226
1227 int __bpf_prog_charge(struct user_struct *user, u32 pages)
1228 {
1229 unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
1230 unsigned long user_bufs;
1231
1232 if (user) {
1233 user_bufs = atomic_long_add_return(pages, &user->locked_vm);
1234 if (user_bufs > memlock_limit) {
1235 atomic_long_sub(pages, &user->locked_vm);
1236 return -EPERM;
1237 }
1238 }
1239
1240 return 0;
1241 }
1242
1243 void __bpf_prog_uncharge(struct user_struct *user, u32 pages)
1244 {
1245 if (user)
1246 atomic_long_sub(pages, &user->locked_vm);
1247 }
1248
1249 static int bpf_prog_charge_memlock(struct bpf_prog *prog)
1250 {
1251 struct user_struct *user = get_current_user();
1252 int ret;
1253
1254 ret = __bpf_prog_charge(user, prog->pages);
1255 if (ret) {
1256 free_uid(user);
1257 return ret;
1258 }
1259
1260 prog->aux->user = user;
1261 return 0;
1262 }
1263
1264 static void bpf_prog_uncharge_memlock(struct bpf_prog *prog)
1265 {
1266 struct user_struct *user = prog->aux->user;
1267
1268 __bpf_prog_uncharge(user, prog->pages);
1269 free_uid(user);
1270 }
1271
1272 static int bpf_prog_alloc_id(struct bpf_prog *prog)
1273 {
1274 int id;
1275
1276 idr_preload(GFP_KERNEL);
1277 spin_lock_bh(&prog_idr_lock);
1278 id = idr_alloc_cyclic(&prog_idr, prog, 1, INT_MAX, GFP_ATOMIC);
1279 if (id > 0)
1280 prog->aux->id = id;
1281 spin_unlock_bh(&prog_idr_lock);
1282 idr_preload_end();
1283
1284 /* id is in [1, INT_MAX) */
1285 if (WARN_ON_ONCE(!id))
1286 return -ENOSPC;
1287
1288 return id > 0 ? 0 : id;
1289 }
1290
1291 void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock)
1292 {
1293 /* cBPF to eBPF migrations are currently not in the idr store.
1294 * Offloaded programs are removed from the store when their device
1295 * disappears - even if someone grabs an fd to them they are unusable,
1296 * simply waiting for refcnt to drop to be freed.
1297 */
1298 if (!prog->aux->id)
1299 return;
1300
1301 if (do_idr_lock)
1302 spin_lock_bh(&prog_idr_lock);
1303 else
1304 __acquire(&prog_idr_lock);
1305
1306 idr_remove(&prog_idr, prog->aux->id);
1307 prog->aux->id = 0;
1308
1309 if (do_idr_lock)
1310 spin_unlock_bh(&prog_idr_lock);
1311 else
1312 __release(&prog_idr_lock);
1313 }
1314
1315 static void __bpf_prog_put_rcu(struct rcu_head *rcu)
1316 {
1317 struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu);
1318
1319 free_used_maps(aux);
1320 bpf_prog_uncharge_memlock(aux->prog);
1321 security_bpf_prog_free(aux);
1322 bpf_prog_free(aux->prog);
1323 }
1324
1325 static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock)
1326 {
1327 if (atomic_dec_and_test(&prog->aux->refcnt)) {
1328 perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_UNLOAD, 0);
1329 /* bpf_prog_free_id() must be called first */
1330 bpf_prog_free_id(prog, do_idr_lock);
1331 bpf_prog_kallsyms_del_all(prog);
1332 btf_put(prog->aux->btf);
1333 kvfree(prog->aux->func_info);
1334 bpf_prog_free_linfo(prog);
1335
1336 call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu);
1337 }
1338 }
1339
1340 void bpf_prog_put(struct bpf_prog *prog)
1341 {
1342 __bpf_prog_put(prog, true);
1343 }
1344 EXPORT_SYMBOL_GPL(bpf_prog_put);
1345
1346 static int bpf_prog_release(struct inode *inode, struct file *filp)
1347 {
1348 struct bpf_prog *prog = filp->private_data;
1349
1350 bpf_prog_put(prog);
1351 return 0;
1352 }
1353
1354 static void bpf_prog_get_stats(const struct bpf_prog *prog,
1355 struct bpf_prog_stats *stats)
1356 {
1357 u64 nsecs = 0, cnt = 0;
1358 int cpu;
1359
1360 for_each_possible_cpu(cpu) {
1361 const struct bpf_prog_stats *st;
1362 unsigned int start;
1363 u64 tnsecs, tcnt;
1364
1365 st = per_cpu_ptr(prog->aux->stats, cpu);
1366 do {
1367 start = u64_stats_fetch_begin_irq(&st->syncp);
1368 tnsecs = st->nsecs;
1369 tcnt = st->cnt;
1370 } while (u64_stats_fetch_retry_irq(&st->syncp, start));
1371 nsecs += tnsecs;
1372 cnt += tcnt;
1373 }
1374 stats->nsecs = nsecs;
1375 stats->cnt = cnt;
1376 }
1377
1378 #ifdef CONFIG_PROC_FS
1379 static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp)
1380 {
1381 const struct bpf_prog *prog = filp->private_data;
1382 char prog_tag[sizeof(prog->tag) * 2 + 1] = { };
1383 struct bpf_prog_stats stats;
1384
1385 bpf_prog_get_stats(prog, &stats);
1386 bin2hex(prog_tag, prog->tag, sizeof(prog->tag));
1387 seq_printf(m,
1388 "prog_type:\t%u\n"
1389 "prog_jited:\t%u\n"
1390 "prog_tag:\t%s\n"
1391 "memlock:\t%llu\n"
1392 "prog_id:\t%u\n"
1393 "run_time_ns:\t%llu\n"
1394 "run_cnt:\t%llu\n",
1395 prog->type,
1396 prog->jited,
1397 prog_tag,
1398 prog->pages * 1ULL << PAGE_SHIFT,
1399 prog->aux->id,
1400 stats.nsecs,
1401 stats.cnt);
1402 }
1403 #endif
1404
1405 const struct file_operations bpf_prog_fops = {
1406 #ifdef CONFIG_PROC_FS
1407 .show_fdinfo = bpf_prog_show_fdinfo,
1408 #endif
1409 .release = bpf_prog_release,
1410 .read = bpf_dummy_read,
1411 .write = bpf_dummy_write,
1412 };
1413
1414 int bpf_prog_new_fd(struct bpf_prog *prog)
1415 {
1416 int ret;
1417
1418 ret = security_bpf_prog(prog);
1419 if (ret < 0)
1420 return ret;
1421
1422 return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog,
1423 O_RDWR | O_CLOEXEC);
1424 }
1425
1426 static struct bpf_prog *____bpf_prog_get(struct fd f)
1427 {
1428 if (!f.file)
1429 return ERR_PTR(-EBADF);
1430 if (f.file->f_op != &bpf_prog_fops) {
1431 fdput(f);
1432 return ERR_PTR(-EINVAL);
1433 }
1434
1435 return f.file->private_data;
1436 }
1437
1438 struct bpf_prog *bpf_prog_add(struct bpf_prog *prog, int i)
1439 {
1440 if (atomic_add_return(i, &prog->aux->refcnt) > BPF_MAX_REFCNT) {
1441 atomic_sub(i, &prog->aux->refcnt);
1442 return ERR_PTR(-EBUSY);
1443 }
1444 return prog;
1445 }
1446 EXPORT_SYMBOL_GPL(bpf_prog_add);
1447
1448 void bpf_prog_sub(struct bpf_prog *prog, int i)
1449 {
1450 /* Only to be used for undoing previous bpf_prog_add() in some
1451 * error path. We still know that another entity in our call
1452 * path holds a reference to the program, thus atomic_sub() can
1453 * be safely used in such cases!
1454 */
1455 WARN_ON(atomic_sub_return(i, &prog->aux->refcnt) == 0);
1456 }
1457 EXPORT_SYMBOL_GPL(bpf_prog_sub);
1458
1459 struct bpf_prog *bpf_prog_inc(struct bpf_prog *prog)
1460 {
1461 return bpf_prog_add(prog, 1);
1462 }
1463 EXPORT_SYMBOL_GPL(bpf_prog_inc);
1464
1465 /* prog_idr_lock should have been held */
1466 struct bpf_prog *bpf_prog_inc_not_zero(struct bpf_prog *prog)
1467 {
1468 int refold;
1469
1470 refold = atomic_fetch_add_unless(&prog->aux->refcnt, 1, 0);
1471
1472 if (refold >= BPF_MAX_REFCNT) {
1473 __bpf_prog_put(prog, false);
1474 return ERR_PTR(-EBUSY);
1475 }
1476
1477 if (!refold)
1478 return ERR_PTR(-ENOENT);
1479
1480 return prog;
1481 }
1482 EXPORT_SYMBOL_GPL(bpf_prog_inc_not_zero);
1483
1484 bool bpf_prog_get_ok(struct bpf_prog *prog,
1485 enum bpf_prog_type *attach_type, bool attach_drv)
1486 {
1487 /* not an attachment, just a refcount inc, always allow */
1488 if (!attach_type)
1489 return true;
1490
1491 if (prog->type != *attach_type)
1492 return false;
1493 if (bpf_prog_is_dev_bound(prog->aux) && !attach_drv)
1494 return false;
1495
1496 return true;
1497 }
1498
1499 static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *attach_type,
1500 bool attach_drv)
1501 {
1502 struct fd f = fdget(ufd);
1503 struct bpf_prog *prog;
1504
1505 prog = ____bpf_prog_get(f);
1506 if (IS_ERR(prog))
1507 return prog;
1508 if (!bpf_prog_get_ok(prog, attach_type, attach_drv)) {
1509 prog = ERR_PTR(-EINVAL);
1510 goto out;
1511 }
1512
1513 prog = bpf_prog_inc(prog);
1514 out:
1515 fdput(f);
1516 return prog;
1517 }
1518
1519 struct bpf_prog *bpf_prog_get(u32 ufd)
1520 {
1521 return __bpf_prog_get(ufd, NULL, false);
1522 }
1523
1524 struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type,
1525 bool attach_drv)
1526 {
1527 return __bpf_prog_get(ufd, &type, attach_drv);
1528 }
1529 EXPORT_SYMBOL_GPL(bpf_prog_get_type_dev);
1530
1531 /* Initially all BPF programs could be loaded w/o specifying
1532 * expected_attach_type. Later for some of them specifying expected_attach_type
1533 * at load time became required so that program could be validated properly.
1534 * Programs of types that are allowed to be loaded both w/ and w/o (for
1535 * backward compatibility) expected_attach_type, should have the default attach
1536 * type assigned to expected_attach_type for the latter case, so that it can be
1537 * validated later at attach time.
1538 *
1539 * bpf_prog_load_fixup_attach_type() sets expected_attach_type in @attr if
1540 * prog type requires it but has some attach types that have to be backward
1541 * compatible.
1542 */
1543 static void bpf_prog_load_fixup_attach_type(union bpf_attr *attr)
1544 {
1545 switch (attr->prog_type) {
1546 case BPF_PROG_TYPE_CGROUP_SOCK:
1547 /* Unfortunately BPF_ATTACH_TYPE_UNSPEC enumeration doesn't
1548 * exist so checking for non-zero is the way to go here.
1549 */
1550 if (!attr->expected_attach_type)
1551 attr->expected_attach_type =
1552 BPF_CGROUP_INET_SOCK_CREATE;
1553 break;
1554 }
1555 }
1556
1557 static int
1558 bpf_prog_load_check_attach_type(enum bpf_prog_type prog_type,
1559 enum bpf_attach_type expected_attach_type)
1560 {
1561 switch (prog_type) {
1562 case BPF_PROG_TYPE_CGROUP_SOCK:
1563 switch (expected_attach_type) {
1564 case BPF_CGROUP_INET_SOCK_CREATE:
1565 case BPF_CGROUP_INET4_POST_BIND:
1566 case BPF_CGROUP_INET6_POST_BIND:
1567 return 0;
1568 default:
1569 return -EINVAL;
1570 }
1571 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
1572 switch (expected_attach_type) {
1573 case BPF_CGROUP_INET4_BIND:
1574 case BPF_CGROUP_INET6_BIND:
1575 case BPF_CGROUP_INET4_CONNECT:
1576 case BPF_CGROUP_INET6_CONNECT:
1577 case BPF_CGROUP_UDP4_SENDMSG:
1578 case BPF_CGROUP_UDP6_SENDMSG:
1579 case BPF_CGROUP_UDP4_RECVMSG:
1580 case BPF_CGROUP_UDP6_RECVMSG:
1581 return 0;
1582 default:
1583 return -EINVAL;
1584 }
1585 case BPF_PROG_TYPE_CGROUP_SKB:
1586 switch (expected_attach_type) {
1587 case BPF_CGROUP_INET_INGRESS:
1588 case BPF_CGROUP_INET_EGRESS:
1589 return 0;
1590 default:
1591 return -EINVAL;
1592 }
1593 case BPF_PROG_TYPE_CGROUP_SOCKOPT:
1594 switch (expected_attach_type) {
1595 case BPF_CGROUP_SETSOCKOPT:
1596 case BPF_CGROUP_GETSOCKOPT:
1597 return 0;
1598 default:
1599 return -EINVAL;
1600 }
1601 default:
1602 return 0;
1603 }
1604 }
1605
1606 /* last field in 'union bpf_attr' used by this command */
1607 #define BPF_PROG_LOAD_LAST_FIELD line_info_cnt
1608
1609 static int bpf_prog_load(union bpf_attr *attr, union bpf_attr __user *uattr)
1610 {
1611 enum bpf_prog_type type = attr->prog_type;
1612 struct bpf_prog *prog;
1613 int err;
1614 char license[128];
1615 bool is_gpl;
1616
1617 if (CHECK_ATTR(BPF_PROG_LOAD))
1618 return -EINVAL;
1619
1620 if (attr->prog_flags & ~(BPF_F_STRICT_ALIGNMENT |
1621 BPF_F_ANY_ALIGNMENT |
1622 BPF_F_TEST_RND_HI32))
1623 return -EINVAL;
1624
1625 if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) &&
1626 (attr->prog_flags & BPF_F_ANY_ALIGNMENT) &&
1627 !capable(CAP_SYS_ADMIN))
1628 return -EPERM;
1629
1630 /* copy eBPF program license from user space */
1631 if (strncpy_from_user(license, u64_to_user_ptr(attr->license),
1632 sizeof(license) - 1) < 0)
1633 return -EFAULT;
1634 license[sizeof(license) - 1] = 0;
1635
1636 /* eBPF programs must be GPL compatible to use GPL-ed functions */
1637 is_gpl = license_is_gpl_compatible(license);
1638
1639 if (attr->insn_cnt == 0 ||
1640 attr->insn_cnt > (capable(CAP_SYS_ADMIN) ? BPF_COMPLEXITY_LIMIT_INSNS : BPF_MAXINSNS))
1641 return -E2BIG;
1642 if (type != BPF_PROG_TYPE_SOCKET_FILTER &&
1643 type != BPF_PROG_TYPE_CGROUP_SKB &&
1644 !capable(CAP_SYS_ADMIN))
1645 return -EPERM;
1646
1647 bpf_prog_load_fixup_attach_type(attr);
1648 if (bpf_prog_load_check_attach_type(type, attr->expected_attach_type))
1649 return -EINVAL;
1650
1651 /* plain bpf_prog allocation */
1652 prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER);
1653 if (!prog)
1654 return -ENOMEM;
1655
1656 prog->expected_attach_type = attr->expected_attach_type;
1657
1658 prog->aux->offload_requested = !!attr->prog_ifindex;
1659
1660 err = security_bpf_prog_alloc(prog->aux);
1661 if (err)
1662 goto free_prog_nouncharge;
1663
1664 err = bpf_prog_charge_memlock(prog);
1665 if (err)
1666 goto free_prog_sec;
1667
1668 prog->len = attr->insn_cnt;
1669
1670 err = -EFAULT;
1671 if (copy_from_user(prog->insns, u64_to_user_ptr(attr->insns),
1672 bpf_prog_insn_size(prog)) != 0)
1673 goto free_prog;
1674
1675 prog->orig_prog = NULL;
1676 prog->jited = 0;
1677
1678 atomic_set(&prog->aux->refcnt, 1);
1679 prog->gpl_compatible = is_gpl ? 1 : 0;
1680
1681 if (bpf_prog_is_dev_bound(prog->aux)) {
1682 err = bpf_prog_offload_init(prog, attr);
1683 if (err)
1684 goto free_prog;
1685 }
1686
1687 /* find program type: socket_filter vs tracing_filter */
1688 err = find_prog_type(type, prog);
1689 if (err < 0)
1690 goto free_prog;
1691
1692 prog->aux->load_time = ktime_get_boottime_ns();
1693 err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name);
1694 if (err)
1695 goto free_prog;
1696
1697 /* run eBPF verifier */
1698 err = bpf_check(&prog, attr, uattr);
1699 if (err < 0)
1700 goto free_used_maps;
1701
1702 prog = bpf_prog_select_runtime(prog, &err);
1703 if (err < 0)
1704 goto free_used_maps;
1705
1706 err = bpf_prog_alloc_id(prog);
1707 if (err)
1708 goto free_used_maps;
1709
1710 err = bpf_prog_new_fd(prog);
1711 if (err < 0) {
1712 /* failed to allocate fd.
1713 * bpf_prog_put() is needed because the above
1714 * bpf_prog_alloc_id() has published the prog
1715 * to the userspace and the userspace may
1716 * have refcnt-ed it through BPF_PROG_GET_FD_BY_ID.
1717 */
1718 bpf_prog_put(prog);
1719 return err;
1720 }
1721
1722 bpf_prog_kallsyms_add(prog);
1723 perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_LOAD, 0);
1724 return err;
1725
1726 free_used_maps:
1727 bpf_prog_free_linfo(prog);
1728 kvfree(prog->aux->func_info);
1729 btf_put(prog->aux->btf);
1730 bpf_prog_kallsyms_del_subprogs(prog);
1731 free_used_maps(prog->aux);
1732 free_prog:
1733 bpf_prog_uncharge_memlock(prog);
1734 free_prog_sec:
1735 security_bpf_prog_free(prog->aux);
1736 free_prog_nouncharge:
1737 bpf_prog_free(prog);
1738 return err;
1739 }
1740
1741 #define BPF_OBJ_LAST_FIELD file_flags
1742
1743 static int bpf_obj_pin(const union bpf_attr *attr)
1744 {
1745 if (CHECK_ATTR(BPF_OBJ) || attr->file_flags != 0)
1746 return -EINVAL;
1747
1748 return bpf_obj_pin_user(attr->bpf_fd, u64_to_user_ptr(attr->pathname));
1749 }
1750
1751 static int bpf_obj_get(const union bpf_attr *attr)
1752 {
1753 if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0 ||
1754 attr->file_flags & ~BPF_OBJ_FLAG_MASK)
1755 return -EINVAL;
1756
1757 return bpf_obj_get_user(u64_to_user_ptr(attr->pathname),
1758 attr->file_flags);
1759 }
1760
1761 struct bpf_raw_tracepoint {
1762 struct bpf_raw_event_map *btp;
1763 struct bpf_prog *prog;
1764 };
1765
1766 static int bpf_raw_tracepoint_release(struct inode *inode, struct file *filp)
1767 {
1768 struct bpf_raw_tracepoint *raw_tp = filp->private_data;
1769
1770 if (raw_tp->prog) {
1771 bpf_probe_unregister(raw_tp->btp, raw_tp->prog);
1772 bpf_prog_put(raw_tp->prog);
1773 }
1774 bpf_put_raw_tracepoint(raw_tp->btp);
1775 kfree(raw_tp);
1776 return 0;
1777 }
1778
1779 static const struct file_operations bpf_raw_tp_fops = {
1780 .release = bpf_raw_tracepoint_release,
1781 .read = bpf_dummy_read,
1782 .write = bpf_dummy_write,
1783 };
1784
1785 #define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.prog_fd
1786
1787 static int bpf_raw_tracepoint_open(const union bpf_attr *attr)
1788 {
1789 struct bpf_raw_tracepoint *raw_tp;
1790 struct bpf_raw_event_map *btp;
1791 struct bpf_prog *prog;
1792 char tp_name[128];
1793 int tp_fd, err;
1794
1795 if (strncpy_from_user(tp_name, u64_to_user_ptr(attr->raw_tracepoint.name),
1796 sizeof(tp_name) - 1) < 0)
1797 return -EFAULT;
1798 tp_name[sizeof(tp_name) - 1] = 0;
1799
1800 btp = bpf_get_raw_tracepoint(tp_name);
1801 if (!btp)
1802 return -ENOENT;
1803
1804 raw_tp = kzalloc(sizeof(*raw_tp), GFP_USER);
1805 if (!raw_tp) {
1806 err = -ENOMEM;
1807 goto out_put_btp;
1808 }
1809 raw_tp->btp = btp;
1810
1811 prog = bpf_prog_get(attr->raw_tracepoint.prog_fd);
1812 if (IS_ERR(prog)) {
1813 err = PTR_ERR(prog);
1814 goto out_free_tp;
1815 }
1816 if (prog->type != BPF_PROG_TYPE_RAW_TRACEPOINT &&
1817 prog->type != BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE) {
1818 err = -EINVAL;
1819 goto out_put_prog;
1820 }
1821
1822 err = bpf_probe_register(raw_tp->btp, prog);
1823 if (err)
1824 goto out_put_prog;
1825
1826 raw_tp->prog = prog;
1827 tp_fd = anon_inode_getfd("bpf-raw-tracepoint", &bpf_raw_tp_fops, raw_tp,
1828 O_CLOEXEC);
1829 if (tp_fd < 0) {
1830 bpf_probe_unregister(raw_tp->btp, prog);
1831 err = tp_fd;
1832 goto out_put_prog;
1833 }
1834 return tp_fd;
1835
1836 out_put_prog:
1837 bpf_prog_put(prog);
1838 out_free_tp:
1839 kfree(raw_tp);
1840 out_put_btp:
1841 bpf_put_raw_tracepoint(btp);
1842 return err;
1843 }
1844
1845 static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog,
1846 enum bpf_attach_type attach_type)
1847 {
1848 switch (prog->type) {
1849 case BPF_PROG_TYPE_CGROUP_SOCK:
1850 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
1851 case BPF_PROG_TYPE_CGROUP_SOCKOPT:
1852 return attach_type == prog->expected_attach_type ? 0 : -EINVAL;
1853 case BPF_PROG_TYPE_CGROUP_SKB:
1854 return prog->enforce_expected_attach_type &&
1855 prog->expected_attach_type != attach_type ?
1856 -EINVAL : 0;
1857 default:
1858 return 0;
1859 }
1860 }
1861
1862 #define BPF_PROG_ATTACH_LAST_FIELD attach_flags
1863
1864 #define BPF_F_ATTACH_MASK \
1865 (BPF_F_ALLOW_OVERRIDE | BPF_F_ALLOW_MULTI)
1866
1867 static int bpf_prog_attach(const union bpf_attr *attr)
1868 {
1869 enum bpf_prog_type ptype;
1870 struct bpf_prog *prog;
1871 int ret;
1872
1873 if (!capable(CAP_NET_ADMIN))
1874 return -EPERM;
1875
1876 if (CHECK_ATTR(BPF_PROG_ATTACH))
1877 return -EINVAL;
1878
1879 if (attr->attach_flags & ~BPF_F_ATTACH_MASK)
1880 return -EINVAL;
1881
1882 switch (attr->attach_type) {
1883 case BPF_CGROUP_INET_INGRESS:
1884 case BPF_CGROUP_INET_EGRESS:
1885 ptype = BPF_PROG_TYPE_CGROUP_SKB;
1886 break;
1887 case BPF_CGROUP_INET_SOCK_CREATE:
1888 case BPF_CGROUP_INET4_POST_BIND:
1889 case BPF_CGROUP_INET6_POST_BIND:
1890 ptype = BPF_PROG_TYPE_CGROUP_SOCK;
1891 break;
1892 case BPF_CGROUP_INET4_BIND:
1893 case BPF_CGROUP_INET6_BIND:
1894 case BPF_CGROUP_INET4_CONNECT:
1895 case BPF_CGROUP_INET6_CONNECT:
1896 case BPF_CGROUP_UDP4_SENDMSG:
1897 case BPF_CGROUP_UDP6_SENDMSG:
1898 case BPF_CGROUP_UDP4_RECVMSG:
1899 case BPF_CGROUP_UDP6_RECVMSG:
1900 ptype = BPF_PROG_TYPE_CGROUP_SOCK_ADDR;
1901 break;
1902 case BPF_CGROUP_SOCK_OPS:
1903 ptype = BPF_PROG_TYPE_SOCK_OPS;
1904 break;
1905 case BPF_CGROUP_DEVICE:
1906 ptype = BPF_PROG_TYPE_CGROUP_DEVICE;
1907 break;
1908 case BPF_SK_MSG_VERDICT:
1909 ptype = BPF_PROG_TYPE_SK_MSG;
1910 break;
1911 case BPF_SK_SKB_STREAM_PARSER:
1912 case BPF_SK_SKB_STREAM_VERDICT:
1913 ptype = BPF_PROG_TYPE_SK_SKB;
1914 break;
1915 case BPF_LIRC_MODE2:
1916 ptype = BPF_PROG_TYPE_LIRC_MODE2;
1917 break;
1918 case BPF_FLOW_DISSECTOR:
1919 ptype = BPF_PROG_TYPE_FLOW_DISSECTOR;
1920 break;
1921 case BPF_CGROUP_SYSCTL:
1922 ptype = BPF_PROG_TYPE_CGROUP_SYSCTL;
1923 break;
1924 case BPF_CGROUP_GETSOCKOPT:
1925 case BPF_CGROUP_SETSOCKOPT:
1926 ptype = BPF_PROG_TYPE_CGROUP_SOCKOPT;
1927 break;
1928 default:
1929 return -EINVAL;
1930 }
1931
1932 prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype);
1933 if (IS_ERR(prog))
1934 return PTR_ERR(prog);
1935
1936 if (bpf_prog_attach_check_attach_type(prog, attr->attach_type)) {
1937 bpf_prog_put(prog);
1938 return -EINVAL;
1939 }
1940
1941 switch (ptype) {
1942 case BPF_PROG_TYPE_SK_SKB:
1943 case BPF_PROG_TYPE_SK_MSG:
1944 ret = sock_map_get_from_fd(attr, prog);
1945 break;
1946 case BPF_PROG_TYPE_LIRC_MODE2:
1947 ret = lirc_prog_attach(attr, prog);
1948 break;
1949 case BPF_PROG_TYPE_FLOW_DISSECTOR:
1950 ret = skb_flow_dissector_bpf_prog_attach(attr, prog);
1951 break;
1952 default:
1953 ret = cgroup_bpf_prog_attach(attr, ptype, prog);
1954 }
1955
1956 if (ret)
1957 bpf_prog_put(prog);
1958 return ret;
1959 }
1960
1961 #define BPF_PROG_DETACH_LAST_FIELD attach_type
1962
1963 static int bpf_prog_detach(const union bpf_attr *attr)
1964 {
1965 enum bpf_prog_type ptype;
1966
1967 if (!capable(CAP_NET_ADMIN))
1968 return -EPERM;
1969
1970 if (CHECK_ATTR(BPF_PROG_DETACH))
1971 return -EINVAL;
1972
1973 switch (attr->attach_type) {
1974 case BPF_CGROUP_INET_INGRESS:
1975 case BPF_CGROUP_INET_EGRESS:
1976 ptype = BPF_PROG_TYPE_CGROUP_SKB;
1977 break;
1978 case BPF_CGROUP_INET_SOCK_CREATE:
1979 case BPF_CGROUP_INET4_POST_BIND:
1980 case BPF_CGROUP_INET6_POST_BIND:
1981 ptype = BPF_PROG_TYPE_CGROUP_SOCK;
1982 break;
1983 case BPF_CGROUP_INET4_BIND:
1984 case BPF_CGROUP_INET6_BIND:
1985 case BPF_CGROUP_INET4_CONNECT:
1986 case BPF_CGROUP_INET6_CONNECT:
1987 case BPF_CGROUP_UDP4_SENDMSG:
1988 case BPF_CGROUP_UDP6_SENDMSG:
1989 case BPF_CGROUP_UDP4_RECVMSG:
1990 case BPF_CGROUP_UDP6_RECVMSG:
1991 ptype = BPF_PROG_TYPE_CGROUP_SOCK_ADDR;
1992 break;
1993 case BPF_CGROUP_SOCK_OPS:
1994 ptype = BPF_PROG_TYPE_SOCK_OPS;
1995 break;
1996 case BPF_CGROUP_DEVICE:
1997 ptype = BPF_PROG_TYPE_CGROUP_DEVICE;
1998 break;
1999 case BPF_SK_MSG_VERDICT:
2000 return sock_map_get_from_fd(attr, NULL);
2001 case BPF_SK_SKB_STREAM_PARSER:
2002 case BPF_SK_SKB_STREAM_VERDICT:
2003 return sock_map_get_from_fd(attr, NULL);
2004 case BPF_LIRC_MODE2:
2005 return lirc_prog_detach(attr);
2006 case BPF_FLOW_DISSECTOR:
2007 return skb_flow_dissector_bpf_prog_detach(attr);
2008 case BPF_CGROUP_SYSCTL:
2009 ptype = BPF_PROG_TYPE_CGROUP_SYSCTL;
2010 break;
2011 case BPF_CGROUP_GETSOCKOPT:
2012 case BPF_CGROUP_SETSOCKOPT:
2013 ptype = BPF_PROG_TYPE_CGROUP_SOCKOPT;
2014 break;
2015 default:
2016 return -EINVAL;
2017 }
2018
2019 return cgroup_bpf_prog_detach(attr, ptype);
2020 }
2021
2022 #define BPF_PROG_QUERY_LAST_FIELD query.prog_cnt
2023
2024 static int bpf_prog_query(const union bpf_attr *attr,
2025 union bpf_attr __user *uattr)
2026 {
2027 if (!capable(CAP_NET_ADMIN))
2028 return -EPERM;
2029 if (CHECK_ATTR(BPF_PROG_QUERY))
2030 return -EINVAL;
2031 if (attr->query.query_flags & ~BPF_F_QUERY_EFFECTIVE)
2032 return -EINVAL;
2033
2034 switch (attr->query.attach_type) {
2035 case BPF_CGROUP_INET_INGRESS:
2036 case BPF_CGROUP_INET_EGRESS:
2037 case BPF_CGROUP_INET_SOCK_CREATE:
2038 case BPF_CGROUP_INET4_BIND:
2039 case BPF_CGROUP_INET6_BIND:
2040 case BPF_CGROUP_INET4_POST_BIND:
2041 case BPF_CGROUP_INET6_POST_BIND:
2042 case BPF_CGROUP_INET4_CONNECT:
2043 case BPF_CGROUP_INET6_CONNECT:
2044 case BPF_CGROUP_UDP4_SENDMSG:
2045 case BPF_CGROUP_UDP6_SENDMSG:
2046 case BPF_CGROUP_UDP4_RECVMSG:
2047 case BPF_CGROUP_UDP6_RECVMSG:
2048 case BPF_CGROUP_SOCK_OPS:
2049 case BPF_CGROUP_DEVICE:
2050 case BPF_CGROUP_SYSCTL:
2051 case BPF_CGROUP_GETSOCKOPT:
2052 case BPF_CGROUP_SETSOCKOPT:
2053 break;
2054 case BPF_LIRC_MODE2:
2055 return lirc_prog_query(attr, uattr);
2056 case BPF_FLOW_DISSECTOR:
2057 return skb_flow_dissector_prog_query(attr, uattr);
2058 default:
2059 return -EINVAL;
2060 }
2061
2062 return cgroup_bpf_prog_query(attr, uattr);
2063 }
2064
2065 #define BPF_PROG_TEST_RUN_LAST_FIELD test.ctx_out
2066
2067 static int bpf_prog_test_run(const union bpf_attr *attr,
2068 union bpf_attr __user *uattr)
2069 {
2070 struct bpf_prog *prog;
2071 int ret = -ENOTSUPP;
2072
2073 if (!capable(CAP_SYS_ADMIN))
2074 return -EPERM;
2075 if (CHECK_ATTR(BPF_PROG_TEST_RUN))
2076 return -EINVAL;
2077
2078 if ((attr->test.ctx_size_in && !attr->test.ctx_in) ||
2079 (!attr->test.ctx_size_in && attr->test.ctx_in))
2080 return -EINVAL;
2081
2082 if ((attr->test.ctx_size_out && !attr->test.ctx_out) ||
2083 (!attr->test.ctx_size_out && attr->test.ctx_out))
2084 return -EINVAL;
2085
2086 prog = bpf_prog_get(attr->test.prog_fd);
2087 if (IS_ERR(prog))
2088 return PTR_ERR(prog);
2089
2090 if (prog->aux->ops->test_run)
2091 ret = prog->aux->ops->test_run(prog, attr, uattr);
2092
2093 bpf_prog_put(prog);
2094 return ret;
2095 }
2096
2097 #define BPF_OBJ_GET_NEXT_ID_LAST_FIELD next_id
2098
2099 static int bpf_obj_get_next_id(const union bpf_attr *attr,
2100 union bpf_attr __user *uattr,
2101 struct idr *idr,
2102 spinlock_t *lock)
2103 {
2104 u32 next_id = attr->start_id;
2105 int err = 0;
2106
2107 if (CHECK_ATTR(BPF_OBJ_GET_NEXT_ID) || next_id >= INT_MAX)
2108 return -EINVAL;
2109
2110 if (!capable(CAP_SYS_ADMIN))
2111 return -EPERM;
2112
2113 next_id++;
2114 spin_lock_bh(lock);
2115 if (!idr_get_next(idr, &next_id))
2116 err = -ENOENT;
2117 spin_unlock_bh(lock);
2118
2119 if (!err)
2120 err = put_user(next_id, &uattr->next_id);
2121
2122 return err;
2123 }
2124
2125 #define BPF_PROG_GET_FD_BY_ID_LAST_FIELD prog_id
2126
2127 static int bpf_prog_get_fd_by_id(const union bpf_attr *attr)
2128 {
2129 struct bpf_prog *prog;
2130 u32 id = attr->prog_id;
2131 int fd;
2132
2133 if (CHECK_ATTR(BPF_PROG_GET_FD_BY_ID))
2134 return -EINVAL;
2135
2136 if (!capable(CAP_SYS_ADMIN))
2137 return -EPERM;
2138
2139 spin_lock_bh(&prog_idr_lock);
2140 prog = idr_find(&prog_idr, id);
2141 if (prog)
2142 prog = bpf_prog_inc_not_zero(prog);
2143 else
2144 prog = ERR_PTR(-ENOENT);
2145 spin_unlock_bh(&prog_idr_lock);
2146
2147 if (IS_ERR(prog))
2148 return PTR_ERR(prog);
2149
2150 fd = bpf_prog_new_fd(prog);
2151 if (fd < 0)
2152 bpf_prog_put(prog);
2153
2154 return fd;
2155 }
2156
2157 #define BPF_MAP_GET_FD_BY_ID_LAST_FIELD open_flags
2158
2159 static int bpf_map_get_fd_by_id(const union bpf_attr *attr)
2160 {
2161 struct bpf_map *map;
2162 u32 id = attr->map_id;
2163 int f_flags;
2164 int fd;
2165
2166 if (CHECK_ATTR(BPF_MAP_GET_FD_BY_ID) ||
2167 attr->open_flags & ~BPF_OBJ_FLAG_MASK)
2168 return -EINVAL;
2169
2170 if (!capable(CAP_SYS_ADMIN))
2171 return -EPERM;
2172
2173 f_flags = bpf_get_file_flag(attr->open_flags);
2174 if (f_flags < 0)
2175 return f_flags;
2176
2177 spin_lock_bh(&map_idr_lock);
2178 map = idr_find(&map_idr, id);
2179 if (map)
2180 map = bpf_map_inc_not_zero(map, true);
2181 else
2182 map = ERR_PTR(-ENOENT);
2183 spin_unlock_bh(&map_idr_lock);
2184
2185 if (IS_ERR(map))
2186 return PTR_ERR(map);
2187
2188 fd = bpf_map_new_fd(map, f_flags);
2189 if (fd < 0)
2190 bpf_map_put_with_uref(map);
2191
2192 return fd;
2193 }
2194
2195 static const struct bpf_map *bpf_map_from_imm(const struct bpf_prog *prog,
2196 unsigned long addr, u32 *off,
2197 u32 *type)
2198 {
2199 const struct bpf_map *map;
2200 int i;
2201
2202 for (i = 0, *off = 0; i < prog->aux->used_map_cnt; i++) {
2203 map = prog->aux->used_maps[i];
2204 if (map == (void *)addr) {
2205 *type = BPF_PSEUDO_MAP_FD;
2206 return map;
2207 }
2208 if (!map->ops->map_direct_value_meta)
2209 continue;
2210 if (!map->ops->map_direct_value_meta(map, addr, off)) {
2211 *type = BPF_PSEUDO_MAP_VALUE;
2212 return map;
2213 }
2214 }
2215
2216 return NULL;
2217 }
2218
2219 static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog)
2220 {
2221 const struct bpf_map *map;
2222 struct bpf_insn *insns;
2223 u32 off, type;
2224 u64 imm;
2225 int i;
2226
2227 insns = kmemdup(prog->insnsi, bpf_prog_insn_size(prog),
2228 GFP_USER);
2229 if (!insns)
2230 return insns;
2231
2232 for (i = 0; i < prog->len; i++) {
2233 if (insns[i].code == (BPF_JMP | BPF_TAIL_CALL)) {
2234 insns[i].code = BPF_JMP | BPF_CALL;
2235 insns[i].imm = BPF_FUNC_tail_call;
2236 /* fall-through */
2237 }
2238 if (insns[i].code == (BPF_JMP | BPF_CALL) ||
2239 insns[i].code == (BPF_JMP | BPF_CALL_ARGS)) {
2240 if (insns[i].code == (BPF_JMP | BPF_CALL_ARGS))
2241 insns[i].code = BPF_JMP | BPF_CALL;
2242 if (!bpf_dump_raw_ok())
2243 insns[i].imm = 0;
2244 continue;
2245 }
2246
2247 if (insns[i].code != (BPF_LD | BPF_IMM | BPF_DW))
2248 continue;
2249
2250 imm = ((u64)insns[i + 1].imm << 32) | (u32)insns[i].imm;
2251 map = bpf_map_from_imm(prog, imm, &off, &type);
2252 if (map) {
2253 insns[i].src_reg = type;
2254 insns[i].imm = map->id;
2255 insns[i + 1].imm = off;
2256 continue;
2257 }
2258 }
2259
2260 return insns;
2261 }
2262
2263 static int set_info_rec_size(struct bpf_prog_info *info)
2264 {
2265 /*
2266 * Ensure info.*_rec_size is the same as kernel expected size
2267 *
2268 * or
2269 *
2270 * Only allow zero *_rec_size if both _rec_size and _cnt are
2271 * zero. In this case, the kernel will set the expected
2272 * _rec_size back to the info.
2273 */
2274
2275 if ((info->nr_func_info || info->func_info_rec_size) &&
2276 info->func_info_rec_size != sizeof(struct bpf_func_info))
2277 return -EINVAL;
2278
2279 if ((info->nr_line_info || info->line_info_rec_size) &&
2280 info->line_info_rec_size != sizeof(struct bpf_line_info))
2281 return -EINVAL;
2282
2283 if ((info->nr_jited_line_info || info->jited_line_info_rec_size) &&
2284 info->jited_line_info_rec_size != sizeof(__u64))
2285 return -EINVAL;
2286
2287 info->func_info_rec_size = sizeof(struct bpf_func_info);
2288 info->line_info_rec_size = sizeof(struct bpf_line_info);
2289 info->jited_line_info_rec_size = sizeof(__u64);
2290
2291 return 0;
2292 }
2293
2294 static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
2295 const union bpf_attr *attr,
2296 union bpf_attr __user *uattr)
2297 {
2298 struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info);
2299 struct bpf_prog_info info = {};
2300 u32 info_len = attr->info.info_len;
2301 struct bpf_prog_stats stats;
2302 char __user *uinsns;
2303 u32 ulen;
2304 int err;
2305
2306 err = bpf_check_uarg_tail_zero(uinfo, sizeof(info), info_len);
2307 if (err)
2308 return err;
2309 info_len = min_t(u32, sizeof(info), info_len);
2310
2311 if (copy_from_user(&info, uinfo, info_len))
2312 return -EFAULT;
2313
2314 info.type = prog->type;
2315 info.id = prog->aux->id;
2316 info.load_time = prog->aux->load_time;
2317 info.created_by_uid = from_kuid_munged(current_user_ns(),
2318 prog->aux->user->uid);
2319 info.gpl_compatible = prog->gpl_compatible;
2320
2321 memcpy(info.tag, prog->tag, sizeof(prog->tag));
2322 memcpy(info.name, prog->aux->name, sizeof(prog->aux->name));
2323
2324 ulen = info.nr_map_ids;
2325 info.nr_map_ids = prog->aux->used_map_cnt;
2326 ulen = min_t(u32, info.nr_map_ids, ulen);
2327 if (ulen) {
2328 u32 __user *user_map_ids = u64_to_user_ptr(info.map_ids);
2329 u32 i;
2330
2331 for (i = 0; i < ulen; i++)
2332 if (put_user(prog->aux->used_maps[i]->id,
2333 &user_map_ids[i]))
2334 return -EFAULT;
2335 }
2336
2337 err = set_info_rec_size(&info);
2338 if (err)
2339 return err;
2340
2341 bpf_prog_get_stats(prog, &stats);
2342 info.run_time_ns = stats.nsecs;
2343 info.run_cnt = stats.cnt;
2344
2345 if (!capable(CAP_SYS_ADMIN)) {
2346 info.jited_prog_len = 0;
2347 info.xlated_prog_len = 0;
2348 info.nr_jited_ksyms = 0;
2349 info.nr_jited_func_lens = 0;
2350 info.nr_func_info = 0;
2351 info.nr_line_info = 0;
2352 info.nr_jited_line_info = 0;
2353 goto done;
2354 }
2355
2356 ulen = info.xlated_prog_len;
2357 info.xlated_prog_len = bpf_prog_insn_size(prog);
2358 if (info.xlated_prog_len && ulen) {
2359 struct bpf_insn *insns_sanitized;
2360 bool fault;
2361
2362 if (prog->blinded && !bpf_dump_raw_ok()) {
2363 info.xlated_prog_insns = 0;
2364 goto done;
2365 }
2366 insns_sanitized = bpf_insn_prepare_dump(prog);
2367 if (!insns_sanitized)
2368 return -ENOMEM;
2369 uinsns = u64_to_user_ptr(info.xlated_prog_insns);
2370 ulen = min_t(u32, info.xlated_prog_len, ulen);
2371 fault = copy_to_user(uinsns, insns_sanitized, ulen);
2372 kfree(insns_sanitized);
2373 if (fault)
2374 return -EFAULT;
2375 }
2376
2377 if (bpf_prog_is_dev_bound(prog->aux)) {
2378 err = bpf_prog_offload_info_fill(&info, prog);
2379 if (err)
2380 return err;
2381 goto done;
2382 }
2383
2384 /* NOTE: the following code is supposed to be skipped for offload.
2385 * bpf_prog_offload_info_fill() is the place to fill similar fields
2386 * for offload.
2387 */
2388 ulen = info.jited_prog_len;
2389 if (prog->aux->func_cnt) {
2390 u32 i;
2391
2392 info.jited_prog_len = 0;
2393 for (i = 0; i < prog->aux->func_cnt; i++)
2394 info.jited_prog_len += prog->aux->func[i]->jited_len;
2395 } else {
2396 info.jited_prog_len = prog->jited_len;
2397 }
2398
2399 if (info.jited_prog_len && ulen) {
2400 if (bpf_dump_raw_ok()) {
2401 uinsns = u64_to_user_ptr(info.jited_prog_insns);
2402 ulen = min_t(u32, info.jited_prog_len, ulen);
2403
2404 /* for multi-function programs, copy the JITed
2405 * instructions for all the functions
2406 */
2407 if (prog->aux->func_cnt) {
2408 u32 len, free, i;
2409 u8 *img;
2410
2411 free = ulen;
2412 for (i = 0; i < prog->aux->func_cnt; i++) {
2413 len = prog->aux->func[i]->jited_len;
2414 len = min_t(u32, len, free);
2415 img = (u8 *) prog->aux->func[i]->bpf_func;
2416 if (copy_to_user(uinsns, img, len))
2417 return -EFAULT;
2418 uinsns += len;
2419 free -= len;
2420 if (!free)
2421 break;
2422 }
2423 } else {
2424 if (copy_to_user(uinsns, prog->bpf_func, ulen))
2425 return -EFAULT;
2426 }
2427 } else {
2428 info.jited_prog_insns = 0;
2429 }
2430 }
2431
2432 ulen = info.nr_jited_ksyms;
2433 info.nr_jited_ksyms = prog->aux->func_cnt ? : 1;
2434 if (ulen) {
2435 if (bpf_dump_raw_ok()) {
2436 unsigned long ksym_addr;
2437 u64 __user *user_ksyms;
2438 u32 i;
2439
2440 /* copy the address of the kernel symbol
2441 * corresponding to each function
2442 */
2443 ulen = min_t(u32, info.nr_jited_ksyms, ulen);
2444 user_ksyms = u64_to_user_ptr(info.jited_ksyms);
2445 if (prog->aux->func_cnt) {
2446 for (i = 0; i < ulen; i++) {
2447 ksym_addr = (unsigned long)
2448 prog->aux->func[i]->bpf_func;
2449 if (put_user((u64) ksym_addr,
2450 &user_ksyms[i]))
2451 return -EFAULT;
2452 }
2453 } else {
2454 ksym_addr = (unsigned long) prog->bpf_func;
2455 if (put_user((u64) ksym_addr, &user_ksyms[0]))
2456 return -EFAULT;
2457 }
2458 } else {
2459 info.jited_ksyms = 0;
2460 }
2461 }
2462
2463 ulen = info.nr_jited_func_lens;
2464 info.nr_jited_func_lens = prog->aux->func_cnt ? : 1;
2465 if (ulen) {
2466 if (bpf_dump_raw_ok()) {
2467 u32 __user *user_lens;
2468 u32 func_len, i;
2469
2470 /* copy the JITed image lengths for each function */
2471 ulen = min_t(u32, info.nr_jited_func_lens, ulen);
2472 user_lens = u64_to_user_ptr(info.jited_func_lens);
2473 if (prog->aux->func_cnt) {
2474 for (i = 0; i < ulen; i++) {
2475 func_len =
2476 prog->aux->func[i]->jited_len;
2477 if (put_user(func_len, &user_lens[i]))
2478 return -EFAULT;
2479 }
2480 } else {
2481 func_len = prog->jited_len;
2482 if (put_user(func_len, &user_lens[0]))
2483 return -EFAULT;
2484 }
2485 } else {
2486 info.jited_func_lens = 0;
2487 }
2488 }
2489
2490 if (prog->aux->btf)
2491 info.btf_id = btf_id(prog->aux->btf);
2492
2493 ulen = info.nr_func_info;
2494 info.nr_func_info = prog->aux->func_info_cnt;
2495 if (info.nr_func_info && ulen) {
2496 char __user *user_finfo;
2497
2498 user_finfo = u64_to_user_ptr(info.func_info);
2499 ulen = min_t(u32, info.nr_func_info, ulen);
2500 if (copy_to_user(user_finfo, prog->aux->func_info,
2501 info.func_info_rec_size * ulen))
2502 return -EFAULT;
2503 }
2504
2505 ulen = info.nr_line_info;
2506 info.nr_line_info = prog->aux->nr_linfo;
2507 if (info.nr_line_info && ulen) {
2508 __u8 __user *user_linfo;
2509
2510 user_linfo = u64_to_user_ptr(info.line_info);
2511 ulen = min_t(u32, info.nr_line_info, ulen);
2512 if (copy_to_user(user_linfo, prog->aux->linfo,
2513 info.line_info_rec_size * ulen))
2514 return -EFAULT;
2515 }
2516
2517 ulen = info.nr_jited_line_info;
2518 if (prog->aux->jited_linfo)
2519 info.nr_jited_line_info = prog->aux->nr_linfo;
2520 else
2521 info.nr_jited_line_info = 0;
2522 if (info.nr_jited_line_info && ulen) {
2523 if (bpf_dump_raw_ok()) {
2524 __u64 __user *user_linfo;
2525 u32 i;
2526
2527 user_linfo = u64_to_user_ptr(info.jited_line_info);
2528 ulen = min_t(u32, info.nr_jited_line_info, ulen);
2529 for (i = 0; i < ulen; i++) {
2530 if (put_user((__u64)(long)prog->aux->jited_linfo[i],
2531 &user_linfo[i]))
2532 return -EFAULT;
2533 }
2534 } else {
2535 info.jited_line_info = 0;
2536 }
2537 }
2538
2539 ulen = info.nr_prog_tags;
2540 info.nr_prog_tags = prog->aux->func_cnt ? : 1;
2541 if (ulen) {
2542 __u8 __user (*user_prog_tags)[BPF_TAG_SIZE];
2543 u32 i;
2544
2545 user_prog_tags = u64_to_user_ptr(info.prog_tags);
2546 ulen = min_t(u32, info.nr_prog_tags, ulen);
2547 if (prog->aux->func_cnt) {
2548 for (i = 0; i < ulen; i++) {
2549 if (copy_to_user(user_prog_tags[i],
2550 prog->aux->func[i]->tag,
2551 BPF_TAG_SIZE))
2552 return -EFAULT;
2553 }
2554 } else {
2555 if (copy_to_user(user_prog_tags[0],
2556 prog->tag, BPF_TAG_SIZE))
2557 return -EFAULT;
2558 }
2559 }
2560
2561 done:
2562 if (copy_to_user(uinfo, &info, info_len) ||
2563 put_user(info_len, &uattr->info.info_len))
2564 return -EFAULT;
2565
2566 return 0;
2567 }
2568
2569 static int bpf_map_get_info_by_fd(struct bpf_map *map,
2570 const union bpf_attr *attr,
2571 union bpf_attr __user *uattr)
2572 {
2573 struct bpf_map_info __user *uinfo = u64_to_user_ptr(attr->info.info);
2574 struct bpf_map_info info = {};
2575 u32 info_len = attr->info.info_len;
2576 int err;
2577
2578 err = bpf_check_uarg_tail_zero(uinfo, sizeof(info), info_len);
2579 if (err)
2580 return err;
2581 info_len = min_t(u32, sizeof(info), info_len);
2582
2583 info.type = map->map_type;
2584 info.id = map->id;
2585 info.key_size = map->key_size;
2586 info.value_size = map->value_size;
2587 info.max_entries = map->max_entries;
2588 info.map_flags = map->map_flags;
2589 memcpy(info.name, map->name, sizeof(map->name));
2590
2591 if (map->btf) {
2592 info.btf_id = btf_id(map->btf);
2593 info.btf_key_type_id = map->btf_key_type_id;
2594 info.btf_value_type_id = map->btf_value_type_id;
2595 }
2596
2597 if (bpf_map_is_dev_bound(map)) {
2598 err = bpf_map_offload_info_fill(&info, map);
2599 if (err)
2600 return err;
2601 }
2602
2603 if (copy_to_user(uinfo, &info, info_len) ||
2604 put_user(info_len, &uattr->info.info_len))
2605 return -EFAULT;
2606
2607 return 0;
2608 }
2609
2610 static int bpf_btf_get_info_by_fd(struct btf *btf,
2611 const union bpf_attr *attr,
2612 union bpf_attr __user *uattr)
2613 {
2614 struct bpf_btf_info __user *uinfo = u64_to_user_ptr(attr->info.info);
2615 u32 info_len = attr->info.info_len;
2616 int err;
2617
2618 err = bpf_check_uarg_tail_zero(uinfo, sizeof(*uinfo), info_len);
2619 if (err)
2620 return err;
2621
2622 return btf_get_info_by_fd(btf, attr, uattr);
2623 }
2624
2625 #define BPF_OBJ_GET_INFO_BY_FD_LAST_FIELD info.info
2626
2627 static int bpf_obj_get_info_by_fd(const union bpf_attr *attr,
2628 union bpf_attr __user *uattr)
2629 {
2630 int ufd = attr->info.bpf_fd;
2631 struct fd f;
2632 int err;
2633
2634 if (CHECK_ATTR(BPF_OBJ_GET_INFO_BY_FD))
2635 return -EINVAL;
2636
2637 f = fdget(ufd);
2638 if (!f.file)
2639 return -EBADFD;
2640
2641 if (f.file->f_op == &bpf_prog_fops)
2642 err = bpf_prog_get_info_by_fd(f.file->private_data, attr,
2643 uattr);
2644 else if (f.file->f_op == &bpf_map_fops)
2645 err = bpf_map_get_info_by_fd(f.file->private_data, attr,
2646 uattr);
2647 else if (f.file->f_op == &btf_fops)
2648 err = bpf_btf_get_info_by_fd(f.file->private_data, attr, uattr);
2649 else
2650 err = -EINVAL;
2651
2652 fdput(f);
2653 return err;
2654 }
2655
2656 #define BPF_BTF_LOAD_LAST_FIELD btf_log_level
2657
2658 static int bpf_btf_load(const union bpf_attr *attr)
2659 {
2660 if (CHECK_ATTR(BPF_BTF_LOAD))
2661 return -EINVAL;
2662
2663 if (!capable(CAP_SYS_ADMIN))
2664 return -EPERM;
2665
2666 return btf_new_fd(attr);
2667 }
2668
2669 #define BPF_BTF_GET_FD_BY_ID_LAST_FIELD btf_id
2670
2671 static int bpf_btf_get_fd_by_id(const union bpf_attr *attr)
2672 {
2673 if (CHECK_ATTR(BPF_BTF_GET_FD_BY_ID))
2674 return -EINVAL;
2675
2676 if (!capable(CAP_SYS_ADMIN))
2677 return -EPERM;
2678
2679 return btf_get_fd_by_id(attr->btf_id);
2680 }
2681
2682 static int bpf_task_fd_query_copy(const union bpf_attr *attr,
2683 union bpf_attr __user *uattr,
2684 u32 prog_id, u32 fd_type,
2685 const char *buf, u64 probe_offset,
2686 u64 probe_addr)
2687 {
2688 char __user *ubuf = u64_to_user_ptr(attr->task_fd_query.buf);
2689 u32 len = buf ? strlen(buf) : 0, input_len;
2690 int err = 0;
2691
2692 if (put_user(len, &uattr->task_fd_query.buf_len))
2693 return -EFAULT;
2694 input_len = attr->task_fd_query.buf_len;
2695 if (input_len && ubuf) {
2696 if (!len) {
2697 /* nothing to copy, just make ubuf NULL terminated */
2698 char zero = '\0';
2699
2700 if (put_user(zero, ubuf))
2701 return -EFAULT;
2702 } else if (input_len >= len + 1) {
2703 /* ubuf can hold the string with NULL terminator */
2704 if (copy_to_user(ubuf, buf, len + 1))
2705 return -EFAULT;
2706 } else {
2707 /* ubuf cannot hold the string with NULL terminator,
2708 * do a partial copy with NULL terminator.
2709 */
2710 char zero = '\0';
2711
2712 err = -ENOSPC;
2713 if (copy_to_user(ubuf, buf, input_len - 1))
2714 return -EFAULT;
2715 if (put_user(zero, ubuf + input_len - 1))
2716 return -EFAULT;
2717 }
2718 }
2719
2720 if (put_user(prog_id, &uattr->task_fd_query.prog_id) ||
2721 put_user(fd_type, &uattr->task_fd_query.fd_type) ||
2722 put_user(probe_offset, &uattr->task_fd_query.probe_offset) ||
2723 put_user(probe_addr, &uattr->task_fd_query.probe_addr))
2724 return -EFAULT;
2725
2726 return err;
2727 }
2728
2729 #define BPF_TASK_FD_QUERY_LAST_FIELD task_fd_query.probe_addr
2730
2731 static int bpf_task_fd_query(const union bpf_attr *attr,
2732 union bpf_attr __user *uattr)
2733 {
2734 pid_t pid = attr->task_fd_query.pid;
2735 u32 fd = attr->task_fd_query.fd;
2736 const struct perf_event *event;
2737 struct files_struct *files;
2738 struct task_struct *task;
2739 struct file *file;
2740 int err;
2741
2742 if (CHECK_ATTR(BPF_TASK_FD_QUERY))
2743 return -EINVAL;
2744
2745 if (!capable(CAP_SYS_ADMIN))
2746 return -EPERM;
2747
2748 if (attr->task_fd_query.flags != 0)
2749 return -EINVAL;
2750
2751 task = get_pid_task(find_vpid(pid), PIDTYPE_PID);
2752 if (!task)
2753 return -ENOENT;
2754
2755 files = get_files_struct(task);
2756 put_task_struct(task);
2757 if (!files)
2758 return -ENOENT;
2759
2760 err = 0;
2761 spin_lock(&files->file_lock);
2762 file = fcheck_files(files, fd);
2763 if (!file)
2764 err = -EBADF;
2765 else
2766 get_file(file);
2767 spin_unlock(&files->file_lock);
2768 put_files_struct(files);
2769
2770 if (err)
2771 goto out;
2772
2773 if (file->f_op == &bpf_raw_tp_fops) {
2774 struct bpf_raw_tracepoint *raw_tp = file->private_data;
2775 struct bpf_raw_event_map *btp = raw_tp->btp;
2776
2777 err = bpf_task_fd_query_copy(attr, uattr,
2778 raw_tp->prog->aux->id,
2779 BPF_FD_TYPE_RAW_TRACEPOINT,
2780 btp->tp->name, 0, 0);
2781 goto put_file;
2782 }
2783
2784 event = perf_get_event(file);
2785 if (!IS_ERR(event)) {
2786 u64 probe_offset, probe_addr;
2787 u32 prog_id, fd_type;
2788 const char *buf;
2789
2790 err = bpf_get_perf_event_info(event, &prog_id, &fd_type,
2791 &buf, &probe_offset,
2792 &probe_addr);
2793 if (!err)
2794 err = bpf_task_fd_query_copy(attr, uattr, prog_id,
2795 fd_type, buf,
2796 probe_offset,
2797 probe_addr);
2798 goto put_file;
2799 }
2800
2801 err = -ENOTSUPP;
2802 put_file:
2803 fput(file);
2804 out:
2805 return err;
2806 }
2807
2808 SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size)
2809 {
2810 union bpf_attr attr = {};
2811 int err;
2812
2813 if (sysctl_unprivileged_bpf_disabled && !capable(CAP_SYS_ADMIN))
2814 return -EPERM;
2815
2816 err = bpf_check_uarg_tail_zero(uattr, sizeof(attr), size);
2817 if (err)
2818 return err;
2819 size = min_t(u32, size, sizeof(attr));
2820
2821 /* copy attributes from user space, may be less than sizeof(bpf_attr) */
2822 if (copy_from_user(&attr, uattr, size) != 0)
2823 return -EFAULT;
2824
2825 err = security_bpf(cmd, &attr, size);
2826 if (err < 0)
2827 return err;
2828
2829 switch (cmd) {
2830 case BPF_MAP_CREATE:
2831 err = map_create(&attr);
2832 break;
2833 case BPF_MAP_LOOKUP_ELEM:
2834 err = map_lookup_elem(&attr);
2835 break;
2836 case BPF_MAP_UPDATE_ELEM:
2837 err = map_update_elem(&attr);
2838 break;
2839 case BPF_MAP_DELETE_ELEM:
2840 err = map_delete_elem(&attr);
2841 break;
2842 case BPF_MAP_GET_NEXT_KEY:
2843 err = map_get_next_key(&attr);
2844 break;
2845 case BPF_MAP_FREEZE:
2846 err = map_freeze(&attr);
2847 break;
2848 case BPF_PROG_LOAD:
2849 err = bpf_prog_load(&attr, uattr);
2850 break;
2851 case BPF_OBJ_PIN:
2852 err = bpf_obj_pin(&attr);
2853 break;
2854 case BPF_OBJ_GET:
2855 err = bpf_obj_get(&attr);
2856 break;
2857 case BPF_PROG_ATTACH:
2858 err = bpf_prog_attach(&attr);
2859 break;
2860 case BPF_PROG_DETACH:
2861 err = bpf_prog_detach(&attr);
2862 break;
2863 case BPF_PROG_QUERY:
2864 err = bpf_prog_query(&attr, uattr);
2865 break;
2866 case BPF_PROG_TEST_RUN:
2867 err = bpf_prog_test_run(&attr, uattr);
2868 break;
2869 case BPF_PROG_GET_NEXT_ID:
2870 err = bpf_obj_get_next_id(&attr, uattr,
2871 &prog_idr, &prog_idr_lock);
2872 break;
2873 case BPF_MAP_GET_NEXT_ID:
2874 err = bpf_obj_get_next_id(&attr, uattr,
2875 &map_idr, &map_idr_lock);
2876 break;
2877 case BPF_PROG_GET_FD_BY_ID:
2878 err = bpf_prog_get_fd_by_id(&attr);
2879 break;
2880 case BPF_MAP_GET_FD_BY_ID:
2881 err = bpf_map_get_fd_by_id(&attr);
2882 break;
2883 case BPF_OBJ_GET_INFO_BY_FD:
2884 err = bpf_obj_get_info_by_fd(&attr, uattr);
2885 break;
2886 case BPF_RAW_TRACEPOINT_OPEN:
2887 err = bpf_raw_tracepoint_open(&attr);
2888 break;
2889 case BPF_BTF_LOAD:
2890 err = bpf_btf_load(&attr);
2891 break;
2892 case BPF_BTF_GET_FD_BY_ID:
2893 err = bpf_btf_get_fd_by_id(&attr);
2894 break;
2895 case BPF_TASK_FD_QUERY:
2896 err = bpf_task_fd_query(&attr, uattr);
2897 break;
2898 case BPF_MAP_LOOKUP_AND_DELETE_ELEM:
2899 err = map_lookup_and_delete_elem(&attr);
2900 break;
2901 default:
2902 err = -EINVAL;
2903 break;
2904 }
2905
2906 return err;
2907 }
Linux with added FPC-III support