1 /* SPDX-License-Identifier: GPL-2.0 */
2 /* Copyright (c) 2018 Facebook */
7 #include <linux/types.h>
8 #include <linux/bpfptr.h>
9 #include <linux/bsearch.h>
10 #include <linux/btf_ids.h>
11 #include <uapi/linux/btf.h>
12 #include <uapi/linux/bpf.h>
14 #define BTF_TYPE_EMIT(type) ((void)(type *)0)
15 #define BTF_TYPE_EMIT_ENUM(enum_val) ((void)enum_val)
17 /* These need to be macros, as the expressions are used in assembler input */
18 #define KF_ACQUIRE (1 << 0) /* kfunc is an acquire function */
19 #define KF_RELEASE (1 << 1) /* kfunc is a release function */
20 #define KF_RET_NULL (1 << 2) /* kfunc returns a pointer that may be NULL */
21 /* Trusted arguments are those which are guaranteed to be valid when passed to
22 * the kfunc. It is used to enforce that pointers obtained from either acquire
23 * kfuncs, or from the main kernel on a tracepoint or struct_ops callback
24 * invocation, remain unmodified when being passed to helpers taking trusted
27 * Consider, for example, the following new task tracepoint:
29 * SEC("tp_btf/task_newtask")
30 * int BPF_PROG(new_task_tp, struct task_struct *task, u64 clone_flags)
35 * And the following kfunc:
37 * BTF_ID_FLAGS(func, bpf_task_acquire, KF_ACQUIRE | KF_TRUSTED_ARGS)
39 * All invocations to the kfunc must pass the unmodified, unwalked task:
41 * bpf_task_acquire(task); // Allowed
42 * bpf_task_acquire(task->last_wakee); // Rejected, walked task
44 * Programs may also pass referenced tasks directly to the kfunc:
46 * struct task_struct *acquired;
48 * acquired = bpf_task_acquire(task); // Allowed, same as above
49 * bpf_task_acquire(acquired); // Allowed
50 * bpf_task_acquire(task); // Allowed
51 * bpf_task_acquire(acquired->last_wakee); // Rejected, walked task
53 * Programs may _not_, however, pass a task from an arbitrary fentry/fexit, or
54 * kprobe/kretprobe to the kfunc, as BPF cannot guarantee that all of these
55 * pointers are guaranteed to be safe. For example, the following BPF program
58 * SEC("kretprobe/free_task")
59 * int BPF_PROG(free_task_probe, struct task_struct *tsk)
61 * struct task_struct *acquired;
63 * acquired = bpf_task_acquire(acquired); // Rejected, not a trusted pointer
64 * bpf_task_release(acquired);
69 #define KF_TRUSTED_ARGS (1 << 4) /* kfunc only takes trusted pointer arguments */
70 #define KF_SLEEPABLE (1 << 5) /* kfunc may sleep */
71 #define KF_DESTRUCTIVE (1 << 6) /* kfunc performs destructive actions */
72 #define KF_RCU (1 << 7) /* kfunc takes either rcu or trusted pointer arguments */
73 /* only one of KF_ITER_{NEW,NEXT,DESTROY} could be specified per kfunc */
74 #define KF_ITER_NEW (1 << 8) /* kfunc implements BPF iter constructor */
75 #define KF_ITER_NEXT (1 << 9) /* kfunc implements BPF iter next method */
76 #define KF_ITER_DESTROY (1 << 10) /* kfunc implements BPF iter destructor */
77 #define KF_RCU_PROTECTED (1 << 11) /* kfunc should be protected by rcu cs when they are invoked */
78 #define KF_FASTCALL (1 << 12) /* kfunc supports bpf_fastcall protocol */
81 * Tag marking a kernel function as a kfunc. This is meant to minimize the
82 * amount of copy-paste that kfunc authors have to include for correctness so
83 * as to avoid issues such as the compiler inlining or eliding either a static
84 * kfunc, or a global kfunc in an LTO build.
86 #define __bpf_kfunc __used __retain noinline
88 #define __bpf_kfunc_start_defs() \
90 __diag_ignore_all("-Wmissing-declarations", \
91 "Global kfuncs as their definitions will be in BTF");\
92 __diag_ignore_all("-Wmissing-prototypes", \
93 "Global kfuncs as their definitions will be in BTF")
95 #define __bpf_kfunc_end_defs() __diag_pop()
96 #define __bpf_hook_start() __bpf_kfunc_start_defs()
97 #define __bpf_hook_end() __bpf_kfunc_end_defs()
100 * Return the name of the passed struct, if exists, or halt the build if for
101 * example the structure gets renamed. In this way, developers have to revisit
102 * the code using that structure name, and update it accordingly.
104 #define stringify_struct(x) \
105 ({ BUILD_BUG_ON(sizeof(struct x) < 0); \
116 typedef int (*btf_kfunc_filter_t
)(const struct bpf_prog
*prog
, u32 kfunc_id
);
118 struct btf_kfunc_id_set
{
119 struct module
*owner
;
120 struct btf_id_set8
*set
;
121 btf_kfunc_filter_t filter
;
124 struct btf_id_dtor_kfunc
{
129 struct btf_struct_meta
{
131 struct btf_record
*record
;
134 struct btf_struct_metas
{
136 struct btf_struct_meta types
[];
139 extern const struct file_operations btf_fops
;
141 const char *btf_get_name(const struct btf
*btf
);
142 void btf_get(struct btf
*btf
);
143 void btf_put(struct btf
*btf
);
144 const struct btf_header
*btf_header(const struct btf
*btf
);
145 int btf_new_fd(const union bpf_attr
*attr
, bpfptr_t uattr
, u32 uattr_sz
);
146 struct btf
*btf_get_by_fd(int fd
);
147 int btf_get_info_by_fd(const struct btf
*btf
,
148 const union bpf_attr
*attr
,
149 union bpf_attr __user
*uattr
);
150 /* Figure out the size of a type_id. If type_id is a modifier
151 * (e.g. const), it will be resolved to find out the type with size.
154 * In describing "const void *", type_id is "const" and "const"
155 * refers to "void *". The return type will be "void *".
157 * If type_id is a simple "int", then return type will be "int".
159 * @btf: struct btf object
160 * @type_id: Find out the size of type_id. The type_id of the return
161 * type is set to *type_id.
162 * @ret_size: It can be NULL. If not NULL, the size of the return
163 * type is set to *ret_size.
164 * Return: The btf_type (resolved to another type with size info if needed).
165 * NULL is returned if type_id itself does not have size info
166 * (e.g. void) or it cannot be resolved to another type that
168 * *type_id and *ret_size will not be changed in the
171 const struct btf_type
*btf_type_id_size(const struct btf
*btf
,
176 * Options to control show behaviour.
177 * - BTF_SHOW_COMPACT: no formatting around type information
178 * - BTF_SHOW_NONAME: no struct/union member names/types
179 * - BTF_SHOW_PTR_RAW: show raw (unobfuscated) pointer values;
181 * - BTF_SHOW_ZERO: show zero-valued struct/union members; they
182 * are not displayed by default
183 * - BTF_SHOW_UNSAFE: skip use of bpf_probe_read() to safely read
184 * data before displaying it.
186 #define BTF_SHOW_COMPACT BTF_F_COMPACT
187 #define BTF_SHOW_NONAME BTF_F_NONAME
188 #define BTF_SHOW_PTR_RAW BTF_F_PTR_RAW
189 #define BTF_SHOW_ZERO BTF_F_ZERO
190 #define BTF_SHOW_UNSAFE (1ULL << 4)
192 void btf_type_seq_show(const struct btf
*btf
, u32 type_id
, void *obj
,
194 int btf_type_seq_show_flags(const struct btf
*btf
, u32 type_id
, void *obj
,
195 struct seq_file
*m
, u64 flags
);
198 * Copy len bytes of string representation of obj of BTF type_id into buf.
200 * @btf: struct btf object
201 * @type_id: type id of type obj points to
202 * @obj: pointer to typed data
203 * @buf: buffer to write to
204 * @len: maximum length to write to buf
205 * @flags: show options (see above)
207 * Return: length that would have been/was copied as per snprintf, or
210 int btf_type_snprintf_show(const struct btf
*btf
, u32 type_id
, void *obj
,
211 char *buf
, int len
, u64 flags
);
213 int btf_get_fd_by_id(u32 id
);
214 u32
btf_obj_id(const struct btf
*btf
);
215 bool btf_is_kernel(const struct btf
*btf
);
216 bool btf_is_module(const struct btf
*btf
);
217 bool btf_is_vmlinux(const struct btf
*btf
);
218 struct module
*btf_try_get_module(const struct btf
*btf
);
219 u32
btf_nr_types(const struct btf
*btf
);
220 struct btf
*btf_base_btf(const struct btf
*btf
);
221 bool btf_member_is_reg_int(const struct btf
*btf
, const struct btf_type
*s
,
222 const struct btf_member
*m
,
223 u32 expected_offset
, u32 expected_size
);
224 struct btf_record
*btf_parse_fields(const struct btf
*btf
, const struct btf_type
*t
,
225 u32 field_mask
, u32 value_size
);
226 int btf_check_and_fixup_fields(const struct btf
*btf
, struct btf_record
*rec
);
227 bool btf_type_is_void(const struct btf_type
*t
);
228 s32
btf_find_by_name_kind(const struct btf
*btf
, const char *name
, u8 kind
);
229 s32
bpf_find_btf_id(const char *name
, u32 kind
, struct btf
**btf_p
);
230 const struct btf_type
*btf_type_skip_modifiers(const struct btf
*btf
,
231 u32 id
, u32
*res_id
);
232 const struct btf_type
*btf_type_resolve_ptr(const struct btf
*btf
,
233 u32 id
, u32
*res_id
);
234 const struct btf_type
*btf_type_resolve_func_ptr(const struct btf
*btf
,
235 u32 id
, u32
*res_id
);
236 const struct btf_type
*
237 btf_resolve_size(const struct btf
*btf
, const struct btf_type
*type
,
239 const char *btf_type_str(const struct btf_type
*t
);
241 #define for_each_member(i, struct_type, member) \
242 for (i = 0, member = btf_type_member(struct_type); \
243 i < btf_type_vlen(struct_type); \
246 #define for_each_vsi(i, datasec_type, member) \
247 for (i = 0, member = btf_type_var_secinfo(datasec_type); \
248 i < btf_type_vlen(datasec_type); \
251 static inline bool btf_type_is_ptr(const struct btf_type
*t
)
253 return BTF_INFO_KIND(t
->info
) == BTF_KIND_PTR
;
256 static inline bool btf_type_is_int(const struct btf_type
*t
)
258 return BTF_INFO_KIND(t
->info
) == BTF_KIND_INT
;
261 static inline bool btf_type_is_small_int(const struct btf_type
*t
)
263 return btf_type_is_int(t
) && t
->size
<= sizeof(u64
);
266 static inline u8
btf_int_encoding(const struct btf_type
*t
)
268 return BTF_INT_ENCODING(*(u32
*)(t
+ 1));
271 static inline bool btf_type_is_signed_int(const struct btf_type
*t
)
273 return btf_type_is_int(t
) && (btf_int_encoding(t
) & BTF_INT_SIGNED
);
276 static inline bool btf_type_is_enum(const struct btf_type
*t
)
278 return BTF_INFO_KIND(t
->info
) == BTF_KIND_ENUM
;
281 static inline bool btf_is_any_enum(const struct btf_type
*t
)
283 return BTF_INFO_KIND(t
->info
) == BTF_KIND_ENUM
||
284 BTF_INFO_KIND(t
->info
) == BTF_KIND_ENUM64
;
287 static inline bool btf_kind_core_compat(const struct btf_type
*t1
,
288 const struct btf_type
*t2
)
290 return BTF_INFO_KIND(t1
->info
) == BTF_INFO_KIND(t2
->info
) ||
291 (btf_is_any_enum(t1
) && btf_is_any_enum(t2
));
294 static inline bool str_is_empty(const char *s
)
299 static inline u16
btf_kind(const struct btf_type
*t
)
301 return BTF_INFO_KIND(t
->info
);
304 static inline bool btf_is_enum(const struct btf_type
*t
)
306 return btf_kind(t
) == BTF_KIND_ENUM
;
309 static inline bool btf_is_enum64(const struct btf_type
*t
)
311 return btf_kind(t
) == BTF_KIND_ENUM64
;
314 static inline u64
btf_enum64_value(const struct btf_enum64
*e
)
316 return ((u64
)e
->val_hi32
<< 32) | e
->val_lo32
;
319 static inline bool btf_is_composite(const struct btf_type
*t
)
321 u16 kind
= btf_kind(t
);
323 return kind
== BTF_KIND_STRUCT
|| kind
== BTF_KIND_UNION
;
326 static inline bool btf_is_array(const struct btf_type
*t
)
328 return btf_kind(t
) == BTF_KIND_ARRAY
;
331 static inline bool btf_is_int(const struct btf_type
*t
)
333 return btf_kind(t
) == BTF_KIND_INT
;
336 static inline bool btf_is_ptr(const struct btf_type
*t
)
338 return btf_kind(t
) == BTF_KIND_PTR
;
341 static inline u8
btf_int_offset(const struct btf_type
*t
)
343 return BTF_INT_OFFSET(*(u32
*)(t
+ 1));
346 static inline __u8
btf_int_bits(const struct btf_type
*t
)
348 return BTF_INT_BITS(*(__u32
*)(t
+ 1));
351 static inline bool btf_type_is_scalar(const struct btf_type
*t
)
353 return btf_type_is_int(t
) || btf_type_is_enum(t
);
356 static inline bool btf_type_is_typedef(const struct btf_type
*t
)
358 return BTF_INFO_KIND(t
->info
) == BTF_KIND_TYPEDEF
;
361 static inline bool btf_type_is_volatile(const struct btf_type
*t
)
363 return BTF_INFO_KIND(t
->info
) == BTF_KIND_VOLATILE
;
366 static inline bool btf_type_is_func(const struct btf_type
*t
)
368 return BTF_INFO_KIND(t
->info
) == BTF_KIND_FUNC
;
371 static inline bool btf_type_is_func_proto(const struct btf_type
*t
)
373 return BTF_INFO_KIND(t
->info
) == BTF_KIND_FUNC_PROTO
;
376 static inline bool btf_type_is_var(const struct btf_type
*t
)
378 return BTF_INFO_KIND(t
->info
) == BTF_KIND_VAR
;
381 static inline bool btf_type_is_type_tag(const struct btf_type
*t
)
383 return BTF_INFO_KIND(t
->info
) == BTF_KIND_TYPE_TAG
;
386 /* union is only a special case of struct:
387 * all its offsetof(member) == 0
389 static inline bool btf_type_is_struct(const struct btf_type
*t
)
391 u8 kind
= BTF_INFO_KIND(t
->info
);
393 return kind
== BTF_KIND_STRUCT
|| kind
== BTF_KIND_UNION
;
396 static inline bool __btf_type_is_struct(const struct btf_type
*t
)
398 return BTF_INFO_KIND(t
->info
) == BTF_KIND_STRUCT
;
401 static inline bool btf_type_is_array(const struct btf_type
*t
)
403 return BTF_INFO_KIND(t
->info
) == BTF_KIND_ARRAY
;
406 static inline u16
btf_type_vlen(const struct btf_type
*t
)
408 return BTF_INFO_VLEN(t
->info
);
411 static inline u16
btf_vlen(const struct btf_type
*t
)
413 return btf_type_vlen(t
);
416 static inline u16
btf_func_linkage(const struct btf_type
*t
)
418 return BTF_INFO_VLEN(t
->info
);
421 static inline bool btf_type_kflag(const struct btf_type
*t
)
423 return BTF_INFO_KFLAG(t
->info
);
426 static inline u32
__btf_member_bit_offset(const struct btf_type
*struct_type
,
427 const struct btf_member
*member
)
429 return btf_type_kflag(struct_type
) ? BTF_MEMBER_BIT_OFFSET(member
->offset
)
433 static inline u32
__btf_member_bitfield_size(const struct btf_type
*struct_type
,
434 const struct btf_member
*member
)
436 return btf_type_kflag(struct_type
) ? BTF_MEMBER_BITFIELD_SIZE(member
->offset
)
440 static inline struct btf_member
*btf_members(const struct btf_type
*t
)
442 return (struct btf_member
*)(t
+ 1);
445 static inline u32
btf_member_bit_offset(const struct btf_type
*t
, u32 member_idx
)
447 const struct btf_member
*m
= btf_members(t
) + member_idx
;
449 return __btf_member_bit_offset(t
, m
);
452 static inline u32
btf_member_bitfield_size(const struct btf_type
*t
, u32 member_idx
)
454 const struct btf_member
*m
= btf_members(t
) + member_idx
;
456 return __btf_member_bitfield_size(t
, m
);
459 static inline const struct btf_member
*btf_type_member(const struct btf_type
*t
)
461 return (const struct btf_member
*)(t
+ 1);
464 static inline struct btf_array
*btf_array(const struct btf_type
*t
)
466 return (struct btf_array
*)(t
+ 1);
469 static inline struct btf_enum
*btf_enum(const struct btf_type
*t
)
471 return (struct btf_enum
*)(t
+ 1);
474 static inline struct btf_enum64
*btf_enum64(const struct btf_type
*t
)
476 return (struct btf_enum64
*)(t
+ 1);
479 static inline const struct btf_var_secinfo
*btf_type_var_secinfo(
480 const struct btf_type
*t
)
482 return (const struct btf_var_secinfo
*)(t
+ 1);
485 static inline struct btf_param
*btf_params(const struct btf_type
*t
)
487 return (struct btf_param
*)(t
+ 1);
490 static inline struct btf_decl_tag
*btf_decl_tag(const struct btf_type
*t
)
492 return (struct btf_decl_tag
*)(t
+ 1);
495 static inline int btf_id_cmp_func(const void *a
, const void *b
)
497 const int *pa
= a
, *pb
= b
;
502 static inline bool btf_id_set_contains(const struct btf_id_set
*set
, u32 id
)
504 return bsearch(&id
, set
->ids
, set
->cnt
, sizeof(u32
), btf_id_cmp_func
) != NULL
;
507 static inline void *btf_id_set8_contains(const struct btf_id_set8
*set
, u32 id
)
509 return bsearch(&id
, set
->pairs
, set
->cnt
, sizeof(set
->pairs
[0]), btf_id_cmp_func
);
512 bool btf_param_match_suffix(const struct btf
*btf
,
513 const struct btf_param
*arg
,
515 int btf_ctx_arg_offset(const struct btf
*btf
, const struct btf_type
*func_proto
,
518 struct bpf_verifier_log
;
520 #if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL)
521 struct bpf_struct_ops
;
522 int __register_bpf_struct_ops(struct bpf_struct_ops
*st_ops
);
523 const struct bpf_struct_ops_desc
*bpf_struct_ops_find_value(struct btf
*btf
, u32 value_id
);
524 const struct bpf_struct_ops_desc
*bpf_struct_ops_find(struct btf
*btf
, u32 type_id
);
526 static inline const struct bpf_struct_ops_desc
*bpf_struct_ops_find(struct btf
*btf
, u32 type_id
)
532 enum btf_field_iter_kind
{
537 struct btf_field_desc
{
538 /* once-per-type offsets */
539 int t_off_cnt
, t_offs
[2];
540 /* member struct size, or zero, if no members */
542 /* repeated per-member offsets */
543 int m_off_cnt
, m_offs
[1];
546 struct btf_field_iter
{
547 struct btf_field_desc desc
;
554 #ifdef CONFIG_BPF_SYSCALL
555 const struct btf_type
*btf_type_by_id(const struct btf
*btf
, u32 type_id
);
556 void btf_set_base_btf(struct btf
*btf
, const struct btf
*base_btf
);
557 int btf_relocate(struct btf
*btf
, const struct btf
*base_btf
, __u32
**map_ids
);
558 int btf_field_iter_init(struct btf_field_iter
*it
, struct btf_type
*t
,
559 enum btf_field_iter_kind iter_kind
);
560 __u32
*btf_field_iter_next(struct btf_field_iter
*it
);
562 const char *btf_name_by_offset(const struct btf
*btf
, u32 offset
);
563 const char *btf_str_by_offset(const struct btf
*btf
, u32 offset
);
564 struct btf
*btf_parse_vmlinux(void);
565 struct btf
*bpf_prog_get_target_btf(const struct bpf_prog
*prog
);
566 u32
*btf_kfunc_id_set_contains(const struct btf
*btf
, u32 kfunc_btf_id
,
567 const struct bpf_prog
*prog
);
568 u32
*btf_kfunc_is_modify_return(const struct btf
*btf
, u32 kfunc_btf_id
,
569 const struct bpf_prog
*prog
);
570 int register_btf_kfunc_id_set(enum bpf_prog_type prog_type
,
571 const struct btf_kfunc_id_set
*s
);
572 int register_btf_fmodret_id_set(const struct btf_kfunc_id_set
*kset
);
573 s32
btf_find_dtor_kfunc(struct btf
*btf
, u32 btf_id
);
574 int register_btf_id_dtor_kfuncs(const struct btf_id_dtor_kfunc
*dtors
, u32 add_cnt
,
575 struct module
*owner
);
576 struct btf_struct_meta
*btf_find_struct_meta(const struct btf
*btf
, u32 btf_id
);
577 bool btf_is_projection_of(const char *pname
, const char *tname
);
578 bool btf_is_prog_ctx_type(struct bpf_verifier_log
*log
, const struct btf
*btf
,
579 const struct btf_type
*t
, enum bpf_prog_type prog_type
,
581 int get_kern_ctx_btf_id(struct bpf_verifier_log
*log
, enum bpf_prog_type prog_type
);
582 bool btf_types_are_same(const struct btf
*btf1
, u32 id1
,
583 const struct btf
*btf2
, u32 id2
);
584 int btf_check_iter_arg(struct btf
*btf
, const struct btf_type
*func
, int arg_idx
);
586 static inline bool btf_type_is_struct_ptr(struct btf
*btf
, const struct btf_type
*t
)
588 if (!btf_type_is_ptr(t
))
591 t
= btf_type_skip_modifiers(btf
, t
->type
, NULL
);
593 return btf_type_is_struct(t
);
596 static inline const struct btf_type
*btf_type_by_id(const struct btf
*btf
,
602 static inline void btf_set_base_btf(struct btf
*btf
, const struct btf
*base_btf
)
606 static inline int btf_relocate(void *log
, struct btf
*btf
, const struct btf
*base_btf
,
612 static inline int btf_field_iter_init(struct btf_field_iter
*it
, struct btf_type
*t
,
613 enum btf_field_iter_kind iter_kind
)
618 static inline __u32
*btf_field_iter_next(struct btf_field_iter
*it
)
623 static inline const char *btf_name_by_offset(const struct btf
*btf
,
628 static inline u32
*btf_kfunc_id_set_contains(const struct btf
*btf
,
630 struct bpf_prog
*prog
)
635 static inline int register_btf_kfunc_id_set(enum bpf_prog_type prog_type
,
636 const struct btf_kfunc_id_set
*s
)
640 static inline s32
btf_find_dtor_kfunc(struct btf
*btf
, u32 btf_id
)
644 static inline int register_btf_id_dtor_kfuncs(const struct btf_id_dtor_kfunc
*dtors
,
645 u32 add_cnt
, struct module
*owner
)
649 static inline struct btf_struct_meta
*btf_find_struct_meta(const struct btf
*btf
, u32 btf_id
)
654 btf_is_prog_ctx_type(struct bpf_verifier_log
*log
, const struct btf
*btf
,
655 const struct btf_type
*t
, enum bpf_prog_type prog_type
,
660 static inline int get_kern_ctx_btf_id(struct bpf_verifier_log
*log
,
661 enum bpf_prog_type prog_type
) {
664 static inline bool btf_types_are_same(const struct btf
*btf1
, u32 id1
,
665 const struct btf
*btf2
, u32 id2
)
669 static inline int btf_check_iter_arg(struct btf
*btf
, const struct btf_type
*func
, int arg_idx
)