?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
WIP FPC-III support
[linux/fpc-iii.git] / tools / lib / bpf / libbpf.c
blob6ae748f6ea118f71d900911068583007895b6fc7
1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
2
3 /*
4 * Common eBPF ELF object loading operations.
5 *
6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>
7 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
8 * Copyright (C) 2015 Huawei Inc.
9 * Copyright (C) 2017 Nicira, Inc.
10 * Copyright (C) 2019 Isovalent, Inc.
11 */
12
13 #ifndef _GNU_SOURCE
14 #define _GNU_SOURCE
15 #endif
16 #include <stdlib.h>
17 #include <stdio.h>
18 #include <stdarg.h>
19 #include <libgen.h>
20 #include <inttypes.h>
21 #include <limits.h>
22 #include <string.h>
23 #include <unistd.h>
24 #include <endian.h>
25 #include <fcntl.h>
26 #include <errno.h>
27 #include <ctype.h>
28 #include <asm/unistd.h>
29 #include <linux/err.h>
30 #include <linux/kernel.h>
31 #include <linux/bpf.h>
32 #include <linux/btf.h>
33 #include <linux/filter.h>
34 #include <linux/list.h>
35 #include <linux/limits.h>
36 #include <linux/perf_event.h>
37 #include <linux/ring_buffer.h>
38 #include <linux/version.h>
39 #include <sys/epoll.h>
40 #include <sys/ioctl.h>
41 #include <sys/mman.h>
42 #include <sys/stat.h>
43 #include <sys/types.h>
44 #include <sys/vfs.h>
45 #include <sys/utsname.h>
46 #include <sys/resource.h>
47 #include <libelf.h>
48 #include <gelf.h>
49 #include <zlib.h>
50
51 #include "libbpf.h"
52 #include "bpf.h"
53 #include "btf.h"
54 #include "str_error.h"
55 #include "libbpf_internal.h"
56 #include "hashmap.h"
57
58 #ifndef EM_BPF
59 #define EM_BPF 247
60 #endif
61
62 #ifndef BPF_FS_MAGIC
63 #define BPF_FS_MAGIC 0xcafe4a11
64 #endif
65
66 #define BPF_INSN_SZ (sizeof(struct bpf_insn))
67
68 /* vsprintf() in __base_pr() uses nonliteral format string. It may break
69 * compilation if user enables corresponding warning. Disable it explicitly.
70 */
71 #pragma GCC diagnostic ignored "-Wformat-nonliteral"
72
73 #define __printf(a, b) __attribute__((format(printf, a, b)))
74
75 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj);
76 static const struct btf_type *
77 skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id);
78
79 static int __base_pr(enum libbpf_print_level level, const char *format,
80 va_list args)
81 {
82 if (level == LIBBPF_DEBUG)
83 return 0;
84
85 return vfprintf(stderr, format, args);
86 }
87
88 static libbpf_print_fn_t __libbpf_pr = __base_pr;
89
90 libbpf_print_fn_t libbpf_set_print(libbpf_print_fn_t fn)
91 {
92 libbpf_print_fn_t old_print_fn = __libbpf_pr;
93
94 __libbpf_pr = fn;
95 return old_print_fn;
96 }
97
98 __printf(2, 3)
99 void libbpf_print(enum libbpf_print_level level, const char *format, ...)
100 {
101 va_list args;
102
103 if (!__libbpf_pr)
104 return;
105
106 va_start(args, format);
107 __libbpf_pr(level, format, args);
108 va_end(args);
109 }
110
111 static void pr_perm_msg(int err)
112 {
113 struct rlimit limit;
114 char buf[100];
115
116 if (err != -EPERM || geteuid() != 0)
117 return;
118
119 err = getrlimit(RLIMIT_MEMLOCK, &limit);
120 if (err)
121 return;
122
123 if (limit.rlim_cur == RLIM_INFINITY)
124 return;
125
126 if (limit.rlim_cur < 1024)
127 snprintf(buf, sizeof(buf), "%zu bytes", (size_t)limit.rlim_cur);
128 else if (limit.rlim_cur < 1024*1024)
129 snprintf(buf, sizeof(buf), "%.1f KiB", (double)limit.rlim_cur / 1024);
130 else
131 snprintf(buf, sizeof(buf), "%.1f MiB", (double)limit.rlim_cur / (1024*1024));
132
133 pr_warn("permission error while running as root; try raising 'ulimit -l'? current value: %s\n",
134 buf);
135 }
136
137 #define STRERR_BUFSIZE 128
138
139 /* Copied from tools/perf/util/util.h */
140 #ifndef zfree
141 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; })
142 #endif
143
144 #ifndef zclose
145 # define zclose(fd) ({ \
146 int ___err = 0; \
147 if ((fd) >= 0) \
148 ___err = close((fd)); \
149 fd = -1; \
150 ___err; })
151 #endif
152
153 static inline __u64 ptr_to_u64(const void *ptr)
154 {
155 return (__u64) (unsigned long) ptr;
156 }
157
158 enum kern_feature_id {
159 /* v4.14: kernel support for program & map names. */
160 FEAT_PROG_NAME,
161 /* v5.2: kernel support for global data sections. */
162 FEAT_GLOBAL_DATA,
163 /* BTF support */
164 FEAT_BTF,
165 /* BTF_KIND_FUNC and BTF_KIND_FUNC_PROTO support */
166 FEAT_BTF_FUNC,
167 /* BTF_KIND_VAR and BTF_KIND_DATASEC support */
168 FEAT_BTF_DATASEC,
169 /* BTF_FUNC_GLOBAL is supported */
170 FEAT_BTF_GLOBAL_FUNC,
171 /* BPF_F_MMAPABLE is supported for arrays */
172 FEAT_ARRAY_MMAP,
173 /* kernel support for expected_attach_type in BPF_PROG_LOAD */
174 FEAT_EXP_ATTACH_TYPE,
175 /* bpf_probe_read_{kernel,user}[_str] helpers */
176 FEAT_PROBE_READ_KERN,
177 /* BPF_PROG_BIND_MAP is supported */
178 FEAT_PROG_BIND_MAP,
179 /* Kernel support for module BTFs */
180 FEAT_MODULE_BTF,
181 __FEAT_CNT,
182 };
183
184 static bool kernel_supports(enum kern_feature_id feat_id);
185
186 enum reloc_type {
187 RELO_LD64,
188 RELO_CALL,
189 RELO_DATA,
190 RELO_EXTERN,
191 };
192
193 struct reloc_desc {
194 enum reloc_type type;
195 int insn_idx;
196 int map_idx;
197 int sym_off;
198 bool processed;
199 };
200
201 struct bpf_sec_def;
202
203 typedef struct bpf_link *(*attach_fn_t)(const struct bpf_sec_def *sec,
204 struct bpf_program *prog);
205
206 struct bpf_sec_def {
207 const char *sec;
208 size_t len;
209 enum bpf_prog_type prog_type;
210 enum bpf_attach_type expected_attach_type;
211 bool is_exp_attach_type_optional;
212 bool is_attachable;
213 bool is_attach_btf;
214 bool is_sleepable;
215 attach_fn_t attach_fn;
216 };
217
218 /*
219 * bpf_prog should be a better name but it has been used in
220 * linux/filter.h.
221 */
222 struct bpf_program {
223 const struct bpf_sec_def *sec_def;
224 char *sec_name;
225 size_t sec_idx;
226 /* this program's instruction offset (in number of instructions)
227 * within its containing ELF section
228 */
229 size_t sec_insn_off;
230 /* number of original instructions in ELF section belonging to this
231 * program, not taking into account subprogram instructions possible
232 * appended later during relocation
233 */
234 size_t sec_insn_cnt;
235 /* Offset (in number of instructions) of the start of instruction
236 * belonging to this BPF program within its containing main BPF
237 * program. For the entry-point (main) BPF program, this is always
238 * zero. For a sub-program, this gets reset before each of main BPF
239 * programs are processed and relocated and is used to determined
240 * whether sub-program was already appended to the main program, and
241 * if yes, at which instruction offset.
242 */
243 size_t sub_insn_off;
244
245 char *name;
246 /* sec_name with / replaced by _; makes recursive pinning
247 * in bpf_object__pin_programs easier
248 */
249 char *pin_name;
250
251 /* instructions that belong to BPF program; insns[0] is located at
252 * sec_insn_off instruction within its ELF section in ELF file, so
253 * when mapping ELF file instruction index to the local instruction,
254 * one needs to subtract sec_insn_off; and vice versa.
255 */
256 struct bpf_insn *insns;
257 /* actual number of instruction in this BPF program's image; for
258 * entry-point BPF programs this includes the size of main program
259 * itself plus all the used sub-programs, appended at the end
260 */
261 size_t insns_cnt;
262
263 struct reloc_desc *reloc_desc;
264 int nr_reloc;
265 int log_level;
266
267 struct {
268 int nr;
269 int *fds;
270 } instances;
271 bpf_program_prep_t preprocessor;
272
273 struct bpf_object *obj;
274 void *priv;
275 bpf_program_clear_priv_t clear_priv;
276
277 bool load;
278 enum bpf_prog_type type;
279 enum bpf_attach_type expected_attach_type;
280 int prog_ifindex;
281 __u32 attach_btf_obj_fd;
282 __u32 attach_btf_id;
283 __u32 attach_prog_fd;
284 void *func_info;
285 __u32 func_info_rec_size;
286 __u32 func_info_cnt;
287
288 void *line_info;
289 __u32 line_info_rec_size;
290 __u32 line_info_cnt;
291 __u32 prog_flags;
292 };
293
294 struct bpf_struct_ops {
295 const char *tname;
296 const struct btf_type *type;
297 struct bpf_program **progs;
298 __u32 *kern_func_off;
299 /* e.g. struct tcp_congestion_ops in bpf_prog's btf format */
300 void *data;
301 /* e.g. struct bpf_struct_ops_tcp_congestion_ops in
302 * btf_vmlinux's format.
303 * struct bpf_struct_ops_tcp_congestion_ops {
304 * [... some other kernel fields ...]
305 * struct tcp_congestion_ops data;
306 * }
307 * kern_vdata-size == sizeof(struct bpf_struct_ops_tcp_congestion_ops)
308 * bpf_map__init_kern_struct_ops() will populate the "kern_vdata"
309 * from "data".
310 */
311 void *kern_vdata;
312 __u32 type_id;
313 };
314
315 #define DATA_SEC ".data"
316 #define BSS_SEC ".bss"
317 #define RODATA_SEC ".rodata"
318 #define KCONFIG_SEC ".kconfig"
319 #define KSYMS_SEC ".ksyms"
320 #define STRUCT_OPS_SEC ".struct_ops"
321
322 enum libbpf_map_type {
323 LIBBPF_MAP_UNSPEC,
324 LIBBPF_MAP_DATA,
325 LIBBPF_MAP_BSS,
326 LIBBPF_MAP_RODATA,
327 LIBBPF_MAP_KCONFIG,
328 };
329
330 static const char * const libbpf_type_to_btf_name[] = {
331 [LIBBPF_MAP_DATA] = DATA_SEC,
332 [LIBBPF_MAP_BSS] = BSS_SEC,
333 [LIBBPF_MAP_RODATA] = RODATA_SEC,
334 [LIBBPF_MAP_KCONFIG] = KCONFIG_SEC,
335 };
336
337 struct bpf_map {
338 char *name;
339 int fd;
340 int sec_idx;
341 size_t sec_offset;
342 int map_ifindex;
343 int inner_map_fd;
344 struct bpf_map_def def;
345 __u32 numa_node;
346 __u32 btf_var_idx;
347 __u32 btf_key_type_id;
348 __u32 btf_value_type_id;
349 __u32 btf_vmlinux_value_type_id;
350 void *priv;
351 bpf_map_clear_priv_t clear_priv;
352 enum libbpf_map_type libbpf_type;
353 void *mmaped;
354 struct bpf_struct_ops *st_ops;
355 struct bpf_map *inner_map;
356 void **init_slots;
357 int init_slots_sz;
358 char *pin_path;
359 bool pinned;
360 bool reused;
361 };
362
363 enum extern_type {
364 EXT_UNKNOWN,
365 EXT_KCFG,
366 EXT_KSYM,
367 };
368
369 enum kcfg_type {
370 KCFG_UNKNOWN,
371 KCFG_CHAR,
372 KCFG_BOOL,
373 KCFG_INT,
374 KCFG_TRISTATE,
375 KCFG_CHAR_ARR,
376 };
377
378 struct extern_desc {
379 enum extern_type type;
380 int sym_idx;
381 int btf_id;
382 int sec_btf_id;
383 const char *name;
384 bool is_set;
385 bool is_weak;
386 union {
387 struct {
388 enum kcfg_type type;
389 int sz;
390 int align;
391 int data_off;
392 bool is_signed;
393 } kcfg;
394 struct {
395 unsigned long long addr;
396
397 /* target btf_id of the corresponding kernel var. */
398 int vmlinux_btf_id;
399
400 /* local btf_id of the ksym extern's type. */
401 __u32 type_id;
402 } ksym;
403 };
404 };
405
406 static LIST_HEAD(bpf_objects_list);
407
408 struct module_btf {
409 struct btf *btf;
410 char *name;
411 __u32 id;
412 int fd;
413 };
414
415 struct bpf_object {
416 char name[BPF_OBJ_NAME_LEN];
417 char license[64];
418 __u32 kern_version;
419
420 struct bpf_program *programs;
421 size_t nr_programs;
422 struct bpf_map *maps;
423 size_t nr_maps;
424 size_t maps_cap;
425
426 char *kconfig;
427 struct extern_desc *externs;
428 int nr_extern;
429 int kconfig_map_idx;
430 int rodata_map_idx;
431
432 bool loaded;
433 bool has_subcalls;
434
435 /*
436 * Information when doing elf related work. Only valid if fd
437 * is valid.
438 */
439 struct {
440 int fd;
441 const void *obj_buf;
442 size_t obj_buf_sz;
443 Elf *elf;
444 GElf_Ehdr ehdr;
445 Elf_Data *symbols;
446 Elf_Data *data;
447 Elf_Data *rodata;
448 Elf_Data *bss;
449 Elf_Data *st_ops_data;
450 size_t shstrndx; /* section index for section name strings */
451 size_t strtabidx;
452 struct {
453 GElf_Shdr shdr;
454 Elf_Data *data;
455 } *reloc_sects;
456 int nr_reloc_sects;
457 int maps_shndx;
458 int btf_maps_shndx;
459 __u32 btf_maps_sec_btf_id;
460 int text_shndx;
461 int symbols_shndx;
462 int data_shndx;
463 int rodata_shndx;
464 int bss_shndx;
465 int st_ops_shndx;
466 } efile;
467 /*
468 * All loaded bpf_object is linked in a list, which is
469 * hidden to caller. bpf_objects__<func> handlers deal with
470 * all objects.
471 */
472 struct list_head list;
473
474 struct btf *btf;
475 struct btf_ext *btf_ext;
476
477 /* Parse and load BTF vmlinux if any of the programs in the object need
478 * it at load time.
479 */
480 struct btf *btf_vmlinux;
481 /* vmlinux BTF override for CO-RE relocations */
482 struct btf *btf_vmlinux_override;
483 /* Lazily initialized kernel module BTFs */
484 struct module_btf *btf_modules;
485 bool btf_modules_loaded;
486 size_t btf_module_cnt;
487 size_t btf_module_cap;
488
489 void *priv;
490 bpf_object_clear_priv_t clear_priv;
491
492 char path[];
493 };
494 #define obj_elf_valid(o) ((o)->efile.elf)
495
496 static const char *elf_sym_str(const struct bpf_object *obj, size_t off);
497 static const char *elf_sec_str(const struct bpf_object *obj, size_t off);
498 static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx);
499 static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name);
500 static int elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn, GElf_Shdr *hdr);
501 static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn);
502 static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn);
503 static int elf_sym_by_sec_off(const struct bpf_object *obj, size_t sec_idx,
504 size_t off, __u32 sym_type, GElf_Sym *sym);
505
506 void bpf_program__unload(struct bpf_program *prog)
507 {
508 int i;
509
510 if (!prog)
511 return;
512
513 /*
514 * If the object is opened but the program was never loaded,
515 * it is possible that prog->instances.nr == -1.
516 */
517 if (prog->instances.nr > 0) {
518 for (i = 0; i < prog->instances.nr; i++)
519 zclose(prog->instances.fds[i]);
520 } else if (prog->instances.nr != -1) {
521 pr_warn("Internal error: instances.nr is %d\n",
522 prog->instances.nr);
523 }
524
525 prog->instances.nr = -1;
526 zfree(&prog->instances.fds);
527
528 zfree(&prog->func_info);
529 zfree(&prog->line_info);
530 }
531
532 static void bpf_program__exit(struct bpf_program *prog)
533 {
534 if (!prog)
535 return;
536
537 if (prog->clear_priv)
538 prog->clear_priv(prog, prog->priv);
539
540 prog->priv = NULL;
541 prog->clear_priv = NULL;
542
543 bpf_program__unload(prog);
544 zfree(&prog->name);
545 zfree(&prog->sec_name);
546 zfree(&prog->pin_name);
547 zfree(&prog->insns);
548 zfree(&prog->reloc_desc);
549
550 prog->nr_reloc = 0;
551 prog->insns_cnt = 0;
552 prog->sec_idx = -1;
553 }
554
555 static char *__bpf_program__pin_name(struct bpf_program *prog)
556 {
557 char *name, *p;
558
559 name = p = strdup(prog->sec_name);
560 while ((p = strchr(p, '/')))
561 *p = '_';
562
563 return name;
564 }
565
566 static bool insn_is_subprog_call(const struct bpf_insn *insn)
567 {
568 return BPF_CLASS(insn->code) == BPF_JMP &&
569 BPF_OP(insn->code) == BPF_CALL &&
570 BPF_SRC(insn->code) == BPF_K &&
571 insn->src_reg == BPF_PSEUDO_CALL &&
572 insn->dst_reg == 0 &&
573 insn->off == 0;
574 }
575
576 static int
577 bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog,
578 const char *name, size_t sec_idx, const char *sec_name,
579 size_t sec_off, void *insn_data, size_t insn_data_sz)
580 {
581 if (insn_data_sz == 0 || insn_data_sz % BPF_INSN_SZ || sec_off % BPF_INSN_SZ) {
582 pr_warn("sec '%s': corrupted program '%s', offset %zu, size %zu\n",
583 sec_name, name, sec_off, insn_data_sz);
584 return -EINVAL;
585 }
586
587 memset(prog, 0, sizeof(*prog));
588 prog->obj = obj;
589
590 prog->sec_idx = sec_idx;
591 prog->sec_insn_off = sec_off / BPF_INSN_SZ;
592 prog->sec_insn_cnt = insn_data_sz / BPF_INSN_SZ;
593 /* insns_cnt can later be increased by appending used subprograms */
594 prog->insns_cnt = prog->sec_insn_cnt;
595
596 prog->type = BPF_PROG_TYPE_UNSPEC;
597 prog->load = true;
598
599 prog->instances.fds = NULL;
600 prog->instances.nr = -1;
601
602 prog->sec_name = strdup(sec_name);
603 if (!prog->sec_name)
604 goto errout;
605
606 prog->name = strdup(name);
607 if (!prog->name)
608 goto errout;
609
610 prog->pin_name = __bpf_program__pin_name(prog);
611 if (!prog->pin_name)
612 goto errout;
613
614 prog->insns = malloc(insn_data_sz);
615 if (!prog->insns)
616 goto errout;
617 memcpy(prog->insns, insn_data, insn_data_sz);
618
619 return 0;
620 errout:
621 pr_warn("sec '%s': failed to allocate memory for prog '%s'\n", sec_name, name);
622 bpf_program__exit(prog);
623 return -ENOMEM;
624 }
625
626 static int
627 bpf_object__add_programs(struct bpf_object *obj, Elf_Data *sec_data,
628 const char *sec_name, int sec_idx)
629 {
630 struct bpf_program *prog, *progs;
631 void *data = sec_data->d_buf;
632 size_t sec_sz = sec_data->d_size, sec_off, prog_sz;
633 int nr_progs, err;
634 const char *name;
635 GElf_Sym sym;
636
637 progs = obj->programs;
638 nr_progs = obj->nr_programs;
639 sec_off = 0;
640
641 while (sec_off < sec_sz) {
642 if (elf_sym_by_sec_off(obj, sec_idx, sec_off, STT_FUNC, &sym)) {
643 pr_warn("sec '%s': failed to find program symbol at offset %zu\n",
644 sec_name, sec_off);
645 return -LIBBPF_ERRNO__FORMAT;
646 }
647
648 prog_sz = sym.st_size;
649
650 name = elf_sym_str(obj, sym.st_name);
651 if (!name) {
652 pr_warn("sec '%s': failed to get symbol name for offset %zu\n",
653 sec_name, sec_off);
654 return -LIBBPF_ERRNO__FORMAT;
655 }
656
657 if (sec_off + prog_sz > sec_sz) {
658 pr_warn("sec '%s': program at offset %zu crosses section boundary\n",
659 sec_name, sec_off);
660 return -LIBBPF_ERRNO__FORMAT;
661 }
662
663 pr_debug("sec '%s': found program '%s' at insn offset %zu (%zu bytes), code size %zu insns (%zu bytes)\n",
664 sec_name, name, sec_off / BPF_INSN_SZ, sec_off, prog_sz / BPF_INSN_SZ, prog_sz);
665
666 progs = libbpf_reallocarray(progs, nr_progs + 1, sizeof(*progs));
667 if (!progs) {
668 /*
669 * In this case the original obj->programs
670 * is still valid, so don't need special treat for
671 * bpf_close_object().
672 */
673 pr_warn("sec '%s': failed to alloc memory for new program '%s'\n",
674 sec_name, name);
675 return -ENOMEM;
676 }
677 obj->programs = progs;
678
679 prog = &progs[nr_progs];
680
681 err = bpf_object__init_prog(obj, prog, name, sec_idx, sec_name,
682 sec_off, data + sec_off, prog_sz);
683 if (err)
684 return err;
685
686 nr_progs++;
687 obj->nr_programs = nr_progs;
688
689 sec_off += prog_sz;
690 }
691
692 return 0;
693 }
694
695 static __u32 get_kernel_version(void)
696 {
697 __u32 major, minor, patch;
698 struct utsname info;
699
700 uname(&info);
701 if (sscanf(info.release, "%u.%u.%u", &major, &minor, &patch) != 3)
702 return 0;
703 return KERNEL_VERSION(major, minor, patch);
704 }
705
706 static const struct btf_member *
707 find_member_by_offset(const struct btf_type *t, __u32 bit_offset)
708 {
709 struct btf_member *m;
710 int i;
711
712 for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
713 if (btf_member_bit_offset(t, i) == bit_offset)
714 return m;
715 }
716
717 return NULL;
718 }
719
720 static const struct btf_member *
721 find_member_by_name(const struct btf *btf, const struct btf_type *t,
722 const char *name)
723 {
724 struct btf_member *m;
725 int i;
726
727 for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
728 if (!strcmp(btf__name_by_offset(btf, m->name_off), name))
729 return m;
730 }
731
732 return NULL;
733 }
734
735 #define STRUCT_OPS_VALUE_PREFIX "bpf_struct_ops_"
736 static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix,
737 const char *name, __u32 kind);
738
739 static int
740 find_struct_ops_kern_types(const struct btf *btf, const char *tname,
741 const struct btf_type **type, __u32 *type_id,
742 const struct btf_type **vtype, __u32 *vtype_id,
743 const struct btf_member **data_member)
744 {
745 const struct btf_type *kern_type, *kern_vtype;
746 const struct btf_member *kern_data_member;
747 __s32 kern_vtype_id, kern_type_id;
748 __u32 i;
749
750 kern_type_id = btf__find_by_name_kind(btf, tname, BTF_KIND_STRUCT);
751 if (kern_type_id < 0) {
752 pr_warn("struct_ops init_kern: struct %s is not found in kernel BTF\n",
753 tname);
754 return kern_type_id;
755 }
756 kern_type = btf__type_by_id(btf, kern_type_id);
757
758 /* Find the corresponding "map_value" type that will be used
759 * in map_update(BPF_MAP_TYPE_STRUCT_OPS). For example,
760 * find "struct bpf_struct_ops_tcp_congestion_ops" from the
761 * btf_vmlinux.
762 */
763 kern_vtype_id = find_btf_by_prefix_kind(btf, STRUCT_OPS_VALUE_PREFIX,
764 tname, BTF_KIND_STRUCT);
765 if (kern_vtype_id < 0) {
766 pr_warn("struct_ops init_kern: struct %s%s is not found in kernel BTF\n",
767 STRUCT_OPS_VALUE_PREFIX, tname);
768 return kern_vtype_id;
769 }
770 kern_vtype = btf__type_by_id(btf, kern_vtype_id);
771
772 /* Find "struct tcp_congestion_ops" from
773 * struct bpf_struct_ops_tcp_congestion_ops {
774 * [ ... ]
775 * struct tcp_congestion_ops data;
776 * }
777 */
778 kern_data_member = btf_members(kern_vtype);
779 for (i = 0; i < btf_vlen(kern_vtype); i++, kern_data_member++) {
780 if (kern_data_member->type == kern_type_id)
781 break;
782 }
783 if (i == btf_vlen(kern_vtype)) {
784 pr_warn("struct_ops init_kern: struct %s data is not found in struct %s%s\n",
785 tname, STRUCT_OPS_VALUE_PREFIX, tname);
786 return -EINVAL;
787 }
788
789 *type = kern_type;
790 *type_id = kern_type_id;
791 *vtype = kern_vtype;
792 *vtype_id = kern_vtype_id;
793 *data_member = kern_data_member;
794
795 return 0;
796 }
797
798 static bool bpf_map__is_struct_ops(const struct bpf_map *map)
799 {
800 return map->def.type == BPF_MAP_TYPE_STRUCT_OPS;
801 }
802
803 /* Init the map's fields that depend on kern_btf */
804 static int bpf_map__init_kern_struct_ops(struct bpf_map *map,
805 const struct btf *btf,
806 const struct btf *kern_btf)
807 {
808 const struct btf_member *member, *kern_member, *kern_data_member;
809 const struct btf_type *type, *kern_type, *kern_vtype;
810 __u32 i, kern_type_id, kern_vtype_id, kern_data_off;
811 struct bpf_struct_ops *st_ops;
812 void *data, *kern_data;
813 const char *tname;
814 int err;
815
816 st_ops = map->st_ops;
817 type = st_ops->type;
818 tname = st_ops->tname;
819 err = find_struct_ops_kern_types(kern_btf, tname,
820 &kern_type, &kern_type_id,
821 &kern_vtype, &kern_vtype_id,
822 &kern_data_member);
823 if (err)
824 return err;
825
826 pr_debug("struct_ops init_kern %s: type_id:%u kern_type_id:%u kern_vtype_id:%u\n",
827 map->name, st_ops->type_id, kern_type_id, kern_vtype_id);
828
829 map->def.value_size = kern_vtype->size;
830 map->btf_vmlinux_value_type_id = kern_vtype_id;
831
832 st_ops->kern_vdata = calloc(1, kern_vtype->size);
833 if (!st_ops->kern_vdata)
834 return -ENOMEM;
835
836 data = st_ops->data;
837 kern_data_off = kern_data_member->offset / 8;
838 kern_data = st_ops->kern_vdata + kern_data_off;
839
840 member = btf_members(type);
841 for (i = 0; i < btf_vlen(type); i++, member++) {
842 const struct btf_type *mtype, *kern_mtype;
843 __u32 mtype_id, kern_mtype_id;
844 void *mdata, *kern_mdata;
845 __s64 msize, kern_msize;
846 __u32 moff, kern_moff;
847 __u32 kern_member_idx;
848 const char *mname;
849
850 mname = btf__name_by_offset(btf, member->name_off);
851 kern_member = find_member_by_name(kern_btf, kern_type, mname);
852 if (!kern_member) {
853 pr_warn("struct_ops init_kern %s: Cannot find member %s in kernel BTF\n",
854 map->name, mname);
855 return -ENOTSUP;
856 }
857
858 kern_member_idx = kern_member - btf_members(kern_type);
859 if (btf_member_bitfield_size(type, i) ||
860 btf_member_bitfield_size(kern_type, kern_member_idx)) {
861 pr_warn("struct_ops init_kern %s: bitfield %s is not supported\n",
862 map->name, mname);
863 return -ENOTSUP;
864 }
865
866 moff = member->offset / 8;
867 kern_moff = kern_member->offset / 8;
868
869 mdata = data + moff;
870 kern_mdata = kern_data + kern_moff;
871
872 mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id);
873 kern_mtype = skip_mods_and_typedefs(kern_btf, kern_member->type,
874 &kern_mtype_id);
875 if (BTF_INFO_KIND(mtype->info) !=
876 BTF_INFO_KIND(kern_mtype->info)) {
877 pr_warn("struct_ops init_kern %s: Unmatched member type %s %u != %u(kernel)\n",
878 map->name, mname, BTF_INFO_KIND(mtype->info),
879 BTF_INFO_KIND(kern_mtype->info));
880 return -ENOTSUP;
881 }
882
883 if (btf_is_ptr(mtype)) {
884 struct bpf_program *prog;
885
886 mtype = skip_mods_and_typedefs(btf, mtype->type, &mtype_id);
887 kern_mtype = skip_mods_and_typedefs(kern_btf,
888 kern_mtype->type,
889 &kern_mtype_id);
890 if (!btf_is_func_proto(mtype) ||
891 !btf_is_func_proto(kern_mtype)) {
892 pr_warn("struct_ops init_kern %s: non func ptr %s is not supported\n",
893 map->name, mname);
894 return -ENOTSUP;
895 }
896
897 prog = st_ops->progs[i];
898 if (!prog) {
899 pr_debug("struct_ops init_kern %s: func ptr %s is not set\n",
900 map->name, mname);
901 continue;
902 }
903
904 prog->attach_btf_id = kern_type_id;
905 prog->expected_attach_type = kern_member_idx;
906
907 st_ops->kern_func_off[i] = kern_data_off + kern_moff;
908
909 pr_debug("struct_ops init_kern %s: func ptr %s is set to prog %s from data(+%u) to kern_data(+%u)\n",
910 map->name, mname, prog->name, moff,
911 kern_moff);
912
913 continue;
914 }
915
916 msize = btf__resolve_size(btf, mtype_id);
917 kern_msize = btf__resolve_size(kern_btf, kern_mtype_id);
918 if (msize < 0 || kern_msize < 0 || msize != kern_msize) {
919 pr_warn("struct_ops init_kern %s: Error in size of member %s: %zd != %zd(kernel)\n",
920 map->name, mname, (ssize_t)msize,
921 (ssize_t)kern_msize);
922 return -ENOTSUP;
923 }
924
925 pr_debug("struct_ops init_kern %s: copy %s %u bytes from data(+%u) to kern_data(+%u)\n",
926 map->name, mname, (unsigned int)msize,
927 moff, kern_moff);
928 memcpy(kern_mdata, mdata, msize);
929 }
930
931 return 0;
932 }
933
934 static int bpf_object__init_kern_struct_ops_maps(struct bpf_object *obj)
935 {
936 struct bpf_map *map;
937 size_t i;
938 int err;
939
940 for (i = 0; i < obj->nr_maps; i++) {
941 map = &obj->maps[i];
942
943 if (!bpf_map__is_struct_ops(map))
944 continue;
945
946 err = bpf_map__init_kern_struct_ops(map, obj->btf,
947 obj->btf_vmlinux);
948 if (err)
949 return err;
950 }
951
952 return 0;
953 }
954
955 static int bpf_object__init_struct_ops_maps(struct bpf_object *obj)
956 {
957 const struct btf_type *type, *datasec;
958 const struct btf_var_secinfo *vsi;
959 struct bpf_struct_ops *st_ops;
960 const char *tname, *var_name;
961 __s32 type_id, datasec_id;
962 const struct btf *btf;
963 struct bpf_map *map;
964 __u32 i;
965
966 if (obj->efile.st_ops_shndx == -1)
967 return 0;
968
969 btf = obj->btf;
970 datasec_id = btf__find_by_name_kind(btf, STRUCT_OPS_SEC,
971 BTF_KIND_DATASEC);
972 if (datasec_id < 0) {
973 pr_warn("struct_ops init: DATASEC %s not found\n",
974 STRUCT_OPS_SEC);
975 return -EINVAL;
976 }
977
978 datasec = btf__type_by_id(btf, datasec_id);
979 vsi = btf_var_secinfos(datasec);
980 for (i = 0; i < btf_vlen(datasec); i++, vsi++) {
981 type = btf__type_by_id(obj->btf, vsi->type);
982 var_name = btf__name_by_offset(obj->btf, type->name_off);
983
984 type_id = btf__resolve_type(obj->btf, vsi->type);
985 if (type_id < 0) {
986 pr_warn("struct_ops init: Cannot resolve var type_id %u in DATASEC %s\n",
987 vsi->type, STRUCT_OPS_SEC);
988 return -EINVAL;
989 }
990
991 type = btf__type_by_id(obj->btf, type_id);
992 tname = btf__name_by_offset(obj->btf, type->name_off);
993 if (!tname[0]) {
994 pr_warn("struct_ops init: anonymous type is not supported\n");
995 return -ENOTSUP;
996 }
997 if (!btf_is_struct(type)) {
998 pr_warn("struct_ops init: %s is not a struct\n", tname);
999 return -EINVAL;
1000 }
1001
1002 map = bpf_object__add_map(obj);
1003 if (IS_ERR(map))
1004 return PTR_ERR(map);
1005
1006 map->sec_idx = obj->efile.st_ops_shndx;
1007 map->sec_offset = vsi->offset;
1008 map->name = strdup(var_name);
1009 if (!map->name)
1010 return -ENOMEM;
1011
1012 map->def.type = BPF_MAP_TYPE_STRUCT_OPS;
1013 map->def.key_size = sizeof(int);
1014 map->def.value_size = type->size;
1015 map->def.max_entries = 1;
1016
1017 map->st_ops = calloc(1, sizeof(*map->st_ops));
1018 if (!map->st_ops)
1019 return -ENOMEM;
1020 st_ops = map->st_ops;
1021 st_ops->data = malloc(type->size);
1022 st_ops->progs = calloc(btf_vlen(type), sizeof(*st_ops->progs));
1023 st_ops->kern_func_off = malloc(btf_vlen(type) *
1024 sizeof(*st_ops->kern_func_off));
1025 if (!st_ops->data || !st_ops->progs || !st_ops->kern_func_off)
1026 return -ENOMEM;
1027
1028 if (vsi->offset + type->size > obj->efile.st_ops_data->d_size) {
1029 pr_warn("struct_ops init: var %s is beyond the end of DATASEC %s\n",
1030 var_name, STRUCT_OPS_SEC);
1031 return -EINVAL;
1032 }
1033
1034 memcpy(st_ops->data,
1035 obj->efile.st_ops_data->d_buf + vsi->offset,
1036 type->size);
1037 st_ops->tname = tname;
1038 st_ops->type = type;
1039 st_ops->type_id = type_id;
1040
1041 pr_debug("struct_ops init: struct %s(type_id=%u) %s found at offset %u\n",
1042 tname, type_id, var_name, vsi->offset);
1043 }
1044
1045 return 0;
1046 }
1047
1048 static struct bpf_object *bpf_object__new(const char *path,
1049 const void *obj_buf,
1050 size_t obj_buf_sz,
1051 const char *obj_name)
1052 {
1053 struct bpf_object *obj;
1054 char *end;
1055
1056 obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1);
1057 if (!obj) {
1058 pr_warn("alloc memory failed for %s\n", path);
1059 return ERR_PTR(-ENOMEM);
1060 }
1061
1062 strcpy(obj->path, path);
1063 if (obj_name) {
1064 strncpy(obj->name, obj_name, sizeof(obj->name) - 1);
1065 obj->name[sizeof(obj->name) - 1] = 0;
1066 } else {
1067 /* Using basename() GNU version which doesn't modify arg. */
1068 strncpy(obj->name, basename((void *)path),
1069 sizeof(obj->name) - 1);
1070 end = strchr(obj->name, '.');
1071 if (end)
1072 *end = 0;
1073 }
1074
1075 obj->efile.fd = -1;
1076 /*
1077 * Caller of this function should also call
1078 * bpf_object__elf_finish() after data collection to return
1079 * obj_buf to user. If not, we should duplicate the buffer to
1080 * avoid user freeing them before elf finish.
1081 */
1082 obj->efile.obj_buf = obj_buf;
1083 obj->efile.obj_buf_sz = obj_buf_sz;
1084 obj->efile.maps_shndx = -1;
1085 obj->efile.btf_maps_shndx = -1;
1086 obj->efile.data_shndx = -1;
1087 obj->efile.rodata_shndx = -1;
1088 obj->efile.bss_shndx = -1;
1089 obj->efile.st_ops_shndx = -1;
1090 obj->kconfig_map_idx = -1;
1091 obj->rodata_map_idx = -1;
1092
1093 obj->kern_version = get_kernel_version();
1094 obj->loaded = false;
1095
1096 INIT_LIST_HEAD(&obj->list);
1097 list_add(&obj->list, &bpf_objects_list);
1098 return obj;
1099 }
1100
1101 static void bpf_object__elf_finish(struct bpf_object *obj)
1102 {
1103 if (!obj_elf_valid(obj))
1104 return;
1105
1106 if (obj->efile.elf) {
1107 elf_end(obj->efile.elf);
1108 obj->efile.elf = NULL;
1109 }
1110 obj->efile.symbols = NULL;
1111 obj->efile.data = NULL;
1112 obj->efile.rodata = NULL;
1113 obj->efile.bss = NULL;
1114 obj->efile.st_ops_data = NULL;
1115
1116 zfree(&obj->efile.reloc_sects);
1117 obj->efile.nr_reloc_sects = 0;
1118 zclose(obj->efile.fd);
1119 obj->efile.obj_buf = NULL;
1120 obj->efile.obj_buf_sz = 0;
1121 }
1122
1123 /* if libelf is old and doesn't support mmap(), fall back to read() */
1124 #ifndef ELF_C_READ_MMAP
1125 #define ELF_C_READ_MMAP ELF_C_READ
1126 #endif
1127
1128 static int bpf_object__elf_init(struct bpf_object *obj)
1129 {
1130 int err = 0;
1131 GElf_Ehdr *ep;
1132
1133 if (obj_elf_valid(obj)) {
1134 pr_warn("elf: init internal error\n");
1135 return -LIBBPF_ERRNO__LIBELF;
1136 }
1137
1138 if (obj->efile.obj_buf_sz > 0) {
1139 /*
1140 * obj_buf should have been validated by
1141 * bpf_object__open_buffer().
1142 */
1143 obj->efile.elf = elf_memory((char *)obj->efile.obj_buf,
1144 obj->efile.obj_buf_sz);
1145 } else {
1146 obj->efile.fd = open(obj->path, O_RDONLY);
1147 if (obj->efile.fd < 0) {
1148 char errmsg[STRERR_BUFSIZE], *cp;
1149
1150 err = -errno;
1151 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
1152 pr_warn("elf: failed to open %s: %s\n", obj->path, cp);
1153 return err;
1154 }
1155
1156 obj->efile.elf = elf_begin(obj->efile.fd, ELF_C_READ_MMAP, NULL);
1157 }
1158
1159 if (!obj->efile.elf) {
1160 pr_warn("elf: failed to open %s as ELF file: %s\n", obj->path, elf_errmsg(-1));
1161 err = -LIBBPF_ERRNO__LIBELF;
1162 goto errout;
1163 }
1164
1165 if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) {
1166 pr_warn("elf: failed to get ELF header from %s: %s\n", obj->path, elf_errmsg(-1));
1167 err = -LIBBPF_ERRNO__FORMAT;
1168 goto errout;
1169 }
1170 ep = &obj->efile.ehdr;
1171
1172 if (elf_getshdrstrndx(obj->efile.elf, &obj->efile.shstrndx)) {
1173 pr_warn("elf: failed to get section names section index for %s: %s\n",
1174 obj->path, elf_errmsg(-1));
1175 err = -LIBBPF_ERRNO__FORMAT;
1176 goto errout;
1177 }
1178
1179 /* Elf is corrupted/truncated, avoid calling elf_strptr. */
1180 if (!elf_rawdata(elf_getscn(obj->efile.elf, obj->efile.shstrndx), NULL)) {
1181 pr_warn("elf: failed to get section names strings from %s: %s\n",
1182 obj->path, elf_errmsg(-1));
1183 return -LIBBPF_ERRNO__FORMAT;
1184 }
1185
1186 /* Old LLVM set e_machine to EM_NONE */
1187 if (ep->e_type != ET_REL ||
1188 (ep->e_machine && ep->e_machine != EM_BPF)) {
1189 pr_warn("elf: %s is not a valid eBPF object file\n", obj->path);
1190 err = -LIBBPF_ERRNO__FORMAT;
1191 goto errout;
1192 }
1193
1194 return 0;
1195 errout:
1196 bpf_object__elf_finish(obj);
1197 return err;
1198 }
1199
1200 static int bpf_object__check_endianness(struct bpf_object *obj)
1201 {
1202 #if __BYTE_ORDER == __LITTLE_ENDIAN
1203 if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2LSB)
1204 return 0;
1205 #elif __BYTE_ORDER == __BIG_ENDIAN
1206 if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2MSB)
1207 return 0;
1208 #else
1209 # error "Unrecognized __BYTE_ORDER__"
1210 #endif
1211 pr_warn("elf: endianness mismatch in %s.\n", obj->path);
1212 return -LIBBPF_ERRNO__ENDIAN;
1213 }
1214
1215 static int
1216 bpf_object__init_license(struct bpf_object *obj, void *data, size_t size)
1217 {
1218 memcpy(obj->license, data, min(size, sizeof(obj->license) - 1));
1219 pr_debug("license of %s is %s\n", obj->path, obj->license);
1220 return 0;
1221 }
1222
1223 static int
1224 bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size)
1225 {
1226 __u32 kver;
1227
1228 if (size != sizeof(kver)) {
1229 pr_warn("invalid kver section in %s\n", obj->path);
1230 return -LIBBPF_ERRNO__FORMAT;
1231 }
1232 memcpy(&kver, data, sizeof(kver));
1233 obj->kern_version = kver;
1234 pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version);
1235 return 0;
1236 }
1237
1238 static bool bpf_map_type__is_map_in_map(enum bpf_map_type type)
1239 {
1240 if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS ||
1241 type == BPF_MAP_TYPE_HASH_OF_MAPS)
1242 return true;
1243 return false;
1244 }
1245
1246 int bpf_object__section_size(const struct bpf_object *obj, const char *name,
1247 __u32 *size)
1248 {
1249 int ret = -ENOENT;
1250
1251 *size = 0;
1252 if (!name) {
1253 return -EINVAL;
1254 } else if (!strcmp(name, DATA_SEC)) {
1255 if (obj->efile.data)
1256 *size = obj->efile.data->d_size;
1257 } else if (!strcmp(name, BSS_SEC)) {
1258 if (obj->efile.bss)
1259 *size = obj->efile.bss->d_size;
1260 } else if (!strcmp(name, RODATA_SEC)) {
1261 if (obj->efile.rodata)
1262 *size = obj->efile.rodata->d_size;
1263 } else if (!strcmp(name, STRUCT_OPS_SEC)) {
1264 if (obj->efile.st_ops_data)
1265 *size = obj->efile.st_ops_data->d_size;
1266 } else {
1267 Elf_Scn *scn = elf_sec_by_name(obj, name);
1268 Elf_Data *data = elf_sec_data(obj, scn);
1269
1270 if (data) {
1271 ret = 0; /* found it */
1272 *size = data->d_size;
1273 }
1274 }
1275
1276 return *size ? 0 : ret;
1277 }
1278
1279 int bpf_object__variable_offset(const struct bpf_object *obj, const char *name,
1280 __u32 *off)
1281 {
1282 Elf_Data *symbols = obj->efile.symbols;
1283 const char *sname;
1284 size_t si;
1285
1286 if (!name || !off)
1287 return -EINVAL;
1288
1289 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym); si++) {
1290 GElf_Sym sym;
1291
1292 if (!gelf_getsym(symbols, si, &sym))
1293 continue;
1294 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL ||
1295 GELF_ST_TYPE(sym.st_info) != STT_OBJECT)
1296 continue;
1297
1298 sname = elf_sym_str(obj, sym.st_name);
1299 if (!sname) {
1300 pr_warn("failed to get sym name string for var %s\n",
1301 name);
1302 return -EIO;
1303 }
1304 if (strcmp(name, sname) == 0) {
1305 *off = sym.st_value;
1306 return 0;
1307 }
1308 }
1309
1310 return -ENOENT;
1311 }
1312
1313 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj)
1314 {
1315 struct bpf_map *new_maps;
1316 size_t new_cap;
1317 int i;
1318
1319 if (obj->nr_maps < obj->maps_cap)
1320 return &obj->maps[obj->nr_maps++];
1321
1322 new_cap = max((size_t)4, obj->maps_cap * 3 / 2);
1323 new_maps = libbpf_reallocarray(obj->maps, new_cap, sizeof(*obj->maps));
1324 if (!new_maps) {
1325 pr_warn("alloc maps for object failed\n");
1326 return ERR_PTR(-ENOMEM);
1327 }
1328
1329 obj->maps_cap = new_cap;
1330 obj->maps = new_maps;
1331
1332 /* zero out new maps */
1333 memset(obj->maps + obj->nr_maps, 0,
1334 (obj->maps_cap - obj->nr_maps) * sizeof(*obj->maps));
1335 /*
1336 * fill all fd with -1 so won't close incorrect fd (fd=0 is stdin)
1337 * when failure (zclose won't close negative fd)).
1338 */
1339 for (i = obj->nr_maps; i < obj->maps_cap; i++) {
1340 obj->maps[i].fd = -1;
1341 obj->maps[i].inner_map_fd = -1;
1342 }
1343
1344 return &obj->maps[obj->nr_maps++];
1345 }
1346
1347 static size_t bpf_map_mmap_sz(const struct bpf_map *map)
1348 {
1349 long page_sz = sysconf(_SC_PAGE_SIZE);
1350 size_t map_sz;
1351
1352 map_sz = (size_t)roundup(map->def.value_size, 8) * map->def.max_entries;
1353 map_sz = roundup(map_sz, page_sz);
1354 return map_sz;
1355 }
1356
1357 static char *internal_map_name(struct bpf_object *obj,
1358 enum libbpf_map_type type)
1359 {
1360 char map_name[BPF_OBJ_NAME_LEN], *p;
1361 const char *sfx = libbpf_type_to_btf_name[type];
1362 int sfx_len = max((size_t)7, strlen(sfx));
1363 int pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1,
1364 strlen(obj->name));
1365
1366 snprintf(map_name, sizeof(map_name), "%.*s%.*s", pfx_len, obj->name,
1367 sfx_len, libbpf_type_to_btf_name[type]);
1368
1369 /* sanitise map name to characters allowed by kernel */
1370 for (p = map_name; *p && p < map_name + sizeof(map_name); p++)
1371 if (!isalnum(*p) && *p != '_' && *p != '.')
1372 *p = '_';
1373
1374 return strdup(map_name);
1375 }
1376
1377 static int
1378 bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type,
1379 int sec_idx, void *data, size_t data_sz)
1380 {
1381 struct bpf_map_def *def;
1382 struct bpf_map *map;
1383 int err;
1384
1385 map = bpf_object__add_map(obj);
1386 if (IS_ERR(map))
1387 return PTR_ERR(map);
1388
1389 map->libbpf_type = type;
1390 map->sec_idx = sec_idx;
1391 map->sec_offset = 0;
1392 map->name = internal_map_name(obj, type);
1393 if (!map->name) {
1394 pr_warn("failed to alloc map name\n");
1395 return -ENOMEM;
1396 }
1397
1398 def = &map->def;
1399 def->type = BPF_MAP_TYPE_ARRAY;
1400 def->key_size = sizeof(int);
1401 def->value_size = data_sz;
1402 def->max_entries = 1;
1403 def->map_flags = type == LIBBPF_MAP_RODATA || type == LIBBPF_MAP_KCONFIG
1404 ? BPF_F_RDONLY_PROG : 0;
1405 def->map_flags |= BPF_F_MMAPABLE;
1406
1407 pr_debug("map '%s' (global data): at sec_idx %d, offset %zu, flags %x.\n",
1408 map->name, map->sec_idx, map->sec_offset, def->map_flags);
1409
1410 map->mmaped = mmap(NULL, bpf_map_mmap_sz(map), PROT_READ | PROT_WRITE,
1411 MAP_SHARED | MAP_ANONYMOUS, -1, 0);
1412 if (map->mmaped == MAP_FAILED) {
1413 err = -errno;
1414 map->mmaped = NULL;
1415 pr_warn("failed to alloc map '%s' content buffer: %d\n",
1416 map->name, err);
1417 zfree(&map->name);
1418 return err;
1419 }
1420
1421 if (data)
1422 memcpy(map->mmaped, data, data_sz);
1423
1424 pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name);
1425 return 0;
1426 }
1427
1428 static int bpf_object__init_global_data_maps(struct bpf_object *obj)
1429 {
1430 int err;
1431
1432 /*
1433 * Populate obj->maps with libbpf internal maps.
1434 */
1435 if (obj->efile.data_shndx >= 0) {
1436 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA,
1437 obj->efile.data_shndx,
1438 obj->efile.data->d_buf,
1439 obj->efile.data->d_size);
1440 if (err)
1441 return err;
1442 }
1443 if (obj->efile.rodata_shndx >= 0) {
1444 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA,
1445 obj->efile.rodata_shndx,
1446 obj->efile.rodata->d_buf,
1447 obj->efile.rodata->d_size);
1448 if (err)
1449 return err;
1450
1451 obj->rodata_map_idx = obj->nr_maps - 1;
1452 }
1453 if (obj->efile.bss_shndx >= 0) {
1454 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS,
1455 obj->efile.bss_shndx,
1456 NULL,
1457 obj->efile.bss->d_size);
1458 if (err)
1459 return err;
1460 }
1461 return 0;
1462 }
1463
1464
1465 static struct extern_desc *find_extern_by_name(const struct bpf_object *obj,
1466 const void *name)
1467 {
1468 int i;
1469
1470 for (i = 0; i < obj->nr_extern; i++) {
1471 if (strcmp(obj->externs[i].name, name) == 0)
1472 return &obj->externs[i];
1473 }
1474 return NULL;
1475 }
1476
1477 static int set_kcfg_value_tri(struct extern_desc *ext, void *ext_val,
1478 char value)
1479 {
1480 switch (ext->kcfg.type) {
1481 case KCFG_BOOL:
1482 if (value == 'm') {
1483 pr_warn("extern (kcfg) %s=%c should be tristate or char\n",
1484 ext->name, value);
1485 return -EINVAL;
1486 }
1487 *(bool *)ext_val = value == 'y' ? true : false;
1488 break;
1489 case KCFG_TRISTATE:
1490 if (value == 'y')
1491 *(enum libbpf_tristate *)ext_val = TRI_YES;
1492 else if (value == 'm')
1493 *(enum libbpf_tristate *)ext_val = TRI_MODULE;
1494 else /* value == 'n' */
1495 *(enum libbpf_tristate *)ext_val = TRI_NO;
1496 break;
1497 case KCFG_CHAR:
1498 *(char *)ext_val = value;
1499 break;
1500 case KCFG_UNKNOWN:
1501 case KCFG_INT:
1502 case KCFG_CHAR_ARR:
1503 default:
1504 pr_warn("extern (kcfg) %s=%c should be bool, tristate, or char\n",
1505 ext->name, value);
1506 return -EINVAL;
1507 }
1508 ext->is_set = true;
1509 return 0;
1510 }
1511
1512 static int set_kcfg_value_str(struct extern_desc *ext, char *ext_val,
1513 const char *value)
1514 {
1515 size_t len;
1516
1517 if (ext->kcfg.type != KCFG_CHAR_ARR) {
1518 pr_warn("extern (kcfg) %s=%s should be char array\n", ext->name, value);
1519 return -EINVAL;
1520 }
1521
1522 len = strlen(value);
1523 if (value[len - 1] != '"') {
1524 pr_warn("extern (kcfg) '%s': invalid string config '%s'\n",
1525 ext->name, value);
1526 return -EINVAL;
1527 }
1528
1529 /* strip quotes */
1530 len -= 2;
1531 if (len >= ext->kcfg.sz) {
1532 pr_warn("extern (kcfg) '%s': long string config %s of (%zu bytes) truncated to %d bytes\n",
1533 ext->name, value, len, ext->kcfg.sz - 1);
1534 len = ext->kcfg.sz - 1;
1535 }
1536 memcpy(ext_val, value + 1, len);
1537 ext_val[len] = '\0';
1538 ext->is_set = true;
1539 return 0;
1540 }
1541
1542 static int parse_u64(const char *value, __u64 *res)
1543 {
1544 char *value_end;
1545 int err;
1546
1547 errno = 0;
1548 *res = strtoull(value, &value_end, 0);
1549 if (errno) {
1550 err = -errno;
1551 pr_warn("failed to parse '%s' as integer: %d\n", value, err);
1552 return err;
1553 }
1554 if (*value_end) {
1555 pr_warn("failed to parse '%s' as integer completely\n", value);
1556 return -EINVAL;
1557 }
1558 return 0;
1559 }
1560
1561 static bool is_kcfg_value_in_range(const struct extern_desc *ext, __u64 v)
1562 {
1563 int bit_sz = ext->kcfg.sz * 8;
1564
1565 if (ext->kcfg.sz == 8)
1566 return true;
1567
1568 /* Validate that value stored in u64 fits in integer of `ext->sz`
1569 * bytes size without any loss of information. If the target integer
1570 * is signed, we rely on the following limits of integer type of
1571 * Y bits and subsequent transformation:
1572 *
1573 * -2^(Y-1) <= X <= 2^(Y-1) - 1
1574 * 0 <= X + 2^(Y-1) <= 2^Y - 1
1575 * 0 <= X + 2^(Y-1) < 2^Y
1576 *
1577 * For unsigned target integer, check that all the (64 - Y) bits are
1578 * zero.
1579 */
1580 if (ext->kcfg.is_signed)
1581 return v + (1ULL << (bit_sz - 1)) < (1ULL << bit_sz);
1582 else
1583 return (v >> bit_sz) == 0;
1584 }
1585
1586 static int set_kcfg_value_num(struct extern_desc *ext, void *ext_val,
1587 __u64 value)
1588 {
1589 if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR) {
1590 pr_warn("extern (kcfg) %s=%llu should be integer\n",
1591 ext->name, (unsigned long long)value);
1592 return -EINVAL;
1593 }
1594 if (!is_kcfg_value_in_range(ext, value)) {
1595 pr_warn("extern (kcfg) %s=%llu value doesn't fit in %d bytes\n",
1596 ext->name, (unsigned long long)value, ext->kcfg.sz);
1597 return -ERANGE;
1598 }
1599 switch (ext->kcfg.sz) {
1600 case 1: *(__u8 *)ext_val = value; break;
1601 case 2: *(__u16 *)ext_val = value; break;
1602 case 4: *(__u32 *)ext_val = value; break;
1603 case 8: *(__u64 *)ext_val = value; break;
1604 default:
1605 return -EINVAL;
1606 }
1607 ext->is_set = true;
1608 return 0;
1609 }
1610
1611 static int bpf_object__process_kconfig_line(struct bpf_object *obj,
1612 char *buf, void *data)
1613 {
1614 struct extern_desc *ext;
1615 char *sep, *value;
1616 int len, err = 0;
1617 void *ext_val;
1618 __u64 num;
1619
1620 if (strncmp(buf, "CONFIG_", 7))
1621 return 0;
1622
1623 sep = strchr(buf, '=');
1624 if (!sep) {
1625 pr_warn("failed to parse '%s': no separator\n", buf);
1626 return -EINVAL;
1627 }
1628
1629 /* Trim ending '\n' */
1630 len = strlen(buf);
1631 if (buf[len - 1] == '\n')
1632 buf[len - 1] = '\0';
1633 /* Split on '=' and ensure that a value is present. */
1634 *sep = '\0';
1635 if (!sep[1]) {
1636 *sep = '=';
1637 pr_warn("failed to parse '%s': no value\n", buf);
1638 return -EINVAL;
1639 }
1640
1641 ext = find_extern_by_name(obj, buf);
1642 if (!ext || ext->is_set)
1643 return 0;
1644
1645 ext_val = data + ext->kcfg.data_off;
1646 value = sep + 1;
1647
1648 switch (*value) {
1649 case 'y': case 'n': case 'm':
1650 err = set_kcfg_value_tri(ext, ext_val, *value);
1651 break;
1652 case '"':
1653 err = set_kcfg_value_str(ext, ext_val, value);
1654 break;
1655 default:
1656 /* assume integer */
1657 err = parse_u64(value, &num);
1658 if (err) {
1659 pr_warn("extern (kcfg) %s=%s should be integer\n",
1660 ext->name, value);
1661 return err;
1662 }
1663 err = set_kcfg_value_num(ext, ext_val, num);
1664 break;
1665 }
1666 if (err)
1667 return err;
1668 pr_debug("extern (kcfg) %s=%s\n", ext->name, value);
1669 return 0;
1670 }
1671
1672 static int bpf_object__read_kconfig_file(struct bpf_object *obj, void *data)
1673 {
1674 char buf[PATH_MAX];
1675 struct utsname uts;
1676 int len, err = 0;
1677 gzFile file;
1678
1679 uname(&uts);
1680 len = snprintf(buf, PATH_MAX, "/boot/config-%s", uts.release);
1681 if (len < 0)
1682 return -EINVAL;
1683 else if (len >= PATH_MAX)
1684 return -ENAMETOOLONG;
1685
1686 /* gzopen also accepts uncompressed files. */
1687 file = gzopen(buf, "r");
1688 if (!file)
1689 file = gzopen("/proc/config.gz", "r");
1690
1691 if (!file) {
1692 pr_warn("failed to open system Kconfig\n");
1693 return -ENOENT;
1694 }
1695
1696 while (gzgets(file, buf, sizeof(buf))) {
1697 err = bpf_object__process_kconfig_line(obj, buf, data);
1698 if (err) {
1699 pr_warn("error parsing system Kconfig line '%s': %d\n",
1700 buf, err);
1701 goto out;
1702 }
1703 }
1704
1705 out:
1706 gzclose(file);
1707 return err;
1708 }
1709
1710 static int bpf_object__read_kconfig_mem(struct bpf_object *obj,
1711 const char *config, void *data)
1712 {
1713 char buf[PATH_MAX];
1714 int err = 0;
1715 FILE *file;
1716
1717 file = fmemopen((void *)config, strlen(config), "r");
1718 if (!file) {
1719 err = -errno;
1720 pr_warn("failed to open in-memory Kconfig: %d\n", err);
1721 return err;
1722 }
1723
1724 while (fgets(buf, sizeof(buf), file)) {
1725 err = bpf_object__process_kconfig_line(obj, buf, data);
1726 if (err) {
1727 pr_warn("error parsing in-memory Kconfig line '%s': %d\n",
1728 buf, err);
1729 break;
1730 }
1731 }
1732
1733 fclose(file);
1734 return err;
1735 }
1736
1737 static int bpf_object__init_kconfig_map(struct bpf_object *obj)
1738 {
1739 struct extern_desc *last_ext = NULL, *ext;
1740 size_t map_sz;
1741 int i, err;
1742
1743 for (i = 0; i < obj->nr_extern; i++) {
1744 ext = &obj->externs[i];
1745 if (ext->type == EXT_KCFG)
1746 last_ext = ext;
1747 }
1748
1749 if (!last_ext)
1750 return 0;
1751
1752 map_sz = last_ext->kcfg.data_off + last_ext->kcfg.sz;
1753 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_KCONFIG,
1754 obj->efile.symbols_shndx,
1755 NULL, map_sz);
1756 if (err)
1757 return err;
1758
1759 obj->kconfig_map_idx = obj->nr_maps - 1;
1760
1761 return 0;
1762 }
1763
1764 static int bpf_object__init_user_maps(struct bpf_object *obj, bool strict)
1765 {
1766 Elf_Data *symbols = obj->efile.symbols;
1767 int i, map_def_sz = 0, nr_maps = 0, nr_syms;
1768 Elf_Data *data = NULL;
1769 Elf_Scn *scn;
1770
1771 if (obj->efile.maps_shndx < 0)
1772 return 0;
1773
1774 if (!symbols)
1775 return -EINVAL;
1776
1777
1778 scn = elf_sec_by_idx(obj, obj->efile.maps_shndx);
1779 data = elf_sec_data(obj, scn);
1780 if (!scn || !data) {
1781 pr_warn("elf: failed to get legacy map definitions for %s\n",
1782 obj->path);
1783 return -EINVAL;
1784 }
1785
1786 /*
1787 * Count number of maps. Each map has a name.
1788 * Array of maps is not supported: only the first element is
1789 * considered.
1790 *
1791 * TODO: Detect array of map and report error.
1792 */
1793 nr_syms = symbols->d_size / sizeof(GElf_Sym);
1794 for (i = 0; i < nr_syms; i++) {
1795 GElf_Sym sym;
1796
1797 if (!gelf_getsym(symbols, i, &sym))
1798 continue;
1799 if (sym.st_shndx != obj->efile.maps_shndx)
1800 continue;
1801 nr_maps++;
1802 }
1803 /* Assume equally sized map definitions */
1804 pr_debug("elf: found %d legacy map definitions (%zd bytes) in %s\n",
1805 nr_maps, data->d_size, obj->path);
1806
1807 if (!data->d_size || nr_maps == 0 || (data->d_size % nr_maps) != 0) {
1808 pr_warn("elf: unable to determine legacy map definition size in %s\n",
1809 obj->path);
1810 return -EINVAL;
1811 }
1812 map_def_sz = data->d_size / nr_maps;
1813
1814 /* Fill obj->maps using data in "maps" section. */
1815 for (i = 0; i < nr_syms; i++) {
1816 GElf_Sym sym;
1817 const char *map_name;
1818 struct bpf_map_def *def;
1819 struct bpf_map *map;
1820
1821 if (!gelf_getsym(symbols, i, &sym))
1822 continue;
1823 if (sym.st_shndx != obj->efile.maps_shndx)
1824 continue;
1825
1826 map = bpf_object__add_map(obj);
1827 if (IS_ERR(map))
1828 return PTR_ERR(map);
1829
1830 map_name = elf_sym_str(obj, sym.st_name);
1831 if (!map_name) {
1832 pr_warn("failed to get map #%d name sym string for obj %s\n",
1833 i, obj->path);
1834 return -LIBBPF_ERRNO__FORMAT;
1835 }
1836
1837 map->libbpf_type = LIBBPF_MAP_UNSPEC;
1838 map->sec_idx = sym.st_shndx;
1839 map->sec_offset = sym.st_value;
1840 pr_debug("map '%s' (legacy): at sec_idx %d, offset %zu.\n",
1841 map_name, map->sec_idx, map->sec_offset);
1842 if (sym.st_value + map_def_sz > data->d_size) {
1843 pr_warn("corrupted maps section in %s: last map \"%s\" too small\n",
1844 obj->path, map_name);
1845 return -EINVAL;
1846 }
1847
1848 map->name = strdup(map_name);
1849 if (!map->name) {
1850 pr_warn("failed to alloc map name\n");
1851 return -ENOMEM;
1852 }
1853 pr_debug("map %d is \"%s\"\n", i, map->name);
1854 def = (struct bpf_map_def *)(data->d_buf + sym.st_value);
1855 /*
1856 * If the definition of the map in the object file fits in
1857 * bpf_map_def, copy it. Any extra fields in our version
1858 * of bpf_map_def will default to zero as a result of the
1859 * calloc above.
1860 */
1861 if (map_def_sz <= sizeof(struct bpf_map_def)) {
1862 memcpy(&map->def, def, map_def_sz);
1863 } else {
1864 /*
1865 * Here the map structure being read is bigger than what
1866 * we expect, truncate if the excess bits are all zero.
1867 * If they are not zero, reject this map as
1868 * incompatible.
1869 */
1870 char *b;
1871
1872 for (b = ((char *)def) + sizeof(struct bpf_map_def);
1873 b < ((char *)def) + map_def_sz; b++) {
1874 if (*b != 0) {
1875 pr_warn("maps section in %s: \"%s\" has unrecognized, non-zero options\n",
1876 obj->path, map_name);
1877 if (strict)
1878 return -EINVAL;
1879 }
1880 }
1881 memcpy(&map->def, def, sizeof(struct bpf_map_def));
1882 }
1883 }
1884 return 0;
1885 }
1886
1887 static const struct btf_type *
1888 skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id)
1889 {
1890 const struct btf_type *t = btf__type_by_id(btf, id);
1891
1892 if (res_id)
1893 *res_id = id;
1894
1895 while (btf_is_mod(t) || btf_is_typedef(t)) {
1896 if (res_id)
1897 *res_id = t->type;
1898 t = btf__type_by_id(btf, t->type);
1899 }
1900
1901 return t;
1902 }
1903
1904 static const struct btf_type *
1905 resolve_func_ptr(const struct btf *btf, __u32 id, __u32 *res_id)
1906 {
1907 const struct btf_type *t;
1908
1909 t = skip_mods_and_typedefs(btf, id, NULL);
1910 if (!btf_is_ptr(t))
1911 return NULL;
1912
1913 t = skip_mods_and_typedefs(btf, t->type, res_id);
1914
1915 return btf_is_func_proto(t) ? t : NULL;
1916 }
1917
1918 static const char *btf_kind_str(const struct btf_type *t)
1919 {
1920 switch (btf_kind(t)) {
1921 case BTF_KIND_UNKN: return "void";
1922 case BTF_KIND_INT: return "int";
1923 case BTF_KIND_PTR: return "ptr";
1924 case BTF_KIND_ARRAY: return "array";
1925 case BTF_KIND_STRUCT: return "struct";
1926 case BTF_KIND_UNION: return "union";
1927 case BTF_KIND_ENUM: return "enum";
1928 case BTF_KIND_FWD: return "fwd";
1929 case BTF_KIND_TYPEDEF: return "typedef";
1930 case BTF_KIND_VOLATILE: return "volatile";
1931 case BTF_KIND_CONST: return "const";
1932 case BTF_KIND_RESTRICT: return "restrict";
1933 case BTF_KIND_FUNC: return "func";
1934 case BTF_KIND_FUNC_PROTO: return "func_proto";
1935 case BTF_KIND_VAR: return "var";
1936 case BTF_KIND_DATASEC: return "datasec";
1937 default: return "unknown";
1938 }
1939 }
1940
1941 /*
1942 * Fetch integer attribute of BTF map definition. Such attributes are
1943 * represented using a pointer to an array, in which dimensionality of array
1944 * encodes specified integer value. E.g., int (*type)[BPF_MAP_TYPE_ARRAY];
1945 * encodes `type => BPF_MAP_TYPE_ARRAY` key/value pair completely using BTF
1946 * type definition, while using only sizeof(void *) space in ELF data section.
1947 */
1948 static bool get_map_field_int(const char *map_name, const struct btf *btf,
1949 const struct btf_member *m, __u32 *res)
1950 {
1951 const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL);
1952 const char *name = btf__name_by_offset(btf, m->name_off);
1953 const struct btf_array *arr_info;
1954 const struct btf_type *arr_t;
1955
1956 if (!btf_is_ptr(t)) {
1957 pr_warn("map '%s': attr '%s': expected PTR, got %s.\n",
1958 map_name, name, btf_kind_str(t));
1959 return false;
1960 }
1961
1962 arr_t = btf__type_by_id(btf, t->type);
1963 if (!arr_t) {
1964 pr_warn("map '%s': attr '%s': type [%u] not found.\n",
1965 map_name, name, t->type);
1966 return false;
1967 }
1968 if (!btf_is_array(arr_t)) {
1969 pr_warn("map '%s': attr '%s': expected ARRAY, got %s.\n",
1970 map_name, name, btf_kind_str(arr_t));
1971 return false;
1972 }
1973 arr_info = btf_array(arr_t);
1974 *res = arr_info->nelems;
1975 return true;
1976 }
1977
1978 static int build_map_pin_path(struct bpf_map *map, const char *path)
1979 {
1980 char buf[PATH_MAX];
1981 int len;
1982
1983 if (!path)
1984 path = "/sys/fs/bpf";
1985
1986 len = snprintf(buf, PATH_MAX, "%s/%s", path, bpf_map__name(map));
1987 if (len < 0)
1988 return -EINVAL;
1989 else if (len >= PATH_MAX)
1990 return -ENAMETOOLONG;
1991
1992 return bpf_map__set_pin_path(map, buf);
1993 }
1994
1995
1996 static int parse_btf_map_def(struct bpf_object *obj,
1997 struct bpf_map *map,
1998 const struct btf_type *def,
1999 bool strict, bool is_inner,
2000 const char *pin_root_path)
2001 {
2002 const struct btf_type *t;
2003 const struct btf_member *m;
2004 int vlen, i;
2005
2006 vlen = btf_vlen(def);
2007 m = btf_members(def);
2008 for (i = 0; i < vlen; i++, m++) {
2009 const char *name = btf__name_by_offset(obj->btf, m->name_off);
2010
2011 if (!name) {
2012 pr_warn("map '%s': invalid field #%d.\n", map->name, i);
2013 return -EINVAL;
2014 }
2015 if (strcmp(name, "type") == 0) {
2016 if (!get_map_field_int(map->name, obj->btf, m,
2017 &map->def.type))
2018 return -EINVAL;
2019 pr_debug("map '%s': found type = %u.\n",
2020 map->name, map->def.type);
2021 } else if (strcmp(name, "max_entries") == 0) {
2022 if (!get_map_field_int(map->name, obj->btf, m,
2023 &map->def.max_entries))
2024 return -EINVAL;
2025 pr_debug("map '%s': found max_entries = %u.\n",
2026 map->name, map->def.max_entries);
2027 } else if (strcmp(name, "map_flags") == 0) {
2028 if (!get_map_field_int(map->name, obj->btf, m,
2029 &map->def.map_flags))
2030 return -EINVAL;
2031 pr_debug("map '%s': found map_flags = %u.\n",
2032 map->name, map->def.map_flags);
2033 } else if (strcmp(name, "numa_node") == 0) {
2034 if (!get_map_field_int(map->name, obj->btf, m, &map->numa_node))
2035 return -EINVAL;
2036 pr_debug("map '%s': found numa_node = %u.\n", map->name, map->numa_node);
2037 } else if (strcmp(name, "key_size") == 0) {
2038 __u32 sz;
2039
2040 if (!get_map_field_int(map->name, obj->btf, m, &sz))
2041 return -EINVAL;
2042 pr_debug("map '%s': found key_size = %u.\n",
2043 map->name, sz);
2044 if (map->def.key_size && map->def.key_size != sz) {
2045 pr_warn("map '%s': conflicting key size %u != %u.\n",
2046 map->name, map->def.key_size, sz);
2047 return -EINVAL;
2048 }
2049 map->def.key_size = sz;
2050 } else if (strcmp(name, "key") == 0) {
2051 __s64 sz;
2052
2053 t = btf__type_by_id(obj->btf, m->type);
2054 if (!t) {
2055 pr_warn("map '%s': key type [%d] not found.\n",
2056 map->name, m->type);
2057 return -EINVAL;
2058 }
2059 if (!btf_is_ptr(t)) {
2060 pr_warn("map '%s': key spec is not PTR: %s.\n",
2061 map->name, btf_kind_str(t));
2062 return -EINVAL;
2063 }
2064 sz = btf__resolve_size(obj->btf, t->type);
2065 if (sz < 0) {
2066 pr_warn("map '%s': can't determine key size for type [%u]: %zd.\n",
2067 map->name, t->type, (ssize_t)sz);
2068 return sz;
2069 }
2070 pr_debug("map '%s': found key [%u], sz = %zd.\n",
2071 map->name, t->type, (ssize_t)sz);
2072 if (map->def.key_size && map->def.key_size != sz) {
2073 pr_warn("map '%s': conflicting key size %u != %zd.\n",
2074 map->name, map->def.key_size, (ssize_t)sz);
2075 return -EINVAL;
2076 }
2077 map->def.key_size = sz;
2078 map->btf_key_type_id = t->type;
2079 } else if (strcmp(name, "value_size") == 0) {
2080 __u32 sz;
2081
2082 if (!get_map_field_int(map->name, obj->btf, m, &sz))
2083 return -EINVAL;
2084 pr_debug("map '%s': found value_size = %u.\n",
2085 map->name, sz);
2086 if (map->def.value_size && map->def.value_size != sz) {
2087 pr_warn("map '%s': conflicting value size %u != %u.\n",
2088 map->name, map->def.value_size, sz);
2089 return -EINVAL;
2090 }
2091 map->def.value_size = sz;
2092 } else if (strcmp(name, "value") == 0) {
2093 __s64 sz;
2094
2095 t = btf__type_by_id(obj->btf, m->type);
2096 if (!t) {
2097 pr_warn("map '%s': value type [%d] not found.\n",
2098 map->name, m->type);
2099 return -EINVAL;
2100 }
2101 if (!btf_is_ptr(t)) {
2102 pr_warn("map '%s': value spec is not PTR: %s.\n",
2103 map->name, btf_kind_str(t));
2104 return -EINVAL;
2105 }
2106 sz = btf__resolve_size(obj->btf, t->type);
2107 if (sz < 0) {
2108 pr_warn("map '%s': can't determine value size for type [%u]: %zd.\n",
2109 map->name, t->type, (ssize_t)sz);
2110 return sz;
2111 }
2112 pr_debug("map '%s': found value [%u], sz = %zd.\n",
2113 map->name, t->type, (ssize_t)sz);
2114 if (map->def.value_size && map->def.value_size != sz) {
2115 pr_warn("map '%s': conflicting value size %u != %zd.\n",
2116 map->name, map->def.value_size, (ssize_t)sz);
2117 return -EINVAL;
2118 }
2119 map->def.value_size = sz;
2120 map->btf_value_type_id = t->type;
2121 }
2122 else if (strcmp(name, "values") == 0) {
2123 int err;
2124
2125 if (is_inner) {
2126 pr_warn("map '%s': multi-level inner maps not supported.\n",
2127 map->name);
2128 return -ENOTSUP;
2129 }
2130 if (i != vlen - 1) {
2131 pr_warn("map '%s': '%s' member should be last.\n",
2132 map->name, name);
2133 return -EINVAL;
2134 }
2135 if (!bpf_map_type__is_map_in_map(map->def.type)) {
2136 pr_warn("map '%s': should be map-in-map.\n",
2137 map->name);
2138 return -ENOTSUP;
2139 }
2140 if (map->def.value_size && map->def.value_size != 4) {
2141 pr_warn("map '%s': conflicting value size %u != 4.\n",
2142 map->name, map->def.value_size);
2143 return -EINVAL;
2144 }
2145 map->def.value_size = 4;
2146 t = btf__type_by_id(obj->btf, m->type);
2147 if (!t) {
2148 pr_warn("map '%s': map-in-map inner type [%d] not found.\n",
2149 map->name, m->type);
2150 return -EINVAL;
2151 }
2152 if (!btf_is_array(t) || btf_array(t)->nelems) {
2153 pr_warn("map '%s': map-in-map inner spec is not a zero-sized array.\n",
2154 map->name);
2155 return -EINVAL;
2156 }
2157 t = skip_mods_and_typedefs(obj->btf, btf_array(t)->type,
2158 NULL);
2159 if (!btf_is_ptr(t)) {
2160 pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n",
2161 map->name, btf_kind_str(t));
2162 return -EINVAL;
2163 }
2164 t = skip_mods_and_typedefs(obj->btf, t->type, NULL);
2165 if (!btf_is_struct(t)) {
2166 pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n",
2167 map->name, btf_kind_str(t));
2168 return -EINVAL;
2169 }
2170
2171 map->inner_map = calloc(1, sizeof(*map->inner_map));
2172 if (!map->inner_map)
2173 return -ENOMEM;
2174 map->inner_map->sec_idx = obj->efile.btf_maps_shndx;
2175 map->inner_map->name = malloc(strlen(map->name) +
2176 sizeof(".inner") + 1);
2177 if (!map->inner_map->name)
2178 return -ENOMEM;
2179 sprintf(map->inner_map->name, "%s.inner", map->name);
2180
2181 err = parse_btf_map_def(obj, map->inner_map, t, strict,
2182 true /* is_inner */, NULL);
2183 if (err)
2184 return err;
2185 } else if (strcmp(name, "pinning") == 0) {
2186 __u32 val;
2187 int err;
2188
2189 if (is_inner) {
2190 pr_debug("map '%s': inner def can't be pinned.\n",
2191 map->name);
2192 return -EINVAL;
2193 }
2194 if (!get_map_field_int(map->name, obj->btf, m, &val))
2195 return -EINVAL;
2196 pr_debug("map '%s': found pinning = %u.\n",
2197 map->name, val);
2198
2199 if (val != LIBBPF_PIN_NONE &&
2200 val != LIBBPF_PIN_BY_NAME) {
2201 pr_warn("map '%s': invalid pinning value %u.\n",
2202 map->name, val);
2203 return -EINVAL;
2204 }
2205 if (val == LIBBPF_PIN_BY_NAME) {
2206 err = build_map_pin_path(map, pin_root_path);
2207 if (err) {
2208 pr_warn("map '%s': couldn't build pin path.\n",
2209 map->name);
2210 return err;
2211 }
2212 }
2213 } else {
2214 if (strict) {
2215 pr_warn("map '%s': unknown field '%s'.\n",
2216 map->name, name);
2217 return -ENOTSUP;
2218 }
2219 pr_debug("map '%s': ignoring unknown field '%s'.\n",
2220 map->name, name);
2221 }
2222 }
2223
2224 if (map->def.type == BPF_MAP_TYPE_UNSPEC) {
2225 pr_warn("map '%s': map type isn't specified.\n", map->name);
2226 return -EINVAL;
2227 }
2228
2229 return 0;
2230 }
2231
2232 static int bpf_object__init_user_btf_map(struct bpf_object *obj,
2233 const struct btf_type *sec,
2234 int var_idx, int sec_idx,
2235 const Elf_Data *data, bool strict,
2236 const char *pin_root_path)
2237 {
2238 const struct btf_type *var, *def;
2239 const struct btf_var_secinfo *vi;
2240 const struct btf_var *var_extra;
2241 const char *map_name;
2242 struct bpf_map *map;
2243
2244 vi = btf_var_secinfos(sec) + var_idx;
2245 var = btf__type_by_id(obj->btf, vi->type);
2246 var_extra = btf_var(var);
2247 map_name = btf__name_by_offset(obj->btf, var->name_off);
2248
2249 if (map_name == NULL || map_name[0] == '\0') {
2250 pr_warn("map #%d: empty name.\n", var_idx);
2251 return -EINVAL;
2252 }
2253 if ((__u64)vi->offset + vi->size > data->d_size) {
2254 pr_warn("map '%s' BTF data is corrupted.\n", map_name);
2255 return -EINVAL;
2256 }
2257 if (!btf_is_var(var)) {
2258 pr_warn("map '%s': unexpected var kind %s.\n",
2259 map_name, btf_kind_str(var));
2260 return -EINVAL;
2261 }
2262 if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED &&
2263 var_extra->linkage != BTF_VAR_STATIC) {
2264 pr_warn("map '%s': unsupported var linkage %u.\n",
2265 map_name, var_extra->linkage);
2266 return -EOPNOTSUPP;
2267 }
2268
2269 def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
2270 if (!btf_is_struct(def)) {
2271 pr_warn("map '%s': unexpected def kind %s.\n",
2272 map_name, btf_kind_str(var));
2273 return -EINVAL;
2274 }
2275 if (def->size > vi->size) {
2276 pr_warn("map '%s': invalid def size.\n", map_name);
2277 return -EINVAL;
2278 }
2279
2280 map = bpf_object__add_map(obj);
2281 if (IS_ERR(map))
2282 return PTR_ERR(map);
2283 map->name = strdup(map_name);
2284 if (!map->name) {
2285 pr_warn("map '%s': failed to alloc map name.\n", map_name);
2286 return -ENOMEM;
2287 }
2288 map->libbpf_type = LIBBPF_MAP_UNSPEC;
2289 map->def.type = BPF_MAP_TYPE_UNSPEC;
2290 map->sec_idx = sec_idx;
2291 map->sec_offset = vi->offset;
2292 map->btf_var_idx = var_idx;
2293 pr_debug("map '%s': at sec_idx %d, offset %zu.\n",
2294 map_name, map->sec_idx, map->sec_offset);
2295
2296 return parse_btf_map_def(obj, map, def, strict, false, pin_root_path);
2297 }
2298
2299 static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict,
2300 const char *pin_root_path)
2301 {
2302 const struct btf_type *sec = NULL;
2303 int nr_types, i, vlen, err;
2304 const struct btf_type *t;
2305 const char *name;
2306 Elf_Data *data;
2307 Elf_Scn *scn;
2308
2309 if (obj->efile.btf_maps_shndx < 0)
2310 return 0;
2311
2312 scn = elf_sec_by_idx(obj, obj->efile.btf_maps_shndx);
2313 data = elf_sec_data(obj, scn);
2314 if (!scn || !data) {
2315 pr_warn("elf: failed to get %s map definitions for %s\n",
2316 MAPS_ELF_SEC, obj->path);
2317 return -EINVAL;
2318 }
2319
2320 nr_types = btf__get_nr_types(obj->btf);
2321 for (i = 1; i <= nr_types; i++) {
2322 t = btf__type_by_id(obj->btf, i);
2323 if (!btf_is_datasec(t))
2324 continue;
2325 name = btf__name_by_offset(obj->btf, t->name_off);
2326 if (strcmp(name, MAPS_ELF_SEC) == 0) {
2327 sec = t;
2328 obj->efile.btf_maps_sec_btf_id = i;
2329 break;
2330 }
2331 }
2332
2333 if (!sec) {
2334 pr_warn("DATASEC '%s' not found.\n", MAPS_ELF_SEC);
2335 return -ENOENT;
2336 }
2337
2338 vlen = btf_vlen(sec);
2339 for (i = 0; i < vlen; i++) {
2340 err = bpf_object__init_user_btf_map(obj, sec, i,
2341 obj->efile.btf_maps_shndx,
2342 data, strict,
2343 pin_root_path);
2344 if (err)
2345 return err;
2346 }
2347
2348 return 0;
2349 }
2350
2351 static int bpf_object__init_maps(struct bpf_object *obj,
2352 const struct bpf_object_open_opts *opts)
2353 {
2354 const char *pin_root_path;
2355 bool strict;
2356 int err;
2357
2358 strict = !OPTS_GET(opts, relaxed_maps, false);
2359 pin_root_path = OPTS_GET(opts, pin_root_path, NULL);
2360
2361 err = bpf_object__init_user_maps(obj, strict);
2362 err = err ?: bpf_object__init_user_btf_maps(obj, strict, pin_root_path);
2363 err = err ?: bpf_object__init_global_data_maps(obj);
2364 err = err ?: bpf_object__init_kconfig_map(obj);
2365 err = err ?: bpf_object__init_struct_ops_maps(obj);
2366 if (err)
2367 return err;
2368
2369 return 0;
2370 }
2371
2372 static bool section_have_execinstr(struct bpf_object *obj, int idx)
2373 {
2374 GElf_Shdr sh;
2375
2376 if (elf_sec_hdr(obj, elf_sec_by_idx(obj, idx), &sh))
2377 return false;
2378
2379 return sh.sh_flags & SHF_EXECINSTR;
2380 }
2381
2382 static bool btf_needs_sanitization(struct bpf_object *obj)
2383 {
2384 bool has_func_global = kernel_supports(FEAT_BTF_GLOBAL_FUNC);
2385 bool has_datasec = kernel_supports(FEAT_BTF_DATASEC);
2386 bool has_func = kernel_supports(FEAT_BTF_FUNC);
2387
2388 return !has_func || !has_datasec || !has_func_global;
2389 }
2390
2391 static void bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf)
2392 {
2393 bool has_func_global = kernel_supports(FEAT_BTF_GLOBAL_FUNC);
2394 bool has_datasec = kernel_supports(FEAT_BTF_DATASEC);
2395 bool has_func = kernel_supports(FEAT_BTF_FUNC);
2396 struct btf_type *t;
2397 int i, j, vlen;
2398
2399 for (i = 1; i <= btf__get_nr_types(btf); i++) {
2400 t = (struct btf_type *)btf__type_by_id(btf, i);
2401
2402 if (!has_datasec && btf_is_var(t)) {
2403 /* replace VAR with INT */
2404 t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0);
2405 /*
2406 * using size = 1 is the safest choice, 4 will be too
2407 * big and cause kernel BTF validation failure if
2408 * original variable took less than 4 bytes
2409 */
2410 t->size = 1;
2411 *(int *)(t + 1) = BTF_INT_ENC(0, 0, 8);
2412 } else if (!has_datasec && btf_is_datasec(t)) {
2413 /* replace DATASEC with STRUCT */
2414 const struct btf_var_secinfo *v = btf_var_secinfos(t);
2415 struct btf_member *m = btf_members(t);
2416 struct btf_type *vt;
2417 char *name;
2418
2419 name = (char *)btf__name_by_offset(btf, t->name_off);
2420 while (*name) {
2421 if (*name == '.')
2422 *name = '_';
2423 name++;
2424 }
2425
2426 vlen = btf_vlen(t);
2427 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen);
2428 for (j = 0; j < vlen; j++, v++, m++) {
2429 /* order of field assignments is important */
2430 m->offset = v->offset * 8;
2431 m->type = v->type;
2432 /* preserve variable name as member name */
2433 vt = (void *)btf__type_by_id(btf, v->type);
2434 m->name_off = vt->name_off;
2435 }
2436 } else if (!has_func && btf_is_func_proto(t)) {
2437 /* replace FUNC_PROTO with ENUM */
2438 vlen = btf_vlen(t);
2439 t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen);
2440 t->size = sizeof(__u32); /* kernel enforced */
2441 } else if (!has_func && btf_is_func(t)) {
2442 /* replace FUNC with TYPEDEF */
2443 t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0);
2444 } else if (!has_func_global && btf_is_func(t)) {
2445 /* replace BTF_FUNC_GLOBAL with BTF_FUNC_STATIC */
2446 t->info = BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0);
2447 }
2448 }
2449 }
2450
2451 static bool libbpf_needs_btf(const struct bpf_object *obj)
2452 {
2453 return obj->efile.btf_maps_shndx >= 0 ||
2454 obj->efile.st_ops_shndx >= 0 ||
2455 obj->nr_extern > 0;
2456 }
2457
2458 static bool kernel_needs_btf(const struct bpf_object *obj)
2459 {
2460 return obj->efile.st_ops_shndx >= 0;
2461 }
2462
2463 static int bpf_object__init_btf(struct bpf_object *obj,
2464 Elf_Data *btf_data,
2465 Elf_Data *btf_ext_data)
2466 {
2467 int err = -ENOENT;
2468
2469 if (btf_data) {
2470 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size);
2471 if (IS_ERR(obj->btf)) {
2472 err = PTR_ERR(obj->btf);
2473 obj->btf = NULL;
2474 pr_warn("Error loading ELF section %s: %d.\n",
2475 BTF_ELF_SEC, err);
2476 goto out;
2477 }
2478 /* enforce 8-byte pointers for BPF-targeted BTFs */
2479 btf__set_pointer_size(obj->btf, 8);
2480 err = 0;
2481 }
2482 if (btf_ext_data) {
2483 if (!obj->btf) {
2484 pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n",
2485 BTF_EXT_ELF_SEC, BTF_ELF_SEC);
2486 goto out;
2487 }
2488 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf,
2489 btf_ext_data->d_size);
2490 if (IS_ERR(obj->btf_ext)) {
2491 pr_warn("Error loading ELF section %s: %ld. Ignored and continue.\n",
2492 BTF_EXT_ELF_SEC, PTR_ERR(obj->btf_ext));
2493 obj->btf_ext = NULL;
2494 goto out;
2495 }
2496 }
2497 out:
2498 if (err && libbpf_needs_btf(obj)) {
2499 pr_warn("BTF is required, but is missing or corrupted.\n");
2500 return err;
2501 }
2502 return 0;
2503 }
2504
2505 static int bpf_object__finalize_btf(struct bpf_object *obj)
2506 {
2507 int err;
2508
2509 if (!obj->btf)
2510 return 0;
2511
2512 err = btf__finalize_data(obj, obj->btf);
2513 if (err) {
2514 pr_warn("Error finalizing %s: %d.\n", BTF_ELF_SEC, err);
2515 return err;
2516 }
2517
2518 return 0;
2519 }
2520
2521 static bool prog_needs_vmlinux_btf(struct bpf_program *prog)
2522 {
2523 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS ||
2524 prog->type == BPF_PROG_TYPE_LSM)
2525 return true;
2526
2527 /* BPF_PROG_TYPE_TRACING programs which do not attach to other programs
2528 * also need vmlinux BTF
2529 */
2530 if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd)
2531 return true;
2532
2533 return false;
2534 }
2535
2536 static bool obj_needs_vmlinux_btf(const struct bpf_object *obj)
2537 {
2538 struct bpf_program *prog;
2539 int i;
2540
2541 /* CO-RE relocations need kernel BTF */
2542 if (obj->btf_ext && obj->btf_ext->core_relo_info.len)
2543 return true;
2544
2545 /* Support for typed ksyms needs kernel BTF */
2546 for (i = 0; i < obj->nr_extern; i++) {
2547 const struct extern_desc *ext;
2548
2549 ext = &obj->externs[i];
2550 if (ext->type == EXT_KSYM && ext->ksym.type_id)
2551 return true;
2552 }
2553
2554 bpf_object__for_each_program(prog, obj) {
2555 if (!prog->load)
2556 continue;
2557 if (prog_needs_vmlinux_btf(prog))
2558 return true;
2559 }
2560
2561 return false;
2562 }
2563
2564 static int bpf_object__load_vmlinux_btf(struct bpf_object *obj, bool force)
2565 {
2566 int err;
2567
2568 /* btf_vmlinux could be loaded earlier */
2569 if (obj->btf_vmlinux)
2570 return 0;
2571
2572 if (!force && !obj_needs_vmlinux_btf(obj))
2573 return 0;
2574
2575 obj->btf_vmlinux = libbpf_find_kernel_btf();
2576 if (IS_ERR(obj->btf_vmlinux)) {
2577 err = PTR_ERR(obj->btf_vmlinux);
2578 pr_warn("Error loading vmlinux BTF: %d\n", err);
2579 obj->btf_vmlinux = NULL;
2580 return err;
2581 }
2582 return 0;
2583 }
2584
2585 static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj)
2586 {
2587 struct btf *kern_btf = obj->btf;
2588 bool btf_mandatory, sanitize;
2589 int err = 0;
2590
2591 if (!obj->btf)
2592 return 0;
2593
2594 if (!kernel_supports(FEAT_BTF)) {
2595 if (kernel_needs_btf(obj)) {
2596 err = -EOPNOTSUPP;
2597 goto report;
2598 }
2599 pr_debug("Kernel doesn't support BTF, skipping uploading it.\n");
2600 return 0;
2601 }
2602
2603 sanitize = btf_needs_sanitization(obj);
2604 if (sanitize) {
2605 const void *raw_data;
2606 __u32 sz;
2607
2608 /* clone BTF to sanitize a copy and leave the original intact */
2609 raw_data = btf__get_raw_data(obj->btf, &sz);
2610 kern_btf = btf__new(raw_data, sz);
2611 if (IS_ERR(kern_btf))
2612 return PTR_ERR(kern_btf);
2613
2614 /* enforce 8-byte pointers for BPF-targeted BTFs */
2615 btf__set_pointer_size(obj->btf, 8);
2616 bpf_object__sanitize_btf(obj, kern_btf);
2617 }
2618
2619 err = btf__load(kern_btf);
2620 if (sanitize) {
2621 if (!err) {
2622 /* move fd to libbpf's BTF */
2623 btf__set_fd(obj->btf, btf__fd(kern_btf));
2624 btf__set_fd(kern_btf, -1);
2625 }
2626 btf__free(kern_btf);
2627 }
2628 report:
2629 if (err) {
2630 btf_mandatory = kernel_needs_btf(obj);
2631 pr_warn("Error loading .BTF into kernel: %d. %s\n", err,
2632 btf_mandatory ? "BTF is mandatory, can't proceed."
2633 : "BTF is optional, ignoring.");
2634 if (!btf_mandatory)
2635 err = 0;
2636 }
2637 return err;
2638 }
2639
2640 static const char *elf_sym_str(const struct bpf_object *obj, size_t off)
2641 {
2642 const char *name;
2643
2644 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, off);
2645 if (!name) {
2646 pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n",
2647 off, obj->path, elf_errmsg(-1));
2648 return NULL;
2649 }
2650
2651 return name;
2652 }
2653
2654 static const char *elf_sec_str(const struct bpf_object *obj, size_t off)
2655 {
2656 const char *name;
2657
2658 name = elf_strptr(obj->efile.elf, obj->efile.shstrndx, off);
2659 if (!name) {
2660 pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n",
2661 off, obj->path, elf_errmsg(-1));
2662 return NULL;
2663 }
2664
2665 return name;
2666 }
2667
2668 static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx)
2669 {
2670 Elf_Scn *scn;
2671
2672 scn = elf_getscn(obj->efile.elf, idx);
2673 if (!scn) {
2674 pr_warn("elf: failed to get section(%zu) from %s: %s\n",
2675 idx, obj->path, elf_errmsg(-1));
2676 return NULL;
2677 }
2678 return scn;
2679 }
2680
2681 static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name)
2682 {
2683 Elf_Scn *scn = NULL;
2684 Elf *elf = obj->efile.elf;
2685 const char *sec_name;
2686
2687 while ((scn = elf_nextscn(elf, scn)) != NULL) {
2688 sec_name = elf_sec_name(obj, scn);
2689 if (!sec_name)
2690 return NULL;
2691
2692 if (strcmp(sec_name, name) != 0)
2693 continue;
2694
2695 return scn;
2696 }
2697 return NULL;
2698 }
2699
2700 static int elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn, GElf_Shdr *hdr)
2701 {
2702 if (!scn)
2703 return -EINVAL;
2704
2705 if (gelf_getshdr(scn, hdr) != hdr) {
2706 pr_warn("elf: failed to get section(%zu) header from %s: %s\n",
2707 elf_ndxscn(scn), obj->path, elf_errmsg(-1));
2708 return -EINVAL;
2709 }
2710
2711 return 0;
2712 }
2713
2714 static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn)
2715 {
2716 const char *name;
2717 GElf_Shdr sh;
2718
2719 if (!scn)
2720 return NULL;
2721
2722 if (elf_sec_hdr(obj, scn, &sh))
2723 return NULL;
2724
2725 name = elf_sec_str(obj, sh.sh_name);
2726 if (!name) {
2727 pr_warn("elf: failed to get section(%zu) name from %s: %s\n",
2728 elf_ndxscn(scn), obj->path, elf_errmsg(-1));
2729 return NULL;
2730 }
2731
2732 return name;
2733 }
2734
2735 static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn)
2736 {
2737 Elf_Data *data;
2738
2739 if (!scn)
2740 return NULL;
2741
2742 data = elf_getdata(scn, 0);
2743 if (!data) {
2744 pr_warn("elf: failed to get section(%zu) %s data from %s: %s\n",
2745 elf_ndxscn(scn), elf_sec_name(obj, scn) ?: "<?>",
2746 obj->path, elf_errmsg(-1));
2747 return NULL;
2748 }
2749
2750 return data;
2751 }
2752
2753 static int elf_sym_by_sec_off(const struct bpf_object *obj, size_t sec_idx,
2754 size_t off, __u32 sym_type, GElf_Sym *sym)
2755 {
2756 Elf_Data *symbols = obj->efile.symbols;
2757 size_t n = symbols->d_size / sizeof(GElf_Sym);
2758 int i;
2759
2760 for (i = 0; i < n; i++) {
2761 if (!gelf_getsym(symbols, i, sym))
2762 continue;
2763 if (sym->st_shndx != sec_idx || sym->st_value != off)
2764 continue;
2765 if (GELF_ST_TYPE(sym->st_info) != sym_type)
2766 continue;
2767 return 0;
2768 }
2769
2770 return -ENOENT;
2771 }
2772
2773 static bool is_sec_name_dwarf(const char *name)
2774 {
2775 /* approximation, but the actual list is too long */
2776 return strncmp(name, ".debug_", sizeof(".debug_") - 1) == 0;
2777 }
2778
2779 static bool ignore_elf_section(GElf_Shdr *hdr, const char *name)
2780 {
2781 /* no special handling of .strtab */
2782 if (hdr->sh_type == SHT_STRTAB)
2783 return true;
2784
2785 /* ignore .llvm_addrsig section as well */
2786 if (hdr->sh_type == 0x6FFF4C03 /* SHT_LLVM_ADDRSIG */)
2787 return true;
2788
2789 /* no subprograms will lead to an empty .text section, ignore it */
2790 if (hdr->sh_type == SHT_PROGBITS && hdr->sh_size == 0 &&
2791 strcmp(name, ".text") == 0)
2792 return true;
2793
2794 /* DWARF sections */
2795 if (is_sec_name_dwarf(name))
2796 return true;
2797
2798 if (strncmp(name, ".rel", sizeof(".rel") - 1) == 0) {
2799 name += sizeof(".rel") - 1;
2800 /* DWARF section relocations */
2801 if (is_sec_name_dwarf(name))
2802 return true;
2803
2804 /* .BTF and .BTF.ext don't need relocations */
2805 if (strcmp(name, BTF_ELF_SEC) == 0 ||
2806 strcmp(name, BTF_EXT_ELF_SEC) == 0)
2807 return true;
2808 }
2809
2810 return false;
2811 }
2812
2813 static int cmp_progs(const void *_a, const void *_b)
2814 {
2815 const struct bpf_program *a = _a;
2816 const struct bpf_program *b = _b;
2817
2818 if (a->sec_idx != b->sec_idx)
2819 return a->sec_idx < b->sec_idx ? -1 : 1;
2820
2821 /* sec_insn_off can't be the same within the section */
2822 return a->sec_insn_off < b->sec_insn_off ? -1 : 1;
2823 }
2824
2825 static int bpf_object__elf_collect(struct bpf_object *obj)
2826 {
2827 Elf *elf = obj->efile.elf;
2828 Elf_Data *btf_ext_data = NULL;
2829 Elf_Data *btf_data = NULL;
2830 int idx = 0, err = 0;
2831 const char *name;
2832 Elf_Data *data;
2833 Elf_Scn *scn;
2834 GElf_Shdr sh;
2835
2836 /* a bunch of ELF parsing functionality depends on processing symbols,
2837 * so do the first pass and find the symbol table
2838 */
2839 scn = NULL;
2840 while ((scn = elf_nextscn(elf, scn)) != NULL) {
2841 if (elf_sec_hdr(obj, scn, &sh))
2842 return -LIBBPF_ERRNO__FORMAT;
2843
2844 if (sh.sh_type == SHT_SYMTAB) {
2845 if (obj->efile.symbols) {
2846 pr_warn("elf: multiple symbol tables in %s\n", obj->path);
2847 return -LIBBPF_ERRNO__FORMAT;
2848 }
2849
2850 data = elf_sec_data(obj, scn);
2851 if (!data)
2852 return -LIBBPF_ERRNO__FORMAT;
2853
2854 obj->efile.symbols = data;
2855 obj->efile.symbols_shndx = elf_ndxscn(scn);
2856 obj->efile.strtabidx = sh.sh_link;
2857 }
2858 }
2859
2860 scn = NULL;
2861 while ((scn = elf_nextscn(elf, scn)) != NULL) {
2862 idx++;
2863
2864 if (elf_sec_hdr(obj, scn, &sh))
2865 return -LIBBPF_ERRNO__FORMAT;
2866
2867 name = elf_sec_str(obj, sh.sh_name);
2868 if (!name)
2869 return -LIBBPF_ERRNO__FORMAT;
2870
2871 if (ignore_elf_section(&sh, name))
2872 continue;
2873
2874 data = elf_sec_data(obj, scn);
2875 if (!data)
2876 return -LIBBPF_ERRNO__FORMAT;
2877
2878 pr_debug("elf: section(%d) %s, size %ld, link %d, flags %lx, type=%d\n",
2879 idx, name, (unsigned long)data->d_size,
2880 (int)sh.sh_link, (unsigned long)sh.sh_flags,
2881 (int)sh.sh_type);
2882
2883 if (strcmp(name, "license") == 0) {
2884 err = bpf_object__init_license(obj, data->d_buf, data->d_size);
2885 if (err)
2886 return err;
2887 } else if (strcmp(name, "version") == 0) {
2888 err = bpf_object__init_kversion(obj, data->d_buf, data->d_size);
2889 if (err)
2890 return err;
2891 } else if (strcmp(name, "maps") == 0) {
2892 obj->efile.maps_shndx = idx;
2893 } else if (strcmp(name, MAPS_ELF_SEC) == 0) {
2894 obj->efile.btf_maps_shndx = idx;
2895 } else if (strcmp(name, BTF_ELF_SEC) == 0) {
2896 btf_data = data;
2897 } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) {
2898 btf_ext_data = data;
2899 } else if (sh.sh_type == SHT_SYMTAB) {
2900 /* already processed during the first pass above */
2901 } else if (sh.sh_type == SHT_PROGBITS && data->d_size > 0) {
2902 if (sh.sh_flags & SHF_EXECINSTR) {
2903 if (strcmp(name, ".text") == 0)
2904 obj->efile.text_shndx = idx;
2905 err = bpf_object__add_programs(obj, data, name, idx);
2906 if (err)
2907 return err;
2908 } else if (strcmp(name, DATA_SEC) == 0) {
2909 obj->efile.data = data;
2910 obj->efile.data_shndx = idx;
2911 } else if (strcmp(name, RODATA_SEC) == 0) {
2912 obj->efile.rodata = data;
2913 obj->efile.rodata_shndx = idx;
2914 } else if (strcmp(name, STRUCT_OPS_SEC) == 0) {
2915 obj->efile.st_ops_data = data;
2916 obj->efile.st_ops_shndx = idx;
2917 } else {
2918 pr_info("elf: skipping unrecognized data section(%d) %s\n",
2919 idx, name);
2920 }
2921 } else if (sh.sh_type == SHT_REL) {
2922 int nr_sects = obj->efile.nr_reloc_sects;
2923 void *sects = obj->efile.reloc_sects;
2924 int sec = sh.sh_info; /* points to other section */
2925
2926 /* Only do relo for section with exec instructions */
2927 if (!section_have_execinstr(obj, sec) &&
2928 strcmp(name, ".rel" STRUCT_OPS_SEC) &&
2929 strcmp(name, ".rel" MAPS_ELF_SEC)) {
2930 pr_info("elf: skipping relo section(%d) %s for section(%d) %s\n",
2931 idx, name, sec,
2932 elf_sec_name(obj, elf_sec_by_idx(obj, sec)) ?: "<?>");
2933 continue;
2934 }
2935
2936 sects = libbpf_reallocarray(sects, nr_sects + 1,
2937 sizeof(*obj->efile.reloc_sects));
2938 if (!sects)
2939 return -ENOMEM;
2940
2941 obj->efile.reloc_sects = sects;
2942 obj->efile.nr_reloc_sects++;
2943
2944 obj->efile.reloc_sects[nr_sects].shdr = sh;
2945 obj->efile.reloc_sects[nr_sects].data = data;
2946 } else if (sh.sh_type == SHT_NOBITS && strcmp(name, BSS_SEC) == 0) {
2947 obj->efile.bss = data;
2948 obj->efile.bss_shndx = idx;
2949 } else {
2950 pr_info("elf: skipping section(%d) %s (size %zu)\n", idx, name,
2951 (size_t)sh.sh_size);
2952 }
2953 }
2954
2955 if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) {
2956 pr_warn("elf: symbol strings section missing or invalid in %s\n", obj->path);
2957 return -LIBBPF_ERRNO__FORMAT;
2958 }
2959
2960 /* sort BPF programs by section name and in-section instruction offset
2961 * for faster search */
2962 qsort(obj->programs, obj->nr_programs, sizeof(*obj->programs), cmp_progs);
2963
2964 return bpf_object__init_btf(obj, btf_data, btf_ext_data);
2965 }
2966
2967 static bool sym_is_extern(const GElf_Sym *sym)
2968 {
2969 int bind = GELF_ST_BIND(sym->st_info);
2970 /* externs are symbols w/ type=NOTYPE, bind=GLOBAL|WEAK, section=UND */
2971 return sym->st_shndx == SHN_UNDEF &&
2972 (bind == STB_GLOBAL || bind == STB_WEAK) &&
2973 GELF_ST_TYPE(sym->st_info) == STT_NOTYPE;
2974 }
2975
2976 static int find_extern_btf_id(const struct btf *btf, const char *ext_name)
2977 {
2978 const struct btf_type *t;
2979 const char *var_name;
2980 int i, n;
2981
2982 if (!btf)
2983 return -ESRCH;
2984
2985 n = btf__get_nr_types(btf);
2986 for (i = 1; i <= n; i++) {
2987 t = btf__type_by_id(btf, i);
2988
2989 if (!btf_is_var(t))
2990 continue;
2991
2992 var_name = btf__name_by_offset(btf, t->name_off);
2993 if (strcmp(var_name, ext_name))
2994 continue;
2995
2996 if (btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN)
2997 return -EINVAL;
2998
2999 return i;
3000 }
3001
3002 return -ENOENT;
3003 }
3004
3005 static int find_extern_sec_btf_id(struct btf *btf, int ext_btf_id) {
3006 const struct btf_var_secinfo *vs;
3007 const struct btf_type *t;
3008 int i, j, n;
3009
3010 if (!btf)
3011 return -ESRCH;
3012
3013 n = btf__get_nr_types(btf);
3014 for (i = 1; i <= n; i++) {
3015 t = btf__type_by_id(btf, i);
3016
3017 if (!btf_is_datasec(t))
3018 continue;
3019
3020 vs = btf_var_secinfos(t);
3021 for (j = 0; j < btf_vlen(t); j++, vs++) {
3022 if (vs->type == ext_btf_id)
3023 return i;
3024 }
3025 }
3026
3027 return -ENOENT;
3028 }
3029
3030 static enum kcfg_type find_kcfg_type(const struct btf *btf, int id,
3031 bool *is_signed)
3032 {
3033 const struct btf_type *t;
3034 const char *name;
3035
3036 t = skip_mods_and_typedefs(btf, id, NULL);
3037 name = btf__name_by_offset(btf, t->name_off);
3038
3039 if (is_signed)
3040 *is_signed = false;
3041 switch (btf_kind(t)) {
3042 case BTF_KIND_INT: {
3043 int enc = btf_int_encoding(t);
3044
3045 if (enc & BTF_INT_BOOL)
3046 return t->size == 1 ? KCFG_BOOL : KCFG_UNKNOWN;
3047 if (is_signed)
3048 *is_signed = enc & BTF_INT_SIGNED;
3049 if (t->size == 1)
3050 return KCFG_CHAR;
3051 if (t->size < 1 || t->size > 8 || (t->size & (t->size - 1)))
3052 return KCFG_UNKNOWN;
3053 return KCFG_INT;
3054 }
3055 case BTF_KIND_ENUM:
3056 if (t->size != 4)
3057 return KCFG_UNKNOWN;
3058 if (strcmp(name, "libbpf_tristate"))
3059 return KCFG_UNKNOWN;
3060 return KCFG_TRISTATE;
3061 case BTF_KIND_ARRAY:
3062 if (btf_array(t)->nelems == 0)
3063 return KCFG_UNKNOWN;
3064 if (find_kcfg_type(btf, btf_array(t)->type, NULL) != KCFG_CHAR)
3065 return KCFG_UNKNOWN;
3066 return KCFG_CHAR_ARR;
3067 default:
3068 return KCFG_UNKNOWN;
3069 }
3070 }
3071
3072 static int cmp_externs(const void *_a, const void *_b)
3073 {
3074 const struct extern_desc *a = _a;
3075 const struct extern_desc *b = _b;
3076
3077 if (a->type != b->type)
3078 return a->type < b->type ? -1 : 1;
3079
3080 if (a->type == EXT_KCFG) {
3081 /* descending order by alignment requirements */
3082 if (a->kcfg.align != b->kcfg.align)
3083 return a->kcfg.align > b->kcfg.align ? -1 : 1;
3084 /* ascending order by size, within same alignment class */
3085 if (a->kcfg.sz != b->kcfg.sz)
3086 return a->kcfg.sz < b->kcfg.sz ? -1 : 1;
3087 }
3088
3089 /* resolve ties by name */
3090 return strcmp(a->name, b->name);
3091 }
3092
3093 static int find_int_btf_id(const struct btf *btf)
3094 {
3095 const struct btf_type *t;
3096 int i, n;
3097
3098 n = btf__get_nr_types(btf);
3099 for (i = 1; i <= n; i++) {
3100 t = btf__type_by_id(btf, i);
3101
3102 if (btf_is_int(t) && btf_int_bits(t) == 32)
3103 return i;
3104 }
3105
3106 return 0;
3107 }
3108
3109 static int bpf_object__collect_externs(struct bpf_object *obj)
3110 {
3111 struct btf_type *sec, *kcfg_sec = NULL, *ksym_sec = NULL;
3112 const struct btf_type *t;
3113 struct extern_desc *ext;
3114 int i, n, off;
3115 const char *ext_name, *sec_name;
3116 Elf_Scn *scn;
3117 GElf_Shdr sh;
3118
3119 if (!obj->efile.symbols)
3120 return 0;
3121
3122 scn = elf_sec_by_idx(obj, obj->efile.symbols_shndx);
3123 if (elf_sec_hdr(obj, scn, &sh))
3124 return -LIBBPF_ERRNO__FORMAT;
3125
3126 n = sh.sh_size / sh.sh_entsize;
3127 pr_debug("looking for externs among %d symbols...\n", n);
3128
3129 for (i = 0; i < n; i++) {
3130 GElf_Sym sym;
3131
3132 if (!gelf_getsym(obj->efile.symbols, i, &sym))
3133 return -LIBBPF_ERRNO__FORMAT;
3134 if (!sym_is_extern(&sym))
3135 continue;
3136 ext_name = elf_sym_str(obj, sym.st_name);
3137 if (!ext_name || !ext_name[0])
3138 continue;
3139
3140 ext = obj->externs;
3141 ext = libbpf_reallocarray(ext, obj->nr_extern + 1, sizeof(*ext));
3142 if (!ext)
3143 return -ENOMEM;
3144 obj->externs = ext;
3145 ext = &ext[obj->nr_extern];
3146 memset(ext, 0, sizeof(*ext));
3147 obj->nr_extern++;
3148
3149 ext->btf_id = find_extern_btf_id(obj->btf, ext_name);
3150 if (ext->btf_id <= 0) {
3151 pr_warn("failed to find BTF for extern '%s': %d\n",
3152 ext_name, ext->btf_id);
3153 return ext->btf_id;
3154 }
3155 t = btf__type_by_id(obj->btf, ext->btf_id);
3156 ext->name = btf__name_by_offset(obj->btf, t->name_off);
3157 ext->sym_idx = i;
3158 ext->is_weak = GELF_ST_BIND(sym.st_info) == STB_WEAK;
3159
3160 ext->sec_btf_id = find_extern_sec_btf_id(obj->btf, ext->btf_id);
3161 if (ext->sec_btf_id <= 0) {
3162 pr_warn("failed to find BTF for extern '%s' [%d] section: %d\n",
3163 ext_name, ext->btf_id, ext->sec_btf_id);
3164 return ext->sec_btf_id;
3165 }
3166 sec = (void *)btf__type_by_id(obj->btf, ext->sec_btf_id);
3167 sec_name = btf__name_by_offset(obj->btf, sec->name_off);
3168
3169 if (strcmp(sec_name, KCONFIG_SEC) == 0) {
3170 kcfg_sec = sec;
3171 ext->type = EXT_KCFG;
3172 ext->kcfg.sz = btf__resolve_size(obj->btf, t->type);
3173 if (ext->kcfg.sz <= 0) {
3174 pr_warn("failed to resolve size of extern (kcfg) '%s': %d\n",
3175 ext_name, ext->kcfg.sz);
3176 return ext->kcfg.sz;
3177 }
3178 ext->kcfg.align = btf__align_of(obj->btf, t->type);
3179 if (ext->kcfg.align <= 0) {
3180 pr_warn("failed to determine alignment of extern (kcfg) '%s': %d\n",
3181 ext_name, ext->kcfg.align);
3182 return -EINVAL;
3183 }
3184 ext->kcfg.type = find_kcfg_type(obj->btf, t->type,
3185 &ext->kcfg.is_signed);
3186 if (ext->kcfg.type == KCFG_UNKNOWN) {
3187 pr_warn("extern (kcfg) '%s' type is unsupported\n", ext_name);
3188 return -ENOTSUP;
3189 }
3190 } else if (strcmp(sec_name, KSYMS_SEC) == 0) {
3191 ksym_sec = sec;
3192 ext->type = EXT_KSYM;
3193 skip_mods_and_typedefs(obj->btf, t->type,
3194 &ext->ksym.type_id);
3195 } else {
3196 pr_warn("unrecognized extern section '%s'\n", sec_name);
3197 return -ENOTSUP;
3198 }
3199 }
3200 pr_debug("collected %d externs total\n", obj->nr_extern);
3201
3202 if (!obj->nr_extern)
3203 return 0;
3204
3205 /* sort externs by type, for kcfg ones also by (align, size, name) */
3206 qsort(obj->externs, obj->nr_extern, sizeof(*ext), cmp_externs);
3207
3208 /* for .ksyms section, we need to turn all externs into allocated
3209 * variables in BTF to pass kernel verification; we do this by
3210 * pretending that each extern is a 8-byte variable
3211 */
3212 if (ksym_sec) {
3213 /* find existing 4-byte integer type in BTF to use for fake
3214 * extern variables in DATASEC
3215 */
3216 int int_btf_id = find_int_btf_id(obj->btf);
3217
3218 for (i = 0; i < obj->nr_extern; i++) {
3219 ext = &obj->externs[i];
3220 if (ext->type != EXT_KSYM)
3221 continue;
3222 pr_debug("extern (ksym) #%d: symbol %d, name %s\n",
3223 i, ext->sym_idx, ext->name);
3224 }
3225
3226 sec = ksym_sec;
3227 n = btf_vlen(sec);
3228 for (i = 0, off = 0; i < n; i++, off += sizeof(int)) {
3229 struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i;
3230 struct btf_type *vt;
3231
3232 vt = (void *)btf__type_by_id(obj->btf, vs->type);
3233 ext_name = btf__name_by_offset(obj->btf, vt->name_off);
3234 ext = find_extern_by_name(obj, ext_name);
3235 if (!ext) {
3236 pr_warn("failed to find extern definition for BTF var '%s'\n",
3237 ext_name);
3238 return -ESRCH;
3239 }
3240 btf_var(vt)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
3241 vt->type = int_btf_id;
3242 vs->offset = off;
3243 vs->size = sizeof(int);
3244 }
3245 sec->size = off;
3246 }
3247
3248 if (kcfg_sec) {
3249 sec = kcfg_sec;
3250 /* for kcfg externs calculate their offsets within a .kconfig map */
3251 off = 0;
3252 for (i = 0; i < obj->nr_extern; i++) {
3253 ext = &obj->externs[i];
3254 if (ext->type != EXT_KCFG)
3255 continue;
3256
3257 ext->kcfg.data_off = roundup(off, ext->kcfg.align);
3258 off = ext->kcfg.data_off + ext->kcfg.sz;
3259 pr_debug("extern (kcfg) #%d: symbol %d, off %u, name %s\n",
3260 i, ext->sym_idx, ext->kcfg.data_off, ext->name);
3261 }
3262 sec->size = off;
3263 n = btf_vlen(sec);
3264 for (i = 0; i < n; i++) {
3265 struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i;
3266
3267 t = btf__type_by_id(obj->btf, vs->type);
3268 ext_name = btf__name_by_offset(obj->btf, t->name_off);
3269 ext = find_extern_by_name(obj, ext_name);
3270 if (!ext) {
3271 pr_warn("failed to find extern definition for BTF var '%s'\n",
3272 ext_name);
3273 return -ESRCH;
3274 }
3275 btf_var(t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
3276 vs->offset = ext->kcfg.data_off;
3277 }
3278 }
3279 return 0;
3280 }
3281
3282 struct bpf_program *
3283 bpf_object__find_program_by_title(const struct bpf_object *obj,
3284 const char *title)
3285 {
3286 struct bpf_program *pos;
3287
3288 bpf_object__for_each_program(pos, obj) {
3289 if (pos->sec_name && !strcmp(pos->sec_name, title))
3290 return pos;
3291 }
3292 return NULL;
3293 }
3294
3295 static bool prog_is_subprog(const struct bpf_object *obj,
3296 const struct bpf_program *prog)
3297 {
3298 /* For legacy reasons, libbpf supports an entry-point BPF programs
3299 * without SEC() attribute, i.e., those in the .text section. But if
3300 * there are 2 or more such programs in the .text section, they all
3301 * must be subprograms called from entry-point BPF programs in
3302 * designated SEC()'tions, otherwise there is no way to distinguish
3303 * which of those programs should be loaded vs which are a subprogram.
3304 * Similarly, if there is a function/program in .text and at least one
3305 * other BPF program with custom SEC() attribute, then we just assume
3306 * .text programs are subprograms (even if they are not called from
3307 * other programs), because libbpf never explicitly supported mixing
3308 * SEC()-designated BPF programs and .text entry-point BPF programs.
3309 */
3310 return prog->sec_idx == obj->efile.text_shndx && obj->nr_programs > 1;
3311 }
3312
3313 struct bpf_program *
3314 bpf_object__find_program_by_name(const struct bpf_object *obj,
3315 const char *name)
3316 {
3317 struct bpf_program *prog;
3318
3319 bpf_object__for_each_program(prog, obj) {
3320 if (prog_is_subprog(obj, prog))
3321 continue;
3322 if (!strcmp(prog->name, name))
3323 return prog;
3324 }
3325 return NULL;
3326 }
3327
3328 static bool bpf_object__shndx_is_data(const struct bpf_object *obj,
3329 int shndx)
3330 {
3331 return shndx == obj->efile.data_shndx ||
3332 shndx == obj->efile.bss_shndx ||
3333 shndx == obj->efile.rodata_shndx;
3334 }
3335
3336 static bool bpf_object__shndx_is_maps(const struct bpf_object *obj,
3337 int shndx)
3338 {
3339 return shndx == obj->efile.maps_shndx ||
3340 shndx == obj->efile.btf_maps_shndx;
3341 }
3342
3343 static enum libbpf_map_type
3344 bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx)
3345 {
3346 if (shndx == obj->efile.data_shndx)
3347 return LIBBPF_MAP_DATA;
3348 else if (shndx == obj->efile.bss_shndx)
3349 return LIBBPF_MAP_BSS;
3350 else if (shndx == obj->efile.rodata_shndx)
3351 return LIBBPF_MAP_RODATA;
3352 else if (shndx == obj->efile.symbols_shndx)
3353 return LIBBPF_MAP_KCONFIG;
3354 else
3355 return LIBBPF_MAP_UNSPEC;
3356 }
3357
3358 static int bpf_program__record_reloc(struct bpf_program *prog,
3359 struct reloc_desc *reloc_desc,
3360 __u32 insn_idx, const char *sym_name,
3361 const GElf_Sym *sym, const GElf_Rel *rel)
3362 {
3363 struct bpf_insn *insn = &prog->insns[insn_idx];
3364 size_t map_idx, nr_maps = prog->obj->nr_maps;
3365 struct bpf_object *obj = prog->obj;
3366 __u32 shdr_idx = sym->st_shndx;
3367 enum libbpf_map_type type;
3368 const char *sym_sec_name;
3369 struct bpf_map *map;
3370
3371 reloc_desc->processed = false;
3372
3373 /* sub-program call relocation */
3374 if (insn->code == (BPF_JMP | BPF_CALL)) {
3375 if (insn->src_reg != BPF_PSEUDO_CALL) {
3376 pr_warn("prog '%s': incorrect bpf_call opcode\n", prog->name);
3377 return -LIBBPF_ERRNO__RELOC;
3378 }
3379 /* text_shndx can be 0, if no default "main" program exists */
3380 if (!shdr_idx || shdr_idx != obj->efile.text_shndx) {
3381 sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx));
3382 pr_warn("prog '%s': bad call relo against '%s' in section '%s'\n",
3383 prog->name, sym_name, sym_sec_name);
3384 return -LIBBPF_ERRNO__RELOC;
3385 }
3386 if (sym->st_value % BPF_INSN_SZ) {
3387 pr_warn("prog '%s': bad call relo against '%s' at offset %zu\n",
3388 prog->name, sym_name, (size_t)sym->st_value);
3389 return -LIBBPF_ERRNO__RELOC;
3390 }
3391 reloc_desc->type = RELO_CALL;
3392 reloc_desc->insn_idx = insn_idx;
3393 reloc_desc->sym_off = sym->st_value;
3394 return 0;
3395 }
3396
3397 if (insn->code != (BPF_LD | BPF_IMM | BPF_DW)) {
3398 pr_warn("prog '%s': invalid relo against '%s' for insns[%d].code 0x%x\n",
3399 prog->name, sym_name, insn_idx, insn->code);
3400 return -LIBBPF_ERRNO__RELOC;
3401 }
3402
3403 if (sym_is_extern(sym)) {
3404 int sym_idx = GELF_R_SYM(rel->r_info);
3405 int i, n = obj->nr_extern;
3406 struct extern_desc *ext;
3407
3408 for (i = 0; i < n; i++) {
3409 ext = &obj->externs[i];
3410 if (ext->sym_idx == sym_idx)
3411 break;
3412 }
3413 if (i >= n) {
3414 pr_warn("prog '%s': extern relo failed to find extern for '%s' (%d)\n",
3415 prog->name, sym_name, sym_idx);
3416 return -LIBBPF_ERRNO__RELOC;
3417 }
3418 pr_debug("prog '%s': found extern #%d '%s' (sym %d) for insn #%u\n",
3419 prog->name, i, ext->name, ext->sym_idx, insn_idx);
3420 reloc_desc->type = RELO_EXTERN;
3421 reloc_desc->insn_idx = insn_idx;
3422 reloc_desc->sym_off = i; /* sym_off stores extern index */
3423 return 0;
3424 }
3425
3426 if (!shdr_idx || shdr_idx >= SHN_LORESERVE) {
3427 pr_warn("prog '%s': invalid relo against '%s' in special section 0x%x; forgot to initialize global var?..\n",
3428 prog->name, sym_name, shdr_idx);
3429 return -LIBBPF_ERRNO__RELOC;
3430 }
3431
3432 type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx);
3433 sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx));
3434
3435 /* generic map reference relocation */
3436 if (type == LIBBPF_MAP_UNSPEC) {
3437 if (!bpf_object__shndx_is_maps(obj, shdr_idx)) {
3438 pr_warn("prog '%s': bad map relo against '%s' in section '%s'\n",
3439 prog->name, sym_name, sym_sec_name);
3440 return -LIBBPF_ERRNO__RELOC;
3441 }
3442 for (map_idx = 0; map_idx < nr_maps; map_idx++) {
3443 map = &obj->maps[map_idx];
3444 if (map->libbpf_type != type ||
3445 map->sec_idx != sym->st_shndx ||
3446 map->sec_offset != sym->st_value)
3447 continue;
3448 pr_debug("prog '%s': found map %zd (%s, sec %d, off %zu) for insn #%u\n",
3449 prog->name, map_idx, map->name, map->sec_idx,
3450 map->sec_offset, insn_idx);
3451 break;
3452 }
3453 if (map_idx >= nr_maps) {
3454 pr_warn("prog '%s': map relo failed to find map for section '%s', off %zu\n",
3455 prog->name, sym_sec_name, (size_t)sym->st_value);
3456 return -LIBBPF_ERRNO__RELOC;
3457 }
3458 reloc_desc->type = RELO_LD64;
3459 reloc_desc->insn_idx = insn_idx;
3460 reloc_desc->map_idx = map_idx;
3461 reloc_desc->sym_off = 0; /* sym->st_value determines map_idx */
3462 return 0;
3463 }
3464
3465 /* global data map relocation */
3466 if (!bpf_object__shndx_is_data(obj, shdr_idx)) {
3467 pr_warn("prog '%s': bad data relo against section '%s'\n",
3468 prog->name, sym_sec_name);
3469 return -LIBBPF_ERRNO__RELOC;
3470 }
3471 for (map_idx = 0; map_idx < nr_maps; map_idx++) {
3472 map = &obj->maps[map_idx];
3473 if (map->libbpf_type != type)
3474 continue;
3475 pr_debug("prog '%s': found data map %zd (%s, sec %d, off %zu) for insn %u\n",
3476 prog->name, map_idx, map->name, map->sec_idx,
3477 map->sec_offset, insn_idx);
3478 break;
3479 }
3480 if (map_idx >= nr_maps) {
3481 pr_warn("prog '%s': data relo failed to find map for section '%s'\n",
3482 prog->name, sym_sec_name);
3483 return -LIBBPF_ERRNO__RELOC;
3484 }
3485
3486 reloc_desc->type = RELO_DATA;
3487 reloc_desc->insn_idx = insn_idx;
3488 reloc_desc->map_idx = map_idx;
3489 reloc_desc->sym_off = sym->st_value;
3490 return 0;
3491 }
3492
3493 static bool prog_contains_insn(const struct bpf_program *prog, size_t insn_idx)
3494 {
3495 return insn_idx >= prog->sec_insn_off &&
3496 insn_idx < prog->sec_insn_off + prog->sec_insn_cnt;
3497 }
3498
3499 static struct bpf_program *find_prog_by_sec_insn(const struct bpf_object *obj,
3500 size_t sec_idx, size_t insn_idx)
3501 {
3502 int l = 0, r = obj->nr_programs - 1, m;
3503 struct bpf_program *prog;
3504
3505 while (l < r) {
3506 m = l + (r - l + 1) / 2;
3507 prog = &obj->programs[m];
3508
3509 if (prog->sec_idx < sec_idx ||
3510 (prog->sec_idx == sec_idx && prog->sec_insn_off <= insn_idx))
3511 l = m;
3512 else
3513 r = m - 1;
3514 }
3515 /* matching program could be at index l, but it still might be the
3516 * wrong one, so we need to double check conditions for the last time
3517 */
3518 prog = &obj->programs[l];
3519 if (prog->sec_idx == sec_idx && prog_contains_insn(prog, insn_idx))
3520 return prog;
3521 return NULL;
3522 }
3523
3524 static int
3525 bpf_object__collect_prog_relos(struct bpf_object *obj, GElf_Shdr *shdr, Elf_Data *data)
3526 {
3527 Elf_Data *symbols = obj->efile.symbols;
3528 const char *relo_sec_name, *sec_name;
3529 size_t sec_idx = shdr->sh_info;
3530 struct bpf_program *prog;
3531 struct reloc_desc *relos;
3532 int err, i, nrels;
3533 const char *sym_name;
3534 __u32 insn_idx;
3535 GElf_Sym sym;
3536 GElf_Rel rel;
3537
3538 relo_sec_name = elf_sec_str(obj, shdr->sh_name);
3539 sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));
3540 if (!relo_sec_name || !sec_name)
3541 return -EINVAL;
3542
3543 pr_debug("sec '%s': collecting relocation for section(%zu) '%s'\n",
3544 relo_sec_name, sec_idx, sec_name);
3545 nrels = shdr->sh_size / shdr->sh_entsize;
3546
3547 for (i = 0; i < nrels; i++) {
3548 if (!gelf_getrel(data, i, &rel)) {
3549 pr_warn("sec '%s': failed to get relo #%d\n", relo_sec_name, i);
3550 return -LIBBPF_ERRNO__FORMAT;
3551 }
3552 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) {
3553 pr_warn("sec '%s': symbol 0x%zx not found for relo #%d\n",
3554 relo_sec_name, (size_t)GELF_R_SYM(rel.r_info), i);
3555 return -LIBBPF_ERRNO__FORMAT;
3556 }
3557 if (rel.r_offset % BPF_INSN_SZ) {
3558 pr_warn("sec '%s': invalid offset 0x%zx for relo #%d\n",
3559 relo_sec_name, (size_t)GELF_R_SYM(rel.r_info), i);
3560 return -LIBBPF_ERRNO__FORMAT;
3561 }
3562
3563 insn_idx = rel.r_offset / BPF_INSN_SZ;
3564 /* relocations against static functions are recorded as
3565 * relocations against the section that contains a function;
3566 * in such case, symbol will be STT_SECTION and sym.st_name
3567 * will point to empty string (0), so fetch section name
3568 * instead
3569 */
3570 if (GELF_ST_TYPE(sym.st_info) == STT_SECTION && sym.st_name == 0)
3571 sym_name = elf_sec_name(obj, elf_sec_by_idx(obj, sym.st_shndx));
3572 else
3573 sym_name = elf_sym_str(obj, sym.st_name);
3574 sym_name = sym_name ?: "<?";
3575
3576 pr_debug("sec '%s': relo #%d: insn #%u against '%s'\n",
3577 relo_sec_name, i, insn_idx, sym_name);
3578
3579 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx);
3580 if (!prog) {
3581 pr_warn("sec '%s': relo #%d: program not found in section '%s' for insn #%u\n",
3582 relo_sec_name, i, sec_name, insn_idx);
3583 return -LIBBPF_ERRNO__RELOC;
3584 }
3585
3586 relos = libbpf_reallocarray(prog->reloc_desc,
3587 prog->nr_reloc + 1, sizeof(*relos));
3588 if (!relos)
3589 return -ENOMEM;
3590 prog->reloc_desc = relos;
3591
3592 /* adjust insn_idx to local BPF program frame of reference */
3593 insn_idx -= prog->sec_insn_off;
3594 err = bpf_program__record_reloc(prog, &relos[prog->nr_reloc],
3595 insn_idx, sym_name, &sym, &rel);
3596 if (err)
3597 return err;
3598
3599 prog->nr_reloc++;
3600 }
3601 return 0;
3602 }
3603
3604 static int bpf_map_find_btf_info(struct bpf_object *obj, struct bpf_map *map)
3605 {
3606 struct bpf_map_def *def = &map->def;
3607 __u32 key_type_id = 0, value_type_id = 0;
3608 int ret;
3609
3610 /* if it's BTF-defined map, we don't need to search for type IDs.
3611 * For struct_ops map, it does not need btf_key_type_id and
3612 * btf_value_type_id.
3613 */
3614 if (map->sec_idx == obj->efile.btf_maps_shndx ||
3615 bpf_map__is_struct_ops(map))
3616 return 0;
3617
3618 if (!bpf_map__is_internal(map)) {
3619 ret = btf__get_map_kv_tids(obj->btf, map->name, def->key_size,
3620 def->value_size, &key_type_id,
3621 &value_type_id);
3622 } else {
3623 /*
3624 * LLVM annotates global data differently in BTF, that is,
3625 * only as '.data', '.bss' or '.rodata'.
3626 */
3627 ret = btf__find_by_name(obj->btf,
3628 libbpf_type_to_btf_name[map->libbpf_type]);
3629 }
3630 if (ret < 0)
3631 return ret;
3632
3633 map->btf_key_type_id = key_type_id;
3634 map->btf_value_type_id = bpf_map__is_internal(map) ?
3635 ret : value_type_id;
3636 return 0;
3637 }
3638
3639 int bpf_map__reuse_fd(struct bpf_map *map, int fd)
3640 {
3641 struct bpf_map_info info = {};
3642 __u32 len = sizeof(info);
3643 int new_fd, err;
3644 char *new_name;
3645
3646 err = bpf_obj_get_info_by_fd(fd, &info, &len);
3647 if (err)
3648 return err;
3649
3650 new_name = strdup(info.name);
3651 if (!new_name)
3652 return -errno;
3653
3654 new_fd = open("/", O_RDONLY | O_CLOEXEC);
3655 if (new_fd < 0) {
3656 err = -errno;
3657 goto err_free_new_name;
3658 }
3659
3660 new_fd = dup3(fd, new_fd, O_CLOEXEC);
3661 if (new_fd < 0) {
3662 err = -errno;
3663 goto err_close_new_fd;
3664 }
3665
3666 err = zclose(map->fd);
3667 if (err) {
3668 err = -errno;
3669 goto err_close_new_fd;
3670 }
3671 free(map->name);
3672
3673 map->fd = new_fd;
3674 map->name = new_name;
3675 map->def.type = info.type;
3676 map->def.key_size = info.key_size;
3677 map->def.value_size = info.value_size;
3678 map->def.max_entries = info.max_entries;
3679 map->def.map_flags = info.map_flags;
3680 map->btf_key_type_id = info.btf_key_type_id;
3681 map->btf_value_type_id = info.btf_value_type_id;
3682 map->reused = true;
3683
3684 return 0;
3685
3686 err_close_new_fd:
3687 close(new_fd);
3688 err_free_new_name:
3689 free(new_name);
3690 return err;
3691 }
3692
3693 __u32 bpf_map__max_entries(const struct bpf_map *map)
3694 {
3695 return map->def.max_entries;
3696 }
3697
3698 int bpf_map__set_max_entries(struct bpf_map *map, __u32 max_entries)
3699 {
3700 if (map->fd >= 0)
3701 return -EBUSY;
3702 map->def.max_entries = max_entries;
3703 return 0;
3704 }
3705
3706 int bpf_map__resize(struct bpf_map *map, __u32 max_entries)
3707 {
3708 if (!map || !max_entries)
3709 return -EINVAL;
3710
3711 return bpf_map__set_max_entries(map, max_entries);
3712 }
3713
3714 static int
3715 bpf_object__probe_loading(struct bpf_object *obj)
3716 {
3717 struct bpf_load_program_attr attr;
3718 char *cp, errmsg[STRERR_BUFSIZE];
3719 struct bpf_insn insns[] = {
3720 BPF_MOV64_IMM(BPF_REG_0, 0),
3721 BPF_EXIT_INSN(),
3722 };
3723 int ret;
3724
3725 /* make sure basic loading works */
3726
3727 memset(&attr, 0, sizeof(attr));
3728 attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
3729 attr.insns = insns;
3730 attr.insns_cnt = ARRAY_SIZE(insns);
3731 attr.license = "GPL";
3732
3733 ret = bpf_load_program_xattr(&attr, NULL, 0);
3734 if (ret < 0) {
3735 ret = errno;
3736 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
3737 pr_warn("Error in %s():%s(%d). Couldn't load trivial BPF "
3738 "program. Make sure your kernel supports BPF "
3739 "(CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is "
3740 "set to big enough value.\n", __func__, cp, ret);
3741 return -ret;
3742 }
3743 close(ret);
3744
3745 return 0;
3746 }
3747
3748 static int probe_fd(int fd)
3749 {
3750 if (fd >= 0)
3751 close(fd);
3752 return fd >= 0;
3753 }
3754
3755 static int probe_kern_prog_name(void)
3756 {
3757 struct bpf_load_program_attr attr;
3758 struct bpf_insn insns[] = {
3759 BPF_MOV64_IMM(BPF_REG_0, 0),
3760 BPF_EXIT_INSN(),
3761 };
3762 int ret;
3763
3764 /* make sure loading with name works */
3765
3766 memset(&attr, 0, sizeof(attr));
3767 attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
3768 attr.insns = insns;
3769 attr.insns_cnt = ARRAY_SIZE(insns);
3770 attr.license = "GPL";
3771 attr.name = "test";
3772 ret = bpf_load_program_xattr(&attr, NULL, 0);
3773 return probe_fd(ret);
3774 }
3775
3776 static int probe_kern_global_data(void)
3777 {
3778 struct bpf_load_program_attr prg_attr;
3779 struct bpf_create_map_attr map_attr;
3780 char *cp, errmsg[STRERR_BUFSIZE];
3781 struct bpf_insn insns[] = {
3782 BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16),
3783 BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42),
3784 BPF_MOV64_IMM(BPF_REG_0, 0),
3785 BPF_EXIT_INSN(),
3786 };
3787 int ret, map;
3788
3789 memset(&map_attr, 0, sizeof(map_attr));
3790 map_attr.map_type = BPF_MAP_TYPE_ARRAY;
3791 map_attr.key_size = sizeof(int);
3792 map_attr.value_size = 32;
3793 map_attr.max_entries = 1;
3794
3795 map = bpf_create_map_xattr(&map_attr);
3796 if (map < 0) {
3797 ret = -errno;
3798 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
3799 pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n",
3800 __func__, cp, -ret);
3801 return ret;
3802 }
3803
3804 insns[0].imm = map;
3805
3806 memset(&prg_attr, 0, sizeof(prg_attr));
3807 prg_attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
3808 prg_attr.insns = insns;
3809 prg_attr.insns_cnt = ARRAY_SIZE(insns);
3810 prg_attr.license = "GPL";
3811
3812 ret = bpf_load_program_xattr(&prg_attr, NULL, 0);
3813 close(map);
3814 return probe_fd(ret);
3815 }
3816
3817 static int probe_kern_btf(void)
3818 {
3819 static const char strs[] = "\0int";
3820 __u32 types[] = {
3821 /* int */
3822 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),
3823 };
3824
3825 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
3826 strs, sizeof(strs)));
3827 }
3828
3829 static int probe_kern_btf_func(void)
3830 {
3831 static const char strs[] = "\0int\0x\0a";
3832 /* void x(int a) {} */
3833 __u32 types[] = {
3834 /* int */
3835 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
3836 /* FUNC_PROTO */ /* [2] */
3837 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0),
3838 BTF_PARAM_ENC(7, 1),
3839 /* FUNC x */ /* [3] */
3840 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), 2),
3841 };
3842
3843 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
3844 strs, sizeof(strs)));
3845 }
3846
3847 static int probe_kern_btf_func_global(void)
3848 {
3849 static const char strs[] = "\0int\0x\0a";
3850 /* static void x(int a) {} */
3851 __u32 types[] = {
3852 /* int */
3853 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
3854 /* FUNC_PROTO */ /* [2] */
3855 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0),
3856 BTF_PARAM_ENC(7, 1),
3857 /* FUNC x BTF_FUNC_GLOBAL */ /* [3] */
3858 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, BTF_FUNC_GLOBAL), 2),
3859 };
3860
3861 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
3862 strs, sizeof(strs)));
3863 }
3864
3865 static int probe_kern_btf_datasec(void)
3866 {
3867 static const char strs[] = "\0x\0.data";
3868 /* static int a; */
3869 __u32 types[] = {
3870 /* int */
3871 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
3872 /* VAR x */ /* [2] */
3873 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1),
3874 BTF_VAR_STATIC,
3875 /* DATASEC val */ /* [3] */
3876 BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4),
3877 BTF_VAR_SECINFO_ENC(2, 0, 4),
3878 };
3879
3880 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
3881 strs, sizeof(strs)));
3882 }
3883
3884 static int probe_kern_array_mmap(void)
3885 {
3886 struct bpf_create_map_attr attr = {
3887 .map_type = BPF_MAP_TYPE_ARRAY,
3888 .map_flags = BPF_F_MMAPABLE,
3889 .key_size = sizeof(int),
3890 .value_size = sizeof(int),
3891 .max_entries = 1,
3892 };
3893
3894 return probe_fd(bpf_create_map_xattr(&attr));
3895 }
3896
3897 static int probe_kern_exp_attach_type(void)
3898 {
3899 struct bpf_load_program_attr attr;
3900 struct bpf_insn insns[] = {
3901 BPF_MOV64_IMM(BPF_REG_0, 0),
3902 BPF_EXIT_INSN(),
3903 };
3904
3905 memset(&attr, 0, sizeof(attr));
3906 /* use any valid combination of program type and (optional)
3907 * non-zero expected attach type (i.e., not a BPF_CGROUP_INET_INGRESS)
3908 * to see if kernel supports expected_attach_type field for
3909 * BPF_PROG_LOAD command
3910 */
3911 attr.prog_type = BPF_PROG_TYPE_CGROUP_SOCK;
3912 attr.expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE;
3913 attr.insns = insns;
3914 attr.insns_cnt = ARRAY_SIZE(insns);
3915 attr.license = "GPL";
3916
3917 return probe_fd(bpf_load_program_xattr(&attr, NULL, 0));
3918 }
3919
3920 static int probe_kern_probe_read_kernel(void)
3921 {
3922 struct bpf_load_program_attr attr;
3923 struct bpf_insn insns[] = {
3924 BPF_MOV64_REG(BPF_REG_1, BPF_REG_10), /* r1 = r10 (fp) */
3925 BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8), /* r1 += -8 */
3926 BPF_MOV64_IMM(BPF_REG_2, 8), /* r2 = 8 */
3927 BPF_MOV64_IMM(BPF_REG_3, 0), /* r3 = 0 */
3928 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_probe_read_kernel),
3929 BPF_EXIT_INSN(),
3930 };
3931
3932 memset(&attr, 0, sizeof(attr));
3933 attr.prog_type = BPF_PROG_TYPE_KPROBE;
3934 attr.insns = insns;
3935 attr.insns_cnt = ARRAY_SIZE(insns);
3936 attr.license = "GPL";
3937
3938 return probe_fd(bpf_load_program_xattr(&attr, NULL, 0));
3939 }
3940
3941 static int probe_prog_bind_map(void)
3942 {
3943 struct bpf_load_program_attr prg_attr;
3944 struct bpf_create_map_attr map_attr;
3945 char *cp, errmsg[STRERR_BUFSIZE];
3946 struct bpf_insn insns[] = {
3947 BPF_MOV64_IMM(BPF_REG_0, 0),
3948 BPF_EXIT_INSN(),
3949 };
3950 int ret, map, prog;
3951
3952 memset(&map_attr, 0, sizeof(map_attr));
3953 map_attr.map_type = BPF_MAP_TYPE_ARRAY;
3954 map_attr.key_size = sizeof(int);
3955 map_attr.value_size = 32;
3956 map_attr.max_entries = 1;
3957
3958 map = bpf_create_map_xattr(&map_attr);
3959 if (map < 0) {
3960 ret = -errno;
3961 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
3962 pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n",
3963 __func__, cp, -ret);
3964 return ret;
3965 }
3966
3967 memset(&prg_attr, 0, sizeof(prg_attr));
3968 prg_attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
3969 prg_attr.insns = insns;
3970 prg_attr.insns_cnt = ARRAY_SIZE(insns);
3971 prg_attr.license = "GPL";
3972
3973 prog = bpf_load_program_xattr(&prg_attr, NULL, 0);
3974 if (prog < 0) {
3975 close(map);
3976 return 0;
3977 }
3978
3979 ret = bpf_prog_bind_map(prog, map, NULL);
3980
3981 close(map);
3982 close(prog);
3983
3984 return ret >= 0;
3985 }
3986
3987 static int probe_module_btf(void)
3988 {
3989 static const char strs[] = "\0int";
3990 __u32 types[] = {
3991 /* int */
3992 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),
3993 };
3994 struct bpf_btf_info info;
3995 __u32 len = sizeof(info);
3996 char name[16];
3997 int fd, err;
3998
3999 fd = libbpf__load_raw_btf((char *)types, sizeof(types), strs, sizeof(strs));
4000 if (fd < 0)
4001 return 0; /* BTF not supported at all */
4002
4003 memset(&info, 0, sizeof(info));
4004 info.name = ptr_to_u64(name);
4005 info.name_len = sizeof(name);
4006
4007 /* check that BPF_OBJ_GET_INFO_BY_FD supports specifying name pointer;
4008 * kernel's module BTF support coincides with support for
4009 * name/name_len fields in struct bpf_btf_info.
4010 */
4011 err = bpf_obj_get_info_by_fd(fd, &info, &len);
4012 close(fd);
4013 return !err;
4014 }
4015
4016 enum kern_feature_result {
4017 FEAT_UNKNOWN = 0,
4018 FEAT_SUPPORTED = 1,
4019 FEAT_MISSING = 2,
4020 };
4021
4022 typedef int (*feature_probe_fn)(void);
4023
4024 static struct kern_feature_desc {
4025 const char *desc;
4026 feature_probe_fn probe;
4027 enum kern_feature_result res;
4028 } feature_probes[__FEAT_CNT] = {
4029 [FEAT_PROG_NAME] = {
4030 "BPF program name", probe_kern_prog_name,
4031 },
4032 [FEAT_GLOBAL_DATA] = {
4033 "global variables", probe_kern_global_data,
4034 },
4035 [FEAT_BTF] = {
4036 "minimal BTF", probe_kern_btf,
4037 },
4038 [FEAT_BTF_FUNC] = {
4039 "BTF functions", probe_kern_btf_func,
4040 },
4041 [FEAT_BTF_GLOBAL_FUNC] = {
4042 "BTF global function", probe_kern_btf_func_global,
4043 },
4044 [FEAT_BTF_DATASEC] = {
4045 "BTF data section and variable", probe_kern_btf_datasec,
4046 },
4047 [FEAT_ARRAY_MMAP] = {
4048 "ARRAY map mmap()", probe_kern_array_mmap,
4049 },
4050 [FEAT_EXP_ATTACH_TYPE] = {
4051 "BPF_PROG_LOAD expected_attach_type attribute",
4052 probe_kern_exp_attach_type,
4053 },
4054 [FEAT_PROBE_READ_KERN] = {
4055 "bpf_probe_read_kernel() helper", probe_kern_probe_read_kernel,
4056 },
4057 [FEAT_PROG_BIND_MAP] = {
4058 "BPF_PROG_BIND_MAP support", probe_prog_bind_map,
4059 },
4060 [FEAT_MODULE_BTF] = {
4061 "module BTF support", probe_module_btf,
4062 },
4063 };
4064
4065 static bool kernel_supports(enum kern_feature_id feat_id)
4066 {
4067 struct kern_feature_desc *feat = &feature_probes[feat_id];
4068 int ret;
4069
4070 if (READ_ONCE(feat->res) == FEAT_UNKNOWN) {
4071 ret = feat->probe();
4072 if (ret > 0) {
4073 WRITE_ONCE(feat->res, FEAT_SUPPORTED);
4074 } else if (ret == 0) {
4075 WRITE_ONCE(feat->res, FEAT_MISSING);
4076 } else {
4077 pr_warn("Detection of kernel %s support failed: %d\n", feat->desc, ret);
4078 WRITE_ONCE(feat->res, FEAT_MISSING);
4079 }
4080 }
4081
4082 return READ_ONCE(feat->res) == FEAT_SUPPORTED;
4083 }
4084
4085 static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd)
4086 {
4087 struct bpf_map_info map_info = {};
4088 char msg[STRERR_BUFSIZE];
4089 __u32 map_info_len;
4090
4091 map_info_len = sizeof(map_info);
4092
4093 if (bpf_obj_get_info_by_fd(map_fd, &map_info, &map_info_len)) {
4094 pr_warn("failed to get map info for map FD %d: %s\n",
4095 map_fd, libbpf_strerror_r(errno, msg, sizeof(msg)));
4096 return false;
4097 }
4098
4099 return (map_info.type == map->def.type &&
4100 map_info.key_size == map->def.key_size &&
4101 map_info.value_size == map->def.value_size &&
4102 map_info.max_entries == map->def.max_entries &&
4103 map_info.map_flags == map->def.map_flags);
4104 }
4105
4106 static int
4107 bpf_object__reuse_map(struct bpf_map *map)
4108 {
4109 char *cp, errmsg[STRERR_BUFSIZE];
4110 int err, pin_fd;
4111
4112 pin_fd = bpf_obj_get(map->pin_path);
4113 if (pin_fd < 0) {
4114 err = -errno;
4115 if (err == -ENOENT) {
4116 pr_debug("found no pinned map to reuse at '%s'\n",
4117 map->pin_path);
4118 return 0;
4119 }
4120
4121 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
4122 pr_warn("couldn't retrieve pinned map '%s': %s\n",
4123 map->pin_path, cp);
4124 return err;
4125 }
4126
4127 if (!map_is_reuse_compat(map, pin_fd)) {
4128 pr_warn("couldn't reuse pinned map at '%s': parameter mismatch\n",
4129 map->pin_path);
4130 close(pin_fd);
4131 return -EINVAL;
4132 }
4133
4134 err = bpf_map__reuse_fd(map, pin_fd);
4135 if (err) {
4136 close(pin_fd);
4137 return err;
4138 }
4139 map->pinned = true;
4140 pr_debug("reused pinned map at '%s'\n", map->pin_path);
4141
4142 return 0;
4143 }
4144
4145 static int
4146 bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map)
4147 {
4148 enum libbpf_map_type map_type = map->libbpf_type;
4149 char *cp, errmsg[STRERR_BUFSIZE];
4150 int err, zero = 0;
4151
4152 err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0);
4153 if (err) {
4154 err = -errno;
4155 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
4156 pr_warn("Error setting initial map(%s) contents: %s\n",
4157 map->name, cp);
4158 return err;
4159 }
4160
4161 /* Freeze .rodata and .kconfig map as read-only from syscall side. */
4162 if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG) {
4163 err = bpf_map_freeze(map->fd);
4164 if (err) {
4165 err = -errno;
4166 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
4167 pr_warn("Error freezing map(%s) as read-only: %s\n",
4168 map->name, cp);
4169 return err;
4170 }
4171 }
4172 return 0;
4173 }
4174
4175 static void bpf_map__destroy(struct bpf_map *map);
4176
4177 static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map)
4178 {
4179 struct bpf_create_map_attr create_attr;
4180 struct bpf_map_def *def = &map->def;
4181
4182 memset(&create_attr, 0, sizeof(create_attr));
4183
4184 if (kernel_supports(FEAT_PROG_NAME))
4185 create_attr.name = map->name;
4186 create_attr.map_ifindex = map->map_ifindex;
4187 create_attr.map_type = def->type;
4188 create_attr.map_flags = def->map_flags;
4189 create_attr.key_size = def->key_size;
4190 create_attr.value_size = def->value_size;
4191 create_attr.numa_node = map->numa_node;
4192
4193 if (def->type == BPF_MAP_TYPE_PERF_EVENT_ARRAY && !def->max_entries) {
4194 int nr_cpus;
4195
4196 nr_cpus = libbpf_num_possible_cpus();
4197 if (nr_cpus < 0) {
4198 pr_warn("map '%s': failed to determine number of system CPUs: %d\n",
4199 map->name, nr_cpus);
4200 return nr_cpus;
4201 }
4202 pr_debug("map '%s': setting size to %d\n", map->name, nr_cpus);
4203 create_attr.max_entries = nr_cpus;
4204 } else {
4205 create_attr.max_entries = def->max_entries;
4206 }
4207
4208 if (bpf_map__is_struct_ops(map))
4209 create_attr.btf_vmlinux_value_type_id =
4210 map->btf_vmlinux_value_type_id;
4211
4212 create_attr.btf_fd = 0;
4213 create_attr.btf_key_type_id = 0;
4214 create_attr.btf_value_type_id = 0;
4215 if (obj->btf && btf__fd(obj->btf) >= 0 && !bpf_map_find_btf_info(obj, map)) {
4216 create_attr.btf_fd = btf__fd(obj->btf);
4217 create_attr.btf_key_type_id = map->btf_key_type_id;
4218 create_attr.btf_value_type_id = map->btf_value_type_id;
4219 }
4220
4221 if (bpf_map_type__is_map_in_map(def->type)) {
4222 if (map->inner_map) {
4223 int err;
4224
4225 err = bpf_object__create_map(obj, map->inner_map);
4226 if (err) {
4227 pr_warn("map '%s': failed to create inner map: %d\n",
4228 map->name, err);
4229 return err;
4230 }
4231 map->inner_map_fd = bpf_map__fd(map->inner_map);
4232 }
4233 if (map->inner_map_fd >= 0)
4234 create_attr.inner_map_fd = map->inner_map_fd;
4235 }
4236
4237 map->fd = bpf_create_map_xattr(&create_attr);
4238 if (map->fd < 0 && (create_attr.btf_key_type_id ||
4239 create_attr.btf_value_type_id)) {
4240 char *cp, errmsg[STRERR_BUFSIZE];
4241 int err = -errno;
4242
4243 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
4244 pr_warn("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n",
4245 map->name, cp, err);
4246 create_attr.btf_fd = 0;
4247 create_attr.btf_key_type_id = 0;
4248 create_attr.btf_value_type_id = 0;
4249 map->btf_key_type_id = 0;
4250 map->btf_value_type_id = 0;
4251 map->fd = bpf_create_map_xattr(&create_attr);
4252 }
4253
4254 if (map->fd < 0)
4255 return -errno;
4256
4257 if (bpf_map_type__is_map_in_map(def->type) && map->inner_map) {
4258 bpf_map__destroy(map->inner_map);
4259 zfree(&map->inner_map);
4260 }
4261
4262 return 0;
4263 }
4264
4265 static int init_map_slots(struct bpf_map *map)
4266 {
4267 const struct bpf_map *targ_map;
4268 unsigned int i;
4269 int fd, err;
4270
4271 for (i = 0; i < map->init_slots_sz; i++) {
4272 if (!map->init_slots[i])
4273 continue;
4274
4275 targ_map = map->init_slots[i];
4276 fd = bpf_map__fd(targ_map);
4277 err = bpf_map_update_elem(map->fd, &i, &fd, 0);
4278 if (err) {
4279 err = -errno;
4280 pr_warn("map '%s': failed to initialize slot [%d] to map '%s' fd=%d: %d\n",
4281 map->name, i, targ_map->name,
4282 fd, err);
4283 return err;
4284 }
4285 pr_debug("map '%s': slot [%d] set to map '%s' fd=%d\n",
4286 map->name, i, targ_map->name, fd);
4287 }
4288
4289 zfree(&map->init_slots);
4290 map->init_slots_sz = 0;
4291
4292 return 0;
4293 }
4294
4295 static int
4296 bpf_object__create_maps(struct bpf_object *obj)
4297 {
4298 struct bpf_map *map;
4299 char *cp, errmsg[STRERR_BUFSIZE];
4300 unsigned int i, j;
4301 int err;
4302
4303 for (i = 0; i < obj->nr_maps; i++) {
4304 map = &obj->maps[i];
4305
4306 if (map->pin_path) {
4307 err = bpf_object__reuse_map(map);
4308 if (err) {
4309 pr_warn("map '%s': error reusing pinned map\n",
4310 map->name);
4311 goto err_out;
4312 }
4313 }
4314
4315 if (map->fd >= 0) {
4316 pr_debug("map '%s': skipping creation (preset fd=%d)\n",
4317 map->name, map->fd);
4318 } else {
4319 err = bpf_object__create_map(obj, map);
4320 if (err)
4321 goto err_out;
4322
4323 pr_debug("map '%s': created successfully, fd=%d\n",
4324 map->name, map->fd);
4325
4326 if (bpf_map__is_internal(map)) {
4327 err = bpf_object__populate_internal_map(obj, map);
4328 if (err < 0) {
4329 zclose(map->fd);
4330 goto err_out;
4331 }
4332 }
4333
4334 if (map->init_slots_sz) {
4335 err = init_map_slots(map);
4336 if (err < 0) {
4337 zclose(map->fd);
4338 goto err_out;
4339 }
4340 }
4341 }
4342
4343 if (map->pin_path && !map->pinned) {
4344 err = bpf_map__pin(map, NULL);
4345 if (err) {
4346 pr_warn("map '%s': failed to auto-pin at '%s': %d\n",
4347 map->name, map->pin_path, err);
4348 zclose(map->fd);
4349 goto err_out;
4350 }
4351 }
4352 }
4353
4354 return 0;
4355
4356 err_out:
4357 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
4358 pr_warn("map '%s': failed to create: %s(%d)\n", map->name, cp, err);
4359 pr_perm_msg(err);
4360 for (j = 0; j < i; j++)
4361 zclose(obj->maps[j].fd);
4362 return err;
4363 }
4364
4365 #define BPF_CORE_SPEC_MAX_LEN 64
4366
4367 /* represents BPF CO-RE field or array element accessor */
4368 struct bpf_core_accessor {
4369 __u32 type_id; /* struct/union type or array element type */
4370 __u32 idx; /* field index or array index */
4371 const char *name; /* field name or NULL for array accessor */
4372 };
4373
4374 struct bpf_core_spec {
4375 const struct btf *btf;
4376 /* high-level spec: named fields and array indices only */
4377 struct bpf_core_accessor spec[BPF_CORE_SPEC_MAX_LEN];
4378 /* original unresolved (no skip_mods_or_typedefs) root type ID */
4379 __u32 root_type_id;
4380 /* CO-RE relocation kind */
4381 enum bpf_core_relo_kind relo_kind;
4382 /* high-level spec length */
4383 int len;
4384 /* raw, low-level spec: 1-to-1 with accessor spec string */
4385 int raw_spec[BPF_CORE_SPEC_MAX_LEN];
4386 /* raw spec length */
4387 int raw_len;
4388 /* field bit offset represented by spec */
4389 __u32 bit_offset;
4390 };
4391
4392 static bool str_is_empty(const char *s)
4393 {
4394 return !s || !s[0];
4395 }
4396
4397 static bool is_flex_arr(const struct btf *btf,
4398 const struct bpf_core_accessor *acc,
4399 const struct btf_array *arr)
4400 {
4401 const struct btf_type *t;
4402
4403 /* not a flexible array, if not inside a struct or has non-zero size */
4404 if (!acc->name || arr->nelems > 0)
4405 return false;
4406
4407 /* has to be the last member of enclosing struct */
4408 t = btf__type_by_id(btf, acc->type_id);
4409 return acc->idx == btf_vlen(t) - 1;
4410 }
4411
4412 static const char *core_relo_kind_str(enum bpf_core_relo_kind kind)
4413 {
4414 switch (kind) {
4415 case BPF_FIELD_BYTE_OFFSET: return "byte_off";
4416 case BPF_FIELD_BYTE_SIZE: return "byte_sz";
4417 case BPF_FIELD_EXISTS: return "field_exists";
4418 case BPF_FIELD_SIGNED: return "signed";
4419 case BPF_FIELD_LSHIFT_U64: return "lshift_u64";
4420 case BPF_FIELD_RSHIFT_U64: return "rshift_u64";
4421 case BPF_TYPE_ID_LOCAL: return "local_type_id";
4422 case BPF_TYPE_ID_TARGET: return "target_type_id";
4423 case BPF_TYPE_EXISTS: return "type_exists";
4424 case BPF_TYPE_SIZE: return "type_size";
4425 case BPF_ENUMVAL_EXISTS: return "enumval_exists";
4426 case BPF_ENUMVAL_VALUE: return "enumval_value";
4427 default: return "unknown";
4428 }
4429 }
4430
4431 static bool core_relo_is_field_based(enum bpf_core_relo_kind kind)
4432 {
4433 switch (kind) {
4434 case BPF_FIELD_BYTE_OFFSET:
4435 case BPF_FIELD_BYTE_SIZE:
4436 case BPF_FIELD_EXISTS:
4437 case BPF_FIELD_SIGNED:
4438 case BPF_FIELD_LSHIFT_U64:
4439 case BPF_FIELD_RSHIFT_U64:
4440 return true;
4441 default:
4442 return false;
4443 }
4444 }
4445
4446 static bool core_relo_is_type_based(enum bpf_core_relo_kind kind)
4447 {
4448 switch (kind) {
4449 case BPF_TYPE_ID_LOCAL:
4450 case BPF_TYPE_ID_TARGET:
4451 case BPF_TYPE_EXISTS:
4452 case BPF_TYPE_SIZE:
4453 return true;
4454 default:
4455 return false;
4456 }
4457 }
4458
4459 static bool core_relo_is_enumval_based(enum bpf_core_relo_kind kind)
4460 {
4461 switch (kind) {
4462 case BPF_ENUMVAL_EXISTS:
4463 case BPF_ENUMVAL_VALUE:
4464 return true;
4465 default:
4466 return false;
4467 }
4468 }
4469
4470 /*
4471 * Turn bpf_core_relo into a low- and high-level spec representation,
4472 * validating correctness along the way, as well as calculating resulting
4473 * field bit offset, specified by accessor string. Low-level spec captures
4474 * every single level of nestedness, including traversing anonymous
4475 * struct/union members. High-level one only captures semantically meaningful
4476 * "turning points": named fields and array indicies.
4477 * E.g., for this case:
4478 *
4479 * struct sample {
4480 * int __unimportant;
4481 * struct {
4482 * int __1;
4483 * int __2;
4484 * int a[7];
4485 * };
4486 * };
4487 *
4488 * struct sample *s = ...;
4489 *
4490 * int x = &s->a[3]; // access string = '0:1:2:3'
4491 *
4492 * Low-level spec has 1:1 mapping with each element of access string (it's
4493 * just a parsed access string representation): [0, 1, 2, 3].
4494 *
4495 * High-level spec will capture only 3 points:
4496 * - intial zero-index access by pointer (&s->... is the same as &s[0]...);
4497 * - field 'a' access (corresponds to '2' in low-level spec);
4498 * - array element #3 access (corresponds to '3' in low-level spec).
4499 *
4500 * Type-based relocations (TYPE_EXISTS/TYPE_SIZE,
4501 * TYPE_ID_LOCAL/TYPE_ID_TARGET) don't capture any field information. Their
4502 * spec and raw_spec are kept empty.
4503 *
4504 * Enum value-based relocations (ENUMVAL_EXISTS/ENUMVAL_VALUE) use access
4505 * string to specify enumerator's value index that need to be relocated.
4506 */
4507 static int bpf_core_parse_spec(const struct btf *btf,
4508 __u32 type_id,
4509 const char *spec_str,
4510 enum bpf_core_relo_kind relo_kind,
4511 struct bpf_core_spec *spec)
4512 {
4513 int access_idx, parsed_len, i;
4514 struct bpf_core_accessor *acc;
4515 const struct btf_type *t;
4516 const char *name;
4517 __u32 id;
4518 __s64 sz;
4519
4520 if (str_is_empty(spec_str) || *spec_str == ':')
4521 return -EINVAL;
4522
4523 memset(spec, 0, sizeof(*spec));
4524 spec->btf = btf;
4525 spec->root_type_id = type_id;
4526 spec->relo_kind = relo_kind;
4527
4528 /* type-based relocations don't have a field access string */
4529 if (core_relo_is_type_based(relo_kind)) {
4530 if (strcmp(spec_str, "0"))
4531 return -EINVAL;
4532 return 0;
4533 }
4534
4535 /* parse spec_str="0:1:2:3:4" into array raw_spec=[0, 1, 2, 3, 4] */
4536 while (*spec_str) {
4537 if (*spec_str == ':')
4538 ++spec_str;
4539 if (sscanf(spec_str, "%d%n", &access_idx, &parsed_len) != 1)
4540 return -EINVAL;
4541 if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN)
4542 return -E2BIG;
4543 spec_str += parsed_len;
4544 spec->raw_spec[spec->raw_len++] = access_idx;
4545 }
4546
4547 if (spec->raw_len == 0)
4548 return -EINVAL;
4549
4550 t = skip_mods_and_typedefs(btf, type_id, &id);
4551 if (!t)
4552 return -EINVAL;
4553
4554 access_idx = spec->raw_spec[0];
4555 acc = &spec->spec[0];
4556 acc->type_id = id;
4557 acc->idx = access_idx;
4558 spec->len++;
4559
4560 if (core_relo_is_enumval_based(relo_kind)) {
4561 if (!btf_is_enum(t) || spec->raw_len > 1 || access_idx >= btf_vlen(t))
4562 return -EINVAL;
4563
4564 /* record enumerator name in a first accessor */
4565 acc->name = btf__name_by_offset(btf, btf_enum(t)[access_idx].name_off);
4566 return 0;
4567 }
4568
4569 if (!core_relo_is_field_based(relo_kind))
4570 return -EINVAL;
4571
4572 sz = btf__resolve_size(btf, id);
4573 if (sz < 0)
4574 return sz;
4575 spec->bit_offset = access_idx * sz * 8;
4576
4577 for (i = 1; i < spec->raw_len; i++) {
4578 t = skip_mods_and_typedefs(btf, id, &id);
4579 if (!t)
4580 return -EINVAL;
4581
4582 access_idx = spec->raw_spec[i];
4583 acc = &spec->spec[spec->len];
4584
4585 if (btf_is_composite(t)) {
4586 const struct btf_member *m;
4587 __u32 bit_offset;
4588
4589 if (access_idx >= btf_vlen(t))
4590 return -EINVAL;
4591
4592 bit_offset = btf_member_bit_offset(t, access_idx);
4593 spec->bit_offset += bit_offset;
4594
4595 m = btf_members(t) + access_idx;
4596 if (m->name_off) {
4597 name = btf__name_by_offset(btf, m->name_off);
4598 if (str_is_empty(name))
4599 return -EINVAL;
4600
4601 acc->type_id = id;
4602 acc->idx = access_idx;
4603 acc->name = name;
4604 spec->len++;
4605 }
4606
4607 id = m->type;
4608 } else if (btf_is_array(t)) {
4609 const struct btf_array *a = btf_array(t);
4610 bool flex;
4611
4612 t = skip_mods_and_typedefs(btf, a->type, &id);
4613 if (!t)
4614 return -EINVAL;
4615
4616 flex = is_flex_arr(btf, acc - 1, a);
4617 if (!flex && access_idx >= a->nelems)
4618 return -EINVAL;
4619
4620 spec->spec[spec->len].type_id = id;
4621 spec->spec[spec->len].idx = access_idx;
4622 spec->len++;
4623
4624 sz = btf__resolve_size(btf, id);
4625 if (sz < 0)
4626 return sz;
4627 spec->bit_offset += access_idx * sz * 8;
4628 } else {
4629 pr_warn("relo for [%u] %s (at idx %d) captures type [%d] of unexpected kind %s\n",
4630 type_id, spec_str, i, id, btf_kind_str(t));
4631 return -EINVAL;
4632 }
4633 }
4634
4635 return 0;
4636 }
4637
4638 static bool bpf_core_is_flavor_sep(const char *s)
4639 {
4640 /* check X___Y name pattern, where X and Y are not underscores */
4641 return s[0] != '_' && /* X */
4642 s[1] == '_' && s[2] == '_' && s[3] == '_' && /* ___ */
4643 s[4] != '_'; /* Y */
4644 }
4645
4646 /* Given 'some_struct_name___with_flavor' return the length of a name prefix
4647 * before last triple underscore. Struct name part after last triple
4648 * underscore is ignored by BPF CO-RE relocation during relocation matching.
4649 */
4650 static size_t bpf_core_essential_name_len(const char *name)
4651 {
4652 size_t n = strlen(name);
4653 int i;
4654
4655 for (i = n - 5; i >= 0; i--) {
4656 if (bpf_core_is_flavor_sep(name + i))
4657 return i + 1;
4658 }
4659 return n;
4660 }
4661
4662 struct core_cand
4663 {
4664 const struct btf *btf;
4665 const struct btf_type *t;
4666 const char *name;
4667 __u32 id;
4668 };
4669
4670 /* dynamically sized list of type IDs and its associated struct btf */
4671 struct core_cand_list {
4672 struct core_cand *cands;
4673 int len;
4674 };
4675
4676 static void bpf_core_free_cands(struct core_cand_list *cands)
4677 {
4678 free(cands->cands);
4679 free(cands);
4680 }
4681
4682 static int bpf_core_add_cands(struct core_cand *local_cand,
4683 size_t local_essent_len,
4684 const struct btf *targ_btf,
4685 const char *targ_btf_name,
4686 int targ_start_id,
4687 struct core_cand_list *cands)
4688 {
4689 struct core_cand *new_cands, *cand;
4690 const struct btf_type *t;
4691 const char *targ_name;
4692 size_t targ_essent_len;
4693 int n, i;
4694
4695 n = btf__get_nr_types(targ_btf);
4696 for (i = targ_start_id; i <= n; i++) {
4697 t = btf__type_by_id(targ_btf, i);
4698 if (btf_kind(t) != btf_kind(local_cand->t))
4699 continue;
4700
4701 targ_name = btf__name_by_offset(targ_btf, t->name_off);
4702 if (str_is_empty(targ_name))
4703 continue;
4704
4705 targ_essent_len = bpf_core_essential_name_len(targ_name);
4706 if (targ_essent_len != local_essent_len)
4707 continue;
4708
4709 if (strncmp(local_cand->name, targ_name, local_essent_len) != 0)
4710 continue;
4711
4712 pr_debug("CO-RE relocating [%d] %s %s: found target candidate [%d] %s %s in [%s]\n",
4713 local_cand->id, btf_kind_str(local_cand->t),
4714 local_cand->name, i, btf_kind_str(t), targ_name,
4715 targ_btf_name);
4716 new_cands = libbpf_reallocarray(cands->cands, cands->len + 1,
4717 sizeof(*cands->cands));
4718 if (!new_cands)
4719 return -ENOMEM;
4720
4721 cand = &new_cands[cands->len];
4722 cand->btf = targ_btf;
4723 cand->t = t;
4724 cand->name = targ_name;
4725 cand->id = i;
4726
4727 cands->cands = new_cands;
4728 cands->len++;
4729 }
4730 return 0;
4731 }
4732
4733 static int load_module_btfs(struct bpf_object *obj)
4734 {
4735 struct bpf_btf_info info;
4736 struct module_btf *mod_btf;
4737 struct btf *btf;
4738 char name[64];
4739 __u32 id = 0, len;
4740 int err, fd;
4741
4742 if (obj->btf_modules_loaded)
4743 return 0;
4744
4745 /* don't do this again, even if we find no module BTFs */
4746 obj->btf_modules_loaded = true;
4747
4748 /* kernel too old to support module BTFs */
4749 if (!kernel_supports(FEAT_MODULE_BTF))
4750 return 0;
4751
4752 while (true) {
4753 err = bpf_btf_get_next_id(id, &id);
4754 if (err && errno == ENOENT)
4755 return 0;
4756 if (err) {
4757 err = -errno;
4758 pr_warn("failed to iterate BTF objects: %d\n", err);
4759 return err;
4760 }
4761
4762 fd = bpf_btf_get_fd_by_id(id);
4763 if (fd < 0) {
4764 if (errno == ENOENT)
4765 continue; /* expected race: BTF was unloaded */
4766 err = -errno;
4767 pr_warn("failed to get BTF object #%d FD: %d\n", id, err);
4768 return err;
4769 }
4770
4771 len = sizeof(info);
4772 memset(&info, 0, sizeof(info));
4773 info.name = ptr_to_u64(name);
4774 info.name_len = sizeof(name);
4775
4776 err = bpf_obj_get_info_by_fd(fd, &info, &len);
4777 if (err) {
4778 err = -errno;
4779 pr_warn("failed to get BTF object #%d info: %d\n", id, err);
4780 goto err_out;
4781 }
4782
4783 /* ignore non-module BTFs */
4784 if (!info.kernel_btf || strcmp(name, "vmlinux") == 0) {
4785 close(fd);
4786 continue;
4787 }
4788
4789 btf = btf_get_from_fd(fd, obj->btf_vmlinux);
4790 if (IS_ERR(btf)) {
4791 pr_warn("failed to load module [%s]'s BTF object #%d: %ld\n",
4792 name, id, PTR_ERR(btf));
4793 err = PTR_ERR(btf);
4794 goto err_out;
4795 }
4796
4797 err = btf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap,
4798 sizeof(*obj->btf_modules), obj->btf_module_cnt + 1);
4799 if (err)
4800 goto err_out;
4801
4802 mod_btf = &obj->btf_modules[obj->btf_module_cnt++];
4803
4804 mod_btf->btf = btf;
4805 mod_btf->id = id;
4806 mod_btf->fd = fd;
4807 mod_btf->name = strdup(name);
4808 if (!mod_btf->name) {
4809 err = -ENOMEM;
4810 goto err_out;
4811 }
4812 continue;
4813
4814 err_out:
4815 close(fd);
4816 return err;
4817 }
4818
4819 return 0;
4820 }
4821
4822 static struct core_cand_list *
4823 bpf_core_find_cands(struct bpf_object *obj, const struct btf *local_btf, __u32 local_type_id)
4824 {
4825 struct core_cand local_cand = {};
4826 struct core_cand_list *cands;
4827 const struct btf *main_btf;
4828 size_t local_essent_len;
4829 int err, i;
4830
4831 local_cand.btf = local_btf;
4832 local_cand.t = btf__type_by_id(local_btf, local_type_id);
4833 if (!local_cand.t)
4834 return ERR_PTR(-EINVAL);
4835
4836 local_cand.name = btf__name_by_offset(local_btf, local_cand.t->name_off);
4837 if (str_is_empty(local_cand.name))
4838 return ERR_PTR(-EINVAL);
4839 local_essent_len = bpf_core_essential_name_len(local_cand.name);
4840
4841 cands = calloc(1, sizeof(*cands));
4842 if (!cands)
4843 return ERR_PTR(-ENOMEM);
4844
4845 /* Attempt to find target candidates in vmlinux BTF first */
4846 main_btf = obj->btf_vmlinux_override ?: obj->btf_vmlinux;
4847 err = bpf_core_add_cands(&local_cand, local_essent_len, main_btf, "vmlinux", 1, cands);
4848 if (err)
4849 goto err_out;
4850
4851 /* if vmlinux BTF has any candidate, don't got for module BTFs */
4852 if (cands->len)
4853 return cands;
4854
4855 /* if vmlinux BTF was overridden, don't attempt to load module BTFs */
4856 if (obj->btf_vmlinux_override)
4857 return cands;
4858
4859 /* now look through module BTFs, trying to still find candidates */
4860 err = load_module_btfs(obj);
4861 if (err)
4862 goto err_out;
4863
4864 for (i = 0; i < obj->btf_module_cnt; i++) {
4865 err = bpf_core_add_cands(&local_cand, local_essent_len,
4866 obj->btf_modules[i].btf,
4867 obj->btf_modules[i].name,
4868 btf__get_nr_types(obj->btf_vmlinux) + 1,
4869 cands);
4870 if (err)
4871 goto err_out;
4872 }
4873
4874 return cands;
4875 err_out:
4876 bpf_core_free_cands(cands);
4877 return ERR_PTR(err);
4878 }
4879
4880 /* Check two types for compatibility for the purpose of field access
4881 * relocation. const/volatile/restrict and typedefs are skipped to ensure we
4882 * are relocating semantically compatible entities:
4883 * - any two STRUCTs/UNIONs are compatible and can be mixed;
4884 * - any two FWDs are compatible, if their names match (modulo flavor suffix);
4885 * - any two PTRs are always compatible;
4886 * - for ENUMs, names should be the same (ignoring flavor suffix) or at
4887 * least one of enums should be anonymous;
4888 * - for ENUMs, check sizes, names are ignored;
4889 * - for INT, size and signedness are ignored;
4890 * - for ARRAY, dimensionality is ignored, element types are checked for
4891 * compatibility recursively;
4892 * - everything else shouldn't be ever a target of relocation.
4893 * These rules are not set in stone and probably will be adjusted as we get
4894 * more experience with using BPF CO-RE relocations.
4895 */
4896 static int bpf_core_fields_are_compat(const struct btf *local_btf,
4897 __u32 local_id,
4898 const struct btf *targ_btf,
4899 __u32 targ_id)
4900 {
4901 const struct btf_type *local_type, *targ_type;
4902
4903 recur:
4904 local_type = skip_mods_and_typedefs(local_btf, local_id, &local_id);
4905 targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id);
4906 if (!local_type || !targ_type)
4907 return -EINVAL;
4908
4909 if (btf_is_composite(local_type) && btf_is_composite(targ_type))
4910 return 1;
4911 if (btf_kind(local_type) != btf_kind(targ_type))
4912 return 0;
4913
4914 switch (btf_kind(local_type)) {
4915 case BTF_KIND_PTR:
4916 return 1;
4917 case BTF_KIND_FWD:
4918 case BTF_KIND_ENUM: {
4919 const char *local_name, *targ_name;
4920 size_t local_len, targ_len;
4921
4922 local_name = btf__name_by_offset(local_btf,
4923 local_type->name_off);
4924 targ_name = btf__name_by_offset(targ_btf, targ_type->name_off);
4925 local_len = bpf_core_essential_name_len(local_name);
4926 targ_len = bpf_core_essential_name_len(targ_name);
4927 /* one of them is anonymous or both w/ same flavor-less names */
4928 return local_len == 0 || targ_len == 0 ||
4929 (local_len == targ_len &&
4930 strncmp(local_name, targ_name, local_len) == 0);
4931 }
4932 case BTF_KIND_INT:
4933 /* just reject deprecated bitfield-like integers; all other
4934 * integers are by default compatible between each other
4935 */
4936 return btf_int_offset(local_type) == 0 &&
4937 btf_int_offset(targ_type) == 0;
4938 case BTF_KIND_ARRAY:
4939 local_id = btf_array(local_type)->type;
4940 targ_id = btf_array(targ_type)->type;
4941 goto recur;
4942 default:
4943 pr_warn("unexpected kind %d relocated, local [%d], target [%d]\n",
4944 btf_kind(local_type), local_id, targ_id);
4945 return 0;
4946 }
4947 }
4948
4949 /*
4950 * Given single high-level named field accessor in local type, find
4951 * corresponding high-level accessor for a target type. Along the way,
4952 * maintain low-level spec for target as well. Also keep updating target
4953 * bit offset.
4954 *
4955 * Searching is performed through recursive exhaustive enumeration of all
4956 * fields of a struct/union. If there are any anonymous (embedded)
4957 * structs/unions, they are recursively searched as well. If field with
4958 * desired name is found, check compatibility between local and target types,
4959 * before returning result.
4960 *
4961 * 1 is returned, if field is found.
4962 * 0 is returned if no compatible field is found.
4963 * <0 is returned on error.
4964 */
4965 static int bpf_core_match_member(const struct btf *local_btf,
4966 const struct bpf_core_accessor *local_acc,
4967 const struct btf *targ_btf,
4968 __u32 targ_id,
4969 struct bpf_core_spec *spec,
4970 __u32 *next_targ_id)
4971 {
4972 const struct btf_type *local_type, *targ_type;
4973 const struct btf_member *local_member, *m;
4974 const char *local_name, *targ_name;
4975 __u32 local_id;
4976 int i, n, found;
4977
4978 targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id);
4979 if (!targ_type)
4980 return -EINVAL;
4981 if (!btf_is_composite(targ_type))
4982 return 0;
4983
4984 local_id = local_acc->type_id;
4985 local_type = btf__type_by_id(local_btf, local_id);
4986 local_member = btf_members(local_type) + local_acc->idx;
4987 local_name = btf__name_by_offset(local_btf, local_member->name_off);
4988
4989 n = btf_vlen(targ_type);
4990 m = btf_members(targ_type);
4991 for (i = 0; i < n; i++, m++) {
4992 __u32 bit_offset;
4993
4994 bit_offset = btf_member_bit_offset(targ_type, i);
4995
4996 /* too deep struct/union/array nesting */
4997 if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN)
4998 return -E2BIG;
4999
5000 /* speculate this member will be the good one */
5001 spec->bit_offset += bit_offset;
5002 spec->raw_spec[spec->raw_len++] = i;
5003
5004 targ_name = btf__name_by_offset(targ_btf, m->name_off);
5005 if (str_is_empty(targ_name)) {
5006 /* embedded struct/union, we need to go deeper */
5007 found = bpf_core_match_member(local_btf, local_acc,
5008 targ_btf, m->type,
5009 spec, next_targ_id);
5010 if (found) /* either found or error */
5011 return found;
5012 } else if (strcmp(local_name, targ_name) == 0) {
5013 /* matching named field */
5014 struct bpf_core_accessor *targ_acc;
5015
5016 targ_acc = &spec->spec[spec->len++];
5017 targ_acc->type_id = targ_id;
5018 targ_acc->idx = i;
5019 targ_acc->name = targ_name;
5020
5021 *next_targ_id = m->type;
5022 found = bpf_core_fields_are_compat(local_btf,
5023 local_member->type,
5024 targ_btf, m->type);
5025 if (!found)
5026 spec->len--; /* pop accessor */
5027 return found;
5028 }
5029 /* member turned out not to be what we looked for */
5030 spec->bit_offset -= bit_offset;
5031 spec->raw_len--;
5032 }
5033
5034 return 0;
5035 }
5036
5037 /* Check local and target types for compatibility. This check is used for
5038 * type-based CO-RE relocations and follow slightly different rules than
5039 * field-based relocations. This function assumes that root types were already
5040 * checked for name match. Beyond that initial root-level name check, names
5041 * are completely ignored. Compatibility rules are as follows:
5042 * - any two STRUCTs/UNIONs/FWDs/ENUMs/INTs are considered compatible, but
5043 * kind should match for local and target types (i.e., STRUCT is not
5044 * compatible with UNION);
5045 * - for ENUMs, the size is ignored;
5046 * - for INT, size and signedness are ignored;
5047 * - for ARRAY, dimensionality is ignored, element types are checked for
5048 * compatibility recursively;
5049 * - CONST/VOLATILE/RESTRICT modifiers are ignored;
5050 * - TYPEDEFs/PTRs are compatible if types they pointing to are compatible;
5051 * - FUNC_PROTOs are compatible if they have compatible signature: same
5052 * number of input args and compatible return and argument types.
5053 * These rules are not set in stone and probably will be adjusted as we get
5054 * more experience with using BPF CO-RE relocations.
5055 */
5056 static int bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id,
5057 const struct btf *targ_btf, __u32 targ_id)
5058 {
5059 const struct btf_type *local_type, *targ_type;
5060 int depth = 32; /* max recursion depth */
5061
5062 /* caller made sure that names match (ignoring flavor suffix) */
5063 local_type = btf__type_by_id(local_btf, local_id);
5064 targ_type = btf__type_by_id(targ_btf, targ_id);
5065 if (btf_kind(local_type) != btf_kind(targ_type))
5066 return 0;
5067
5068 recur:
5069 depth--;
5070 if (depth < 0)
5071 return -EINVAL;
5072
5073 local_type = skip_mods_and_typedefs(local_btf, local_id, &local_id);
5074 targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id);
5075 if (!local_type || !targ_type)
5076 return -EINVAL;
5077
5078 if (btf_kind(local_type) != btf_kind(targ_type))
5079 return 0;
5080
5081 switch (btf_kind(local_type)) {
5082 case BTF_KIND_UNKN:
5083 case BTF_KIND_STRUCT:
5084 case BTF_KIND_UNION:
5085 case BTF_KIND_ENUM:
5086 case BTF_KIND_FWD:
5087 return 1;
5088 case BTF_KIND_INT:
5089 /* just reject deprecated bitfield-like integers; all other
5090 * integers are by default compatible between each other
5091 */
5092 return btf_int_offset(local_type) == 0 && btf_int_offset(targ_type) == 0;
5093 case BTF_KIND_PTR:
5094 local_id = local_type->type;
5095 targ_id = targ_type->type;
5096 goto recur;
5097 case BTF_KIND_ARRAY:
5098 local_id = btf_array(local_type)->type;
5099 targ_id = btf_array(targ_type)->type;
5100 goto recur;
5101 case BTF_KIND_FUNC_PROTO: {
5102 struct btf_param *local_p = btf_params(local_type);
5103 struct btf_param *targ_p = btf_params(targ_type);
5104 __u16 local_vlen = btf_vlen(local_type);
5105 __u16 targ_vlen = btf_vlen(targ_type);
5106 int i, err;
5107
5108 if (local_vlen != targ_vlen)
5109 return 0;
5110
5111 for (i = 0; i < local_vlen; i++, local_p++, targ_p++) {
5112 skip_mods_and_typedefs(local_btf, local_p->type, &local_id);
5113 skip_mods_and_typedefs(targ_btf, targ_p->type, &targ_id);
5114 err = bpf_core_types_are_compat(local_btf, local_id, targ_btf, targ_id);
5115 if (err <= 0)
5116 return err;
5117 }
5118
5119 /* tail recurse for return type check */
5120 skip_mods_and_typedefs(local_btf, local_type->type, &local_id);
5121 skip_mods_and_typedefs(targ_btf, targ_type->type, &targ_id);
5122 goto recur;
5123 }
5124 default:
5125 pr_warn("unexpected kind %s relocated, local [%d], target [%d]\n",
5126 btf_kind_str(local_type), local_id, targ_id);
5127 return 0;
5128 }
5129 }
5130
5131 /*
5132 * Try to match local spec to a target type and, if successful, produce full
5133 * target spec (high-level, low-level + bit offset).
5134 */
5135 static int bpf_core_spec_match(struct bpf_core_spec *local_spec,
5136 const struct btf *targ_btf, __u32 targ_id,
5137 struct bpf_core_spec *targ_spec)
5138 {
5139 const struct btf_type *targ_type;
5140 const struct bpf_core_accessor *local_acc;
5141 struct bpf_core_accessor *targ_acc;
5142 int i, sz, matched;
5143
5144 memset(targ_spec, 0, sizeof(*targ_spec));
5145 targ_spec->btf = targ_btf;
5146 targ_spec->root_type_id = targ_id;
5147 targ_spec->relo_kind = local_spec->relo_kind;
5148
5149 if (core_relo_is_type_based(local_spec->relo_kind)) {
5150 return bpf_core_types_are_compat(local_spec->btf,
5151 local_spec->root_type_id,
5152 targ_btf, targ_id);
5153 }
5154
5155 local_acc = &local_spec->spec[0];
5156 targ_acc = &targ_spec->spec[0];
5157
5158 if (core_relo_is_enumval_based(local_spec->relo_kind)) {
5159 size_t local_essent_len, targ_essent_len;
5160 const struct btf_enum *e;
5161 const char *targ_name;
5162
5163 /* has to resolve to an enum */
5164 targ_type = skip_mods_and_typedefs(targ_spec->btf, targ_id, &targ_id);
5165 if (!btf_is_enum(targ_type))
5166 return 0;
5167
5168 local_essent_len = bpf_core_essential_name_len(local_acc->name);
5169
5170 for (i = 0, e = btf_enum(targ_type); i < btf_vlen(targ_type); i++, e++) {
5171 targ_name = btf__name_by_offset(targ_spec->btf, e->name_off);
5172 targ_essent_len = bpf_core_essential_name_len(targ_name);
5173 if (targ_essent_len != local_essent_len)
5174 continue;
5175 if (strncmp(local_acc->name, targ_name, local_essent_len) == 0) {
5176 targ_acc->type_id = targ_id;
5177 targ_acc->idx = i;
5178 targ_acc->name = targ_name;
5179 targ_spec->len++;
5180 targ_spec->raw_spec[targ_spec->raw_len] = targ_acc->idx;
5181 targ_spec->raw_len++;
5182 return 1;
5183 }
5184 }
5185 return 0;
5186 }
5187
5188 if (!core_relo_is_field_based(local_spec->relo_kind))
5189 return -EINVAL;
5190
5191 for (i = 0; i < local_spec->len; i++, local_acc++, targ_acc++) {
5192 targ_type = skip_mods_and_typedefs(targ_spec->btf, targ_id,
5193 &targ_id);
5194 if (!targ_type)
5195 return -EINVAL;
5196
5197 if (local_acc->name) {
5198 matched = bpf_core_match_member(local_spec->btf,
5199 local_acc,
5200 targ_btf, targ_id,
5201 targ_spec, &targ_id);
5202 if (matched <= 0)
5203 return matched;
5204 } else {
5205 /* for i=0, targ_id is already treated as array element
5206 * type (because it's the original struct), for others
5207 * we should find array element type first
5208 */
5209 if (i > 0) {
5210 const struct btf_array *a;
5211 bool flex;
5212
5213 if (!btf_is_array(targ_type))
5214 return 0;
5215
5216 a = btf_array(targ_type);
5217 flex = is_flex_arr(targ_btf, targ_acc - 1, a);
5218 if (!flex && local_acc->idx >= a->nelems)
5219 return 0;
5220 if (!skip_mods_and_typedefs(targ_btf, a->type,
5221 &targ_id))
5222 return -EINVAL;
5223 }
5224
5225 /* too deep struct/union/array nesting */
5226 if (targ_spec->raw_len == BPF_CORE_SPEC_MAX_LEN)
5227 return -E2BIG;
5228
5229 targ_acc->type_id = targ_id;
5230 targ_acc->idx = local_acc->idx;
5231 targ_acc->name = NULL;
5232 targ_spec->len++;
5233 targ_spec->raw_spec[targ_spec->raw_len] = targ_acc->idx;
5234 targ_spec->raw_len++;
5235
5236 sz = btf__resolve_size(targ_btf, targ_id);
5237 if (sz < 0)
5238 return sz;
5239 targ_spec->bit_offset += local_acc->idx * sz * 8;
5240 }
5241 }
5242
5243 return 1;
5244 }
5245
5246 static int bpf_core_calc_field_relo(const struct bpf_program *prog,
5247 const struct bpf_core_relo *relo,
5248 const struct bpf_core_spec *spec,
5249 __u32 *val, __u32 *field_sz, __u32 *type_id,
5250 bool *validate)
5251 {
5252 const struct bpf_core_accessor *acc;
5253 const struct btf_type *t;
5254 __u32 byte_off, byte_sz, bit_off, bit_sz, field_type_id;
5255 const struct btf_member *m;
5256 const struct btf_type *mt;
5257 bool bitfield;
5258 __s64 sz;
5259
5260 *field_sz = 0;
5261
5262 if (relo->kind == BPF_FIELD_EXISTS) {
5263 *val = spec ? 1 : 0;
5264 return 0;
5265 }
5266
5267 if (!spec)
5268 return -EUCLEAN; /* request instruction poisoning */
5269
5270 acc = &spec->spec[spec->len - 1];
5271 t = btf__type_by_id(spec->btf, acc->type_id);
5272
5273 /* a[n] accessor needs special handling */
5274 if (!acc->name) {
5275 if (relo->kind == BPF_FIELD_BYTE_OFFSET) {
5276 *val = spec->bit_offset / 8;
5277 /* remember field size for load/store mem size */
5278 sz = btf__resolve_size(spec->btf, acc->type_id);
5279 if (sz < 0)
5280 return -EINVAL;
5281 *field_sz = sz;
5282 *type_id = acc->type_id;
5283 } else if (relo->kind == BPF_FIELD_BYTE_SIZE) {
5284 sz = btf__resolve_size(spec->btf, acc->type_id);
5285 if (sz < 0)
5286 return -EINVAL;
5287 *val = sz;
5288 } else {
5289 pr_warn("prog '%s': relo %d at insn #%d can't be applied to array access\n",
5290 prog->name, relo->kind, relo->insn_off / 8);
5291 return -EINVAL;
5292 }
5293 if (validate)
5294 *validate = true;
5295 return 0;
5296 }
5297
5298 m = btf_members(t) + acc->idx;
5299 mt = skip_mods_and_typedefs(spec->btf, m->type, &field_type_id);
5300 bit_off = spec->bit_offset;
5301 bit_sz = btf_member_bitfield_size(t, acc->idx);
5302
5303 bitfield = bit_sz > 0;
5304 if (bitfield) {
5305 byte_sz = mt->size;
5306 byte_off = bit_off / 8 / byte_sz * byte_sz;
5307 /* figure out smallest int size necessary for bitfield load */
5308 while (bit_off + bit_sz - byte_off * 8 > byte_sz * 8) {
5309 if (byte_sz >= 8) {
5310 /* bitfield can't be read with 64-bit read */
5311 pr_warn("prog '%s': relo %d at insn #%d can't be satisfied for bitfield\n",
5312 prog->name, relo->kind, relo->insn_off / 8);
5313 return -E2BIG;
5314 }
5315 byte_sz *= 2;
5316 byte_off = bit_off / 8 / byte_sz * byte_sz;
5317 }
5318 } else {
5319 sz = btf__resolve_size(spec->btf, field_type_id);
5320 if (sz < 0)
5321 return -EINVAL;
5322 byte_sz = sz;
5323 byte_off = spec->bit_offset / 8;
5324 bit_sz = byte_sz * 8;
5325 }
5326
5327 /* for bitfields, all the relocatable aspects are ambiguous and we
5328 * might disagree with compiler, so turn off validation of expected
5329 * value, except for signedness
5330 */
5331 if (validate)
5332 *validate = !bitfield;
5333
5334 switch (relo->kind) {
5335 case BPF_FIELD_BYTE_OFFSET:
5336 *val = byte_off;
5337 if (!bitfield) {
5338 *field_sz = byte_sz;
5339 *type_id = field_type_id;
5340 }
5341 break;
5342 case BPF_FIELD_BYTE_SIZE:
5343 *val = byte_sz;
5344 break;
5345 case BPF_FIELD_SIGNED:
5346 /* enums will be assumed unsigned */
5347 *val = btf_is_enum(mt) ||
5348 (btf_int_encoding(mt) & BTF_INT_SIGNED);
5349 if (validate)
5350 *validate = true; /* signedness is never ambiguous */
5351 break;
5352 case BPF_FIELD_LSHIFT_U64:
5353 #if __BYTE_ORDER == __LITTLE_ENDIAN
5354 *val = 64 - (bit_off + bit_sz - byte_off * 8);
5355 #else
5356 *val = (8 - byte_sz) * 8 + (bit_off - byte_off * 8);
5357 #endif
5358 break;
5359 case BPF_FIELD_RSHIFT_U64:
5360 *val = 64 - bit_sz;
5361 if (validate)
5362 *validate = true; /* right shift is never ambiguous */
5363 break;
5364 case BPF_FIELD_EXISTS:
5365 default:
5366 return -EOPNOTSUPP;
5367 }
5368
5369 return 0;
5370 }
5371
5372 static int bpf_core_calc_type_relo(const struct bpf_core_relo *relo,
5373 const struct bpf_core_spec *spec,
5374 __u32 *val)
5375 {
5376 __s64 sz;
5377
5378 /* type-based relos return zero when target type is not found */
5379 if (!spec) {
5380 *val = 0;
5381 return 0;
5382 }
5383
5384 switch (relo->kind) {
5385 case BPF_TYPE_ID_TARGET:
5386 *val = spec->root_type_id;
5387 break;
5388 case BPF_TYPE_EXISTS:
5389 *val = 1;
5390 break;
5391 case BPF_TYPE_SIZE:
5392 sz = btf__resolve_size(spec->btf, spec->root_type_id);
5393 if (sz < 0)
5394 return -EINVAL;
5395 *val = sz;
5396 break;
5397 case BPF_TYPE_ID_LOCAL:
5398 /* BPF_TYPE_ID_LOCAL is handled specially and shouldn't get here */
5399 default:
5400 return -EOPNOTSUPP;
5401 }
5402
5403 return 0;
5404 }
5405
5406 static int bpf_core_calc_enumval_relo(const struct bpf_core_relo *relo,
5407 const struct bpf_core_spec *spec,
5408 __u32 *val)
5409 {
5410 const struct btf_type *t;
5411 const struct btf_enum *e;
5412
5413 switch (relo->kind) {
5414 case BPF_ENUMVAL_EXISTS:
5415 *val = spec ? 1 : 0;
5416 break;
5417 case BPF_ENUMVAL_VALUE:
5418 if (!spec)
5419 return -EUCLEAN; /* request instruction poisoning */
5420 t = btf__type_by_id(spec->btf, spec->spec[0].type_id);
5421 e = btf_enum(t) + spec->spec[0].idx;
5422 *val = e->val;
5423 break;
5424 default:
5425 return -EOPNOTSUPP;
5426 }
5427
5428 return 0;
5429 }
5430
5431 struct bpf_core_relo_res
5432 {
5433 /* expected value in the instruction, unless validate == false */
5434 __u32 orig_val;
5435 /* new value that needs to be patched up to */
5436 __u32 new_val;
5437 /* relocation unsuccessful, poison instruction, but don't fail load */
5438 bool poison;
5439 /* some relocations can't be validated against orig_val */
5440 bool validate;
5441 /* for field byte offset relocations or the forms:
5442 * *(T *)(rX + <off>) = rY
5443 * rX = *(T *)(rY + <off>),
5444 * we remember original and resolved field size to adjust direct
5445 * memory loads of pointers and integers; this is necessary for 32-bit
5446 * host kernel architectures, but also allows to automatically
5447 * relocate fields that were resized from, e.g., u32 to u64, etc.
5448 */
5449 bool fail_memsz_adjust;
5450 __u32 orig_sz;
5451 __u32 orig_type_id;
5452 __u32 new_sz;
5453 __u32 new_type_id;
5454 };
5455
5456 /* Calculate original and target relocation values, given local and target
5457 * specs and relocation kind. These values are calculated for each candidate.
5458 * If there are multiple candidates, resulting values should all be consistent
5459 * with each other. Otherwise, libbpf will refuse to proceed due to ambiguity.
5460 * If instruction has to be poisoned, *poison will be set to true.
5461 */
5462 static int bpf_core_calc_relo(const struct bpf_program *prog,
5463 const struct bpf_core_relo *relo,
5464 int relo_idx,
5465 const struct bpf_core_spec *local_spec,
5466 const struct bpf_core_spec *targ_spec,
5467 struct bpf_core_relo_res *res)
5468 {
5469 int err = -EOPNOTSUPP;
5470
5471 res->orig_val = 0;
5472 res->new_val = 0;
5473 res->poison = false;
5474 res->validate = true;
5475 res->fail_memsz_adjust = false;
5476 res->orig_sz = res->new_sz = 0;
5477 res->orig_type_id = res->new_type_id = 0;
5478
5479 if (core_relo_is_field_based(relo->kind)) {
5480 err = bpf_core_calc_field_relo(prog, relo, local_spec,
5481 &res->orig_val, &res->orig_sz,
5482 &res->orig_type_id, &res->validate);
5483 err = err ?: bpf_core_calc_field_relo(prog, relo, targ_spec,
5484 &res->new_val, &res->new_sz,
5485 &res->new_type_id, NULL);
5486 if (err)
5487 goto done;
5488 /* Validate if it's safe to adjust load/store memory size.
5489 * Adjustments are performed only if original and new memory
5490 * sizes differ.
5491 */
5492 res->fail_memsz_adjust = false;
5493 if (res->orig_sz != res->new_sz) {
5494 const struct btf_type *orig_t, *new_t;
5495
5496 orig_t = btf__type_by_id(local_spec->btf, res->orig_type_id);
5497 new_t = btf__type_by_id(targ_spec->btf, res->new_type_id);
5498
5499 /* There are two use cases in which it's safe to
5500 * adjust load/store's mem size:
5501 * - reading a 32-bit kernel pointer, while on BPF
5502 * size pointers are always 64-bit; in this case
5503 * it's safe to "downsize" instruction size due to
5504 * pointer being treated as unsigned integer with
5505 * zero-extended upper 32-bits;
5506 * - reading unsigned integers, again due to
5507 * zero-extension is preserving the value correctly.
5508 *
5509 * In all other cases it's incorrect to attempt to
5510 * load/store field because read value will be
5511 * incorrect, so we poison relocated instruction.
5512 */
5513 if (btf_is_ptr(orig_t) && btf_is_ptr(new_t))
5514 goto done;
5515 if (btf_is_int(orig_t) && btf_is_int(new_t) &&
5516 btf_int_encoding(orig_t) != BTF_INT_SIGNED &&
5517 btf_int_encoding(new_t) != BTF_INT_SIGNED)
5518 goto done;
5519
5520 /* mark as invalid mem size adjustment, but this will
5521 * only be checked for LDX/STX/ST insns
5522 */
5523 res->fail_memsz_adjust = true;
5524 }
5525 } else if (core_relo_is_type_based(relo->kind)) {
5526 err = bpf_core_calc_type_relo(relo, local_spec, &res->orig_val);
5527 err = err ?: bpf_core_calc_type_relo(relo, targ_spec, &res->new_val);
5528 } else if (core_relo_is_enumval_based(relo->kind)) {
5529 err = bpf_core_calc_enumval_relo(relo, local_spec, &res->orig_val);
5530 err = err ?: bpf_core_calc_enumval_relo(relo, targ_spec, &res->new_val);
5531 }
5532
5533 done:
5534 if (err == -EUCLEAN) {
5535 /* EUCLEAN is used to signal instruction poisoning request */
5536 res->poison = true;
5537 err = 0;
5538 } else if (err == -EOPNOTSUPP) {
5539 /* EOPNOTSUPP means unknown/unsupported relocation */
5540 pr_warn("prog '%s': relo #%d: unrecognized CO-RE relocation %s (%d) at insn #%d\n",
5541 prog->name, relo_idx, core_relo_kind_str(relo->kind),
5542 relo->kind, relo->insn_off / 8);
5543 }
5544
5545 return err;
5546 }
5547
5548 /*
5549 * Turn instruction for which CO_RE relocation failed into invalid one with
5550 * distinct signature.
5551 */
5552 static void bpf_core_poison_insn(struct bpf_program *prog, int relo_idx,
5553 int insn_idx, struct bpf_insn *insn)
5554 {
5555 pr_debug("prog '%s': relo #%d: substituting insn #%d w/ invalid insn\n",
5556 prog->name, relo_idx, insn_idx);
5557 insn->code = BPF_JMP | BPF_CALL;
5558 insn->dst_reg = 0;
5559 insn->src_reg = 0;
5560 insn->off = 0;
5561 /* if this instruction is reachable (not a dead code),
5562 * verifier will complain with the following message:
5563 * invalid func unknown#195896080
5564 */
5565 insn->imm = 195896080; /* => 0xbad2310 => "bad relo" */
5566 }
5567
5568 static bool is_ldimm64(struct bpf_insn *insn)
5569 {
5570 return insn->code == (BPF_LD | BPF_IMM | BPF_DW);
5571 }
5572
5573 static int insn_bpf_size_to_bytes(struct bpf_insn *insn)
5574 {
5575 switch (BPF_SIZE(insn->code)) {
5576 case BPF_DW: return 8;
5577 case BPF_W: return 4;
5578 case BPF_H: return 2;
5579 case BPF_B: return 1;
5580 default: return -1;
5581 }
5582 }
5583
5584 static int insn_bytes_to_bpf_size(__u32 sz)
5585 {
5586 switch (sz) {
5587 case 8: return BPF_DW;
5588 case 4: return BPF_W;
5589 case 2: return BPF_H;
5590 case 1: return BPF_B;
5591 default: return -1;
5592 }
5593 }
5594
5595 /*
5596 * Patch relocatable BPF instruction.
5597 *
5598 * Patched value is determined by relocation kind and target specification.
5599 * For existence relocations target spec will be NULL if field/type is not found.
5600 * Expected insn->imm value is determined using relocation kind and local
5601 * spec, and is checked before patching instruction. If actual insn->imm value
5602 * is wrong, bail out with error.
5603 *
5604 * Currently supported classes of BPF instruction are:
5605 * 1. rX = <imm> (assignment with immediate operand);
5606 * 2. rX += <imm> (arithmetic operations with immediate operand);
5607 * 3. rX = <imm64> (load with 64-bit immediate value);
5608 * 4. rX = *(T *)(rY + <off>), where T is one of {u8, u16, u32, u64};
5609 * 5. *(T *)(rX + <off>) = rY, where T is one of {u8, u16, u32, u64};
5610 * 6. *(T *)(rX + <off>) = <imm>, where T is one of {u8, u16, u32, u64}.
5611 */
5612 static int bpf_core_patch_insn(struct bpf_program *prog,
5613 const struct bpf_core_relo *relo,
5614 int relo_idx,
5615 const struct bpf_core_relo_res *res)
5616 {
5617 __u32 orig_val, new_val;
5618 struct bpf_insn *insn;
5619 int insn_idx;
5620 __u8 class;
5621
5622 if (relo->insn_off % BPF_INSN_SZ)
5623 return -EINVAL;
5624 insn_idx = relo->insn_off / BPF_INSN_SZ;
5625 /* adjust insn_idx from section frame of reference to the local
5626 * program's frame of reference; (sub-)program code is not yet
5627 * relocated, so it's enough to just subtract in-section offset
5628 */
5629 insn_idx = insn_idx - prog->sec_insn_off;
5630 insn = &prog->insns[insn_idx];
5631 class = BPF_CLASS(insn->code);
5632
5633 if (res->poison) {
5634 poison:
5635 /* poison second part of ldimm64 to avoid confusing error from
5636 * verifier about "unknown opcode 00"
5637 */
5638 if (is_ldimm64(insn))
5639 bpf_core_poison_insn(prog, relo_idx, insn_idx + 1, insn + 1);
5640 bpf_core_poison_insn(prog, relo_idx, insn_idx, insn);
5641 return 0;
5642 }
5643
5644 orig_val = res->orig_val;
5645 new_val = res->new_val;
5646
5647 switch (class) {
5648 case BPF_ALU:
5649 case BPF_ALU64:
5650 if (BPF_SRC(insn->code) != BPF_K)
5651 return -EINVAL;
5652 if (res->validate && insn->imm != orig_val) {
5653 pr_warn("prog '%s': relo #%d: unexpected insn #%d (ALU/ALU64) value: got %u, exp %u -> %u\n",
5654 prog->name, relo_idx,
5655 insn_idx, insn->imm, orig_val, new_val);
5656 return -EINVAL;
5657 }
5658 orig_val = insn->imm;
5659 insn->imm = new_val;
5660 pr_debug("prog '%s': relo #%d: patched insn #%d (ALU/ALU64) imm %u -> %u\n",
5661 prog->name, relo_idx, insn_idx,
5662 orig_val, new_val);
5663 break;
5664 case BPF_LDX:
5665 case BPF_ST:
5666 case BPF_STX:
5667 if (res->validate && insn->off != orig_val) {
5668 pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDX/ST/STX) value: got %u, exp %u -> %u\n",
5669 prog->name, relo_idx, insn_idx, insn->off, orig_val, new_val);
5670 return -EINVAL;
5671 }
5672 if (new_val > SHRT_MAX) {
5673 pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) value too big: %u\n",
5674 prog->name, relo_idx, insn_idx, new_val);
5675 return -ERANGE;
5676 }
5677 if (res->fail_memsz_adjust) {
5678 pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) accesses field incorrectly. "
5679 "Make sure you are accessing pointers, unsigned integers, or fields of matching type and size.\n",
5680 prog->name, relo_idx, insn_idx);
5681 goto poison;
5682 }
5683
5684 orig_val = insn->off;
5685 insn->off = new_val;
5686 pr_debug("prog '%s': relo #%d: patched insn #%d (LDX/ST/STX) off %u -> %u\n",
5687 prog->name, relo_idx, insn_idx, orig_val, new_val);
5688
5689 if (res->new_sz != res->orig_sz) {
5690 int insn_bytes_sz, insn_bpf_sz;
5691
5692 insn_bytes_sz = insn_bpf_size_to_bytes(insn);
5693 if (insn_bytes_sz != res->orig_sz) {
5694 pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) unexpected mem size: got %d, exp %u\n",
5695 prog->name, relo_idx, insn_idx, insn_bytes_sz, res->orig_sz);
5696 return -EINVAL;
5697 }
5698
5699 insn_bpf_sz = insn_bytes_to_bpf_size(res->new_sz);
5700 if (insn_bpf_sz < 0) {
5701 pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) invalid new mem size: %u\n",
5702 prog->name, relo_idx, insn_idx, res->new_sz);
5703 return -EINVAL;
5704 }
5705
5706 insn->code = BPF_MODE(insn->code) | insn_bpf_sz | BPF_CLASS(insn->code);
5707 pr_debug("prog '%s': relo #%d: patched insn #%d (LDX/ST/STX) mem_sz %u -> %u\n",
5708 prog->name, relo_idx, insn_idx, res->orig_sz, res->new_sz);
5709 }
5710 break;
5711 case BPF_LD: {
5712 __u64 imm;
5713
5714 if (!is_ldimm64(insn) ||
5715 insn[0].src_reg != 0 || insn[0].off != 0 ||
5716 insn_idx + 1 >= prog->insns_cnt ||
5717 insn[1].code != 0 || insn[1].dst_reg != 0 ||
5718 insn[1].src_reg != 0 || insn[1].off != 0) {
5719 pr_warn("prog '%s': relo #%d: insn #%d (LDIMM64) has unexpected form\n",
5720 prog->name, relo_idx, insn_idx);
5721 return -EINVAL;
5722 }
5723
5724 imm = insn[0].imm + ((__u64)insn[1].imm << 32);
5725 if (res->validate && imm != orig_val) {
5726 pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDIMM64) value: got %llu, exp %u -> %u\n",
5727 prog->name, relo_idx,
5728 insn_idx, (unsigned long long)imm,
5729 orig_val, new_val);
5730 return -EINVAL;
5731 }
5732
5733 insn[0].imm = new_val;
5734 insn[1].imm = 0; /* currently only 32-bit values are supported */
5735 pr_debug("prog '%s': relo #%d: patched insn #%d (LDIMM64) imm64 %llu -> %u\n",
5736 prog->name, relo_idx, insn_idx,
5737 (unsigned long long)imm, new_val);
5738 break;
5739 }
5740 default:
5741 pr_warn("prog '%s': relo #%d: trying to relocate unrecognized insn #%d, code:0x%x, src:0x%x, dst:0x%x, off:0x%x, imm:0x%x\n",
5742 prog->name, relo_idx, insn_idx, insn->code,
5743 insn->src_reg, insn->dst_reg, insn->off, insn->imm);
5744 return -EINVAL;
5745 }
5746
5747 return 0;
5748 }
5749
5750 /* Output spec definition in the format:
5751 * [<type-id>] (<type-name>) + <raw-spec> => <offset>@<spec>,
5752 * where <spec> is a C-syntax view of recorded field access, e.g.: x.a[3].b
5753 */
5754 static void bpf_core_dump_spec(int level, const struct bpf_core_spec *spec)
5755 {
5756 const struct btf_type *t;
5757 const struct btf_enum *e;
5758 const char *s;
5759 __u32 type_id;
5760 int i;
5761
5762 type_id = spec->root_type_id;
5763 t = btf__type_by_id(spec->btf, type_id);
5764 s = btf__name_by_offset(spec->btf, t->name_off);
5765
5766 libbpf_print(level, "[%u] %s %s", type_id, btf_kind_str(t), str_is_empty(s) ? "<anon>" : s);
5767
5768 if (core_relo_is_type_based(spec->relo_kind))
5769 return;
5770
5771 if (core_relo_is_enumval_based(spec->relo_kind)) {
5772 t = skip_mods_and_typedefs(spec->btf, type_id, NULL);
5773 e = btf_enum(t) + spec->raw_spec[0];
5774 s = btf__name_by_offset(spec->btf, e->name_off);
5775
5776 libbpf_print(level, "::%s = %u", s, e->val);
5777 return;
5778 }
5779
5780 if (core_relo_is_field_based(spec->relo_kind)) {
5781 for (i = 0; i < spec->len; i++) {
5782 if (spec->spec[i].name)
5783 libbpf_print(level, ".%s", spec->spec[i].name);
5784 else if (i > 0 || spec->spec[i].idx > 0)
5785 libbpf_print(level, "[%u]", spec->spec[i].idx);
5786 }
5787
5788 libbpf_print(level, " (");
5789 for (i = 0; i < spec->raw_len; i++)
5790 libbpf_print(level, "%s%d", i == 0 ? "" : ":", spec->raw_spec[i]);
5791
5792 if (spec->bit_offset % 8)
5793 libbpf_print(level, " @ offset %u.%u)",
5794 spec->bit_offset / 8, spec->bit_offset % 8);
5795 else
5796 libbpf_print(level, " @ offset %u)", spec->bit_offset / 8);
5797 return;
5798 }
5799 }
5800
5801 static size_t bpf_core_hash_fn(const void *key, void *ctx)
5802 {
5803 return (size_t)key;
5804 }
5805
5806 static bool bpf_core_equal_fn(const void *k1, const void *k2, void *ctx)
5807 {
5808 return k1 == k2;
5809 }
5810
5811 static void *u32_as_hash_key(__u32 x)
5812 {
5813 return (void *)(uintptr_t)x;
5814 }
5815
5816 /*
5817 * CO-RE relocate single instruction.
5818 *
5819 * The outline and important points of the algorithm:
5820 * 1. For given local type, find corresponding candidate target types.
5821 * Candidate type is a type with the same "essential" name, ignoring
5822 * everything after last triple underscore (___). E.g., `sample`,
5823 * `sample___flavor_one`, `sample___flavor_another_one`, are all candidates
5824 * for each other. Names with triple underscore are referred to as
5825 * "flavors" and are useful, among other things, to allow to
5826 * specify/support incompatible variations of the same kernel struct, which
5827 * might differ between different kernel versions and/or build
5828 * configurations.
5829 *
5830 * N.B. Struct "flavors" could be generated by bpftool's BTF-to-C
5831 * converter, when deduplicated BTF of a kernel still contains more than
5832 * one different types with the same name. In that case, ___2, ___3, etc
5833 * are appended starting from second name conflict. But start flavors are
5834 * also useful to be defined "locally", in BPF program, to extract same
5835 * data from incompatible changes between different kernel
5836 * versions/configurations. For instance, to handle field renames between
5837 * kernel versions, one can use two flavors of the struct name with the
5838 * same common name and use conditional relocations to extract that field,
5839 * depending on target kernel version.
5840 * 2. For each candidate type, try to match local specification to this
5841 * candidate target type. Matching involves finding corresponding
5842 * high-level spec accessors, meaning that all named fields should match,
5843 * as well as all array accesses should be within the actual bounds. Also,
5844 * types should be compatible (see bpf_core_fields_are_compat for details).
5845 * 3. It is supported and expected that there might be multiple flavors
5846 * matching the spec. As long as all the specs resolve to the same set of
5847 * offsets across all candidates, there is no error. If there is any
5848 * ambiguity, CO-RE relocation will fail. This is necessary to accomodate
5849 * imprefection of BTF deduplication, which can cause slight duplication of
5850 * the same BTF type, if some directly or indirectly referenced (by
5851 * pointer) type gets resolved to different actual types in different
5852 * object files. If such situation occurs, deduplicated BTF will end up
5853 * with two (or more) structurally identical types, which differ only in
5854 * types they refer to through pointer. This should be OK in most cases and
5855 * is not an error.
5856 * 4. Candidate types search is performed by linearly scanning through all
5857 * types in target BTF. It is anticipated that this is overall more
5858 * efficient memory-wise and not significantly worse (if not better)
5859 * CPU-wise compared to prebuilding a map from all local type names to
5860 * a list of candidate type names. It's also sped up by caching resolved
5861 * list of matching candidates per each local "root" type ID, that has at
5862 * least one bpf_core_relo associated with it. This list is shared
5863 * between multiple relocations for the same type ID and is updated as some
5864 * of the candidates are pruned due to structural incompatibility.
5865 */
5866 static int bpf_core_apply_relo(struct bpf_program *prog,
5867 const struct bpf_core_relo *relo,
5868 int relo_idx,
5869 const struct btf *local_btf,
5870 struct hashmap *cand_cache)
5871 {
5872 struct bpf_core_spec local_spec, cand_spec, targ_spec = {};
5873 const void *type_key = u32_as_hash_key(relo->type_id);
5874 struct bpf_core_relo_res cand_res, targ_res;
5875 const struct btf_type *local_type;
5876 const char *local_name;
5877 struct core_cand_list *cands = NULL;
5878 __u32 local_id;
5879 const char *spec_str;
5880 int i, j, err;
5881
5882 local_id = relo->type_id;
5883 local_type = btf__type_by_id(local_btf, local_id);
5884 if (!local_type)
5885 return -EINVAL;
5886
5887 local_name = btf__name_by_offset(local_btf, local_type->name_off);
5888 if (!local_name)
5889 return -EINVAL;
5890
5891 spec_str = btf__name_by_offset(local_btf, relo->access_str_off);
5892 if (str_is_empty(spec_str))
5893 return -EINVAL;
5894
5895 err = bpf_core_parse_spec(local_btf, local_id, spec_str, relo->kind, &local_spec);
5896 if (err) {
5897 pr_warn("prog '%s': relo #%d: parsing [%d] %s %s + %s failed: %d\n",
5898 prog->name, relo_idx, local_id, btf_kind_str(local_type),
5899 str_is_empty(local_name) ? "<anon>" : local_name,
5900 spec_str, err);
5901 return -EINVAL;
5902 }
5903
5904 pr_debug("prog '%s': relo #%d: kind <%s> (%d), spec is ", prog->name,
5905 relo_idx, core_relo_kind_str(relo->kind), relo->kind);
5906 bpf_core_dump_spec(LIBBPF_DEBUG, &local_spec);
5907 libbpf_print(LIBBPF_DEBUG, "\n");
5908
5909 /* TYPE_ID_LOCAL relo is special and doesn't need candidate search */
5910 if (relo->kind == BPF_TYPE_ID_LOCAL) {
5911 targ_res.validate = true;
5912 targ_res.poison = false;
5913 targ_res.orig_val = local_spec.root_type_id;
5914 targ_res.new_val = local_spec.root_type_id;
5915 goto patch_insn;
5916 }
5917
5918 /* libbpf doesn't support candidate search for anonymous types */
5919 if (str_is_empty(spec_str)) {
5920 pr_warn("prog '%s': relo #%d: <%s> (%d) relocation doesn't support anonymous types\n",
5921 prog->name, relo_idx, core_relo_kind_str(relo->kind), relo->kind);
5922 return -EOPNOTSUPP;
5923 }
5924
5925 if (!hashmap__find(cand_cache, type_key, (void **)&cands)) {
5926 cands = bpf_core_find_cands(prog->obj, local_btf, local_id);
5927 if (IS_ERR(cands)) {
5928 pr_warn("prog '%s': relo #%d: target candidate search failed for [%d] %s %s: %ld\n",
5929 prog->name, relo_idx, local_id, btf_kind_str(local_type),
5930 local_name, PTR_ERR(cands));
5931 return PTR_ERR(cands);
5932 }
5933 err = hashmap__set(cand_cache, type_key, cands, NULL, NULL);
5934 if (err) {
5935 bpf_core_free_cands(cands);
5936 return err;
5937 }
5938 }
5939
5940 for (i = 0, j = 0; i < cands->len; i++) {
5941 err = bpf_core_spec_match(&local_spec, cands->cands[i].btf,
5942 cands->cands[i].id, &cand_spec);
5943 if (err < 0) {
5944 pr_warn("prog '%s': relo #%d: error matching candidate #%d ",
5945 prog->name, relo_idx, i);
5946 bpf_core_dump_spec(LIBBPF_WARN, &cand_spec);
5947 libbpf_print(LIBBPF_WARN, ": %d\n", err);
5948 return err;
5949 }
5950
5951 pr_debug("prog '%s': relo #%d: %s candidate #%d ", prog->name,
5952 relo_idx, err == 0 ? "non-matching" : "matching", i);
5953 bpf_core_dump_spec(LIBBPF_DEBUG, &cand_spec);
5954 libbpf_print(LIBBPF_DEBUG, "\n");
5955
5956 if (err == 0)
5957 continue;
5958
5959 err = bpf_core_calc_relo(prog, relo, relo_idx, &local_spec, &cand_spec, &cand_res);
5960 if (err)
5961 return err;
5962
5963 if (j == 0) {
5964 targ_res = cand_res;
5965 targ_spec = cand_spec;
5966 } else if (cand_spec.bit_offset != targ_spec.bit_offset) {
5967 /* if there are many field relo candidates, they
5968 * should all resolve to the same bit offset
5969 */
5970 pr_warn("prog '%s': relo #%d: field offset ambiguity: %u != %u\n",
5971 prog->name, relo_idx, cand_spec.bit_offset,
5972 targ_spec.bit_offset);
5973 return -EINVAL;
5974 } else if (cand_res.poison != targ_res.poison || cand_res.new_val != targ_res.new_val) {
5975 /* all candidates should result in the same relocation
5976 * decision and value, otherwise it's dangerous to
5977 * proceed due to ambiguity
5978 */
5979 pr_warn("prog '%s': relo #%d: relocation decision ambiguity: %s %u != %s %u\n",
5980 prog->name, relo_idx,
5981 cand_res.poison ? "failure" : "success", cand_res.new_val,
5982 targ_res.poison ? "failure" : "success", targ_res.new_val);
5983 return -EINVAL;
5984 }
5985
5986 cands->cands[j++] = cands->cands[i];
5987 }
5988
5989 /*
5990 * For BPF_FIELD_EXISTS relo or when used BPF program has field
5991 * existence checks or kernel version/config checks, it's expected
5992 * that we might not find any candidates. In this case, if field
5993 * wasn't found in any candidate, the list of candidates shouldn't
5994 * change at all, we'll just handle relocating appropriately,
5995 * depending on relo's kind.
5996 */
5997 if (j > 0)
5998 cands->len = j;
5999
6000 /*
6001 * If no candidates were found, it might be both a programmer error,
6002 * as well as expected case, depending whether instruction w/
6003 * relocation is guarded in some way that makes it unreachable (dead
6004 * code) if relocation can't be resolved. This is handled in
6005 * bpf_core_patch_insn() uniformly by replacing that instruction with
6006 * BPF helper call insn (using invalid helper ID). If that instruction
6007 * is indeed unreachable, then it will be ignored and eliminated by
6008 * verifier. If it was an error, then verifier will complain and point
6009 * to a specific instruction number in its log.
6010 */
6011 if (j == 0) {
6012 pr_debug("prog '%s': relo #%d: no matching targets found\n",
6013 prog->name, relo_idx);
6014
6015 /* calculate single target relo result explicitly */
6016 err = bpf_core_calc_relo(prog, relo, relo_idx, &local_spec, NULL, &targ_res);
6017 if (err)
6018 return err;
6019 }
6020
6021 patch_insn:
6022 /* bpf_core_patch_insn() should know how to handle missing targ_spec */
6023 err = bpf_core_patch_insn(prog, relo, relo_idx, &targ_res);
6024 if (err) {
6025 pr_warn("prog '%s': relo #%d: failed to patch insn at offset %d: %d\n",
6026 prog->name, relo_idx, relo->insn_off, err);
6027 return -EINVAL;
6028 }
6029
6030 return 0;
6031 }
6032
6033 static int
6034 bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path)
6035 {
6036 const struct btf_ext_info_sec *sec;
6037 const struct bpf_core_relo *rec;
6038 const struct btf_ext_info *seg;
6039 struct hashmap_entry *entry;
6040 struct hashmap *cand_cache = NULL;
6041 struct bpf_program *prog;
6042 const char *sec_name;
6043 int i, err = 0, insn_idx, sec_idx;
6044
6045 if (obj->btf_ext->core_relo_info.len == 0)
6046 return 0;
6047
6048 if (targ_btf_path) {
6049 obj->btf_vmlinux_override = btf__parse(targ_btf_path, NULL);
6050 if (IS_ERR_OR_NULL(obj->btf_vmlinux_override)) {
6051 err = PTR_ERR(obj->btf_vmlinux_override);
6052 pr_warn("failed to parse target BTF: %d\n", err);
6053 return err;
6054 }
6055 }
6056
6057 cand_cache = hashmap__new(bpf_core_hash_fn, bpf_core_equal_fn, NULL);
6058 if (IS_ERR(cand_cache)) {
6059 err = PTR_ERR(cand_cache);
6060 goto out;
6061 }
6062
6063 seg = &obj->btf_ext->core_relo_info;
6064 for_each_btf_ext_sec(seg, sec) {
6065 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
6066 if (str_is_empty(sec_name)) {
6067 err = -EINVAL;
6068 goto out;
6069 }
6070 /* bpf_object's ELF is gone by now so it's not easy to find
6071 * section index by section name, but we can find *any*
6072 * bpf_program within desired section name and use it's
6073 * prog->sec_idx to do a proper search by section index and
6074 * instruction offset
6075 */
6076 prog = NULL;
6077 for (i = 0; i < obj->nr_programs; i++) {
6078 prog = &obj->programs[i];
6079 if (strcmp(prog->sec_name, sec_name) == 0)
6080 break;
6081 }
6082 if (!prog) {
6083 pr_warn("sec '%s': failed to find a BPF program\n", sec_name);
6084 return -ENOENT;
6085 }
6086 sec_idx = prog->sec_idx;
6087
6088 pr_debug("sec '%s': found %d CO-RE relocations\n",
6089 sec_name, sec->num_info);
6090
6091 for_each_btf_ext_rec(seg, sec, i, rec) {
6092 insn_idx = rec->insn_off / BPF_INSN_SZ;
6093 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx);
6094 if (!prog) {
6095 pr_warn("sec '%s': failed to find program at insn #%d for CO-RE offset relocation #%d\n",
6096 sec_name, insn_idx, i);
6097 err = -EINVAL;
6098 goto out;
6099 }
6100 /* no need to apply CO-RE relocation if the program is
6101 * not going to be loaded
6102 */
6103 if (!prog->load)
6104 continue;
6105
6106 err = bpf_core_apply_relo(prog, rec, i, obj->btf, cand_cache);
6107 if (err) {
6108 pr_warn("prog '%s': relo #%d: failed to relocate: %d\n",
6109 prog->name, i, err);
6110 goto out;
6111 }
6112 }
6113 }
6114
6115 out:
6116 /* obj->btf_vmlinux and module BTFs are freed after object load */
6117 btf__free(obj->btf_vmlinux_override);
6118 obj->btf_vmlinux_override = NULL;
6119
6120 if (!IS_ERR_OR_NULL(cand_cache)) {
6121 hashmap__for_each_entry(cand_cache, entry, i) {
6122 bpf_core_free_cands(entry->value);
6123 }
6124 hashmap__free(cand_cache);
6125 }
6126 return err;
6127 }
6128
6129 /* Relocate data references within program code:
6130 * - map references;
6131 * - global variable references;
6132 * - extern references.
6133 */
6134 static int
6135 bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog)
6136 {
6137 int i;
6138
6139 for (i = 0; i < prog->nr_reloc; i++) {
6140 struct reloc_desc *relo = &prog->reloc_desc[i];
6141 struct bpf_insn *insn = &prog->insns[relo->insn_idx];
6142 struct extern_desc *ext;
6143
6144 switch (relo->type) {
6145 case RELO_LD64:
6146 insn[0].src_reg = BPF_PSEUDO_MAP_FD;
6147 insn[0].imm = obj->maps[relo->map_idx].fd;
6148 relo->processed = true;
6149 break;
6150 case RELO_DATA:
6151 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE;
6152 insn[1].imm = insn[0].imm + relo->sym_off;
6153 insn[0].imm = obj->maps[relo->map_idx].fd;
6154 relo->processed = true;
6155 break;
6156 case RELO_EXTERN:
6157 ext = &obj->externs[relo->sym_off];
6158 if (ext->type == EXT_KCFG) {
6159 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE;
6160 insn[0].imm = obj->maps[obj->kconfig_map_idx].fd;
6161 insn[1].imm = ext->kcfg.data_off;
6162 } else /* EXT_KSYM */ {
6163 if (ext->ksym.type_id) { /* typed ksyms */
6164 insn[0].src_reg = BPF_PSEUDO_BTF_ID;
6165 insn[0].imm = ext->ksym.vmlinux_btf_id;
6166 } else { /* typeless ksyms */
6167 insn[0].imm = (__u32)ext->ksym.addr;
6168 insn[1].imm = ext->ksym.addr >> 32;
6169 }
6170 }
6171 relo->processed = true;
6172 break;
6173 case RELO_CALL:
6174 /* will be handled as a follow up pass */
6175 break;
6176 default:
6177 pr_warn("prog '%s': relo #%d: bad relo type %d\n",
6178 prog->name, i, relo->type);
6179 return -EINVAL;
6180 }
6181 }
6182
6183 return 0;
6184 }
6185
6186 static int adjust_prog_btf_ext_info(const struct bpf_object *obj,
6187 const struct bpf_program *prog,
6188 const struct btf_ext_info *ext_info,
6189 void **prog_info, __u32 *prog_rec_cnt,
6190 __u32 *prog_rec_sz)
6191 {
6192 void *copy_start = NULL, *copy_end = NULL;
6193 void *rec, *rec_end, *new_prog_info;
6194 const struct btf_ext_info_sec *sec;
6195 size_t old_sz, new_sz;
6196 const char *sec_name;
6197 int i, off_adj;
6198
6199 for_each_btf_ext_sec(ext_info, sec) {
6200 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
6201 if (!sec_name)
6202 return -EINVAL;
6203 if (strcmp(sec_name, prog->sec_name) != 0)
6204 continue;
6205
6206 for_each_btf_ext_rec(ext_info, sec, i, rec) {
6207 __u32 insn_off = *(__u32 *)rec / BPF_INSN_SZ;
6208
6209 if (insn_off < prog->sec_insn_off)
6210 continue;
6211 if (insn_off >= prog->sec_insn_off + prog->sec_insn_cnt)
6212 break;
6213
6214 if (!copy_start)
6215 copy_start = rec;
6216 copy_end = rec + ext_info->rec_size;
6217 }
6218
6219 if (!copy_start)
6220 return -ENOENT;
6221
6222 /* append func/line info of a given (sub-)program to the main
6223 * program func/line info
6224 */
6225 old_sz = (size_t)(*prog_rec_cnt) * ext_info->rec_size;
6226 new_sz = old_sz + (copy_end - copy_start);
6227 new_prog_info = realloc(*prog_info, new_sz);
6228 if (!new_prog_info)
6229 return -ENOMEM;
6230 *prog_info = new_prog_info;
6231 *prog_rec_cnt = new_sz / ext_info->rec_size;
6232 memcpy(new_prog_info + old_sz, copy_start, copy_end - copy_start);
6233
6234 /* Kernel instruction offsets are in units of 8-byte
6235 * instructions, while .BTF.ext instruction offsets generated
6236 * by Clang are in units of bytes. So convert Clang offsets
6237 * into kernel offsets and adjust offset according to program
6238 * relocated position.
6239 */
6240 off_adj = prog->sub_insn_off - prog->sec_insn_off;
6241 rec = new_prog_info + old_sz;
6242 rec_end = new_prog_info + new_sz;
6243 for (; rec < rec_end; rec += ext_info->rec_size) {
6244 __u32 *insn_off = rec;
6245
6246 *insn_off = *insn_off / BPF_INSN_SZ + off_adj;
6247 }
6248 *prog_rec_sz = ext_info->rec_size;
6249 return 0;
6250 }
6251
6252 return -ENOENT;
6253 }
6254
6255 static int
6256 reloc_prog_func_and_line_info(const struct bpf_object *obj,
6257 struct bpf_program *main_prog,
6258 const struct bpf_program *prog)
6259 {
6260 int err;
6261
6262 /* no .BTF.ext relocation if .BTF.ext is missing or kernel doesn't
6263 * supprot func/line info
6264 */
6265 if (!obj->btf_ext || !kernel_supports(FEAT_BTF_FUNC))
6266 return 0;
6267
6268 /* only attempt func info relocation if main program's func_info
6269 * relocation was successful
6270 */
6271 if (main_prog != prog && !main_prog->func_info)
6272 goto line_info;
6273
6274 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->func_info,
6275 &main_prog->func_info,
6276 &main_prog->func_info_cnt,
6277 &main_prog->func_info_rec_size);
6278 if (err) {
6279 if (err != -ENOENT) {
6280 pr_warn("prog '%s': error relocating .BTF.ext function info: %d\n",
6281 prog->name, err);
6282 return err;
6283 }
6284 if (main_prog->func_info) {
6285 /*
6286 * Some info has already been found but has problem
6287 * in the last btf_ext reloc. Must have to error out.
6288 */
6289 pr_warn("prog '%s': missing .BTF.ext function info.\n", prog->name);
6290 return err;
6291 }
6292 /* Have problem loading the very first info. Ignore the rest. */
6293 pr_warn("prog '%s': missing .BTF.ext function info for the main program, skipping all of .BTF.ext func info.\n",
6294 prog->name);
6295 }
6296
6297 line_info:
6298 /* don't relocate line info if main program's relocation failed */
6299 if (main_prog != prog && !main_prog->line_info)
6300 return 0;
6301
6302 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->line_info,
6303 &main_prog->line_info,
6304 &main_prog->line_info_cnt,
6305 &main_prog->line_info_rec_size);
6306 if (err) {
6307 if (err != -ENOENT) {
6308 pr_warn("prog '%s': error relocating .BTF.ext line info: %d\n",
6309 prog->name, err);
6310 return err;
6311 }
6312 if (main_prog->line_info) {
6313 /*
6314 * Some info has already been found but has problem
6315 * in the last btf_ext reloc. Must have to error out.
6316 */
6317 pr_warn("prog '%s': missing .BTF.ext line info.\n", prog->name);
6318 return err;
6319 }
6320 /* Have problem loading the very first info. Ignore the rest. */
6321 pr_warn("prog '%s': missing .BTF.ext line info for the main program, skipping all of .BTF.ext line info.\n",
6322 prog->name);
6323 }
6324 return 0;
6325 }
6326
6327 static int cmp_relo_by_insn_idx(const void *key, const void *elem)
6328 {
6329 size_t insn_idx = *(const size_t *)key;
6330 const struct reloc_desc *relo = elem;
6331
6332 if (insn_idx == relo->insn_idx)
6333 return 0;
6334 return insn_idx < relo->insn_idx ? -1 : 1;
6335 }
6336
6337 static struct reloc_desc *find_prog_insn_relo(const struct bpf_program *prog, size_t insn_idx)
6338 {
6339 return bsearch(&insn_idx, prog->reloc_desc, prog->nr_reloc,
6340 sizeof(*prog->reloc_desc), cmp_relo_by_insn_idx);
6341 }
6342
6343 static int
6344 bpf_object__reloc_code(struct bpf_object *obj, struct bpf_program *main_prog,
6345 struct bpf_program *prog)
6346 {
6347 size_t sub_insn_idx, insn_idx, new_cnt;
6348 struct bpf_program *subprog;
6349 struct bpf_insn *insns, *insn;
6350 struct reloc_desc *relo;
6351 int err;
6352
6353 err = reloc_prog_func_and_line_info(obj, main_prog, prog);
6354 if (err)
6355 return err;
6356
6357 for (insn_idx = 0; insn_idx < prog->sec_insn_cnt; insn_idx++) {
6358 insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
6359 if (!insn_is_subprog_call(insn))
6360 continue;
6361
6362 relo = find_prog_insn_relo(prog, insn_idx);
6363 if (relo && relo->type != RELO_CALL) {
6364 pr_warn("prog '%s': unexpected relo for insn #%zu, type %d\n",
6365 prog->name, insn_idx, relo->type);
6366 return -LIBBPF_ERRNO__RELOC;
6367 }
6368 if (relo) {
6369 /* sub-program instruction index is a combination of
6370 * an offset of a symbol pointed to by relocation and
6371 * call instruction's imm field; for global functions,
6372 * call always has imm = -1, but for static functions
6373 * relocation is against STT_SECTION and insn->imm
6374 * points to a start of a static function
6375 */
6376 sub_insn_idx = relo->sym_off / BPF_INSN_SZ + insn->imm + 1;
6377 } else {
6378 /* if subprogram call is to a static function within
6379 * the same ELF section, there won't be any relocation
6380 * emitted, but it also means there is no additional
6381 * offset necessary, insns->imm is relative to
6382 * instruction's original position within the section
6383 */
6384 sub_insn_idx = prog->sec_insn_off + insn_idx + insn->imm + 1;
6385 }
6386
6387 /* we enforce that sub-programs should be in .text section */
6388 subprog = find_prog_by_sec_insn(obj, obj->efile.text_shndx, sub_insn_idx);
6389 if (!subprog) {
6390 pr_warn("prog '%s': no .text section found yet sub-program call exists\n",
6391 prog->name);
6392 return -LIBBPF_ERRNO__RELOC;
6393 }
6394
6395 /* if it's the first call instruction calling into this
6396 * subprogram (meaning this subprog hasn't been processed
6397 * yet) within the context of current main program:
6398 * - append it at the end of main program's instructions blog;
6399 * - process is recursively, while current program is put on hold;
6400 * - if that subprogram calls some other not yet processes
6401 * subprogram, same thing will happen recursively until
6402 * there are no more unprocesses subprograms left to append
6403 * and relocate.
6404 */
6405 if (subprog->sub_insn_off == 0) {
6406 subprog->sub_insn_off = main_prog->insns_cnt;
6407
6408 new_cnt = main_prog->insns_cnt + subprog->insns_cnt;
6409 insns = libbpf_reallocarray(main_prog->insns, new_cnt, sizeof(*insns));
6410 if (!insns) {
6411 pr_warn("prog '%s': failed to realloc prog code\n", main_prog->name);
6412 return -ENOMEM;
6413 }
6414 main_prog->insns = insns;
6415 main_prog->insns_cnt = new_cnt;
6416
6417 memcpy(main_prog->insns + subprog->sub_insn_off, subprog->insns,
6418 subprog->insns_cnt * sizeof(*insns));
6419
6420 pr_debug("prog '%s': added %zu insns from sub-prog '%s'\n",
6421 main_prog->name, subprog->insns_cnt, subprog->name);
6422
6423 err = bpf_object__reloc_code(obj, main_prog, subprog);
6424 if (err)
6425 return err;
6426 }
6427
6428 /* main_prog->insns memory could have been re-allocated, so
6429 * calculate pointer again
6430 */
6431 insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
6432 /* calculate correct instruction position within current main
6433 * prog; each main prog can have a different set of
6434 * subprograms appended (potentially in different order as
6435 * well), so position of any subprog can be different for
6436 * different main programs */
6437 insn->imm = subprog->sub_insn_off - (prog->sub_insn_off + insn_idx) - 1;
6438
6439 if (relo)
6440 relo->processed = true;
6441
6442 pr_debug("prog '%s': insn #%zu relocated, imm %d points to subprog '%s' (now at %zu offset)\n",
6443 prog->name, insn_idx, insn->imm, subprog->name, subprog->sub_insn_off);
6444 }
6445
6446 return 0;
6447 }
6448
6449 /*
6450 * Relocate sub-program calls.
6451 *
6452 * Algorithm operates as follows. Each entry-point BPF program (referred to as
6453 * main prog) is processed separately. For each subprog (non-entry functions,
6454 * that can be called from either entry progs or other subprogs) gets their
6455 * sub_insn_off reset to zero. This serves as indicator that this subprogram
6456 * hasn't been yet appended and relocated within current main prog. Once its
6457 * relocated, sub_insn_off will point at the position within current main prog
6458 * where given subprog was appended. This will further be used to relocate all
6459 * the call instructions jumping into this subprog.
6460 *
6461 * We start with main program and process all call instructions. If the call
6462 * is into a subprog that hasn't been processed (i.e., subprog->sub_insn_off
6463 * is zero), subprog instructions are appended at the end of main program's
6464 * instruction array. Then main program is "put on hold" while we recursively
6465 * process newly appended subprogram. If that subprogram calls into another
6466 * subprogram that hasn't been appended, new subprogram is appended again to
6467 * the *main* prog's instructions (subprog's instructions are always left
6468 * untouched, as they need to be in unmodified state for subsequent main progs
6469 * and subprog instructions are always sent only as part of a main prog) and
6470 * the process continues recursively. Once all the subprogs called from a main
6471 * prog or any of its subprogs are appended (and relocated), all their
6472 * positions within finalized instructions array are known, so it's easy to
6473 * rewrite call instructions with correct relative offsets, corresponding to
6474 * desired target subprog.
6475 *
6476 * Its important to realize that some subprogs might not be called from some
6477 * main prog and any of its called/used subprogs. Those will keep their
6478 * subprog->sub_insn_off as zero at all times and won't be appended to current
6479 * main prog and won't be relocated within the context of current main prog.
6480 * They might still be used from other main progs later.
6481 *
6482 * Visually this process can be shown as below. Suppose we have two main
6483 * programs mainA and mainB and BPF object contains three subprogs: subA,
6484 * subB, and subC. mainA calls only subA, mainB calls only subC, but subA and
6485 * subC both call subB:
6486 *
6487 * +--------+ +-------+
6488 * | v v |
6489 * +--+---+ +--+-+-+ +---+--+
6490 * | subA | | subB | | subC |
6491 * +--+---+ +------+ +---+--+
6492 * ^ ^
6493 * | |
6494 * +---+-------+ +------+----+
6495 * | mainA | | mainB |
6496 * +-----------+ +-----------+
6497 *
6498 * We'll start relocating mainA, will find subA, append it and start
6499 * processing sub A recursively:
6500 *
6501 * +-----------+------+
6502 * | mainA | subA |
6503 * +-----------+------+
6504 *
6505 * At this point we notice that subB is used from subA, so we append it and
6506 * relocate (there are no further subcalls from subB):
6507 *
6508 * +-----------+------+------+
6509 * | mainA | subA | subB |
6510 * +-----------+------+------+
6511 *
6512 * At this point, we relocate subA calls, then go one level up and finish with
6513 * relocatin mainA calls. mainA is done.
6514 *
6515 * For mainB process is similar but results in different order. We start with
6516 * mainB and skip subA and subB, as mainB never calls them (at least
6517 * directly), but we see subC is needed, so we append and start processing it:
6518 *
6519 * +-----------+------+
6520 * | mainB | subC |
6521 * +-----------+------+
6522 * Now we see subC needs subB, so we go back to it, append and relocate it:
6523 *
6524 * +-----------+------+------+
6525 * | mainB | subC | subB |
6526 * +-----------+------+------+
6527 *
6528 * At this point we unwind recursion, relocate calls in subC, then in mainB.
6529 */
6530 static int
6531 bpf_object__relocate_calls(struct bpf_object *obj, struct bpf_program *prog)
6532 {
6533 struct bpf_program *subprog;
6534 int i, j, err;
6535
6536 /* mark all subprogs as not relocated (yet) within the context of
6537 * current main program
6538 */
6539 for (i = 0; i < obj->nr_programs; i++) {
6540 subprog = &obj->programs[i];
6541 if (!prog_is_subprog(obj, subprog))
6542 continue;
6543
6544 subprog->sub_insn_off = 0;
6545 for (j = 0; j < subprog->nr_reloc; j++)
6546 if (subprog->reloc_desc[j].type == RELO_CALL)
6547 subprog->reloc_desc[j].processed = false;
6548 }
6549
6550 err = bpf_object__reloc_code(obj, prog, prog);
6551 if (err)
6552 return err;
6553
6554
6555 return 0;
6556 }
6557
6558 static int
6559 bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path)
6560 {
6561 struct bpf_program *prog;
6562 size_t i;
6563 int err;
6564
6565 if (obj->btf_ext) {
6566 err = bpf_object__relocate_core(obj, targ_btf_path);
6567 if (err) {
6568 pr_warn("failed to perform CO-RE relocations: %d\n",
6569 err);
6570 return err;
6571 }
6572 }
6573 /* relocate data references first for all programs and sub-programs,
6574 * as they don't change relative to code locations, so subsequent
6575 * subprogram processing won't need to re-calculate any of them
6576 */
6577 for (i = 0; i < obj->nr_programs; i++) {
6578 prog = &obj->programs[i];
6579 err = bpf_object__relocate_data(obj, prog);
6580 if (err) {
6581 pr_warn("prog '%s': failed to relocate data references: %d\n",
6582 prog->name, err);
6583 return err;
6584 }
6585 }
6586 /* now relocate subprogram calls and append used subprograms to main
6587 * programs; each copy of subprogram code needs to be relocated
6588 * differently for each main program, because its code location might
6589 * have changed
6590 */
6591 for (i = 0; i < obj->nr_programs; i++) {
6592 prog = &obj->programs[i];
6593 /* sub-program's sub-calls are relocated within the context of
6594 * its main program only
6595 */
6596 if (prog_is_subprog(obj, prog))
6597 continue;
6598
6599 err = bpf_object__relocate_calls(obj, prog);
6600 if (err) {
6601 pr_warn("prog '%s': failed to relocate calls: %d\n",
6602 prog->name, err);
6603 return err;
6604 }
6605 }
6606 /* free up relocation descriptors */
6607 for (i = 0; i < obj->nr_programs; i++) {
6608 prog = &obj->programs[i];
6609 zfree(&prog->reloc_desc);
6610 prog->nr_reloc = 0;
6611 }
6612 return 0;
6613 }
6614
6615 static int bpf_object__collect_st_ops_relos(struct bpf_object *obj,
6616 GElf_Shdr *shdr, Elf_Data *data);
6617
6618 static int bpf_object__collect_map_relos(struct bpf_object *obj,
6619 GElf_Shdr *shdr, Elf_Data *data)
6620 {
6621 const int bpf_ptr_sz = 8, host_ptr_sz = sizeof(void *);
6622 int i, j, nrels, new_sz;
6623 const struct btf_var_secinfo *vi = NULL;
6624 const struct btf_type *sec, *var, *def;
6625 struct bpf_map *map = NULL, *targ_map;
6626 const struct btf_member *member;
6627 const char *name, *mname;
6628 Elf_Data *symbols;
6629 unsigned int moff;
6630 GElf_Sym sym;
6631 GElf_Rel rel;
6632 void *tmp;
6633
6634 if (!obj->efile.btf_maps_sec_btf_id || !obj->btf)
6635 return -EINVAL;
6636 sec = btf__type_by_id(obj->btf, obj->efile.btf_maps_sec_btf_id);
6637 if (!sec)
6638 return -EINVAL;
6639
6640 symbols = obj->efile.symbols;
6641 nrels = shdr->sh_size / shdr->sh_entsize;
6642 for (i = 0; i < nrels; i++) {
6643 if (!gelf_getrel(data, i, &rel)) {
6644 pr_warn(".maps relo #%d: failed to get ELF relo\n", i);
6645 return -LIBBPF_ERRNO__FORMAT;
6646 }
6647 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) {
6648 pr_warn(".maps relo #%d: symbol %zx not found\n",
6649 i, (size_t)GELF_R_SYM(rel.r_info));
6650 return -LIBBPF_ERRNO__FORMAT;
6651 }
6652 name = elf_sym_str(obj, sym.st_name) ?: "<?>";
6653 if (sym.st_shndx != obj->efile.btf_maps_shndx) {
6654 pr_warn(".maps relo #%d: '%s' isn't a BTF-defined map\n",
6655 i, name);
6656 return -LIBBPF_ERRNO__RELOC;
6657 }
6658
6659 pr_debug(".maps relo #%d: for %zd value %zd rel.r_offset %zu name %d ('%s')\n",
6660 i, (ssize_t)(rel.r_info >> 32), (size_t)sym.st_value,
6661 (size_t)rel.r_offset, sym.st_name, name);
6662
6663 for (j = 0; j < obj->nr_maps; j++) {
6664 map = &obj->maps[j];
6665 if (map->sec_idx != obj->efile.btf_maps_shndx)
6666 continue;
6667
6668 vi = btf_var_secinfos(sec) + map->btf_var_idx;
6669 if (vi->offset <= rel.r_offset &&
6670 rel.r_offset + bpf_ptr_sz <= vi->offset + vi->size)
6671 break;
6672 }
6673 if (j == obj->nr_maps) {
6674 pr_warn(".maps relo #%d: cannot find map '%s' at rel.r_offset %zu\n",
6675 i, name, (size_t)rel.r_offset);
6676 return -EINVAL;
6677 }
6678
6679 if (!bpf_map_type__is_map_in_map(map->def.type))
6680 return -EINVAL;
6681 if (map->def.type == BPF_MAP_TYPE_HASH_OF_MAPS &&
6682 map->def.key_size != sizeof(int)) {
6683 pr_warn(".maps relo #%d: hash-of-maps '%s' should have key size %zu.\n",
6684 i, map->name, sizeof(int));
6685 return -EINVAL;
6686 }
6687
6688 targ_map = bpf_object__find_map_by_name(obj, name);
6689 if (!targ_map)
6690 return -ESRCH;
6691
6692 var = btf__type_by_id(obj->btf, vi->type);
6693 def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
6694 if (btf_vlen(def) == 0)
6695 return -EINVAL;
6696 member = btf_members(def) + btf_vlen(def) - 1;
6697 mname = btf__name_by_offset(obj->btf, member->name_off);
6698 if (strcmp(mname, "values"))
6699 return -EINVAL;
6700
6701 moff = btf_member_bit_offset(def, btf_vlen(def) - 1) / 8;
6702 if (rel.r_offset - vi->offset < moff)
6703 return -EINVAL;
6704
6705 moff = rel.r_offset - vi->offset - moff;
6706 /* here we use BPF pointer size, which is always 64 bit, as we
6707 * are parsing ELF that was built for BPF target
6708 */
6709 if (moff % bpf_ptr_sz)
6710 return -EINVAL;
6711 moff /= bpf_ptr_sz;
6712 if (moff >= map->init_slots_sz) {
6713 new_sz = moff + 1;
6714 tmp = libbpf_reallocarray(map->init_slots, new_sz, host_ptr_sz);
6715 if (!tmp)
6716 return -ENOMEM;
6717 map->init_slots = tmp;
6718 memset(map->init_slots + map->init_slots_sz, 0,
6719 (new_sz - map->init_slots_sz) * host_ptr_sz);
6720 map->init_slots_sz = new_sz;
6721 }
6722 map->init_slots[moff] = targ_map;
6723
6724 pr_debug(".maps relo #%d: map '%s' slot [%d] points to map '%s'\n",
6725 i, map->name, moff, name);
6726 }
6727
6728 return 0;
6729 }
6730
6731 static int cmp_relocs(const void *_a, const void *_b)
6732 {
6733 const struct reloc_desc *a = _a;
6734 const struct reloc_desc *b = _b;
6735
6736 if (a->insn_idx != b->insn_idx)
6737 return a->insn_idx < b->insn_idx ? -1 : 1;
6738
6739 /* no two relocations should have the same insn_idx, but ... */
6740 if (a->type != b->type)
6741 return a->type < b->type ? -1 : 1;
6742
6743 return 0;
6744 }
6745
6746 static int bpf_object__collect_relos(struct bpf_object *obj)
6747 {
6748 int i, err;
6749
6750 for (i = 0; i < obj->efile.nr_reloc_sects; i++) {
6751 GElf_Shdr *shdr = &obj->efile.reloc_sects[i].shdr;
6752 Elf_Data *data = obj->efile.reloc_sects[i].data;
6753 int idx = shdr->sh_info;
6754
6755 if (shdr->sh_type != SHT_REL) {
6756 pr_warn("internal error at %d\n", __LINE__);
6757 return -LIBBPF_ERRNO__INTERNAL;
6758 }
6759
6760 if (idx == obj->efile.st_ops_shndx)
6761 err = bpf_object__collect_st_ops_relos(obj, shdr, data);
6762 else if (idx == obj->efile.btf_maps_shndx)
6763 err = bpf_object__collect_map_relos(obj, shdr, data);
6764 else
6765 err = bpf_object__collect_prog_relos(obj, shdr, data);
6766 if (err)
6767 return err;
6768 }
6769
6770 for (i = 0; i < obj->nr_programs; i++) {
6771 struct bpf_program *p = &obj->programs[i];
6772
6773 if (!p->nr_reloc)
6774 continue;
6775
6776 qsort(p->reloc_desc, p->nr_reloc, sizeof(*p->reloc_desc), cmp_relocs);
6777 }
6778 return 0;
6779 }
6780
6781 static bool insn_is_helper_call(struct bpf_insn *insn, enum bpf_func_id *func_id)
6782 {
6783 if (BPF_CLASS(insn->code) == BPF_JMP &&
6784 BPF_OP(insn->code) == BPF_CALL &&
6785 BPF_SRC(insn->code) == BPF_K &&
6786 insn->src_reg == 0 &&
6787 insn->dst_reg == 0) {
6788 *func_id = insn->imm;
6789 return true;
6790 }
6791 return false;
6792 }
6793
6794 static int bpf_object__sanitize_prog(struct bpf_object* obj, struct bpf_program *prog)
6795 {
6796 struct bpf_insn *insn = prog->insns;
6797 enum bpf_func_id func_id;
6798 int i;
6799
6800 for (i = 0; i < prog->insns_cnt; i++, insn++) {
6801 if (!insn_is_helper_call(insn, &func_id))
6802 continue;
6803
6804 /* on kernels that don't yet support
6805 * bpf_probe_read_{kernel,user}[_str] helpers, fall back
6806 * to bpf_probe_read() which works well for old kernels
6807 */
6808 switch (func_id) {
6809 case BPF_FUNC_probe_read_kernel:
6810 case BPF_FUNC_probe_read_user:
6811 if (!kernel_supports(FEAT_PROBE_READ_KERN))
6812 insn->imm = BPF_FUNC_probe_read;
6813 break;
6814 case BPF_FUNC_probe_read_kernel_str:
6815 case BPF_FUNC_probe_read_user_str:
6816 if (!kernel_supports(FEAT_PROBE_READ_KERN))
6817 insn->imm = BPF_FUNC_probe_read_str;
6818 break;
6819 default:
6820 break;
6821 }
6822 }
6823 return 0;
6824 }
6825
6826 static int
6827 load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt,
6828 char *license, __u32 kern_version, int *pfd)
6829 {
6830 struct bpf_prog_load_params load_attr = {};
6831 char *cp, errmsg[STRERR_BUFSIZE];
6832 size_t log_buf_size = 0;
6833 char *log_buf = NULL;
6834 int btf_fd, ret;
6835
6836 if (prog->type == BPF_PROG_TYPE_UNSPEC) {
6837 /*
6838 * The program type must be set. Most likely we couldn't find a proper
6839 * section definition at load time, and thus we didn't infer the type.
6840 */
6841 pr_warn("prog '%s': missing BPF prog type, check ELF section name '%s'\n",
6842 prog->name, prog->sec_name);
6843 return -EINVAL;
6844 }
6845
6846 if (!insns || !insns_cnt)
6847 return -EINVAL;
6848
6849 load_attr.prog_type = prog->type;
6850 /* old kernels might not support specifying expected_attach_type */
6851 if (!kernel_supports(FEAT_EXP_ATTACH_TYPE) && prog->sec_def &&
6852 prog->sec_def->is_exp_attach_type_optional)
6853 load_attr.expected_attach_type = 0;
6854 else
6855 load_attr.expected_attach_type = prog->expected_attach_type;
6856 if (kernel_supports(FEAT_PROG_NAME))
6857 load_attr.name = prog->name;
6858 load_attr.insns = insns;
6859 load_attr.insn_cnt = insns_cnt;
6860 load_attr.license = license;
6861 load_attr.attach_btf_id = prog->attach_btf_id;
6862 if (prog->attach_prog_fd)
6863 load_attr.attach_prog_fd = prog->attach_prog_fd;
6864 else
6865 load_attr.attach_btf_obj_fd = prog->attach_btf_obj_fd;
6866 load_attr.attach_btf_id = prog->attach_btf_id;
6867 load_attr.kern_version = kern_version;
6868 load_attr.prog_ifindex = prog->prog_ifindex;
6869
6870 /* specify func_info/line_info only if kernel supports them */
6871 btf_fd = bpf_object__btf_fd(prog->obj);
6872 if (btf_fd >= 0 && kernel_supports(FEAT_BTF_FUNC)) {
6873 load_attr.prog_btf_fd = btf_fd;
6874 load_attr.func_info = prog->func_info;
6875 load_attr.func_info_rec_size = prog->func_info_rec_size;
6876 load_attr.func_info_cnt = prog->func_info_cnt;
6877 load_attr.line_info = prog->line_info;
6878 load_attr.line_info_rec_size = prog->line_info_rec_size;
6879 load_attr.line_info_cnt = prog->line_info_cnt;
6880 }
6881 load_attr.log_level = prog->log_level;
6882 load_attr.prog_flags = prog->prog_flags;
6883
6884 retry_load:
6885 if (log_buf_size) {
6886 log_buf = malloc(log_buf_size);
6887 if (!log_buf)
6888 return -ENOMEM;
6889
6890 *log_buf = 0;
6891 }
6892
6893 load_attr.log_buf = log_buf;
6894 load_attr.log_buf_sz = log_buf_size;
6895 ret = libbpf__bpf_prog_load(&load_attr);
6896
6897 if (ret >= 0) {
6898 if (log_buf && load_attr.log_level)
6899 pr_debug("verifier log:\n%s", log_buf);
6900
6901 if (prog->obj->rodata_map_idx >= 0 &&
6902 kernel_supports(FEAT_PROG_BIND_MAP)) {
6903 struct bpf_map *rodata_map =
6904 &prog->obj->maps[prog->obj->rodata_map_idx];
6905
6906 if (bpf_prog_bind_map(ret, bpf_map__fd(rodata_map), NULL)) {
6907 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
6908 pr_warn("prog '%s': failed to bind .rodata map: %s\n",
6909 prog->name, cp);
6910 /* Don't fail hard if can't bind rodata. */
6911 }
6912 }
6913
6914 *pfd = ret;
6915 ret = 0;
6916 goto out;
6917 }
6918
6919 if (!log_buf || errno == ENOSPC) {
6920 log_buf_size = max((size_t)BPF_LOG_BUF_SIZE,
6921 log_buf_size << 1);
6922
6923 free(log_buf);
6924 goto retry_load;
6925 }
6926 ret = errno ? -errno : -LIBBPF_ERRNO__LOAD;
6927 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
6928 pr_warn("load bpf program failed: %s\n", cp);
6929 pr_perm_msg(ret);
6930
6931 if (log_buf && log_buf[0] != '\0') {
6932 ret = -LIBBPF_ERRNO__VERIFY;
6933 pr_warn("-- BEGIN DUMP LOG ---\n");
6934 pr_warn("\n%s\n", log_buf);
6935 pr_warn("-- END LOG --\n");
6936 } else if (load_attr.insn_cnt >= BPF_MAXINSNS) {
6937 pr_warn("Program too large (%zu insns), at most %d insns\n",
6938 load_attr.insn_cnt, BPF_MAXINSNS);
6939 ret = -LIBBPF_ERRNO__PROG2BIG;
6940 } else if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) {
6941 /* Wrong program type? */
6942 int fd;
6943
6944 load_attr.prog_type = BPF_PROG_TYPE_KPROBE;
6945 load_attr.expected_attach_type = 0;
6946 load_attr.log_buf = NULL;
6947 load_attr.log_buf_sz = 0;
6948 fd = libbpf__bpf_prog_load(&load_attr);
6949 if (fd >= 0) {
6950 close(fd);
6951 ret = -LIBBPF_ERRNO__PROGTYPE;
6952 goto out;
6953 }
6954 }
6955
6956 out:
6957 free(log_buf);
6958 return ret;
6959 }
6960
6961 static int libbpf_find_attach_btf_id(struct bpf_program *prog, int *btf_obj_fd, int *btf_type_id);
6962
6963 int bpf_program__load(struct bpf_program *prog, char *license, __u32 kern_ver)
6964 {
6965 int err = 0, fd, i;
6966
6967 if (prog->obj->loaded) {
6968 pr_warn("prog '%s': can't load after object was loaded\n", prog->name);
6969 return -EINVAL;
6970 }
6971
6972 if ((prog->type == BPF_PROG_TYPE_TRACING ||
6973 prog->type == BPF_PROG_TYPE_LSM ||
6974 prog->type == BPF_PROG_TYPE_EXT) && !prog->attach_btf_id) {
6975 int btf_obj_fd = 0, btf_type_id = 0;
6976
6977 err = libbpf_find_attach_btf_id(prog, &btf_obj_fd, &btf_type_id);
6978 if (err)
6979 return err;
6980
6981 prog->attach_btf_obj_fd = btf_obj_fd;
6982 prog->attach_btf_id = btf_type_id;
6983 }
6984
6985 if (prog->instances.nr < 0 || !prog->instances.fds) {
6986 if (prog->preprocessor) {
6987 pr_warn("Internal error: can't load program '%s'\n",
6988 prog->name);
6989 return -LIBBPF_ERRNO__INTERNAL;
6990 }
6991
6992 prog->instances.fds = malloc(sizeof(int));
6993 if (!prog->instances.fds) {
6994 pr_warn("Not enough memory for BPF fds\n");
6995 return -ENOMEM;
6996 }
6997 prog->instances.nr = 1;
6998 prog->instances.fds[0] = -1;
6999 }
7000
7001 if (!prog->preprocessor) {
7002 if (prog->instances.nr != 1) {
7003 pr_warn("prog '%s': inconsistent nr(%d) != 1\n",
7004 prog->name, prog->instances.nr);
7005 }
7006 err = load_program(prog, prog->insns, prog->insns_cnt,
7007 license, kern_ver, &fd);
7008 if (!err)
7009 prog->instances.fds[0] = fd;
7010 goto out;
7011 }
7012
7013 for (i = 0; i < prog->instances.nr; i++) {
7014 struct bpf_prog_prep_result result;
7015 bpf_program_prep_t preprocessor = prog->preprocessor;
7016
7017 memset(&result, 0, sizeof(result));
7018 err = preprocessor(prog, i, prog->insns,
7019 prog->insns_cnt, &result);
7020 if (err) {
7021 pr_warn("Preprocessing the %dth instance of program '%s' failed\n",
7022 i, prog->name);
7023 goto out;
7024 }
7025
7026 if (!result.new_insn_ptr || !result.new_insn_cnt) {
7027 pr_debug("Skip loading the %dth instance of program '%s'\n",
7028 i, prog->name);
7029 prog->instances.fds[i] = -1;
7030 if (result.pfd)
7031 *result.pfd = -1;
7032 continue;
7033 }
7034
7035 err = load_program(prog, result.new_insn_ptr,
7036 result.new_insn_cnt, license, kern_ver, &fd);
7037 if (err) {
7038 pr_warn("Loading the %dth instance of program '%s' failed\n",
7039 i, prog->name);
7040 goto out;
7041 }
7042
7043 if (result.pfd)
7044 *result.pfd = fd;
7045 prog->instances.fds[i] = fd;
7046 }
7047 out:
7048 if (err)
7049 pr_warn("failed to load program '%s'\n", prog->name);
7050 zfree(&prog->insns);
7051 prog->insns_cnt = 0;
7052 return err;
7053 }
7054
7055 static int
7056 bpf_object__load_progs(struct bpf_object *obj, int log_level)
7057 {
7058 struct bpf_program *prog;
7059 size_t i;
7060 int err;
7061
7062 for (i = 0; i < obj->nr_programs; i++) {
7063 prog = &obj->programs[i];
7064 err = bpf_object__sanitize_prog(obj, prog);
7065 if (err)
7066 return err;
7067 }
7068
7069 for (i = 0; i < obj->nr_programs; i++) {
7070 prog = &obj->programs[i];
7071 if (prog_is_subprog(obj, prog))
7072 continue;
7073 if (!prog->load) {
7074 pr_debug("prog '%s': skipped loading\n", prog->name);
7075 continue;
7076 }
7077 prog->log_level |= log_level;
7078 err = bpf_program__load(prog, obj->license, obj->kern_version);
7079 if (err)
7080 return err;
7081 }
7082 return 0;
7083 }
7084
7085 static const struct bpf_sec_def *find_sec_def(const char *sec_name);
7086
7087 static struct bpf_object *
7088 __bpf_object__open(const char *path, const void *obj_buf, size_t obj_buf_sz,
7089 const struct bpf_object_open_opts *opts)
7090 {
7091 const char *obj_name, *kconfig;
7092 struct bpf_program *prog;
7093 struct bpf_object *obj;
7094 char tmp_name[64];
7095 int err;
7096
7097 if (elf_version(EV_CURRENT) == EV_NONE) {
7098 pr_warn("failed to init libelf for %s\n",
7099 path ? : "(mem buf)");
7100 return ERR_PTR(-LIBBPF_ERRNO__LIBELF);
7101 }
7102
7103 if (!OPTS_VALID(opts, bpf_object_open_opts))
7104 return ERR_PTR(-EINVAL);
7105
7106 obj_name = OPTS_GET(opts, object_name, NULL);
7107 if (obj_buf) {
7108 if (!obj_name) {
7109 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx",
7110 (unsigned long)obj_buf,
7111 (unsigned long)obj_buf_sz);
7112 obj_name = tmp_name;
7113 }
7114 path = obj_name;
7115 pr_debug("loading object '%s' from buffer\n", obj_name);
7116 }
7117
7118 obj = bpf_object__new(path, obj_buf, obj_buf_sz, obj_name);
7119 if (IS_ERR(obj))
7120 return obj;
7121
7122 kconfig = OPTS_GET(opts, kconfig, NULL);
7123 if (kconfig) {
7124 obj->kconfig = strdup(kconfig);
7125 if (!obj->kconfig)
7126 return ERR_PTR(-ENOMEM);
7127 }
7128
7129 err = bpf_object__elf_init(obj);
7130 err = err ? : bpf_object__check_endianness(obj);
7131 err = err ? : bpf_object__elf_collect(obj);
7132 err = err ? : bpf_object__collect_externs(obj);
7133 err = err ? : bpf_object__finalize_btf(obj);
7134 err = err ? : bpf_object__init_maps(obj, opts);
7135 err = err ? : bpf_object__collect_relos(obj);
7136 if (err)
7137 goto out;
7138 bpf_object__elf_finish(obj);
7139
7140 bpf_object__for_each_program(prog, obj) {
7141 prog->sec_def = find_sec_def(prog->sec_name);
7142 if (!prog->sec_def) {
7143 /* couldn't guess, but user might manually specify */
7144 pr_debug("prog '%s': unrecognized ELF section name '%s'\n",
7145 prog->name, prog->sec_name);
7146 continue;
7147 }
7148
7149 if (prog->sec_def->is_sleepable)
7150 prog->prog_flags |= BPF_F_SLEEPABLE;
7151 bpf_program__set_type(prog, prog->sec_def->prog_type);
7152 bpf_program__set_expected_attach_type(prog,
7153 prog->sec_def->expected_attach_type);
7154
7155 if (prog->sec_def->prog_type == BPF_PROG_TYPE_TRACING ||
7156 prog->sec_def->prog_type == BPF_PROG_TYPE_EXT)
7157 prog->attach_prog_fd = OPTS_GET(opts, attach_prog_fd, 0);
7158 }
7159
7160 return obj;
7161 out:
7162 bpf_object__close(obj);
7163 return ERR_PTR(err);
7164 }
7165
7166 static struct bpf_object *
7167 __bpf_object__open_xattr(struct bpf_object_open_attr *attr, int flags)
7168 {
7169 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts,
7170 .relaxed_maps = flags & MAPS_RELAX_COMPAT,
7171 );
7172
7173 /* param validation */
7174 if (!attr->file)
7175 return NULL;
7176
7177 pr_debug("loading %s\n", attr->file);
7178 return __bpf_object__open(attr->file, NULL, 0, &opts);
7179 }
7180
7181 struct bpf_object *bpf_object__open_xattr(struct bpf_object_open_attr *attr)
7182 {
7183 return __bpf_object__open_xattr(attr, 0);
7184 }
7185
7186 struct bpf_object *bpf_object__open(const char *path)
7187 {
7188 struct bpf_object_open_attr attr = {
7189 .file = path,
7190 .prog_type = BPF_PROG_TYPE_UNSPEC,
7191 };
7192
7193 return bpf_object__open_xattr(&attr);
7194 }
7195
7196 struct bpf_object *
7197 bpf_object__open_file(const char *path, const struct bpf_object_open_opts *opts)
7198 {
7199 if (!path)
7200 return ERR_PTR(-EINVAL);
7201
7202 pr_debug("loading %s\n", path);
7203
7204 return __bpf_object__open(path, NULL, 0, opts);
7205 }
7206
7207 struct bpf_object *
7208 bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz,
7209 const struct bpf_object_open_opts *opts)
7210 {
7211 if (!obj_buf || obj_buf_sz == 0)
7212 return ERR_PTR(-EINVAL);
7213
7214 return __bpf_object__open(NULL, obj_buf, obj_buf_sz, opts);
7215 }
7216
7217 struct bpf_object *
7218 bpf_object__open_buffer(const void *obj_buf, size_t obj_buf_sz,
7219 const char *name)
7220 {
7221 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts,
7222 .object_name = name,
7223 /* wrong default, but backwards-compatible */
7224 .relaxed_maps = true,
7225 );
7226
7227 /* returning NULL is wrong, but backwards-compatible */
7228 if (!obj_buf || obj_buf_sz == 0)
7229 return NULL;
7230
7231 return bpf_object__open_mem(obj_buf, obj_buf_sz, &opts);
7232 }
7233
7234 int bpf_object__unload(struct bpf_object *obj)
7235 {
7236 size_t i;
7237
7238 if (!obj)
7239 return -EINVAL;
7240
7241 for (i = 0; i < obj->nr_maps; i++) {
7242 zclose(obj->maps[i].fd);
7243 if (obj->maps[i].st_ops)
7244 zfree(&obj->maps[i].st_ops->kern_vdata);
7245 }
7246
7247 for (i = 0; i < obj->nr_programs; i++)
7248 bpf_program__unload(&obj->programs[i]);
7249
7250 return 0;
7251 }
7252
7253 static int bpf_object__sanitize_maps(struct bpf_object *obj)
7254 {
7255 struct bpf_map *m;
7256
7257 bpf_object__for_each_map(m, obj) {
7258 if (!bpf_map__is_internal(m))
7259 continue;
7260 if (!kernel_supports(FEAT_GLOBAL_DATA)) {
7261 pr_warn("kernel doesn't support global data\n");
7262 return -ENOTSUP;
7263 }
7264 if (!kernel_supports(FEAT_ARRAY_MMAP))
7265 m->def.map_flags ^= BPF_F_MMAPABLE;
7266 }
7267
7268 return 0;
7269 }
7270
7271 static int bpf_object__read_kallsyms_file(struct bpf_object *obj)
7272 {
7273 char sym_type, sym_name[500];
7274 unsigned long long sym_addr;
7275 struct extern_desc *ext;
7276 int ret, err = 0;
7277 FILE *f;
7278
7279 f = fopen("/proc/kallsyms", "r");
7280 if (!f) {
7281 err = -errno;
7282 pr_warn("failed to open /proc/kallsyms: %d\n", err);
7283 return err;
7284 }
7285
7286 while (true) {
7287 ret = fscanf(f, "%llx %c %499s%*[^\n]\n",
7288 &sym_addr, &sym_type, sym_name);
7289 if (ret == EOF && feof(f))
7290 break;
7291 if (ret != 3) {
7292 pr_warn("failed to read kallsyms entry: %d\n", ret);
7293 err = -EINVAL;
7294 goto out;
7295 }
7296
7297 ext = find_extern_by_name(obj, sym_name);
7298 if (!ext || ext->type != EXT_KSYM)
7299 continue;
7300
7301 if (ext->is_set && ext->ksym.addr != sym_addr) {
7302 pr_warn("extern (ksym) '%s' resolution is ambiguous: 0x%llx or 0x%llx\n",
7303 sym_name, ext->ksym.addr, sym_addr);
7304 err = -EINVAL;
7305 goto out;
7306 }
7307 if (!ext->is_set) {
7308 ext->is_set = true;
7309 ext->ksym.addr = sym_addr;
7310 pr_debug("extern (ksym) %s=0x%llx\n", sym_name, sym_addr);
7311 }
7312 }
7313
7314 out:
7315 fclose(f);
7316 return err;
7317 }
7318
7319 static int bpf_object__resolve_ksyms_btf_id(struct bpf_object *obj)
7320 {
7321 struct extern_desc *ext;
7322 int i, id;
7323
7324 for (i = 0; i < obj->nr_extern; i++) {
7325 const struct btf_type *targ_var, *targ_type;
7326 __u32 targ_type_id, local_type_id;
7327 const char *targ_var_name;
7328 int ret;
7329
7330 ext = &obj->externs[i];
7331 if (ext->type != EXT_KSYM || !ext->ksym.type_id)
7332 continue;
7333
7334 id = btf__find_by_name_kind(obj->btf_vmlinux, ext->name,
7335 BTF_KIND_VAR);
7336 if (id <= 0) {
7337 pr_warn("extern (ksym) '%s': failed to find BTF ID in vmlinux BTF.\n",
7338 ext->name);
7339 return -ESRCH;
7340 }
7341
7342 /* find local type_id */
7343 local_type_id = ext->ksym.type_id;
7344
7345 /* find target type_id */
7346 targ_var = btf__type_by_id(obj->btf_vmlinux, id);
7347 targ_var_name = btf__name_by_offset(obj->btf_vmlinux,
7348 targ_var->name_off);
7349 targ_type = skip_mods_and_typedefs(obj->btf_vmlinux,
7350 targ_var->type,
7351 &targ_type_id);
7352
7353 ret = bpf_core_types_are_compat(obj->btf, local_type_id,
7354 obj->btf_vmlinux, targ_type_id);
7355 if (ret <= 0) {
7356 const struct btf_type *local_type;
7357 const char *targ_name, *local_name;
7358
7359 local_type = btf__type_by_id(obj->btf, local_type_id);
7360 local_name = btf__name_by_offset(obj->btf,
7361 local_type->name_off);
7362 targ_name = btf__name_by_offset(obj->btf_vmlinux,
7363 targ_type->name_off);
7364
7365 pr_warn("extern (ksym) '%s': incompatible types, expected [%d] %s %s, but kernel has [%d] %s %s\n",
7366 ext->name, local_type_id,
7367 btf_kind_str(local_type), local_name, targ_type_id,
7368 btf_kind_str(targ_type), targ_name);
7369 return -EINVAL;
7370 }
7371
7372 ext->is_set = true;
7373 ext->ksym.vmlinux_btf_id = id;
7374 pr_debug("extern (ksym) '%s': resolved to [%d] %s %s\n",
7375 ext->name, id, btf_kind_str(targ_var), targ_var_name);
7376 }
7377 return 0;
7378 }
7379
7380 static int bpf_object__resolve_externs(struct bpf_object *obj,
7381 const char *extra_kconfig)
7382 {
7383 bool need_config = false, need_kallsyms = false;
7384 bool need_vmlinux_btf = false;
7385 struct extern_desc *ext;
7386 void *kcfg_data = NULL;
7387 int err, i;
7388
7389 if (obj->nr_extern == 0)
7390 return 0;
7391
7392 if (obj->kconfig_map_idx >= 0)
7393 kcfg_data = obj->maps[obj->kconfig_map_idx].mmaped;
7394
7395 for (i = 0; i < obj->nr_extern; i++) {
7396 ext = &obj->externs[i];
7397
7398 if (ext->type == EXT_KCFG &&
7399 strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) {
7400 void *ext_val = kcfg_data + ext->kcfg.data_off;
7401 __u32 kver = get_kernel_version();
7402
7403 if (!kver) {
7404 pr_warn("failed to get kernel version\n");
7405 return -EINVAL;
7406 }
7407 err = set_kcfg_value_num(ext, ext_val, kver);
7408 if (err)
7409 return err;
7410 pr_debug("extern (kcfg) %s=0x%x\n", ext->name, kver);
7411 } else if (ext->type == EXT_KCFG &&
7412 strncmp(ext->name, "CONFIG_", 7) == 0) {
7413 need_config = true;
7414 } else if (ext->type == EXT_KSYM) {
7415 if (ext->ksym.type_id)
7416 need_vmlinux_btf = true;
7417 else
7418 need_kallsyms = true;
7419 } else {
7420 pr_warn("unrecognized extern '%s'\n", ext->name);
7421 return -EINVAL;
7422 }
7423 }
7424 if (need_config && extra_kconfig) {
7425 err = bpf_object__read_kconfig_mem(obj, extra_kconfig, kcfg_data);
7426 if (err)
7427 return -EINVAL;
7428 need_config = false;
7429 for (i = 0; i < obj->nr_extern; i++) {
7430 ext = &obj->externs[i];
7431 if (ext->type == EXT_KCFG && !ext->is_set) {
7432 need_config = true;
7433 break;
7434 }
7435 }
7436 }
7437 if (need_config) {
7438 err = bpf_object__read_kconfig_file(obj, kcfg_data);
7439 if (err)
7440 return -EINVAL;
7441 }
7442 if (need_kallsyms) {
7443 err = bpf_object__read_kallsyms_file(obj);
7444 if (err)
7445 return -EINVAL;
7446 }
7447 if (need_vmlinux_btf) {
7448 err = bpf_object__resolve_ksyms_btf_id(obj);
7449 if (err)
7450 return -EINVAL;
7451 }
7452 for (i = 0; i < obj->nr_extern; i++) {
7453 ext = &obj->externs[i];
7454
7455 if (!ext->is_set && !ext->is_weak) {
7456 pr_warn("extern %s (strong) not resolved\n", ext->name);
7457 return -ESRCH;
7458 } else if (!ext->is_set) {
7459 pr_debug("extern %s (weak) not resolved, defaulting to zero\n",
7460 ext->name);
7461 }
7462 }
7463
7464 return 0;
7465 }
7466
7467 int bpf_object__load_xattr(struct bpf_object_load_attr *attr)
7468 {
7469 struct bpf_object *obj;
7470 int err, i;
7471
7472 if (!attr)
7473 return -EINVAL;
7474 obj = attr->obj;
7475 if (!obj)
7476 return -EINVAL;
7477
7478 if (obj->loaded) {
7479 pr_warn("object '%s': load can't be attempted twice\n", obj->name);
7480 return -EINVAL;
7481 }
7482
7483 err = bpf_object__probe_loading(obj);
7484 err = err ? : bpf_object__load_vmlinux_btf(obj, false);
7485 err = err ? : bpf_object__resolve_externs(obj, obj->kconfig);
7486 err = err ? : bpf_object__sanitize_and_load_btf(obj);
7487 err = err ? : bpf_object__sanitize_maps(obj);
7488 err = err ? : bpf_object__init_kern_struct_ops_maps(obj);
7489 err = err ? : bpf_object__create_maps(obj);
7490 err = err ? : bpf_object__relocate(obj, attr->target_btf_path);
7491 err = err ? : bpf_object__load_progs(obj, attr->log_level);
7492
7493 /* clean up module BTFs */
7494 for (i = 0; i < obj->btf_module_cnt; i++) {
7495 close(obj->btf_modules[i].fd);
7496 btf__free(obj->btf_modules[i].btf);
7497 free(obj->btf_modules[i].name);
7498 }
7499 free(obj->btf_modules);
7500
7501 /* clean up vmlinux BTF */
7502 btf__free(obj->btf_vmlinux);
7503 obj->btf_vmlinux = NULL;
7504
7505 obj->loaded = true; /* doesn't matter if successfully or not */
7506
7507 if (err)
7508 goto out;
7509
7510 return 0;
7511 out:
7512 /* unpin any maps that were auto-pinned during load */
7513 for (i = 0; i < obj->nr_maps; i++)
7514 if (obj->maps[i].pinned && !obj->maps[i].reused)
7515 bpf_map__unpin(&obj->maps[i], NULL);
7516
7517 bpf_object__unload(obj);
7518 pr_warn("failed to load object '%s'\n", obj->path);
7519 return err;
7520 }
7521
7522 int bpf_object__load(struct bpf_object *obj)
7523 {
7524 struct bpf_object_load_attr attr = {
7525 .obj = obj,
7526 };
7527
7528 return bpf_object__load_xattr(&attr);
7529 }
7530
7531 static int make_parent_dir(const char *path)
7532 {
7533 char *cp, errmsg[STRERR_BUFSIZE];
7534 char *dname, *dir;
7535 int err = 0;
7536
7537 dname = strdup(path);
7538 if (dname == NULL)
7539 return -ENOMEM;
7540
7541 dir = dirname(dname);
7542 if (mkdir(dir, 0700) && errno != EEXIST)
7543 err = -errno;
7544
7545 free(dname);
7546 if (err) {
7547 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
7548 pr_warn("failed to mkdir %s: %s\n", path, cp);
7549 }
7550 return err;
7551 }
7552
7553 static int check_path(const char *path)
7554 {
7555 char *cp, errmsg[STRERR_BUFSIZE];
7556 struct statfs st_fs;
7557 char *dname, *dir;
7558 int err = 0;
7559
7560 if (path == NULL)
7561 return -EINVAL;
7562
7563 dname = strdup(path);
7564 if (dname == NULL)
7565 return -ENOMEM;
7566
7567 dir = dirname(dname);
7568 if (statfs(dir, &st_fs)) {
7569 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
7570 pr_warn("failed to statfs %s: %s\n", dir, cp);
7571 err = -errno;
7572 }
7573 free(dname);
7574
7575 if (!err && st_fs.f_type != BPF_FS_MAGIC) {
7576 pr_warn("specified path %s is not on BPF FS\n", path);
7577 err = -EINVAL;
7578 }
7579
7580 return err;
7581 }
7582
7583 int bpf_program__pin_instance(struct bpf_program *prog, const char *path,
7584 int instance)
7585 {
7586 char *cp, errmsg[STRERR_BUFSIZE];
7587 int err;
7588
7589 err = make_parent_dir(path);
7590 if (err)
7591 return err;
7592
7593 err = check_path(path);
7594 if (err)
7595 return err;
7596
7597 if (prog == NULL) {
7598 pr_warn("invalid program pointer\n");
7599 return -EINVAL;
7600 }
7601
7602 if (instance < 0 || instance >= prog->instances.nr) {
7603 pr_warn("invalid prog instance %d of prog %s (max %d)\n",
7604 instance, prog->name, prog->instances.nr);
7605 return -EINVAL;
7606 }
7607
7608 if (bpf_obj_pin(prog->instances.fds[instance], path)) {
7609 err = -errno;
7610 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
7611 pr_warn("failed to pin program: %s\n", cp);
7612 return err;
7613 }
7614 pr_debug("pinned program '%s'\n", path);
7615
7616 return 0;
7617 }
7618
7619 int bpf_program__unpin_instance(struct bpf_program *prog, const char *path,
7620 int instance)
7621 {
7622 int err;
7623
7624 err = check_path(path);
7625 if (err)
7626 return err;
7627
7628 if (prog == NULL) {
7629 pr_warn("invalid program pointer\n");
7630 return -EINVAL;
7631 }
7632
7633 if (instance < 0 || instance >= prog->instances.nr) {
7634 pr_warn("invalid prog instance %d of prog %s (max %d)\n",
7635 instance, prog->name, prog->instances.nr);
7636 return -EINVAL;
7637 }
7638
7639 err = unlink(path);
7640 if (err != 0)
7641 return -errno;
7642 pr_debug("unpinned program '%s'\n", path);
7643
7644 return 0;
7645 }
7646
7647 int bpf_program__pin(struct bpf_program *prog, const char *path)
7648 {
7649 int i, err;
7650
7651 err = make_parent_dir(path);
7652 if (err)
7653 return err;
7654
7655 err = check_path(path);
7656 if (err)
7657 return err;
7658
7659 if (prog == NULL) {
7660 pr_warn("invalid program pointer\n");
7661 return -EINVAL;
7662 }
7663
7664 if (prog->instances.nr <= 0) {
7665 pr_warn("no instances of prog %s to pin\n", prog->name);
7666 return -EINVAL;
7667 }
7668
7669 if (prog->instances.nr == 1) {
7670 /* don't create subdirs when pinning single instance */
7671 return bpf_program__pin_instance(prog, path, 0);
7672 }
7673
7674 for (i = 0; i < prog->instances.nr; i++) {
7675 char buf[PATH_MAX];
7676 int len;
7677
7678 len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
7679 if (len < 0) {
7680 err = -EINVAL;
7681 goto err_unpin;
7682 } else if (len >= PATH_MAX) {
7683 err = -ENAMETOOLONG;
7684 goto err_unpin;
7685 }
7686
7687 err = bpf_program__pin_instance(prog, buf, i);
7688 if (err)
7689 goto err_unpin;
7690 }
7691
7692 return 0;
7693
7694 err_unpin:
7695 for (i = i - 1; i >= 0; i--) {
7696 char buf[PATH_MAX];
7697 int len;
7698
7699 len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
7700 if (len < 0)
7701 continue;
7702 else if (len >= PATH_MAX)
7703 continue;
7704
7705 bpf_program__unpin_instance(prog, buf, i);
7706 }
7707
7708 rmdir(path);
7709
7710 return err;
7711 }
7712
7713 int bpf_program__unpin(struct bpf_program *prog, const char *path)
7714 {
7715 int i, err;
7716
7717 err = check_path(path);
7718 if (err)
7719 return err;
7720
7721 if (prog == NULL) {
7722 pr_warn("invalid program pointer\n");
7723 return -EINVAL;
7724 }
7725
7726 if (prog->instances.nr <= 0) {
7727 pr_warn("no instances of prog %s to pin\n", prog->name);
7728 return -EINVAL;
7729 }
7730
7731 if (prog->instances.nr == 1) {
7732 /* don't create subdirs when pinning single instance */
7733 return bpf_program__unpin_instance(prog, path, 0);
7734 }
7735
7736 for (i = 0; i < prog->instances.nr; i++) {
7737 char buf[PATH_MAX];
7738 int len;
7739
7740 len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
7741 if (len < 0)
7742 return -EINVAL;
7743 else if (len >= PATH_MAX)
7744 return -ENAMETOOLONG;
7745
7746 err = bpf_program__unpin_instance(prog, buf, i);
7747 if (err)
7748 return err;
7749 }
7750
7751 err = rmdir(path);
7752 if (err)
7753 return -errno;
7754
7755 return 0;
7756 }
7757
7758 int bpf_map__pin(struct bpf_map *map, const char *path)
7759 {
7760 char *cp, errmsg[STRERR_BUFSIZE];
7761 int err;
7762
7763 if (map == NULL) {
7764 pr_warn("invalid map pointer\n");
7765 return -EINVAL;
7766 }
7767
7768 if (map->pin_path) {
7769 if (path && strcmp(path, map->pin_path)) {
7770 pr_warn("map '%s' already has pin path '%s' different from '%s'\n",
7771 bpf_map__name(map), map->pin_path, path);
7772 return -EINVAL;
7773 } else if (map->pinned) {
7774 pr_debug("map '%s' already pinned at '%s'; not re-pinning\n",
7775 bpf_map__name(map), map->pin_path);
7776 return 0;
7777 }
7778 } else {
7779 if (!path) {
7780 pr_warn("missing a path to pin map '%s' at\n",
7781 bpf_map__name(map));
7782 return -EINVAL;
7783 } else if (map->pinned) {
7784 pr_warn("map '%s' already pinned\n", bpf_map__name(map));
7785 return -EEXIST;
7786 }
7787
7788 map->pin_path = strdup(path);
7789 if (!map->pin_path) {
7790 err = -errno;
7791 goto out_err;
7792 }
7793 }
7794
7795 err = make_parent_dir(map->pin_path);
7796 if (err)
7797 return err;
7798
7799 err = check_path(map->pin_path);
7800 if (err)
7801 return err;
7802
7803 if (bpf_obj_pin(map->fd, map->pin_path)) {
7804 err = -errno;
7805 goto out_err;
7806 }
7807
7808 map->pinned = true;
7809 pr_debug("pinned map '%s'\n", map->pin_path);
7810
7811 return 0;
7812
7813 out_err:
7814 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
7815 pr_warn("failed to pin map: %s\n", cp);
7816 return err;
7817 }
7818
7819 int bpf_map__unpin(struct bpf_map *map, const char *path)
7820 {
7821 int err;
7822
7823 if (map == NULL) {
7824 pr_warn("invalid map pointer\n");
7825 return -EINVAL;
7826 }
7827
7828 if (map->pin_path) {
7829 if (path && strcmp(path, map->pin_path)) {
7830 pr_warn("map '%s' already has pin path '%s' different from '%s'\n",
7831 bpf_map__name(map), map->pin_path, path);
7832 return -EINVAL;
7833 }
7834 path = map->pin_path;
7835 } else if (!path) {
7836 pr_warn("no path to unpin map '%s' from\n",
7837 bpf_map__name(map));
7838 return -EINVAL;
7839 }
7840
7841 err = check_path(path);
7842 if (err)
7843 return err;
7844
7845 err = unlink(path);
7846 if (err != 0)
7847 return -errno;
7848
7849 map->pinned = false;
7850 pr_debug("unpinned map '%s' from '%s'\n", bpf_map__name(map), path);
7851
7852 return 0;
7853 }
7854
7855 int bpf_map__set_pin_path(struct bpf_map *map, const char *path)
7856 {
7857 char *new = NULL;
7858
7859 if (path) {
7860 new = strdup(path);
7861 if (!new)
7862 return -errno;
7863 }
7864
7865 free(map->pin_path);
7866 map->pin_path = new;
7867 return 0;
7868 }
7869
7870 const char *bpf_map__get_pin_path(const struct bpf_map *map)
7871 {
7872 return map->pin_path;
7873 }
7874
7875 bool bpf_map__is_pinned(const struct bpf_map *map)
7876 {
7877 return map->pinned;
7878 }
7879
7880 static void sanitize_pin_path(char *s)
7881 {
7882 /* bpffs disallows periods in path names */
7883 while (*s) {
7884 if (*s == '.')
7885 *s = '_';
7886 s++;
7887 }
7888 }
7889
7890 int bpf_object__pin_maps(struct bpf_object *obj, const char *path)
7891 {
7892 struct bpf_map *map;
7893 int err;
7894
7895 if (!obj)
7896 return -ENOENT;
7897
7898 if (!obj->loaded) {
7899 pr_warn("object not yet loaded; load it first\n");
7900 return -ENOENT;
7901 }
7902
7903 bpf_object__for_each_map(map, obj) {
7904 char *pin_path = NULL;
7905 char buf[PATH_MAX];
7906
7907 if (path) {
7908 int len;
7909
7910 len = snprintf(buf, PATH_MAX, "%s/%s", path,
7911 bpf_map__name(map));
7912 if (len < 0) {
7913 err = -EINVAL;
7914 goto err_unpin_maps;
7915 } else if (len >= PATH_MAX) {
7916 err = -ENAMETOOLONG;
7917 goto err_unpin_maps;
7918 }
7919 sanitize_pin_path(buf);
7920 pin_path = buf;
7921 } else if (!map->pin_path) {
7922 continue;
7923 }
7924
7925 err = bpf_map__pin(map, pin_path);
7926 if (err)
7927 goto err_unpin_maps;
7928 }
7929
7930 return 0;
7931
7932 err_unpin_maps:
7933 while ((map = bpf_map__prev(map, obj))) {
7934 if (!map->pin_path)
7935 continue;
7936
7937 bpf_map__unpin(map, NULL);
7938 }
7939
7940 return err;
7941 }
7942
7943 int bpf_object__unpin_maps(struct bpf_object *obj, const char *path)
7944 {
7945 struct bpf_map *map;
7946 int err;
7947
7948 if (!obj)
7949 return -ENOENT;
7950
7951 bpf_object__for_each_map(map, obj) {
7952 char *pin_path = NULL;
7953 char buf[PATH_MAX];
7954
7955 if (path) {
7956 int len;
7957
7958 len = snprintf(buf, PATH_MAX, "%s/%s", path,
7959 bpf_map__name(map));
7960 if (len < 0)
7961 return -EINVAL;
7962 else if (len >= PATH_MAX)
7963 return -ENAMETOOLONG;
7964 sanitize_pin_path(buf);
7965 pin_path = buf;
7966 } else if (!map->pin_path) {
7967 continue;
7968 }
7969
7970 err = bpf_map__unpin(map, pin_path);
7971 if (err)
7972 return err;
7973 }
7974
7975 return 0;
7976 }
7977
7978 int bpf_object__pin_programs(struct bpf_object *obj, const char *path)
7979 {
7980 struct bpf_program *prog;
7981 int err;
7982
7983 if (!obj)
7984 return -ENOENT;
7985
7986 if (!obj->loaded) {
7987 pr_warn("object not yet loaded; load it first\n");
7988 return -ENOENT;
7989 }
7990
7991 bpf_object__for_each_program(prog, obj) {
7992 char buf[PATH_MAX];
7993 int len;
7994
7995 len = snprintf(buf, PATH_MAX, "%s/%s", path,
7996 prog->pin_name);
7997 if (len < 0) {
7998 err = -EINVAL;
7999 goto err_unpin_programs;
8000 } else if (len >= PATH_MAX) {
8001 err = -ENAMETOOLONG;
8002 goto err_unpin_programs;
8003 }
8004
8005 err = bpf_program__pin(prog, buf);
8006 if (err)
8007 goto err_unpin_programs;
8008 }
8009
8010 return 0;
8011
8012 err_unpin_programs:
8013 while ((prog = bpf_program__prev(prog, obj))) {
8014 char buf[PATH_MAX];
8015 int len;
8016
8017 len = snprintf(buf, PATH_MAX, "%s/%s", path,
8018 prog->pin_name);
8019 if (len < 0)
8020 continue;
8021 else if (len >= PATH_MAX)
8022 continue;
8023
8024 bpf_program__unpin(prog, buf);
8025 }
8026
8027 return err;
8028 }
8029
8030 int bpf_object__unpin_programs(struct bpf_object *obj, const char *path)
8031 {
8032 struct bpf_program *prog;
8033 int err;
8034
8035 if (!obj)
8036 return -ENOENT;
8037
8038 bpf_object__for_each_program(prog, obj) {
8039 char buf[PATH_MAX];
8040 int len;
8041
8042 len = snprintf(buf, PATH_MAX, "%s/%s", path,
8043 prog->pin_name);
8044 if (len < 0)
8045 return -EINVAL;
8046 else if (len >= PATH_MAX)
8047 return -ENAMETOOLONG;
8048
8049 err = bpf_program__unpin(prog, buf);
8050 if (err)
8051 return err;
8052 }
8053
8054 return 0;
8055 }
8056
8057 int bpf_object__pin(struct bpf_object *obj, const char *path)
8058 {
8059 int err;
8060
8061 err = bpf_object__pin_maps(obj, path);
8062 if (err)
8063 return err;
8064
8065 err = bpf_object__pin_programs(obj, path);
8066 if (err) {
8067 bpf_object__unpin_maps(obj, path);
8068 return err;
8069 }
8070
8071 return 0;
8072 }
8073
8074 static void bpf_map__destroy(struct bpf_map *map)
8075 {
8076 if (map->clear_priv)
8077 map->clear_priv(map, map->priv);
8078 map->priv = NULL;
8079 map->clear_priv = NULL;
8080
8081 if (map->inner_map) {
8082 bpf_map__destroy(map->inner_map);
8083 zfree(&map->inner_map);
8084 }
8085
8086 zfree(&map->init_slots);
8087 map->init_slots_sz = 0;
8088
8089 if (map->mmaped) {
8090 munmap(map->mmaped, bpf_map_mmap_sz(map));
8091 map->mmaped = NULL;
8092 }
8093
8094 if (map->st_ops) {
8095 zfree(&map->st_ops->data);
8096 zfree(&map->st_ops->progs);
8097 zfree(&map->st_ops->kern_func_off);
8098 zfree(&map->st_ops);
8099 }
8100
8101 zfree(&map->name);
8102 zfree(&map->pin_path);
8103
8104 if (map->fd >= 0)
8105 zclose(map->fd);
8106 }
8107
8108 void bpf_object__close(struct bpf_object *obj)
8109 {
8110 size_t i;
8111
8112 if (IS_ERR_OR_NULL(obj))
8113 return;
8114
8115 if (obj->clear_priv)
8116 obj->clear_priv(obj, obj->priv);
8117
8118 bpf_object__elf_finish(obj);
8119 bpf_object__unload(obj);
8120 btf__free(obj->btf);
8121 btf_ext__free(obj->btf_ext);
8122
8123 for (i = 0; i < obj->nr_maps; i++)
8124 bpf_map__destroy(&obj->maps[i]);
8125
8126 zfree(&obj->kconfig);
8127 zfree(&obj->externs);
8128 obj->nr_extern = 0;
8129
8130 zfree(&obj->maps);
8131 obj->nr_maps = 0;
8132
8133 if (obj->programs && obj->nr_programs) {
8134 for (i = 0; i < obj->nr_programs; i++)
8135 bpf_program__exit(&obj->programs[i]);
8136 }
8137 zfree(&obj->programs);
8138
8139 list_del(&obj->list);
8140 free(obj);
8141 }
8142
8143 struct bpf_object *
8144 bpf_object__next(struct bpf_object *prev)
8145 {
8146 struct bpf_object *next;
8147
8148 if (!prev)
8149 next = list_first_entry(&bpf_objects_list,
8150 struct bpf_object,
8151 list);
8152 else
8153 next = list_next_entry(prev, list);
8154
8155 /* Empty list is noticed here so don't need checking on entry. */
8156 if (&next->list == &bpf_objects_list)
8157 return NULL;
8158
8159 return next;
8160 }
8161
8162 const char *bpf_object__name(const struct bpf_object *obj)
8163 {
8164 return obj ? obj->name : ERR_PTR(-EINVAL);
8165 }
8166
8167 unsigned int bpf_object__kversion(const struct bpf_object *obj)
8168 {
8169 return obj ? obj->kern_version : 0;
8170 }
8171
8172 struct btf *bpf_object__btf(const struct bpf_object *obj)
8173 {
8174 return obj ? obj->btf : NULL;
8175 }
8176
8177 int bpf_object__btf_fd(const struct bpf_object *obj)
8178 {
8179 return obj->btf ? btf__fd(obj->btf) : -1;
8180 }
8181
8182 int bpf_object__set_priv(struct bpf_object *obj, void *priv,
8183 bpf_object_clear_priv_t clear_priv)
8184 {
8185 if (obj->priv && obj->clear_priv)
8186 obj->clear_priv(obj, obj->priv);
8187
8188 obj->priv = priv;
8189 obj->clear_priv = clear_priv;
8190 return 0;
8191 }
8192
8193 void *bpf_object__priv(const struct bpf_object *obj)
8194 {
8195 return obj ? obj->priv : ERR_PTR(-EINVAL);
8196 }
8197
8198 static struct bpf_program *
8199 __bpf_program__iter(const struct bpf_program *p, const struct bpf_object *obj,
8200 bool forward)
8201 {
8202 size_t nr_programs = obj->nr_programs;
8203 ssize_t idx;
8204
8205 if (!nr_programs)
8206 return NULL;
8207
8208 if (!p)
8209 /* Iter from the beginning */
8210 return forward ? &obj->programs[0] :
8211 &obj->programs[nr_programs - 1];
8212
8213 if (p->obj != obj) {
8214 pr_warn("error: program handler doesn't match object\n");
8215 return NULL;
8216 }
8217
8218 idx = (p - obj->programs) + (forward ? 1 : -1);
8219 if (idx >= obj->nr_programs || idx < 0)
8220 return NULL;
8221 return &obj->programs[idx];
8222 }
8223
8224 struct bpf_program *
8225 bpf_program__next(struct bpf_program *prev, const struct bpf_object *obj)
8226 {
8227 struct bpf_program *prog = prev;
8228
8229 do {
8230 prog = __bpf_program__iter(prog, obj, true);
8231 } while (prog && prog_is_subprog(obj, prog));
8232
8233 return prog;
8234 }
8235
8236 struct bpf_program *
8237 bpf_program__prev(struct bpf_program *next, const struct bpf_object *obj)
8238 {
8239 struct bpf_program *prog = next;
8240
8241 do {
8242 prog = __bpf_program__iter(prog, obj, false);
8243 } while (prog && prog_is_subprog(obj, prog));
8244
8245 return prog;
8246 }
8247
8248 int bpf_program__set_priv(struct bpf_program *prog, void *priv,
8249 bpf_program_clear_priv_t clear_priv)
8250 {
8251 if (prog->priv && prog->clear_priv)
8252 prog->clear_priv(prog, prog->priv);
8253
8254 prog->priv = priv;
8255 prog->clear_priv = clear_priv;
8256 return 0;
8257 }
8258
8259 void *bpf_program__priv(const struct bpf_program *prog)
8260 {
8261 return prog ? prog->priv : ERR_PTR(-EINVAL);
8262 }
8263
8264 void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex)
8265 {
8266 prog->prog_ifindex = ifindex;
8267 }
8268
8269 const char *bpf_program__name(const struct bpf_program *prog)
8270 {
8271 return prog->name;
8272 }
8273
8274 const char *bpf_program__section_name(const struct bpf_program *prog)
8275 {
8276 return prog->sec_name;
8277 }
8278
8279 const char *bpf_program__title(const struct bpf_program *prog, bool needs_copy)
8280 {
8281 const char *title;
8282
8283 title = prog->sec_name;
8284 if (needs_copy) {
8285 title = strdup(title);
8286 if (!title) {
8287 pr_warn("failed to strdup program title\n");
8288 return ERR_PTR(-ENOMEM);
8289 }
8290 }
8291
8292 return title;
8293 }
8294
8295 bool bpf_program__autoload(const struct bpf_program *prog)
8296 {
8297 return prog->load;
8298 }
8299
8300 int bpf_program__set_autoload(struct bpf_program *prog, bool autoload)
8301 {
8302 if (prog->obj->loaded)
8303 return -EINVAL;
8304
8305 prog->load = autoload;
8306 return 0;
8307 }
8308
8309 int bpf_program__fd(const struct bpf_program *prog)
8310 {
8311 return bpf_program__nth_fd(prog, 0);
8312 }
8313
8314 size_t bpf_program__size(const struct bpf_program *prog)
8315 {
8316 return prog->insns_cnt * BPF_INSN_SZ;
8317 }
8318
8319 int bpf_program__set_prep(struct bpf_program *prog, int nr_instances,
8320 bpf_program_prep_t prep)
8321 {
8322 int *instances_fds;
8323
8324 if (nr_instances <= 0 || !prep)
8325 return -EINVAL;
8326
8327 if (prog->instances.nr > 0 || prog->instances.fds) {
8328 pr_warn("Can't set pre-processor after loading\n");
8329 return -EINVAL;
8330 }
8331
8332 instances_fds = malloc(sizeof(int) * nr_instances);
8333 if (!instances_fds) {
8334 pr_warn("alloc memory failed for fds\n");
8335 return -ENOMEM;
8336 }
8337
8338 /* fill all fd with -1 */
8339 memset(instances_fds, -1, sizeof(int) * nr_instances);
8340
8341 prog->instances.nr = nr_instances;
8342 prog->instances.fds = instances_fds;
8343 prog->preprocessor = prep;
8344 return 0;
8345 }
8346
8347 int bpf_program__nth_fd(const struct bpf_program *prog, int n)
8348 {
8349 int fd;
8350
8351 if (!prog)
8352 return -EINVAL;
8353
8354 if (n >= prog->instances.nr || n < 0) {
8355 pr_warn("Can't get the %dth fd from program %s: only %d instances\n",
8356 n, prog->name, prog->instances.nr);
8357 return -EINVAL;
8358 }
8359
8360 fd = prog->instances.fds[n];
8361 if (fd < 0) {
8362 pr_warn("%dth instance of program '%s' is invalid\n",
8363 n, prog->name);
8364 return -ENOENT;
8365 }
8366
8367 return fd;
8368 }
8369
8370 enum bpf_prog_type bpf_program__get_type(struct bpf_program *prog)
8371 {
8372 return prog->type;
8373 }
8374
8375 void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type)
8376 {
8377 prog->type = type;
8378 }
8379
8380 static bool bpf_program__is_type(const struct bpf_program *prog,
8381 enum bpf_prog_type type)
8382 {
8383 return prog ? (prog->type == type) : false;
8384 }
8385
8386 #define BPF_PROG_TYPE_FNS(NAME, TYPE) \
8387 int bpf_program__set_##NAME(struct bpf_program *prog) \
8388 { \
8389 if (!prog) \
8390 return -EINVAL; \
8391 bpf_program__set_type(prog, TYPE); \
8392 return 0; \
8393 } \
8394 \
8395 bool bpf_program__is_##NAME(const struct bpf_program *prog) \
8396 { \
8397 return bpf_program__is_type(prog, TYPE); \
8398 } \
8399
8400 BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER);
8401 BPF_PROG_TYPE_FNS(lsm, BPF_PROG_TYPE_LSM);
8402 BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE);
8403 BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS);
8404 BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT);
8405 BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT);
8406 BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT);
8407 BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP);
8408 BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT);
8409 BPF_PROG_TYPE_FNS(tracing, BPF_PROG_TYPE_TRACING);
8410 BPF_PROG_TYPE_FNS(struct_ops, BPF_PROG_TYPE_STRUCT_OPS);
8411 BPF_PROG_TYPE_FNS(extension, BPF_PROG_TYPE_EXT);
8412 BPF_PROG_TYPE_FNS(sk_lookup, BPF_PROG_TYPE_SK_LOOKUP);
8413
8414 enum bpf_attach_type
8415 bpf_program__get_expected_attach_type(struct bpf_program *prog)
8416 {
8417 return prog->expected_attach_type;
8418 }
8419
8420 void bpf_program__set_expected_attach_type(struct bpf_program *prog,
8421 enum bpf_attach_type type)
8422 {
8423 prog->expected_attach_type = type;
8424 }
8425
8426 #define BPF_PROG_SEC_IMPL(string, ptype, eatype, eatype_optional, \
8427 attachable, attach_btf) \
8428 { \
8429 .sec = string, \
8430 .len = sizeof(string) - 1, \
8431 .prog_type = ptype, \
8432 .expected_attach_type = eatype, \
8433 .is_exp_attach_type_optional = eatype_optional, \
8434 .is_attachable = attachable, \
8435 .is_attach_btf = attach_btf, \
8436 }
8437
8438 /* Programs that can NOT be attached. */
8439 #define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_IMPL(string, ptype, 0, 0, 0, 0)
8440
8441 /* Programs that can be attached. */
8442 #define BPF_APROG_SEC(string, ptype, atype) \
8443 BPF_PROG_SEC_IMPL(string, ptype, atype, true, 1, 0)
8444
8445 /* Programs that must specify expected attach type at load time. */
8446 #define BPF_EAPROG_SEC(string, ptype, eatype) \
8447 BPF_PROG_SEC_IMPL(string, ptype, eatype, false, 1, 0)
8448
8449 /* Programs that use BTF to identify attach point */
8450 #define BPF_PROG_BTF(string, ptype, eatype) \
8451 BPF_PROG_SEC_IMPL(string, ptype, eatype, false, 0, 1)
8452
8453 /* Programs that can be attached but attach type can't be identified by section
8454 * name. Kept for backward compatibility.
8455 */
8456 #define BPF_APROG_COMPAT(string, ptype) BPF_PROG_SEC(string, ptype)
8457
8458 #define SEC_DEF(sec_pfx, ptype, ...) { \
8459 .sec = sec_pfx, \
8460 .len = sizeof(sec_pfx) - 1, \
8461 .prog_type = BPF_PROG_TYPE_##ptype, \
8462 __VA_ARGS__ \
8463 }
8464
8465 static struct bpf_link *attach_kprobe(const struct bpf_sec_def *sec,
8466 struct bpf_program *prog);
8467 static struct bpf_link *attach_tp(const struct bpf_sec_def *sec,
8468 struct bpf_program *prog);
8469 static struct bpf_link *attach_raw_tp(const struct bpf_sec_def *sec,
8470 struct bpf_program *prog);
8471 static struct bpf_link *attach_trace(const struct bpf_sec_def *sec,
8472 struct bpf_program *prog);
8473 static struct bpf_link *attach_lsm(const struct bpf_sec_def *sec,
8474 struct bpf_program *prog);
8475 static struct bpf_link *attach_iter(const struct bpf_sec_def *sec,
8476 struct bpf_program *prog);
8477
8478 static const struct bpf_sec_def section_defs[] = {
8479 BPF_PROG_SEC("socket", BPF_PROG_TYPE_SOCKET_FILTER),
8480 BPF_PROG_SEC("sk_reuseport", BPF_PROG_TYPE_SK_REUSEPORT),
8481 SEC_DEF("kprobe/", KPROBE,
8482 .attach_fn = attach_kprobe),
8483 BPF_PROG_SEC("uprobe/", BPF_PROG_TYPE_KPROBE),
8484 SEC_DEF("kretprobe/", KPROBE,
8485 .attach_fn = attach_kprobe),
8486 BPF_PROG_SEC("uretprobe/", BPF_PROG_TYPE_KPROBE),
8487 BPF_PROG_SEC("classifier", BPF_PROG_TYPE_SCHED_CLS),
8488 BPF_PROG_SEC("action", BPF_PROG_TYPE_SCHED_ACT),
8489 SEC_DEF("tracepoint/", TRACEPOINT,
8490 .attach_fn = attach_tp),
8491 SEC_DEF("tp/", TRACEPOINT,
8492 .attach_fn = attach_tp),
8493 SEC_DEF("raw_tracepoint/", RAW_TRACEPOINT,
8494 .attach_fn = attach_raw_tp),
8495 SEC_DEF("raw_tp/", RAW_TRACEPOINT,
8496 .attach_fn = attach_raw_tp),
8497 SEC_DEF("tp_btf/", TRACING,
8498 .expected_attach_type = BPF_TRACE_RAW_TP,
8499 .is_attach_btf = true,
8500 .attach_fn = attach_trace),
8501 SEC_DEF("fentry/", TRACING,
8502 .expected_attach_type = BPF_TRACE_FENTRY,
8503 .is_attach_btf = true,
8504 .attach_fn = attach_trace),
8505 SEC_DEF("fmod_ret/", TRACING,
8506 .expected_attach_type = BPF_MODIFY_RETURN,
8507 .is_attach_btf = true,
8508 .attach_fn = attach_trace),
8509 SEC_DEF("fexit/", TRACING,
8510 .expected_attach_type = BPF_TRACE_FEXIT,
8511 .is_attach_btf = true,
8512 .attach_fn = attach_trace),
8513 SEC_DEF("fentry.s/", TRACING,
8514 .expected_attach_type = BPF_TRACE_FENTRY,
8515 .is_attach_btf = true,
8516 .is_sleepable = true,
8517 .attach_fn = attach_trace),
8518 SEC_DEF("fmod_ret.s/", TRACING,
8519 .expected_attach_type = BPF_MODIFY_RETURN,
8520 .is_attach_btf = true,
8521 .is_sleepable = true,
8522 .attach_fn = attach_trace),
8523 SEC_DEF("fexit.s/", TRACING,
8524 .expected_attach_type = BPF_TRACE_FEXIT,
8525 .is_attach_btf = true,
8526 .is_sleepable = true,
8527 .attach_fn = attach_trace),
8528 SEC_DEF("freplace/", EXT,
8529 .is_attach_btf = true,
8530 .attach_fn = attach_trace),
8531 SEC_DEF("lsm/", LSM,
8532 .is_attach_btf = true,
8533 .expected_attach_type = BPF_LSM_MAC,
8534 .attach_fn = attach_lsm),
8535 SEC_DEF("lsm.s/", LSM,
8536 .is_attach_btf = true,
8537 .is_sleepable = true,
8538 .expected_attach_type = BPF_LSM_MAC,
8539 .attach_fn = attach_lsm),
8540 SEC_DEF("iter/", TRACING,
8541 .expected_attach_type = BPF_TRACE_ITER,
8542 .is_attach_btf = true,
8543 .attach_fn = attach_iter),
8544 BPF_EAPROG_SEC("xdp_devmap/", BPF_PROG_TYPE_XDP,
8545 BPF_XDP_DEVMAP),
8546 BPF_EAPROG_SEC("xdp_cpumap/", BPF_PROG_TYPE_XDP,
8547 BPF_XDP_CPUMAP),
8548 BPF_APROG_SEC("xdp", BPF_PROG_TYPE_XDP,
8549 BPF_XDP),
8550 BPF_PROG_SEC("perf_event", BPF_PROG_TYPE_PERF_EVENT),
8551 BPF_PROG_SEC("lwt_in", BPF_PROG_TYPE_LWT_IN),
8552 BPF_PROG_SEC("lwt_out", BPF_PROG_TYPE_LWT_OUT),
8553 BPF_PROG_SEC("lwt_xmit", BPF_PROG_TYPE_LWT_XMIT),
8554 BPF_PROG_SEC("lwt_seg6local", BPF_PROG_TYPE_LWT_SEG6LOCAL),
8555 BPF_APROG_SEC("cgroup_skb/ingress", BPF_PROG_TYPE_CGROUP_SKB,
8556 BPF_CGROUP_INET_INGRESS),
8557 BPF_APROG_SEC("cgroup_skb/egress", BPF_PROG_TYPE_CGROUP_SKB,
8558 BPF_CGROUP_INET_EGRESS),
8559 BPF_APROG_COMPAT("cgroup/skb", BPF_PROG_TYPE_CGROUP_SKB),
8560 BPF_EAPROG_SEC("cgroup/sock_create", BPF_PROG_TYPE_CGROUP_SOCK,
8561 BPF_CGROUP_INET_SOCK_CREATE),
8562 BPF_EAPROG_SEC("cgroup/sock_release", BPF_PROG_TYPE_CGROUP_SOCK,
8563 BPF_CGROUP_INET_SOCK_RELEASE),
8564 BPF_APROG_SEC("cgroup/sock", BPF_PROG_TYPE_CGROUP_SOCK,
8565 BPF_CGROUP_INET_SOCK_CREATE),
8566 BPF_EAPROG_SEC("cgroup/post_bind4", BPF_PROG_TYPE_CGROUP_SOCK,
8567 BPF_CGROUP_INET4_POST_BIND),
8568 BPF_EAPROG_SEC("cgroup/post_bind6", BPF_PROG_TYPE_CGROUP_SOCK,
8569 BPF_CGROUP_INET6_POST_BIND),
8570 BPF_APROG_SEC("cgroup/dev", BPF_PROG_TYPE_CGROUP_DEVICE,
8571 BPF_CGROUP_DEVICE),
8572 BPF_APROG_SEC("sockops", BPF_PROG_TYPE_SOCK_OPS,
8573 BPF_CGROUP_SOCK_OPS),
8574 BPF_APROG_SEC("sk_skb/stream_parser", BPF_PROG_TYPE_SK_SKB,
8575 BPF_SK_SKB_STREAM_PARSER),
8576 BPF_APROG_SEC("sk_skb/stream_verdict", BPF_PROG_TYPE_SK_SKB,
8577 BPF_SK_SKB_STREAM_VERDICT),
8578 BPF_APROG_COMPAT("sk_skb", BPF_PROG_TYPE_SK_SKB),
8579 BPF_APROG_SEC("sk_msg", BPF_PROG_TYPE_SK_MSG,
8580 BPF_SK_MSG_VERDICT),
8581 BPF_APROG_SEC("lirc_mode2", BPF_PROG_TYPE_LIRC_MODE2,
8582 BPF_LIRC_MODE2),
8583 BPF_APROG_SEC("flow_dissector", BPF_PROG_TYPE_FLOW_DISSECTOR,
8584 BPF_FLOW_DISSECTOR),
8585 BPF_EAPROG_SEC("cgroup/bind4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8586 BPF_CGROUP_INET4_BIND),
8587 BPF_EAPROG_SEC("cgroup/bind6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8588 BPF_CGROUP_INET6_BIND),
8589 BPF_EAPROG_SEC("cgroup/connect4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8590 BPF_CGROUP_INET4_CONNECT),
8591 BPF_EAPROG_SEC("cgroup/connect6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8592 BPF_CGROUP_INET6_CONNECT),
8593 BPF_EAPROG_SEC("cgroup/sendmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8594 BPF_CGROUP_UDP4_SENDMSG),
8595 BPF_EAPROG_SEC("cgroup/sendmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8596 BPF_CGROUP_UDP6_SENDMSG),
8597 BPF_EAPROG_SEC("cgroup/recvmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8598 BPF_CGROUP_UDP4_RECVMSG),
8599 BPF_EAPROG_SEC("cgroup/recvmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8600 BPF_CGROUP_UDP6_RECVMSG),
8601 BPF_EAPROG_SEC("cgroup/getpeername4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8602 BPF_CGROUP_INET4_GETPEERNAME),
8603 BPF_EAPROG_SEC("cgroup/getpeername6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8604 BPF_CGROUP_INET6_GETPEERNAME),
8605 BPF_EAPROG_SEC("cgroup/getsockname4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8606 BPF_CGROUP_INET4_GETSOCKNAME),
8607 BPF_EAPROG_SEC("cgroup/getsockname6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8608 BPF_CGROUP_INET6_GETSOCKNAME),
8609 BPF_EAPROG_SEC("cgroup/sysctl", BPF_PROG_TYPE_CGROUP_SYSCTL,
8610 BPF_CGROUP_SYSCTL),
8611 BPF_EAPROG_SEC("cgroup/getsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT,
8612 BPF_CGROUP_GETSOCKOPT),
8613 BPF_EAPROG_SEC("cgroup/setsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT,
8614 BPF_CGROUP_SETSOCKOPT),
8615 BPF_PROG_SEC("struct_ops", BPF_PROG_TYPE_STRUCT_OPS),
8616 BPF_EAPROG_SEC("sk_lookup/", BPF_PROG_TYPE_SK_LOOKUP,
8617 BPF_SK_LOOKUP),
8618 };
8619
8620 #undef BPF_PROG_SEC_IMPL
8621 #undef BPF_PROG_SEC
8622 #undef BPF_APROG_SEC
8623 #undef BPF_EAPROG_SEC
8624 #undef BPF_APROG_COMPAT
8625 #undef SEC_DEF
8626
8627 #define MAX_TYPE_NAME_SIZE 32
8628
8629 static const struct bpf_sec_def *find_sec_def(const char *sec_name)
8630 {
8631 int i, n = ARRAY_SIZE(section_defs);
8632
8633 for (i = 0; i < n; i++) {
8634 if (strncmp(sec_name,
8635 section_defs[i].sec, section_defs[i].len))
8636 continue;
8637 return &section_defs[i];
8638 }
8639 return NULL;
8640 }
8641
8642 static char *libbpf_get_type_names(bool attach_type)
8643 {
8644 int i, len = ARRAY_SIZE(section_defs) * MAX_TYPE_NAME_SIZE;
8645 char *buf;
8646
8647 buf = malloc(len);
8648 if (!buf)
8649 return NULL;
8650
8651 buf[0] = '\0';
8652 /* Forge string buf with all available names */
8653 for (i = 0; i < ARRAY_SIZE(section_defs); i++) {
8654 if (attach_type && !section_defs[i].is_attachable)
8655 continue;
8656
8657 if (strlen(buf) + strlen(section_defs[i].sec) + 2 > len) {
8658 free(buf);
8659 return NULL;
8660 }
8661 strcat(buf, " ");
8662 strcat(buf, section_defs[i].sec);
8663 }
8664
8665 return buf;
8666 }
8667
8668 int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type,
8669 enum bpf_attach_type *expected_attach_type)
8670 {
8671 const struct bpf_sec_def *sec_def;
8672 char *type_names;
8673
8674 if (!name)
8675 return -EINVAL;
8676
8677 sec_def = find_sec_def(name);
8678 if (sec_def) {
8679 *prog_type = sec_def->prog_type;
8680 *expected_attach_type = sec_def->expected_attach_type;
8681 return 0;
8682 }
8683
8684 pr_debug("failed to guess program type from ELF section '%s'\n", name);
8685 type_names = libbpf_get_type_names(false);
8686 if (type_names != NULL) {
8687 pr_debug("supported section(type) names are:%s\n", type_names);
8688 free(type_names);
8689 }
8690
8691 return -ESRCH;
8692 }
8693
8694 static struct bpf_map *find_struct_ops_map_by_offset(struct bpf_object *obj,
8695 size_t offset)
8696 {
8697 struct bpf_map *map;
8698 size_t i;
8699
8700 for (i = 0; i < obj->nr_maps; i++) {
8701 map = &obj->maps[i];
8702 if (!bpf_map__is_struct_ops(map))
8703 continue;
8704 if (map->sec_offset <= offset &&
8705 offset - map->sec_offset < map->def.value_size)
8706 return map;
8707 }
8708
8709 return NULL;
8710 }
8711
8712 /* Collect the reloc from ELF and populate the st_ops->progs[] */
8713 static int bpf_object__collect_st_ops_relos(struct bpf_object *obj,
8714 GElf_Shdr *shdr, Elf_Data *data)
8715 {
8716 const struct btf_member *member;
8717 struct bpf_struct_ops *st_ops;
8718 struct bpf_program *prog;
8719 unsigned int shdr_idx;
8720 const struct btf *btf;
8721 struct bpf_map *map;
8722 Elf_Data *symbols;
8723 unsigned int moff, insn_idx;
8724 const char *name;
8725 __u32 member_idx;
8726 GElf_Sym sym;
8727 GElf_Rel rel;
8728 int i, nrels;
8729
8730 symbols = obj->efile.symbols;
8731 btf = obj->btf;
8732 nrels = shdr->sh_size / shdr->sh_entsize;
8733 for (i = 0; i < nrels; i++) {
8734 if (!gelf_getrel(data, i, &rel)) {
8735 pr_warn("struct_ops reloc: failed to get %d reloc\n", i);
8736 return -LIBBPF_ERRNO__FORMAT;
8737 }
8738
8739 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) {
8740 pr_warn("struct_ops reloc: symbol %zx not found\n",
8741 (size_t)GELF_R_SYM(rel.r_info));
8742 return -LIBBPF_ERRNO__FORMAT;
8743 }
8744
8745 name = elf_sym_str(obj, sym.st_name) ?: "<?>";
8746 map = find_struct_ops_map_by_offset(obj, rel.r_offset);
8747 if (!map) {
8748 pr_warn("struct_ops reloc: cannot find map at rel.r_offset %zu\n",
8749 (size_t)rel.r_offset);
8750 return -EINVAL;
8751 }
8752
8753 moff = rel.r_offset - map->sec_offset;
8754 shdr_idx = sym.st_shndx;
8755 st_ops = map->st_ops;
8756 pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel.r_offset %zu map->sec_offset %zu name %d (\'%s\')\n",
8757 map->name,
8758 (long long)(rel.r_info >> 32),
8759 (long long)sym.st_value,
8760 shdr_idx, (size_t)rel.r_offset,
8761 map->sec_offset, sym.st_name, name);
8762
8763 if (shdr_idx >= SHN_LORESERVE) {
8764 pr_warn("struct_ops reloc %s: rel.r_offset %zu shdr_idx %u unsupported non-static function\n",
8765 map->name, (size_t)rel.r_offset, shdr_idx);
8766 return -LIBBPF_ERRNO__RELOC;
8767 }
8768 if (sym.st_value % BPF_INSN_SZ) {
8769 pr_warn("struct_ops reloc %s: invalid target program offset %llu\n",
8770 map->name, (unsigned long long)sym.st_value);
8771 return -LIBBPF_ERRNO__FORMAT;
8772 }
8773 insn_idx = sym.st_value / BPF_INSN_SZ;
8774
8775 member = find_member_by_offset(st_ops->type, moff * 8);
8776 if (!member) {
8777 pr_warn("struct_ops reloc %s: cannot find member at moff %u\n",
8778 map->name, moff);
8779 return -EINVAL;
8780 }
8781 member_idx = member - btf_members(st_ops->type);
8782 name = btf__name_by_offset(btf, member->name_off);
8783
8784 if (!resolve_func_ptr(btf, member->type, NULL)) {
8785 pr_warn("struct_ops reloc %s: cannot relocate non func ptr %s\n",
8786 map->name, name);
8787 return -EINVAL;
8788 }
8789
8790 prog = find_prog_by_sec_insn(obj, shdr_idx, insn_idx);
8791 if (!prog) {
8792 pr_warn("struct_ops reloc %s: cannot find prog at shdr_idx %u to relocate func ptr %s\n",
8793 map->name, shdr_idx, name);
8794 return -EINVAL;
8795 }
8796
8797 if (prog->type == BPF_PROG_TYPE_UNSPEC) {
8798 const struct bpf_sec_def *sec_def;
8799
8800 sec_def = find_sec_def(prog->sec_name);
8801 if (sec_def &&
8802 sec_def->prog_type != BPF_PROG_TYPE_STRUCT_OPS) {
8803 /* for pr_warn */
8804 prog->type = sec_def->prog_type;
8805 goto invalid_prog;
8806 }
8807
8808 prog->type = BPF_PROG_TYPE_STRUCT_OPS;
8809 prog->attach_btf_id = st_ops->type_id;
8810 prog->expected_attach_type = member_idx;
8811 } else if (prog->type != BPF_PROG_TYPE_STRUCT_OPS ||
8812 prog->attach_btf_id != st_ops->type_id ||
8813 prog->expected_attach_type != member_idx) {
8814 goto invalid_prog;
8815 }
8816 st_ops->progs[member_idx] = prog;
8817 }
8818
8819 return 0;
8820
8821 invalid_prog:
8822 pr_warn("struct_ops reloc %s: cannot use prog %s in sec %s with type %u attach_btf_id %u expected_attach_type %u for func ptr %s\n",
8823 map->name, prog->name, prog->sec_name, prog->type,
8824 prog->attach_btf_id, prog->expected_attach_type, name);
8825 return -EINVAL;
8826 }
8827
8828 #define BTF_TRACE_PREFIX "btf_trace_"
8829 #define BTF_LSM_PREFIX "bpf_lsm_"
8830 #define BTF_ITER_PREFIX "bpf_iter_"
8831 #define BTF_MAX_NAME_SIZE 128
8832
8833 static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix,
8834 const char *name, __u32 kind)
8835 {
8836 char btf_type_name[BTF_MAX_NAME_SIZE];
8837 int ret;
8838
8839 ret = snprintf(btf_type_name, sizeof(btf_type_name),
8840 "%s%s", prefix, name);
8841 /* snprintf returns the number of characters written excluding the
8842 * the terminating null. So, if >= BTF_MAX_NAME_SIZE are written, it
8843 * indicates truncation.
8844 */
8845 if (ret < 0 || ret >= sizeof(btf_type_name))
8846 return -ENAMETOOLONG;
8847 return btf__find_by_name_kind(btf, btf_type_name, kind);
8848 }
8849
8850 static inline int find_attach_btf_id(struct btf *btf, const char *name,
8851 enum bpf_attach_type attach_type)
8852 {
8853 int err;
8854
8855 if (attach_type == BPF_TRACE_RAW_TP)
8856 err = find_btf_by_prefix_kind(btf, BTF_TRACE_PREFIX, name,
8857 BTF_KIND_TYPEDEF);
8858 else if (attach_type == BPF_LSM_MAC)
8859 err = find_btf_by_prefix_kind(btf, BTF_LSM_PREFIX, name,
8860 BTF_KIND_FUNC);
8861 else if (attach_type == BPF_TRACE_ITER)
8862 err = find_btf_by_prefix_kind(btf, BTF_ITER_PREFIX, name,
8863 BTF_KIND_FUNC);
8864 else
8865 err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC);
8866
8867 return err;
8868 }
8869
8870 int libbpf_find_vmlinux_btf_id(const char *name,
8871 enum bpf_attach_type attach_type)
8872 {
8873 struct btf *btf;
8874 int err;
8875
8876 btf = libbpf_find_kernel_btf();
8877 if (IS_ERR(btf)) {
8878 pr_warn("vmlinux BTF is not found\n");
8879 return -EINVAL;
8880 }
8881
8882 err = find_attach_btf_id(btf, name, attach_type);
8883 if (err <= 0)
8884 pr_warn("%s is not found in vmlinux BTF\n", name);
8885
8886 btf__free(btf);
8887 return err;
8888 }
8889
8890 static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd)
8891 {
8892 struct bpf_prog_info_linear *info_linear;
8893 struct bpf_prog_info *info;
8894 struct btf *btf = NULL;
8895 int err = -EINVAL;
8896
8897 info_linear = bpf_program__get_prog_info_linear(attach_prog_fd, 0);
8898 if (IS_ERR_OR_NULL(info_linear)) {
8899 pr_warn("failed get_prog_info_linear for FD %d\n",
8900 attach_prog_fd);
8901 return -EINVAL;
8902 }
8903 info = &info_linear->info;
8904 if (!info->btf_id) {
8905 pr_warn("The target program doesn't have BTF\n");
8906 goto out;
8907 }
8908 if (btf__get_from_id(info->btf_id, &btf)) {
8909 pr_warn("Failed to get BTF of the program\n");
8910 goto out;
8911 }
8912 err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC);
8913 btf__free(btf);
8914 if (err <= 0) {
8915 pr_warn("%s is not found in prog's BTF\n", name);
8916 goto out;
8917 }
8918 out:
8919 free(info_linear);
8920 return err;
8921 }
8922
8923 static int find_kernel_btf_id(struct bpf_object *obj, const char *attach_name,
8924 enum bpf_attach_type attach_type,
8925 int *btf_obj_fd, int *btf_type_id)
8926 {
8927 int ret, i;
8928
8929 ret = find_attach_btf_id(obj->btf_vmlinux, attach_name, attach_type);
8930 if (ret > 0) {
8931 *btf_obj_fd = 0; /* vmlinux BTF */
8932 *btf_type_id = ret;
8933 return 0;
8934 }
8935 if (ret != -ENOENT)
8936 return ret;
8937
8938 ret = load_module_btfs(obj);
8939 if (ret)
8940 return ret;
8941
8942 for (i = 0; i < obj->btf_module_cnt; i++) {
8943 const struct module_btf *mod = &obj->btf_modules[i];
8944
8945 ret = find_attach_btf_id(mod->btf, attach_name, attach_type);
8946 if (ret > 0) {
8947 *btf_obj_fd = mod->fd;
8948 *btf_type_id = ret;
8949 return 0;
8950 }
8951 if (ret == -ENOENT)
8952 continue;
8953
8954 return ret;
8955 }
8956
8957 return -ESRCH;
8958 }
8959
8960 static int libbpf_find_attach_btf_id(struct bpf_program *prog, int *btf_obj_fd, int *btf_type_id)
8961 {
8962 enum bpf_attach_type attach_type = prog->expected_attach_type;
8963 __u32 attach_prog_fd = prog->attach_prog_fd;
8964 const char *name = prog->sec_name, *attach_name;
8965 const struct bpf_sec_def *sec = NULL;
8966 int i, err;
8967
8968 if (!name)
8969 return -EINVAL;
8970
8971 for (i = 0; i < ARRAY_SIZE(section_defs); i++) {
8972 if (!section_defs[i].is_attach_btf)
8973 continue;
8974 if (strncmp(name, section_defs[i].sec, section_defs[i].len))
8975 continue;
8976
8977 sec = &section_defs[i];
8978 break;
8979 }
8980
8981 if (!sec) {
8982 pr_warn("failed to identify BTF ID based on ELF section name '%s'\n", name);
8983 return -ESRCH;
8984 }
8985 attach_name = name + sec->len;
8986
8987 /* BPF program's BTF ID */
8988 if (attach_prog_fd) {
8989 err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd);
8990 if (err < 0) {
8991 pr_warn("failed to find BPF program (FD %d) BTF ID for '%s': %d\n",
8992 attach_prog_fd, attach_name, err);
8993 return err;
8994 }
8995 *btf_obj_fd = 0;
8996 *btf_type_id = err;
8997 return 0;
8998 }
8999
9000 /* kernel/module BTF ID */
9001 err = find_kernel_btf_id(prog->obj, attach_name, attach_type, btf_obj_fd, btf_type_id);
9002 if (err) {
9003 pr_warn("failed to find kernel BTF type ID of '%s': %d\n", attach_name, err);
9004 return err;
9005 }
9006 return 0;
9007 }
9008
9009 int libbpf_attach_type_by_name(const char *name,
9010 enum bpf_attach_type *attach_type)
9011 {
9012 char *type_names;
9013 int i;
9014
9015 if (!name)
9016 return -EINVAL;
9017
9018 for (i = 0; i < ARRAY_SIZE(section_defs); i++) {
9019 if (strncmp(name, section_defs[i].sec, section_defs[i].len))
9020 continue;
9021 if (!section_defs[i].is_attachable)
9022 return -EINVAL;
9023 *attach_type = section_defs[i].expected_attach_type;
9024 return 0;
9025 }
9026 pr_debug("failed to guess attach type based on ELF section name '%s'\n", name);
9027 type_names = libbpf_get_type_names(true);
9028 if (type_names != NULL) {
9029 pr_debug("attachable section(type) names are:%s\n", type_names);
9030 free(type_names);
9031 }
9032
9033 return -EINVAL;
9034 }
9035
9036 int bpf_map__fd(const struct bpf_map *map)
9037 {
9038 return map ? map->fd : -EINVAL;
9039 }
9040
9041 const struct bpf_map_def *bpf_map__def(const struct bpf_map *map)
9042 {
9043 return map ? &map->def : ERR_PTR(-EINVAL);
9044 }
9045
9046 const char *bpf_map__name(const struct bpf_map *map)
9047 {
9048 return map ? map->name : NULL;
9049 }
9050
9051 enum bpf_map_type bpf_map__type(const struct bpf_map *map)
9052 {
9053 return map->def.type;
9054 }
9055
9056 int bpf_map__set_type(struct bpf_map *map, enum bpf_map_type type)
9057 {
9058 if (map->fd >= 0)
9059 return -EBUSY;
9060 map->def.type = type;
9061 return 0;
9062 }
9063
9064 __u32 bpf_map__map_flags(const struct bpf_map *map)
9065 {
9066 return map->def.map_flags;
9067 }
9068
9069 int bpf_map__set_map_flags(struct bpf_map *map, __u32 flags)
9070 {
9071 if (map->fd >= 0)
9072 return -EBUSY;
9073 map->def.map_flags = flags;
9074 return 0;
9075 }
9076
9077 __u32 bpf_map__numa_node(const struct bpf_map *map)
9078 {
9079 return map->numa_node;
9080 }
9081
9082 int bpf_map__set_numa_node(struct bpf_map *map, __u32 numa_node)
9083 {
9084 if (map->fd >= 0)
9085 return -EBUSY;
9086 map->numa_node = numa_node;
9087 return 0;
9088 }
9089
9090 __u32 bpf_map__key_size(const struct bpf_map *map)
9091 {
9092 return map->def.key_size;
9093 }
9094
9095 int bpf_map__set_key_size(struct bpf_map *map, __u32 size)
9096 {
9097 if (map->fd >= 0)
9098 return -EBUSY;
9099 map->def.key_size = size;
9100 return 0;
9101 }
9102
9103 __u32 bpf_map__value_size(const struct bpf_map *map)
9104 {
9105 return map->def.value_size;
9106 }
9107
9108 int bpf_map__set_value_size(struct bpf_map *map, __u32 size)
9109 {
9110 if (map->fd >= 0)
9111 return -EBUSY;
9112 map->def.value_size = size;
9113 return 0;
9114 }
9115
9116 __u32 bpf_map__btf_key_type_id(const struct bpf_map *map)
9117 {
9118 return map ? map->btf_key_type_id : 0;
9119 }
9120
9121 __u32 bpf_map__btf_value_type_id(const struct bpf_map *map)
9122 {
9123 return map ? map->btf_value_type_id : 0;
9124 }
9125
9126 int bpf_map__set_priv(struct bpf_map *map, void *priv,
9127 bpf_map_clear_priv_t clear_priv)
9128 {
9129 if (!map)
9130 return -EINVAL;
9131
9132 if (map->priv) {
9133 if (map->clear_priv)
9134 map->clear_priv(map, map->priv);
9135 }
9136
9137 map->priv = priv;
9138 map->clear_priv = clear_priv;
9139 return 0;
9140 }
9141
9142 void *bpf_map__priv(const struct bpf_map *map)
9143 {
9144 return map ? map->priv : ERR_PTR(-EINVAL);
9145 }
9146
9147 int bpf_map__set_initial_value(struct bpf_map *map,
9148 const void *data, size_t size)
9149 {
9150 if (!map->mmaped || map->libbpf_type == LIBBPF_MAP_KCONFIG ||
9151 size != map->def.value_size || map->fd >= 0)
9152 return -EINVAL;
9153
9154 memcpy(map->mmaped, data, size);
9155 return 0;
9156 }
9157
9158 bool bpf_map__is_offload_neutral(const struct bpf_map *map)
9159 {
9160 return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY;
9161 }
9162
9163 bool bpf_map__is_internal(const struct bpf_map *map)
9164 {
9165 return map->libbpf_type != LIBBPF_MAP_UNSPEC;
9166 }
9167
9168 __u32 bpf_map__ifindex(const struct bpf_map *map)
9169 {
9170 return map->map_ifindex;
9171 }
9172
9173 int bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex)
9174 {
9175 if (map->fd >= 0)
9176 return -EBUSY;
9177 map->map_ifindex = ifindex;
9178 return 0;
9179 }
9180
9181 int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd)
9182 {
9183 if (!bpf_map_type__is_map_in_map(map->def.type)) {
9184 pr_warn("error: unsupported map type\n");
9185 return -EINVAL;
9186 }
9187 if (map->inner_map_fd != -1) {
9188 pr_warn("error: inner_map_fd already specified\n");
9189 return -EINVAL;
9190 }
9191 map->inner_map_fd = fd;
9192 return 0;
9193 }
9194
9195 static struct bpf_map *
9196 __bpf_map__iter(const struct bpf_map *m, const struct bpf_object *obj, int i)
9197 {
9198 ssize_t idx;
9199 struct bpf_map *s, *e;
9200
9201 if (!obj || !obj->maps)
9202 return NULL;
9203
9204 s = obj->maps;
9205 e = obj->maps + obj->nr_maps;
9206
9207 if ((m < s) || (m >= e)) {
9208 pr_warn("error in %s: map handler doesn't belong to object\n",
9209 __func__);
9210 return NULL;
9211 }
9212
9213 idx = (m - obj->maps) + i;
9214 if (idx >= obj->nr_maps || idx < 0)
9215 return NULL;
9216 return &obj->maps[idx];
9217 }
9218
9219 struct bpf_map *
9220 bpf_map__next(const struct bpf_map *prev, const struct bpf_object *obj)
9221 {
9222 if (prev == NULL)
9223 return obj->maps;
9224
9225 return __bpf_map__iter(prev, obj, 1);
9226 }
9227
9228 struct bpf_map *
9229 bpf_map__prev(const struct bpf_map *next, const struct bpf_object *obj)
9230 {
9231 if (next == NULL) {
9232 if (!obj->nr_maps)
9233 return NULL;
9234 return obj->maps + obj->nr_maps - 1;
9235 }
9236
9237 return __bpf_map__iter(next, obj, -1);
9238 }
9239
9240 struct bpf_map *
9241 bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name)
9242 {
9243 struct bpf_map *pos;
9244
9245 bpf_object__for_each_map(pos, obj) {
9246 if (pos->name && !strcmp(pos->name, name))
9247 return pos;
9248 }
9249 return NULL;
9250 }
9251
9252 int
9253 bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name)
9254 {
9255 return bpf_map__fd(bpf_object__find_map_by_name(obj, name));
9256 }
9257
9258 struct bpf_map *
9259 bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset)
9260 {
9261 return ERR_PTR(-ENOTSUP);
9262 }
9263
9264 long libbpf_get_error(const void *ptr)
9265 {
9266 return PTR_ERR_OR_ZERO(ptr);
9267 }
9268
9269 int bpf_prog_load(const char *file, enum bpf_prog_type type,
9270 struct bpf_object **pobj, int *prog_fd)
9271 {
9272 struct bpf_prog_load_attr attr;
9273
9274 memset(&attr, 0, sizeof(struct bpf_prog_load_attr));
9275 attr.file = file;
9276 attr.prog_type = type;
9277 attr.expected_attach_type = 0;
9278
9279 return bpf_prog_load_xattr(&attr, pobj, prog_fd);
9280 }
9281
9282 int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr,
9283 struct bpf_object **pobj, int *prog_fd)
9284 {
9285 struct bpf_object_open_attr open_attr = {};
9286 struct bpf_program *prog, *first_prog = NULL;
9287 struct bpf_object *obj;
9288 struct bpf_map *map;
9289 int err;
9290
9291 if (!attr)
9292 return -EINVAL;
9293 if (!attr->file)
9294 return -EINVAL;
9295
9296 open_attr.file = attr->file;
9297 open_attr.prog_type = attr->prog_type;
9298
9299 obj = bpf_object__open_xattr(&open_attr);
9300 if (IS_ERR_OR_NULL(obj))
9301 return -ENOENT;
9302
9303 bpf_object__for_each_program(prog, obj) {
9304 enum bpf_attach_type attach_type = attr->expected_attach_type;
9305 /*
9306 * to preserve backwards compatibility, bpf_prog_load treats
9307 * attr->prog_type, if specified, as an override to whatever
9308 * bpf_object__open guessed
9309 */
9310 if (attr->prog_type != BPF_PROG_TYPE_UNSPEC) {
9311 bpf_program__set_type(prog, attr->prog_type);
9312 bpf_program__set_expected_attach_type(prog,
9313 attach_type);
9314 }
9315 if (bpf_program__get_type(prog) == BPF_PROG_TYPE_UNSPEC) {
9316 /*
9317 * we haven't guessed from section name and user
9318 * didn't provide a fallback type, too bad...
9319 */
9320 bpf_object__close(obj);
9321 return -EINVAL;
9322 }
9323
9324 prog->prog_ifindex = attr->ifindex;
9325 prog->log_level = attr->log_level;
9326 prog->prog_flags |= attr->prog_flags;
9327 if (!first_prog)
9328 first_prog = prog;
9329 }
9330
9331 bpf_object__for_each_map(map, obj) {
9332 if (!bpf_map__is_offload_neutral(map))
9333 map->map_ifindex = attr->ifindex;
9334 }
9335
9336 if (!first_prog) {
9337 pr_warn("object file doesn't contain bpf program\n");
9338 bpf_object__close(obj);
9339 return -ENOENT;
9340 }
9341
9342 err = bpf_object__load(obj);
9343 if (err) {
9344 bpf_object__close(obj);
9345 return err;
9346 }
9347
9348 *pobj = obj;
9349 *prog_fd = bpf_program__fd(first_prog);
9350 return 0;
9351 }
9352
9353 struct bpf_link {
9354 int (*detach)(struct bpf_link *link);
9355 int (*destroy)(struct bpf_link *link);
9356 char *pin_path; /* NULL, if not pinned */
9357 int fd; /* hook FD, -1 if not applicable */
9358 bool disconnected;
9359 };
9360
9361 /* Replace link's underlying BPF program with the new one */
9362 int bpf_link__update_program(struct bpf_link *link, struct bpf_program *prog)
9363 {
9364 return bpf_link_update(bpf_link__fd(link), bpf_program__fd(prog), NULL);
9365 }
9366
9367 /* Release "ownership" of underlying BPF resource (typically, BPF program
9368 * attached to some BPF hook, e.g., tracepoint, kprobe, etc). Disconnected
9369 * link, when destructed through bpf_link__destroy() call won't attempt to
9370 * detach/unregisted that BPF resource. This is useful in situations where,
9371 * say, attached BPF program has to outlive userspace program that attached it
9372 * in the system. Depending on type of BPF program, though, there might be
9373 * additional steps (like pinning BPF program in BPF FS) necessary to ensure
9374 * exit of userspace program doesn't trigger automatic detachment and clean up
9375 * inside the kernel.
9376 */
9377 void bpf_link__disconnect(struct bpf_link *link)
9378 {
9379 link->disconnected = true;
9380 }
9381
9382 int bpf_link__destroy(struct bpf_link *link)
9383 {
9384 int err = 0;
9385
9386 if (IS_ERR_OR_NULL(link))
9387 return 0;
9388
9389 if (!link->disconnected && link->detach)
9390 err = link->detach(link);
9391 if (link->destroy)
9392 link->destroy(link);
9393 if (link->pin_path)
9394 free(link->pin_path);
9395 free(link);
9396
9397 return err;
9398 }
9399
9400 int bpf_link__fd(const struct bpf_link *link)
9401 {
9402 return link->fd;
9403 }
9404
9405 const char *bpf_link__pin_path(const struct bpf_link *link)
9406 {
9407 return link->pin_path;
9408 }
9409
9410 static int bpf_link__detach_fd(struct bpf_link *link)
9411 {
9412 return close(link->fd);
9413 }
9414
9415 struct bpf_link *bpf_link__open(const char *path)
9416 {
9417 struct bpf_link *link;
9418 int fd;
9419
9420 fd = bpf_obj_get(path);
9421 if (fd < 0) {
9422 fd = -errno;
9423 pr_warn("failed to open link at %s: %d\n", path, fd);
9424 return ERR_PTR(fd);
9425 }
9426
9427 link = calloc(1, sizeof(*link));
9428 if (!link) {
9429 close(fd);
9430 return ERR_PTR(-ENOMEM);
9431 }
9432 link->detach = &bpf_link__detach_fd;
9433 link->fd = fd;
9434
9435 link->pin_path = strdup(path);
9436 if (!link->pin_path) {
9437 bpf_link__destroy(link);
9438 return ERR_PTR(-ENOMEM);
9439 }
9440
9441 return link;
9442 }
9443
9444 int bpf_link__detach(struct bpf_link *link)
9445 {
9446 return bpf_link_detach(link->fd) ? -errno : 0;
9447 }
9448
9449 int bpf_link__pin(struct bpf_link *link, const char *path)
9450 {
9451 int err;
9452
9453 if (link->pin_path)
9454 return -EBUSY;
9455 err = make_parent_dir(path);
9456 if (err)
9457 return err;
9458 err = check_path(path);
9459 if (err)
9460 return err;
9461
9462 link->pin_path = strdup(path);
9463 if (!link->pin_path)
9464 return -ENOMEM;
9465
9466 if (bpf_obj_pin(link->fd, link->pin_path)) {
9467 err = -errno;
9468 zfree(&link->pin_path);
9469 return err;
9470 }
9471
9472 pr_debug("link fd=%d: pinned at %s\n", link->fd, link->pin_path);
9473 return 0;
9474 }
9475
9476 int bpf_link__unpin(struct bpf_link *link)
9477 {
9478 int err;
9479
9480 if (!link->pin_path)
9481 return -EINVAL;
9482
9483 err = unlink(link->pin_path);
9484 if (err != 0)
9485 return -errno;
9486
9487 pr_debug("link fd=%d: unpinned from %s\n", link->fd, link->pin_path);
9488 zfree(&link->pin_path);
9489 return 0;
9490 }
9491
9492 static int bpf_link__detach_perf_event(struct bpf_link *link)
9493 {
9494 int err;
9495
9496 err = ioctl(link->fd, PERF_EVENT_IOC_DISABLE, 0);
9497 if (err)
9498 err = -errno;
9499
9500 close(link->fd);
9501 return err;
9502 }
9503
9504 struct bpf_link *bpf_program__attach_perf_event(struct bpf_program *prog,
9505 int pfd)
9506 {
9507 char errmsg[STRERR_BUFSIZE];
9508 struct bpf_link *link;
9509 int prog_fd, err;
9510
9511 if (pfd < 0) {
9512 pr_warn("prog '%s': invalid perf event FD %d\n",
9513 prog->name, pfd);
9514 return ERR_PTR(-EINVAL);
9515 }
9516 prog_fd = bpf_program__fd(prog);
9517 if (prog_fd < 0) {
9518 pr_warn("prog '%s': can't attach BPF program w/o FD (did you load it?)\n",
9519 prog->name);
9520 return ERR_PTR(-EINVAL);
9521 }
9522
9523 link = calloc(1, sizeof(*link));
9524 if (!link)
9525 return ERR_PTR(-ENOMEM);
9526 link->detach = &bpf_link__detach_perf_event;
9527 link->fd = pfd;
9528
9529 if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) {
9530 err = -errno;
9531 free(link);
9532 pr_warn("prog '%s': failed to attach to pfd %d: %s\n",
9533 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9534 if (err == -EPROTO)
9535 pr_warn("prog '%s': try add PERF_SAMPLE_CALLCHAIN to or remove exclude_callchain_[kernel|user] from pfd %d\n",
9536 prog->name, pfd);
9537 return ERR_PTR(err);
9538 }
9539 if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
9540 err = -errno;
9541 free(link);
9542 pr_warn("prog '%s': failed to enable pfd %d: %s\n",
9543 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9544 return ERR_PTR(err);
9545 }
9546 return link;
9547 }
9548
9549 /*
9550 * this function is expected to parse integer in the range of [0, 2^31-1] from
9551 * given file using scanf format string fmt. If actual parsed value is
9552 * negative, the result might be indistinguishable from error
9553 */
9554 static int parse_uint_from_file(const char *file, const char *fmt)
9555 {
9556 char buf[STRERR_BUFSIZE];
9557 int err, ret;
9558 FILE *f;
9559
9560 f = fopen(file, "r");
9561 if (!f) {
9562 err = -errno;
9563 pr_debug("failed to open '%s': %s\n", file,
9564 libbpf_strerror_r(err, buf, sizeof(buf)));
9565 return err;
9566 }
9567 err = fscanf(f, fmt, &ret);
9568 if (err != 1) {
9569 err = err == EOF ? -EIO : -errno;
9570 pr_debug("failed to parse '%s': %s\n", file,
9571 libbpf_strerror_r(err, buf, sizeof(buf)));
9572 fclose(f);
9573 return err;
9574 }
9575 fclose(f);
9576 return ret;
9577 }
9578
9579 static int determine_kprobe_perf_type(void)
9580 {
9581 const char *file = "/sys/bus/event_source/devices/kprobe/type";
9582
9583 return parse_uint_from_file(file, "%d\n");
9584 }
9585
9586 static int determine_uprobe_perf_type(void)
9587 {
9588 const char *file = "/sys/bus/event_source/devices/uprobe/type";
9589
9590 return parse_uint_from_file(file, "%d\n");
9591 }
9592
9593 static int determine_kprobe_retprobe_bit(void)
9594 {
9595 const char *file = "/sys/bus/event_source/devices/kprobe/format/retprobe";
9596
9597 return parse_uint_from_file(file, "config:%d\n");
9598 }
9599
9600 static int determine_uprobe_retprobe_bit(void)
9601 {
9602 const char *file = "/sys/bus/event_source/devices/uprobe/format/retprobe";
9603
9604 return parse_uint_from_file(file, "config:%d\n");
9605 }
9606
9607 static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name,
9608 uint64_t offset, int pid)
9609 {
9610 struct perf_event_attr attr = {};
9611 char errmsg[STRERR_BUFSIZE];
9612 int type, pfd, err;
9613
9614 type = uprobe ? determine_uprobe_perf_type()
9615 : determine_kprobe_perf_type();
9616 if (type < 0) {
9617 pr_warn("failed to determine %s perf type: %s\n",
9618 uprobe ? "uprobe" : "kprobe",
9619 libbpf_strerror_r(type, errmsg, sizeof(errmsg)));
9620 return type;
9621 }
9622 if (retprobe) {
9623 int bit = uprobe ? determine_uprobe_retprobe_bit()
9624 : determine_kprobe_retprobe_bit();
9625
9626 if (bit < 0) {
9627 pr_warn("failed to determine %s retprobe bit: %s\n",
9628 uprobe ? "uprobe" : "kprobe",
9629 libbpf_strerror_r(bit, errmsg, sizeof(errmsg)));
9630 return bit;
9631 }
9632 attr.config |= 1 << bit;
9633 }
9634 attr.size = sizeof(attr);
9635 attr.type = type;
9636 attr.config1 = ptr_to_u64(name); /* kprobe_func or uprobe_path */
9637 attr.config2 = offset; /* kprobe_addr or probe_offset */
9638
9639 /* pid filter is meaningful only for uprobes */
9640 pfd = syscall(__NR_perf_event_open, &attr,
9641 pid < 0 ? -1 : pid /* pid */,
9642 pid == -1 ? 0 : -1 /* cpu */,
9643 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
9644 if (pfd < 0) {
9645 err = -errno;
9646 pr_warn("%s perf_event_open() failed: %s\n",
9647 uprobe ? "uprobe" : "kprobe",
9648 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9649 return err;
9650 }
9651 return pfd;
9652 }
9653
9654 struct bpf_link *bpf_program__attach_kprobe(struct bpf_program *prog,
9655 bool retprobe,
9656 const char *func_name)
9657 {
9658 char errmsg[STRERR_BUFSIZE];
9659 struct bpf_link *link;
9660 int pfd, err;
9661
9662 pfd = perf_event_open_probe(false /* uprobe */, retprobe, func_name,
9663 0 /* offset */, -1 /* pid */);
9664 if (pfd < 0) {
9665 pr_warn("prog '%s': failed to create %s '%s' perf event: %s\n",
9666 prog->name, retprobe ? "kretprobe" : "kprobe", func_name,
9667 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
9668 return ERR_PTR(pfd);
9669 }
9670 link = bpf_program__attach_perf_event(prog, pfd);
9671 if (IS_ERR(link)) {
9672 close(pfd);
9673 err = PTR_ERR(link);
9674 pr_warn("prog '%s': failed to attach to %s '%s': %s\n",
9675 prog->name, retprobe ? "kretprobe" : "kprobe", func_name,
9676 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9677 return link;
9678 }
9679 return link;
9680 }
9681
9682 static struct bpf_link *attach_kprobe(const struct bpf_sec_def *sec,
9683 struct bpf_program *prog)
9684 {
9685 const char *func_name;
9686 bool retprobe;
9687
9688 func_name = prog->sec_name + sec->len;
9689 retprobe = strcmp(sec->sec, "kretprobe/") == 0;
9690
9691 return bpf_program__attach_kprobe(prog, retprobe, func_name);
9692 }
9693
9694 struct bpf_link *bpf_program__attach_uprobe(struct bpf_program *prog,
9695 bool retprobe, pid_t pid,
9696 const char *binary_path,
9697 size_t func_offset)
9698 {
9699 char errmsg[STRERR_BUFSIZE];
9700 struct bpf_link *link;
9701 int pfd, err;
9702
9703 pfd = perf_event_open_probe(true /* uprobe */, retprobe,
9704 binary_path, func_offset, pid);
9705 if (pfd < 0) {
9706 pr_warn("prog '%s': failed to create %s '%s:0x%zx' perf event: %s\n",
9707 prog->name, retprobe ? "uretprobe" : "uprobe",
9708 binary_path, func_offset,
9709 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
9710 return ERR_PTR(pfd);
9711 }
9712 link = bpf_program__attach_perf_event(prog, pfd);
9713 if (IS_ERR(link)) {
9714 close(pfd);
9715 err = PTR_ERR(link);
9716 pr_warn("prog '%s': failed to attach to %s '%s:0x%zx': %s\n",
9717 prog->name, retprobe ? "uretprobe" : "uprobe",
9718 binary_path, func_offset,
9719 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9720 return link;
9721 }
9722 return link;
9723 }
9724
9725 static int determine_tracepoint_id(const char *tp_category,
9726 const char *tp_name)
9727 {
9728 char file[PATH_MAX];
9729 int ret;
9730
9731 ret = snprintf(file, sizeof(file),
9732 "/sys/kernel/debug/tracing/events/%s/%s/id",
9733 tp_category, tp_name);
9734 if (ret < 0)
9735 return -errno;
9736 if (ret >= sizeof(file)) {
9737 pr_debug("tracepoint %s/%s path is too long\n",
9738 tp_category, tp_name);
9739 return -E2BIG;
9740 }
9741 return parse_uint_from_file(file, "%d\n");
9742 }
9743
9744 static int perf_event_open_tracepoint(const char *tp_category,
9745 const char *tp_name)
9746 {
9747 struct perf_event_attr attr = {};
9748 char errmsg[STRERR_BUFSIZE];
9749 int tp_id, pfd, err;
9750
9751 tp_id = determine_tracepoint_id(tp_category, tp_name);
9752 if (tp_id < 0) {
9753 pr_warn("failed to determine tracepoint '%s/%s' perf event ID: %s\n",
9754 tp_category, tp_name,
9755 libbpf_strerror_r(tp_id, errmsg, sizeof(errmsg)));
9756 return tp_id;
9757 }
9758
9759 attr.type = PERF_TYPE_TRACEPOINT;
9760 attr.size = sizeof(attr);
9761 attr.config = tp_id;
9762
9763 pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */,
9764 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
9765 if (pfd < 0) {
9766 err = -errno;
9767 pr_warn("tracepoint '%s/%s' perf_event_open() failed: %s\n",
9768 tp_category, tp_name,
9769 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9770 return err;
9771 }
9772 return pfd;
9773 }
9774
9775 struct bpf_link *bpf_program__attach_tracepoint(struct bpf_program *prog,
9776 const char *tp_category,
9777 const char *tp_name)
9778 {
9779 char errmsg[STRERR_BUFSIZE];
9780 struct bpf_link *link;
9781 int pfd, err;
9782
9783 pfd = perf_event_open_tracepoint(tp_category, tp_name);
9784 if (pfd < 0) {
9785 pr_warn("prog '%s': failed to create tracepoint '%s/%s' perf event: %s\n",
9786 prog->name, tp_category, tp_name,
9787 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
9788 return ERR_PTR(pfd);
9789 }
9790 link = bpf_program__attach_perf_event(prog, pfd);
9791 if (IS_ERR(link)) {
9792 close(pfd);
9793 err = PTR_ERR(link);
9794 pr_warn("prog '%s': failed to attach to tracepoint '%s/%s': %s\n",
9795 prog->name, tp_category, tp_name,
9796 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9797 return link;
9798 }
9799 return link;
9800 }
9801
9802 static struct bpf_link *attach_tp(const struct bpf_sec_def *sec,
9803 struct bpf_program *prog)
9804 {
9805 char *sec_name, *tp_cat, *tp_name;
9806 struct bpf_link *link;
9807
9808 sec_name = strdup(prog->sec_name);
9809 if (!sec_name)
9810 return ERR_PTR(-ENOMEM);
9811
9812 /* extract "tp/<category>/<name>" */
9813 tp_cat = sec_name + sec->len;
9814 tp_name = strchr(tp_cat, '/');
9815 if (!tp_name) {
9816 link = ERR_PTR(-EINVAL);
9817 goto out;
9818 }
9819 *tp_name = '\0';
9820 tp_name++;
9821
9822 link = bpf_program__attach_tracepoint(prog, tp_cat, tp_name);
9823 out:
9824 free(sec_name);
9825 return link;
9826 }
9827
9828 struct bpf_link *bpf_program__attach_raw_tracepoint(struct bpf_program *prog,
9829 const char *tp_name)
9830 {
9831 char errmsg[STRERR_BUFSIZE];
9832 struct bpf_link *link;
9833 int prog_fd, pfd;
9834
9835 prog_fd = bpf_program__fd(prog);
9836 if (prog_fd < 0) {
9837 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
9838 return ERR_PTR(-EINVAL);
9839 }
9840
9841 link = calloc(1, sizeof(*link));
9842 if (!link)
9843 return ERR_PTR(-ENOMEM);
9844 link->detach = &bpf_link__detach_fd;
9845
9846 pfd = bpf_raw_tracepoint_open(tp_name, prog_fd);
9847 if (pfd < 0) {
9848 pfd = -errno;
9849 free(link);
9850 pr_warn("prog '%s': failed to attach to raw tracepoint '%s': %s\n",
9851 prog->name, tp_name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
9852 return ERR_PTR(pfd);
9853 }
9854 link->fd = pfd;
9855 return link;
9856 }
9857
9858 static struct bpf_link *attach_raw_tp(const struct bpf_sec_def *sec,
9859 struct bpf_program *prog)
9860 {
9861 const char *tp_name = prog->sec_name + sec->len;
9862
9863 return bpf_program__attach_raw_tracepoint(prog, tp_name);
9864 }
9865
9866 /* Common logic for all BPF program types that attach to a btf_id */
9867 static struct bpf_link *bpf_program__attach_btf_id(struct bpf_program *prog)
9868 {
9869 char errmsg[STRERR_BUFSIZE];
9870 struct bpf_link *link;
9871 int prog_fd, pfd;
9872
9873 prog_fd = bpf_program__fd(prog);
9874 if (prog_fd < 0) {
9875 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
9876 return ERR_PTR(-EINVAL);
9877 }
9878
9879 link = calloc(1, sizeof(*link));
9880 if (!link)
9881 return ERR_PTR(-ENOMEM);
9882 link->detach = &bpf_link__detach_fd;
9883
9884 pfd = bpf_raw_tracepoint_open(NULL, prog_fd);
9885 if (pfd < 0) {
9886 pfd = -errno;
9887 free(link);
9888 pr_warn("prog '%s': failed to attach: %s\n",
9889 prog->name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
9890 return ERR_PTR(pfd);
9891 }
9892 link->fd = pfd;
9893 return (struct bpf_link *)link;
9894 }
9895
9896 struct bpf_link *bpf_program__attach_trace(struct bpf_program *prog)
9897 {
9898 return bpf_program__attach_btf_id(prog);
9899 }
9900
9901 struct bpf_link *bpf_program__attach_lsm(struct bpf_program *prog)
9902 {
9903 return bpf_program__attach_btf_id(prog);
9904 }
9905
9906 static struct bpf_link *attach_trace(const struct bpf_sec_def *sec,
9907 struct bpf_program *prog)
9908 {
9909 return bpf_program__attach_trace(prog);
9910 }
9911
9912 static struct bpf_link *attach_lsm(const struct bpf_sec_def *sec,
9913 struct bpf_program *prog)
9914 {
9915 return bpf_program__attach_lsm(prog);
9916 }
9917
9918 static struct bpf_link *attach_iter(const struct bpf_sec_def *sec,
9919 struct bpf_program *prog)
9920 {
9921 return bpf_program__attach_iter(prog, NULL);
9922 }
9923
9924 static struct bpf_link *
9925 bpf_program__attach_fd(struct bpf_program *prog, int target_fd, int btf_id,
9926 const char *target_name)
9927 {
9928 DECLARE_LIBBPF_OPTS(bpf_link_create_opts, opts,
9929 .target_btf_id = btf_id);
9930 enum bpf_attach_type attach_type;
9931 char errmsg[STRERR_BUFSIZE];
9932 struct bpf_link *link;
9933 int prog_fd, link_fd;
9934
9935 prog_fd = bpf_program__fd(prog);
9936 if (prog_fd < 0) {
9937 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
9938 return ERR_PTR(-EINVAL);
9939 }
9940
9941 link = calloc(1, sizeof(*link));
9942 if (!link)
9943 return ERR_PTR(-ENOMEM);
9944 link->detach = &bpf_link__detach_fd;
9945
9946 attach_type = bpf_program__get_expected_attach_type(prog);
9947 link_fd = bpf_link_create(prog_fd, target_fd, attach_type, &opts);
9948 if (link_fd < 0) {
9949 link_fd = -errno;
9950 free(link);
9951 pr_warn("prog '%s': failed to attach to %s: %s\n",
9952 prog->name, target_name,
9953 libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
9954 return ERR_PTR(link_fd);
9955 }
9956 link->fd = link_fd;
9957 return link;
9958 }
9959
9960 struct bpf_link *
9961 bpf_program__attach_cgroup(struct bpf_program *prog, int cgroup_fd)
9962 {
9963 return bpf_program__attach_fd(prog, cgroup_fd, 0, "cgroup");
9964 }
9965
9966 struct bpf_link *
9967 bpf_program__attach_netns(struct bpf_program *prog, int netns_fd)
9968 {
9969 return bpf_program__attach_fd(prog, netns_fd, 0, "netns");
9970 }
9971
9972 struct bpf_link *bpf_program__attach_xdp(struct bpf_program *prog, int ifindex)
9973 {
9974 /* target_fd/target_ifindex use the same field in LINK_CREATE */
9975 return bpf_program__attach_fd(prog, ifindex, 0, "xdp");
9976 }
9977
9978 struct bpf_link *bpf_program__attach_freplace(struct bpf_program *prog,
9979 int target_fd,
9980 const char *attach_func_name)
9981 {
9982 int btf_id;
9983
9984 if (!!target_fd != !!attach_func_name) {
9985 pr_warn("prog '%s': supply none or both of target_fd and attach_func_name\n",
9986 prog->name);
9987 return ERR_PTR(-EINVAL);
9988 }
9989
9990 if (prog->type != BPF_PROG_TYPE_EXT) {
9991 pr_warn("prog '%s': only BPF_PROG_TYPE_EXT can attach as freplace",
9992 prog->name);
9993 return ERR_PTR(-EINVAL);
9994 }
9995
9996 if (target_fd) {
9997 btf_id = libbpf_find_prog_btf_id(attach_func_name, target_fd);
9998 if (btf_id < 0)
9999 return ERR_PTR(btf_id);
10000
10001 return bpf_program__attach_fd(prog, target_fd, btf_id, "freplace");
10002 } else {
10003 /* no target, so use raw_tracepoint_open for compatibility
10004 * with old kernels
10005 */
10006 return bpf_program__attach_trace(prog);
10007 }
10008 }
10009
10010 struct bpf_link *
10011 bpf_program__attach_iter(struct bpf_program *prog,
10012 const struct bpf_iter_attach_opts *opts)
10013 {
10014 DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_create_opts);
10015 char errmsg[STRERR_BUFSIZE];
10016 struct bpf_link *link;
10017 int prog_fd, link_fd;
10018 __u32 target_fd = 0;
10019
10020 if (!OPTS_VALID(opts, bpf_iter_attach_opts))
10021 return ERR_PTR(-EINVAL);
10022
10023 link_create_opts.iter_info = OPTS_GET(opts, link_info, (void *)0);
10024 link_create_opts.iter_info_len = OPTS_GET(opts, link_info_len, 0);
10025
10026 prog_fd = bpf_program__fd(prog);
10027 if (prog_fd < 0) {
10028 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
10029 return ERR_PTR(-EINVAL);
10030 }
10031
10032 link = calloc(1, sizeof(*link));
10033 if (!link)
10034 return ERR_PTR(-ENOMEM);
10035 link->detach = &bpf_link__detach_fd;
10036
10037 link_fd = bpf_link_create(prog_fd, target_fd, BPF_TRACE_ITER,
10038 &link_create_opts);
10039 if (link_fd < 0) {
10040 link_fd = -errno;
10041 free(link);
10042 pr_warn("prog '%s': failed to attach to iterator: %s\n",
10043 prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
10044 return ERR_PTR(link_fd);
10045 }
10046 link->fd = link_fd;
10047 return link;
10048 }
10049
10050 struct bpf_link *bpf_program__attach(struct bpf_program *prog)
10051 {
10052 const struct bpf_sec_def *sec_def;
10053
10054 sec_def = find_sec_def(prog->sec_name);
10055 if (!sec_def || !sec_def->attach_fn)
10056 return ERR_PTR(-ESRCH);
10057
10058 return sec_def->attach_fn(sec_def, prog);
10059 }
10060
10061 static int bpf_link__detach_struct_ops(struct bpf_link *link)
10062 {
10063 __u32 zero = 0;
10064
10065 if (bpf_map_delete_elem(link->fd, &zero))
10066 return -errno;
10067
10068 return 0;
10069 }
10070
10071 struct bpf_link *bpf_map__attach_struct_ops(struct bpf_map *map)
10072 {
10073 struct bpf_struct_ops *st_ops;
10074 struct bpf_link *link;
10075 __u32 i, zero = 0;
10076 int err;
10077
10078 if (!bpf_map__is_struct_ops(map) || map->fd == -1)
10079 return ERR_PTR(-EINVAL);
10080
10081 link = calloc(1, sizeof(*link));
10082 if (!link)
10083 return ERR_PTR(-EINVAL);
10084
10085 st_ops = map->st_ops;
10086 for (i = 0; i < btf_vlen(st_ops->type); i++) {
10087 struct bpf_program *prog = st_ops->progs[i];
10088 void *kern_data;
10089 int prog_fd;
10090
10091 if (!prog)
10092 continue;
10093
10094 prog_fd = bpf_program__fd(prog);
10095 kern_data = st_ops->kern_vdata + st_ops->kern_func_off[i];
10096 *(unsigned long *)kern_data = prog_fd;
10097 }
10098
10099 err = bpf_map_update_elem(map->fd, &zero, st_ops->kern_vdata, 0);
10100 if (err) {
10101 err = -errno;
10102 free(link);
10103 return ERR_PTR(err);
10104 }
10105
10106 link->detach = bpf_link__detach_struct_ops;
10107 link->fd = map->fd;
10108
10109 return link;
10110 }
10111
10112 enum bpf_perf_event_ret
10113 bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size,
10114 void **copy_mem, size_t *copy_size,
10115 bpf_perf_event_print_t fn, void *private_data)
10116 {
10117 struct perf_event_mmap_page *header = mmap_mem;
10118 __u64 data_head = ring_buffer_read_head(header);
10119 __u64 data_tail = header->data_tail;
10120 void *base = ((__u8 *)header) + page_size;
10121 int ret = LIBBPF_PERF_EVENT_CONT;
10122 struct perf_event_header *ehdr;
10123 size_t ehdr_size;
10124
10125 while (data_head != data_tail) {
10126 ehdr = base + (data_tail & (mmap_size - 1));
10127 ehdr_size = ehdr->size;
10128
10129 if (((void *)ehdr) + ehdr_size > base + mmap_size) {
10130 void *copy_start = ehdr;
10131 size_t len_first = base + mmap_size - copy_start;
10132 size_t len_secnd = ehdr_size - len_first;
10133
10134 if (*copy_size < ehdr_size) {
10135 free(*copy_mem);
10136 *copy_mem = malloc(ehdr_size);
10137 if (!*copy_mem) {
10138 *copy_size = 0;
10139 ret = LIBBPF_PERF_EVENT_ERROR;
10140 break;
10141 }
10142 *copy_size = ehdr_size;
10143 }
10144
10145 memcpy(*copy_mem, copy_start, len_first);
10146 memcpy(*copy_mem + len_first, base, len_secnd);
10147 ehdr = *copy_mem;
10148 }
10149
10150 ret = fn(ehdr, private_data);
10151 data_tail += ehdr_size;
10152 if (ret != LIBBPF_PERF_EVENT_CONT)
10153 break;
10154 }
10155
10156 ring_buffer_write_tail(header, data_tail);
10157 return ret;
10158 }
10159
10160 struct perf_buffer;
10161
10162 struct perf_buffer_params {
10163 struct perf_event_attr *attr;
10164 /* if event_cb is specified, it takes precendence */
10165 perf_buffer_event_fn event_cb;
10166 /* sample_cb and lost_cb are higher-level common-case callbacks */
10167 perf_buffer_sample_fn sample_cb;
10168 perf_buffer_lost_fn lost_cb;
10169 void *ctx;
10170 int cpu_cnt;
10171 int *cpus;
10172 int *map_keys;
10173 };
10174
10175 struct perf_cpu_buf {
10176 struct perf_buffer *pb;
10177 void *base; /* mmap()'ed memory */
10178 void *buf; /* for reconstructing segmented data */
10179 size_t buf_size;
10180 int fd;
10181 int cpu;
10182 int map_key;
10183 };
10184
10185 struct perf_buffer {
10186 perf_buffer_event_fn event_cb;
10187 perf_buffer_sample_fn sample_cb;
10188 perf_buffer_lost_fn lost_cb;
10189 void *ctx; /* passed into callbacks */
10190
10191 size_t page_size;
10192 size_t mmap_size;
10193 struct perf_cpu_buf **cpu_bufs;
10194 struct epoll_event *events;
10195 int cpu_cnt; /* number of allocated CPU buffers */
10196 int epoll_fd; /* perf event FD */
10197 int map_fd; /* BPF_MAP_TYPE_PERF_EVENT_ARRAY BPF map FD */
10198 };
10199
10200 static void perf_buffer__free_cpu_buf(struct perf_buffer *pb,
10201 struct perf_cpu_buf *cpu_buf)
10202 {
10203 if (!cpu_buf)
10204 return;
10205 if (cpu_buf->base &&
10206 munmap(cpu_buf->base, pb->mmap_size + pb->page_size))
10207 pr_warn("failed to munmap cpu_buf #%d\n", cpu_buf->cpu);
10208 if (cpu_buf->fd >= 0) {
10209 ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0);
10210 close(cpu_buf->fd);
10211 }
10212 free(cpu_buf->buf);
10213 free(cpu_buf);
10214 }
10215
10216 void perf_buffer__free(struct perf_buffer *pb)
10217 {
10218 int i;
10219
10220 if (IS_ERR_OR_NULL(pb))
10221 return;
10222 if (pb->cpu_bufs) {
10223 for (i = 0; i < pb->cpu_cnt; i++) {
10224 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];
10225
10226 if (!cpu_buf)
10227 continue;
10228
10229 bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key);
10230 perf_buffer__free_cpu_buf(pb, cpu_buf);
10231 }
10232 free(pb->cpu_bufs);
10233 }
10234 if (pb->epoll_fd >= 0)
10235 close(pb->epoll_fd);
10236 free(pb->events);
10237 free(pb);
10238 }
10239
10240 static struct perf_cpu_buf *
10241 perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr,
10242 int cpu, int map_key)
10243 {
10244 struct perf_cpu_buf *cpu_buf;
10245 char msg[STRERR_BUFSIZE];
10246 int err;
10247
10248 cpu_buf = calloc(1, sizeof(*cpu_buf));
10249 if (!cpu_buf)
10250 return ERR_PTR(-ENOMEM);
10251
10252 cpu_buf->pb = pb;
10253 cpu_buf->cpu = cpu;
10254 cpu_buf->map_key = map_key;
10255
10256 cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 /* pid */, cpu,
10257 -1, PERF_FLAG_FD_CLOEXEC);
10258 if (cpu_buf->fd < 0) {
10259 err = -errno;
10260 pr_warn("failed to open perf buffer event on cpu #%d: %s\n",
10261 cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
10262 goto error;
10263 }
10264
10265 cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size,
10266 PROT_READ | PROT_WRITE, MAP_SHARED,
10267 cpu_buf->fd, 0);
10268 if (cpu_buf->base == MAP_FAILED) {
10269 cpu_buf->base = NULL;
10270 err = -errno;
10271 pr_warn("failed to mmap perf buffer on cpu #%d: %s\n",
10272 cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
10273 goto error;
10274 }
10275
10276 if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
10277 err = -errno;
10278 pr_warn("failed to enable perf buffer event on cpu #%d: %s\n",
10279 cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
10280 goto error;
10281 }
10282
10283 return cpu_buf;
10284
10285 error:
10286 perf_buffer__free_cpu_buf(pb, cpu_buf);
10287 return (struct perf_cpu_buf *)ERR_PTR(err);
10288 }
10289
10290 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt,
10291 struct perf_buffer_params *p);
10292
10293 struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt,
10294 const struct perf_buffer_opts *opts)
10295 {
10296 struct perf_buffer_params p = {};
10297 struct perf_event_attr attr = { 0, };
10298
10299 attr.config = PERF_COUNT_SW_BPF_OUTPUT;
10300 attr.type = PERF_TYPE_SOFTWARE;
10301 attr.sample_type = PERF_SAMPLE_RAW;
10302 attr.sample_period = 1;
10303 attr.wakeup_events = 1;
10304
10305 p.attr = &attr;
10306 p.sample_cb = opts ? opts->sample_cb : NULL;
10307 p.lost_cb = opts ? opts->lost_cb : NULL;
10308 p.ctx = opts ? opts->ctx : NULL;
10309
10310 return __perf_buffer__new(map_fd, page_cnt, &p);
10311 }
10312
10313 struct perf_buffer *
10314 perf_buffer__new_raw(int map_fd, size_t page_cnt,
10315 const struct perf_buffer_raw_opts *opts)
10316 {
10317 struct perf_buffer_params p = {};
10318
10319 p.attr = opts->attr;
10320 p.event_cb = opts->event_cb;
10321 p.ctx = opts->ctx;
10322 p.cpu_cnt = opts->cpu_cnt;
10323 p.cpus = opts->cpus;
10324 p.map_keys = opts->map_keys;
10325
10326 return __perf_buffer__new(map_fd, page_cnt, &p);
10327 }
10328
10329 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt,
10330 struct perf_buffer_params *p)
10331 {
10332 const char *online_cpus_file = "/sys/devices/system/cpu/online";
10333 struct bpf_map_info map;
10334 char msg[STRERR_BUFSIZE];
10335 struct perf_buffer *pb;
10336 bool *online = NULL;
10337 __u32 map_info_len;
10338 int err, i, j, n;
10339
10340 if (page_cnt & (page_cnt - 1)) {
10341 pr_warn("page count should be power of two, but is %zu\n",
10342 page_cnt);
10343 return ERR_PTR(-EINVAL);
10344 }
10345
10346 /* best-effort sanity checks */
10347 memset(&map, 0, sizeof(map));
10348 map_info_len = sizeof(map);
10349 err = bpf_obj_get_info_by_fd(map_fd, &map, &map_info_len);
10350 if (err) {
10351 err = -errno;
10352 /* if BPF_OBJ_GET_INFO_BY_FD is supported, will return
10353 * -EBADFD, -EFAULT, or -E2BIG on real error
10354 */
10355 if (err != -EINVAL) {
10356 pr_warn("failed to get map info for map FD %d: %s\n",
10357 map_fd, libbpf_strerror_r(err, msg, sizeof(msg)));
10358 return ERR_PTR(err);
10359 }
10360 pr_debug("failed to get map info for FD %d; API not supported? Ignoring...\n",
10361 map_fd);
10362 } else {
10363 if (map.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) {
10364 pr_warn("map '%s' should be BPF_MAP_TYPE_PERF_EVENT_ARRAY\n",
10365 map.name);
10366 return ERR_PTR(-EINVAL);
10367 }
10368 }
10369
10370 pb = calloc(1, sizeof(*pb));
10371 if (!pb)
10372 return ERR_PTR(-ENOMEM);
10373
10374 pb->event_cb = p->event_cb;
10375 pb->sample_cb = p->sample_cb;
10376 pb->lost_cb = p->lost_cb;
10377 pb->ctx = p->ctx;
10378
10379 pb->page_size = getpagesize();
10380 pb->mmap_size = pb->page_size * page_cnt;
10381 pb->map_fd = map_fd;
10382
10383 pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
10384 if (pb->epoll_fd < 0) {
10385 err = -errno;
10386 pr_warn("failed to create epoll instance: %s\n",
10387 libbpf_strerror_r(err, msg, sizeof(msg)));
10388 goto error;
10389 }
10390
10391 if (p->cpu_cnt > 0) {
10392 pb->cpu_cnt = p->cpu_cnt;
10393 } else {
10394 pb->cpu_cnt = libbpf_num_possible_cpus();
10395 if (pb->cpu_cnt < 0) {
10396 err = pb->cpu_cnt;
10397 goto error;
10398 }
10399 if (map.max_entries && map.max_entries < pb->cpu_cnt)
10400 pb->cpu_cnt = map.max_entries;
10401 }
10402
10403 pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events));
10404 if (!pb->events) {
10405 err = -ENOMEM;
10406 pr_warn("failed to allocate events: out of memory\n");
10407 goto error;
10408 }
10409 pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs));
10410 if (!pb->cpu_bufs) {
10411 err = -ENOMEM;
10412 pr_warn("failed to allocate buffers: out of memory\n");
10413 goto error;
10414 }
10415
10416 err = parse_cpu_mask_file(online_cpus_file, &online, &n);
10417 if (err) {
10418 pr_warn("failed to get online CPU mask: %d\n", err);
10419 goto error;
10420 }
10421
10422 for (i = 0, j = 0; i < pb->cpu_cnt; i++) {
10423 struct perf_cpu_buf *cpu_buf;
10424 int cpu, map_key;
10425
10426 cpu = p->cpu_cnt > 0 ? p->cpus[i] : i;
10427 map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i;
10428
10429 /* in case user didn't explicitly requested particular CPUs to
10430 * be attached to, skip offline/not present CPUs
10431 */
10432 if (p->cpu_cnt <= 0 && (cpu >= n || !online[cpu]))
10433 continue;
10434
10435 cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key);
10436 if (IS_ERR(cpu_buf)) {
10437 err = PTR_ERR(cpu_buf);
10438 goto error;
10439 }
10440
10441 pb->cpu_bufs[j] = cpu_buf;
10442
10443 err = bpf_map_update_elem(pb->map_fd, &map_key,
10444 &cpu_buf->fd, 0);
10445 if (err) {
10446 err = -errno;
10447 pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n",
10448 cpu, map_key, cpu_buf->fd,
10449 libbpf_strerror_r(err, msg, sizeof(msg)));
10450 goto error;
10451 }
10452
10453 pb->events[j].events = EPOLLIN;
10454 pb->events[j].data.ptr = cpu_buf;
10455 if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd,
10456 &pb->events[j]) < 0) {
10457 err = -errno;
10458 pr_warn("failed to epoll_ctl cpu #%d perf FD %d: %s\n",
10459 cpu, cpu_buf->fd,
10460 libbpf_strerror_r(err, msg, sizeof(msg)));
10461 goto error;
10462 }
10463 j++;
10464 }
10465 pb->cpu_cnt = j;
10466 free(online);
10467
10468 return pb;
10469
10470 error:
10471 free(online);
10472 if (pb)
10473 perf_buffer__free(pb);
10474 return ERR_PTR(err);
10475 }
10476
10477 struct perf_sample_raw {
10478 struct perf_event_header header;
10479 uint32_t size;
10480 char data[];
10481 };
10482
10483 struct perf_sample_lost {
10484 struct perf_event_header header;
10485 uint64_t id;
10486 uint64_t lost;
10487 uint64_t sample_id;
10488 };
10489
10490 static enum bpf_perf_event_ret
10491 perf_buffer__process_record(struct perf_event_header *e, void *ctx)
10492 {
10493 struct perf_cpu_buf *cpu_buf = ctx;
10494 struct perf_buffer *pb = cpu_buf->pb;
10495 void *data = e;
10496
10497 /* user wants full control over parsing perf event */
10498 if (pb->event_cb)
10499 return pb->event_cb(pb->ctx, cpu_buf->cpu, e);
10500
10501 switch (e->type) {
10502 case PERF_RECORD_SAMPLE: {
10503 struct perf_sample_raw *s = data;
10504
10505 if (pb->sample_cb)
10506 pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size);
10507 break;
10508 }
10509 case PERF_RECORD_LOST: {
10510 struct perf_sample_lost *s = data;
10511
10512 if (pb->lost_cb)
10513 pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost);
10514 break;
10515 }
10516 default:
10517 pr_warn("unknown perf sample type %d\n", e->type);
10518 return LIBBPF_PERF_EVENT_ERROR;
10519 }
10520 return LIBBPF_PERF_EVENT_CONT;
10521 }
10522
10523 static int perf_buffer__process_records(struct perf_buffer *pb,
10524 struct perf_cpu_buf *cpu_buf)
10525 {
10526 enum bpf_perf_event_ret ret;
10527
10528 ret = bpf_perf_event_read_simple(cpu_buf->base, pb->mmap_size,
10529 pb->page_size, &cpu_buf->buf,
10530 &cpu_buf->buf_size,
10531 perf_buffer__process_record, cpu_buf);
10532 if (ret != LIBBPF_PERF_EVENT_CONT)
10533 return ret;
10534 return 0;
10535 }
10536
10537 int perf_buffer__epoll_fd(const struct perf_buffer *pb)
10538 {
10539 return pb->epoll_fd;
10540 }
10541
10542 int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms)
10543 {
10544 int i, cnt, err;
10545
10546 cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms);
10547 for (i = 0; i < cnt; i++) {
10548 struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr;
10549
10550 err = perf_buffer__process_records(pb, cpu_buf);
10551 if (err) {
10552 pr_warn("error while processing records: %d\n", err);
10553 return err;
10554 }
10555 }
10556 return cnt < 0 ? -errno : cnt;
10557 }
10558
10559 /* Return number of PERF_EVENT_ARRAY map slots set up by this perf_buffer
10560 * manager.
10561 */
10562 size_t perf_buffer__buffer_cnt(const struct perf_buffer *pb)
10563 {
10564 return pb->cpu_cnt;
10565 }
10566
10567 /*
10568 * Return perf_event FD of a ring buffer in *buf_idx* slot of
10569 * PERF_EVENT_ARRAY BPF map. This FD can be polled for new data using
10570 * select()/poll()/epoll() Linux syscalls.
10571 */
10572 int perf_buffer__buffer_fd(const struct perf_buffer *pb, size_t buf_idx)
10573 {
10574 struct perf_cpu_buf *cpu_buf;
10575
10576 if (buf_idx >= pb->cpu_cnt)
10577 return -EINVAL;
10578
10579 cpu_buf = pb->cpu_bufs[buf_idx];
10580 if (!cpu_buf)
10581 return -ENOENT;
10582
10583 return cpu_buf->fd;
10584 }
10585
10586 /*
10587 * Consume data from perf ring buffer corresponding to slot *buf_idx* in
10588 * PERF_EVENT_ARRAY BPF map without waiting/polling. If there is no data to
10589 * consume, do nothing and return success.
10590 * Returns:
10591 * - 0 on success;
10592 * - <0 on failure.
10593 */
10594 int perf_buffer__consume_buffer(struct perf_buffer *pb, size_t buf_idx)
10595 {
10596 struct perf_cpu_buf *cpu_buf;
10597
10598 if (buf_idx >= pb->cpu_cnt)
10599 return -EINVAL;
10600
10601 cpu_buf = pb->cpu_bufs[buf_idx];
10602 if (!cpu_buf)
10603 return -ENOENT;
10604
10605 return perf_buffer__process_records(pb, cpu_buf);
10606 }
10607
10608 int perf_buffer__consume(struct perf_buffer *pb)
10609 {
10610 int i, err;
10611
10612 for (i = 0; i < pb->cpu_cnt; i++) {
10613 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];
10614
10615 if (!cpu_buf)
10616 continue;
10617
10618 err = perf_buffer__process_records(pb, cpu_buf);
10619 if (err) {
10620 pr_warn("perf_buffer: failed to process records in buffer #%d: %d\n", i, err);
10621 return err;
10622 }
10623 }
10624 return 0;
10625 }
10626
10627 struct bpf_prog_info_array_desc {
10628 int array_offset; /* e.g. offset of jited_prog_insns */
10629 int count_offset; /* e.g. offset of jited_prog_len */
10630 int size_offset; /* > 0: offset of rec size,
10631 * < 0: fix size of -size_offset
10632 */
10633 };
10634
10635 static struct bpf_prog_info_array_desc bpf_prog_info_array_desc[] = {
10636 [BPF_PROG_INFO_JITED_INSNS] = {
10637 offsetof(struct bpf_prog_info, jited_prog_insns),
10638 offsetof(struct bpf_prog_info, jited_prog_len),
10639 -1,
10640 },
10641 [BPF_PROG_INFO_XLATED_INSNS] = {
10642 offsetof(struct bpf_prog_info, xlated_prog_insns),
10643 offsetof(struct bpf_prog_info, xlated_prog_len),
10644 -1,
10645 },
10646 [BPF_PROG_INFO_MAP_IDS] = {
10647 offsetof(struct bpf_prog_info, map_ids),
10648 offsetof(struct bpf_prog_info, nr_map_ids),
10649 -(int)sizeof(__u32),
10650 },
10651 [BPF_PROG_INFO_JITED_KSYMS] = {
10652 offsetof(struct bpf_prog_info, jited_ksyms),
10653 offsetof(struct bpf_prog_info, nr_jited_ksyms),
10654 -(int)sizeof(__u64),
10655 },
10656 [BPF_PROG_INFO_JITED_FUNC_LENS] = {
10657 offsetof(struct bpf_prog_info, jited_func_lens),
10658 offsetof(struct bpf_prog_info, nr_jited_func_lens),
10659 -(int)sizeof(__u32),
10660 },
10661 [BPF_PROG_INFO_FUNC_INFO] = {
10662 offsetof(struct bpf_prog_info, func_info),
10663 offsetof(struct bpf_prog_info, nr_func_info),
10664 offsetof(struct bpf_prog_info, func_info_rec_size),
10665 },
10666 [BPF_PROG_INFO_LINE_INFO] = {
10667 offsetof(struct bpf_prog_info, line_info),
10668 offsetof(struct bpf_prog_info, nr_line_info),
10669 offsetof(struct bpf_prog_info, line_info_rec_size),
10670 },
10671 [BPF_PROG_INFO_JITED_LINE_INFO] = {
10672 offsetof(struct bpf_prog_info, jited_line_info),
10673 offsetof(struct bpf_prog_info, nr_jited_line_info),
10674 offsetof(struct bpf_prog_info, jited_line_info_rec_size),
10675 },
10676 [BPF_PROG_INFO_PROG_TAGS] = {
10677 offsetof(struct bpf_prog_info, prog_tags),
10678 offsetof(struct bpf_prog_info, nr_prog_tags),
10679 -(int)sizeof(__u8) * BPF_TAG_SIZE,
10680 },
10681
10682 };
10683
10684 static __u32 bpf_prog_info_read_offset_u32(struct bpf_prog_info *info,
10685 int offset)
10686 {
10687 __u32 *array = (__u32 *)info;
10688
10689 if (offset >= 0)
10690 return array[offset / sizeof(__u32)];
10691 return -(int)offset;
10692 }
10693
10694 static __u64 bpf_prog_info_read_offset_u64(struct bpf_prog_info *info,
10695 int offset)
10696 {
10697 __u64 *array = (__u64 *)info;
10698
10699 if (offset >= 0)
10700 return array[offset / sizeof(__u64)];
10701 return -(int)offset;
10702 }
10703
10704 static void bpf_prog_info_set_offset_u32(struct bpf_prog_info *info, int offset,
10705 __u32 val)
10706 {
10707 __u32 *array = (__u32 *)info;
10708
10709 if (offset >= 0)
10710 array[offset / sizeof(__u32)] = val;
10711 }
10712
10713 static void bpf_prog_info_set_offset_u64(struct bpf_prog_info *info, int offset,
10714 __u64 val)
10715 {
10716 __u64 *array = (__u64 *)info;
10717
10718 if (offset >= 0)
10719 array[offset / sizeof(__u64)] = val;
10720 }
10721
10722 struct bpf_prog_info_linear *
10723 bpf_program__get_prog_info_linear(int fd, __u64 arrays)
10724 {
10725 struct bpf_prog_info_linear *info_linear;
10726 struct bpf_prog_info info = {};
10727 __u32 info_len = sizeof(info);
10728 __u32 data_len = 0;
10729 int i, err;
10730 void *ptr;
10731
10732 if (arrays >> BPF_PROG_INFO_LAST_ARRAY)
10733 return ERR_PTR(-EINVAL);
10734
10735 /* step 1: get array dimensions */
10736 err = bpf_obj_get_info_by_fd(fd, &info, &info_len);
10737 if (err) {
10738 pr_debug("can't get prog info: %s", strerror(errno));
10739 return ERR_PTR(-EFAULT);
10740 }
10741
10742 /* step 2: calculate total size of all arrays */
10743 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
10744 bool include_array = (arrays & (1UL << i)) > 0;
10745 struct bpf_prog_info_array_desc *desc;
10746 __u32 count, size;
10747
10748 desc = bpf_prog_info_array_desc + i;
10749
10750 /* kernel is too old to support this field */
10751 if (info_len < desc->array_offset + sizeof(__u32) ||
10752 info_len < desc->count_offset + sizeof(__u32) ||
10753 (desc->size_offset > 0 && info_len < desc->size_offset))
10754 include_array = false;
10755
10756 if (!include_array) {
10757 arrays &= ~(1UL << i); /* clear the bit */
10758 continue;
10759 }
10760
10761 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset);
10762 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset);
10763
10764 data_len += count * size;
10765 }
10766
10767 /* step 3: allocate continuous memory */
10768 data_len = roundup(data_len, sizeof(__u64));
10769 info_linear = malloc(sizeof(struct bpf_prog_info_linear) + data_len);
10770 if (!info_linear)
10771 return ERR_PTR(-ENOMEM);
10772
10773 /* step 4: fill data to info_linear->info */
10774 info_linear->arrays = arrays;
10775 memset(&info_linear->info, 0, sizeof(info));
10776 ptr = info_linear->data;
10777
10778 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
10779 struct bpf_prog_info_array_desc *desc;
10780 __u32 count, size;
10781
10782 if ((arrays & (1UL << i)) == 0)
10783 continue;
10784
10785 desc = bpf_prog_info_array_desc + i;
10786 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset);
10787 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset);
10788 bpf_prog_info_set_offset_u32(&info_linear->info,
10789 desc->count_offset, count);
10790 bpf_prog_info_set_offset_u32(&info_linear->info,
10791 desc->size_offset, size);
10792 bpf_prog_info_set_offset_u64(&info_linear->info,
10793 desc->array_offset,
10794 ptr_to_u64(ptr));
10795 ptr += count * size;
10796 }
10797
10798 /* step 5: call syscall again to get required arrays */
10799 err = bpf_obj_get_info_by_fd(fd, &info_linear->info, &info_len);
10800 if (err) {
10801 pr_debug("can't get prog info: %s", strerror(errno));
10802 free(info_linear);
10803 return ERR_PTR(-EFAULT);
10804 }
10805
10806 /* step 6: verify the data */
10807 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
10808 struct bpf_prog_info_array_desc *desc;
10809 __u32 v1, v2;
10810
10811 if ((arrays & (1UL << i)) == 0)
10812 continue;
10813
10814 desc = bpf_prog_info_array_desc + i;
10815 v1 = bpf_prog_info_read_offset_u32(&info, desc->count_offset);
10816 v2 = bpf_prog_info_read_offset_u32(&info_linear->info,
10817 desc->count_offset);
10818 if (v1 != v2)
10819 pr_warn("%s: mismatch in element count\n", __func__);
10820
10821 v1 = bpf_prog_info_read_offset_u32(&info, desc->size_offset);
10822 v2 = bpf_prog_info_read_offset_u32(&info_linear->info,
10823 desc->size_offset);
10824 if (v1 != v2)
10825 pr_warn("%s: mismatch in rec size\n", __func__);
10826 }
10827
10828 /* step 7: update info_len and data_len */
10829 info_linear->info_len = sizeof(struct bpf_prog_info);
10830 info_linear->data_len = data_len;
10831
10832 return info_linear;
10833 }
10834
10835 void bpf_program__bpil_addr_to_offs(struct bpf_prog_info_linear *info_linear)
10836 {
10837 int i;
10838
10839 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
10840 struct bpf_prog_info_array_desc *desc;
10841 __u64 addr, offs;
10842
10843 if ((info_linear->arrays & (1UL << i)) == 0)
10844 continue;
10845
10846 desc = bpf_prog_info_array_desc + i;
10847 addr = bpf_prog_info_read_offset_u64(&info_linear->info,
10848 desc->array_offset);
10849 offs = addr - ptr_to_u64(info_linear->data);
10850 bpf_prog_info_set_offset_u64(&info_linear->info,
10851 desc->array_offset, offs);
10852 }
10853 }
10854
10855 void bpf_program__bpil_offs_to_addr(struct bpf_prog_info_linear *info_linear)
10856 {
10857 int i;
10858
10859 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
10860 struct bpf_prog_info_array_desc *desc;
10861 __u64 addr, offs;
10862
10863 if ((info_linear->arrays & (1UL << i)) == 0)
10864 continue;
10865
10866 desc = bpf_prog_info_array_desc + i;
10867 offs = bpf_prog_info_read_offset_u64(&info_linear->info,
10868 desc->array_offset);
10869 addr = offs + ptr_to_u64(info_linear->data);
10870 bpf_prog_info_set_offset_u64(&info_linear->info,
10871 desc->array_offset, addr);
10872 }
10873 }
10874
10875 int bpf_program__set_attach_target(struct bpf_program *prog,
10876 int attach_prog_fd,
10877 const char *attach_func_name)
10878 {
10879 int btf_obj_fd = 0, btf_id = 0, err;
10880
10881 if (!prog || attach_prog_fd < 0 || !attach_func_name)
10882 return -EINVAL;
10883
10884 if (prog->obj->loaded)
10885 return -EINVAL;
10886
10887 if (attach_prog_fd) {
10888 btf_id = libbpf_find_prog_btf_id(attach_func_name,
10889 attach_prog_fd);
10890 if (btf_id < 0)
10891 return btf_id;
10892 } else {
10893 /* load btf_vmlinux, if not yet */
10894 err = bpf_object__load_vmlinux_btf(prog->obj, true);
10895 if (err)
10896 return err;
10897 err = find_kernel_btf_id(prog->obj, attach_func_name,
10898 prog->expected_attach_type,
10899 &btf_obj_fd, &btf_id);
10900 if (err)
10901 return err;
10902 }
10903
10904 prog->attach_btf_id = btf_id;
10905 prog->attach_btf_obj_fd = btf_obj_fd;
10906 prog->attach_prog_fd = attach_prog_fd;
10907 return 0;
10908 }
10909
10910 int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz)
10911 {
10912 int err = 0, n, len, start, end = -1;
10913 bool *tmp;
10914
10915 *mask = NULL;
10916 *mask_sz = 0;
10917
10918 /* Each sub string separated by ',' has format \d+-\d+ or \d+ */
10919 while (*s) {
10920 if (*s == ',' || *s == '\n') {
10921 s++;
10922 continue;
10923 }
10924 n = sscanf(s, "%d%n-%d%n", &start, &len, &end, &len);
10925 if (n <= 0 || n > 2) {
10926 pr_warn("Failed to get CPU range %s: %d\n", s, n);
10927 err = -EINVAL;
10928 goto cleanup;
10929 } else if (n == 1) {
10930 end = start;
10931 }
10932 if (start < 0 || start > end) {
10933 pr_warn("Invalid CPU range [%d,%d] in %s\n",
10934 start, end, s);
10935 err = -EINVAL;
10936 goto cleanup;
10937 }
10938 tmp = realloc(*mask, end + 1);
10939 if (!tmp) {
10940 err = -ENOMEM;
10941 goto cleanup;
10942 }
10943 *mask = tmp;
10944 memset(tmp + *mask_sz, 0, start - *mask_sz);
10945 memset(tmp + start, 1, end - start + 1);
10946 *mask_sz = end + 1;
10947 s += len;
10948 }
10949 if (!*mask_sz) {
10950 pr_warn("Empty CPU range\n");
10951 return -EINVAL;
10952 }
10953 return 0;
10954 cleanup:
10955 free(*mask);
10956 *mask = NULL;
10957 return err;
10958 }
10959
10960 int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz)
10961 {
10962 int fd, err = 0, len;
10963 char buf[128];
10964
10965 fd = open(fcpu, O_RDONLY);
10966 if (fd < 0) {
10967 err = -errno;
10968 pr_warn("Failed to open cpu mask file %s: %d\n", fcpu, err);
10969 return err;
10970 }
10971 len = read(fd, buf, sizeof(buf));
10972 close(fd);
10973 if (len <= 0) {
10974 err = len ? -errno : -EINVAL;
10975 pr_warn("Failed to read cpu mask from %s: %d\n", fcpu, err);
10976 return err;
10977 }
10978 if (len >= sizeof(buf)) {
10979 pr_warn("CPU mask is too big in file %s\n", fcpu);
10980 return -E2BIG;
10981 }
10982 buf[len] = '\0';
10983
10984 return parse_cpu_mask_str(buf, mask, mask_sz);
10985 }
10986
10987 int libbpf_num_possible_cpus(void)
10988 {
10989 static const char *fcpu = "/sys/devices/system/cpu/possible";
10990 static int cpus;
10991 int err, n, i, tmp_cpus;
10992 bool *mask;
10993
10994 tmp_cpus = READ_ONCE(cpus);
10995 if (tmp_cpus > 0)
10996 return tmp_cpus;
10997
10998 err = parse_cpu_mask_file(fcpu, &mask, &n);
10999 if (err)
11000 return err;
11001
11002 tmp_cpus = 0;
11003 for (i = 0; i < n; i++) {
11004 if (mask[i])
11005 tmp_cpus++;
11006 }
11007 free(mask);
11008
11009 WRITE_ONCE(cpus, tmp_cpus);
11010 return tmp_cpus;
11011 }
11012
11013 int bpf_object__open_skeleton(struct bpf_object_skeleton *s,
11014 const struct bpf_object_open_opts *opts)
11015 {
11016 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, skel_opts,
11017 .object_name = s->name,
11018 );
11019 struct bpf_object *obj;
11020 int i;
11021
11022 /* Attempt to preserve opts->object_name, unless overriden by user
11023 * explicitly. Overwriting object name for skeletons is discouraged,
11024 * as it breaks global data maps, because they contain object name
11025 * prefix as their own map name prefix. When skeleton is generated,
11026 * bpftool is making an assumption that this name will stay the same.
11027 */
11028 if (opts) {
11029 memcpy(&skel_opts, opts, sizeof(*opts));
11030 if (!opts->object_name)
11031 skel_opts.object_name = s->name;
11032 }
11033
11034 obj = bpf_object__open_mem(s->data, s->data_sz, &skel_opts);
11035 if (IS_ERR(obj)) {
11036 pr_warn("failed to initialize skeleton BPF object '%s': %ld\n",
11037 s->name, PTR_ERR(obj));
11038 return PTR_ERR(obj);
11039 }
11040
11041 *s->obj = obj;
11042
11043 for (i = 0; i < s->map_cnt; i++) {
11044 struct bpf_map **map = s->maps[i].map;
11045 const char *name = s->maps[i].name;
11046 void **mmaped = s->maps[i].mmaped;
11047
11048 *map = bpf_object__find_map_by_name(obj, name);
11049 if (!*map) {
11050 pr_warn("failed to find skeleton map '%s'\n", name);
11051 return -ESRCH;
11052 }
11053
11054 /* externs shouldn't be pre-setup from user code */
11055 if (mmaped && (*map)->libbpf_type != LIBBPF_MAP_KCONFIG)
11056 *mmaped = (*map)->mmaped;
11057 }
11058
11059 for (i = 0; i < s->prog_cnt; i++) {
11060 struct bpf_program **prog = s->progs[i].prog;
11061 const char *name = s->progs[i].name;
11062
11063 *prog = bpf_object__find_program_by_name(obj, name);
11064 if (!*prog) {
11065 pr_warn("failed to find skeleton program '%s'\n", name);
11066 return -ESRCH;
11067 }
11068 }
11069
11070 return 0;
11071 }
11072
11073 int bpf_object__load_skeleton(struct bpf_object_skeleton *s)
11074 {
11075 int i, err;
11076
11077 err = bpf_object__load(*s->obj);
11078 if (err) {
11079 pr_warn("failed to load BPF skeleton '%s': %d\n", s->name, err);
11080 return err;
11081 }
11082
11083 for (i = 0; i < s->map_cnt; i++) {
11084 struct bpf_map *map = *s->maps[i].map;
11085 size_t mmap_sz = bpf_map_mmap_sz(map);
11086 int prot, map_fd = bpf_map__fd(map);
11087 void **mmaped = s->maps[i].mmaped;
11088
11089 if (!mmaped)
11090 continue;
11091
11092 if (!(map->def.map_flags & BPF_F_MMAPABLE)) {
11093 *mmaped = NULL;
11094 continue;
11095 }
11096
11097 if (map->def.map_flags & BPF_F_RDONLY_PROG)
11098 prot = PROT_READ;
11099 else
11100 prot = PROT_READ | PROT_WRITE;
11101
11102 /* Remap anonymous mmap()-ed "map initialization image" as
11103 * a BPF map-backed mmap()-ed memory, but preserving the same
11104 * memory address. This will cause kernel to change process'
11105 * page table to point to a different piece of kernel memory,
11106 * but from userspace point of view memory address (and its
11107 * contents, being identical at this point) will stay the
11108 * same. This mapping will be released by bpf_object__close()
11109 * as per normal clean up procedure, so we don't need to worry
11110 * about it from skeleton's clean up perspective.
11111 */
11112 *mmaped = mmap(map->mmaped, mmap_sz, prot,
11113 MAP_SHARED | MAP_FIXED, map_fd, 0);
11114 if (*mmaped == MAP_FAILED) {
11115 err = -errno;
11116 *mmaped = NULL;
11117 pr_warn("failed to re-mmap() map '%s': %d\n",
11118 bpf_map__name(map), err);
11119 return err;
11120 }
11121 }
11122
11123 return 0;
11124 }
11125
11126 int bpf_object__attach_skeleton(struct bpf_object_skeleton *s)
11127 {
11128 int i;
11129
11130 for (i = 0; i < s->prog_cnt; i++) {
11131 struct bpf_program *prog = *s->progs[i].prog;
11132 struct bpf_link **link = s->progs[i].link;
11133 const struct bpf_sec_def *sec_def;
11134
11135 if (!prog->load)
11136 continue;
11137
11138 sec_def = find_sec_def(prog->sec_name);
11139 if (!sec_def || !sec_def->attach_fn)
11140 continue;
11141
11142 *link = sec_def->attach_fn(sec_def, prog);
11143 if (IS_ERR(*link)) {
11144 pr_warn("failed to auto-attach program '%s': %ld\n",
11145 bpf_program__name(prog), PTR_ERR(*link));
11146 return PTR_ERR(*link);
11147 }
11148 }
11149
11150 return 0;
11151 }
11152
11153 void bpf_object__detach_skeleton(struct bpf_object_skeleton *s)
11154 {
11155 int i;
11156
11157 for (i = 0; i < s->prog_cnt; i++) {
11158 struct bpf_link **link = s->progs[i].link;
11159
11160 bpf_link__destroy(*link);
11161 *link = NULL;
11162 }
11163 }
11164
11165 void bpf_object__destroy_skeleton(struct bpf_object_skeleton *s)
11166 {
11167 if (s->progs)
11168 bpf_object__detach_skeleton(s);
11169 if (s->obj)
11170 bpf_object__close(*s->obj);
11171 free(s->maps);
11172 free(s->progs);
11173 free(s);
11174 }
Linux with added FPC-III support