?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
RDMA/rtrs: server: Fix some error return code
[linux/fpc-iii.git] / tools / lib / bpf / libbpf.c
blobff9174282a8c6afdfeb79015a7ccbe0c7b33ac7b
1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
2
3 /*
4 * Common eBPF ELF object loading operations.
5 *
6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>
7 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
8 * Copyright (C) 2015 Huawei Inc.
9 * Copyright (C) 2017 Nicira, Inc.
10 * Copyright (C) 2019 Isovalent, Inc.
11 */
12
13 #ifndef _GNU_SOURCE
14 #define _GNU_SOURCE
15 #endif
16 #include <stdlib.h>
17 #include <stdio.h>
18 #include <stdarg.h>
19 #include <libgen.h>
20 #include <inttypes.h>
21 #include <limits.h>
22 #include <string.h>
23 #include <unistd.h>
24 #include <endian.h>
25 #include <fcntl.h>
26 #include <errno.h>
27 #include <ctype.h>
28 #include <asm/unistd.h>
29 #include <linux/err.h>
30 #include <linux/kernel.h>
31 #include <linux/bpf.h>
32 #include <linux/btf.h>
33 #include <linux/filter.h>
34 #include <linux/list.h>
35 #include <linux/limits.h>
36 #include <linux/perf_event.h>
37 #include <linux/ring_buffer.h>
38 #include <linux/version.h>
39 #include <sys/epoll.h>
40 #include <sys/ioctl.h>
41 #include <sys/mman.h>
42 #include <sys/stat.h>
43 #include <sys/types.h>
44 #include <sys/vfs.h>
45 #include <sys/utsname.h>
46 #include <sys/resource.h>
47 #include <tools/libc_compat.h>
48 #include <libelf.h>
49 #include <gelf.h>
50 #include <zlib.h>
51
52 #include "libbpf.h"
53 #include "bpf.h"
54 #include "btf.h"
55 #include "str_error.h"
56 #include "libbpf_internal.h"
57 #include "hashmap.h"
58
59 /* make sure libbpf doesn't use kernel-only integer typedefs */
60 #pragma GCC poison u8 u16 u32 u64 s8 s16 s32 s64
61
62 #ifndef EM_BPF
63 #define EM_BPF 247
64 #endif
65
66 #ifndef BPF_FS_MAGIC
67 #define BPF_FS_MAGIC 0xcafe4a11
68 #endif
69
70 /* vsprintf() in __base_pr() uses nonliteral format string. It may break
71 * compilation if user enables corresponding warning. Disable it explicitly.
72 */
73 #pragma GCC diagnostic ignored "-Wformat-nonliteral"
74
75 #define __printf(a, b) __attribute__((format(printf, a, b)))
76
77 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj);
78 static struct bpf_program *bpf_object__find_prog_by_idx(struct bpf_object *obj,
79 int idx);
80 static const struct btf_type *
81 skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id);
82
83 static int __base_pr(enum libbpf_print_level level, const char *format,
84 va_list args)
85 {
86 if (level == LIBBPF_DEBUG)
87 return 0;
88
89 return vfprintf(stderr, format, args);
90 }
91
92 static libbpf_print_fn_t __libbpf_pr = __base_pr;
93
94 libbpf_print_fn_t libbpf_set_print(libbpf_print_fn_t fn)
95 {
96 libbpf_print_fn_t old_print_fn = __libbpf_pr;
97
98 __libbpf_pr = fn;
99 return old_print_fn;
100 }
101
102 __printf(2, 3)
103 void libbpf_print(enum libbpf_print_level level, const char *format, ...)
104 {
105 va_list args;
106
107 if (!__libbpf_pr)
108 return;
109
110 va_start(args, format);
111 __libbpf_pr(level, format, args);
112 va_end(args);
113 }
114
115 static void pr_perm_msg(int err)
116 {
117 struct rlimit limit;
118 char buf[100];
119
120 if (err != -EPERM || geteuid() != 0)
121 return;
122
123 err = getrlimit(RLIMIT_MEMLOCK, &limit);
124 if (err)
125 return;
126
127 if (limit.rlim_cur == RLIM_INFINITY)
128 return;
129
130 if (limit.rlim_cur < 1024)
131 snprintf(buf, sizeof(buf), "%zu bytes", (size_t)limit.rlim_cur);
132 else if (limit.rlim_cur < 1024*1024)
133 snprintf(buf, sizeof(buf), "%.1f KiB", (double)limit.rlim_cur / 1024);
134 else
135 snprintf(buf, sizeof(buf), "%.1f MiB", (double)limit.rlim_cur / (1024*1024));
136
137 pr_warn("permission error while running as root; try raising 'ulimit -l'? current value: %s\n",
138 buf);
139 }
140
141 #define STRERR_BUFSIZE 128
142
143 /* Copied from tools/perf/util/util.h */
144 #ifndef zfree
145 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; })
146 #endif
147
148 #ifndef zclose
149 # define zclose(fd) ({ \
150 int ___err = 0; \
151 if ((fd) >= 0) \
152 ___err = close((fd)); \
153 fd = -1; \
154 ___err; })
155 #endif
156
157 #ifdef HAVE_LIBELF_MMAP_SUPPORT
158 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP
159 #else
160 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ
161 #endif
162
163 static inline __u64 ptr_to_u64(const void *ptr)
164 {
165 return (__u64) (unsigned long) ptr;
166 }
167
168 struct bpf_capabilities {
169 /* v4.14: kernel support for program & map names. */
170 __u32 name:1;
171 /* v5.2: kernel support for global data sections. */
172 __u32 global_data:1;
173 /* BTF_KIND_FUNC and BTF_KIND_FUNC_PROTO support */
174 __u32 btf_func:1;
175 /* BTF_KIND_VAR and BTF_KIND_DATASEC support */
176 __u32 btf_datasec:1;
177 /* BPF_F_MMAPABLE is supported for arrays */
178 __u32 array_mmap:1;
179 /* BTF_FUNC_GLOBAL is supported */
180 __u32 btf_func_global:1;
181 };
182
183 enum reloc_type {
184 RELO_LD64,
185 RELO_CALL,
186 RELO_DATA,
187 RELO_EXTERN,
188 };
189
190 struct reloc_desc {
191 enum reloc_type type;
192 int insn_idx;
193 int map_idx;
194 int sym_off;
195 };
196
197 /*
198 * bpf_prog should be a better name but it has been used in
199 * linux/filter.h.
200 */
201 struct bpf_program {
202 /* Index in elf obj file, for relocation use. */
203 int idx;
204 char *name;
205 int prog_ifindex;
206 char *section_name;
207 /* section_name with / replaced by _; makes recursive pinning
208 * in bpf_object__pin_programs easier
209 */
210 char *pin_name;
211 struct bpf_insn *insns;
212 size_t insns_cnt, main_prog_cnt;
213 enum bpf_prog_type type;
214
215 struct reloc_desc *reloc_desc;
216 int nr_reloc;
217 int log_level;
218
219 struct {
220 int nr;
221 int *fds;
222 } instances;
223 bpf_program_prep_t preprocessor;
224
225 struct bpf_object *obj;
226 void *priv;
227 bpf_program_clear_priv_t clear_priv;
228
229 enum bpf_attach_type expected_attach_type;
230 __u32 attach_btf_id;
231 __u32 attach_prog_fd;
232 void *func_info;
233 __u32 func_info_rec_size;
234 __u32 func_info_cnt;
235
236 struct bpf_capabilities *caps;
237
238 void *line_info;
239 __u32 line_info_rec_size;
240 __u32 line_info_cnt;
241 __u32 prog_flags;
242 };
243
244 struct bpf_struct_ops {
245 const char *tname;
246 const struct btf_type *type;
247 struct bpf_program **progs;
248 __u32 *kern_func_off;
249 /* e.g. struct tcp_congestion_ops in bpf_prog's btf format */
250 void *data;
251 /* e.g. struct bpf_struct_ops_tcp_congestion_ops in
252 * btf_vmlinux's format.
253 * struct bpf_struct_ops_tcp_congestion_ops {
254 * [... some other kernel fields ...]
255 * struct tcp_congestion_ops data;
256 * }
257 * kern_vdata-size == sizeof(struct bpf_struct_ops_tcp_congestion_ops)
258 * bpf_map__init_kern_struct_ops() will populate the "kern_vdata"
259 * from "data".
260 */
261 void *kern_vdata;
262 __u32 type_id;
263 };
264
265 #define DATA_SEC ".data"
266 #define BSS_SEC ".bss"
267 #define RODATA_SEC ".rodata"
268 #define KCONFIG_SEC ".kconfig"
269 #define STRUCT_OPS_SEC ".struct_ops"
270
271 enum libbpf_map_type {
272 LIBBPF_MAP_UNSPEC,
273 LIBBPF_MAP_DATA,
274 LIBBPF_MAP_BSS,
275 LIBBPF_MAP_RODATA,
276 LIBBPF_MAP_KCONFIG,
277 };
278
279 static const char * const libbpf_type_to_btf_name[] = {
280 [LIBBPF_MAP_DATA] = DATA_SEC,
281 [LIBBPF_MAP_BSS] = BSS_SEC,
282 [LIBBPF_MAP_RODATA] = RODATA_SEC,
283 [LIBBPF_MAP_KCONFIG] = KCONFIG_SEC,
284 };
285
286 struct bpf_map {
287 char *name;
288 int fd;
289 int sec_idx;
290 size_t sec_offset;
291 int map_ifindex;
292 int inner_map_fd;
293 struct bpf_map_def def;
294 __u32 btf_key_type_id;
295 __u32 btf_value_type_id;
296 __u32 btf_vmlinux_value_type_id;
297 void *priv;
298 bpf_map_clear_priv_t clear_priv;
299 enum libbpf_map_type libbpf_type;
300 void *mmaped;
301 struct bpf_struct_ops *st_ops;
302 char *pin_path;
303 bool pinned;
304 bool reused;
305 };
306
307 enum extern_type {
308 EXT_UNKNOWN,
309 EXT_CHAR,
310 EXT_BOOL,
311 EXT_INT,
312 EXT_TRISTATE,
313 EXT_CHAR_ARR,
314 };
315
316 struct extern_desc {
317 const char *name;
318 int sym_idx;
319 int btf_id;
320 enum extern_type type;
321 int sz;
322 int align;
323 int data_off;
324 bool is_signed;
325 bool is_weak;
326 bool is_set;
327 };
328
329 static LIST_HEAD(bpf_objects_list);
330
331 struct bpf_object {
332 char name[BPF_OBJ_NAME_LEN];
333 char license[64];
334 __u32 kern_version;
335
336 struct bpf_program *programs;
337 size_t nr_programs;
338 struct bpf_map *maps;
339 size_t nr_maps;
340 size_t maps_cap;
341
342 char *kconfig;
343 struct extern_desc *externs;
344 int nr_extern;
345 int kconfig_map_idx;
346
347 bool loaded;
348 bool has_pseudo_calls;
349
350 /*
351 * Information when doing elf related work. Only valid if fd
352 * is valid.
353 */
354 struct {
355 int fd;
356 const void *obj_buf;
357 size_t obj_buf_sz;
358 Elf *elf;
359 GElf_Ehdr ehdr;
360 Elf_Data *symbols;
361 Elf_Data *data;
362 Elf_Data *rodata;
363 Elf_Data *bss;
364 Elf_Data *st_ops_data;
365 size_t strtabidx;
366 struct {
367 GElf_Shdr shdr;
368 Elf_Data *data;
369 } *reloc_sects;
370 int nr_reloc_sects;
371 int maps_shndx;
372 int btf_maps_shndx;
373 int text_shndx;
374 int symbols_shndx;
375 int data_shndx;
376 int rodata_shndx;
377 int bss_shndx;
378 int st_ops_shndx;
379 } efile;
380 /*
381 * All loaded bpf_object is linked in a list, which is
382 * hidden to caller. bpf_objects__<func> handlers deal with
383 * all objects.
384 */
385 struct list_head list;
386
387 struct btf *btf;
388 /* Parse and load BTF vmlinux if any of the programs in the object need
389 * it at load time.
390 */
391 struct btf *btf_vmlinux;
392 struct btf_ext *btf_ext;
393
394 void *priv;
395 bpf_object_clear_priv_t clear_priv;
396
397 struct bpf_capabilities caps;
398
399 char path[];
400 };
401 #define obj_elf_valid(o) ((o)->efile.elf)
402
403 void bpf_program__unload(struct bpf_program *prog)
404 {
405 int i;
406
407 if (!prog)
408 return;
409
410 /*
411 * If the object is opened but the program was never loaded,
412 * it is possible that prog->instances.nr == -1.
413 */
414 if (prog->instances.nr > 0) {
415 for (i = 0; i < prog->instances.nr; i++)
416 zclose(prog->instances.fds[i]);
417 } else if (prog->instances.nr != -1) {
418 pr_warn("Internal error: instances.nr is %d\n",
419 prog->instances.nr);
420 }
421
422 prog->instances.nr = -1;
423 zfree(&prog->instances.fds);
424
425 zfree(&prog->func_info);
426 zfree(&prog->line_info);
427 }
428
429 static void bpf_program__exit(struct bpf_program *prog)
430 {
431 if (!prog)
432 return;
433
434 if (prog->clear_priv)
435 prog->clear_priv(prog, prog->priv);
436
437 prog->priv = NULL;
438 prog->clear_priv = NULL;
439
440 bpf_program__unload(prog);
441 zfree(&prog->name);
442 zfree(&prog->section_name);
443 zfree(&prog->pin_name);
444 zfree(&prog->insns);
445 zfree(&prog->reloc_desc);
446
447 prog->nr_reloc = 0;
448 prog->insns_cnt = 0;
449 prog->idx = -1;
450 }
451
452 static char *__bpf_program__pin_name(struct bpf_program *prog)
453 {
454 char *name, *p;
455
456 name = p = strdup(prog->section_name);
457 while ((p = strchr(p, '/')))
458 *p = '_';
459
460 return name;
461 }
462
463 static int
464 bpf_program__init(void *data, size_t size, char *section_name, int idx,
465 struct bpf_program *prog)
466 {
467 const size_t bpf_insn_sz = sizeof(struct bpf_insn);
468
469 if (size == 0 || size % bpf_insn_sz) {
470 pr_warn("corrupted section '%s', size: %zu\n",
471 section_name, size);
472 return -EINVAL;
473 }
474
475 memset(prog, 0, sizeof(*prog));
476
477 prog->section_name = strdup(section_name);
478 if (!prog->section_name) {
479 pr_warn("failed to alloc name for prog under section(%d) %s\n",
480 idx, section_name);
481 goto errout;
482 }
483
484 prog->pin_name = __bpf_program__pin_name(prog);
485 if (!prog->pin_name) {
486 pr_warn("failed to alloc pin name for prog under section(%d) %s\n",
487 idx, section_name);
488 goto errout;
489 }
490
491 prog->insns = malloc(size);
492 if (!prog->insns) {
493 pr_warn("failed to alloc insns for prog under section %s\n",
494 section_name);
495 goto errout;
496 }
497 prog->insns_cnt = size / bpf_insn_sz;
498 memcpy(prog->insns, data, size);
499 prog->idx = idx;
500 prog->instances.fds = NULL;
501 prog->instances.nr = -1;
502 prog->type = BPF_PROG_TYPE_UNSPEC;
503
504 return 0;
505 errout:
506 bpf_program__exit(prog);
507 return -ENOMEM;
508 }
509
510 static int
511 bpf_object__add_program(struct bpf_object *obj, void *data, size_t size,
512 char *section_name, int idx)
513 {
514 struct bpf_program prog, *progs;
515 int nr_progs, err;
516
517 err = bpf_program__init(data, size, section_name, idx, &prog);
518 if (err)
519 return err;
520
521 prog.caps = &obj->caps;
522 progs = obj->programs;
523 nr_progs = obj->nr_programs;
524
525 progs = reallocarray(progs, nr_progs + 1, sizeof(progs[0]));
526 if (!progs) {
527 /*
528 * In this case the original obj->programs
529 * is still valid, so don't need special treat for
530 * bpf_close_object().
531 */
532 pr_warn("failed to alloc a new program under section '%s'\n",
533 section_name);
534 bpf_program__exit(&prog);
535 return -ENOMEM;
536 }
537
538 pr_debug("found program %s\n", prog.section_name);
539 obj->programs = progs;
540 obj->nr_programs = nr_progs + 1;
541 prog.obj = obj;
542 progs[nr_progs] = prog;
543 return 0;
544 }
545
546 static int
547 bpf_object__init_prog_names(struct bpf_object *obj)
548 {
549 Elf_Data *symbols = obj->efile.symbols;
550 struct bpf_program *prog;
551 size_t pi, si;
552
553 for (pi = 0; pi < obj->nr_programs; pi++) {
554 const char *name = NULL;
555
556 prog = &obj->programs[pi];
557
558 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym) && !name;
559 si++) {
560 GElf_Sym sym;
561
562 if (!gelf_getsym(symbols, si, &sym))
563 continue;
564 if (sym.st_shndx != prog->idx)
565 continue;
566 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL)
567 continue;
568
569 name = elf_strptr(obj->efile.elf,
570 obj->efile.strtabidx,
571 sym.st_name);
572 if (!name) {
573 pr_warn("failed to get sym name string for prog %s\n",
574 prog->section_name);
575 return -LIBBPF_ERRNO__LIBELF;
576 }
577 }
578
579 if (!name && prog->idx == obj->efile.text_shndx)
580 name = ".text";
581
582 if (!name) {
583 pr_warn("failed to find sym for prog %s\n",
584 prog->section_name);
585 return -EINVAL;
586 }
587
588 prog->name = strdup(name);
589 if (!prog->name) {
590 pr_warn("failed to allocate memory for prog sym %s\n",
591 name);
592 return -ENOMEM;
593 }
594 }
595
596 return 0;
597 }
598
599 static __u32 get_kernel_version(void)
600 {
601 __u32 major, minor, patch;
602 struct utsname info;
603
604 uname(&info);
605 if (sscanf(info.release, "%u.%u.%u", &major, &minor, &patch) != 3)
606 return 0;
607 return KERNEL_VERSION(major, minor, patch);
608 }
609
610 static const struct btf_member *
611 find_member_by_offset(const struct btf_type *t, __u32 bit_offset)
612 {
613 struct btf_member *m;
614 int i;
615
616 for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
617 if (btf_member_bit_offset(t, i) == bit_offset)
618 return m;
619 }
620
621 return NULL;
622 }
623
624 static const struct btf_member *
625 find_member_by_name(const struct btf *btf, const struct btf_type *t,
626 const char *name)
627 {
628 struct btf_member *m;
629 int i;
630
631 for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
632 if (!strcmp(btf__name_by_offset(btf, m->name_off), name))
633 return m;
634 }
635
636 return NULL;
637 }
638
639 #define STRUCT_OPS_VALUE_PREFIX "bpf_struct_ops_"
640 static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix,
641 const char *name, __u32 kind);
642
643 static int
644 find_struct_ops_kern_types(const struct btf *btf, const char *tname,
645 const struct btf_type **type, __u32 *type_id,
646 const struct btf_type **vtype, __u32 *vtype_id,
647 const struct btf_member **data_member)
648 {
649 const struct btf_type *kern_type, *kern_vtype;
650 const struct btf_member *kern_data_member;
651 __s32 kern_vtype_id, kern_type_id;
652 __u32 i;
653
654 kern_type_id = btf__find_by_name_kind(btf, tname, BTF_KIND_STRUCT);
655 if (kern_type_id < 0) {
656 pr_warn("struct_ops init_kern: struct %s is not found in kernel BTF\n",
657 tname);
658 return kern_type_id;
659 }
660 kern_type = btf__type_by_id(btf, kern_type_id);
661
662 /* Find the corresponding "map_value" type that will be used
663 * in map_update(BPF_MAP_TYPE_STRUCT_OPS). For example,
664 * find "struct bpf_struct_ops_tcp_congestion_ops" from the
665 * btf_vmlinux.
666 */
667 kern_vtype_id = find_btf_by_prefix_kind(btf, STRUCT_OPS_VALUE_PREFIX,
668 tname, BTF_KIND_STRUCT);
669 if (kern_vtype_id < 0) {
670 pr_warn("struct_ops init_kern: struct %s%s is not found in kernel BTF\n",
671 STRUCT_OPS_VALUE_PREFIX, tname);
672 return kern_vtype_id;
673 }
674 kern_vtype = btf__type_by_id(btf, kern_vtype_id);
675
676 /* Find "struct tcp_congestion_ops" from
677 * struct bpf_struct_ops_tcp_congestion_ops {
678 * [ ... ]
679 * struct tcp_congestion_ops data;
680 * }
681 */
682 kern_data_member = btf_members(kern_vtype);
683 for (i = 0; i < btf_vlen(kern_vtype); i++, kern_data_member++) {
684 if (kern_data_member->type == kern_type_id)
685 break;
686 }
687 if (i == btf_vlen(kern_vtype)) {
688 pr_warn("struct_ops init_kern: struct %s data is not found in struct %s%s\n",
689 tname, STRUCT_OPS_VALUE_PREFIX, tname);
690 return -EINVAL;
691 }
692
693 *type = kern_type;
694 *type_id = kern_type_id;
695 *vtype = kern_vtype;
696 *vtype_id = kern_vtype_id;
697 *data_member = kern_data_member;
698
699 return 0;
700 }
701
702 static bool bpf_map__is_struct_ops(const struct bpf_map *map)
703 {
704 return map->def.type == BPF_MAP_TYPE_STRUCT_OPS;
705 }
706
707 /* Init the map's fields that depend on kern_btf */
708 static int bpf_map__init_kern_struct_ops(struct bpf_map *map,
709 const struct btf *btf,
710 const struct btf *kern_btf)
711 {
712 const struct btf_member *member, *kern_member, *kern_data_member;
713 const struct btf_type *type, *kern_type, *kern_vtype;
714 __u32 i, kern_type_id, kern_vtype_id, kern_data_off;
715 struct bpf_struct_ops *st_ops;
716 void *data, *kern_data;
717 const char *tname;
718 int err;
719
720 st_ops = map->st_ops;
721 type = st_ops->type;
722 tname = st_ops->tname;
723 err = find_struct_ops_kern_types(kern_btf, tname,
724 &kern_type, &kern_type_id,
725 &kern_vtype, &kern_vtype_id,
726 &kern_data_member);
727 if (err)
728 return err;
729
730 pr_debug("struct_ops init_kern %s: type_id:%u kern_type_id:%u kern_vtype_id:%u\n",
731 map->name, st_ops->type_id, kern_type_id, kern_vtype_id);
732
733 map->def.value_size = kern_vtype->size;
734 map->btf_vmlinux_value_type_id = kern_vtype_id;
735
736 st_ops->kern_vdata = calloc(1, kern_vtype->size);
737 if (!st_ops->kern_vdata)
738 return -ENOMEM;
739
740 data = st_ops->data;
741 kern_data_off = kern_data_member->offset / 8;
742 kern_data = st_ops->kern_vdata + kern_data_off;
743
744 member = btf_members(type);
745 for (i = 0; i < btf_vlen(type); i++, member++) {
746 const struct btf_type *mtype, *kern_mtype;
747 __u32 mtype_id, kern_mtype_id;
748 void *mdata, *kern_mdata;
749 __s64 msize, kern_msize;
750 __u32 moff, kern_moff;
751 __u32 kern_member_idx;
752 const char *mname;
753
754 mname = btf__name_by_offset(btf, member->name_off);
755 kern_member = find_member_by_name(kern_btf, kern_type, mname);
756 if (!kern_member) {
757 pr_warn("struct_ops init_kern %s: Cannot find member %s in kernel BTF\n",
758 map->name, mname);
759 return -ENOTSUP;
760 }
761
762 kern_member_idx = kern_member - btf_members(kern_type);
763 if (btf_member_bitfield_size(type, i) ||
764 btf_member_bitfield_size(kern_type, kern_member_idx)) {
765 pr_warn("struct_ops init_kern %s: bitfield %s is not supported\n",
766 map->name, mname);
767 return -ENOTSUP;
768 }
769
770 moff = member->offset / 8;
771 kern_moff = kern_member->offset / 8;
772
773 mdata = data + moff;
774 kern_mdata = kern_data + kern_moff;
775
776 mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id);
777 kern_mtype = skip_mods_and_typedefs(kern_btf, kern_member->type,
778 &kern_mtype_id);
779 if (BTF_INFO_KIND(mtype->info) !=
780 BTF_INFO_KIND(kern_mtype->info)) {
781 pr_warn("struct_ops init_kern %s: Unmatched member type %s %u != %u(kernel)\n",
782 map->name, mname, BTF_INFO_KIND(mtype->info),
783 BTF_INFO_KIND(kern_mtype->info));
784 return -ENOTSUP;
785 }
786
787 if (btf_is_ptr(mtype)) {
788 struct bpf_program *prog;
789
790 mtype = skip_mods_and_typedefs(btf, mtype->type, &mtype_id);
791 kern_mtype = skip_mods_and_typedefs(kern_btf,
792 kern_mtype->type,
793 &kern_mtype_id);
794 if (!btf_is_func_proto(mtype) ||
795 !btf_is_func_proto(kern_mtype)) {
796 pr_warn("struct_ops init_kern %s: non func ptr %s is not supported\n",
797 map->name, mname);
798 return -ENOTSUP;
799 }
800
801 prog = st_ops->progs[i];
802 if (!prog) {
803 pr_debug("struct_ops init_kern %s: func ptr %s is not set\n",
804 map->name, mname);
805 continue;
806 }
807
808 prog->attach_btf_id = kern_type_id;
809 prog->expected_attach_type = kern_member_idx;
810
811 st_ops->kern_func_off[i] = kern_data_off + kern_moff;
812
813 pr_debug("struct_ops init_kern %s: func ptr %s is set to prog %s from data(+%u) to kern_data(+%u)\n",
814 map->name, mname, prog->name, moff,
815 kern_moff);
816
817 continue;
818 }
819
820 msize = btf__resolve_size(btf, mtype_id);
821 kern_msize = btf__resolve_size(kern_btf, kern_mtype_id);
822 if (msize < 0 || kern_msize < 0 || msize != kern_msize) {
823 pr_warn("struct_ops init_kern %s: Error in size of member %s: %zd != %zd(kernel)\n",
824 map->name, mname, (ssize_t)msize,
825 (ssize_t)kern_msize);
826 return -ENOTSUP;
827 }
828
829 pr_debug("struct_ops init_kern %s: copy %s %u bytes from data(+%u) to kern_data(+%u)\n",
830 map->name, mname, (unsigned int)msize,
831 moff, kern_moff);
832 memcpy(kern_mdata, mdata, msize);
833 }
834
835 return 0;
836 }
837
838 static int bpf_object__init_kern_struct_ops_maps(struct bpf_object *obj)
839 {
840 struct bpf_map *map;
841 size_t i;
842 int err;
843
844 for (i = 0; i < obj->nr_maps; i++) {
845 map = &obj->maps[i];
846
847 if (!bpf_map__is_struct_ops(map))
848 continue;
849
850 err = bpf_map__init_kern_struct_ops(map, obj->btf,
851 obj->btf_vmlinux);
852 if (err)
853 return err;
854 }
855
856 return 0;
857 }
858
859 static int bpf_object__init_struct_ops_maps(struct bpf_object *obj)
860 {
861 const struct btf_type *type, *datasec;
862 const struct btf_var_secinfo *vsi;
863 struct bpf_struct_ops *st_ops;
864 const char *tname, *var_name;
865 __s32 type_id, datasec_id;
866 const struct btf *btf;
867 struct bpf_map *map;
868 __u32 i;
869
870 if (obj->efile.st_ops_shndx == -1)
871 return 0;
872
873 btf = obj->btf;
874 datasec_id = btf__find_by_name_kind(btf, STRUCT_OPS_SEC,
875 BTF_KIND_DATASEC);
876 if (datasec_id < 0) {
877 pr_warn("struct_ops init: DATASEC %s not found\n",
878 STRUCT_OPS_SEC);
879 return -EINVAL;
880 }
881
882 datasec = btf__type_by_id(btf, datasec_id);
883 vsi = btf_var_secinfos(datasec);
884 for (i = 0; i < btf_vlen(datasec); i++, vsi++) {
885 type = btf__type_by_id(obj->btf, vsi->type);
886 var_name = btf__name_by_offset(obj->btf, type->name_off);
887
888 type_id = btf__resolve_type(obj->btf, vsi->type);
889 if (type_id < 0) {
890 pr_warn("struct_ops init: Cannot resolve var type_id %u in DATASEC %s\n",
891 vsi->type, STRUCT_OPS_SEC);
892 return -EINVAL;
893 }
894
895 type = btf__type_by_id(obj->btf, type_id);
896 tname = btf__name_by_offset(obj->btf, type->name_off);
897 if (!tname[0]) {
898 pr_warn("struct_ops init: anonymous type is not supported\n");
899 return -ENOTSUP;
900 }
901 if (!btf_is_struct(type)) {
902 pr_warn("struct_ops init: %s is not a struct\n", tname);
903 return -EINVAL;
904 }
905
906 map = bpf_object__add_map(obj);
907 if (IS_ERR(map))
908 return PTR_ERR(map);
909
910 map->sec_idx = obj->efile.st_ops_shndx;
911 map->sec_offset = vsi->offset;
912 map->name = strdup(var_name);
913 if (!map->name)
914 return -ENOMEM;
915
916 map->def.type = BPF_MAP_TYPE_STRUCT_OPS;
917 map->def.key_size = sizeof(int);
918 map->def.value_size = type->size;
919 map->def.max_entries = 1;
920
921 map->st_ops = calloc(1, sizeof(*map->st_ops));
922 if (!map->st_ops)
923 return -ENOMEM;
924 st_ops = map->st_ops;
925 st_ops->data = malloc(type->size);
926 st_ops->progs = calloc(btf_vlen(type), sizeof(*st_ops->progs));
927 st_ops->kern_func_off = malloc(btf_vlen(type) *
928 sizeof(*st_ops->kern_func_off));
929 if (!st_ops->data || !st_ops->progs || !st_ops->kern_func_off)
930 return -ENOMEM;
931
932 if (vsi->offset + type->size > obj->efile.st_ops_data->d_size) {
933 pr_warn("struct_ops init: var %s is beyond the end of DATASEC %s\n",
934 var_name, STRUCT_OPS_SEC);
935 return -EINVAL;
936 }
937
938 memcpy(st_ops->data,
939 obj->efile.st_ops_data->d_buf + vsi->offset,
940 type->size);
941 st_ops->tname = tname;
942 st_ops->type = type;
943 st_ops->type_id = type_id;
944
945 pr_debug("struct_ops init: struct %s(type_id=%u) %s found at offset %u\n",
946 tname, type_id, var_name, vsi->offset);
947 }
948
949 return 0;
950 }
951
952 static struct bpf_object *bpf_object__new(const char *path,
953 const void *obj_buf,
954 size_t obj_buf_sz,
955 const char *obj_name)
956 {
957 struct bpf_object *obj;
958 char *end;
959
960 obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1);
961 if (!obj) {
962 pr_warn("alloc memory failed for %s\n", path);
963 return ERR_PTR(-ENOMEM);
964 }
965
966 strcpy(obj->path, path);
967 if (obj_name) {
968 strncpy(obj->name, obj_name, sizeof(obj->name) - 1);
969 obj->name[sizeof(obj->name) - 1] = 0;
970 } else {
971 /* Using basename() GNU version which doesn't modify arg. */
972 strncpy(obj->name, basename((void *)path),
973 sizeof(obj->name) - 1);
974 end = strchr(obj->name, '.');
975 if (end)
976 *end = 0;
977 }
978
979 obj->efile.fd = -1;
980 /*
981 * Caller of this function should also call
982 * bpf_object__elf_finish() after data collection to return
983 * obj_buf to user. If not, we should duplicate the buffer to
984 * avoid user freeing them before elf finish.
985 */
986 obj->efile.obj_buf = obj_buf;
987 obj->efile.obj_buf_sz = obj_buf_sz;
988 obj->efile.maps_shndx = -1;
989 obj->efile.btf_maps_shndx = -1;
990 obj->efile.data_shndx = -1;
991 obj->efile.rodata_shndx = -1;
992 obj->efile.bss_shndx = -1;
993 obj->efile.st_ops_shndx = -1;
994 obj->kconfig_map_idx = -1;
995
996 obj->kern_version = get_kernel_version();
997 obj->loaded = false;
998
999 INIT_LIST_HEAD(&obj->list);
1000 list_add(&obj->list, &bpf_objects_list);
1001 return obj;
1002 }
1003
1004 static void bpf_object__elf_finish(struct bpf_object *obj)
1005 {
1006 if (!obj_elf_valid(obj))
1007 return;
1008
1009 if (obj->efile.elf) {
1010 elf_end(obj->efile.elf);
1011 obj->efile.elf = NULL;
1012 }
1013 obj->efile.symbols = NULL;
1014 obj->efile.data = NULL;
1015 obj->efile.rodata = NULL;
1016 obj->efile.bss = NULL;
1017 obj->efile.st_ops_data = NULL;
1018
1019 zfree(&obj->efile.reloc_sects);
1020 obj->efile.nr_reloc_sects = 0;
1021 zclose(obj->efile.fd);
1022 obj->efile.obj_buf = NULL;
1023 obj->efile.obj_buf_sz = 0;
1024 }
1025
1026 static int bpf_object__elf_init(struct bpf_object *obj)
1027 {
1028 int err = 0;
1029 GElf_Ehdr *ep;
1030
1031 if (obj_elf_valid(obj)) {
1032 pr_warn("elf init: internal error\n");
1033 return -LIBBPF_ERRNO__LIBELF;
1034 }
1035
1036 if (obj->efile.obj_buf_sz > 0) {
1037 /*
1038 * obj_buf should have been validated by
1039 * bpf_object__open_buffer().
1040 */
1041 obj->efile.elf = elf_memory((char *)obj->efile.obj_buf,
1042 obj->efile.obj_buf_sz);
1043 } else {
1044 obj->efile.fd = open(obj->path, O_RDONLY);
1045 if (obj->efile.fd < 0) {
1046 char errmsg[STRERR_BUFSIZE], *cp;
1047
1048 err = -errno;
1049 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
1050 pr_warn("failed to open %s: %s\n", obj->path, cp);
1051 return err;
1052 }
1053
1054 obj->efile.elf = elf_begin(obj->efile.fd,
1055 LIBBPF_ELF_C_READ_MMAP, NULL);
1056 }
1057
1058 if (!obj->efile.elf) {
1059 pr_warn("failed to open %s as ELF file\n", obj->path);
1060 err = -LIBBPF_ERRNO__LIBELF;
1061 goto errout;
1062 }
1063
1064 if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) {
1065 pr_warn("failed to get EHDR from %s\n", obj->path);
1066 err = -LIBBPF_ERRNO__FORMAT;
1067 goto errout;
1068 }
1069 ep = &obj->efile.ehdr;
1070
1071 /* Old LLVM set e_machine to EM_NONE */
1072 if (ep->e_type != ET_REL ||
1073 (ep->e_machine && ep->e_machine != EM_BPF)) {
1074 pr_warn("%s is not an eBPF object file\n", obj->path);
1075 err = -LIBBPF_ERRNO__FORMAT;
1076 goto errout;
1077 }
1078
1079 return 0;
1080 errout:
1081 bpf_object__elf_finish(obj);
1082 return err;
1083 }
1084
1085 static int bpf_object__check_endianness(struct bpf_object *obj)
1086 {
1087 #if __BYTE_ORDER == __LITTLE_ENDIAN
1088 if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2LSB)
1089 return 0;
1090 #elif __BYTE_ORDER == __BIG_ENDIAN
1091 if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2MSB)
1092 return 0;
1093 #else
1094 # error "Unrecognized __BYTE_ORDER__"
1095 #endif
1096 pr_warn("endianness mismatch.\n");
1097 return -LIBBPF_ERRNO__ENDIAN;
1098 }
1099
1100 static int
1101 bpf_object__init_license(struct bpf_object *obj, void *data, size_t size)
1102 {
1103 memcpy(obj->license, data, min(size, sizeof(obj->license) - 1));
1104 pr_debug("license of %s is %s\n", obj->path, obj->license);
1105 return 0;
1106 }
1107
1108 static int
1109 bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size)
1110 {
1111 __u32 kver;
1112
1113 if (size != sizeof(kver)) {
1114 pr_warn("invalid kver section in %s\n", obj->path);
1115 return -LIBBPF_ERRNO__FORMAT;
1116 }
1117 memcpy(&kver, data, sizeof(kver));
1118 obj->kern_version = kver;
1119 pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version);
1120 return 0;
1121 }
1122
1123 static bool bpf_map_type__is_map_in_map(enum bpf_map_type type)
1124 {
1125 if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS ||
1126 type == BPF_MAP_TYPE_HASH_OF_MAPS)
1127 return true;
1128 return false;
1129 }
1130
1131 static int bpf_object_search_section_size(const struct bpf_object *obj,
1132 const char *name, size_t *d_size)
1133 {
1134 const GElf_Ehdr *ep = &obj->efile.ehdr;
1135 Elf *elf = obj->efile.elf;
1136 Elf_Scn *scn = NULL;
1137 int idx = 0;
1138
1139 while ((scn = elf_nextscn(elf, scn)) != NULL) {
1140 const char *sec_name;
1141 Elf_Data *data;
1142 GElf_Shdr sh;
1143
1144 idx++;
1145 if (gelf_getshdr(scn, &sh) != &sh) {
1146 pr_warn("failed to get section(%d) header from %s\n",
1147 idx, obj->path);
1148 return -EIO;
1149 }
1150
1151 sec_name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name);
1152 if (!sec_name) {
1153 pr_warn("failed to get section(%d) name from %s\n",
1154 idx, obj->path);
1155 return -EIO;
1156 }
1157
1158 if (strcmp(name, sec_name))
1159 continue;
1160
1161 data = elf_getdata(scn, 0);
1162 if (!data) {
1163 pr_warn("failed to get section(%d) data from %s(%s)\n",
1164 idx, name, obj->path);
1165 return -EIO;
1166 }
1167
1168 *d_size = data->d_size;
1169 return 0;
1170 }
1171
1172 return -ENOENT;
1173 }
1174
1175 int bpf_object__section_size(const struct bpf_object *obj, const char *name,
1176 __u32 *size)
1177 {
1178 int ret = -ENOENT;
1179 size_t d_size;
1180
1181 *size = 0;
1182 if (!name) {
1183 return -EINVAL;
1184 } else if (!strcmp(name, DATA_SEC)) {
1185 if (obj->efile.data)
1186 *size = obj->efile.data->d_size;
1187 } else if (!strcmp(name, BSS_SEC)) {
1188 if (obj->efile.bss)
1189 *size = obj->efile.bss->d_size;
1190 } else if (!strcmp(name, RODATA_SEC)) {
1191 if (obj->efile.rodata)
1192 *size = obj->efile.rodata->d_size;
1193 } else if (!strcmp(name, STRUCT_OPS_SEC)) {
1194 if (obj->efile.st_ops_data)
1195 *size = obj->efile.st_ops_data->d_size;
1196 } else {
1197 ret = bpf_object_search_section_size(obj, name, &d_size);
1198 if (!ret)
1199 *size = d_size;
1200 }
1201
1202 return *size ? 0 : ret;
1203 }
1204
1205 int bpf_object__variable_offset(const struct bpf_object *obj, const char *name,
1206 __u32 *off)
1207 {
1208 Elf_Data *symbols = obj->efile.symbols;
1209 const char *sname;
1210 size_t si;
1211
1212 if (!name || !off)
1213 return -EINVAL;
1214
1215 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym); si++) {
1216 GElf_Sym sym;
1217
1218 if (!gelf_getsym(symbols, si, &sym))
1219 continue;
1220 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL ||
1221 GELF_ST_TYPE(sym.st_info) != STT_OBJECT)
1222 continue;
1223
1224 sname = elf_strptr(obj->efile.elf, obj->efile.strtabidx,
1225 sym.st_name);
1226 if (!sname) {
1227 pr_warn("failed to get sym name string for var %s\n",
1228 name);
1229 return -EIO;
1230 }
1231 if (strcmp(name, sname) == 0) {
1232 *off = sym.st_value;
1233 return 0;
1234 }
1235 }
1236
1237 return -ENOENT;
1238 }
1239
1240 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj)
1241 {
1242 struct bpf_map *new_maps;
1243 size_t new_cap;
1244 int i;
1245
1246 if (obj->nr_maps < obj->maps_cap)
1247 return &obj->maps[obj->nr_maps++];
1248
1249 new_cap = max((size_t)4, obj->maps_cap * 3 / 2);
1250 new_maps = realloc(obj->maps, new_cap * sizeof(*obj->maps));
1251 if (!new_maps) {
1252 pr_warn("alloc maps for object failed\n");
1253 return ERR_PTR(-ENOMEM);
1254 }
1255
1256 obj->maps_cap = new_cap;
1257 obj->maps = new_maps;
1258
1259 /* zero out new maps */
1260 memset(obj->maps + obj->nr_maps, 0,
1261 (obj->maps_cap - obj->nr_maps) * sizeof(*obj->maps));
1262 /*
1263 * fill all fd with -1 so won't close incorrect fd (fd=0 is stdin)
1264 * when failure (zclose won't close negative fd)).
1265 */
1266 for (i = obj->nr_maps; i < obj->maps_cap; i++) {
1267 obj->maps[i].fd = -1;
1268 obj->maps[i].inner_map_fd = -1;
1269 }
1270
1271 return &obj->maps[obj->nr_maps++];
1272 }
1273
1274 static size_t bpf_map_mmap_sz(const struct bpf_map *map)
1275 {
1276 long page_sz = sysconf(_SC_PAGE_SIZE);
1277 size_t map_sz;
1278
1279 map_sz = (size_t)roundup(map->def.value_size, 8) * map->def.max_entries;
1280 map_sz = roundup(map_sz, page_sz);
1281 return map_sz;
1282 }
1283
1284 static char *internal_map_name(struct bpf_object *obj,
1285 enum libbpf_map_type type)
1286 {
1287 char map_name[BPF_OBJ_NAME_LEN], *p;
1288 const char *sfx = libbpf_type_to_btf_name[type];
1289 int sfx_len = max((size_t)7, strlen(sfx));
1290 int pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1,
1291 strlen(obj->name));
1292
1293 snprintf(map_name, sizeof(map_name), "%.*s%.*s", pfx_len, obj->name,
1294 sfx_len, libbpf_type_to_btf_name[type]);
1295
1296 /* sanitise map name to characters allowed by kernel */
1297 for (p = map_name; *p && p < map_name + sizeof(map_name); p++)
1298 if (!isalnum(*p) && *p != '_' && *p != '.')
1299 *p = '_';
1300
1301 return strdup(map_name);
1302 }
1303
1304 static int
1305 bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type,
1306 int sec_idx, void *data, size_t data_sz)
1307 {
1308 struct bpf_map_def *def;
1309 struct bpf_map *map;
1310 int err;
1311
1312 map = bpf_object__add_map(obj);
1313 if (IS_ERR(map))
1314 return PTR_ERR(map);
1315
1316 map->libbpf_type = type;
1317 map->sec_idx = sec_idx;
1318 map->sec_offset = 0;
1319 map->name = internal_map_name(obj, type);
1320 if (!map->name) {
1321 pr_warn("failed to alloc map name\n");
1322 return -ENOMEM;
1323 }
1324
1325 def = &map->def;
1326 def->type = BPF_MAP_TYPE_ARRAY;
1327 def->key_size = sizeof(int);
1328 def->value_size = data_sz;
1329 def->max_entries = 1;
1330 def->map_flags = type == LIBBPF_MAP_RODATA || type == LIBBPF_MAP_KCONFIG
1331 ? BPF_F_RDONLY_PROG : 0;
1332 def->map_flags |= BPF_F_MMAPABLE;
1333
1334 pr_debug("map '%s' (global data): at sec_idx %d, offset %zu, flags %x.\n",
1335 map->name, map->sec_idx, map->sec_offset, def->map_flags);
1336
1337 map->mmaped = mmap(NULL, bpf_map_mmap_sz(map), PROT_READ | PROT_WRITE,
1338 MAP_SHARED | MAP_ANONYMOUS, -1, 0);
1339 if (map->mmaped == MAP_FAILED) {
1340 err = -errno;
1341 map->mmaped = NULL;
1342 pr_warn("failed to alloc map '%s' content buffer: %d\n",
1343 map->name, err);
1344 zfree(&map->name);
1345 return err;
1346 }
1347
1348 if (data)
1349 memcpy(map->mmaped, data, data_sz);
1350
1351 pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name);
1352 return 0;
1353 }
1354
1355 static int bpf_object__init_global_data_maps(struct bpf_object *obj)
1356 {
1357 int err;
1358
1359 /*
1360 * Populate obj->maps with libbpf internal maps.
1361 */
1362 if (obj->efile.data_shndx >= 0) {
1363 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA,
1364 obj->efile.data_shndx,
1365 obj->efile.data->d_buf,
1366 obj->efile.data->d_size);
1367 if (err)
1368 return err;
1369 }
1370 if (obj->efile.rodata_shndx >= 0) {
1371 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA,
1372 obj->efile.rodata_shndx,
1373 obj->efile.rodata->d_buf,
1374 obj->efile.rodata->d_size);
1375 if (err)
1376 return err;
1377 }
1378 if (obj->efile.bss_shndx >= 0) {
1379 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS,
1380 obj->efile.bss_shndx,
1381 NULL,
1382 obj->efile.bss->d_size);
1383 if (err)
1384 return err;
1385 }
1386 return 0;
1387 }
1388
1389
1390 static struct extern_desc *find_extern_by_name(const struct bpf_object *obj,
1391 const void *name)
1392 {
1393 int i;
1394
1395 for (i = 0; i < obj->nr_extern; i++) {
1396 if (strcmp(obj->externs[i].name, name) == 0)
1397 return &obj->externs[i];
1398 }
1399 return NULL;
1400 }
1401
1402 static int set_ext_value_tri(struct extern_desc *ext, void *ext_val,
1403 char value)
1404 {
1405 switch (ext->type) {
1406 case EXT_BOOL:
1407 if (value == 'm') {
1408 pr_warn("extern %s=%c should be tristate or char\n",
1409 ext->name, value);
1410 return -EINVAL;
1411 }
1412 *(bool *)ext_val = value == 'y' ? true : false;
1413 break;
1414 case EXT_TRISTATE:
1415 if (value == 'y')
1416 *(enum libbpf_tristate *)ext_val = TRI_YES;
1417 else if (value == 'm')
1418 *(enum libbpf_tristate *)ext_val = TRI_MODULE;
1419 else /* value == 'n' */
1420 *(enum libbpf_tristate *)ext_val = TRI_NO;
1421 break;
1422 case EXT_CHAR:
1423 *(char *)ext_val = value;
1424 break;
1425 case EXT_UNKNOWN:
1426 case EXT_INT:
1427 case EXT_CHAR_ARR:
1428 default:
1429 pr_warn("extern %s=%c should be bool, tristate, or char\n",
1430 ext->name, value);
1431 return -EINVAL;
1432 }
1433 ext->is_set = true;
1434 return 0;
1435 }
1436
1437 static int set_ext_value_str(struct extern_desc *ext, char *ext_val,
1438 const char *value)
1439 {
1440 size_t len;
1441
1442 if (ext->type != EXT_CHAR_ARR) {
1443 pr_warn("extern %s=%s should char array\n", ext->name, value);
1444 return -EINVAL;
1445 }
1446
1447 len = strlen(value);
1448 if (value[len - 1] != '"') {
1449 pr_warn("extern '%s': invalid string config '%s'\n",
1450 ext->name, value);
1451 return -EINVAL;
1452 }
1453
1454 /* strip quotes */
1455 len -= 2;
1456 if (len >= ext->sz) {
1457 pr_warn("extern '%s': long string config %s of (%zu bytes) truncated to %d bytes\n",
1458 ext->name, value, len, ext->sz - 1);
1459 len = ext->sz - 1;
1460 }
1461 memcpy(ext_val, value + 1, len);
1462 ext_val[len] = '\0';
1463 ext->is_set = true;
1464 return 0;
1465 }
1466
1467 static int parse_u64(const char *value, __u64 *res)
1468 {
1469 char *value_end;
1470 int err;
1471
1472 errno = 0;
1473 *res = strtoull(value, &value_end, 0);
1474 if (errno) {
1475 err = -errno;
1476 pr_warn("failed to parse '%s' as integer: %d\n", value, err);
1477 return err;
1478 }
1479 if (*value_end) {
1480 pr_warn("failed to parse '%s' as integer completely\n", value);
1481 return -EINVAL;
1482 }
1483 return 0;
1484 }
1485
1486 static bool is_ext_value_in_range(const struct extern_desc *ext, __u64 v)
1487 {
1488 int bit_sz = ext->sz * 8;
1489
1490 if (ext->sz == 8)
1491 return true;
1492
1493 /* Validate that value stored in u64 fits in integer of `ext->sz`
1494 * bytes size without any loss of information. If the target integer
1495 * is signed, we rely on the following limits of integer type of
1496 * Y bits and subsequent transformation:
1497 *
1498 * -2^(Y-1) <= X <= 2^(Y-1) - 1
1499 * 0 <= X + 2^(Y-1) <= 2^Y - 1
1500 * 0 <= X + 2^(Y-1) < 2^Y
1501 *
1502 * For unsigned target integer, check that all the (64 - Y) bits are
1503 * zero.
1504 */
1505 if (ext->is_signed)
1506 return v + (1ULL << (bit_sz - 1)) < (1ULL << bit_sz);
1507 else
1508 return (v >> bit_sz) == 0;
1509 }
1510
1511 static int set_ext_value_num(struct extern_desc *ext, void *ext_val,
1512 __u64 value)
1513 {
1514 if (ext->type != EXT_INT && ext->type != EXT_CHAR) {
1515 pr_warn("extern %s=%llu should be integer\n",
1516 ext->name, (unsigned long long)value);
1517 return -EINVAL;
1518 }
1519 if (!is_ext_value_in_range(ext, value)) {
1520 pr_warn("extern %s=%llu value doesn't fit in %d bytes\n",
1521 ext->name, (unsigned long long)value, ext->sz);
1522 return -ERANGE;
1523 }
1524 switch (ext->sz) {
1525 case 1: *(__u8 *)ext_val = value; break;
1526 case 2: *(__u16 *)ext_val = value; break;
1527 case 4: *(__u32 *)ext_val = value; break;
1528 case 8: *(__u64 *)ext_val = value; break;
1529 default:
1530 return -EINVAL;
1531 }
1532 ext->is_set = true;
1533 return 0;
1534 }
1535
1536 static int bpf_object__process_kconfig_line(struct bpf_object *obj,
1537 char *buf, void *data)
1538 {
1539 struct extern_desc *ext;
1540 char *sep, *value;
1541 int len, err = 0;
1542 void *ext_val;
1543 __u64 num;
1544
1545 if (strncmp(buf, "CONFIG_", 7))
1546 return 0;
1547
1548 sep = strchr(buf, '=');
1549 if (!sep) {
1550 pr_warn("failed to parse '%s': no separator\n", buf);
1551 return -EINVAL;
1552 }
1553
1554 /* Trim ending '\n' */
1555 len = strlen(buf);
1556 if (buf[len - 1] == '\n')
1557 buf[len - 1] = '\0';
1558 /* Split on '=' and ensure that a value is present. */
1559 *sep = '\0';
1560 if (!sep[1]) {
1561 *sep = '=';
1562 pr_warn("failed to parse '%s': no value\n", buf);
1563 return -EINVAL;
1564 }
1565
1566 ext = find_extern_by_name(obj, buf);
1567 if (!ext || ext->is_set)
1568 return 0;
1569
1570 ext_val = data + ext->data_off;
1571 value = sep + 1;
1572
1573 switch (*value) {
1574 case 'y': case 'n': case 'm':
1575 err = set_ext_value_tri(ext, ext_val, *value);
1576 break;
1577 case '"':
1578 err = set_ext_value_str(ext, ext_val, value);
1579 break;
1580 default:
1581 /* assume integer */
1582 err = parse_u64(value, &num);
1583 if (err) {
1584 pr_warn("extern %s=%s should be integer\n",
1585 ext->name, value);
1586 return err;
1587 }
1588 err = set_ext_value_num(ext, ext_val, num);
1589 break;
1590 }
1591 if (err)
1592 return err;
1593 pr_debug("extern %s=%s\n", ext->name, value);
1594 return 0;
1595 }
1596
1597 static int bpf_object__read_kconfig_file(struct bpf_object *obj, void *data)
1598 {
1599 char buf[PATH_MAX];
1600 struct utsname uts;
1601 int len, err = 0;
1602 gzFile file;
1603
1604 uname(&uts);
1605 len = snprintf(buf, PATH_MAX, "/boot/config-%s", uts.release);
1606 if (len < 0)
1607 return -EINVAL;
1608 else if (len >= PATH_MAX)
1609 return -ENAMETOOLONG;
1610
1611 /* gzopen also accepts uncompressed files. */
1612 file = gzopen(buf, "r");
1613 if (!file)
1614 file = gzopen("/proc/config.gz", "r");
1615
1616 if (!file) {
1617 pr_warn("failed to open system Kconfig\n");
1618 return -ENOENT;
1619 }
1620
1621 while (gzgets(file, buf, sizeof(buf))) {
1622 err = bpf_object__process_kconfig_line(obj, buf, data);
1623 if (err) {
1624 pr_warn("error parsing system Kconfig line '%s': %d\n",
1625 buf, err);
1626 goto out;
1627 }
1628 }
1629
1630 out:
1631 gzclose(file);
1632 return err;
1633 }
1634
1635 static int bpf_object__read_kconfig_mem(struct bpf_object *obj,
1636 const char *config, void *data)
1637 {
1638 char buf[PATH_MAX];
1639 int err = 0;
1640 FILE *file;
1641
1642 file = fmemopen((void *)config, strlen(config), "r");
1643 if (!file) {
1644 err = -errno;
1645 pr_warn("failed to open in-memory Kconfig: %d\n", err);
1646 return err;
1647 }
1648
1649 while (fgets(buf, sizeof(buf), file)) {
1650 err = bpf_object__process_kconfig_line(obj, buf, data);
1651 if (err) {
1652 pr_warn("error parsing in-memory Kconfig line '%s': %d\n",
1653 buf, err);
1654 break;
1655 }
1656 }
1657
1658 fclose(file);
1659 return err;
1660 }
1661
1662 static int bpf_object__init_kconfig_map(struct bpf_object *obj)
1663 {
1664 struct extern_desc *last_ext;
1665 size_t map_sz;
1666 int err;
1667
1668 if (obj->nr_extern == 0)
1669 return 0;
1670
1671 last_ext = &obj->externs[obj->nr_extern - 1];
1672 map_sz = last_ext->data_off + last_ext->sz;
1673
1674 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_KCONFIG,
1675 obj->efile.symbols_shndx,
1676 NULL, map_sz);
1677 if (err)
1678 return err;
1679
1680 obj->kconfig_map_idx = obj->nr_maps - 1;
1681
1682 return 0;
1683 }
1684
1685 static int bpf_object__init_user_maps(struct bpf_object *obj, bool strict)
1686 {
1687 Elf_Data *symbols = obj->efile.symbols;
1688 int i, map_def_sz = 0, nr_maps = 0, nr_syms;
1689 Elf_Data *data = NULL;
1690 Elf_Scn *scn;
1691
1692 if (obj->efile.maps_shndx < 0)
1693 return 0;
1694
1695 if (!symbols)
1696 return -EINVAL;
1697
1698 scn = elf_getscn(obj->efile.elf, obj->efile.maps_shndx);
1699 if (scn)
1700 data = elf_getdata(scn, NULL);
1701 if (!scn || !data) {
1702 pr_warn("failed to get Elf_Data from map section %d\n",
1703 obj->efile.maps_shndx);
1704 return -EINVAL;
1705 }
1706
1707 /*
1708 * Count number of maps. Each map has a name.
1709 * Array of maps is not supported: only the first element is
1710 * considered.
1711 *
1712 * TODO: Detect array of map and report error.
1713 */
1714 nr_syms = symbols->d_size / sizeof(GElf_Sym);
1715 for (i = 0; i < nr_syms; i++) {
1716 GElf_Sym sym;
1717
1718 if (!gelf_getsym(symbols, i, &sym))
1719 continue;
1720 if (sym.st_shndx != obj->efile.maps_shndx)
1721 continue;
1722 nr_maps++;
1723 }
1724 /* Assume equally sized map definitions */
1725 pr_debug("maps in %s: %d maps in %zd bytes\n",
1726 obj->path, nr_maps, data->d_size);
1727
1728 if (!data->d_size || nr_maps == 0 || (data->d_size % nr_maps) != 0) {
1729 pr_warn("unable to determine map definition size section %s, %d maps in %zd bytes\n",
1730 obj->path, nr_maps, data->d_size);
1731 return -EINVAL;
1732 }
1733 map_def_sz = data->d_size / nr_maps;
1734
1735 /* Fill obj->maps using data in "maps" section. */
1736 for (i = 0; i < nr_syms; i++) {
1737 GElf_Sym sym;
1738 const char *map_name;
1739 struct bpf_map_def *def;
1740 struct bpf_map *map;
1741
1742 if (!gelf_getsym(symbols, i, &sym))
1743 continue;
1744 if (sym.st_shndx != obj->efile.maps_shndx)
1745 continue;
1746
1747 map = bpf_object__add_map(obj);
1748 if (IS_ERR(map))
1749 return PTR_ERR(map);
1750
1751 map_name = elf_strptr(obj->efile.elf, obj->efile.strtabidx,
1752 sym.st_name);
1753 if (!map_name) {
1754 pr_warn("failed to get map #%d name sym string for obj %s\n",
1755 i, obj->path);
1756 return -LIBBPF_ERRNO__FORMAT;
1757 }
1758
1759 map->libbpf_type = LIBBPF_MAP_UNSPEC;
1760 map->sec_idx = sym.st_shndx;
1761 map->sec_offset = sym.st_value;
1762 pr_debug("map '%s' (legacy): at sec_idx %d, offset %zu.\n",
1763 map_name, map->sec_idx, map->sec_offset);
1764 if (sym.st_value + map_def_sz > data->d_size) {
1765 pr_warn("corrupted maps section in %s: last map \"%s\" too small\n",
1766 obj->path, map_name);
1767 return -EINVAL;
1768 }
1769
1770 map->name = strdup(map_name);
1771 if (!map->name) {
1772 pr_warn("failed to alloc map name\n");
1773 return -ENOMEM;
1774 }
1775 pr_debug("map %d is \"%s\"\n", i, map->name);
1776 def = (struct bpf_map_def *)(data->d_buf + sym.st_value);
1777 /*
1778 * If the definition of the map in the object file fits in
1779 * bpf_map_def, copy it. Any extra fields in our version
1780 * of bpf_map_def will default to zero as a result of the
1781 * calloc above.
1782 */
1783 if (map_def_sz <= sizeof(struct bpf_map_def)) {
1784 memcpy(&map->def, def, map_def_sz);
1785 } else {
1786 /*
1787 * Here the map structure being read is bigger than what
1788 * we expect, truncate if the excess bits are all zero.
1789 * If they are not zero, reject this map as
1790 * incompatible.
1791 */
1792 char *b;
1793
1794 for (b = ((char *)def) + sizeof(struct bpf_map_def);
1795 b < ((char *)def) + map_def_sz; b++) {
1796 if (*b != 0) {
1797 pr_warn("maps section in %s: \"%s\" has unrecognized, non-zero options\n",
1798 obj->path, map_name);
1799 if (strict)
1800 return -EINVAL;
1801 }
1802 }
1803 memcpy(&map->def, def, sizeof(struct bpf_map_def));
1804 }
1805 }
1806 return 0;
1807 }
1808
1809 static const struct btf_type *
1810 skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id)
1811 {
1812 const struct btf_type *t = btf__type_by_id(btf, id);
1813
1814 if (res_id)
1815 *res_id = id;
1816
1817 while (btf_is_mod(t) || btf_is_typedef(t)) {
1818 if (res_id)
1819 *res_id = t->type;
1820 t = btf__type_by_id(btf, t->type);
1821 }
1822
1823 return t;
1824 }
1825
1826 static const struct btf_type *
1827 resolve_func_ptr(const struct btf *btf, __u32 id, __u32 *res_id)
1828 {
1829 const struct btf_type *t;
1830
1831 t = skip_mods_and_typedefs(btf, id, NULL);
1832 if (!btf_is_ptr(t))
1833 return NULL;
1834
1835 t = skip_mods_and_typedefs(btf, t->type, res_id);
1836
1837 return btf_is_func_proto(t) ? t : NULL;
1838 }
1839
1840 /*
1841 * Fetch integer attribute of BTF map definition. Such attributes are
1842 * represented using a pointer to an array, in which dimensionality of array
1843 * encodes specified integer value. E.g., int (*type)[BPF_MAP_TYPE_ARRAY];
1844 * encodes `type => BPF_MAP_TYPE_ARRAY` key/value pair completely using BTF
1845 * type definition, while using only sizeof(void *) space in ELF data section.
1846 */
1847 static bool get_map_field_int(const char *map_name, const struct btf *btf,
1848 const struct btf_member *m, __u32 *res)
1849 {
1850 const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL);
1851 const char *name = btf__name_by_offset(btf, m->name_off);
1852 const struct btf_array *arr_info;
1853 const struct btf_type *arr_t;
1854
1855 if (!btf_is_ptr(t)) {
1856 pr_warn("map '%s': attr '%s': expected PTR, got %u.\n",
1857 map_name, name, btf_kind(t));
1858 return false;
1859 }
1860
1861 arr_t = btf__type_by_id(btf, t->type);
1862 if (!arr_t) {
1863 pr_warn("map '%s': attr '%s': type [%u] not found.\n",
1864 map_name, name, t->type);
1865 return false;
1866 }
1867 if (!btf_is_array(arr_t)) {
1868 pr_warn("map '%s': attr '%s': expected ARRAY, got %u.\n",
1869 map_name, name, btf_kind(arr_t));
1870 return false;
1871 }
1872 arr_info = btf_array(arr_t);
1873 *res = arr_info->nelems;
1874 return true;
1875 }
1876
1877 static int build_map_pin_path(struct bpf_map *map, const char *path)
1878 {
1879 char buf[PATH_MAX];
1880 int err, len;
1881
1882 if (!path)
1883 path = "/sys/fs/bpf";
1884
1885 len = snprintf(buf, PATH_MAX, "%s/%s", path, bpf_map__name(map));
1886 if (len < 0)
1887 return -EINVAL;
1888 else if (len >= PATH_MAX)
1889 return -ENAMETOOLONG;
1890
1891 err = bpf_map__set_pin_path(map, buf);
1892 if (err)
1893 return err;
1894
1895 return 0;
1896 }
1897
1898 static int bpf_object__init_user_btf_map(struct bpf_object *obj,
1899 const struct btf_type *sec,
1900 int var_idx, int sec_idx,
1901 const Elf_Data *data, bool strict,
1902 const char *pin_root_path)
1903 {
1904 const struct btf_type *var, *def, *t;
1905 const struct btf_var_secinfo *vi;
1906 const struct btf_var *var_extra;
1907 const struct btf_member *m;
1908 const char *map_name;
1909 struct bpf_map *map;
1910 int vlen, i;
1911
1912 vi = btf_var_secinfos(sec) + var_idx;
1913 var = btf__type_by_id(obj->btf, vi->type);
1914 var_extra = btf_var(var);
1915 map_name = btf__name_by_offset(obj->btf, var->name_off);
1916 vlen = btf_vlen(var);
1917
1918 if (map_name == NULL || map_name[0] == '\0') {
1919 pr_warn("map #%d: empty name.\n", var_idx);
1920 return -EINVAL;
1921 }
1922 if ((__u64)vi->offset + vi->size > data->d_size) {
1923 pr_warn("map '%s' BTF data is corrupted.\n", map_name);
1924 return -EINVAL;
1925 }
1926 if (!btf_is_var(var)) {
1927 pr_warn("map '%s': unexpected var kind %u.\n",
1928 map_name, btf_kind(var));
1929 return -EINVAL;
1930 }
1931 if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED &&
1932 var_extra->linkage != BTF_VAR_STATIC) {
1933 pr_warn("map '%s': unsupported var linkage %u.\n",
1934 map_name, var_extra->linkage);
1935 return -EOPNOTSUPP;
1936 }
1937
1938 def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
1939 if (!btf_is_struct(def)) {
1940 pr_warn("map '%s': unexpected def kind %u.\n",
1941 map_name, btf_kind(var));
1942 return -EINVAL;
1943 }
1944 if (def->size > vi->size) {
1945 pr_warn("map '%s': invalid def size.\n", map_name);
1946 return -EINVAL;
1947 }
1948
1949 map = bpf_object__add_map(obj);
1950 if (IS_ERR(map))
1951 return PTR_ERR(map);
1952 map->name = strdup(map_name);
1953 if (!map->name) {
1954 pr_warn("map '%s': failed to alloc map name.\n", map_name);
1955 return -ENOMEM;
1956 }
1957 map->libbpf_type = LIBBPF_MAP_UNSPEC;
1958 map->def.type = BPF_MAP_TYPE_UNSPEC;
1959 map->sec_idx = sec_idx;
1960 map->sec_offset = vi->offset;
1961 pr_debug("map '%s': at sec_idx %d, offset %zu.\n",
1962 map_name, map->sec_idx, map->sec_offset);
1963
1964 vlen = btf_vlen(def);
1965 m = btf_members(def);
1966 for (i = 0; i < vlen; i++, m++) {
1967 const char *name = btf__name_by_offset(obj->btf, m->name_off);
1968
1969 if (!name) {
1970 pr_warn("map '%s': invalid field #%d.\n", map_name, i);
1971 return -EINVAL;
1972 }
1973 if (strcmp(name, "type") == 0) {
1974 if (!get_map_field_int(map_name, obj->btf, m,
1975 &map->def.type))
1976 return -EINVAL;
1977 pr_debug("map '%s': found type = %u.\n",
1978 map_name, map->def.type);
1979 } else if (strcmp(name, "max_entries") == 0) {
1980 if (!get_map_field_int(map_name, obj->btf, m,
1981 &map->def.max_entries))
1982 return -EINVAL;
1983 pr_debug("map '%s': found max_entries = %u.\n",
1984 map_name, map->def.max_entries);
1985 } else if (strcmp(name, "map_flags") == 0) {
1986 if (!get_map_field_int(map_name, obj->btf, m,
1987 &map->def.map_flags))
1988 return -EINVAL;
1989 pr_debug("map '%s': found map_flags = %u.\n",
1990 map_name, map->def.map_flags);
1991 } else if (strcmp(name, "key_size") == 0) {
1992 __u32 sz;
1993
1994 if (!get_map_field_int(map_name, obj->btf, m, &sz))
1995 return -EINVAL;
1996 pr_debug("map '%s': found key_size = %u.\n",
1997 map_name, sz);
1998 if (map->def.key_size && map->def.key_size != sz) {
1999 pr_warn("map '%s': conflicting key size %u != %u.\n",
2000 map_name, map->def.key_size, sz);
2001 return -EINVAL;
2002 }
2003 map->def.key_size = sz;
2004 } else if (strcmp(name, "key") == 0) {
2005 __s64 sz;
2006
2007 t = btf__type_by_id(obj->btf, m->type);
2008 if (!t) {
2009 pr_warn("map '%s': key type [%d] not found.\n",
2010 map_name, m->type);
2011 return -EINVAL;
2012 }
2013 if (!btf_is_ptr(t)) {
2014 pr_warn("map '%s': key spec is not PTR: %u.\n",
2015 map_name, btf_kind(t));
2016 return -EINVAL;
2017 }
2018 sz = btf__resolve_size(obj->btf, t->type);
2019 if (sz < 0) {
2020 pr_warn("map '%s': can't determine key size for type [%u]: %zd.\n",
2021 map_name, t->type, (ssize_t)sz);
2022 return sz;
2023 }
2024 pr_debug("map '%s': found key [%u], sz = %zd.\n",
2025 map_name, t->type, (ssize_t)sz);
2026 if (map->def.key_size && map->def.key_size != sz) {
2027 pr_warn("map '%s': conflicting key size %u != %zd.\n",
2028 map_name, map->def.key_size, (ssize_t)sz);
2029 return -EINVAL;
2030 }
2031 map->def.key_size = sz;
2032 map->btf_key_type_id = t->type;
2033 } else if (strcmp(name, "value_size") == 0) {
2034 __u32 sz;
2035
2036 if (!get_map_field_int(map_name, obj->btf, m, &sz))
2037 return -EINVAL;
2038 pr_debug("map '%s': found value_size = %u.\n",
2039 map_name, sz);
2040 if (map->def.value_size && map->def.value_size != sz) {
2041 pr_warn("map '%s': conflicting value size %u != %u.\n",
2042 map_name, map->def.value_size, sz);
2043 return -EINVAL;
2044 }
2045 map->def.value_size = sz;
2046 } else if (strcmp(name, "value") == 0) {
2047 __s64 sz;
2048
2049 t = btf__type_by_id(obj->btf, m->type);
2050 if (!t) {
2051 pr_warn("map '%s': value type [%d] not found.\n",
2052 map_name, m->type);
2053 return -EINVAL;
2054 }
2055 if (!btf_is_ptr(t)) {
2056 pr_warn("map '%s': value spec is not PTR: %u.\n",
2057 map_name, btf_kind(t));
2058 return -EINVAL;
2059 }
2060 sz = btf__resolve_size(obj->btf, t->type);
2061 if (sz < 0) {
2062 pr_warn("map '%s': can't determine value size for type [%u]: %zd.\n",
2063 map_name, t->type, (ssize_t)sz);
2064 return sz;
2065 }
2066 pr_debug("map '%s': found value [%u], sz = %zd.\n",
2067 map_name, t->type, (ssize_t)sz);
2068 if (map->def.value_size && map->def.value_size != sz) {
2069 pr_warn("map '%s': conflicting value size %u != %zd.\n",
2070 map_name, map->def.value_size, (ssize_t)sz);
2071 return -EINVAL;
2072 }
2073 map->def.value_size = sz;
2074 map->btf_value_type_id = t->type;
2075 } else if (strcmp(name, "pinning") == 0) {
2076 __u32 val;
2077 int err;
2078
2079 if (!get_map_field_int(map_name, obj->btf, m, &val))
2080 return -EINVAL;
2081 pr_debug("map '%s': found pinning = %u.\n",
2082 map_name, val);
2083
2084 if (val != LIBBPF_PIN_NONE &&
2085 val != LIBBPF_PIN_BY_NAME) {
2086 pr_warn("map '%s': invalid pinning value %u.\n",
2087 map_name, val);
2088 return -EINVAL;
2089 }
2090 if (val == LIBBPF_PIN_BY_NAME) {
2091 err = build_map_pin_path(map, pin_root_path);
2092 if (err) {
2093 pr_warn("map '%s': couldn't build pin path.\n",
2094 map_name);
2095 return err;
2096 }
2097 }
2098 } else {
2099 if (strict) {
2100 pr_warn("map '%s': unknown field '%s'.\n",
2101 map_name, name);
2102 return -ENOTSUP;
2103 }
2104 pr_debug("map '%s': ignoring unknown field '%s'.\n",
2105 map_name, name);
2106 }
2107 }
2108
2109 if (map->def.type == BPF_MAP_TYPE_UNSPEC) {
2110 pr_warn("map '%s': map type isn't specified.\n", map_name);
2111 return -EINVAL;
2112 }
2113
2114 return 0;
2115 }
2116
2117 static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict,
2118 const char *pin_root_path)
2119 {
2120 const struct btf_type *sec = NULL;
2121 int nr_types, i, vlen, err;
2122 const struct btf_type *t;
2123 const char *name;
2124 Elf_Data *data;
2125 Elf_Scn *scn;
2126
2127 if (obj->efile.btf_maps_shndx < 0)
2128 return 0;
2129
2130 scn = elf_getscn(obj->efile.elf, obj->efile.btf_maps_shndx);
2131 if (scn)
2132 data = elf_getdata(scn, NULL);
2133 if (!scn || !data) {
2134 pr_warn("failed to get Elf_Data from map section %d (%s)\n",
2135 obj->efile.maps_shndx, MAPS_ELF_SEC);
2136 return -EINVAL;
2137 }
2138
2139 nr_types = btf__get_nr_types(obj->btf);
2140 for (i = 1; i <= nr_types; i++) {
2141 t = btf__type_by_id(obj->btf, i);
2142 if (!btf_is_datasec(t))
2143 continue;
2144 name = btf__name_by_offset(obj->btf, t->name_off);
2145 if (strcmp(name, MAPS_ELF_SEC) == 0) {
2146 sec = t;
2147 break;
2148 }
2149 }
2150
2151 if (!sec) {
2152 pr_warn("DATASEC '%s' not found.\n", MAPS_ELF_SEC);
2153 return -ENOENT;
2154 }
2155
2156 vlen = btf_vlen(sec);
2157 for (i = 0; i < vlen; i++) {
2158 err = bpf_object__init_user_btf_map(obj, sec, i,
2159 obj->efile.btf_maps_shndx,
2160 data, strict,
2161 pin_root_path);
2162 if (err)
2163 return err;
2164 }
2165
2166 return 0;
2167 }
2168
2169 static int bpf_object__init_maps(struct bpf_object *obj,
2170 const struct bpf_object_open_opts *opts)
2171 {
2172 const char *pin_root_path;
2173 bool strict;
2174 int err;
2175
2176 strict = !OPTS_GET(opts, relaxed_maps, false);
2177 pin_root_path = OPTS_GET(opts, pin_root_path, NULL);
2178
2179 err = bpf_object__init_user_maps(obj, strict);
2180 err = err ?: bpf_object__init_user_btf_maps(obj, strict, pin_root_path);
2181 err = err ?: bpf_object__init_global_data_maps(obj);
2182 err = err ?: bpf_object__init_kconfig_map(obj);
2183 err = err ?: bpf_object__init_struct_ops_maps(obj);
2184 if (err)
2185 return err;
2186
2187 return 0;
2188 }
2189
2190 static bool section_have_execinstr(struct bpf_object *obj, int idx)
2191 {
2192 Elf_Scn *scn;
2193 GElf_Shdr sh;
2194
2195 scn = elf_getscn(obj->efile.elf, idx);
2196 if (!scn)
2197 return false;
2198
2199 if (gelf_getshdr(scn, &sh) != &sh)
2200 return false;
2201
2202 if (sh.sh_flags & SHF_EXECINSTR)
2203 return true;
2204
2205 return false;
2206 }
2207
2208 static void bpf_object__sanitize_btf(struct bpf_object *obj)
2209 {
2210 bool has_func_global = obj->caps.btf_func_global;
2211 bool has_datasec = obj->caps.btf_datasec;
2212 bool has_func = obj->caps.btf_func;
2213 struct btf *btf = obj->btf;
2214 struct btf_type *t;
2215 int i, j, vlen;
2216
2217 if (!obj->btf || (has_func && has_datasec && has_func_global))
2218 return;
2219
2220 for (i = 1; i <= btf__get_nr_types(btf); i++) {
2221 t = (struct btf_type *)btf__type_by_id(btf, i);
2222
2223 if (!has_datasec && btf_is_var(t)) {
2224 /* replace VAR with INT */
2225 t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0);
2226 /*
2227 * using size = 1 is the safest choice, 4 will be too
2228 * big and cause kernel BTF validation failure if
2229 * original variable took less than 4 bytes
2230 */
2231 t->size = 1;
2232 *(int *)(t + 1) = BTF_INT_ENC(0, 0, 8);
2233 } else if (!has_datasec && btf_is_datasec(t)) {
2234 /* replace DATASEC with STRUCT */
2235 const struct btf_var_secinfo *v = btf_var_secinfos(t);
2236 struct btf_member *m = btf_members(t);
2237 struct btf_type *vt;
2238 char *name;
2239
2240 name = (char *)btf__name_by_offset(btf, t->name_off);
2241 while (*name) {
2242 if (*name == '.')
2243 *name = '_';
2244 name++;
2245 }
2246
2247 vlen = btf_vlen(t);
2248 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen);
2249 for (j = 0; j < vlen; j++, v++, m++) {
2250 /* order of field assignments is important */
2251 m->offset = v->offset * 8;
2252 m->type = v->type;
2253 /* preserve variable name as member name */
2254 vt = (void *)btf__type_by_id(btf, v->type);
2255 m->name_off = vt->name_off;
2256 }
2257 } else if (!has_func && btf_is_func_proto(t)) {
2258 /* replace FUNC_PROTO with ENUM */
2259 vlen = btf_vlen(t);
2260 t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen);
2261 t->size = sizeof(__u32); /* kernel enforced */
2262 } else if (!has_func && btf_is_func(t)) {
2263 /* replace FUNC with TYPEDEF */
2264 t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0);
2265 } else if (!has_func_global && btf_is_func(t)) {
2266 /* replace BTF_FUNC_GLOBAL with BTF_FUNC_STATIC */
2267 t->info = BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0);
2268 }
2269 }
2270 }
2271
2272 static void bpf_object__sanitize_btf_ext(struct bpf_object *obj)
2273 {
2274 if (!obj->btf_ext)
2275 return;
2276
2277 if (!obj->caps.btf_func) {
2278 btf_ext__free(obj->btf_ext);
2279 obj->btf_ext = NULL;
2280 }
2281 }
2282
2283 static bool libbpf_needs_btf(const struct bpf_object *obj)
2284 {
2285 return obj->efile.btf_maps_shndx >= 0 ||
2286 obj->efile.st_ops_shndx >= 0 ||
2287 obj->nr_extern > 0;
2288 }
2289
2290 static bool kernel_needs_btf(const struct bpf_object *obj)
2291 {
2292 return obj->efile.st_ops_shndx >= 0;
2293 }
2294
2295 static int bpf_object__init_btf(struct bpf_object *obj,
2296 Elf_Data *btf_data,
2297 Elf_Data *btf_ext_data)
2298 {
2299 int err = -ENOENT;
2300
2301 if (btf_data) {
2302 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size);
2303 if (IS_ERR(obj->btf)) {
2304 err = PTR_ERR(obj->btf);
2305 obj->btf = NULL;
2306 pr_warn("Error loading ELF section %s: %d.\n",
2307 BTF_ELF_SEC, err);
2308 goto out;
2309 }
2310 err = 0;
2311 }
2312 if (btf_ext_data) {
2313 if (!obj->btf) {
2314 pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n",
2315 BTF_EXT_ELF_SEC, BTF_ELF_SEC);
2316 goto out;
2317 }
2318 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf,
2319 btf_ext_data->d_size);
2320 if (IS_ERR(obj->btf_ext)) {
2321 pr_warn("Error loading ELF section %s: %ld. Ignored and continue.\n",
2322 BTF_EXT_ELF_SEC, PTR_ERR(obj->btf_ext));
2323 obj->btf_ext = NULL;
2324 goto out;
2325 }
2326 }
2327 out:
2328 if (err && libbpf_needs_btf(obj)) {
2329 pr_warn("BTF is required, but is missing or corrupted.\n");
2330 return err;
2331 }
2332 return 0;
2333 }
2334
2335 static int bpf_object__finalize_btf(struct bpf_object *obj)
2336 {
2337 int err;
2338
2339 if (!obj->btf)
2340 return 0;
2341
2342 err = btf__finalize_data(obj, obj->btf);
2343 if (!err)
2344 return 0;
2345
2346 pr_warn("Error finalizing %s: %d.\n", BTF_ELF_SEC, err);
2347 btf__free(obj->btf);
2348 obj->btf = NULL;
2349 btf_ext__free(obj->btf_ext);
2350 obj->btf_ext = NULL;
2351
2352 if (libbpf_needs_btf(obj)) {
2353 pr_warn("BTF is required, but is missing or corrupted.\n");
2354 return -ENOENT;
2355 }
2356 return 0;
2357 }
2358
2359 static inline bool libbpf_prog_needs_vmlinux_btf(struct bpf_program *prog)
2360 {
2361 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS ||
2362 prog->type == BPF_PROG_TYPE_LSM)
2363 return true;
2364
2365 /* BPF_PROG_TYPE_TRACING programs which do not attach to other programs
2366 * also need vmlinux BTF
2367 */
2368 if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd)
2369 return true;
2370
2371 return false;
2372 }
2373
2374 static int bpf_object__load_vmlinux_btf(struct bpf_object *obj)
2375 {
2376 struct bpf_program *prog;
2377 int err;
2378
2379 bpf_object__for_each_program(prog, obj) {
2380 if (libbpf_prog_needs_vmlinux_btf(prog)) {
2381 obj->btf_vmlinux = libbpf_find_kernel_btf();
2382 if (IS_ERR(obj->btf_vmlinux)) {
2383 err = PTR_ERR(obj->btf_vmlinux);
2384 pr_warn("Error loading vmlinux BTF: %d\n", err);
2385 obj->btf_vmlinux = NULL;
2386 return err;
2387 }
2388 return 0;
2389 }
2390 }
2391
2392 return 0;
2393 }
2394
2395 static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj)
2396 {
2397 int err = 0;
2398
2399 if (!obj->btf)
2400 return 0;
2401
2402 bpf_object__sanitize_btf(obj);
2403 bpf_object__sanitize_btf_ext(obj);
2404
2405 err = btf__load(obj->btf);
2406 if (err) {
2407 pr_warn("Error loading %s into kernel: %d.\n",
2408 BTF_ELF_SEC, err);
2409 btf__free(obj->btf);
2410 obj->btf = NULL;
2411 /* btf_ext can't exist without btf, so free it as well */
2412 if (obj->btf_ext) {
2413 btf_ext__free(obj->btf_ext);
2414 obj->btf_ext = NULL;
2415 }
2416
2417 if (kernel_needs_btf(obj))
2418 return err;
2419 }
2420 return 0;
2421 }
2422
2423 static int bpf_object__elf_collect(struct bpf_object *obj)
2424 {
2425 Elf *elf = obj->efile.elf;
2426 GElf_Ehdr *ep = &obj->efile.ehdr;
2427 Elf_Data *btf_ext_data = NULL;
2428 Elf_Data *btf_data = NULL;
2429 Elf_Scn *scn = NULL;
2430 int idx = 0, err = 0;
2431
2432 /* Elf is corrupted/truncated, avoid calling elf_strptr. */
2433 if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) {
2434 pr_warn("failed to get e_shstrndx from %s\n", obj->path);
2435 return -LIBBPF_ERRNO__FORMAT;
2436 }
2437
2438 while ((scn = elf_nextscn(elf, scn)) != NULL) {
2439 char *name;
2440 GElf_Shdr sh;
2441 Elf_Data *data;
2442
2443 idx++;
2444 if (gelf_getshdr(scn, &sh) != &sh) {
2445 pr_warn("failed to get section(%d) header from %s\n",
2446 idx, obj->path);
2447 return -LIBBPF_ERRNO__FORMAT;
2448 }
2449
2450 name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name);
2451 if (!name) {
2452 pr_warn("failed to get section(%d) name from %s\n",
2453 idx, obj->path);
2454 return -LIBBPF_ERRNO__FORMAT;
2455 }
2456
2457 data = elf_getdata(scn, 0);
2458 if (!data) {
2459 pr_warn("failed to get section(%d) data from %s(%s)\n",
2460 idx, name, obj->path);
2461 return -LIBBPF_ERRNO__FORMAT;
2462 }
2463 pr_debug("section(%d) %s, size %ld, link %d, flags %lx, type=%d\n",
2464 idx, name, (unsigned long)data->d_size,
2465 (int)sh.sh_link, (unsigned long)sh.sh_flags,
2466 (int)sh.sh_type);
2467
2468 if (strcmp(name, "license") == 0) {
2469 err = bpf_object__init_license(obj,
2470 data->d_buf,
2471 data->d_size);
2472 if (err)
2473 return err;
2474 } else if (strcmp(name, "version") == 0) {
2475 err = bpf_object__init_kversion(obj,
2476 data->d_buf,
2477 data->d_size);
2478 if (err)
2479 return err;
2480 } else if (strcmp(name, "maps") == 0) {
2481 obj->efile.maps_shndx = idx;
2482 } else if (strcmp(name, MAPS_ELF_SEC) == 0) {
2483 obj->efile.btf_maps_shndx = idx;
2484 } else if (strcmp(name, BTF_ELF_SEC) == 0) {
2485 btf_data = data;
2486 } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) {
2487 btf_ext_data = data;
2488 } else if (sh.sh_type == SHT_SYMTAB) {
2489 if (obj->efile.symbols) {
2490 pr_warn("bpf: multiple SYMTAB in %s\n",
2491 obj->path);
2492 return -LIBBPF_ERRNO__FORMAT;
2493 }
2494 obj->efile.symbols = data;
2495 obj->efile.symbols_shndx = idx;
2496 obj->efile.strtabidx = sh.sh_link;
2497 } else if (sh.sh_type == SHT_PROGBITS && data->d_size > 0) {
2498 if (sh.sh_flags & SHF_EXECINSTR) {
2499 if (strcmp(name, ".text") == 0)
2500 obj->efile.text_shndx = idx;
2501 err = bpf_object__add_program(obj, data->d_buf,
2502 data->d_size,
2503 name, idx);
2504 if (err) {
2505 char errmsg[STRERR_BUFSIZE];
2506 char *cp;
2507
2508 cp = libbpf_strerror_r(-err, errmsg,
2509 sizeof(errmsg));
2510 pr_warn("failed to alloc program %s (%s): %s",
2511 name, obj->path, cp);
2512 return err;
2513 }
2514 } else if (strcmp(name, DATA_SEC) == 0) {
2515 obj->efile.data = data;
2516 obj->efile.data_shndx = idx;
2517 } else if (strcmp(name, RODATA_SEC) == 0) {
2518 obj->efile.rodata = data;
2519 obj->efile.rodata_shndx = idx;
2520 } else if (strcmp(name, STRUCT_OPS_SEC) == 0) {
2521 obj->efile.st_ops_data = data;
2522 obj->efile.st_ops_shndx = idx;
2523 } else {
2524 pr_debug("skip section(%d) %s\n", idx, name);
2525 }
2526 } else if (sh.sh_type == SHT_REL) {
2527 int nr_sects = obj->efile.nr_reloc_sects;
2528 void *sects = obj->efile.reloc_sects;
2529 int sec = sh.sh_info; /* points to other section */
2530
2531 /* Only do relo for section with exec instructions */
2532 if (!section_have_execinstr(obj, sec) &&
2533 strcmp(name, ".rel" STRUCT_OPS_SEC)) {
2534 pr_debug("skip relo %s(%d) for section(%d)\n",
2535 name, idx, sec);
2536 continue;
2537 }
2538
2539 sects = reallocarray(sects, nr_sects + 1,
2540 sizeof(*obj->efile.reloc_sects));
2541 if (!sects) {
2542 pr_warn("reloc_sects realloc failed\n");
2543 return -ENOMEM;
2544 }
2545
2546 obj->efile.reloc_sects = sects;
2547 obj->efile.nr_reloc_sects++;
2548
2549 obj->efile.reloc_sects[nr_sects].shdr = sh;
2550 obj->efile.reloc_sects[nr_sects].data = data;
2551 } else if (sh.sh_type == SHT_NOBITS &&
2552 strcmp(name, BSS_SEC) == 0) {
2553 obj->efile.bss = data;
2554 obj->efile.bss_shndx = idx;
2555 } else {
2556 pr_debug("skip section(%d) %s\n", idx, name);
2557 }
2558 }
2559
2560 if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) {
2561 pr_warn("Corrupted ELF file: index of strtab invalid\n");
2562 return -LIBBPF_ERRNO__FORMAT;
2563 }
2564 return bpf_object__init_btf(obj, btf_data, btf_ext_data);
2565 }
2566
2567 static bool sym_is_extern(const GElf_Sym *sym)
2568 {
2569 int bind = GELF_ST_BIND(sym->st_info);
2570 /* externs are symbols w/ type=NOTYPE, bind=GLOBAL|WEAK, section=UND */
2571 return sym->st_shndx == SHN_UNDEF &&
2572 (bind == STB_GLOBAL || bind == STB_WEAK) &&
2573 GELF_ST_TYPE(sym->st_info) == STT_NOTYPE;
2574 }
2575
2576 static int find_extern_btf_id(const struct btf *btf, const char *ext_name)
2577 {
2578 const struct btf_type *t;
2579 const char *var_name;
2580 int i, n;
2581
2582 if (!btf)
2583 return -ESRCH;
2584
2585 n = btf__get_nr_types(btf);
2586 for (i = 1; i <= n; i++) {
2587 t = btf__type_by_id(btf, i);
2588
2589 if (!btf_is_var(t))
2590 continue;
2591
2592 var_name = btf__name_by_offset(btf, t->name_off);
2593 if (strcmp(var_name, ext_name))
2594 continue;
2595
2596 if (btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN)
2597 return -EINVAL;
2598
2599 return i;
2600 }
2601
2602 return -ENOENT;
2603 }
2604
2605 static enum extern_type find_extern_type(const struct btf *btf, int id,
2606 bool *is_signed)
2607 {
2608 const struct btf_type *t;
2609 const char *name;
2610
2611 t = skip_mods_and_typedefs(btf, id, NULL);
2612 name = btf__name_by_offset(btf, t->name_off);
2613
2614 if (is_signed)
2615 *is_signed = false;
2616 switch (btf_kind(t)) {
2617 case BTF_KIND_INT: {
2618 int enc = btf_int_encoding(t);
2619
2620 if (enc & BTF_INT_BOOL)
2621 return t->size == 1 ? EXT_BOOL : EXT_UNKNOWN;
2622 if (is_signed)
2623 *is_signed = enc & BTF_INT_SIGNED;
2624 if (t->size == 1)
2625 return EXT_CHAR;
2626 if (t->size < 1 || t->size > 8 || (t->size & (t->size - 1)))
2627 return EXT_UNKNOWN;
2628 return EXT_INT;
2629 }
2630 case BTF_KIND_ENUM:
2631 if (t->size != 4)
2632 return EXT_UNKNOWN;
2633 if (strcmp(name, "libbpf_tristate"))
2634 return EXT_UNKNOWN;
2635 return EXT_TRISTATE;
2636 case BTF_KIND_ARRAY:
2637 if (btf_array(t)->nelems == 0)
2638 return EXT_UNKNOWN;
2639 if (find_extern_type(btf, btf_array(t)->type, NULL) != EXT_CHAR)
2640 return EXT_UNKNOWN;
2641 return EXT_CHAR_ARR;
2642 default:
2643 return EXT_UNKNOWN;
2644 }
2645 }
2646
2647 static int cmp_externs(const void *_a, const void *_b)
2648 {
2649 const struct extern_desc *a = _a;
2650 const struct extern_desc *b = _b;
2651
2652 /* descending order by alignment requirements */
2653 if (a->align != b->align)
2654 return a->align > b->align ? -1 : 1;
2655 /* ascending order by size, within same alignment class */
2656 if (a->sz != b->sz)
2657 return a->sz < b->sz ? -1 : 1;
2658 /* resolve ties by name */
2659 return strcmp(a->name, b->name);
2660 }
2661
2662 static int bpf_object__collect_externs(struct bpf_object *obj)
2663 {
2664 const struct btf_type *t;
2665 struct extern_desc *ext;
2666 int i, n, off, btf_id;
2667 struct btf_type *sec;
2668 const char *ext_name;
2669 Elf_Scn *scn;
2670 GElf_Shdr sh;
2671
2672 if (!obj->efile.symbols)
2673 return 0;
2674
2675 scn = elf_getscn(obj->efile.elf, obj->efile.symbols_shndx);
2676 if (!scn)
2677 return -LIBBPF_ERRNO__FORMAT;
2678 if (gelf_getshdr(scn, &sh) != &sh)
2679 return -LIBBPF_ERRNO__FORMAT;
2680 n = sh.sh_size / sh.sh_entsize;
2681
2682 pr_debug("looking for externs among %d symbols...\n", n);
2683 for (i = 0; i < n; i++) {
2684 GElf_Sym sym;
2685
2686 if (!gelf_getsym(obj->efile.symbols, i, &sym))
2687 return -LIBBPF_ERRNO__FORMAT;
2688 if (!sym_is_extern(&sym))
2689 continue;
2690 ext_name = elf_strptr(obj->efile.elf, obj->efile.strtabidx,
2691 sym.st_name);
2692 if (!ext_name || !ext_name[0])
2693 continue;
2694
2695 ext = obj->externs;
2696 ext = reallocarray(ext, obj->nr_extern + 1, sizeof(*ext));
2697 if (!ext)
2698 return -ENOMEM;
2699 obj->externs = ext;
2700 ext = &ext[obj->nr_extern];
2701 memset(ext, 0, sizeof(*ext));
2702 obj->nr_extern++;
2703
2704 ext->btf_id = find_extern_btf_id(obj->btf, ext_name);
2705 if (ext->btf_id <= 0) {
2706 pr_warn("failed to find BTF for extern '%s': %d\n",
2707 ext_name, ext->btf_id);
2708 return ext->btf_id;
2709 }
2710 t = btf__type_by_id(obj->btf, ext->btf_id);
2711 ext->name = btf__name_by_offset(obj->btf, t->name_off);
2712 ext->sym_idx = i;
2713 ext->is_weak = GELF_ST_BIND(sym.st_info) == STB_WEAK;
2714 ext->sz = btf__resolve_size(obj->btf, t->type);
2715 if (ext->sz <= 0) {
2716 pr_warn("failed to resolve size of extern '%s': %d\n",
2717 ext_name, ext->sz);
2718 return ext->sz;
2719 }
2720 ext->align = btf__align_of(obj->btf, t->type);
2721 if (ext->align <= 0) {
2722 pr_warn("failed to determine alignment of extern '%s': %d\n",
2723 ext_name, ext->align);
2724 return -EINVAL;
2725 }
2726 ext->type = find_extern_type(obj->btf, t->type,
2727 &ext->is_signed);
2728 if (ext->type == EXT_UNKNOWN) {
2729 pr_warn("extern '%s' type is unsupported\n", ext_name);
2730 return -ENOTSUP;
2731 }
2732 }
2733 pr_debug("collected %d externs total\n", obj->nr_extern);
2734
2735 if (!obj->nr_extern)
2736 return 0;
2737
2738 /* sort externs by (alignment, size, name) and calculate their offsets
2739 * within a map */
2740 qsort(obj->externs, obj->nr_extern, sizeof(*ext), cmp_externs);
2741 off = 0;
2742 for (i = 0; i < obj->nr_extern; i++) {
2743 ext = &obj->externs[i];
2744 ext->data_off = roundup(off, ext->align);
2745 off = ext->data_off + ext->sz;
2746 pr_debug("extern #%d: symbol %d, off %u, name %s\n",
2747 i, ext->sym_idx, ext->data_off, ext->name);
2748 }
2749
2750 btf_id = btf__find_by_name(obj->btf, KCONFIG_SEC);
2751 if (btf_id <= 0) {
2752 pr_warn("no BTF info found for '%s' datasec\n", KCONFIG_SEC);
2753 return -ESRCH;
2754 }
2755
2756 sec = (struct btf_type *)btf__type_by_id(obj->btf, btf_id);
2757 sec->size = off;
2758 n = btf_vlen(sec);
2759 for (i = 0; i < n; i++) {
2760 struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i;
2761
2762 t = btf__type_by_id(obj->btf, vs->type);
2763 ext_name = btf__name_by_offset(obj->btf, t->name_off);
2764 ext = find_extern_by_name(obj, ext_name);
2765 if (!ext) {
2766 pr_warn("failed to find extern definition for BTF var '%s'\n",
2767 ext_name);
2768 return -ESRCH;
2769 }
2770 vs->offset = ext->data_off;
2771 btf_var(t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
2772 }
2773
2774 return 0;
2775 }
2776
2777 static struct bpf_program *
2778 bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx)
2779 {
2780 struct bpf_program *prog;
2781 size_t i;
2782
2783 for (i = 0; i < obj->nr_programs; i++) {
2784 prog = &obj->programs[i];
2785 if (prog->idx == idx)
2786 return prog;
2787 }
2788 return NULL;
2789 }
2790
2791 struct bpf_program *
2792 bpf_object__find_program_by_title(const struct bpf_object *obj,
2793 const char *title)
2794 {
2795 struct bpf_program *pos;
2796
2797 bpf_object__for_each_program(pos, obj) {
2798 if (pos->section_name && !strcmp(pos->section_name, title))
2799 return pos;
2800 }
2801 return NULL;
2802 }
2803
2804 struct bpf_program *
2805 bpf_object__find_program_by_name(const struct bpf_object *obj,
2806 const char *name)
2807 {
2808 struct bpf_program *prog;
2809
2810 bpf_object__for_each_program(prog, obj) {
2811 if (!strcmp(prog->name, name))
2812 return prog;
2813 }
2814 return NULL;
2815 }
2816
2817 static bool bpf_object__shndx_is_data(const struct bpf_object *obj,
2818 int shndx)
2819 {
2820 return shndx == obj->efile.data_shndx ||
2821 shndx == obj->efile.bss_shndx ||
2822 shndx == obj->efile.rodata_shndx;
2823 }
2824
2825 static bool bpf_object__shndx_is_maps(const struct bpf_object *obj,
2826 int shndx)
2827 {
2828 return shndx == obj->efile.maps_shndx ||
2829 shndx == obj->efile.btf_maps_shndx;
2830 }
2831
2832 static enum libbpf_map_type
2833 bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx)
2834 {
2835 if (shndx == obj->efile.data_shndx)
2836 return LIBBPF_MAP_DATA;
2837 else if (shndx == obj->efile.bss_shndx)
2838 return LIBBPF_MAP_BSS;
2839 else if (shndx == obj->efile.rodata_shndx)
2840 return LIBBPF_MAP_RODATA;
2841 else if (shndx == obj->efile.symbols_shndx)
2842 return LIBBPF_MAP_KCONFIG;
2843 else
2844 return LIBBPF_MAP_UNSPEC;
2845 }
2846
2847 static int bpf_program__record_reloc(struct bpf_program *prog,
2848 struct reloc_desc *reloc_desc,
2849 __u32 insn_idx, const char *name,
2850 const GElf_Sym *sym, const GElf_Rel *rel)
2851 {
2852 struct bpf_insn *insn = &prog->insns[insn_idx];
2853 size_t map_idx, nr_maps = prog->obj->nr_maps;
2854 struct bpf_object *obj = prog->obj;
2855 __u32 shdr_idx = sym->st_shndx;
2856 enum libbpf_map_type type;
2857 struct bpf_map *map;
2858
2859 /* sub-program call relocation */
2860 if (insn->code == (BPF_JMP | BPF_CALL)) {
2861 if (insn->src_reg != BPF_PSEUDO_CALL) {
2862 pr_warn("incorrect bpf_call opcode\n");
2863 return -LIBBPF_ERRNO__RELOC;
2864 }
2865 /* text_shndx can be 0, if no default "main" program exists */
2866 if (!shdr_idx || shdr_idx != obj->efile.text_shndx) {
2867 pr_warn("bad call relo against section %u\n", shdr_idx);
2868 return -LIBBPF_ERRNO__RELOC;
2869 }
2870 if (sym->st_value % 8) {
2871 pr_warn("bad call relo offset: %zu\n",
2872 (size_t)sym->st_value);
2873 return -LIBBPF_ERRNO__RELOC;
2874 }
2875 reloc_desc->type = RELO_CALL;
2876 reloc_desc->insn_idx = insn_idx;
2877 reloc_desc->sym_off = sym->st_value;
2878 obj->has_pseudo_calls = true;
2879 return 0;
2880 }
2881
2882 if (insn->code != (BPF_LD | BPF_IMM | BPF_DW)) {
2883 pr_warn("invalid relo for insns[%d].code 0x%x\n",
2884 insn_idx, insn->code);
2885 return -LIBBPF_ERRNO__RELOC;
2886 }
2887
2888 if (sym_is_extern(sym)) {
2889 int sym_idx = GELF_R_SYM(rel->r_info);
2890 int i, n = obj->nr_extern;
2891 struct extern_desc *ext;
2892
2893 for (i = 0; i < n; i++) {
2894 ext = &obj->externs[i];
2895 if (ext->sym_idx == sym_idx)
2896 break;
2897 }
2898 if (i >= n) {
2899 pr_warn("extern relo failed to find extern for sym %d\n",
2900 sym_idx);
2901 return -LIBBPF_ERRNO__RELOC;
2902 }
2903 pr_debug("found extern #%d '%s' (sym %d, off %u) for insn %u\n",
2904 i, ext->name, ext->sym_idx, ext->data_off, insn_idx);
2905 reloc_desc->type = RELO_EXTERN;
2906 reloc_desc->insn_idx = insn_idx;
2907 reloc_desc->sym_off = ext->data_off;
2908 return 0;
2909 }
2910
2911 if (!shdr_idx || shdr_idx >= SHN_LORESERVE) {
2912 pr_warn("invalid relo for \'%s\' in special section 0x%x; forgot to initialize global var?..\n",
2913 name, shdr_idx);
2914 return -LIBBPF_ERRNO__RELOC;
2915 }
2916
2917 type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx);
2918
2919 /* generic map reference relocation */
2920 if (type == LIBBPF_MAP_UNSPEC) {
2921 if (!bpf_object__shndx_is_maps(obj, shdr_idx)) {
2922 pr_warn("bad map relo against section %u\n",
2923 shdr_idx);
2924 return -LIBBPF_ERRNO__RELOC;
2925 }
2926 for (map_idx = 0; map_idx < nr_maps; map_idx++) {
2927 map = &obj->maps[map_idx];
2928 if (map->libbpf_type != type ||
2929 map->sec_idx != sym->st_shndx ||
2930 map->sec_offset != sym->st_value)
2931 continue;
2932 pr_debug("found map %zd (%s, sec %d, off %zu) for insn %u\n",
2933 map_idx, map->name, map->sec_idx,
2934 map->sec_offset, insn_idx);
2935 break;
2936 }
2937 if (map_idx >= nr_maps) {
2938 pr_warn("map relo failed to find map for sec %u, off %zu\n",
2939 shdr_idx, (size_t)sym->st_value);
2940 return -LIBBPF_ERRNO__RELOC;
2941 }
2942 reloc_desc->type = RELO_LD64;
2943 reloc_desc->insn_idx = insn_idx;
2944 reloc_desc->map_idx = map_idx;
2945 reloc_desc->sym_off = 0; /* sym->st_value determines map_idx */
2946 return 0;
2947 }
2948
2949 /* global data map relocation */
2950 if (!bpf_object__shndx_is_data(obj, shdr_idx)) {
2951 pr_warn("bad data relo against section %u\n", shdr_idx);
2952 return -LIBBPF_ERRNO__RELOC;
2953 }
2954 for (map_idx = 0; map_idx < nr_maps; map_idx++) {
2955 map = &obj->maps[map_idx];
2956 if (map->libbpf_type != type)
2957 continue;
2958 pr_debug("found data map %zd (%s, sec %d, off %zu) for insn %u\n",
2959 map_idx, map->name, map->sec_idx, map->sec_offset,
2960 insn_idx);
2961 break;
2962 }
2963 if (map_idx >= nr_maps) {
2964 pr_warn("data relo failed to find map for sec %u\n",
2965 shdr_idx);
2966 return -LIBBPF_ERRNO__RELOC;
2967 }
2968
2969 reloc_desc->type = RELO_DATA;
2970 reloc_desc->insn_idx = insn_idx;
2971 reloc_desc->map_idx = map_idx;
2972 reloc_desc->sym_off = sym->st_value;
2973 return 0;
2974 }
2975
2976 static int
2977 bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr,
2978 Elf_Data *data, struct bpf_object *obj)
2979 {
2980 Elf_Data *symbols = obj->efile.symbols;
2981 int err, i, nrels;
2982
2983 pr_debug("collecting relocating info for: '%s'\n", prog->section_name);
2984 nrels = shdr->sh_size / shdr->sh_entsize;
2985
2986 prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels);
2987 if (!prog->reloc_desc) {
2988 pr_warn("failed to alloc memory in relocation\n");
2989 return -ENOMEM;
2990 }
2991 prog->nr_reloc = nrels;
2992
2993 for (i = 0; i < nrels; i++) {
2994 const char *name;
2995 __u32 insn_idx;
2996 GElf_Sym sym;
2997 GElf_Rel rel;
2998
2999 if (!gelf_getrel(data, i, &rel)) {
3000 pr_warn("relocation: failed to get %d reloc\n", i);
3001 return -LIBBPF_ERRNO__FORMAT;
3002 }
3003 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) {
3004 pr_warn("relocation: symbol %"PRIx64" not found\n",
3005 GELF_R_SYM(rel.r_info));
3006 return -LIBBPF_ERRNO__FORMAT;
3007 }
3008 if (rel.r_offset % sizeof(struct bpf_insn))
3009 return -LIBBPF_ERRNO__FORMAT;
3010
3011 insn_idx = rel.r_offset / sizeof(struct bpf_insn);
3012 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx,
3013 sym.st_name) ? : "<?>";
3014
3015 pr_debug("relo for shdr %u, symb %zu, value %zu, type %d, bind %d, name %d (\'%s\'), insn %u\n",
3016 (__u32)sym.st_shndx, (size_t)GELF_R_SYM(rel.r_info),
3017 (size_t)sym.st_value, GELF_ST_TYPE(sym.st_info),
3018 GELF_ST_BIND(sym.st_info), sym.st_name, name,
3019 insn_idx);
3020
3021 err = bpf_program__record_reloc(prog, &prog->reloc_desc[i],
3022 insn_idx, name, &sym, &rel);
3023 if (err)
3024 return err;
3025 }
3026 return 0;
3027 }
3028
3029 static int bpf_map_find_btf_info(struct bpf_object *obj, struct bpf_map *map)
3030 {
3031 struct bpf_map_def *def = &map->def;
3032 __u32 key_type_id = 0, value_type_id = 0;
3033 int ret;
3034
3035 /* if it's BTF-defined map, we don't need to search for type IDs.
3036 * For struct_ops map, it does not need btf_key_type_id and
3037 * btf_value_type_id.
3038 */
3039 if (map->sec_idx == obj->efile.btf_maps_shndx ||
3040 bpf_map__is_struct_ops(map))
3041 return 0;
3042
3043 if (!bpf_map__is_internal(map)) {
3044 ret = btf__get_map_kv_tids(obj->btf, map->name, def->key_size,
3045 def->value_size, &key_type_id,
3046 &value_type_id);
3047 } else {
3048 /*
3049 * LLVM annotates global data differently in BTF, that is,
3050 * only as '.data', '.bss' or '.rodata'.
3051 */
3052 ret = btf__find_by_name(obj->btf,
3053 libbpf_type_to_btf_name[map->libbpf_type]);
3054 }
3055 if (ret < 0)
3056 return ret;
3057
3058 map->btf_key_type_id = key_type_id;
3059 map->btf_value_type_id = bpf_map__is_internal(map) ?
3060 ret : value_type_id;
3061 return 0;
3062 }
3063
3064 int bpf_map__reuse_fd(struct bpf_map *map, int fd)
3065 {
3066 struct bpf_map_info info = {};
3067 __u32 len = sizeof(info);
3068 int new_fd, err;
3069 char *new_name;
3070
3071 err = bpf_obj_get_info_by_fd(fd, &info, &len);
3072 if (err)
3073 return err;
3074
3075 new_name = strdup(info.name);
3076 if (!new_name)
3077 return -errno;
3078
3079 new_fd = open("/", O_RDONLY | O_CLOEXEC);
3080 if (new_fd < 0) {
3081 err = -errno;
3082 goto err_free_new_name;
3083 }
3084
3085 new_fd = dup3(fd, new_fd, O_CLOEXEC);
3086 if (new_fd < 0) {
3087 err = -errno;
3088 goto err_close_new_fd;
3089 }
3090
3091 err = zclose(map->fd);
3092 if (err) {
3093 err = -errno;
3094 goto err_close_new_fd;
3095 }
3096 free(map->name);
3097
3098 map->fd = new_fd;
3099 map->name = new_name;
3100 map->def.type = info.type;
3101 map->def.key_size = info.key_size;
3102 map->def.value_size = info.value_size;
3103 map->def.max_entries = info.max_entries;
3104 map->def.map_flags = info.map_flags;
3105 map->btf_key_type_id = info.btf_key_type_id;
3106 map->btf_value_type_id = info.btf_value_type_id;
3107 map->reused = true;
3108
3109 return 0;
3110
3111 err_close_new_fd:
3112 close(new_fd);
3113 err_free_new_name:
3114 free(new_name);
3115 return err;
3116 }
3117
3118 int bpf_map__resize(struct bpf_map *map, __u32 max_entries)
3119 {
3120 if (!map || !max_entries)
3121 return -EINVAL;
3122
3123 /* If map already created, its attributes can't be changed. */
3124 if (map->fd >= 0)
3125 return -EBUSY;
3126
3127 map->def.max_entries = max_entries;
3128
3129 return 0;
3130 }
3131
3132 static int
3133 bpf_object__probe_name(struct bpf_object *obj)
3134 {
3135 struct bpf_load_program_attr attr;
3136 char *cp, errmsg[STRERR_BUFSIZE];
3137 struct bpf_insn insns[] = {
3138 BPF_MOV64_IMM(BPF_REG_0, 0),
3139 BPF_EXIT_INSN(),
3140 };
3141 int ret;
3142
3143 /* make sure basic loading works */
3144
3145 memset(&attr, 0, sizeof(attr));
3146 attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
3147 attr.insns = insns;
3148 attr.insns_cnt = ARRAY_SIZE(insns);
3149 attr.license = "GPL";
3150
3151 ret = bpf_load_program_xattr(&attr, NULL, 0);
3152 if (ret < 0) {
3153 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
3154 pr_warn("Error in %s():%s(%d). Couldn't load basic 'r0 = 0' BPF program.\n",
3155 __func__, cp, errno);
3156 return -errno;
3157 }
3158 close(ret);
3159
3160 /* now try the same program, but with the name */
3161
3162 attr.name = "test";
3163 ret = bpf_load_program_xattr(&attr, NULL, 0);
3164 if (ret >= 0) {
3165 obj->caps.name = 1;
3166 close(ret);
3167 }
3168
3169 return 0;
3170 }
3171
3172 static int
3173 bpf_object__probe_global_data(struct bpf_object *obj)
3174 {
3175 struct bpf_load_program_attr prg_attr;
3176 struct bpf_create_map_attr map_attr;
3177 char *cp, errmsg[STRERR_BUFSIZE];
3178 struct bpf_insn insns[] = {
3179 BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16),
3180 BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42),
3181 BPF_MOV64_IMM(BPF_REG_0, 0),
3182 BPF_EXIT_INSN(),
3183 };
3184 int ret, map;
3185
3186 memset(&map_attr, 0, sizeof(map_attr));
3187 map_attr.map_type = BPF_MAP_TYPE_ARRAY;
3188 map_attr.key_size = sizeof(int);
3189 map_attr.value_size = 32;
3190 map_attr.max_entries = 1;
3191
3192 map = bpf_create_map_xattr(&map_attr);
3193 if (map < 0) {
3194 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
3195 pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n",
3196 __func__, cp, errno);
3197 return -errno;
3198 }
3199
3200 insns[0].imm = map;
3201
3202 memset(&prg_attr, 0, sizeof(prg_attr));
3203 prg_attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
3204 prg_attr.insns = insns;
3205 prg_attr.insns_cnt = ARRAY_SIZE(insns);
3206 prg_attr.license = "GPL";
3207
3208 ret = bpf_load_program_xattr(&prg_attr, NULL, 0);
3209 if (ret >= 0) {
3210 obj->caps.global_data = 1;
3211 close(ret);
3212 }
3213
3214 close(map);
3215 return 0;
3216 }
3217
3218 static int bpf_object__probe_btf_func(struct bpf_object *obj)
3219 {
3220 static const char strs[] = "\0int\0x\0a";
3221 /* void x(int a) {} */
3222 __u32 types[] = {
3223 /* int */
3224 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
3225 /* FUNC_PROTO */ /* [2] */
3226 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0),
3227 BTF_PARAM_ENC(7, 1),
3228 /* FUNC x */ /* [3] */
3229 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), 2),
3230 };
3231 int btf_fd;
3232
3233 btf_fd = libbpf__load_raw_btf((char *)types, sizeof(types),
3234 strs, sizeof(strs));
3235 if (btf_fd >= 0) {
3236 obj->caps.btf_func = 1;
3237 close(btf_fd);
3238 return 1;
3239 }
3240
3241 return 0;
3242 }
3243
3244 static int bpf_object__probe_btf_func_global(struct bpf_object *obj)
3245 {
3246 static const char strs[] = "\0int\0x\0a";
3247 /* static void x(int a) {} */
3248 __u32 types[] = {
3249 /* int */
3250 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
3251 /* FUNC_PROTO */ /* [2] */
3252 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0),
3253 BTF_PARAM_ENC(7, 1),
3254 /* FUNC x BTF_FUNC_GLOBAL */ /* [3] */
3255 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, BTF_FUNC_GLOBAL), 2),
3256 };
3257 int btf_fd;
3258
3259 btf_fd = libbpf__load_raw_btf((char *)types, sizeof(types),
3260 strs, sizeof(strs));
3261 if (btf_fd >= 0) {
3262 obj->caps.btf_func_global = 1;
3263 close(btf_fd);
3264 return 1;
3265 }
3266
3267 return 0;
3268 }
3269
3270 static int bpf_object__probe_btf_datasec(struct bpf_object *obj)
3271 {
3272 static const char strs[] = "\0x\0.data";
3273 /* static int a; */
3274 __u32 types[] = {
3275 /* int */
3276 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
3277 /* VAR x */ /* [2] */
3278 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1),
3279 BTF_VAR_STATIC,
3280 /* DATASEC val */ /* [3] */
3281 BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4),
3282 BTF_VAR_SECINFO_ENC(2, 0, 4),
3283 };
3284 int btf_fd;
3285
3286 btf_fd = libbpf__load_raw_btf((char *)types, sizeof(types),
3287 strs, sizeof(strs));
3288 if (btf_fd >= 0) {
3289 obj->caps.btf_datasec = 1;
3290 close(btf_fd);
3291 return 1;
3292 }
3293
3294 return 0;
3295 }
3296
3297 static int bpf_object__probe_array_mmap(struct bpf_object *obj)
3298 {
3299 struct bpf_create_map_attr attr = {
3300 .map_type = BPF_MAP_TYPE_ARRAY,
3301 .map_flags = BPF_F_MMAPABLE,
3302 .key_size = sizeof(int),
3303 .value_size = sizeof(int),
3304 .max_entries = 1,
3305 };
3306 int fd;
3307
3308 fd = bpf_create_map_xattr(&attr);
3309 if (fd >= 0) {
3310 obj->caps.array_mmap = 1;
3311 close(fd);
3312 return 1;
3313 }
3314
3315 return 0;
3316 }
3317
3318 static int
3319 bpf_object__probe_caps(struct bpf_object *obj)
3320 {
3321 int (*probe_fn[])(struct bpf_object *obj) = {
3322 bpf_object__probe_name,
3323 bpf_object__probe_global_data,
3324 bpf_object__probe_btf_func,
3325 bpf_object__probe_btf_func_global,
3326 bpf_object__probe_btf_datasec,
3327 bpf_object__probe_array_mmap,
3328 };
3329 int i, ret;
3330
3331 for (i = 0; i < ARRAY_SIZE(probe_fn); i++) {
3332 ret = probe_fn[i](obj);
3333 if (ret < 0)
3334 pr_debug("Probe #%d failed with %d.\n", i, ret);
3335 }
3336
3337 return 0;
3338 }
3339
3340 static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd)
3341 {
3342 struct bpf_map_info map_info = {};
3343 char msg[STRERR_BUFSIZE];
3344 __u32 map_info_len;
3345
3346 map_info_len = sizeof(map_info);
3347
3348 if (bpf_obj_get_info_by_fd(map_fd, &map_info, &map_info_len)) {
3349 pr_warn("failed to get map info for map FD %d: %s\n",
3350 map_fd, libbpf_strerror_r(errno, msg, sizeof(msg)));
3351 return false;
3352 }
3353
3354 return (map_info.type == map->def.type &&
3355 map_info.key_size == map->def.key_size &&
3356 map_info.value_size == map->def.value_size &&
3357 map_info.max_entries == map->def.max_entries &&
3358 map_info.map_flags == map->def.map_flags);
3359 }
3360
3361 static int
3362 bpf_object__reuse_map(struct bpf_map *map)
3363 {
3364 char *cp, errmsg[STRERR_BUFSIZE];
3365 int err, pin_fd;
3366
3367 pin_fd = bpf_obj_get(map->pin_path);
3368 if (pin_fd < 0) {
3369 err = -errno;
3370 if (err == -ENOENT) {
3371 pr_debug("found no pinned map to reuse at '%s'\n",
3372 map->pin_path);
3373 return 0;
3374 }
3375
3376 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
3377 pr_warn("couldn't retrieve pinned map '%s': %s\n",
3378 map->pin_path, cp);
3379 return err;
3380 }
3381
3382 if (!map_is_reuse_compat(map, pin_fd)) {
3383 pr_warn("couldn't reuse pinned map at '%s': parameter mismatch\n",
3384 map->pin_path);
3385 close(pin_fd);
3386 return -EINVAL;
3387 }
3388
3389 err = bpf_map__reuse_fd(map, pin_fd);
3390 if (err) {
3391 close(pin_fd);
3392 return err;
3393 }
3394 map->pinned = true;
3395 pr_debug("reused pinned map at '%s'\n", map->pin_path);
3396
3397 return 0;
3398 }
3399
3400 static int
3401 bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map)
3402 {
3403 enum libbpf_map_type map_type = map->libbpf_type;
3404 char *cp, errmsg[STRERR_BUFSIZE];
3405 int err, zero = 0;
3406
3407 /* kernel already zero-initializes .bss map. */
3408 if (map_type == LIBBPF_MAP_BSS)
3409 return 0;
3410
3411 err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0);
3412 if (err) {
3413 err = -errno;
3414 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
3415 pr_warn("Error setting initial map(%s) contents: %s\n",
3416 map->name, cp);
3417 return err;
3418 }
3419
3420 /* Freeze .rodata and .kconfig map as read-only from syscall side. */
3421 if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG) {
3422 err = bpf_map_freeze(map->fd);
3423 if (err) {
3424 err = -errno;
3425 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
3426 pr_warn("Error freezing map(%s) as read-only: %s\n",
3427 map->name, cp);
3428 return err;
3429 }
3430 }
3431 return 0;
3432 }
3433
3434 static int
3435 bpf_object__create_maps(struct bpf_object *obj)
3436 {
3437 struct bpf_create_map_attr create_attr = {};
3438 int nr_cpus = 0;
3439 unsigned int i;
3440 int err;
3441
3442 for (i = 0; i < obj->nr_maps; i++) {
3443 struct bpf_map *map = &obj->maps[i];
3444 struct bpf_map_def *def = &map->def;
3445 char *cp, errmsg[STRERR_BUFSIZE];
3446 int *pfd = &map->fd;
3447
3448 if (map->pin_path) {
3449 err = bpf_object__reuse_map(map);
3450 if (err) {
3451 pr_warn("error reusing pinned map %s\n",
3452 map->name);
3453 return err;
3454 }
3455 }
3456
3457 if (map->fd >= 0) {
3458 pr_debug("skip map create (preset) %s: fd=%d\n",
3459 map->name, map->fd);
3460 continue;
3461 }
3462
3463 if (obj->caps.name)
3464 create_attr.name = map->name;
3465 create_attr.map_ifindex = map->map_ifindex;
3466 create_attr.map_type = def->type;
3467 create_attr.map_flags = def->map_flags;
3468 create_attr.key_size = def->key_size;
3469 create_attr.value_size = def->value_size;
3470 if (def->type == BPF_MAP_TYPE_PERF_EVENT_ARRAY &&
3471 !def->max_entries) {
3472 if (!nr_cpus)
3473 nr_cpus = libbpf_num_possible_cpus();
3474 if (nr_cpus < 0) {
3475 pr_warn("failed to determine number of system CPUs: %d\n",
3476 nr_cpus);
3477 err = nr_cpus;
3478 goto err_out;
3479 }
3480 pr_debug("map '%s': setting size to %d\n",
3481 map->name, nr_cpus);
3482 create_attr.max_entries = nr_cpus;
3483 } else {
3484 create_attr.max_entries = def->max_entries;
3485 }
3486 create_attr.btf_fd = 0;
3487 create_attr.btf_key_type_id = 0;
3488 create_attr.btf_value_type_id = 0;
3489 if (bpf_map_type__is_map_in_map(def->type) &&
3490 map->inner_map_fd >= 0)
3491 create_attr.inner_map_fd = map->inner_map_fd;
3492 if (bpf_map__is_struct_ops(map))
3493 create_attr.btf_vmlinux_value_type_id =
3494 map->btf_vmlinux_value_type_id;
3495
3496 if (obj->btf && !bpf_map_find_btf_info(obj, map)) {
3497 create_attr.btf_fd = btf__fd(obj->btf);
3498 create_attr.btf_key_type_id = map->btf_key_type_id;
3499 create_attr.btf_value_type_id = map->btf_value_type_id;
3500 }
3501
3502 *pfd = bpf_create_map_xattr(&create_attr);
3503 if (*pfd < 0 && (create_attr.btf_key_type_id ||
3504 create_attr.btf_value_type_id)) {
3505 err = -errno;
3506 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
3507 pr_warn("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n",
3508 map->name, cp, err);
3509 create_attr.btf_fd = 0;
3510 create_attr.btf_key_type_id = 0;
3511 create_attr.btf_value_type_id = 0;
3512 map->btf_key_type_id = 0;
3513 map->btf_value_type_id = 0;
3514 *pfd = bpf_create_map_xattr(&create_attr);
3515 }
3516
3517 if (*pfd < 0) {
3518 size_t j;
3519
3520 err = -errno;
3521 err_out:
3522 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
3523 pr_warn("failed to create map (name: '%s'): %s(%d)\n",
3524 map->name, cp, err);
3525 pr_perm_msg(err);
3526 for (j = 0; j < i; j++)
3527 zclose(obj->maps[j].fd);
3528 return err;
3529 }
3530
3531 if (bpf_map__is_internal(map)) {
3532 err = bpf_object__populate_internal_map(obj, map);
3533 if (err < 0) {
3534 zclose(*pfd);
3535 goto err_out;
3536 }
3537 }
3538
3539 if (map->pin_path && !map->pinned) {
3540 err = bpf_map__pin(map, NULL);
3541 if (err) {
3542 pr_warn("failed to auto-pin map name '%s' at '%s'\n",
3543 map->name, map->pin_path);
3544 return err;
3545 }
3546 }
3547
3548 pr_debug("created map %s: fd=%d\n", map->name, *pfd);
3549 }
3550
3551 return 0;
3552 }
3553
3554 static int
3555 check_btf_ext_reloc_err(struct bpf_program *prog, int err,
3556 void *btf_prog_info, const char *info_name)
3557 {
3558 if (err != -ENOENT) {
3559 pr_warn("Error in loading %s for sec %s.\n",
3560 info_name, prog->section_name);
3561 return err;
3562 }
3563
3564 /* err == -ENOENT (i.e. prog->section_name not found in btf_ext) */
3565
3566 if (btf_prog_info) {
3567 /*
3568 * Some info has already been found but has problem
3569 * in the last btf_ext reloc. Must have to error out.
3570 */
3571 pr_warn("Error in relocating %s for sec %s.\n",
3572 info_name, prog->section_name);
3573 return err;
3574 }
3575
3576 /* Have problem loading the very first info. Ignore the rest. */
3577 pr_warn("Cannot find %s for main program sec %s. Ignore all %s.\n",
3578 info_name, prog->section_name, info_name);
3579 return 0;
3580 }
3581
3582 static int
3583 bpf_program_reloc_btf_ext(struct bpf_program *prog, struct bpf_object *obj,
3584 const char *section_name, __u32 insn_offset)
3585 {
3586 int err;
3587
3588 if (!insn_offset || prog->func_info) {
3589 /*
3590 * !insn_offset => main program
3591 *
3592 * For sub prog, the main program's func_info has to
3593 * be loaded first (i.e. prog->func_info != NULL)
3594 */
3595 err = btf_ext__reloc_func_info(obj->btf, obj->btf_ext,
3596 section_name, insn_offset,
3597 &prog->func_info,
3598 &prog->func_info_cnt);
3599 if (err)
3600 return check_btf_ext_reloc_err(prog, err,
3601 prog->func_info,
3602 "bpf_func_info");
3603
3604 prog->func_info_rec_size = btf_ext__func_info_rec_size(obj->btf_ext);
3605 }
3606
3607 if (!insn_offset || prog->line_info) {
3608 err = btf_ext__reloc_line_info(obj->btf, obj->btf_ext,
3609 section_name, insn_offset,
3610 &prog->line_info,
3611 &prog->line_info_cnt);
3612 if (err)
3613 return check_btf_ext_reloc_err(prog, err,
3614 prog->line_info,
3615 "bpf_line_info");
3616
3617 prog->line_info_rec_size = btf_ext__line_info_rec_size(obj->btf_ext);
3618 }
3619
3620 return 0;
3621 }
3622
3623 #define BPF_CORE_SPEC_MAX_LEN 64
3624
3625 /* represents BPF CO-RE field or array element accessor */
3626 struct bpf_core_accessor {
3627 __u32 type_id; /* struct/union type or array element type */
3628 __u32 idx; /* field index or array index */
3629 const char *name; /* field name or NULL for array accessor */
3630 };
3631
3632 struct bpf_core_spec {
3633 const struct btf *btf;
3634 /* high-level spec: named fields and array indices only */
3635 struct bpf_core_accessor spec[BPF_CORE_SPEC_MAX_LEN];
3636 /* high-level spec length */
3637 int len;
3638 /* raw, low-level spec: 1-to-1 with accessor spec string */
3639 int raw_spec[BPF_CORE_SPEC_MAX_LEN];
3640 /* raw spec length */
3641 int raw_len;
3642 /* field bit offset represented by spec */
3643 __u32 bit_offset;
3644 };
3645
3646 static bool str_is_empty(const char *s)
3647 {
3648 return !s || !s[0];
3649 }
3650
3651 static bool is_flex_arr(const struct btf *btf,
3652 const struct bpf_core_accessor *acc,
3653 const struct btf_array *arr)
3654 {
3655 const struct btf_type *t;
3656
3657 /* not a flexible array, if not inside a struct or has non-zero size */
3658 if (!acc->name || arr->nelems > 0)
3659 return false;
3660
3661 /* has to be the last member of enclosing struct */
3662 t = btf__type_by_id(btf, acc->type_id);
3663 return acc->idx == btf_vlen(t) - 1;
3664 }
3665
3666 /*
3667 * Turn bpf_field_reloc into a low- and high-level spec representation,
3668 * validating correctness along the way, as well as calculating resulting
3669 * field bit offset, specified by accessor string. Low-level spec captures
3670 * every single level of nestedness, including traversing anonymous
3671 * struct/union members. High-level one only captures semantically meaningful
3672 * "turning points": named fields and array indicies.
3673 * E.g., for this case:
3674 *
3675 * struct sample {
3676 * int __unimportant;
3677 * struct {
3678 * int __1;
3679 * int __2;
3680 * int a[7];
3681 * };
3682 * };
3683 *
3684 * struct sample *s = ...;
3685 *
3686 * int x = &s->a[3]; // access string = '0:1:2:3'
3687 *
3688 * Low-level spec has 1:1 mapping with each element of access string (it's
3689 * just a parsed access string representation): [0, 1, 2, 3].
3690 *
3691 * High-level spec will capture only 3 points:
3692 * - intial zero-index access by pointer (&s->... is the same as &s[0]...);
3693 * - field 'a' access (corresponds to '2' in low-level spec);
3694 * - array element #3 access (corresponds to '3' in low-level spec).
3695 *
3696 */
3697 static int bpf_core_spec_parse(const struct btf *btf,
3698 __u32 type_id,
3699 const char *spec_str,
3700 struct bpf_core_spec *spec)
3701 {
3702 int access_idx, parsed_len, i;
3703 struct bpf_core_accessor *acc;
3704 const struct btf_type *t;
3705 const char *name;
3706 __u32 id;
3707 __s64 sz;
3708
3709 if (str_is_empty(spec_str) || *spec_str == ':')
3710 return -EINVAL;
3711
3712 memset(spec, 0, sizeof(*spec));
3713 spec->btf = btf;
3714
3715 /* parse spec_str="0:1:2:3:4" into array raw_spec=[0, 1, 2, 3, 4] */
3716 while (*spec_str) {
3717 if (*spec_str == ':')
3718 ++spec_str;
3719 if (sscanf(spec_str, "%d%n", &access_idx, &parsed_len) != 1)
3720 return -EINVAL;
3721 if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN)
3722 return -E2BIG;
3723 spec_str += parsed_len;
3724 spec->raw_spec[spec->raw_len++] = access_idx;
3725 }
3726
3727 if (spec->raw_len == 0)
3728 return -EINVAL;
3729
3730 /* first spec value is always reloc type array index */
3731 t = skip_mods_and_typedefs(btf, type_id, &id);
3732 if (!t)
3733 return -EINVAL;
3734
3735 access_idx = spec->raw_spec[0];
3736 spec->spec[0].type_id = id;
3737 spec->spec[0].idx = access_idx;
3738 spec->len++;
3739
3740 sz = btf__resolve_size(btf, id);
3741 if (sz < 0)
3742 return sz;
3743 spec->bit_offset = access_idx * sz * 8;
3744
3745 for (i = 1; i < spec->raw_len; i++) {
3746 t = skip_mods_and_typedefs(btf, id, &id);
3747 if (!t)
3748 return -EINVAL;
3749
3750 access_idx = spec->raw_spec[i];
3751 acc = &spec->spec[spec->len];
3752
3753 if (btf_is_composite(t)) {
3754 const struct btf_member *m;
3755 __u32 bit_offset;
3756
3757 if (access_idx >= btf_vlen(t))
3758 return -EINVAL;
3759
3760 bit_offset = btf_member_bit_offset(t, access_idx);
3761 spec->bit_offset += bit_offset;
3762
3763 m = btf_members(t) + access_idx;
3764 if (m->name_off) {
3765 name = btf__name_by_offset(btf, m->name_off);
3766 if (str_is_empty(name))
3767 return -EINVAL;
3768
3769 acc->type_id = id;
3770 acc->idx = access_idx;
3771 acc->name = name;
3772 spec->len++;
3773 }
3774
3775 id = m->type;
3776 } else if (btf_is_array(t)) {
3777 const struct btf_array *a = btf_array(t);
3778 bool flex;
3779
3780 t = skip_mods_and_typedefs(btf, a->type, &id);
3781 if (!t)
3782 return -EINVAL;
3783
3784 flex = is_flex_arr(btf, acc - 1, a);
3785 if (!flex && access_idx >= a->nelems)
3786 return -EINVAL;
3787
3788 spec->spec[spec->len].type_id = id;
3789 spec->spec[spec->len].idx = access_idx;
3790 spec->len++;
3791
3792 sz = btf__resolve_size(btf, id);
3793 if (sz < 0)
3794 return sz;
3795 spec->bit_offset += access_idx * sz * 8;
3796 } else {
3797 pr_warn("relo for [%u] %s (at idx %d) captures type [%d] of unexpected kind %d\n",
3798 type_id, spec_str, i, id, btf_kind(t));
3799 return -EINVAL;
3800 }
3801 }
3802
3803 return 0;
3804 }
3805
3806 static bool bpf_core_is_flavor_sep(const char *s)
3807 {
3808 /* check X___Y name pattern, where X and Y are not underscores */
3809 return s[0] != '_' && /* X */
3810 s[1] == '_' && s[2] == '_' && s[3] == '_' && /* ___ */
3811 s[4] != '_'; /* Y */
3812 }
3813
3814 /* Given 'some_struct_name___with_flavor' return the length of a name prefix
3815 * before last triple underscore. Struct name part after last triple
3816 * underscore is ignored by BPF CO-RE relocation during relocation matching.
3817 */
3818 static size_t bpf_core_essential_name_len(const char *name)
3819 {
3820 size_t n = strlen(name);
3821 int i;
3822
3823 for (i = n - 5; i >= 0; i--) {
3824 if (bpf_core_is_flavor_sep(name + i))
3825 return i + 1;
3826 }
3827 return n;
3828 }
3829
3830 /* dynamically sized list of type IDs */
3831 struct ids_vec {
3832 __u32 *data;
3833 int len;
3834 };
3835
3836 static void bpf_core_free_cands(struct ids_vec *cand_ids)
3837 {
3838 free(cand_ids->data);
3839 free(cand_ids);
3840 }
3841
3842 static struct ids_vec *bpf_core_find_cands(const struct btf *local_btf,
3843 __u32 local_type_id,
3844 const struct btf *targ_btf)
3845 {
3846 size_t local_essent_len, targ_essent_len;
3847 const char *local_name, *targ_name;
3848 const struct btf_type *t;
3849 struct ids_vec *cand_ids;
3850 __u32 *new_ids;
3851 int i, err, n;
3852
3853 t = btf__type_by_id(local_btf, local_type_id);
3854 if (!t)
3855 return ERR_PTR(-EINVAL);
3856
3857 local_name = btf__name_by_offset(local_btf, t->name_off);
3858 if (str_is_empty(local_name))
3859 return ERR_PTR(-EINVAL);
3860 local_essent_len = bpf_core_essential_name_len(local_name);
3861
3862 cand_ids = calloc(1, sizeof(*cand_ids));
3863 if (!cand_ids)
3864 return ERR_PTR(-ENOMEM);
3865
3866 n = btf__get_nr_types(targ_btf);
3867 for (i = 1; i <= n; i++) {
3868 t = btf__type_by_id(targ_btf, i);
3869 targ_name = btf__name_by_offset(targ_btf, t->name_off);
3870 if (str_is_empty(targ_name))
3871 continue;
3872
3873 t = skip_mods_and_typedefs(targ_btf, i, NULL);
3874 if (!btf_is_composite(t) && !btf_is_array(t))
3875 continue;
3876
3877 targ_essent_len = bpf_core_essential_name_len(targ_name);
3878 if (targ_essent_len != local_essent_len)
3879 continue;
3880
3881 if (strncmp(local_name, targ_name, local_essent_len) == 0) {
3882 pr_debug("[%d] %s: found candidate [%d] %s\n",
3883 local_type_id, local_name, i, targ_name);
3884 new_ids = reallocarray(cand_ids->data,
3885 cand_ids->len + 1,
3886 sizeof(*cand_ids->data));
3887 if (!new_ids) {
3888 err = -ENOMEM;
3889 goto err_out;
3890 }
3891 cand_ids->data = new_ids;
3892 cand_ids->data[cand_ids->len++] = i;
3893 }
3894 }
3895 return cand_ids;
3896 err_out:
3897 bpf_core_free_cands(cand_ids);
3898 return ERR_PTR(err);
3899 }
3900
3901 /* Check two types for compatibility, skipping const/volatile/restrict and
3902 * typedefs, to ensure we are relocating compatible entities:
3903 * - any two STRUCTs/UNIONs are compatible and can be mixed;
3904 * - any two FWDs are compatible, if their names match (modulo flavor suffix);
3905 * - any two PTRs are always compatible;
3906 * - for ENUMs, names should be the same (ignoring flavor suffix) or at
3907 * least one of enums should be anonymous;
3908 * - for ENUMs, check sizes, names are ignored;
3909 * - for INT, size and signedness are ignored;
3910 * - for ARRAY, dimensionality is ignored, element types are checked for
3911 * compatibility recursively;
3912 * - everything else shouldn't be ever a target of relocation.
3913 * These rules are not set in stone and probably will be adjusted as we get
3914 * more experience with using BPF CO-RE relocations.
3915 */
3916 static int bpf_core_fields_are_compat(const struct btf *local_btf,
3917 __u32 local_id,
3918 const struct btf *targ_btf,
3919 __u32 targ_id)
3920 {
3921 const struct btf_type *local_type, *targ_type;
3922
3923 recur:
3924 local_type = skip_mods_and_typedefs(local_btf, local_id, &local_id);
3925 targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id);
3926 if (!local_type || !targ_type)
3927 return -EINVAL;
3928
3929 if (btf_is_composite(local_type) && btf_is_composite(targ_type))
3930 return 1;
3931 if (btf_kind(local_type) != btf_kind(targ_type))
3932 return 0;
3933
3934 switch (btf_kind(local_type)) {
3935 case BTF_KIND_PTR:
3936 return 1;
3937 case BTF_KIND_FWD:
3938 case BTF_KIND_ENUM: {
3939 const char *local_name, *targ_name;
3940 size_t local_len, targ_len;
3941
3942 local_name = btf__name_by_offset(local_btf,
3943 local_type->name_off);
3944 targ_name = btf__name_by_offset(targ_btf, targ_type->name_off);
3945 local_len = bpf_core_essential_name_len(local_name);
3946 targ_len = bpf_core_essential_name_len(targ_name);
3947 /* one of them is anonymous or both w/ same flavor-less names */
3948 return local_len == 0 || targ_len == 0 ||
3949 (local_len == targ_len &&
3950 strncmp(local_name, targ_name, local_len) == 0);
3951 }
3952 case BTF_KIND_INT:
3953 /* just reject deprecated bitfield-like integers; all other
3954 * integers are by default compatible between each other
3955 */
3956 return btf_int_offset(local_type) == 0 &&
3957 btf_int_offset(targ_type) == 0;
3958 case BTF_KIND_ARRAY:
3959 local_id = btf_array(local_type)->type;
3960 targ_id = btf_array(targ_type)->type;
3961 goto recur;
3962 default:
3963 pr_warn("unexpected kind %d relocated, local [%d], target [%d]\n",
3964 btf_kind(local_type), local_id, targ_id);
3965 return 0;
3966 }
3967 }
3968
3969 /*
3970 * Given single high-level named field accessor in local type, find
3971 * corresponding high-level accessor for a target type. Along the way,
3972 * maintain low-level spec for target as well. Also keep updating target
3973 * bit offset.
3974 *
3975 * Searching is performed through recursive exhaustive enumeration of all
3976 * fields of a struct/union. If there are any anonymous (embedded)
3977 * structs/unions, they are recursively searched as well. If field with
3978 * desired name is found, check compatibility between local and target types,
3979 * before returning result.
3980 *
3981 * 1 is returned, if field is found.
3982 * 0 is returned if no compatible field is found.
3983 * <0 is returned on error.
3984 */
3985 static int bpf_core_match_member(const struct btf *local_btf,
3986 const struct bpf_core_accessor *local_acc,
3987 const struct btf *targ_btf,
3988 __u32 targ_id,
3989 struct bpf_core_spec *spec,
3990 __u32 *next_targ_id)
3991 {
3992 const struct btf_type *local_type, *targ_type;
3993 const struct btf_member *local_member, *m;
3994 const char *local_name, *targ_name;
3995 __u32 local_id;
3996 int i, n, found;
3997
3998 targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id);
3999 if (!targ_type)
4000 return -EINVAL;
4001 if (!btf_is_composite(targ_type))
4002 return 0;
4003
4004 local_id = local_acc->type_id;
4005 local_type = btf__type_by_id(local_btf, local_id);
4006 local_member = btf_members(local_type) + local_acc->idx;
4007 local_name = btf__name_by_offset(local_btf, local_member->name_off);
4008
4009 n = btf_vlen(targ_type);
4010 m = btf_members(targ_type);
4011 for (i = 0; i < n; i++, m++) {
4012 __u32 bit_offset;
4013
4014 bit_offset = btf_member_bit_offset(targ_type, i);
4015
4016 /* too deep struct/union/array nesting */
4017 if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN)
4018 return -E2BIG;
4019
4020 /* speculate this member will be the good one */
4021 spec->bit_offset += bit_offset;
4022 spec->raw_spec[spec->raw_len++] = i;
4023
4024 targ_name = btf__name_by_offset(targ_btf, m->name_off);
4025 if (str_is_empty(targ_name)) {
4026 /* embedded struct/union, we need to go deeper */
4027 found = bpf_core_match_member(local_btf, local_acc,
4028 targ_btf, m->type,
4029 spec, next_targ_id);
4030 if (found) /* either found or error */
4031 return found;
4032 } else if (strcmp(local_name, targ_name) == 0) {
4033 /* matching named field */
4034 struct bpf_core_accessor *targ_acc;
4035
4036 targ_acc = &spec->spec[spec->len++];
4037 targ_acc->type_id = targ_id;
4038 targ_acc->idx = i;
4039 targ_acc->name = targ_name;
4040
4041 *next_targ_id = m->type;
4042 found = bpf_core_fields_are_compat(local_btf,
4043 local_member->type,
4044 targ_btf, m->type);
4045 if (!found)
4046 spec->len--; /* pop accessor */
4047 return found;
4048 }
4049 /* member turned out not to be what we looked for */
4050 spec->bit_offset -= bit_offset;
4051 spec->raw_len--;
4052 }
4053
4054 return 0;
4055 }
4056
4057 /*
4058 * Try to match local spec to a target type and, if successful, produce full
4059 * target spec (high-level, low-level + bit offset).
4060 */
4061 static int bpf_core_spec_match(struct bpf_core_spec *local_spec,
4062 const struct btf *targ_btf, __u32 targ_id,
4063 struct bpf_core_spec *targ_spec)
4064 {
4065 const struct btf_type *targ_type;
4066 const struct bpf_core_accessor *local_acc;
4067 struct bpf_core_accessor *targ_acc;
4068 int i, sz, matched;
4069
4070 memset(targ_spec, 0, sizeof(*targ_spec));
4071 targ_spec->btf = targ_btf;
4072
4073 local_acc = &local_spec->spec[0];
4074 targ_acc = &targ_spec->spec[0];
4075
4076 for (i = 0; i < local_spec->len; i++, local_acc++, targ_acc++) {
4077 targ_type = skip_mods_and_typedefs(targ_spec->btf, targ_id,
4078 &targ_id);
4079 if (!targ_type)
4080 return -EINVAL;
4081
4082 if (local_acc->name) {
4083 matched = bpf_core_match_member(local_spec->btf,
4084 local_acc,
4085 targ_btf, targ_id,
4086 targ_spec, &targ_id);
4087 if (matched <= 0)
4088 return matched;
4089 } else {
4090 /* for i=0, targ_id is already treated as array element
4091 * type (because it's the original struct), for others
4092 * we should find array element type first
4093 */
4094 if (i > 0) {
4095 const struct btf_array *a;
4096 bool flex;
4097
4098 if (!btf_is_array(targ_type))
4099 return 0;
4100
4101 a = btf_array(targ_type);
4102 flex = is_flex_arr(targ_btf, targ_acc - 1, a);
4103 if (!flex && local_acc->idx >= a->nelems)
4104 return 0;
4105 if (!skip_mods_and_typedefs(targ_btf, a->type,
4106 &targ_id))
4107 return -EINVAL;
4108 }
4109
4110 /* too deep struct/union/array nesting */
4111 if (targ_spec->raw_len == BPF_CORE_SPEC_MAX_LEN)
4112 return -E2BIG;
4113
4114 targ_acc->type_id = targ_id;
4115 targ_acc->idx = local_acc->idx;
4116 targ_acc->name = NULL;
4117 targ_spec->len++;
4118 targ_spec->raw_spec[targ_spec->raw_len] = targ_acc->idx;
4119 targ_spec->raw_len++;
4120
4121 sz = btf__resolve_size(targ_btf, targ_id);
4122 if (sz < 0)
4123 return sz;
4124 targ_spec->bit_offset += local_acc->idx * sz * 8;
4125 }
4126 }
4127
4128 return 1;
4129 }
4130
4131 static int bpf_core_calc_field_relo(const struct bpf_program *prog,
4132 const struct bpf_field_reloc *relo,
4133 const struct bpf_core_spec *spec,
4134 __u32 *val, bool *validate)
4135 {
4136 const struct bpf_core_accessor *acc = &spec->spec[spec->len - 1];
4137 const struct btf_type *t = btf__type_by_id(spec->btf, acc->type_id);
4138 __u32 byte_off, byte_sz, bit_off, bit_sz;
4139 const struct btf_member *m;
4140 const struct btf_type *mt;
4141 bool bitfield;
4142 __s64 sz;
4143
4144 /* a[n] accessor needs special handling */
4145 if (!acc->name) {
4146 if (relo->kind == BPF_FIELD_BYTE_OFFSET) {
4147 *val = spec->bit_offset / 8;
4148 } else if (relo->kind == BPF_FIELD_BYTE_SIZE) {
4149 sz = btf__resolve_size(spec->btf, acc->type_id);
4150 if (sz < 0)
4151 return -EINVAL;
4152 *val = sz;
4153 } else {
4154 pr_warn("prog '%s': relo %d at insn #%d can't be applied to array access\n",
4155 bpf_program__title(prog, false),
4156 relo->kind, relo->insn_off / 8);
4157 return -EINVAL;
4158 }
4159 if (validate)
4160 *validate = true;
4161 return 0;
4162 }
4163
4164 m = btf_members(t) + acc->idx;
4165 mt = skip_mods_and_typedefs(spec->btf, m->type, NULL);
4166 bit_off = spec->bit_offset;
4167 bit_sz = btf_member_bitfield_size(t, acc->idx);
4168
4169 bitfield = bit_sz > 0;
4170 if (bitfield) {
4171 byte_sz = mt->size;
4172 byte_off = bit_off / 8 / byte_sz * byte_sz;
4173 /* figure out smallest int size necessary for bitfield load */
4174 while (bit_off + bit_sz - byte_off * 8 > byte_sz * 8) {
4175 if (byte_sz >= 8) {
4176 /* bitfield can't be read with 64-bit read */
4177 pr_warn("prog '%s': relo %d at insn #%d can't be satisfied for bitfield\n",
4178 bpf_program__title(prog, false),
4179 relo->kind, relo->insn_off / 8);
4180 return -E2BIG;
4181 }
4182 byte_sz *= 2;
4183 byte_off = bit_off / 8 / byte_sz * byte_sz;
4184 }
4185 } else {
4186 sz = btf__resolve_size(spec->btf, m->type);
4187 if (sz < 0)
4188 return -EINVAL;
4189 byte_sz = sz;
4190 byte_off = spec->bit_offset / 8;
4191 bit_sz = byte_sz * 8;
4192 }
4193
4194 /* for bitfields, all the relocatable aspects are ambiguous and we
4195 * might disagree with compiler, so turn off validation of expected
4196 * value, except for signedness
4197 */
4198 if (validate)
4199 *validate = !bitfield;
4200
4201 switch (relo->kind) {
4202 case BPF_FIELD_BYTE_OFFSET:
4203 *val = byte_off;
4204 break;
4205 case BPF_FIELD_BYTE_SIZE:
4206 *val = byte_sz;
4207 break;
4208 case BPF_FIELD_SIGNED:
4209 /* enums will be assumed unsigned */
4210 *val = btf_is_enum(mt) ||
4211 (btf_int_encoding(mt) & BTF_INT_SIGNED);
4212 if (validate)
4213 *validate = true; /* signedness is never ambiguous */
4214 break;
4215 case BPF_FIELD_LSHIFT_U64:
4216 #if __BYTE_ORDER == __LITTLE_ENDIAN
4217 *val = 64 - (bit_off + bit_sz - byte_off * 8);
4218 #else
4219 *val = (8 - byte_sz) * 8 + (bit_off - byte_off * 8);
4220 #endif
4221 break;
4222 case BPF_FIELD_RSHIFT_U64:
4223 *val = 64 - bit_sz;
4224 if (validate)
4225 *validate = true; /* right shift is never ambiguous */
4226 break;
4227 case BPF_FIELD_EXISTS:
4228 default:
4229 pr_warn("prog '%s': unknown relo %d at insn #%d\n",
4230 bpf_program__title(prog, false),
4231 relo->kind, relo->insn_off / 8);
4232 return -EINVAL;
4233 }
4234
4235 return 0;
4236 }
4237
4238 /*
4239 * Patch relocatable BPF instruction.
4240 *
4241 * Patched value is determined by relocation kind and target specification.
4242 * For field existence relocation target spec will be NULL if field is not
4243 * found.
4244 * Expected insn->imm value is determined using relocation kind and local
4245 * spec, and is checked before patching instruction. If actual insn->imm value
4246 * is wrong, bail out with error.
4247 *
4248 * Currently three kinds of BPF instructions are supported:
4249 * 1. rX = <imm> (assignment with immediate operand);
4250 * 2. rX += <imm> (arithmetic operations with immediate operand);
4251 */
4252 static int bpf_core_reloc_insn(struct bpf_program *prog,
4253 const struct bpf_field_reloc *relo,
4254 int relo_idx,
4255 const struct bpf_core_spec *local_spec,
4256 const struct bpf_core_spec *targ_spec)
4257 {
4258 __u32 orig_val, new_val;
4259 struct bpf_insn *insn;
4260 bool validate = true;
4261 int insn_idx, err;
4262 __u8 class;
4263
4264 if (relo->insn_off % sizeof(struct bpf_insn))
4265 return -EINVAL;
4266 insn_idx = relo->insn_off / sizeof(struct bpf_insn);
4267 insn = &prog->insns[insn_idx];
4268 class = BPF_CLASS(insn->code);
4269
4270 if (relo->kind == BPF_FIELD_EXISTS) {
4271 orig_val = 1; /* can't generate EXISTS relo w/o local field */
4272 new_val = targ_spec ? 1 : 0;
4273 } else if (!targ_spec) {
4274 pr_debug("prog '%s': relo #%d: substituting insn #%d w/ invalid insn\n",
4275 bpf_program__title(prog, false), relo_idx, insn_idx);
4276 insn->code = BPF_JMP | BPF_CALL;
4277 insn->dst_reg = 0;
4278 insn->src_reg = 0;
4279 insn->off = 0;
4280 /* if this instruction is reachable (not a dead code),
4281 * verifier will complain with the following message:
4282 * invalid func unknown#195896080
4283 */
4284 insn->imm = 195896080; /* => 0xbad2310 => "bad relo" */
4285 return 0;
4286 } else {
4287 err = bpf_core_calc_field_relo(prog, relo, local_spec,
4288 &orig_val, &validate);
4289 if (err)
4290 return err;
4291 err = bpf_core_calc_field_relo(prog, relo, targ_spec,
4292 &new_val, NULL);
4293 if (err)
4294 return err;
4295 }
4296
4297 switch (class) {
4298 case BPF_ALU:
4299 case BPF_ALU64:
4300 if (BPF_SRC(insn->code) != BPF_K)
4301 return -EINVAL;
4302 if (validate && insn->imm != orig_val) {
4303 pr_warn("prog '%s': relo #%d: unexpected insn #%d (ALU/ALU64) value: got %u, exp %u -> %u\n",
4304 bpf_program__title(prog, false), relo_idx,
4305 insn_idx, insn->imm, orig_val, new_val);
4306 return -EINVAL;
4307 }
4308 orig_val = insn->imm;
4309 insn->imm = new_val;
4310 pr_debug("prog '%s': relo #%d: patched insn #%d (ALU/ALU64) imm %u -> %u\n",
4311 bpf_program__title(prog, false), relo_idx, insn_idx,
4312 orig_val, new_val);
4313 break;
4314 case BPF_LDX:
4315 case BPF_ST:
4316 case BPF_STX:
4317 if (validate && insn->off != orig_val) {
4318 pr_warn("prog '%s': relo #%d: unexpected insn #%d (LD/LDX/ST/STX) value: got %u, exp %u -> %u\n",
4319 bpf_program__title(prog, false), relo_idx,
4320 insn_idx, insn->off, orig_val, new_val);
4321 return -EINVAL;
4322 }
4323 if (new_val > SHRT_MAX) {
4324 pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) value too big: %u\n",
4325 bpf_program__title(prog, false), relo_idx,
4326 insn_idx, new_val);
4327 return -ERANGE;
4328 }
4329 orig_val = insn->off;
4330 insn->off = new_val;
4331 pr_debug("prog '%s': relo #%d: patched insn #%d (LDX/ST/STX) off %u -> %u\n",
4332 bpf_program__title(prog, false), relo_idx, insn_idx,
4333 orig_val, new_val);
4334 break;
4335 default:
4336 pr_warn("prog '%s': relo #%d: trying to relocate unrecognized insn #%d, code:%x, src:%x, dst:%x, off:%x, imm:%x\n",
4337 bpf_program__title(prog, false), relo_idx,
4338 insn_idx, insn->code, insn->src_reg, insn->dst_reg,
4339 insn->off, insn->imm);
4340 return -EINVAL;
4341 }
4342
4343 return 0;
4344 }
4345
4346 /* Output spec definition in the format:
4347 * [<type-id>] (<type-name>) + <raw-spec> => <offset>@<spec>,
4348 * where <spec> is a C-syntax view of recorded field access, e.g.: x.a[3].b
4349 */
4350 static void bpf_core_dump_spec(int level, const struct bpf_core_spec *spec)
4351 {
4352 const struct btf_type *t;
4353 const char *s;
4354 __u32 type_id;
4355 int i;
4356
4357 type_id = spec->spec[0].type_id;
4358 t = btf__type_by_id(spec->btf, type_id);
4359 s = btf__name_by_offset(spec->btf, t->name_off);
4360 libbpf_print(level, "[%u] %s + ", type_id, s);
4361
4362 for (i = 0; i < spec->raw_len; i++)
4363 libbpf_print(level, "%d%s", spec->raw_spec[i],
4364 i == spec->raw_len - 1 ? " => " : ":");
4365
4366 libbpf_print(level, "%u.%u @ &x",
4367 spec->bit_offset / 8, spec->bit_offset % 8);
4368
4369 for (i = 0; i < spec->len; i++) {
4370 if (spec->spec[i].name)
4371 libbpf_print(level, ".%s", spec->spec[i].name);
4372 else
4373 libbpf_print(level, "[%u]", spec->spec[i].idx);
4374 }
4375
4376 }
4377
4378 static size_t bpf_core_hash_fn(const void *key, void *ctx)
4379 {
4380 return (size_t)key;
4381 }
4382
4383 static bool bpf_core_equal_fn(const void *k1, const void *k2, void *ctx)
4384 {
4385 return k1 == k2;
4386 }
4387
4388 static void *u32_as_hash_key(__u32 x)
4389 {
4390 return (void *)(uintptr_t)x;
4391 }
4392
4393 /*
4394 * CO-RE relocate single instruction.
4395 *
4396 * The outline and important points of the algorithm:
4397 * 1. For given local type, find corresponding candidate target types.
4398 * Candidate type is a type with the same "essential" name, ignoring
4399 * everything after last triple underscore (___). E.g., `sample`,
4400 * `sample___flavor_one`, `sample___flavor_another_one`, are all candidates
4401 * for each other. Names with triple underscore are referred to as
4402 * "flavors" and are useful, among other things, to allow to
4403 * specify/support incompatible variations of the same kernel struct, which
4404 * might differ between different kernel versions and/or build
4405 * configurations.
4406 *
4407 * N.B. Struct "flavors" could be generated by bpftool's BTF-to-C
4408 * converter, when deduplicated BTF of a kernel still contains more than
4409 * one different types with the same name. In that case, ___2, ___3, etc
4410 * are appended starting from second name conflict. But start flavors are
4411 * also useful to be defined "locally", in BPF program, to extract same
4412 * data from incompatible changes between different kernel
4413 * versions/configurations. For instance, to handle field renames between
4414 * kernel versions, one can use two flavors of the struct name with the
4415 * same common name and use conditional relocations to extract that field,
4416 * depending on target kernel version.
4417 * 2. For each candidate type, try to match local specification to this
4418 * candidate target type. Matching involves finding corresponding
4419 * high-level spec accessors, meaning that all named fields should match,
4420 * as well as all array accesses should be within the actual bounds. Also,
4421 * types should be compatible (see bpf_core_fields_are_compat for details).
4422 * 3. It is supported and expected that there might be multiple flavors
4423 * matching the spec. As long as all the specs resolve to the same set of
4424 * offsets across all candidates, there is no error. If there is any
4425 * ambiguity, CO-RE relocation will fail. This is necessary to accomodate
4426 * imprefection of BTF deduplication, which can cause slight duplication of
4427 * the same BTF type, if some directly or indirectly referenced (by
4428 * pointer) type gets resolved to different actual types in different
4429 * object files. If such situation occurs, deduplicated BTF will end up
4430 * with two (or more) structurally identical types, which differ only in
4431 * types they refer to through pointer. This should be OK in most cases and
4432 * is not an error.
4433 * 4. Candidate types search is performed by linearly scanning through all
4434 * types in target BTF. It is anticipated that this is overall more
4435 * efficient memory-wise and not significantly worse (if not better)
4436 * CPU-wise compared to prebuilding a map from all local type names to
4437 * a list of candidate type names. It's also sped up by caching resolved
4438 * list of matching candidates per each local "root" type ID, that has at
4439 * least one bpf_field_reloc associated with it. This list is shared
4440 * between multiple relocations for the same type ID and is updated as some
4441 * of the candidates are pruned due to structural incompatibility.
4442 */
4443 static int bpf_core_reloc_field(struct bpf_program *prog,
4444 const struct bpf_field_reloc *relo,
4445 int relo_idx,
4446 const struct btf *local_btf,
4447 const struct btf *targ_btf,
4448 struct hashmap *cand_cache)
4449 {
4450 const char *prog_name = bpf_program__title(prog, false);
4451 struct bpf_core_spec local_spec, cand_spec, targ_spec;
4452 const void *type_key = u32_as_hash_key(relo->type_id);
4453 const struct btf_type *local_type, *cand_type;
4454 const char *local_name, *cand_name;
4455 struct ids_vec *cand_ids;
4456 __u32 local_id, cand_id;
4457 const char *spec_str;
4458 int i, j, err;
4459
4460 local_id = relo->type_id;
4461 local_type = btf__type_by_id(local_btf, local_id);
4462 if (!local_type)
4463 return -EINVAL;
4464
4465 local_name = btf__name_by_offset(local_btf, local_type->name_off);
4466 if (str_is_empty(local_name))
4467 return -EINVAL;
4468
4469 spec_str = btf__name_by_offset(local_btf, relo->access_str_off);
4470 if (str_is_empty(spec_str))
4471 return -EINVAL;
4472
4473 err = bpf_core_spec_parse(local_btf, local_id, spec_str, &local_spec);
4474 if (err) {
4475 pr_warn("prog '%s': relo #%d: parsing [%d] %s + %s failed: %d\n",
4476 prog_name, relo_idx, local_id, local_name, spec_str,
4477 err);
4478 return -EINVAL;
4479 }
4480
4481 pr_debug("prog '%s': relo #%d: kind %d, spec is ", prog_name, relo_idx,
4482 relo->kind);
4483 bpf_core_dump_spec(LIBBPF_DEBUG, &local_spec);
4484 libbpf_print(LIBBPF_DEBUG, "\n");
4485
4486 if (!hashmap__find(cand_cache, type_key, (void **)&cand_ids)) {
4487 cand_ids = bpf_core_find_cands(local_btf, local_id, targ_btf);
4488 if (IS_ERR(cand_ids)) {
4489 pr_warn("prog '%s': relo #%d: target candidate search failed for [%d] %s: %ld",
4490 prog_name, relo_idx, local_id, local_name,
4491 PTR_ERR(cand_ids));
4492 return PTR_ERR(cand_ids);
4493 }
4494 err = hashmap__set(cand_cache, type_key, cand_ids, NULL, NULL);
4495 if (err) {
4496 bpf_core_free_cands(cand_ids);
4497 return err;
4498 }
4499 }
4500
4501 for (i = 0, j = 0; i < cand_ids->len; i++) {
4502 cand_id = cand_ids->data[i];
4503 cand_type = btf__type_by_id(targ_btf, cand_id);
4504 cand_name = btf__name_by_offset(targ_btf, cand_type->name_off);
4505
4506 err = bpf_core_spec_match(&local_spec, targ_btf,
4507 cand_id, &cand_spec);
4508 pr_debug("prog '%s': relo #%d: matching candidate #%d %s against spec ",
4509 prog_name, relo_idx, i, cand_name);
4510 bpf_core_dump_spec(LIBBPF_DEBUG, &cand_spec);
4511 libbpf_print(LIBBPF_DEBUG, ": %d\n", err);
4512 if (err < 0) {
4513 pr_warn("prog '%s': relo #%d: matching error: %d\n",
4514 prog_name, relo_idx, err);
4515 return err;
4516 }
4517 if (err == 0)
4518 continue;
4519
4520 if (j == 0) {
4521 targ_spec = cand_spec;
4522 } else if (cand_spec.bit_offset != targ_spec.bit_offset) {
4523 /* if there are many candidates, they should all
4524 * resolve to the same bit offset
4525 */
4526 pr_warn("prog '%s': relo #%d: offset ambiguity: %u != %u\n",
4527 prog_name, relo_idx, cand_spec.bit_offset,
4528 targ_spec.bit_offset);
4529 return -EINVAL;
4530 }
4531
4532 cand_ids->data[j++] = cand_spec.spec[0].type_id;
4533 }
4534
4535 /*
4536 * For BPF_FIELD_EXISTS relo or when used BPF program has field
4537 * existence checks or kernel version/config checks, it's expected
4538 * that we might not find any candidates. In this case, if field
4539 * wasn't found in any candidate, the list of candidates shouldn't
4540 * change at all, we'll just handle relocating appropriately,
4541 * depending on relo's kind.
4542 */
4543 if (j > 0)
4544 cand_ids->len = j;
4545
4546 /*
4547 * If no candidates were found, it might be both a programmer error,
4548 * as well as expected case, depending whether instruction w/
4549 * relocation is guarded in some way that makes it unreachable (dead
4550 * code) if relocation can't be resolved. This is handled in
4551 * bpf_core_reloc_insn() uniformly by replacing that instruction with
4552 * BPF helper call insn (using invalid helper ID). If that instruction
4553 * is indeed unreachable, then it will be ignored and eliminated by
4554 * verifier. If it was an error, then verifier will complain and point
4555 * to a specific instruction number in its log.
4556 */
4557 if (j == 0)
4558 pr_debug("prog '%s': relo #%d: no matching targets found for [%d] %s + %s\n",
4559 prog_name, relo_idx, local_id, local_name, spec_str);
4560
4561 /* bpf_core_reloc_insn should know how to handle missing targ_spec */
4562 err = bpf_core_reloc_insn(prog, relo, relo_idx, &local_spec,
4563 j ? &targ_spec : NULL);
4564 if (err) {
4565 pr_warn("prog '%s': relo #%d: failed to patch insn at offset %d: %d\n",
4566 prog_name, relo_idx, relo->insn_off, err);
4567 return -EINVAL;
4568 }
4569
4570 return 0;
4571 }
4572
4573 static int
4574 bpf_core_reloc_fields(struct bpf_object *obj, const char *targ_btf_path)
4575 {
4576 const struct btf_ext_info_sec *sec;
4577 const struct bpf_field_reloc *rec;
4578 const struct btf_ext_info *seg;
4579 struct hashmap_entry *entry;
4580 struct hashmap *cand_cache = NULL;
4581 struct bpf_program *prog;
4582 struct btf *targ_btf;
4583 const char *sec_name;
4584 int i, err = 0;
4585
4586 if (targ_btf_path)
4587 targ_btf = btf__parse_elf(targ_btf_path, NULL);
4588 else
4589 targ_btf = libbpf_find_kernel_btf();
4590 if (IS_ERR(targ_btf)) {
4591 pr_warn("failed to get target BTF: %ld\n", PTR_ERR(targ_btf));
4592 return PTR_ERR(targ_btf);
4593 }
4594
4595 cand_cache = hashmap__new(bpf_core_hash_fn, bpf_core_equal_fn, NULL);
4596 if (IS_ERR(cand_cache)) {
4597 err = PTR_ERR(cand_cache);
4598 goto out;
4599 }
4600
4601 seg = &obj->btf_ext->field_reloc_info;
4602 for_each_btf_ext_sec(seg, sec) {
4603 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
4604 if (str_is_empty(sec_name)) {
4605 err = -EINVAL;
4606 goto out;
4607 }
4608 prog = bpf_object__find_program_by_title(obj, sec_name);
4609 if (!prog) {
4610 pr_warn("failed to find program '%s' for CO-RE offset relocation\n",
4611 sec_name);
4612 err = -EINVAL;
4613 goto out;
4614 }
4615
4616 pr_debug("prog '%s': performing %d CO-RE offset relocs\n",
4617 sec_name, sec->num_info);
4618
4619 for_each_btf_ext_rec(seg, sec, i, rec) {
4620 err = bpf_core_reloc_field(prog, rec, i, obj->btf,
4621 targ_btf, cand_cache);
4622 if (err) {
4623 pr_warn("prog '%s': relo #%d: failed to relocate: %d\n",
4624 sec_name, i, err);
4625 goto out;
4626 }
4627 }
4628 }
4629
4630 out:
4631 btf__free(targ_btf);
4632 if (!IS_ERR_OR_NULL(cand_cache)) {
4633 hashmap__for_each_entry(cand_cache, entry, i) {
4634 bpf_core_free_cands(entry->value);
4635 }
4636 hashmap__free(cand_cache);
4637 }
4638 return err;
4639 }
4640
4641 static int
4642 bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path)
4643 {
4644 int err = 0;
4645
4646 if (obj->btf_ext->field_reloc_info.len)
4647 err = bpf_core_reloc_fields(obj, targ_btf_path);
4648
4649 return err;
4650 }
4651
4652 static int
4653 bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj,
4654 struct reloc_desc *relo)
4655 {
4656 struct bpf_insn *insn, *new_insn;
4657 struct bpf_program *text;
4658 size_t new_cnt;
4659 int err;
4660
4661 if (prog->idx != obj->efile.text_shndx && prog->main_prog_cnt == 0) {
4662 text = bpf_object__find_prog_by_idx(obj, obj->efile.text_shndx);
4663 if (!text) {
4664 pr_warn("no .text section found yet relo into text exist\n");
4665 return -LIBBPF_ERRNO__RELOC;
4666 }
4667 new_cnt = prog->insns_cnt + text->insns_cnt;
4668 new_insn = reallocarray(prog->insns, new_cnt, sizeof(*insn));
4669 if (!new_insn) {
4670 pr_warn("oom in prog realloc\n");
4671 return -ENOMEM;
4672 }
4673 prog->insns = new_insn;
4674
4675 if (obj->btf_ext) {
4676 err = bpf_program_reloc_btf_ext(prog, obj,
4677 text->section_name,
4678 prog->insns_cnt);
4679 if (err)
4680 return err;
4681 }
4682
4683 memcpy(new_insn + prog->insns_cnt, text->insns,
4684 text->insns_cnt * sizeof(*insn));
4685 prog->main_prog_cnt = prog->insns_cnt;
4686 prog->insns_cnt = new_cnt;
4687 pr_debug("added %zd insn from %s to prog %s\n",
4688 text->insns_cnt, text->section_name,
4689 prog->section_name);
4690 }
4691
4692 insn = &prog->insns[relo->insn_idx];
4693 insn->imm += relo->sym_off / 8 + prog->main_prog_cnt - relo->insn_idx;
4694 return 0;
4695 }
4696
4697 static int
4698 bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj)
4699 {
4700 int i, err;
4701
4702 if (!prog)
4703 return 0;
4704
4705 if (obj->btf_ext) {
4706 err = bpf_program_reloc_btf_ext(prog, obj,
4707 prog->section_name, 0);
4708 if (err)
4709 return err;
4710 }
4711
4712 if (!prog->reloc_desc)
4713 return 0;
4714
4715 for (i = 0; i < prog->nr_reloc; i++) {
4716 struct reloc_desc *relo = &prog->reloc_desc[i];
4717 struct bpf_insn *insn = &prog->insns[relo->insn_idx];
4718
4719 if (relo->insn_idx + 1 >= (int)prog->insns_cnt) {
4720 pr_warn("relocation out of range: '%s'\n",
4721 prog->section_name);
4722 return -LIBBPF_ERRNO__RELOC;
4723 }
4724
4725 switch (relo->type) {
4726 case RELO_LD64:
4727 insn[0].src_reg = BPF_PSEUDO_MAP_FD;
4728 insn[0].imm = obj->maps[relo->map_idx].fd;
4729 break;
4730 case RELO_DATA:
4731 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE;
4732 insn[1].imm = insn[0].imm + relo->sym_off;
4733 insn[0].imm = obj->maps[relo->map_idx].fd;
4734 break;
4735 case RELO_EXTERN:
4736 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE;
4737 insn[0].imm = obj->maps[obj->kconfig_map_idx].fd;
4738 insn[1].imm = relo->sym_off;
4739 break;
4740 case RELO_CALL:
4741 err = bpf_program__reloc_text(prog, obj, relo);
4742 if (err)
4743 return err;
4744 break;
4745 default:
4746 pr_warn("relo #%d: bad relo type %d\n", i, relo->type);
4747 return -EINVAL;
4748 }
4749 }
4750
4751 zfree(&prog->reloc_desc);
4752 prog->nr_reloc = 0;
4753 return 0;
4754 }
4755
4756 static int
4757 bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path)
4758 {
4759 struct bpf_program *prog;
4760 size_t i;
4761 int err;
4762
4763 if (obj->btf_ext) {
4764 err = bpf_object__relocate_core(obj, targ_btf_path);
4765 if (err) {
4766 pr_warn("failed to perform CO-RE relocations: %d\n",
4767 err);
4768 return err;
4769 }
4770 }
4771 /* ensure .text is relocated first, as it's going to be copied as-is
4772 * later for sub-program calls
4773 */
4774 for (i = 0; i < obj->nr_programs; i++) {
4775 prog = &obj->programs[i];
4776 if (prog->idx != obj->efile.text_shndx)
4777 continue;
4778
4779 err = bpf_program__relocate(prog, obj);
4780 if (err) {
4781 pr_warn("failed to relocate '%s'\n", prog->section_name);
4782 return err;
4783 }
4784 break;
4785 }
4786 /* now relocate everything but .text, which by now is relocated
4787 * properly, so we can copy raw sub-program instructions as is safely
4788 */
4789 for (i = 0; i < obj->nr_programs; i++) {
4790 prog = &obj->programs[i];
4791 if (prog->idx == obj->efile.text_shndx)
4792 continue;
4793
4794 err = bpf_program__relocate(prog, obj);
4795 if (err) {
4796 pr_warn("failed to relocate '%s'\n", prog->section_name);
4797 return err;
4798 }
4799 }
4800 return 0;
4801 }
4802
4803 static int bpf_object__collect_struct_ops_map_reloc(struct bpf_object *obj,
4804 GElf_Shdr *shdr,
4805 Elf_Data *data);
4806
4807 static int bpf_object__collect_reloc(struct bpf_object *obj)
4808 {
4809 int i, err;
4810
4811 if (!obj_elf_valid(obj)) {
4812 pr_warn("Internal error: elf object is closed\n");
4813 return -LIBBPF_ERRNO__INTERNAL;
4814 }
4815
4816 for (i = 0; i < obj->efile.nr_reloc_sects; i++) {
4817 GElf_Shdr *shdr = &obj->efile.reloc_sects[i].shdr;
4818 Elf_Data *data = obj->efile.reloc_sects[i].data;
4819 int idx = shdr->sh_info;
4820 struct bpf_program *prog;
4821
4822 if (shdr->sh_type != SHT_REL) {
4823 pr_warn("internal error at %d\n", __LINE__);
4824 return -LIBBPF_ERRNO__INTERNAL;
4825 }
4826
4827 if (idx == obj->efile.st_ops_shndx) {
4828 err = bpf_object__collect_struct_ops_map_reloc(obj,
4829 shdr,
4830 data);
4831 if (err)
4832 return err;
4833 continue;
4834 }
4835
4836 prog = bpf_object__find_prog_by_idx(obj, idx);
4837 if (!prog) {
4838 pr_warn("relocation failed: no section(%d)\n", idx);
4839 return -LIBBPF_ERRNO__RELOC;
4840 }
4841
4842 err = bpf_program__collect_reloc(prog, shdr, data, obj);
4843 if (err)
4844 return err;
4845 }
4846 return 0;
4847 }
4848
4849 static int
4850 load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt,
4851 char *license, __u32 kern_version, int *pfd)
4852 {
4853 struct bpf_load_program_attr load_attr;
4854 char *cp, errmsg[STRERR_BUFSIZE];
4855 size_t log_buf_size = 0;
4856 char *log_buf = NULL;
4857 int btf_fd, ret;
4858
4859 if (!insns || !insns_cnt)
4860 return -EINVAL;
4861
4862 memset(&load_attr, 0, sizeof(struct bpf_load_program_attr));
4863 load_attr.prog_type = prog->type;
4864 load_attr.expected_attach_type = prog->expected_attach_type;
4865 if (prog->caps->name)
4866 load_attr.name = prog->name;
4867 load_attr.insns = insns;
4868 load_attr.insns_cnt = insns_cnt;
4869 load_attr.license = license;
4870 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS ||
4871 prog->type == BPF_PROG_TYPE_LSM) {
4872 load_attr.attach_btf_id = prog->attach_btf_id;
4873 } else if (prog->type == BPF_PROG_TYPE_TRACING ||
4874 prog->type == BPF_PROG_TYPE_EXT) {
4875 load_attr.attach_prog_fd = prog->attach_prog_fd;
4876 load_attr.attach_btf_id = prog->attach_btf_id;
4877 } else {
4878 load_attr.kern_version = kern_version;
4879 load_attr.prog_ifindex = prog->prog_ifindex;
4880 }
4881 /* if .BTF.ext was loaded, kernel supports associated BTF for prog */
4882 if (prog->obj->btf_ext)
4883 btf_fd = bpf_object__btf_fd(prog->obj);
4884 else
4885 btf_fd = -1;
4886 load_attr.prog_btf_fd = btf_fd >= 0 ? btf_fd : 0;
4887 load_attr.func_info = prog->func_info;
4888 load_attr.func_info_rec_size = prog->func_info_rec_size;
4889 load_attr.func_info_cnt = prog->func_info_cnt;
4890 load_attr.line_info = prog->line_info;
4891 load_attr.line_info_rec_size = prog->line_info_rec_size;
4892 load_attr.line_info_cnt = prog->line_info_cnt;
4893 load_attr.log_level = prog->log_level;
4894 load_attr.prog_flags = prog->prog_flags;
4895
4896 retry_load:
4897 if (log_buf_size) {
4898 log_buf = malloc(log_buf_size);
4899 if (!log_buf)
4900 return -ENOMEM;
4901
4902 *log_buf = 0;
4903 }
4904
4905 ret = bpf_load_program_xattr(&load_attr, log_buf, log_buf_size);
4906
4907 if (ret >= 0) {
4908 if (log_buf && load_attr.log_level)
4909 pr_debug("verifier log:\n%s", log_buf);
4910 *pfd = ret;
4911 ret = 0;
4912 goto out;
4913 }
4914
4915 if (!log_buf || errno == ENOSPC) {
4916 log_buf_size = max((size_t)BPF_LOG_BUF_SIZE,
4917 log_buf_size << 1);
4918
4919 free(log_buf);
4920 goto retry_load;
4921 }
4922 ret = -errno;
4923 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
4924 pr_warn("load bpf program failed: %s\n", cp);
4925 pr_perm_msg(ret);
4926
4927 if (log_buf && log_buf[0] != '\0') {
4928 ret = -LIBBPF_ERRNO__VERIFY;
4929 pr_warn("-- BEGIN DUMP LOG ---\n");
4930 pr_warn("\n%s\n", log_buf);
4931 pr_warn("-- END LOG --\n");
4932 } else if (load_attr.insns_cnt >= BPF_MAXINSNS) {
4933 pr_warn("Program too large (%zu insns), at most %d insns\n",
4934 load_attr.insns_cnt, BPF_MAXINSNS);
4935 ret = -LIBBPF_ERRNO__PROG2BIG;
4936 } else if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) {
4937 /* Wrong program type? */
4938 int fd;
4939
4940 load_attr.prog_type = BPF_PROG_TYPE_KPROBE;
4941 load_attr.expected_attach_type = 0;
4942 fd = bpf_load_program_xattr(&load_attr, NULL, 0);
4943 if (fd >= 0) {
4944 close(fd);
4945 ret = -LIBBPF_ERRNO__PROGTYPE;
4946 goto out;
4947 }
4948 }
4949
4950 out:
4951 free(log_buf);
4952 return ret;
4953 }
4954
4955 static int libbpf_find_attach_btf_id(struct bpf_program *prog);
4956
4957 int bpf_program__load(struct bpf_program *prog, char *license, __u32 kern_ver)
4958 {
4959 int err = 0, fd, i, btf_id;
4960
4961 if ((prog->type == BPF_PROG_TYPE_TRACING ||
4962 prog->type == BPF_PROG_TYPE_LSM ||
4963 prog->type == BPF_PROG_TYPE_EXT) && !prog->attach_btf_id) {
4964 btf_id = libbpf_find_attach_btf_id(prog);
4965 if (btf_id <= 0)
4966 return btf_id;
4967 prog->attach_btf_id = btf_id;
4968 }
4969
4970 if (prog->instances.nr < 0 || !prog->instances.fds) {
4971 if (prog->preprocessor) {
4972 pr_warn("Internal error: can't load program '%s'\n",
4973 prog->section_name);
4974 return -LIBBPF_ERRNO__INTERNAL;
4975 }
4976
4977 prog->instances.fds = malloc(sizeof(int));
4978 if (!prog->instances.fds) {
4979 pr_warn("Not enough memory for BPF fds\n");
4980 return -ENOMEM;
4981 }
4982 prog->instances.nr = 1;
4983 prog->instances.fds[0] = -1;
4984 }
4985
4986 if (!prog->preprocessor) {
4987 if (prog->instances.nr != 1) {
4988 pr_warn("Program '%s' is inconsistent: nr(%d) != 1\n",
4989 prog->section_name, prog->instances.nr);
4990 }
4991 err = load_program(prog, prog->insns, prog->insns_cnt,
4992 license, kern_ver, &fd);
4993 if (!err)
4994 prog->instances.fds[0] = fd;
4995 goto out;
4996 }
4997
4998 for (i = 0; i < prog->instances.nr; i++) {
4999 struct bpf_prog_prep_result result;
5000 bpf_program_prep_t preprocessor = prog->preprocessor;
5001
5002 memset(&result, 0, sizeof(result));
5003 err = preprocessor(prog, i, prog->insns,
5004 prog->insns_cnt, &result);
5005 if (err) {
5006 pr_warn("Preprocessing the %dth instance of program '%s' failed\n",
5007 i, prog->section_name);
5008 goto out;
5009 }
5010
5011 if (!result.new_insn_ptr || !result.new_insn_cnt) {
5012 pr_debug("Skip loading the %dth instance of program '%s'\n",
5013 i, prog->section_name);
5014 prog->instances.fds[i] = -1;
5015 if (result.pfd)
5016 *result.pfd = -1;
5017 continue;
5018 }
5019
5020 err = load_program(prog, result.new_insn_ptr,
5021 result.new_insn_cnt, license, kern_ver, &fd);
5022 if (err) {
5023 pr_warn("Loading the %dth instance of program '%s' failed\n",
5024 i, prog->section_name);
5025 goto out;
5026 }
5027
5028 if (result.pfd)
5029 *result.pfd = fd;
5030 prog->instances.fds[i] = fd;
5031 }
5032 out:
5033 if (err)
5034 pr_warn("failed to load program '%s'\n", prog->section_name);
5035 zfree(&prog->insns);
5036 prog->insns_cnt = 0;
5037 return err;
5038 }
5039
5040 static bool bpf_program__is_function_storage(const struct bpf_program *prog,
5041 const struct bpf_object *obj)
5042 {
5043 return prog->idx == obj->efile.text_shndx && obj->has_pseudo_calls;
5044 }
5045
5046 static int
5047 bpf_object__load_progs(struct bpf_object *obj, int log_level)
5048 {
5049 size_t i;
5050 int err;
5051
5052 for (i = 0; i < obj->nr_programs; i++) {
5053 if (bpf_program__is_function_storage(&obj->programs[i], obj))
5054 continue;
5055 obj->programs[i].log_level |= log_level;
5056 err = bpf_program__load(&obj->programs[i],
5057 obj->license,
5058 obj->kern_version);
5059 if (err)
5060 return err;
5061 }
5062 return 0;
5063 }
5064
5065 static struct bpf_object *
5066 __bpf_object__open(const char *path, const void *obj_buf, size_t obj_buf_sz,
5067 const struct bpf_object_open_opts *opts)
5068 {
5069 const char *obj_name, *kconfig;
5070 struct bpf_program *prog;
5071 struct bpf_object *obj;
5072 char tmp_name[64];
5073 int err;
5074
5075 if (elf_version(EV_CURRENT) == EV_NONE) {
5076 pr_warn("failed to init libelf for %s\n",
5077 path ? : "(mem buf)");
5078 return ERR_PTR(-LIBBPF_ERRNO__LIBELF);
5079 }
5080
5081 if (!OPTS_VALID(opts, bpf_object_open_opts))
5082 return ERR_PTR(-EINVAL);
5083
5084 obj_name = OPTS_GET(opts, object_name, NULL);
5085 if (obj_buf) {
5086 if (!obj_name) {
5087 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx",
5088 (unsigned long)obj_buf,
5089 (unsigned long)obj_buf_sz);
5090 obj_name = tmp_name;
5091 }
5092 path = obj_name;
5093 pr_debug("loading object '%s' from buffer\n", obj_name);
5094 }
5095
5096 obj = bpf_object__new(path, obj_buf, obj_buf_sz, obj_name);
5097 if (IS_ERR(obj))
5098 return obj;
5099
5100 kconfig = OPTS_GET(opts, kconfig, NULL);
5101 if (kconfig) {
5102 obj->kconfig = strdup(kconfig);
5103 if (!obj->kconfig)
5104 return ERR_PTR(-ENOMEM);
5105 }
5106
5107 err = bpf_object__elf_init(obj);
5108 err = err ? : bpf_object__check_endianness(obj);
5109 err = err ? : bpf_object__elf_collect(obj);
5110 err = err ? : bpf_object__collect_externs(obj);
5111 err = err ? : bpf_object__finalize_btf(obj);
5112 err = err ? : bpf_object__init_maps(obj, opts);
5113 err = err ? : bpf_object__init_prog_names(obj);
5114 err = err ? : bpf_object__collect_reloc(obj);
5115 if (err)
5116 goto out;
5117 bpf_object__elf_finish(obj);
5118
5119 bpf_object__for_each_program(prog, obj) {
5120 enum bpf_prog_type prog_type;
5121 enum bpf_attach_type attach_type;
5122
5123 if (prog->type != BPF_PROG_TYPE_UNSPEC)
5124 continue;
5125
5126 err = libbpf_prog_type_by_name(prog->section_name, &prog_type,
5127 &attach_type);
5128 if (err == -ESRCH)
5129 /* couldn't guess, but user might manually specify */
5130 continue;
5131 if (err)
5132 goto out;
5133
5134 bpf_program__set_type(prog, prog_type);
5135 bpf_program__set_expected_attach_type(prog, attach_type);
5136 if (prog_type == BPF_PROG_TYPE_TRACING ||
5137 prog_type == BPF_PROG_TYPE_EXT)
5138 prog->attach_prog_fd = OPTS_GET(opts, attach_prog_fd, 0);
5139 }
5140
5141 return obj;
5142 out:
5143 bpf_object__close(obj);
5144 return ERR_PTR(err);
5145 }
5146
5147 static struct bpf_object *
5148 __bpf_object__open_xattr(struct bpf_object_open_attr *attr, int flags)
5149 {
5150 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts,
5151 .relaxed_maps = flags & MAPS_RELAX_COMPAT,
5152 );
5153
5154 /* param validation */
5155 if (!attr->file)
5156 return NULL;
5157
5158 pr_debug("loading %s\n", attr->file);
5159 return __bpf_object__open(attr->file, NULL, 0, &opts);
5160 }
5161
5162 struct bpf_object *bpf_object__open_xattr(struct bpf_object_open_attr *attr)
5163 {
5164 return __bpf_object__open_xattr(attr, 0);
5165 }
5166
5167 struct bpf_object *bpf_object__open(const char *path)
5168 {
5169 struct bpf_object_open_attr attr = {
5170 .file = path,
5171 .prog_type = BPF_PROG_TYPE_UNSPEC,
5172 };
5173
5174 return bpf_object__open_xattr(&attr);
5175 }
5176
5177 struct bpf_object *
5178 bpf_object__open_file(const char *path, const struct bpf_object_open_opts *opts)
5179 {
5180 if (!path)
5181 return ERR_PTR(-EINVAL);
5182
5183 pr_debug("loading %s\n", path);
5184
5185 return __bpf_object__open(path, NULL, 0, opts);
5186 }
5187
5188 struct bpf_object *
5189 bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz,
5190 const struct bpf_object_open_opts *opts)
5191 {
5192 if (!obj_buf || obj_buf_sz == 0)
5193 return ERR_PTR(-EINVAL);
5194
5195 return __bpf_object__open(NULL, obj_buf, obj_buf_sz, opts);
5196 }
5197
5198 struct bpf_object *
5199 bpf_object__open_buffer(const void *obj_buf, size_t obj_buf_sz,
5200 const char *name)
5201 {
5202 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts,
5203 .object_name = name,
5204 /* wrong default, but backwards-compatible */
5205 .relaxed_maps = true,
5206 );
5207
5208 /* returning NULL is wrong, but backwards-compatible */
5209 if (!obj_buf || obj_buf_sz == 0)
5210 return NULL;
5211
5212 return bpf_object__open_mem(obj_buf, obj_buf_sz, &opts);
5213 }
5214
5215 int bpf_object__unload(struct bpf_object *obj)
5216 {
5217 size_t i;
5218
5219 if (!obj)
5220 return -EINVAL;
5221
5222 for (i = 0; i < obj->nr_maps; i++) {
5223 zclose(obj->maps[i].fd);
5224 if (obj->maps[i].st_ops)
5225 zfree(&obj->maps[i].st_ops->kern_vdata);
5226 }
5227
5228 for (i = 0; i < obj->nr_programs; i++)
5229 bpf_program__unload(&obj->programs[i]);
5230
5231 return 0;
5232 }
5233
5234 static int bpf_object__sanitize_maps(struct bpf_object *obj)
5235 {
5236 struct bpf_map *m;
5237
5238 bpf_object__for_each_map(m, obj) {
5239 if (!bpf_map__is_internal(m))
5240 continue;
5241 if (!obj->caps.global_data) {
5242 pr_warn("kernel doesn't support global data\n");
5243 return -ENOTSUP;
5244 }
5245 if (!obj->caps.array_mmap)
5246 m->def.map_flags ^= BPF_F_MMAPABLE;
5247 }
5248
5249 return 0;
5250 }
5251
5252 static int bpf_object__resolve_externs(struct bpf_object *obj,
5253 const char *extra_kconfig)
5254 {
5255 bool need_config = false;
5256 struct extern_desc *ext;
5257 int err, i;
5258 void *data;
5259
5260 if (obj->nr_extern == 0)
5261 return 0;
5262
5263 data = obj->maps[obj->kconfig_map_idx].mmaped;
5264
5265 for (i = 0; i < obj->nr_extern; i++) {
5266 ext = &obj->externs[i];
5267
5268 if (strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) {
5269 void *ext_val = data + ext->data_off;
5270 __u32 kver = get_kernel_version();
5271
5272 if (!kver) {
5273 pr_warn("failed to get kernel version\n");
5274 return -EINVAL;
5275 }
5276 err = set_ext_value_num(ext, ext_val, kver);
5277 if (err)
5278 return err;
5279 pr_debug("extern %s=0x%x\n", ext->name, kver);
5280 } else if (strncmp(ext->name, "CONFIG_", 7) == 0) {
5281 need_config = true;
5282 } else {
5283 pr_warn("unrecognized extern '%s'\n", ext->name);
5284 return -EINVAL;
5285 }
5286 }
5287 if (need_config && extra_kconfig) {
5288 err = bpf_object__read_kconfig_mem(obj, extra_kconfig, data);
5289 if (err)
5290 return -EINVAL;
5291 need_config = false;
5292 for (i = 0; i < obj->nr_extern; i++) {
5293 ext = &obj->externs[i];
5294 if (!ext->is_set) {
5295 need_config = true;
5296 break;
5297 }
5298 }
5299 }
5300 if (need_config) {
5301 err = bpf_object__read_kconfig_file(obj, data);
5302 if (err)
5303 return -EINVAL;
5304 }
5305 for (i = 0; i < obj->nr_extern; i++) {
5306 ext = &obj->externs[i];
5307
5308 if (!ext->is_set && !ext->is_weak) {
5309 pr_warn("extern %s (strong) not resolved\n", ext->name);
5310 return -ESRCH;
5311 } else if (!ext->is_set) {
5312 pr_debug("extern %s (weak) not resolved, defaulting to zero\n",
5313 ext->name);
5314 }
5315 }
5316
5317 return 0;
5318 }
5319
5320 int bpf_object__load_xattr(struct bpf_object_load_attr *attr)
5321 {
5322 struct bpf_object *obj;
5323 int err, i;
5324
5325 if (!attr)
5326 return -EINVAL;
5327 obj = attr->obj;
5328 if (!obj)
5329 return -EINVAL;
5330
5331 if (obj->loaded) {
5332 pr_warn("object should not be loaded twice\n");
5333 return -EINVAL;
5334 }
5335
5336 obj->loaded = true;
5337
5338 err = bpf_object__probe_caps(obj);
5339 err = err ? : bpf_object__resolve_externs(obj, obj->kconfig);
5340 err = err ? : bpf_object__sanitize_and_load_btf(obj);
5341 err = err ? : bpf_object__sanitize_maps(obj);
5342 err = err ? : bpf_object__load_vmlinux_btf(obj);
5343 err = err ? : bpf_object__init_kern_struct_ops_maps(obj);
5344 err = err ? : bpf_object__create_maps(obj);
5345 err = err ? : bpf_object__relocate(obj, attr->target_btf_path);
5346 err = err ? : bpf_object__load_progs(obj, attr->log_level);
5347
5348 btf__free(obj->btf_vmlinux);
5349 obj->btf_vmlinux = NULL;
5350
5351 if (err)
5352 goto out;
5353
5354 return 0;
5355 out:
5356 /* unpin any maps that were auto-pinned during load */
5357 for (i = 0; i < obj->nr_maps; i++)
5358 if (obj->maps[i].pinned && !obj->maps[i].reused)
5359 bpf_map__unpin(&obj->maps[i], NULL);
5360
5361 bpf_object__unload(obj);
5362 pr_warn("failed to load object '%s'\n", obj->path);
5363 return err;
5364 }
5365
5366 int bpf_object__load(struct bpf_object *obj)
5367 {
5368 struct bpf_object_load_attr attr = {
5369 .obj = obj,
5370 };
5371
5372 return bpf_object__load_xattr(&attr);
5373 }
5374
5375 static int make_parent_dir(const char *path)
5376 {
5377 char *cp, errmsg[STRERR_BUFSIZE];
5378 char *dname, *dir;
5379 int err = 0;
5380
5381 dname = strdup(path);
5382 if (dname == NULL)
5383 return -ENOMEM;
5384
5385 dir = dirname(dname);
5386 if (mkdir(dir, 0700) && errno != EEXIST)
5387 err = -errno;
5388
5389 free(dname);
5390 if (err) {
5391 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
5392 pr_warn("failed to mkdir %s: %s\n", path, cp);
5393 }
5394 return err;
5395 }
5396
5397 static int check_path(const char *path)
5398 {
5399 char *cp, errmsg[STRERR_BUFSIZE];
5400 struct statfs st_fs;
5401 char *dname, *dir;
5402 int err = 0;
5403
5404 if (path == NULL)
5405 return -EINVAL;
5406
5407 dname = strdup(path);
5408 if (dname == NULL)
5409 return -ENOMEM;
5410
5411 dir = dirname(dname);
5412 if (statfs(dir, &st_fs)) {
5413 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
5414 pr_warn("failed to statfs %s: %s\n", dir, cp);
5415 err = -errno;
5416 }
5417 free(dname);
5418
5419 if (!err && st_fs.f_type != BPF_FS_MAGIC) {
5420 pr_warn("specified path %s is not on BPF FS\n", path);
5421 err = -EINVAL;
5422 }
5423
5424 return err;
5425 }
5426
5427 int bpf_program__pin_instance(struct bpf_program *prog, const char *path,
5428 int instance)
5429 {
5430 char *cp, errmsg[STRERR_BUFSIZE];
5431 int err;
5432
5433 err = make_parent_dir(path);
5434 if (err)
5435 return err;
5436
5437 err = check_path(path);
5438 if (err)
5439 return err;
5440
5441 if (prog == NULL) {
5442 pr_warn("invalid program pointer\n");
5443 return -EINVAL;
5444 }
5445
5446 if (instance < 0 || instance >= prog->instances.nr) {
5447 pr_warn("invalid prog instance %d of prog %s (max %d)\n",
5448 instance, prog->section_name, prog->instances.nr);
5449 return -EINVAL;
5450 }
5451
5452 if (bpf_obj_pin(prog->instances.fds[instance], path)) {
5453 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
5454 pr_warn("failed to pin program: %s\n", cp);
5455 return -errno;
5456 }
5457 pr_debug("pinned program '%s'\n", path);
5458
5459 return 0;
5460 }
5461
5462 int bpf_program__unpin_instance(struct bpf_program *prog, const char *path,
5463 int instance)
5464 {
5465 int err;
5466
5467 err = check_path(path);
5468 if (err)
5469 return err;
5470
5471 if (prog == NULL) {
5472 pr_warn("invalid program pointer\n");
5473 return -EINVAL;
5474 }
5475
5476 if (instance < 0 || instance >= prog->instances.nr) {
5477 pr_warn("invalid prog instance %d of prog %s (max %d)\n",
5478 instance, prog->section_name, prog->instances.nr);
5479 return -EINVAL;
5480 }
5481
5482 err = unlink(path);
5483 if (err != 0)
5484 return -errno;
5485 pr_debug("unpinned program '%s'\n", path);
5486
5487 return 0;
5488 }
5489
5490 int bpf_program__pin(struct bpf_program *prog, const char *path)
5491 {
5492 int i, err;
5493
5494 err = make_parent_dir(path);
5495 if (err)
5496 return err;
5497
5498 err = check_path(path);
5499 if (err)
5500 return err;
5501
5502 if (prog == NULL) {
5503 pr_warn("invalid program pointer\n");
5504 return -EINVAL;
5505 }
5506
5507 if (prog->instances.nr <= 0) {
5508 pr_warn("no instances of prog %s to pin\n",
5509 prog->section_name);
5510 return -EINVAL;
5511 }
5512
5513 if (prog->instances.nr == 1) {
5514 /* don't create subdirs when pinning single instance */
5515 return bpf_program__pin_instance(prog, path, 0);
5516 }
5517
5518 for (i = 0; i < prog->instances.nr; i++) {
5519 char buf[PATH_MAX];
5520 int len;
5521
5522 len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
5523 if (len < 0) {
5524 err = -EINVAL;
5525 goto err_unpin;
5526 } else if (len >= PATH_MAX) {
5527 err = -ENAMETOOLONG;
5528 goto err_unpin;
5529 }
5530
5531 err = bpf_program__pin_instance(prog, buf, i);
5532 if (err)
5533 goto err_unpin;
5534 }
5535
5536 return 0;
5537
5538 err_unpin:
5539 for (i = i - 1; i >= 0; i--) {
5540 char buf[PATH_MAX];
5541 int len;
5542
5543 len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
5544 if (len < 0)
5545 continue;
5546 else if (len >= PATH_MAX)
5547 continue;
5548
5549 bpf_program__unpin_instance(prog, buf, i);
5550 }
5551
5552 rmdir(path);
5553
5554 return err;
5555 }
5556
5557 int bpf_program__unpin(struct bpf_program *prog, const char *path)
5558 {
5559 int i, err;
5560
5561 err = check_path(path);
5562 if (err)
5563 return err;
5564
5565 if (prog == NULL) {
5566 pr_warn("invalid program pointer\n");
5567 return -EINVAL;
5568 }
5569
5570 if (prog->instances.nr <= 0) {
5571 pr_warn("no instances of prog %s to pin\n",
5572 prog->section_name);
5573 return -EINVAL;
5574 }
5575
5576 if (prog->instances.nr == 1) {
5577 /* don't create subdirs when pinning single instance */
5578 return bpf_program__unpin_instance(prog, path, 0);
5579 }
5580
5581 for (i = 0; i < prog->instances.nr; i++) {
5582 char buf[PATH_MAX];
5583 int len;
5584
5585 len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
5586 if (len < 0)
5587 return -EINVAL;
5588 else if (len >= PATH_MAX)
5589 return -ENAMETOOLONG;
5590
5591 err = bpf_program__unpin_instance(prog, buf, i);
5592 if (err)
5593 return err;
5594 }
5595
5596 err = rmdir(path);
5597 if (err)
5598 return -errno;
5599
5600 return 0;
5601 }
5602
5603 int bpf_map__pin(struct bpf_map *map, const char *path)
5604 {
5605 char *cp, errmsg[STRERR_BUFSIZE];
5606 int err;
5607
5608 if (map == NULL) {
5609 pr_warn("invalid map pointer\n");
5610 return -EINVAL;
5611 }
5612
5613 if (map->pin_path) {
5614 if (path && strcmp(path, map->pin_path)) {
5615 pr_warn("map '%s' already has pin path '%s' different from '%s'\n",
5616 bpf_map__name(map), map->pin_path, path);
5617 return -EINVAL;
5618 } else if (map->pinned) {
5619 pr_debug("map '%s' already pinned at '%s'; not re-pinning\n",
5620 bpf_map__name(map), map->pin_path);
5621 return 0;
5622 }
5623 } else {
5624 if (!path) {
5625 pr_warn("missing a path to pin map '%s' at\n",
5626 bpf_map__name(map));
5627 return -EINVAL;
5628 } else if (map->pinned) {
5629 pr_warn("map '%s' already pinned\n", bpf_map__name(map));
5630 return -EEXIST;
5631 }
5632
5633 map->pin_path = strdup(path);
5634 if (!map->pin_path) {
5635 err = -errno;
5636 goto out_err;
5637 }
5638 }
5639
5640 err = make_parent_dir(map->pin_path);
5641 if (err)
5642 return err;
5643
5644 err = check_path(map->pin_path);
5645 if (err)
5646 return err;
5647
5648 if (bpf_obj_pin(map->fd, map->pin_path)) {
5649 err = -errno;
5650 goto out_err;
5651 }
5652
5653 map->pinned = true;
5654 pr_debug("pinned map '%s'\n", map->pin_path);
5655
5656 return 0;
5657
5658 out_err:
5659 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
5660 pr_warn("failed to pin map: %s\n", cp);
5661 return err;
5662 }
5663
5664 int bpf_map__unpin(struct bpf_map *map, const char *path)
5665 {
5666 int err;
5667
5668 if (map == NULL) {
5669 pr_warn("invalid map pointer\n");
5670 return -EINVAL;
5671 }
5672
5673 if (map->pin_path) {
5674 if (path && strcmp(path, map->pin_path)) {
5675 pr_warn("map '%s' already has pin path '%s' different from '%s'\n",
5676 bpf_map__name(map), map->pin_path, path);
5677 return -EINVAL;
5678 }
5679 path = map->pin_path;
5680 } else if (!path) {
5681 pr_warn("no path to unpin map '%s' from\n",
5682 bpf_map__name(map));
5683 return -EINVAL;
5684 }
5685
5686 err = check_path(path);
5687 if (err)
5688 return err;
5689
5690 err = unlink(path);
5691 if (err != 0)
5692 return -errno;
5693
5694 map->pinned = false;
5695 pr_debug("unpinned map '%s' from '%s'\n", bpf_map__name(map), path);
5696
5697 return 0;
5698 }
5699
5700 int bpf_map__set_pin_path(struct bpf_map *map, const char *path)
5701 {
5702 char *new = NULL;
5703
5704 if (path) {
5705 new = strdup(path);
5706 if (!new)
5707 return -errno;
5708 }
5709
5710 free(map->pin_path);
5711 map->pin_path = new;
5712 return 0;
5713 }
5714
5715 const char *bpf_map__get_pin_path(const struct bpf_map *map)
5716 {
5717 return map->pin_path;
5718 }
5719
5720 bool bpf_map__is_pinned(const struct bpf_map *map)
5721 {
5722 return map->pinned;
5723 }
5724
5725 int bpf_object__pin_maps(struct bpf_object *obj, const char *path)
5726 {
5727 struct bpf_map *map;
5728 int err;
5729
5730 if (!obj)
5731 return -ENOENT;
5732
5733 if (!obj->loaded) {
5734 pr_warn("object not yet loaded; load it first\n");
5735 return -ENOENT;
5736 }
5737
5738 bpf_object__for_each_map(map, obj) {
5739 char *pin_path = NULL;
5740 char buf[PATH_MAX];
5741
5742 if (path) {
5743 int len;
5744
5745 len = snprintf(buf, PATH_MAX, "%s/%s", path,
5746 bpf_map__name(map));
5747 if (len < 0) {
5748 err = -EINVAL;
5749 goto err_unpin_maps;
5750 } else if (len >= PATH_MAX) {
5751 err = -ENAMETOOLONG;
5752 goto err_unpin_maps;
5753 }
5754 pin_path = buf;
5755 } else if (!map->pin_path) {
5756 continue;
5757 }
5758
5759 err = bpf_map__pin(map, pin_path);
5760 if (err)
5761 goto err_unpin_maps;
5762 }
5763
5764 return 0;
5765
5766 err_unpin_maps:
5767 while ((map = bpf_map__prev(map, obj))) {
5768 if (!map->pin_path)
5769 continue;
5770
5771 bpf_map__unpin(map, NULL);
5772 }
5773
5774 return err;
5775 }
5776
5777 int bpf_object__unpin_maps(struct bpf_object *obj, const char *path)
5778 {
5779 struct bpf_map *map;
5780 int err;
5781
5782 if (!obj)
5783 return -ENOENT;
5784
5785 bpf_object__for_each_map(map, obj) {
5786 char *pin_path = NULL;
5787 char buf[PATH_MAX];
5788
5789 if (path) {
5790 int len;
5791
5792 len = snprintf(buf, PATH_MAX, "%s/%s", path,
5793 bpf_map__name(map));
5794 if (len < 0)
5795 return -EINVAL;
5796 else if (len >= PATH_MAX)
5797 return -ENAMETOOLONG;
5798 pin_path = buf;
5799 } else if (!map->pin_path) {
5800 continue;
5801 }
5802
5803 err = bpf_map__unpin(map, pin_path);
5804 if (err)
5805 return err;
5806 }
5807
5808 return 0;
5809 }
5810
5811 int bpf_object__pin_programs(struct bpf_object *obj, const char *path)
5812 {
5813 struct bpf_program *prog;
5814 int err;
5815
5816 if (!obj)
5817 return -ENOENT;
5818
5819 if (!obj->loaded) {
5820 pr_warn("object not yet loaded; load it first\n");
5821 return -ENOENT;
5822 }
5823
5824 bpf_object__for_each_program(prog, obj) {
5825 char buf[PATH_MAX];
5826 int len;
5827
5828 len = snprintf(buf, PATH_MAX, "%s/%s", path,
5829 prog->pin_name);
5830 if (len < 0) {
5831 err = -EINVAL;
5832 goto err_unpin_programs;
5833 } else if (len >= PATH_MAX) {
5834 err = -ENAMETOOLONG;
5835 goto err_unpin_programs;
5836 }
5837
5838 err = bpf_program__pin(prog, buf);
5839 if (err)
5840 goto err_unpin_programs;
5841 }
5842
5843 return 0;
5844
5845 err_unpin_programs:
5846 while ((prog = bpf_program__prev(prog, obj))) {
5847 char buf[PATH_MAX];
5848 int len;
5849
5850 len = snprintf(buf, PATH_MAX, "%s/%s", path,
5851 prog->pin_name);
5852 if (len < 0)
5853 continue;
5854 else if (len >= PATH_MAX)
5855 continue;
5856
5857 bpf_program__unpin(prog, buf);
5858 }
5859
5860 return err;
5861 }
5862
5863 int bpf_object__unpin_programs(struct bpf_object *obj, const char *path)
5864 {
5865 struct bpf_program *prog;
5866 int err;
5867
5868 if (!obj)
5869 return -ENOENT;
5870
5871 bpf_object__for_each_program(prog, obj) {
5872 char buf[PATH_MAX];
5873 int len;
5874
5875 len = snprintf(buf, PATH_MAX, "%s/%s", path,
5876 prog->pin_name);
5877 if (len < 0)
5878 return -EINVAL;
5879 else if (len >= PATH_MAX)
5880 return -ENAMETOOLONG;
5881
5882 err = bpf_program__unpin(prog, buf);
5883 if (err)
5884 return err;
5885 }
5886
5887 return 0;
5888 }
5889
5890 int bpf_object__pin(struct bpf_object *obj, const char *path)
5891 {
5892 int err;
5893
5894 err = bpf_object__pin_maps(obj, path);
5895 if (err)
5896 return err;
5897
5898 err = bpf_object__pin_programs(obj, path);
5899 if (err) {
5900 bpf_object__unpin_maps(obj, path);
5901 return err;
5902 }
5903
5904 return 0;
5905 }
5906
5907 void bpf_object__close(struct bpf_object *obj)
5908 {
5909 size_t i;
5910
5911 if (!obj)
5912 return;
5913
5914 if (obj->clear_priv)
5915 obj->clear_priv(obj, obj->priv);
5916
5917 bpf_object__elf_finish(obj);
5918 bpf_object__unload(obj);
5919 btf__free(obj->btf);
5920 btf_ext__free(obj->btf_ext);
5921
5922 for (i = 0; i < obj->nr_maps; i++) {
5923 struct bpf_map *map = &obj->maps[i];
5924
5925 if (map->clear_priv)
5926 map->clear_priv(map, map->priv);
5927 map->priv = NULL;
5928 map->clear_priv = NULL;
5929
5930 if (map->mmaped) {
5931 munmap(map->mmaped, bpf_map_mmap_sz(map));
5932 map->mmaped = NULL;
5933 }
5934
5935 if (map->st_ops) {
5936 zfree(&map->st_ops->data);
5937 zfree(&map->st_ops->progs);
5938 zfree(&map->st_ops->kern_func_off);
5939 zfree(&map->st_ops);
5940 }
5941
5942 zfree(&map->name);
5943 zfree(&map->pin_path);
5944 }
5945
5946 zfree(&obj->kconfig);
5947 zfree(&obj->externs);
5948 obj->nr_extern = 0;
5949
5950 zfree(&obj->maps);
5951 obj->nr_maps = 0;
5952
5953 if (obj->programs && obj->nr_programs) {
5954 for (i = 0; i < obj->nr_programs; i++)
5955 bpf_program__exit(&obj->programs[i]);
5956 }
5957 zfree(&obj->programs);
5958
5959 list_del(&obj->list);
5960 free(obj);
5961 }
5962
5963 struct bpf_object *
5964 bpf_object__next(struct bpf_object *prev)
5965 {
5966 struct bpf_object *next;
5967
5968 if (!prev)
5969 next = list_first_entry(&bpf_objects_list,
5970 struct bpf_object,
5971 list);
5972 else
5973 next = list_next_entry(prev, list);
5974
5975 /* Empty list is noticed here so don't need checking on entry. */
5976 if (&next->list == &bpf_objects_list)
5977 return NULL;
5978
5979 return next;
5980 }
5981
5982 const char *bpf_object__name(const struct bpf_object *obj)
5983 {
5984 return obj ? obj->name : ERR_PTR(-EINVAL);
5985 }
5986
5987 unsigned int bpf_object__kversion(const struct bpf_object *obj)
5988 {
5989 return obj ? obj->kern_version : 0;
5990 }
5991
5992 struct btf *bpf_object__btf(const struct bpf_object *obj)
5993 {
5994 return obj ? obj->btf : NULL;
5995 }
5996
5997 int bpf_object__btf_fd(const struct bpf_object *obj)
5998 {
5999 return obj->btf ? btf__fd(obj->btf) : -1;
6000 }
6001
6002 int bpf_object__set_priv(struct bpf_object *obj, void *priv,
6003 bpf_object_clear_priv_t clear_priv)
6004 {
6005 if (obj->priv && obj->clear_priv)
6006 obj->clear_priv(obj, obj->priv);
6007
6008 obj->priv = priv;
6009 obj->clear_priv = clear_priv;
6010 return 0;
6011 }
6012
6013 void *bpf_object__priv(const struct bpf_object *obj)
6014 {
6015 return obj ? obj->priv : ERR_PTR(-EINVAL);
6016 }
6017
6018 static struct bpf_program *
6019 __bpf_program__iter(const struct bpf_program *p, const struct bpf_object *obj,
6020 bool forward)
6021 {
6022 size_t nr_programs = obj->nr_programs;
6023 ssize_t idx;
6024
6025 if (!nr_programs)
6026 return NULL;
6027
6028 if (!p)
6029 /* Iter from the beginning */
6030 return forward ? &obj->programs[0] :
6031 &obj->programs[nr_programs - 1];
6032
6033 if (p->obj != obj) {
6034 pr_warn("error: program handler doesn't match object\n");
6035 return NULL;
6036 }
6037
6038 idx = (p - obj->programs) + (forward ? 1 : -1);
6039 if (idx >= obj->nr_programs || idx < 0)
6040 return NULL;
6041 return &obj->programs[idx];
6042 }
6043
6044 struct bpf_program *
6045 bpf_program__next(struct bpf_program *prev, const struct bpf_object *obj)
6046 {
6047 struct bpf_program *prog = prev;
6048
6049 do {
6050 prog = __bpf_program__iter(prog, obj, true);
6051 } while (prog && bpf_program__is_function_storage(prog, obj));
6052
6053 return prog;
6054 }
6055
6056 struct bpf_program *
6057 bpf_program__prev(struct bpf_program *next, const struct bpf_object *obj)
6058 {
6059 struct bpf_program *prog = next;
6060
6061 do {
6062 prog = __bpf_program__iter(prog, obj, false);
6063 } while (prog && bpf_program__is_function_storage(prog, obj));
6064
6065 return prog;
6066 }
6067
6068 int bpf_program__set_priv(struct bpf_program *prog, void *priv,
6069 bpf_program_clear_priv_t clear_priv)
6070 {
6071 if (prog->priv && prog->clear_priv)
6072 prog->clear_priv(prog, prog->priv);
6073
6074 prog->priv = priv;
6075 prog->clear_priv = clear_priv;
6076 return 0;
6077 }
6078
6079 void *bpf_program__priv(const struct bpf_program *prog)
6080 {
6081 return prog ? prog->priv : ERR_PTR(-EINVAL);
6082 }
6083
6084 void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex)
6085 {
6086 prog->prog_ifindex = ifindex;
6087 }
6088
6089 const char *bpf_program__name(const struct bpf_program *prog)
6090 {
6091 return prog->name;
6092 }
6093
6094 const char *bpf_program__title(const struct bpf_program *prog, bool needs_copy)
6095 {
6096 const char *title;
6097
6098 title = prog->section_name;
6099 if (needs_copy) {
6100 title = strdup(title);
6101 if (!title) {
6102 pr_warn("failed to strdup program title\n");
6103 return ERR_PTR(-ENOMEM);
6104 }
6105 }
6106
6107 return title;
6108 }
6109
6110 int bpf_program__fd(const struct bpf_program *prog)
6111 {
6112 return bpf_program__nth_fd(prog, 0);
6113 }
6114
6115 size_t bpf_program__size(const struct bpf_program *prog)
6116 {
6117 return prog->insns_cnt * sizeof(struct bpf_insn);
6118 }
6119
6120 int bpf_program__set_prep(struct bpf_program *prog, int nr_instances,
6121 bpf_program_prep_t prep)
6122 {
6123 int *instances_fds;
6124
6125 if (nr_instances <= 0 || !prep)
6126 return -EINVAL;
6127
6128 if (prog->instances.nr > 0 || prog->instances.fds) {
6129 pr_warn("Can't set pre-processor after loading\n");
6130 return -EINVAL;
6131 }
6132
6133 instances_fds = malloc(sizeof(int) * nr_instances);
6134 if (!instances_fds) {
6135 pr_warn("alloc memory failed for fds\n");
6136 return -ENOMEM;
6137 }
6138
6139 /* fill all fd with -1 */
6140 memset(instances_fds, -1, sizeof(int) * nr_instances);
6141
6142 prog->instances.nr = nr_instances;
6143 prog->instances.fds = instances_fds;
6144 prog->preprocessor = prep;
6145 return 0;
6146 }
6147
6148 int bpf_program__nth_fd(const struct bpf_program *prog, int n)
6149 {
6150 int fd;
6151
6152 if (!prog)
6153 return -EINVAL;
6154
6155 if (n >= prog->instances.nr || n < 0) {
6156 pr_warn("Can't get the %dth fd from program %s: only %d instances\n",
6157 n, prog->section_name, prog->instances.nr);
6158 return -EINVAL;
6159 }
6160
6161 fd = prog->instances.fds[n];
6162 if (fd < 0) {
6163 pr_warn("%dth instance of program '%s' is invalid\n",
6164 n, prog->section_name);
6165 return -ENOENT;
6166 }
6167
6168 return fd;
6169 }
6170
6171 enum bpf_prog_type bpf_program__get_type(struct bpf_program *prog)
6172 {
6173 return prog->type;
6174 }
6175
6176 void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type)
6177 {
6178 prog->type = type;
6179 }
6180
6181 static bool bpf_program__is_type(const struct bpf_program *prog,
6182 enum bpf_prog_type type)
6183 {
6184 return prog ? (prog->type == type) : false;
6185 }
6186
6187 #define BPF_PROG_TYPE_FNS(NAME, TYPE) \
6188 int bpf_program__set_##NAME(struct bpf_program *prog) \
6189 { \
6190 if (!prog) \
6191 return -EINVAL; \
6192 bpf_program__set_type(prog, TYPE); \
6193 return 0; \
6194 } \
6195 \
6196 bool bpf_program__is_##NAME(const struct bpf_program *prog) \
6197 { \
6198 return bpf_program__is_type(prog, TYPE); \
6199 } \
6200
6201 BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER);
6202 BPF_PROG_TYPE_FNS(lsm, BPF_PROG_TYPE_LSM);
6203 BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE);
6204 BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS);
6205 BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT);
6206 BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT);
6207 BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT);
6208 BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP);
6209 BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT);
6210 BPF_PROG_TYPE_FNS(tracing, BPF_PROG_TYPE_TRACING);
6211 BPF_PROG_TYPE_FNS(struct_ops, BPF_PROG_TYPE_STRUCT_OPS);
6212 BPF_PROG_TYPE_FNS(extension, BPF_PROG_TYPE_EXT);
6213
6214 enum bpf_attach_type
6215 bpf_program__get_expected_attach_type(struct bpf_program *prog)
6216 {
6217 return prog->expected_attach_type;
6218 }
6219
6220 void bpf_program__set_expected_attach_type(struct bpf_program *prog,
6221 enum bpf_attach_type type)
6222 {
6223 prog->expected_attach_type = type;
6224 }
6225
6226 #define BPF_PROG_SEC_IMPL(string, ptype, eatype, is_attachable, btf, atype) \
6227 { string, sizeof(string) - 1, ptype, eatype, is_attachable, btf, atype }
6228
6229 /* Programs that can NOT be attached. */
6230 #define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_IMPL(string, ptype, 0, 0, 0, 0)
6231
6232 /* Programs that can be attached. */
6233 #define BPF_APROG_SEC(string, ptype, atype) \
6234 BPF_PROG_SEC_IMPL(string, ptype, 0, 1, 0, atype)
6235
6236 /* Programs that must specify expected attach type at load time. */
6237 #define BPF_EAPROG_SEC(string, ptype, eatype) \
6238 BPF_PROG_SEC_IMPL(string, ptype, eatype, 1, 0, eatype)
6239
6240 /* Programs that use BTF to identify attach point */
6241 #define BPF_PROG_BTF(string, ptype, eatype) \
6242 BPF_PROG_SEC_IMPL(string, ptype, eatype, 0, 1, 0)
6243
6244 /* Programs that can be attached but attach type can't be identified by section
6245 * name. Kept for backward compatibility.
6246 */
6247 #define BPF_APROG_COMPAT(string, ptype) BPF_PROG_SEC(string, ptype)
6248
6249 #define SEC_DEF(sec_pfx, ptype, ...) { \
6250 .sec = sec_pfx, \
6251 .len = sizeof(sec_pfx) - 1, \
6252 .prog_type = BPF_PROG_TYPE_##ptype, \
6253 __VA_ARGS__ \
6254 }
6255
6256 struct bpf_sec_def;
6257
6258 typedef struct bpf_link *(*attach_fn_t)(const struct bpf_sec_def *sec,
6259 struct bpf_program *prog);
6260
6261 static struct bpf_link *attach_kprobe(const struct bpf_sec_def *sec,
6262 struct bpf_program *prog);
6263 static struct bpf_link *attach_tp(const struct bpf_sec_def *sec,
6264 struct bpf_program *prog);
6265 static struct bpf_link *attach_raw_tp(const struct bpf_sec_def *sec,
6266 struct bpf_program *prog);
6267 static struct bpf_link *attach_trace(const struct bpf_sec_def *sec,
6268 struct bpf_program *prog);
6269 static struct bpf_link *attach_lsm(const struct bpf_sec_def *sec,
6270 struct bpf_program *prog);
6271
6272 struct bpf_sec_def {
6273 const char *sec;
6274 size_t len;
6275 enum bpf_prog_type prog_type;
6276 enum bpf_attach_type expected_attach_type;
6277 bool is_attachable;
6278 bool is_attach_btf;
6279 enum bpf_attach_type attach_type;
6280 attach_fn_t attach_fn;
6281 };
6282
6283 static const struct bpf_sec_def section_defs[] = {
6284 BPF_PROG_SEC("socket", BPF_PROG_TYPE_SOCKET_FILTER),
6285 BPF_PROG_SEC("sk_reuseport", BPF_PROG_TYPE_SK_REUSEPORT),
6286 SEC_DEF("kprobe/", KPROBE,
6287 .attach_fn = attach_kprobe),
6288 BPF_PROG_SEC("uprobe/", BPF_PROG_TYPE_KPROBE),
6289 SEC_DEF("kretprobe/", KPROBE,
6290 .attach_fn = attach_kprobe),
6291 BPF_PROG_SEC("uretprobe/", BPF_PROG_TYPE_KPROBE),
6292 BPF_PROG_SEC("classifier", BPF_PROG_TYPE_SCHED_CLS),
6293 BPF_PROG_SEC("action", BPF_PROG_TYPE_SCHED_ACT),
6294 SEC_DEF("tracepoint/", TRACEPOINT,
6295 .attach_fn = attach_tp),
6296 SEC_DEF("tp/", TRACEPOINT,
6297 .attach_fn = attach_tp),
6298 SEC_DEF("raw_tracepoint/", RAW_TRACEPOINT,
6299 .attach_fn = attach_raw_tp),
6300 SEC_DEF("raw_tp/", RAW_TRACEPOINT,
6301 .attach_fn = attach_raw_tp),
6302 SEC_DEF("tp_btf/", TRACING,
6303 .expected_attach_type = BPF_TRACE_RAW_TP,
6304 .is_attach_btf = true,
6305 .attach_fn = attach_trace),
6306 SEC_DEF("fentry/", TRACING,
6307 .expected_attach_type = BPF_TRACE_FENTRY,
6308 .is_attach_btf = true,
6309 .attach_fn = attach_trace),
6310 SEC_DEF("fmod_ret/", TRACING,
6311 .expected_attach_type = BPF_MODIFY_RETURN,
6312 .is_attach_btf = true,
6313 .attach_fn = attach_trace),
6314 SEC_DEF("fexit/", TRACING,
6315 .expected_attach_type = BPF_TRACE_FEXIT,
6316 .is_attach_btf = true,
6317 .attach_fn = attach_trace),
6318 SEC_DEF("freplace/", EXT,
6319 .is_attach_btf = true,
6320 .attach_fn = attach_trace),
6321 SEC_DEF("lsm/", LSM,
6322 .is_attach_btf = true,
6323 .expected_attach_type = BPF_LSM_MAC,
6324 .attach_fn = attach_lsm),
6325 BPF_PROG_SEC("xdp", BPF_PROG_TYPE_XDP),
6326 BPF_PROG_SEC("perf_event", BPF_PROG_TYPE_PERF_EVENT),
6327 BPF_PROG_SEC("lwt_in", BPF_PROG_TYPE_LWT_IN),
6328 BPF_PROG_SEC("lwt_out", BPF_PROG_TYPE_LWT_OUT),
6329 BPF_PROG_SEC("lwt_xmit", BPF_PROG_TYPE_LWT_XMIT),
6330 BPF_PROG_SEC("lwt_seg6local", BPF_PROG_TYPE_LWT_SEG6LOCAL),
6331 BPF_APROG_SEC("cgroup_skb/ingress", BPF_PROG_TYPE_CGROUP_SKB,
6332 BPF_CGROUP_INET_INGRESS),
6333 BPF_APROG_SEC("cgroup_skb/egress", BPF_PROG_TYPE_CGROUP_SKB,
6334 BPF_CGROUP_INET_EGRESS),
6335 BPF_APROG_COMPAT("cgroup/skb", BPF_PROG_TYPE_CGROUP_SKB),
6336 BPF_APROG_SEC("cgroup/sock", BPF_PROG_TYPE_CGROUP_SOCK,
6337 BPF_CGROUP_INET_SOCK_CREATE),
6338 BPF_EAPROG_SEC("cgroup/post_bind4", BPF_PROG_TYPE_CGROUP_SOCK,
6339 BPF_CGROUP_INET4_POST_BIND),
6340 BPF_EAPROG_SEC("cgroup/post_bind6", BPF_PROG_TYPE_CGROUP_SOCK,
6341 BPF_CGROUP_INET6_POST_BIND),
6342 BPF_APROG_SEC("cgroup/dev", BPF_PROG_TYPE_CGROUP_DEVICE,
6343 BPF_CGROUP_DEVICE),
6344 BPF_APROG_SEC("sockops", BPF_PROG_TYPE_SOCK_OPS,
6345 BPF_CGROUP_SOCK_OPS),
6346 BPF_APROG_SEC("sk_skb/stream_parser", BPF_PROG_TYPE_SK_SKB,
6347 BPF_SK_SKB_STREAM_PARSER),
6348 BPF_APROG_SEC("sk_skb/stream_verdict", BPF_PROG_TYPE_SK_SKB,
6349 BPF_SK_SKB_STREAM_VERDICT),
6350 BPF_APROG_COMPAT("sk_skb", BPF_PROG_TYPE_SK_SKB),
6351 BPF_APROG_SEC("sk_msg", BPF_PROG_TYPE_SK_MSG,
6352 BPF_SK_MSG_VERDICT),
6353 BPF_APROG_SEC("lirc_mode2", BPF_PROG_TYPE_LIRC_MODE2,
6354 BPF_LIRC_MODE2),
6355 BPF_APROG_SEC("flow_dissector", BPF_PROG_TYPE_FLOW_DISSECTOR,
6356 BPF_FLOW_DISSECTOR),
6357 BPF_EAPROG_SEC("cgroup/bind4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
6358 BPF_CGROUP_INET4_BIND),
6359 BPF_EAPROG_SEC("cgroup/bind6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
6360 BPF_CGROUP_INET6_BIND),
6361 BPF_EAPROG_SEC("cgroup/connect4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
6362 BPF_CGROUP_INET4_CONNECT),
6363 BPF_EAPROG_SEC("cgroup/connect6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
6364 BPF_CGROUP_INET6_CONNECT),
6365 BPF_EAPROG_SEC("cgroup/sendmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
6366 BPF_CGROUP_UDP4_SENDMSG),
6367 BPF_EAPROG_SEC("cgroup/sendmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
6368 BPF_CGROUP_UDP6_SENDMSG),
6369 BPF_EAPROG_SEC("cgroup/recvmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
6370 BPF_CGROUP_UDP4_RECVMSG),
6371 BPF_EAPROG_SEC("cgroup/recvmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
6372 BPF_CGROUP_UDP6_RECVMSG),
6373 BPF_EAPROG_SEC("cgroup/sysctl", BPF_PROG_TYPE_CGROUP_SYSCTL,
6374 BPF_CGROUP_SYSCTL),
6375 BPF_EAPROG_SEC("cgroup/getsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT,
6376 BPF_CGROUP_GETSOCKOPT),
6377 BPF_EAPROG_SEC("cgroup/setsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT,
6378 BPF_CGROUP_SETSOCKOPT),
6379 BPF_PROG_SEC("struct_ops", BPF_PROG_TYPE_STRUCT_OPS),
6380 };
6381
6382 #undef BPF_PROG_SEC_IMPL
6383 #undef BPF_PROG_SEC
6384 #undef BPF_APROG_SEC
6385 #undef BPF_EAPROG_SEC
6386 #undef BPF_APROG_COMPAT
6387 #undef SEC_DEF
6388
6389 #define MAX_TYPE_NAME_SIZE 32
6390
6391 static const struct bpf_sec_def *find_sec_def(const char *sec_name)
6392 {
6393 int i, n = ARRAY_SIZE(section_defs);
6394
6395 for (i = 0; i < n; i++) {
6396 if (strncmp(sec_name,
6397 section_defs[i].sec, section_defs[i].len))
6398 continue;
6399 return &section_defs[i];
6400 }
6401 return NULL;
6402 }
6403
6404 static char *libbpf_get_type_names(bool attach_type)
6405 {
6406 int i, len = ARRAY_SIZE(section_defs) * MAX_TYPE_NAME_SIZE;
6407 char *buf;
6408
6409 buf = malloc(len);
6410 if (!buf)
6411 return NULL;
6412
6413 buf[0] = '\0';
6414 /* Forge string buf with all available names */
6415 for (i = 0; i < ARRAY_SIZE(section_defs); i++) {
6416 if (attach_type && !section_defs[i].is_attachable)
6417 continue;
6418
6419 if (strlen(buf) + strlen(section_defs[i].sec) + 2 > len) {
6420 free(buf);
6421 return NULL;
6422 }
6423 strcat(buf, " ");
6424 strcat(buf, section_defs[i].sec);
6425 }
6426
6427 return buf;
6428 }
6429
6430 int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type,
6431 enum bpf_attach_type *expected_attach_type)
6432 {
6433 const struct bpf_sec_def *sec_def;
6434 char *type_names;
6435
6436 if (!name)
6437 return -EINVAL;
6438
6439 sec_def = find_sec_def(name);
6440 if (sec_def) {
6441 *prog_type = sec_def->prog_type;
6442 *expected_attach_type = sec_def->expected_attach_type;
6443 return 0;
6444 }
6445
6446 pr_debug("failed to guess program type from ELF section '%s'\n", name);
6447 type_names = libbpf_get_type_names(false);
6448 if (type_names != NULL) {
6449 pr_debug("supported section(type) names are:%s\n", type_names);
6450 free(type_names);
6451 }
6452
6453 return -ESRCH;
6454 }
6455
6456 static struct bpf_map *find_struct_ops_map_by_offset(struct bpf_object *obj,
6457 size_t offset)
6458 {
6459 struct bpf_map *map;
6460 size_t i;
6461
6462 for (i = 0; i < obj->nr_maps; i++) {
6463 map = &obj->maps[i];
6464 if (!bpf_map__is_struct_ops(map))
6465 continue;
6466 if (map->sec_offset <= offset &&
6467 offset - map->sec_offset < map->def.value_size)
6468 return map;
6469 }
6470
6471 return NULL;
6472 }
6473
6474 /* Collect the reloc from ELF and populate the st_ops->progs[] */
6475 static int bpf_object__collect_struct_ops_map_reloc(struct bpf_object *obj,
6476 GElf_Shdr *shdr,
6477 Elf_Data *data)
6478 {
6479 const struct btf_member *member;
6480 struct bpf_struct_ops *st_ops;
6481 struct bpf_program *prog;
6482 unsigned int shdr_idx;
6483 const struct btf *btf;
6484 struct bpf_map *map;
6485 Elf_Data *symbols;
6486 unsigned int moff;
6487 const char *name;
6488 __u32 member_idx;
6489 GElf_Sym sym;
6490 GElf_Rel rel;
6491 int i, nrels;
6492
6493 symbols = obj->efile.symbols;
6494 btf = obj->btf;
6495 nrels = shdr->sh_size / shdr->sh_entsize;
6496 for (i = 0; i < nrels; i++) {
6497 if (!gelf_getrel(data, i, &rel)) {
6498 pr_warn("struct_ops reloc: failed to get %d reloc\n", i);
6499 return -LIBBPF_ERRNO__FORMAT;
6500 }
6501
6502 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) {
6503 pr_warn("struct_ops reloc: symbol %zx not found\n",
6504 (size_t)GELF_R_SYM(rel.r_info));
6505 return -LIBBPF_ERRNO__FORMAT;
6506 }
6507
6508 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx,
6509 sym.st_name) ? : "<?>";
6510 map = find_struct_ops_map_by_offset(obj, rel.r_offset);
6511 if (!map) {
6512 pr_warn("struct_ops reloc: cannot find map at rel.r_offset %zu\n",
6513 (size_t)rel.r_offset);
6514 return -EINVAL;
6515 }
6516
6517 moff = rel.r_offset - map->sec_offset;
6518 shdr_idx = sym.st_shndx;
6519 st_ops = map->st_ops;
6520 pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel.r_offset %zu map->sec_offset %zu name %d (\'%s\')\n",
6521 map->name,
6522 (long long)(rel.r_info >> 32),
6523 (long long)sym.st_value,
6524 shdr_idx, (size_t)rel.r_offset,
6525 map->sec_offset, sym.st_name, name);
6526
6527 if (shdr_idx >= SHN_LORESERVE) {
6528 pr_warn("struct_ops reloc %s: rel.r_offset %zu shdr_idx %u unsupported non-static function\n",
6529 map->name, (size_t)rel.r_offset, shdr_idx);
6530 return -LIBBPF_ERRNO__RELOC;
6531 }
6532
6533 member = find_member_by_offset(st_ops->type, moff * 8);
6534 if (!member) {
6535 pr_warn("struct_ops reloc %s: cannot find member at moff %u\n",
6536 map->name, moff);
6537 return -EINVAL;
6538 }
6539 member_idx = member - btf_members(st_ops->type);
6540 name = btf__name_by_offset(btf, member->name_off);
6541
6542 if (!resolve_func_ptr(btf, member->type, NULL)) {
6543 pr_warn("struct_ops reloc %s: cannot relocate non func ptr %s\n",
6544 map->name, name);
6545 return -EINVAL;
6546 }
6547
6548 prog = bpf_object__find_prog_by_idx(obj, shdr_idx);
6549 if (!prog) {
6550 pr_warn("struct_ops reloc %s: cannot find prog at shdr_idx %u to relocate func ptr %s\n",
6551 map->name, shdr_idx, name);
6552 return -EINVAL;
6553 }
6554
6555 if (prog->type == BPF_PROG_TYPE_UNSPEC) {
6556 const struct bpf_sec_def *sec_def;
6557
6558 sec_def = find_sec_def(prog->section_name);
6559 if (sec_def &&
6560 sec_def->prog_type != BPF_PROG_TYPE_STRUCT_OPS) {
6561 /* for pr_warn */
6562 prog->type = sec_def->prog_type;
6563 goto invalid_prog;
6564 }
6565
6566 prog->type = BPF_PROG_TYPE_STRUCT_OPS;
6567 prog->attach_btf_id = st_ops->type_id;
6568 prog->expected_attach_type = member_idx;
6569 } else if (prog->type != BPF_PROG_TYPE_STRUCT_OPS ||
6570 prog->attach_btf_id != st_ops->type_id ||
6571 prog->expected_attach_type != member_idx) {
6572 goto invalid_prog;
6573 }
6574 st_ops->progs[member_idx] = prog;
6575 }
6576
6577 return 0;
6578
6579 invalid_prog:
6580 pr_warn("struct_ops reloc %s: cannot use prog %s in sec %s with type %u attach_btf_id %u expected_attach_type %u for func ptr %s\n",
6581 map->name, prog->name, prog->section_name, prog->type,
6582 prog->attach_btf_id, prog->expected_attach_type, name);
6583 return -EINVAL;
6584 }
6585
6586 #define BTF_TRACE_PREFIX "btf_trace_"
6587 #define BTF_LSM_PREFIX "bpf_lsm_"
6588 #define BTF_MAX_NAME_SIZE 128
6589
6590 static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix,
6591 const char *name, __u32 kind)
6592 {
6593 char btf_type_name[BTF_MAX_NAME_SIZE];
6594 int ret;
6595
6596 ret = snprintf(btf_type_name, sizeof(btf_type_name),
6597 "%s%s", prefix, name);
6598 /* snprintf returns the number of characters written excluding the
6599 * the terminating null. So, if >= BTF_MAX_NAME_SIZE are written, it
6600 * indicates truncation.
6601 */
6602 if (ret < 0 || ret >= sizeof(btf_type_name))
6603 return -ENAMETOOLONG;
6604 return btf__find_by_name_kind(btf, btf_type_name, kind);
6605 }
6606
6607 static inline int __find_vmlinux_btf_id(struct btf *btf, const char *name,
6608 enum bpf_attach_type attach_type)
6609 {
6610 int err;
6611
6612 if (attach_type == BPF_TRACE_RAW_TP)
6613 err = find_btf_by_prefix_kind(btf, BTF_TRACE_PREFIX, name,
6614 BTF_KIND_TYPEDEF);
6615 else if (attach_type == BPF_LSM_MAC)
6616 err = find_btf_by_prefix_kind(btf, BTF_LSM_PREFIX, name,
6617 BTF_KIND_FUNC);
6618 else
6619 err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC);
6620
6621 if (err <= 0)
6622 pr_warn("%s is not found in vmlinux BTF\n", name);
6623
6624 return err;
6625 }
6626
6627 int libbpf_find_vmlinux_btf_id(const char *name,
6628 enum bpf_attach_type attach_type)
6629 {
6630 struct btf *btf;
6631
6632 btf = libbpf_find_kernel_btf();
6633 if (IS_ERR(btf)) {
6634 pr_warn("vmlinux BTF is not found\n");
6635 return -EINVAL;
6636 }
6637
6638 return __find_vmlinux_btf_id(btf, name, attach_type);
6639 }
6640
6641 static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd)
6642 {
6643 struct bpf_prog_info_linear *info_linear;
6644 struct bpf_prog_info *info;
6645 struct btf *btf = NULL;
6646 int err = -EINVAL;
6647
6648 info_linear = bpf_program__get_prog_info_linear(attach_prog_fd, 0);
6649 if (IS_ERR_OR_NULL(info_linear)) {
6650 pr_warn("failed get_prog_info_linear for FD %d\n",
6651 attach_prog_fd);
6652 return -EINVAL;
6653 }
6654 info = &info_linear->info;
6655 if (!info->btf_id) {
6656 pr_warn("The target program doesn't have BTF\n");
6657 goto out;
6658 }
6659 if (btf__get_from_id(info->btf_id, &btf)) {
6660 pr_warn("Failed to get BTF of the program\n");
6661 goto out;
6662 }
6663 err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC);
6664 btf__free(btf);
6665 if (err <= 0) {
6666 pr_warn("%s is not found in prog's BTF\n", name);
6667 goto out;
6668 }
6669 out:
6670 free(info_linear);
6671 return err;
6672 }
6673
6674 static int libbpf_find_attach_btf_id(struct bpf_program *prog)
6675 {
6676 enum bpf_attach_type attach_type = prog->expected_attach_type;
6677 __u32 attach_prog_fd = prog->attach_prog_fd;
6678 const char *name = prog->section_name;
6679 int i, err;
6680
6681 if (!name)
6682 return -EINVAL;
6683
6684 for (i = 0; i < ARRAY_SIZE(section_defs); i++) {
6685 if (!section_defs[i].is_attach_btf)
6686 continue;
6687 if (strncmp(name, section_defs[i].sec, section_defs[i].len))
6688 continue;
6689 if (attach_prog_fd)
6690 err = libbpf_find_prog_btf_id(name + section_defs[i].len,
6691 attach_prog_fd);
6692 else
6693 err = __find_vmlinux_btf_id(prog->obj->btf_vmlinux,
6694 name + section_defs[i].len,
6695 attach_type);
6696 return err;
6697 }
6698 pr_warn("failed to identify btf_id based on ELF section name '%s'\n", name);
6699 return -ESRCH;
6700 }
6701
6702 int libbpf_attach_type_by_name(const char *name,
6703 enum bpf_attach_type *attach_type)
6704 {
6705 char *type_names;
6706 int i;
6707
6708 if (!name)
6709 return -EINVAL;
6710
6711 for (i = 0; i < ARRAY_SIZE(section_defs); i++) {
6712 if (strncmp(name, section_defs[i].sec, section_defs[i].len))
6713 continue;
6714 if (!section_defs[i].is_attachable)
6715 return -EINVAL;
6716 *attach_type = section_defs[i].attach_type;
6717 return 0;
6718 }
6719 pr_debug("failed to guess attach type based on ELF section name '%s'\n", name);
6720 type_names = libbpf_get_type_names(true);
6721 if (type_names != NULL) {
6722 pr_debug("attachable section(type) names are:%s\n", type_names);
6723 free(type_names);
6724 }
6725
6726 return -EINVAL;
6727 }
6728
6729 int bpf_map__fd(const struct bpf_map *map)
6730 {
6731 return map ? map->fd : -EINVAL;
6732 }
6733
6734 const struct bpf_map_def *bpf_map__def(const struct bpf_map *map)
6735 {
6736 return map ? &map->def : ERR_PTR(-EINVAL);
6737 }
6738
6739 const char *bpf_map__name(const struct bpf_map *map)
6740 {
6741 return map ? map->name : NULL;
6742 }
6743
6744 __u32 bpf_map__btf_key_type_id(const struct bpf_map *map)
6745 {
6746 return map ? map->btf_key_type_id : 0;
6747 }
6748
6749 __u32 bpf_map__btf_value_type_id(const struct bpf_map *map)
6750 {
6751 return map ? map->btf_value_type_id : 0;
6752 }
6753
6754 int bpf_map__set_priv(struct bpf_map *map, void *priv,
6755 bpf_map_clear_priv_t clear_priv)
6756 {
6757 if (!map)
6758 return -EINVAL;
6759
6760 if (map->priv) {
6761 if (map->clear_priv)
6762 map->clear_priv(map, map->priv);
6763 }
6764
6765 map->priv = priv;
6766 map->clear_priv = clear_priv;
6767 return 0;
6768 }
6769
6770 void *bpf_map__priv(const struct bpf_map *map)
6771 {
6772 return map ? map->priv : ERR_PTR(-EINVAL);
6773 }
6774
6775 int bpf_map__set_initial_value(struct bpf_map *map,
6776 const void *data, size_t size)
6777 {
6778 if (!map->mmaped || map->libbpf_type == LIBBPF_MAP_KCONFIG ||
6779 size != map->def.value_size || map->fd >= 0)
6780 return -EINVAL;
6781
6782 memcpy(map->mmaped, data, size);
6783 return 0;
6784 }
6785
6786 bool bpf_map__is_offload_neutral(const struct bpf_map *map)
6787 {
6788 return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY;
6789 }
6790
6791 bool bpf_map__is_internal(const struct bpf_map *map)
6792 {
6793 return map->libbpf_type != LIBBPF_MAP_UNSPEC;
6794 }
6795
6796 void bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex)
6797 {
6798 map->map_ifindex = ifindex;
6799 }
6800
6801 int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd)
6802 {
6803 if (!bpf_map_type__is_map_in_map(map->def.type)) {
6804 pr_warn("error: unsupported map type\n");
6805 return -EINVAL;
6806 }
6807 if (map->inner_map_fd != -1) {
6808 pr_warn("error: inner_map_fd already specified\n");
6809 return -EINVAL;
6810 }
6811 map->inner_map_fd = fd;
6812 return 0;
6813 }
6814
6815 static struct bpf_map *
6816 __bpf_map__iter(const struct bpf_map *m, const struct bpf_object *obj, int i)
6817 {
6818 ssize_t idx;
6819 struct bpf_map *s, *e;
6820
6821 if (!obj || !obj->maps)
6822 return NULL;
6823
6824 s = obj->maps;
6825 e = obj->maps + obj->nr_maps;
6826
6827 if ((m < s) || (m >= e)) {
6828 pr_warn("error in %s: map handler doesn't belong to object\n",
6829 __func__);
6830 return NULL;
6831 }
6832
6833 idx = (m - obj->maps) + i;
6834 if (idx >= obj->nr_maps || idx < 0)
6835 return NULL;
6836 return &obj->maps[idx];
6837 }
6838
6839 struct bpf_map *
6840 bpf_map__next(const struct bpf_map *prev, const struct bpf_object *obj)
6841 {
6842 if (prev == NULL)
6843 return obj->maps;
6844
6845 return __bpf_map__iter(prev, obj, 1);
6846 }
6847
6848 struct bpf_map *
6849 bpf_map__prev(const struct bpf_map *next, const struct bpf_object *obj)
6850 {
6851 if (next == NULL) {
6852 if (!obj->nr_maps)
6853 return NULL;
6854 return obj->maps + obj->nr_maps - 1;
6855 }
6856
6857 return __bpf_map__iter(next, obj, -1);
6858 }
6859
6860 struct bpf_map *
6861 bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name)
6862 {
6863 struct bpf_map *pos;
6864
6865 bpf_object__for_each_map(pos, obj) {
6866 if (pos->name && !strcmp(pos->name, name))
6867 return pos;
6868 }
6869 return NULL;
6870 }
6871
6872 int
6873 bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name)
6874 {
6875 return bpf_map__fd(bpf_object__find_map_by_name(obj, name));
6876 }
6877
6878 struct bpf_map *
6879 bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset)
6880 {
6881 return ERR_PTR(-ENOTSUP);
6882 }
6883
6884 long libbpf_get_error(const void *ptr)
6885 {
6886 return PTR_ERR_OR_ZERO(ptr);
6887 }
6888
6889 int bpf_prog_load(const char *file, enum bpf_prog_type type,
6890 struct bpf_object **pobj, int *prog_fd)
6891 {
6892 struct bpf_prog_load_attr attr;
6893
6894 memset(&attr, 0, sizeof(struct bpf_prog_load_attr));
6895 attr.file = file;
6896 attr.prog_type = type;
6897 attr.expected_attach_type = 0;
6898
6899 return bpf_prog_load_xattr(&attr, pobj, prog_fd);
6900 }
6901
6902 int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr,
6903 struct bpf_object **pobj, int *prog_fd)
6904 {
6905 struct bpf_object_open_attr open_attr = {};
6906 struct bpf_program *prog, *first_prog = NULL;
6907 struct bpf_object *obj;
6908 struct bpf_map *map;
6909 int err;
6910
6911 if (!attr)
6912 return -EINVAL;
6913 if (!attr->file)
6914 return -EINVAL;
6915
6916 open_attr.file = attr->file;
6917 open_attr.prog_type = attr->prog_type;
6918
6919 obj = bpf_object__open_xattr(&open_attr);
6920 if (IS_ERR_OR_NULL(obj))
6921 return -ENOENT;
6922
6923 bpf_object__for_each_program(prog, obj) {
6924 enum bpf_attach_type attach_type = attr->expected_attach_type;
6925 /*
6926 * to preserve backwards compatibility, bpf_prog_load treats
6927 * attr->prog_type, if specified, as an override to whatever
6928 * bpf_object__open guessed
6929 */
6930 if (attr->prog_type != BPF_PROG_TYPE_UNSPEC) {
6931 bpf_program__set_type(prog, attr->prog_type);
6932 bpf_program__set_expected_attach_type(prog,
6933 attach_type);
6934 }
6935 if (bpf_program__get_type(prog) == BPF_PROG_TYPE_UNSPEC) {
6936 /*
6937 * we haven't guessed from section name and user
6938 * didn't provide a fallback type, too bad...
6939 */
6940 bpf_object__close(obj);
6941 return -EINVAL;
6942 }
6943
6944 prog->prog_ifindex = attr->ifindex;
6945 prog->log_level = attr->log_level;
6946 prog->prog_flags = attr->prog_flags;
6947 if (!first_prog)
6948 first_prog = prog;
6949 }
6950
6951 bpf_object__for_each_map(map, obj) {
6952 if (!bpf_map__is_offload_neutral(map))
6953 map->map_ifindex = attr->ifindex;
6954 }
6955
6956 if (!first_prog) {
6957 pr_warn("object file doesn't contain bpf program\n");
6958 bpf_object__close(obj);
6959 return -ENOENT;
6960 }
6961
6962 err = bpf_object__load(obj);
6963 if (err) {
6964 bpf_object__close(obj);
6965 return -EINVAL;
6966 }
6967
6968 *pobj = obj;
6969 *prog_fd = bpf_program__fd(first_prog);
6970 return 0;
6971 }
6972
6973 struct bpf_link {
6974 int (*detach)(struct bpf_link *link);
6975 int (*destroy)(struct bpf_link *link);
6976 char *pin_path; /* NULL, if not pinned */
6977 int fd; /* hook FD, -1 if not applicable */
6978 bool disconnected;
6979 };
6980
6981 /* Replace link's underlying BPF program with the new one */
6982 int bpf_link__update_program(struct bpf_link *link, struct bpf_program *prog)
6983 {
6984 return bpf_link_update(bpf_link__fd(link), bpf_program__fd(prog), NULL);
6985 }
6986
6987 /* Release "ownership" of underlying BPF resource (typically, BPF program
6988 * attached to some BPF hook, e.g., tracepoint, kprobe, etc). Disconnected
6989 * link, when destructed through bpf_link__destroy() call won't attempt to
6990 * detach/unregisted that BPF resource. This is useful in situations where,
6991 * say, attached BPF program has to outlive userspace program that attached it
6992 * in the system. Depending on type of BPF program, though, there might be
6993 * additional steps (like pinning BPF program in BPF FS) necessary to ensure
6994 * exit of userspace program doesn't trigger automatic detachment and clean up
6995 * inside the kernel.
6996 */
6997 void bpf_link__disconnect(struct bpf_link *link)
6998 {
6999 link->disconnected = true;
7000 }
7001
7002 int bpf_link__destroy(struct bpf_link *link)
7003 {
7004 int err = 0;
7005
7006 if (!link)
7007 return 0;
7008
7009 if (!link->disconnected && link->detach)
7010 err = link->detach(link);
7011 if (link->destroy)
7012 link->destroy(link);
7013 if (link->pin_path)
7014 free(link->pin_path);
7015 free(link);
7016
7017 return err;
7018 }
7019
7020 int bpf_link__fd(const struct bpf_link *link)
7021 {
7022 return link->fd;
7023 }
7024
7025 const char *bpf_link__pin_path(const struct bpf_link *link)
7026 {
7027 return link->pin_path;
7028 }
7029
7030 static int bpf_link__detach_fd(struct bpf_link *link)
7031 {
7032 return close(link->fd);
7033 }
7034
7035 struct bpf_link *bpf_link__open(const char *path)
7036 {
7037 struct bpf_link *link;
7038 int fd;
7039
7040 fd = bpf_obj_get(path);
7041 if (fd < 0) {
7042 fd = -errno;
7043 pr_warn("failed to open link at %s: %d\n", path, fd);
7044 return ERR_PTR(fd);
7045 }
7046
7047 link = calloc(1, sizeof(*link));
7048 if (!link) {
7049 close(fd);
7050 return ERR_PTR(-ENOMEM);
7051 }
7052 link->detach = &bpf_link__detach_fd;
7053 link->fd = fd;
7054
7055 link->pin_path = strdup(path);
7056 if (!link->pin_path) {
7057 bpf_link__destroy(link);
7058 return ERR_PTR(-ENOMEM);
7059 }
7060
7061 return link;
7062 }
7063
7064 int bpf_link__pin(struct bpf_link *link, const char *path)
7065 {
7066 int err;
7067
7068 if (link->pin_path)
7069 return -EBUSY;
7070 err = make_parent_dir(path);
7071 if (err)
7072 return err;
7073 err = check_path(path);
7074 if (err)
7075 return err;
7076
7077 link->pin_path = strdup(path);
7078 if (!link->pin_path)
7079 return -ENOMEM;
7080
7081 if (bpf_obj_pin(link->fd, link->pin_path)) {
7082 err = -errno;
7083 zfree(&link->pin_path);
7084 return err;
7085 }
7086
7087 pr_debug("link fd=%d: pinned at %s\n", link->fd, link->pin_path);
7088 return 0;
7089 }
7090
7091 int bpf_link__unpin(struct bpf_link *link)
7092 {
7093 int err;
7094
7095 if (!link->pin_path)
7096 return -EINVAL;
7097
7098 err = unlink(link->pin_path);
7099 if (err != 0)
7100 return -errno;
7101
7102 pr_debug("link fd=%d: unpinned from %s\n", link->fd, link->pin_path);
7103 zfree(&link->pin_path);
7104 return 0;
7105 }
7106
7107 static int bpf_link__detach_perf_event(struct bpf_link *link)
7108 {
7109 int err;
7110
7111 err = ioctl(link->fd, PERF_EVENT_IOC_DISABLE, 0);
7112 if (err)
7113 err = -errno;
7114
7115 close(link->fd);
7116 return err;
7117 }
7118
7119 struct bpf_link *bpf_program__attach_perf_event(struct bpf_program *prog,
7120 int pfd)
7121 {
7122 char errmsg[STRERR_BUFSIZE];
7123 struct bpf_link *link;
7124 int prog_fd, err;
7125
7126 if (pfd < 0) {
7127 pr_warn("program '%s': invalid perf event FD %d\n",
7128 bpf_program__title(prog, false), pfd);
7129 return ERR_PTR(-EINVAL);
7130 }
7131 prog_fd = bpf_program__fd(prog);
7132 if (prog_fd < 0) {
7133 pr_warn("program '%s': can't attach BPF program w/o FD (did you load it?)\n",
7134 bpf_program__title(prog, false));
7135 return ERR_PTR(-EINVAL);
7136 }
7137
7138 link = calloc(1, sizeof(*link));
7139 if (!link)
7140 return ERR_PTR(-ENOMEM);
7141 link->detach = &bpf_link__detach_perf_event;
7142 link->fd = pfd;
7143
7144 if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) {
7145 err = -errno;
7146 free(link);
7147 pr_warn("program '%s': failed to attach to pfd %d: %s\n",
7148 bpf_program__title(prog, false), pfd,
7149 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
7150 return ERR_PTR(err);
7151 }
7152 if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
7153 err = -errno;
7154 free(link);
7155 pr_warn("program '%s': failed to enable pfd %d: %s\n",
7156 bpf_program__title(prog, false), pfd,
7157 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
7158 return ERR_PTR(err);
7159 }
7160 return link;
7161 }
7162
7163 /*
7164 * this function is expected to parse integer in the range of [0, 2^31-1] from
7165 * given file using scanf format string fmt. If actual parsed value is
7166 * negative, the result might be indistinguishable from error
7167 */
7168 static int parse_uint_from_file(const char *file, const char *fmt)
7169 {
7170 char buf[STRERR_BUFSIZE];
7171 int err, ret;
7172 FILE *f;
7173
7174 f = fopen(file, "r");
7175 if (!f) {
7176 err = -errno;
7177 pr_debug("failed to open '%s': %s\n", file,
7178 libbpf_strerror_r(err, buf, sizeof(buf)));
7179 return err;
7180 }
7181 err = fscanf(f, fmt, &ret);
7182 if (err != 1) {
7183 err = err == EOF ? -EIO : -errno;
7184 pr_debug("failed to parse '%s': %s\n", file,
7185 libbpf_strerror_r(err, buf, sizeof(buf)));
7186 fclose(f);
7187 return err;
7188 }
7189 fclose(f);
7190 return ret;
7191 }
7192
7193 static int determine_kprobe_perf_type(void)
7194 {
7195 const char *file = "/sys/bus/event_source/devices/kprobe/type";
7196
7197 return parse_uint_from_file(file, "%d\n");
7198 }
7199
7200 static int determine_uprobe_perf_type(void)
7201 {
7202 const char *file = "/sys/bus/event_source/devices/uprobe/type";
7203
7204 return parse_uint_from_file(file, "%d\n");
7205 }
7206
7207 static int determine_kprobe_retprobe_bit(void)
7208 {
7209 const char *file = "/sys/bus/event_source/devices/kprobe/format/retprobe";
7210
7211 return parse_uint_from_file(file, "config:%d\n");
7212 }
7213
7214 static int determine_uprobe_retprobe_bit(void)
7215 {
7216 const char *file = "/sys/bus/event_source/devices/uprobe/format/retprobe";
7217
7218 return parse_uint_from_file(file, "config:%d\n");
7219 }
7220
7221 static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name,
7222 uint64_t offset, int pid)
7223 {
7224 struct perf_event_attr attr = {};
7225 char errmsg[STRERR_BUFSIZE];
7226 int type, pfd, err;
7227
7228 type = uprobe ? determine_uprobe_perf_type()
7229 : determine_kprobe_perf_type();
7230 if (type < 0) {
7231 pr_warn("failed to determine %s perf type: %s\n",
7232 uprobe ? "uprobe" : "kprobe",
7233 libbpf_strerror_r(type, errmsg, sizeof(errmsg)));
7234 return type;
7235 }
7236 if (retprobe) {
7237 int bit = uprobe ? determine_uprobe_retprobe_bit()
7238 : determine_kprobe_retprobe_bit();
7239
7240 if (bit < 0) {
7241 pr_warn("failed to determine %s retprobe bit: %s\n",
7242 uprobe ? "uprobe" : "kprobe",
7243 libbpf_strerror_r(bit, errmsg, sizeof(errmsg)));
7244 return bit;
7245 }
7246 attr.config |= 1 << bit;
7247 }
7248 attr.size = sizeof(attr);
7249 attr.type = type;
7250 attr.config1 = ptr_to_u64(name); /* kprobe_func or uprobe_path */
7251 attr.config2 = offset; /* kprobe_addr or probe_offset */
7252
7253 /* pid filter is meaningful only for uprobes */
7254 pfd = syscall(__NR_perf_event_open, &attr,
7255 pid < 0 ? -1 : pid /* pid */,
7256 pid == -1 ? 0 : -1 /* cpu */,
7257 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
7258 if (pfd < 0) {
7259 err = -errno;
7260 pr_warn("%s perf_event_open() failed: %s\n",
7261 uprobe ? "uprobe" : "kprobe",
7262 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
7263 return err;
7264 }
7265 return pfd;
7266 }
7267
7268 struct bpf_link *bpf_program__attach_kprobe(struct bpf_program *prog,
7269 bool retprobe,
7270 const char *func_name)
7271 {
7272 char errmsg[STRERR_BUFSIZE];
7273 struct bpf_link *link;
7274 int pfd, err;
7275
7276 pfd = perf_event_open_probe(false /* uprobe */, retprobe, func_name,
7277 0 /* offset */, -1 /* pid */);
7278 if (pfd < 0) {
7279 pr_warn("program '%s': failed to create %s '%s' perf event: %s\n",
7280 bpf_program__title(prog, false),
7281 retprobe ? "kretprobe" : "kprobe", func_name,
7282 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
7283 return ERR_PTR(pfd);
7284 }
7285 link = bpf_program__attach_perf_event(prog, pfd);
7286 if (IS_ERR(link)) {
7287 close(pfd);
7288 err = PTR_ERR(link);
7289 pr_warn("program '%s': failed to attach to %s '%s': %s\n",
7290 bpf_program__title(prog, false),
7291 retprobe ? "kretprobe" : "kprobe", func_name,
7292 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
7293 return link;
7294 }
7295 return link;
7296 }
7297
7298 static struct bpf_link *attach_kprobe(const struct bpf_sec_def *sec,
7299 struct bpf_program *prog)
7300 {
7301 const char *func_name;
7302 bool retprobe;
7303
7304 func_name = bpf_program__title(prog, false) + sec->len;
7305 retprobe = strcmp(sec->sec, "kretprobe/") == 0;
7306
7307 return bpf_program__attach_kprobe(prog, retprobe, func_name);
7308 }
7309
7310 struct bpf_link *bpf_program__attach_uprobe(struct bpf_program *prog,
7311 bool retprobe, pid_t pid,
7312 const char *binary_path,
7313 size_t func_offset)
7314 {
7315 char errmsg[STRERR_BUFSIZE];
7316 struct bpf_link *link;
7317 int pfd, err;
7318
7319 pfd = perf_event_open_probe(true /* uprobe */, retprobe,
7320 binary_path, func_offset, pid);
7321 if (pfd < 0) {
7322 pr_warn("program '%s': failed to create %s '%s:0x%zx' perf event: %s\n",
7323 bpf_program__title(prog, false),
7324 retprobe ? "uretprobe" : "uprobe",
7325 binary_path, func_offset,
7326 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
7327 return ERR_PTR(pfd);
7328 }
7329 link = bpf_program__attach_perf_event(prog, pfd);
7330 if (IS_ERR(link)) {
7331 close(pfd);
7332 err = PTR_ERR(link);
7333 pr_warn("program '%s': failed to attach to %s '%s:0x%zx': %s\n",
7334 bpf_program__title(prog, false),
7335 retprobe ? "uretprobe" : "uprobe",
7336 binary_path, func_offset,
7337 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
7338 return link;
7339 }
7340 return link;
7341 }
7342
7343 static int determine_tracepoint_id(const char *tp_category,
7344 const char *tp_name)
7345 {
7346 char file[PATH_MAX];
7347 int ret;
7348
7349 ret = snprintf(file, sizeof(file),
7350 "/sys/kernel/debug/tracing/events/%s/%s/id",
7351 tp_category, tp_name);
7352 if (ret < 0)
7353 return -errno;
7354 if (ret >= sizeof(file)) {
7355 pr_debug("tracepoint %s/%s path is too long\n",
7356 tp_category, tp_name);
7357 return -E2BIG;
7358 }
7359 return parse_uint_from_file(file, "%d\n");
7360 }
7361
7362 static int perf_event_open_tracepoint(const char *tp_category,
7363 const char *tp_name)
7364 {
7365 struct perf_event_attr attr = {};
7366 char errmsg[STRERR_BUFSIZE];
7367 int tp_id, pfd, err;
7368
7369 tp_id = determine_tracepoint_id(tp_category, tp_name);
7370 if (tp_id < 0) {
7371 pr_warn("failed to determine tracepoint '%s/%s' perf event ID: %s\n",
7372 tp_category, tp_name,
7373 libbpf_strerror_r(tp_id, errmsg, sizeof(errmsg)));
7374 return tp_id;
7375 }
7376
7377 attr.type = PERF_TYPE_TRACEPOINT;
7378 attr.size = sizeof(attr);
7379 attr.config = tp_id;
7380
7381 pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */,
7382 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
7383 if (pfd < 0) {
7384 err = -errno;
7385 pr_warn("tracepoint '%s/%s' perf_event_open() failed: %s\n",
7386 tp_category, tp_name,
7387 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
7388 return err;
7389 }
7390 return pfd;
7391 }
7392
7393 struct bpf_link *bpf_program__attach_tracepoint(struct bpf_program *prog,
7394 const char *tp_category,
7395 const char *tp_name)
7396 {
7397 char errmsg[STRERR_BUFSIZE];
7398 struct bpf_link *link;
7399 int pfd, err;
7400
7401 pfd = perf_event_open_tracepoint(tp_category, tp_name);
7402 if (pfd < 0) {
7403 pr_warn("program '%s': failed to create tracepoint '%s/%s' perf event: %s\n",
7404 bpf_program__title(prog, false),
7405 tp_category, tp_name,
7406 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
7407 return ERR_PTR(pfd);
7408 }
7409 link = bpf_program__attach_perf_event(prog, pfd);
7410 if (IS_ERR(link)) {
7411 close(pfd);
7412 err = PTR_ERR(link);
7413 pr_warn("program '%s': failed to attach to tracepoint '%s/%s': %s\n",
7414 bpf_program__title(prog, false),
7415 tp_category, tp_name,
7416 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
7417 return link;
7418 }
7419 return link;
7420 }
7421
7422 static struct bpf_link *attach_tp(const struct bpf_sec_def *sec,
7423 struct bpf_program *prog)
7424 {
7425 char *sec_name, *tp_cat, *tp_name;
7426 struct bpf_link *link;
7427
7428 sec_name = strdup(bpf_program__title(prog, false));
7429 if (!sec_name)
7430 return ERR_PTR(-ENOMEM);
7431
7432 /* extract "tp/<category>/<name>" */
7433 tp_cat = sec_name + sec->len;
7434 tp_name = strchr(tp_cat, '/');
7435 if (!tp_name) {
7436 link = ERR_PTR(-EINVAL);
7437 goto out;
7438 }
7439 *tp_name = '\0';
7440 tp_name++;
7441
7442 link = bpf_program__attach_tracepoint(prog, tp_cat, tp_name);
7443 out:
7444 free(sec_name);
7445 return link;
7446 }
7447
7448 struct bpf_link *bpf_program__attach_raw_tracepoint(struct bpf_program *prog,
7449 const char *tp_name)
7450 {
7451 char errmsg[STRERR_BUFSIZE];
7452 struct bpf_link *link;
7453 int prog_fd, pfd;
7454
7455 prog_fd = bpf_program__fd(prog);
7456 if (prog_fd < 0) {
7457 pr_warn("program '%s': can't attach before loaded\n",
7458 bpf_program__title(prog, false));
7459 return ERR_PTR(-EINVAL);
7460 }
7461
7462 link = calloc(1, sizeof(*link));
7463 if (!link)
7464 return ERR_PTR(-ENOMEM);
7465 link->detach = &bpf_link__detach_fd;
7466
7467 pfd = bpf_raw_tracepoint_open(tp_name, prog_fd);
7468 if (pfd < 0) {
7469 pfd = -errno;
7470 free(link);
7471 pr_warn("program '%s': failed to attach to raw tracepoint '%s': %s\n",
7472 bpf_program__title(prog, false), tp_name,
7473 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
7474 return ERR_PTR(pfd);
7475 }
7476 link->fd = pfd;
7477 return link;
7478 }
7479
7480 static struct bpf_link *attach_raw_tp(const struct bpf_sec_def *sec,
7481 struct bpf_program *prog)
7482 {
7483 const char *tp_name = bpf_program__title(prog, false) + sec->len;
7484
7485 return bpf_program__attach_raw_tracepoint(prog, tp_name);
7486 }
7487
7488 /* Common logic for all BPF program types that attach to a btf_id */
7489 static struct bpf_link *bpf_program__attach_btf_id(struct bpf_program *prog)
7490 {
7491 char errmsg[STRERR_BUFSIZE];
7492 struct bpf_link *link;
7493 int prog_fd, pfd;
7494
7495 prog_fd = bpf_program__fd(prog);
7496 if (prog_fd < 0) {
7497 pr_warn("program '%s': can't attach before loaded\n",
7498 bpf_program__title(prog, false));
7499 return ERR_PTR(-EINVAL);
7500 }
7501
7502 link = calloc(1, sizeof(*link));
7503 if (!link)
7504 return ERR_PTR(-ENOMEM);
7505 link->detach = &bpf_link__detach_fd;
7506
7507 pfd = bpf_raw_tracepoint_open(NULL, prog_fd);
7508 if (pfd < 0) {
7509 pfd = -errno;
7510 free(link);
7511 pr_warn("program '%s': failed to attach: %s\n",
7512 bpf_program__title(prog, false),
7513 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
7514 return ERR_PTR(pfd);
7515 }
7516 link->fd = pfd;
7517 return (struct bpf_link *)link;
7518 }
7519
7520 struct bpf_link *bpf_program__attach_trace(struct bpf_program *prog)
7521 {
7522 return bpf_program__attach_btf_id(prog);
7523 }
7524
7525 struct bpf_link *bpf_program__attach_lsm(struct bpf_program *prog)
7526 {
7527 return bpf_program__attach_btf_id(prog);
7528 }
7529
7530 static struct bpf_link *attach_trace(const struct bpf_sec_def *sec,
7531 struct bpf_program *prog)
7532 {
7533 return bpf_program__attach_trace(prog);
7534 }
7535
7536 static struct bpf_link *attach_lsm(const struct bpf_sec_def *sec,
7537 struct bpf_program *prog)
7538 {
7539 return bpf_program__attach_lsm(prog);
7540 }
7541
7542 struct bpf_link *
7543 bpf_program__attach_cgroup(struct bpf_program *prog, int cgroup_fd)
7544 {
7545 const struct bpf_sec_def *sec_def;
7546 enum bpf_attach_type attach_type;
7547 char errmsg[STRERR_BUFSIZE];
7548 struct bpf_link *link;
7549 int prog_fd, link_fd;
7550
7551 prog_fd = bpf_program__fd(prog);
7552 if (prog_fd < 0) {
7553 pr_warn("program '%s': can't attach before loaded\n",
7554 bpf_program__title(prog, false));
7555 return ERR_PTR(-EINVAL);
7556 }
7557
7558 link = calloc(1, sizeof(*link));
7559 if (!link)
7560 return ERR_PTR(-ENOMEM);
7561 link->detach = &bpf_link__detach_fd;
7562
7563 attach_type = bpf_program__get_expected_attach_type(prog);
7564 if (!attach_type) {
7565 sec_def = find_sec_def(bpf_program__title(prog, false));
7566 if (sec_def)
7567 attach_type = sec_def->attach_type;
7568 }
7569 link_fd = bpf_link_create(prog_fd, cgroup_fd, attach_type, NULL);
7570 if (link_fd < 0) {
7571 link_fd = -errno;
7572 free(link);
7573 pr_warn("program '%s': failed to attach to cgroup: %s\n",
7574 bpf_program__title(prog, false),
7575 libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
7576 return ERR_PTR(link_fd);
7577 }
7578 link->fd = link_fd;
7579 return link;
7580 }
7581
7582 struct bpf_link *bpf_program__attach(struct bpf_program *prog)
7583 {
7584 const struct bpf_sec_def *sec_def;
7585
7586 sec_def = find_sec_def(bpf_program__title(prog, false));
7587 if (!sec_def || !sec_def->attach_fn)
7588 return ERR_PTR(-ESRCH);
7589
7590 return sec_def->attach_fn(sec_def, prog);
7591 }
7592
7593 static int bpf_link__detach_struct_ops(struct bpf_link *link)
7594 {
7595 __u32 zero = 0;
7596
7597 if (bpf_map_delete_elem(link->fd, &zero))
7598 return -errno;
7599
7600 return 0;
7601 }
7602
7603 struct bpf_link *bpf_map__attach_struct_ops(struct bpf_map *map)
7604 {
7605 struct bpf_struct_ops *st_ops;
7606 struct bpf_link *link;
7607 __u32 i, zero = 0;
7608 int err;
7609
7610 if (!bpf_map__is_struct_ops(map) || map->fd == -1)
7611 return ERR_PTR(-EINVAL);
7612
7613 link = calloc(1, sizeof(*link));
7614 if (!link)
7615 return ERR_PTR(-EINVAL);
7616
7617 st_ops = map->st_ops;
7618 for (i = 0; i < btf_vlen(st_ops->type); i++) {
7619 struct bpf_program *prog = st_ops->progs[i];
7620 void *kern_data;
7621 int prog_fd;
7622
7623 if (!prog)
7624 continue;
7625
7626 prog_fd = bpf_program__fd(prog);
7627 kern_data = st_ops->kern_vdata + st_ops->kern_func_off[i];
7628 *(unsigned long *)kern_data = prog_fd;
7629 }
7630
7631 err = bpf_map_update_elem(map->fd, &zero, st_ops->kern_vdata, 0);
7632 if (err) {
7633 err = -errno;
7634 free(link);
7635 return ERR_PTR(err);
7636 }
7637
7638 link->detach = bpf_link__detach_struct_ops;
7639 link->fd = map->fd;
7640
7641 return link;
7642 }
7643
7644 enum bpf_perf_event_ret
7645 bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size,
7646 void **copy_mem, size_t *copy_size,
7647 bpf_perf_event_print_t fn, void *private_data)
7648 {
7649 struct perf_event_mmap_page *header = mmap_mem;
7650 __u64 data_head = ring_buffer_read_head(header);
7651 __u64 data_tail = header->data_tail;
7652 void *base = ((__u8 *)header) + page_size;
7653 int ret = LIBBPF_PERF_EVENT_CONT;
7654 struct perf_event_header *ehdr;
7655 size_t ehdr_size;
7656
7657 while (data_head != data_tail) {
7658 ehdr = base + (data_tail & (mmap_size - 1));
7659 ehdr_size = ehdr->size;
7660
7661 if (((void *)ehdr) + ehdr_size > base + mmap_size) {
7662 void *copy_start = ehdr;
7663 size_t len_first = base + mmap_size - copy_start;
7664 size_t len_secnd = ehdr_size - len_first;
7665
7666 if (*copy_size < ehdr_size) {
7667 free(*copy_mem);
7668 *copy_mem = malloc(ehdr_size);
7669 if (!*copy_mem) {
7670 *copy_size = 0;
7671 ret = LIBBPF_PERF_EVENT_ERROR;
7672 break;
7673 }
7674 *copy_size = ehdr_size;
7675 }
7676
7677 memcpy(*copy_mem, copy_start, len_first);
7678 memcpy(*copy_mem + len_first, base, len_secnd);
7679 ehdr = *copy_mem;
7680 }
7681
7682 ret = fn(ehdr, private_data);
7683 data_tail += ehdr_size;
7684 if (ret != LIBBPF_PERF_EVENT_CONT)
7685 break;
7686 }
7687
7688 ring_buffer_write_tail(header, data_tail);
7689 return ret;
7690 }
7691
7692 struct perf_buffer;
7693
7694 struct perf_buffer_params {
7695 struct perf_event_attr *attr;
7696 /* if event_cb is specified, it takes precendence */
7697 perf_buffer_event_fn event_cb;
7698 /* sample_cb and lost_cb are higher-level common-case callbacks */
7699 perf_buffer_sample_fn sample_cb;
7700 perf_buffer_lost_fn lost_cb;
7701 void *ctx;
7702 int cpu_cnt;
7703 int *cpus;
7704 int *map_keys;
7705 };
7706
7707 struct perf_cpu_buf {
7708 struct perf_buffer *pb;
7709 void *base; /* mmap()'ed memory */
7710 void *buf; /* for reconstructing segmented data */
7711 size_t buf_size;
7712 int fd;
7713 int cpu;
7714 int map_key;
7715 };
7716
7717 struct perf_buffer {
7718 perf_buffer_event_fn event_cb;
7719 perf_buffer_sample_fn sample_cb;
7720 perf_buffer_lost_fn lost_cb;
7721 void *ctx; /* passed into callbacks */
7722
7723 size_t page_size;
7724 size_t mmap_size;
7725 struct perf_cpu_buf **cpu_bufs;
7726 struct epoll_event *events;
7727 int cpu_cnt; /* number of allocated CPU buffers */
7728 int epoll_fd; /* perf event FD */
7729 int map_fd; /* BPF_MAP_TYPE_PERF_EVENT_ARRAY BPF map FD */
7730 };
7731
7732 static void perf_buffer__free_cpu_buf(struct perf_buffer *pb,
7733 struct perf_cpu_buf *cpu_buf)
7734 {
7735 if (!cpu_buf)
7736 return;
7737 if (cpu_buf->base &&
7738 munmap(cpu_buf->base, pb->mmap_size + pb->page_size))
7739 pr_warn("failed to munmap cpu_buf #%d\n", cpu_buf->cpu);
7740 if (cpu_buf->fd >= 0) {
7741 ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0);
7742 close(cpu_buf->fd);
7743 }
7744 free(cpu_buf->buf);
7745 free(cpu_buf);
7746 }
7747
7748 void perf_buffer__free(struct perf_buffer *pb)
7749 {
7750 int i;
7751
7752 if (!pb)
7753 return;
7754 if (pb->cpu_bufs) {
7755 for (i = 0; i < pb->cpu_cnt && pb->cpu_bufs[i]; i++) {
7756 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];
7757
7758 bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key);
7759 perf_buffer__free_cpu_buf(pb, cpu_buf);
7760 }
7761 free(pb->cpu_bufs);
7762 }
7763 if (pb->epoll_fd >= 0)
7764 close(pb->epoll_fd);
7765 free(pb->events);
7766 free(pb);
7767 }
7768
7769 static struct perf_cpu_buf *
7770 perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr,
7771 int cpu, int map_key)
7772 {
7773 struct perf_cpu_buf *cpu_buf;
7774 char msg[STRERR_BUFSIZE];
7775 int err;
7776
7777 cpu_buf = calloc(1, sizeof(*cpu_buf));
7778 if (!cpu_buf)
7779 return ERR_PTR(-ENOMEM);
7780
7781 cpu_buf->pb = pb;
7782 cpu_buf->cpu = cpu;
7783 cpu_buf->map_key = map_key;
7784
7785 cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 /* pid */, cpu,
7786 -1, PERF_FLAG_FD_CLOEXEC);
7787 if (cpu_buf->fd < 0) {
7788 err = -errno;
7789 pr_warn("failed to open perf buffer event on cpu #%d: %s\n",
7790 cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
7791 goto error;
7792 }
7793
7794 cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size,
7795 PROT_READ | PROT_WRITE, MAP_SHARED,
7796 cpu_buf->fd, 0);
7797 if (cpu_buf->base == MAP_FAILED) {
7798 cpu_buf->base = NULL;
7799 err = -errno;
7800 pr_warn("failed to mmap perf buffer on cpu #%d: %s\n",
7801 cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
7802 goto error;
7803 }
7804
7805 if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
7806 err = -errno;
7807 pr_warn("failed to enable perf buffer event on cpu #%d: %s\n",
7808 cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
7809 goto error;
7810 }
7811
7812 return cpu_buf;
7813
7814 error:
7815 perf_buffer__free_cpu_buf(pb, cpu_buf);
7816 return (struct perf_cpu_buf *)ERR_PTR(err);
7817 }
7818
7819 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt,
7820 struct perf_buffer_params *p);
7821
7822 struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt,
7823 const struct perf_buffer_opts *opts)
7824 {
7825 struct perf_buffer_params p = {};
7826 struct perf_event_attr attr = { 0, };
7827
7828 attr.config = PERF_COUNT_SW_BPF_OUTPUT,
7829 attr.type = PERF_TYPE_SOFTWARE;
7830 attr.sample_type = PERF_SAMPLE_RAW;
7831 attr.sample_period = 1;
7832 attr.wakeup_events = 1;
7833
7834 p.attr = &attr;
7835 p.sample_cb = opts ? opts->sample_cb : NULL;
7836 p.lost_cb = opts ? opts->lost_cb : NULL;
7837 p.ctx = opts ? opts->ctx : NULL;
7838
7839 return __perf_buffer__new(map_fd, page_cnt, &p);
7840 }
7841
7842 struct perf_buffer *
7843 perf_buffer__new_raw(int map_fd, size_t page_cnt,
7844 const struct perf_buffer_raw_opts *opts)
7845 {
7846 struct perf_buffer_params p = {};
7847
7848 p.attr = opts->attr;
7849 p.event_cb = opts->event_cb;
7850 p.ctx = opts->ctx;
7851 p.cpu_cnt = opts->cpu_cnt;
7852 p.cpus = opts->cpus;
7853 p.map_keys = opts->map_keys;
7854
7855 return __perf_buffer__new(map_fd, page_cnt, &p);
7856 }
7857
7858 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt,
7859 struct perf_buffer_params *p)
7860 {
7861 const char *online_cpus_file = "/sys/devices/system/cpu/online";
7862 struct bpf_map_info map = {};
7863 char msg[STRERR_BUFSIZE];
7864 struct perf_buffer *pb;
7865 bool *online = NULL;
7866 __u32 map_info_len;
7867 int err, i, j, n;
7868
7869 if (page_cnt & (page_cnt - 1)) {
7870 pr_warn("page count should be power of two, but is %zu\n",
7871 page_cnt);
7872 return ERR_PTR(-EINVAL);
7873 }
7874
7875 map_info_len = sizeof(map);
7876 err = bpf_obj_get_info_by_fd(map_fd, &map, &map_info_len);
7877 if (err) {
7878 err = -errno;
7879 pr_warn("failed to get map info for map FD %d: %s\n",
7880 map_fd, libbpf_strerror_r(err, msg, sizeof(msg)));
7881 return ERR_PTR(err);
7882 }
7883
7884 if (map.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) {
7885 pr_warn("map '%s' should be BPF_MAP_TYPE_PERF_EVENT_ARRAY\n",
7886 map.name);
7887 return ERR_PTR(-EINVAL);
7888 }
7889
7890 pb = calloc(1, sizeof(*pb));
7891 if (!pb)
7892 return ERR_PTR(-ENOMEM);
7893
7894 pb->event_cb = p->event_cb;
7895 pb->sample_cb = p->sample_cb;
7896 pb->lost_cb = p->lost_cb;
7897 pb->ctx = p->ctx;
7898
7899 pb->page_size = getpagesize();
7900 pb->mmap_size = pb->page_size * page_cnt;
7901 pb->map_fd = map_fd;
7902
7903 pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
7904 if (pb->epoll_fd < 0) {
7905 err = -errno;
7906 pr_warn("failed to create epoll instance: %s\n",
7907 libbpf_strerror_r(err, msg, sizeof(msg)));
7908 goto error;
7909 }
7910
7911 if (p->cpu_cnt > 0) {
7912 pb->cpu_cnt = p->cpu_cnt;
7913 } else {
7914 pb->cpu_cnt = libbpf_num_possible_cpus();
7915 if (pb->cpu_cnt < 0) {
7916 err = pb->cpu_cnt;
7917 goto error;
7918 }
7919 if (map.max_entries < pb->cpu_cnt)
7920 pb->cpu_cnt = map.max_entries;
7921 }
7922
7923 pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events));
7924 if (!pb->events) {
7925 err = -ENOMEM;
7926 pr_warn("failed to allocate events: out of memory\n");
7927 goto error;
7928 }
7929 pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs));
7930 if (!pb->cpu_bufs) {
7931 err = -ENOMEM;
7932 pr_warn("failed to allocate buffers: out of memory\n");
7933 goto error;
7934 }
7935
7936 err = parse_cpu_mask_file(online_cpus_file, &online, &n);
7937 if (err) {
7938 pr_warn("failed to get online CPU mask: %d\n", err);
7939 goto error;
7940 }
7941
7942 for (i = 0, j = 0; i < pb->cpu_cnt; i++) {
7943 struct perf_cpu_buf *cpu_buf;
7944 int cpu, map_key;
7945
7946 cpu = p->cpu_cnt > 0 ? p->cpus[i] : i;
7947 map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i;
7948
7949 /* in case user didn't explicitly requested particular CPUs to
7950 * be attached to, skip offline/not present CPUs
7951 */
7952 if (p->cpu_cnt <= 0 && (cpu >= n || !online[cpu]))
7953 continue;
7954
7955 cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key);
7956 if (IS_ERR(cpu_buf)) {
7957 err = PTR_ERR(cpu_buf);
7958 goto error;
7959 }
7960
7961 pb->cpu_bufs[j] = cpu_buf;
7962
7963 err = bpf_map_update_elem(pb->map_fd, &map_key,
7964 &cpu_buf->fd, 0);
7965 if (err) {
7966 err = -errno;
7967 pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n",
7968 cpu, map_key, cpu_buf->fd,
7969 libbpf_strerror_r(err, msg, sizeof(msg)));
7970 goto error;
7971 }
7972
7973 pb->events[j].events = EPOLLIN;
7974 pb->events[j].data.ptr = cpu_buf;
7975 if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd,
7976 &pb->events[j]) < 0) {
7977 err = -errno;
7978 pr_warn("failed to epoll_ctl cpu #%d perf FD %d: %s\n",
7979 cpu, cpu_buf->fd,
7980 libbpf_strerror_r(err, msg, sizeof(msg)));
7981 goto error;
7982 }
7983 j++;
7984 }
7985 pb->cpu_cnt = j;
7986 free(online);
7987
7988 return pb;
7989
7990 error:
7991 free(online);
7992 if (pb)
7993 perf_buffer__free(pb);
7994 return ERR_PTR(err);
7995 }
7996
7997 struct perf_sample_raw {
7998 struct perf_event_header header;
7999 uint32_t size;
8000 char data[0];
8001 };
8002
8003 struct perf_sample_lost {
8004 struct perf_event_header header;
8005 uint64_t id;
8006 uint64_t lost;
8007 uint64_t sample_id;
8008 };
8009
8010 static enum bpf_perf_event_ret
8011 perf_buffer__process_record(struct perf_event_header *e, void *ctx)
8012 {
8013 struct perf_cpu_buf *cpu_buf = ctx;
8014 struct perf_buffer *pb = cpu_buf->pb;
8015 void *data = e;
8016
8017 /* user wants full control over parsing perf event */
8018 if (pb->event_cb)
8019 return pb->event_cb(pb->ctx, cpu_buf->cpu, e);
8020
8021 switch (e->type) {
8022 case PERF_RECORD_SAMPLE: {
8023 struct perf_sample_raw *s = data;
8024
8025 if (pb->sample_cb)
8026 pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size);
8027 break;
8028 }
8029 case PERF_RECORD_LOST: {
8030 struct perf_sample_lost *s = data;
8031
8032 if (pb->lost_cb)
8033 pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost);
8034 break;
8035 }
8036 default:
8037 pr_warn("unknown perf sample type %d\n", e->type);
8038 return LIBBPF_PERF_EVENT_ERROR;
8039 }
8040 return LIBBPF_PERF_EVENT_CONT;
8041 }
8042
8043 static int perf_buffer__process_records(struct perf_buffer *pb,
8044 struct perf_cpu_buf *cpu_buf)
8045 {
8046 enum bpf_perf_event_ret ret;
8047
8048 ret = bpf_perf_event_read_simple(cpu_buf->base, pb->mmap_size,
8049 pb->page_size, &cpu_buf->buf,
8050 &cpu_buf->buf_size,
8051 perf_buffer__process_record, cpu_buf);
8052 if (ret != LIBBPF_PERF_EVENT_CONT)
8053 return ret;
8054 return 0;
8055 }
8056
8057 int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms)
8058 {
8059 int i, cnt, err;
8060
8061 cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms);
8062 for (i = 0; i < cnt; i++) {
8063 struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr;
8064
8065 err = perf_buffer__process_records(pb, cpu_buf);
8066 if (err) {
8067 pr_warn("error while processing records: %d\n", err);
8068 return err;
8069 }
8070 }
8071 return cnt < 0 ? -errno : cnt;
8072 }
8073
8074 struct bpf_prog_info_array_desc {
8075 int array_offset; /* e.g. offset of jited_prog_insns */
8076 int count_offset; /* e.g. offset of jited_prog_len */
8077 int size_offset; /* > 0: offset of rec size,
8078 * < 0: fix size of -size_offset
8079 */
8080 };
8081
8082 static struct bpf_prog_info_array_desc bpf_prog_info_array_desc[] = {
8083 [BPF_PROG_INFO_JITED_INSNS] = {
8084 offsetof(struct bpf_prog_info, jited_prog_insns),
8085 offsetof(struct bpf_prog_info, jited_prog_len),
8086 -1,
8087 },
8088 [BPF_PROG_INFO_XLATED_INSNS] = {
8089 offsetof(struct bpf_prog_info, xlated_prog_insns),
8090 offsetof(struct bpf_prog_info, xlated_prog_len),
8091 -1,
8092 },
8093 [BPF_PROG_INFO_MAP_IDS] = {
8094 offsetof(struct bpf_prog_info, map_ids),
8095 offsetof(struct bpf_prog_info, nr_map_ids),
8096 -(int)sizeof(__u32),
8097 },
8098 [BPF_PROG_INFO_JITED_KSYMS] = {
8099 offsetof(struct bpf_prog_info, jited_ksyms),
8100 offsetof(struct bpf_prog_info, nr_jited_ksyms),
8101 -(int)sizeof(__u64),
8102 },
8103 [BPF_PROG_INFO_JITED_FUNC_LENS] = {
8104 offsetof(struct bpf_prog_info, jited_func_lens),
8105 offsetof(struct bpf_prog_info, nr_jited_func_lens),
8106 -(int)sizeof(__u32),
8107 },
8108 [BPF_PROG_INFO_FUNC_INFO] = {
8109 offsetof(struct bpf_prog_info, func_info),
8110 offsetof(struct bpf_prog_info, nr_func_info),
8111 offsetof(struct bpf_prog_info, func_info_rec_size),
8112 },
8113 [BPF_PROG_INFO_LINE_INFO] = {
8114 offsetof(struct bpf_prog_info, line_info),
8115 offsetof(struct bpf_prog_info, nr_line_info),
8116 offsetof(struct bpf_prog_info, line_info_rec_size),
8117 },
8118 [BPF_PROG_INFO_JITED_LINE_INFO] = {
8119 offsetof(struct bpf_prog_info, jited_line_info),
8120 offsetof(struct bpf_prog_info, nr_jited_line_info),
8121 offsetof(struct bpf_prog_info, jited_line_info_rec_size),
8122 },
8123 [BPF_PROG_INFO_PROG_TAGS] = {
8124 offsetof(struct bpf_prog_info, prog_tags),
8125 offsetof(struct bpf_prog_info, nr_prog_tags),
8126 -(int)sizeof(__u8) * BPF_TAG_SIZE,
8127 },
8128
8129 };
8130
8131 static __u32 bpf_prog_info_read_offset_u32(struct bpf_prog_info *info,
8132 int offset)
8133 {
8134 __u32 *array = (__u32 *)info;
8135
8136 if (offset >= 0)
8137 return array[offset / sizeof(__u32)];
8138 return -(int)offset;
8139 }
8140
8141 static __u64 bpf_prog_info_read_offset_u64(struct bpf_prog_info *info,
8142 int offset)
8143 {
8144 __u64 *array = (__u64 *)info;
8145
8146 if (offset >= 0)
8147 return array[offset / sizeof(__u64)];
8148 return -(int)offset;
8149 }
8150
8151 static void bpf_prog_info_set_offset_u32(struct bpf_prog_info *info, int offset,
8152 __u32 val)
8153 {
8154 __u32 *array = (__u32 *)info;
8155
8156 if (offset >= 0)
8157 array[offset / sizeof(__u32)] = val;
8158 }
8159
8160 static void bpf_prog_info_set_offset_u64(struct bpf_prog_info *info, int offset,
8161 __u64 val)
8162 {
8163 __u64 *array = (__u64 *)info;
8164
8165 if (offset >= 0)
8166 array[offset / sizeof(__u64)] = val;
8167 }
8168
8169 struct bpf_prog_info_linear *
8170 bpf_program__get_prog_info_linear(int fd, __u64 arrays)
8171 {
8172 struct bpf_prog_info_linear *info_linear;
8173 struct bpf_prog_info info = {};
8174 __u32 info_len = sizeof(info);
8175 __u32 data_len = 0;
8176 int i, err;
8177 void *ptr;
8178
8179 if (arrays >> BPF_PROG_INFO_LAST_ARRAY)
8180 return ERR_PTR(-EINVAL);
8181
8182 /* step 1: get array dimensions */
8183 err = bpf_obj_get_info_by_fd(fd, &info, &info_len);
8184 if (err) {
8185 pr_debug("can't get prog info: %s", strerror(errno));
8186 return ERR_PTR(-EFAULT);
8187 }
8188
8189 /* step 2: calculate total size of all arrays */
8190 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
8191 bool include_array = (arrays & (1UL << i)) > 0;
8192 struct bpf_prog_info_array_desc *desc;
8193 __u32 count, size;
8194
8195 desc = bpf_prog_info_array_desc + i;
8196
8197 /* kernel is too old to support this field */
8198 if (info_len < desc->array_offset + sizeof(__u32) ||
8199 info_len < desc->count_offset + sizeof(__u32) ||
8200 (desc->size_offset > 0 && info_len < desc->size_offset))
8201 include_array = false;
8202
8203 if (!include_array) {
8204 arrays &= ~(1UL << i); /* clear the bit */
8205 continue;
8206 }
8207
8208 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset);
8209 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset);
8210
8211 data_len += count * size;
8212 }
8213
8214 /* step 3: allocate continuous memory */
8215 data_len = roundup(data_len, sizeof(__u64));
8216 info_linear = malloc(sizeof(struct bpf_prog_info_linear) + data_len);
8217 if (!info_linear)
8218 return ERR_PTR(-ENOMEM);
8219
8220 /* step 4: fill data to info_linear->info */
8221 info_linear->arrays = arrays;
8222 memset(&info_linear->info, 0, sizeof(info));
8223 ptr = info_linear->data;
8224
8225 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
8226 struct bpf_prog_info_array_desc *desc;
8227 __u32 count, size;
8228
8229 if ((arrays & (1UL << i)) == 0)
8230 continue;
8231
8232 desc = bpf_prog_info_array_desc + i;
8233 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset);
8234 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset);
8235 bpf_prog_info_set_offset_u32(&info_linear->info,
8236 desc->count_offset, count);
8237 bpf_prog_info_set_offset_u32(&info_linear->info,
8238 desc->size_offset, size);
8239 bpf_prog_info_set_offset_u64(&info_linear->info,
8240 desc->array_offset,
8241 ptr_to_u64(ptr));
8242 ptr += count * size;
8243 }
8244
8245 /* step 5: call syscall again to get required arrays */
8246 err = bpf_obj_get_info_by_fd(fd, &info_linear->info, &info_len);
8247 if (err) {
8248 pr_debug("can't get prog info: %s", strerror(errno));
8249 free(info_linear);
8250 return ERR_PTR(-EFAULT);
8251 }
8252
8253 /* step 6: verify the data */
8254 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
8255 struct bpf_prog_info_array_desc *desc;
8256 __u32 v1, v2;
8257
8258 if ((arrays & (1UL << i)) == 0)
8259 continue;
8260
8261 desc = bpf_prog_info_array_desc + i;
8262 v1 = bpf_prog_info_read_offset_u32(&info, desc->count_offset);
8263 v2 = bpf_prog_info_read_offset_u32(&info_linear->info,
8264 desc->count_offset);
8265 if (v1 != v2)
8266 pr_warn("%s: mismatch in element count\n", __func__);
8267
8268 v1 = bpf_prog_info_read_offset_u32(&info, desc->size_offset);
8269 v2 = bpf_prog_info_read_offset_u32(&info_linear->info,
8270 desc->size_offset);
8271 if (v1 != v2)
8272 pr_warn("%s: mismatch in rec size\n", __func__);
8273 }
8274
8275 /* step 7: update info_len and data_len */
8276 info_linear->info_len = sizeof(struct bpf_prog_info);
8277 info_linear->data_len = data_len;
8278
8279 return info_linear;
8280 }
8281
8282 void bpf_program__bpil_addr_to_offs(struct bpf_prog_info_linear *info_linear)
8283 {
8284 int i;
8285
8286 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
8287 struct bpf_prog_info_array_desc *desc;
8288 __u64 addr, offs;
8289
8290 if ((info_linear->arrays & (1UL << i)) == 0)
8291 continue;
8292
8293 desc = bpf_prog_info_array_desc + i;
8294 addr = bpf_prog_info_read_offset_u64(&info_linear->info,
8295 desc->array_offset);
8296 offs = addr - ptr_to_u64(info_linear->data);
8297 bpf_prog_info_set_offset_u64(&info_linear->info,
8298 desc->array_offset, offs);
8299 }
8300 }
8301
8302 void bpf_program__bpil_offs_to_addr(struct bpf_prog_info_linear *info_linear)
8303 {
8304 int i;
8305
8306 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
8307 struct bpf_prog_info_array_desc *desc;
8308 __u64 addr, offs;
8309
8310 if ((info_linear->arrays & (1UL << i)) == 0)
8311 continue;
8312
8313 desc = bpf_prog_info_array_desc + i;
8314 offs = bpf_prog_info_read_offset_u64(&info_linear->info,
8315 desc->array_offset);
8316 addr = offs + ptr_to_u64(info_linear->data);
8317 bpf_prog_info_set_offset_u64(&info_linear->info,
8318 desc->array_offset, addr);
8319 }
8320 }
8321
8322 int bpf_program__set_attach_target(struct bpf_program *prog,
8323 int attach_prog_fd,
8324 const char *attach_func_name)
8325 {
8326 int btf_id;
8327
8328 if (!prog || attach_prog_fd < 0 || !attach_func_name)
8329 return -EINVAL;
8330
8331 if (attach_prog_fd)
8332 btf_id = libbpf_find_prog_btf_id(attach_func_name,
8333 attach_prog_fd);
8334 else
8335 btf_id = __find_vmlinux_btf_id(prog->obj->btf_vmlinux,
8336 attach_func_name,
8337 prog->expected_attach_type);
8338
8339 if (btf_id < 0)
8340 return btf_id;
8341
8342 prog->attach_btf_id = btf_id;
8343 prog->attach_prog_fd = attach_prog_fd;
8344 return 0;
8345 }
8346
8347 int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz)
8348 {
8349 int err = 0, n, len, start, end = -1;
8350 bool *tmp;
8351
8352 *mask = NULL;
8353 *mask_sz = 0;
8354
8355 /* Each sub string separated by ',' has format \d+-\d+ or \d+ */
8356 while (*s) {
8357 if (*s == ',' || *s == '\n') {
8358 s++;
8359 continue;
8360 }
8361 n = sscanf(s, "%d%n-%d%n", &start, &len, &end, &len);
8362 if (n <= 0 || n > 2) {
8363 pr_warn("Failed to get CPU range %s: %d\n", s, n);
8364 err = -EINVAL;
8365 goto cleanup;
8366 } else if (n == 1) {
8367 end = start;
8368 }
8369 if (start < 0 || start > end) {
8370 pr_warn("Invalid CPU range [%d,%d] in %s\n",
8371 start, end, s);
8372 err = -EINVAL;
8373 goto cleanup;
8374 }
8375 tmp = realloc(*mask, end + 1);
8376 if (!tmp) {
8377 err = -ENOMEM;
8378 goto cleanup;
8379 }
8380 *mask = tmp;
8381 memset(tmp + *mask_sz, 0, start - *mask_sz);
8382 memset(tmp + start, 1, end - start + 1);
8383 *mask_sz = end + 1;
8384 s += len;
8385 }
8386 if (!*mask_sz) {
8387 pr_warn("Empty CPU range\n");
8388 return -EINVAL;
8389 }
8390 return 0;
8391 cleanup:
8392 free(*mask);
8393 *mask = NULL;
8394 return err;
8395 }
8396
8397 int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz)
8398 {
8399 int fd, err = 0, len;
8400 char buf[128];
8401
8402 fd = open(fcpu, O_RDONLY);
8403 if (fd < 0) {
8404 err = -errno;
8405 pr_warn("Failed to open cpu mask file %s: %d\n", fcpu, err);
8406 return err;
8407 }
8408 len = read(fd, buf, sizeof(buf));
8409 close(fd);
8410 if (len <= 0) {
8411 err = len ? -errno : -EINVAL;
8412 pr_warn("Failed to read cpu mask from %s: %d\n", fcpu, err);
8413 return err;
8414 }
8415 if (len >= sizeof(buf)) {
8416 pr_warn("CPU mask is too big in file %s\n", fcpu);
8417 return -E2BIG;
8418 }
8419 buf[len] = '\0';
8420
8421 return parse_cpu_mask_str(buf, mask, mask_sz);
8422 }
8423
8424 int libbpf_num_possible_cpus(void)
8425 {
8426 static const char *fcpu = "/sys/devices/system/cpu/possible";
8427 static int cpus;
8428 int err, n, i, tmp_cpus;
8429 bool *mask;
8430
8431 tmp_cpus = READ_ONCE(cpus);
8432 if (tmp_cpus > 0)
8433 return tmp_cpus;
8434
8435 err = parse_cpu_mask_file(fcpu, &mask, &n);
8436 if (err)
8437 return err;
8438
8439 tmp_cpus = 0;
8440 for (i = 0; i < n; i++) {
8441 if (mask[i])
8442 tmp_cpus++;
8443 }
8444 free(mask);
8445
8446 WRITE_ONCE(cpus, tmp_cpus);
8447 return tmp_cpus;
8448 }
8449
8450 int bpf_object__open_skeleton(struct bpf_object_skeleton *s,
8451 const struct bpf_object_open_opts *opts)
8452 {
8453 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, skel_opts,
8454 .object_name = s->name,
8455 );
8456 struct bpf_object *obj;
8457 int i;
8458
8459 /* Attempt to preserve opts->object_name, unless overriden by user
8460 * explicitly. Overwriting object name for skeletons is discouraged,
8461 * as it breaks global data maps, because they contain object name
8462 * prefix as their own map name prefix. When skeleton is generated,
8463 * bpftool is making an assumption that this name will stay the same.
8464 */
8465 if (opts) {
8466 memcpy(&skel_opts, opts, sizeof(*opts));
8467 if (!opts->object_name)
8468 skel_opts.object_name = s->name;
8469 }
8470
8471 obj = bpf_object__open_mem(s->data, s->data_sz, &skel_opts);
8472 if (IS_ERR(obj)) {
8473 pr_warn("failed to initialize skeleton BPF object '%s': %ld\n",
8474 s->name, PTR_ERR(obj));
8475 return PTR_ERR(obj);
8476 }
8477
8478 *s->obj = obj;
8479
8480 for (i = 0; i < s->map_cnt; i++) {
8481 struct bpf_map **map = s->maps[i].map;
8482 const char *name = s->maps[i].name;
8483 void **mmaped = s->maps[i].mmaped;
8484
8485 *map = bpf_object__find_map_by_name(obj, name);
8486 if (!*map) {
8487 pr_warn("failed to find skeleton map '%s'\n", name);
8488 return -ESRCH;
8489 }
8490
8491 /* externs shouldn't be pre-setup from user code */
8492 if (mmaped && (*map)->libbpf_type != LIBBPF_MAP_KCONFIG)
8493 *mmaped = (*map)->mmaped;
8494 }
8495
8496 for (i = 0; i < s->prog_cnt; i++) {
8497 struct bpf_program **prog = s->progs[i].prog;
8498 const char *name = s->progs[i].name;
8499
8500 *prog = bpf_object__find_program_by_name(obj, name);
8501 if (!*prog) {
8502 pr_warn("failed to find skeleton program '%s'\n", name);
8503 return -ESRCH;
8504 }
8505 }
8506
8507 return 0;
8508 }
8509
8510 int bpf_object__load_skeleton(struct bpf_object_skeleton *s)
8511 {
8512 int i, err;
8513
8514 err = bpf_object__load(*s->obj);
8515 if (err) {
8516 pr_warn("failed to load BPF skeleton '%s': %d\n", s->name, err);
8517 return err;
8518 }
8519
8520 for (i = 0; i < s->map_cnt; i++) {
8521 struct bpf_map *map = *s->maps[i].map;
8522 size_t mmap_sz = bpf_map_mmap_sz(map);
8523 int prot, map_fd = bpf_map__fd(map);
8524 void **mmaped = s->maps[i].mmaped;
8525
8526 if (!mmaped)
8527 continue;
8528
8529 if (!(map->def.map_flags & BPF_F_MMAPABLE)) {
8530 *mmaped = NULL;
8531 continue;
8532 }
8533
8534 if (map->def.map_flags & BPF_F_RDONLY_PROG)
8535 prot = PROT_READ;
8536 else
8537 prot = PROT_READ | PROT_WRITE;
8538
8539 /* Remap anonymous mmap()-ed "map initialization image" as
8540 * a BPF map-backed mmap()-ed memory, but preserving the same
8541 * memory address. This will cause kernel to change process'
8542 * page table to point to a different piece of kernel memory,
8543 * but from userspace point of view memory address (and its
8544 * contents, being identical at this point) will stay the
8545 * same. This mapping will be released by bpf_object__close()
8546 * as per normal clean up procedure, so we don't need to worry
8547 * about it from skeleton's clean up perspective.
8548 */
8549 *mmaped = mmap(map->mmaped, mmap_sz, prot,
8550 MAP_SHARED | MAP_FIXED, map_fd, 0);
8551 if (*mmaped == MAP_FAILED) {
8552 err = -errno;
8553 *mmaped = NULL;
8554 pr_warn("failed to re-mmap() map '%s': %d\n",
8555 bpf_map__name(map), err);
8556 return err;
8557 }
8558 }
8559
8560 return 0;
8561 }
8562
8563 int bpf_object__attach_skeleton(struct bpf_object_skeleton *s)
8564 {
8565 int i;
8566
8567 for (i = 0; i < s->prog_cnt; i++) {
8568 struct bpf_program *prog = *s->progs[i].prog;
8569 struct bpf_link **link = s->progs[i].link;
8570 const struct bpf_sec_def *sec_def;
8571 const char *sec_name = bpf_program__title(prog, false);
8572
8573 sec_def = find_sec_def(sec_name);
8574 if (!sec_def || !sec_def->attach_fn)
8575 continue;
8576
8577 *link = sec_def->attach_fn(sec_def, prog);
8578 if (IS_ERR(*link)) {
8579 pr_warn("failed to auto-attach program '%s': %ld\n",
8580 bpf_program__name(prog), PTR_ERR(*link));
8581 return PTR_ERR(*link);
8582 }
8583 }
8584
8585 return 0;
8586 }
8587
8588 void bpf_object__detach_skeleton(struct bpf_object_skeleton *s)
8589 {
8590 int i;
8591
8592 for (i = 0; i < s->prog_cnt; i++) {
8593 struct bpf_link **link = s->progs[i].link;
8594
8595 if (!IS_ERR_OR_NULL(*link))
8596 bpf_link__destroy(*link);
8597 *link = NULL;
8598 }
8599 }
8600
8601 void bpf_object__destroy_skeleton(struct bpf_object_skeleton *s)
8602 {
8603 if (s->progs)
8604 bpf_object__detach_skeleton(s);
8605 if (s->obj)
8606 bpf_object__close(*s->obj);
8607 free(s->maps);
8608 free(s->progs);
8609 free(s);
8610 }
Linux with added FPC-III support