tools/lib/bpf/bpf.c

   1 // SPDX-License-Identifier: LGPL-2.1
   2
   3 /*
   4  * common eBPF ELF operations.
   5  *
   6  * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>
   7  * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
   8  * Copyright (C) 2015 Huawei Inc.
   9  *
  10  * This program is free software; you can redistribute it and/or
  11  * modify it under the terms of the GNU Lesser General Public
  12  * License as published by the Free Software Foundation;
  13  * version 2.1 of the License (not later!)
  14  *
  15  * This program is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18  * GNU Lesser General Public License for more details.
  19  *
  20  * You should have received a copy of the GNU Lesser General Public
  21  * License along with this program; if not,  see <http://www.gnu.org/licenses>
  22  */
  23
  24 #include <stdlib.h>
  25 #include <memory.h>
  26 #include <unistd.h>
  27 #include <asm/unistd.h>
  28 #include <linux/bpf.h>
  29 #include "bpf.h"
  30 #include "libbpf.h"
  31 #include "nlattr.h"
  32 #include <linux/rtnetlink.h>
  33 #include <linux/if_link.h>
  34 #include <sys/socket.h>
  35 #include <errno.h>
  36
  37 #ifndef SOL_NETLINK
  38 #define SOL_NETLINK 270
  39 #endif
  40
  41 /*
  42  * When building perf, unistd.h is overridden. __NR_bpf is
  43  * required to be defined explicitly.
  44  */
  45 #ifndef __NR_bpf
  46 # if defined(__i386__)
  47 #  define __NR_bpf 357
  48 # elif defined(__x86_64__)
  49 #  define __NR_bpf 321
  50 # elif defined(__aarch64__)
  51 #  define __NR_bpf 280
  52 # elif defined(__sparc__)
  53 #  define __NR_bpf 349
  54 # elif defined(__s390__)
  55 #  define __NR_bpf 351
  56 # else
  57 #  error __NR_bpf not defined. libbpf does not support your arch.
  58 # endif
  59 #endif
  60
  61 #ifndef min
  62 #define min(x, y) ((x) < (y) ? (x) : (y))
  63 #endif
  64
  65 static inline __u64 ptr_to_u64(const void *ptr)
  66 {
  67         return (__u64) (unsigned long) ptr;
  68 }
  69
  70 static inline int sys_bpf(enum bpf_cmd cmd, union bpf_attr *attr,
  71                           unsigned int size)
  72 {
  73         return syscall(__NR_bpf, cmd, attr, size);
  74 }
  75
  76 int bpf_create_map_node(enum bpf_map_type map_type, const char *name,
  77                         int key_size, int value_size, int max_entries,
  78                         __u32 map_flags, int node)
  79 {
  80         __u32 name_len = name ? strlen(name) : 0;
  81         union bpf_attr attr;
  82
  83         memset(&attr, '\0', sizeof(attr));
  84
  85         attr.map_type = map_type;
  86         attr.key_size = key_size;
  87         attr.value_size = value_size;
  88         attr.max_entries = max_entries;
  89         attr.map_flags = map_flags;
  90         memcpy(attr.map_name, name, min(name_len, BPF_OBJ_NAME_LEN - 1));
  91
  92         if (node >= 0) {
  93                 attr.map_flags |= BPF_F_NUMA_NODE;
  94                 attr.numa_node = node;
  95         }
  96
  97         return sys_bpf(BPF_MAP_CREATE, &attr, sizeof(attr));
  98 }
  99
 100 int bpf_create_map(enum bpf_map_type map_type, int key_size,
 101                    int value_size, int max_entries, __u32 map_flags)
 102 {
 103         return bpf_create_map_node(map_type, NULL, key_size, value_size,
 104                                    max_entries, map_flags, -1);
 105 }
 106
 107 int bpf_create_map_name(enum bpf_map_type map_type, const char *name,
 108                         int key_size, int value_size, int max_entries,
 109                         __u32 map_flags)
 110 {
 111         return bpf_create_map_node(map_type, name, key_size, value_size,
 112                                    max_entries, map_flags, -1);
 113 }
 114
 115 int bpf_create_map_in_map_node(enum bpf_map_type map_type, const char *name,
 116                                int key_size, int inner_map_fd, int max_entries,
 117                                __u32 map_flags, int node)
 118 {
 119         __u32 name_len = name ? strlen(name) : 0;
 120         union bpf_attr attr;
 121
 122         memset(&attr, '\0', sizeof(attr));
 123
 124         attr.map_type = map_type;
 125         attr.key_size = key_size;
 126         attr.value_size = 4;
 127         attr.inner_map_fd = inner_map_fd;
 128         attr.max_entries = max_entries;
 129         attr.map_flags = map_flags;
 130         memcpy(attr.map_name, name, min(name_len, BPF_OBJ_NAME_LEN - 1));
 131
 132         if (node >= 0) {
 133                 attr.map_flags |= BPF_F_NUMA_NODE;
 134                 attr.numa_node = node;
 135         }
 136
 137         return sys_bpf(BPF_MAP_CREATE, &attr, sizeof(attr));
 138 }
 139
 140 int bpf_create_map_in_map(enum bpf_map_type map_type, const char *name,
 141                           int key_size, int inner_map_fd, int max_entries,
 142                           __u32 map_flags)
 143 {
 144         return bpf_create_map_in_map_node(map_type, name, key_size,
 145                                           inner_map_fd, max_entries, map_flags,
 146                                           -1);
 147 }
 148
 149 int bpf_load_program_name(enum bpf_prog_type type, const char *name,
 150                           const struct bpf_insn *insns,
 151                           size_t insns_cnt, const char *license,
 152                           __u32 kern_version, char *log_buf,
 153                           size_t log_buf_sz)
 154 {
 155         int fd;
 156         union bpf_attr attr;
 157         __u32 name_len = name ? strlen(name) : 0;
 158
 159         bzero(&attr, sizeof(attr));
 160         attr.prog_type = type;
 161         attr.insn_cnt = (__u32)insns_cnt;
 162         attr.insns = ptr_to_u64(insns);
 163         attr.license = ptr_to_u64(license);
 164         attr.log_buf = ptr_to_u64(NULL);
 165         attr.log_size = 0;
 166         attr.log_level = 0;
 167         attr.kern_version = kern_version;
 168         memcpy(attr.prog_name, name, min(name_len, BPF_OBJ_NAME_LEN - 1));
 169
 170         fd = sys_bpf(BPF_PROG_LOAD, &attr, sizeof(attr));
 171         if (fd >= 0 || !log_buf || !log_buf_sz)
 172                 return fd;
 173
 174         /* Try again with log */
 175         attr.log_buf = ptr_to_u64(log_buf);
 176         attr.log_size = log_buf_sz;
 177         attr.log_level = 1;
 178         log_buf[0] = 0;
 179         return sys_bpf(BPF_PROG_LOAD, &attr, sizeof(attr));
 180 }
 181
 182 int bpf_load_program(enum bpf_prog_type type, const struct bpf_insn *insns,
 183                      size_t insns_cnt, const char *license,
 184                      __u32 kern_version, char *log_buf,
 185                      size_t log_buf_sz)
 186 {
 187         return bpf_load_program_name(type, NULL, insns, insns_cnt, license,
 188                                      kern_version, log_buf, log_buf_sz);
 189 }
 190
 191 int bpf_verify_program(enum bpf_prog_type type, const struct bpf_insn *insns,
 192                        size_t insns_cnt, int strict_alignment,
 193                        const char *license, __u32 kern_version,
 194                        char *log_buf, size_t log_buf_sz, int log_level)
 195 {
 196         union bpf_attr attr;
 197
 198         bzero(&attr, sizeof(attr));
 199         attr.prog_type = type;
 200         attr.insn_cnt = (__u32)insns_cnt;
 201         attr.insns = ptr_to_u64(insns);
 202         attr.license = ptr_to_u64(license);
 203         attr.log_buf = ptr_to_u64(log_buf);
 204         attr.log_size = log_buf_sz;
 205         attr.log_level = log_level;
 206         log_buf[0] = 0;
 207         attr.kern_version = kern_version;
 208         attr.prog_flags = strict_alignment ? BPF_F_STRICT_ALIGNMENT : 0;
 209
 210         return sys_bpf(BPF_PROG_LOAD, &attr, sizeof(attr));
 211 }
 212
 213 int bpf_map_update_elem(int fd, const void *key, const void *value,
 214                         __u64 flags)
 215 {
 216         union bpf_attr attr;
 217
 218         bzero(&attr, sizeof(attr));
 219         attr.map_fd = fd;
 220         attr.key = ptr_to_u64(key);
 221         attr.value = ptr_to_u64(value);
 222         attr.flags = flags;
 223
 224         return sys_bpf(BPF_MAP_UPDATE_ELEM, &attr, sizeof(attr));
 225 }
 226
 227 int bpf_map_lookup_elem(int fd, const void *key, void *value)
 228 {
 229         union bpf_attr attr;
 230
 231         bzero(&attr, sizeof(attr));
 232         attr.map_fd = fd;
 233         attr.key = ptr_to_u64(key);
 234         attr.value = ptr_to_u64(value);
 235
 236         return sys_bpf(BPF_MAP_LOOKUP_ELEM, &attr, sizeof(attr));
 237 }
 238
 239 int bpf_map_delete_elem(int fd, const void *key)
 240 {
 241         union bpf_attr attr;
 242
 243         bzero(&attr, sizeof(attr));
 244         attr.map_fd = fd;
 245         attr.key = ptr_to_u64(key);
 246
 247         return sys_bpf(BPF_MAP_DELETE_ELEM, &attr, sizeof(attr));
 248 }
 249
 250 int bpf_map_get_next_key(int fd, const void *key, void *next_key)
 251 {
 252         union bpf_attr attr;
 253
 254         bzero(&attr, sizeof(attr));
 255         attr.map_fd = fd;
 256         attr.key = ptr_to_u64(key);
 257         attr.next_key = ptr_to_u64(next_key);
 258
 259         return sys_bpf(BPF_MAP_GET_NEXT_KEY, &attr, sizeof(attr));
 260 }
 261
 262 int bpf_obj_pin(int fd, const char *pathname)
 263 {
 264         union bpf_attr attr;
 265
 266         bzero(&attr, sizeof(attr));
 267         attr.pathname = ptr_to_u64((void *)pathname);
 268         attr.bpf_fd = fd;
 269
 270         return sys_bpf(BPF_OBJ_PIN, &attr, sizeof(attr));
 271 }
 272
 273 int bpf_obj_get(const char *pathname)
 274 {
 275         union bpf_attr attr;
 276
 277         bzero(&attr, sizeof(attr));
 278         attr.pathname = ptr_to_u64((void *)pathname);
 279
 280         return sys_bpf(BPF_OBJ_GET, &attr, sizeof(attr));
 281 }
 282
 283 int bpf_prog_attach(int prog_fd, int target_fd, enum bpf_attach_type type,
 284                     unsigned int flags)
 285 {
 286         union bpf_attr attr;
 287
 288         bzero(&attr, sizeof(attr));
 289         attr.target_fd     = target_fd;
 290         attr.attach_bpf_fd = prog_fd;
 291         attr.attach_type   = type;
 292         attr.attach_flags  = flags;
 293
 294         return sys_bpf(BPF_PROG_ATTACH, &attr, sizeof(attr));
 295 }
 296
 297 int bpf_prog_detach(int target_fd, enum bpf_attach_type type)
 298 {
 299         union bpf_attr attr;
 300
 301         bzero(&attr, sizeof(attr));
 302         attr.target_fd   = target_fd;
 303         attr.attach_type = type;
 304
 305         return sys_bpf(BPF_PROG_DETACH, &attr, sizeof(attr));
 306 }
 307
 308 int bpf_prog_detach2(int prog_fd, int target_fd, enum bpf_attach_type type)
 309 {
 310         union bpf_attr attr;
 311
 312         bzero(&attr, sizeof(attr));
 313         attr.target_fd   = target_fd;
 314         attr.attach_bpf_fd = prog_fd;
 315         attr.attach_type = type;
 316
 317         return sys_bpf(BPF_PROG_DETACH, &attr, sizeof(attr));
 318 }
 319
 320 int bpf_prog_query(int target_fd, enum bpf_attach_type type, __u32 query_flags,
 321                    __u32 *attach_flags, __u32 *prog_ids, __u32 *prog_cnt)
 322 {
 323         union bpf_attr attr;
 324         int ret;
 325
 326         bzero(&attr, sizeof(attr));
 327         attr.query.target_fd    = target_fd;
 328         attr.query.attach_type  = type;
 329         attr.query.query_flags  = query_flags;
 330         attr.query.prog_cnt     = *prog_cnt;
 331         attr.query.prog_ids     = ptr_to_u64(prog_ids);
 332
 333         ret = sys_bpf(BPF_PROG_QUERY, &attr, sizeof(attr));
 334         if (attach_flags)
 335                 *attach_flags = attr.query.attach_flags;
 336         *prog_cnt = attr.query.prog_cnt;
 337         return ret;
 338 }
 339
 340 int bpf_prog_test_run(int prog_fd, int repeat, void *data, __u32 size,
 341                       void *data_out, __u32 *size_out, __u32 *retval,
 342                       __u32 *duration)
 343 {
 344         union bpf_attr attr;
 345         int ret;
 346
 347         bzero(&attr, sizeof(attr));
 348         attr.test.prog_fd = prog_fd;
 349         attr.test.data_in = ptr_to_u64(data);
 350         attr.test.data_out = ptr_to_u64(data_out);
 351         attr.test.data_size_in = size;
 352         attr.test.repeat = repeat;
 353
 354         ret = sys_bpf(BPF_PROG_TEST_RUN, &attr, sizeof(attr));
 355         if (size_out)
 356                 *size_out = attr.test.data_size_out;
 357         if (retval)
 358                 *retval = attr.test.retval;
 359         if (duration)
 360                 *duration = attr.test.duration;
 361         return ret;
 362 }
 363
 364 int bpf_prog_get_next_id(__u32 start_id, __u32 *next_id)
 365 {
 366         union bpf_attr attr;
 367         int err;
 368
 369         bzero(&attr, sizeof(attr));
 370         attr.start_id = start_id;
 371
 372         err = sys_bpf(BPF_PROG_GET_NEXT_ID, &attr, sizeof(attr));
 373         if (!err)
 374                 *next_id = attr.next_id;
 375
 376         return err;
 377 }
 378
 379 int bpf_map_get_next_id(__u32 start_id, __u32 *next_id)
 380 {
 381         union bpf_attr attr;
 382         int err;
 383
 384         bzero(&attr, sizeof(attr));
 385         attr.start_id = start_id;
 386
 387         err = sys_bpf(BPF_MAP_GET_NEXT_ID, &attr, sizeof(attr));
 388         if (!err)
 389                 *next_id = attr.next_id;
 390
 391         return err;
 392 }
 393
 394 int bpf_prog_get_fd_by_id(__u32 id)
 395 {
 396         union bpf_attr attr;
 397
 398         bzero(&attr, sizeof(attr));
 399         attr.prog_id = id;
 400
 401         return sys_bpf(BPF_PROG_GET_FD_BY_ID, &attr, sizeof(attr));
 402 }
 403
 404 int bpf_map_get_fd_by_id(__u32 id)
 405 {
 406         union bpf_attr attr;
 407
 408         bzero(&attr, sizeof(attr));
 409         attr.map_id = id;
 410
 411         return sys_bpf(BPF_MAP_GET_FD_BY_ID, &attr, sizeof(attr));
 412 }
 413
 414 int bpf_obj_get_info_by_fd(int prog_fd, void *info, __u32 *info_len)
 415 {
 416         union bpf_attr attr;
 417         int err;
 418
 419         bzero(&attr, sizeof(attr));
 420         attr.info.bpf_fd = prog_fd;
 421         attr.info.info_len = *info_len;
 422         attr.info.info = ptr_to_u64(info);
 423
 424         err = sys_bpf(BPF_OBJ_GET_INFO_BY_FD, &attr, sizeof(attr));
 425         if (!err)
 426                 *info_len = attr.info.info_len;
 427
 428         return err;
 429 }
 430
 431 int bpf_set_link_xdp_fd(int ifindex, int fd, __u32 flags)
 432 {
 433         struct sockaddr_nl sa;
 434         int sock, seq = 0, len, ret = -1;
 435         char buf[4096];
 436         struct nlattr *nla, *nla_xdp;
 437         struct {
 438                 struct nlmsghdr  nh;
 439                 struct ifinfomsg ifinfo;
 440                 char             attrbuf[64];
 441         } req;
 442         struct nlmsghdr *nh;
 443         struct nlmsgerr *err;
 444         socklen_t addrlen;
 445         int one = 1;
 446
 447         memset(&sa, 0, sizeof(sa));
 448         sa.nl_family = AF_NETLINK;
 449
 450         sock = socket(AF_NETLINK, SOCK_RAW, NETLINK_ROUTE);
 451         if (sock < 0) {
 452                 return -errno;
 453         }
 454
 455         if (setsockopt(sock, SOL_NETLINK, NETLINK_EXT_ACK,
 456                        &one, sizeof(one)) < 0) {
 457                 fprintf(stderr, "Netlink error reporting not supported\n");
 458         }
 459
 460         if (bind(sock, (struct sockaddr *)&sa, sizeof(sa)) < 0) {
 461                 ret = -errno;
 462                 goto cleanup;
 463         }
 464
 465         addrlen = sizeof(sa);
 466         if (getsockname(sock, (struct sockaddr *)&sa, &addrlen) < 0) {
 467                 ret = -errno;
 468                 goto cleanup;
 469         }
 470
 471         if (addrlen != sizeof(sa)) {
 472                 ret = -LIBBPF_ERRNO__INTERNAL;
 473                 goto cleanup;
 474         }
 475
 476         memset(&req, 0, sizeof(req));
 477         req.nh.nlmsg_len = NLMSG_LENGTH(sizeof(struct ifinfomsg));
 478         req.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK;
 479         req.nh.nlmsg_type = RTM_SETLINK;
 480         req.nh.nlmsg_pid = 0;
 481         req.nh.nlmsg_seq = ++seq;
 482         req.ifinfo.ifi_family = AF_UNSPEC;
 483         req.ifinfo.ifi_index = ifindex;
 484
 485         /* started nested attribute for XDP */
 486         nla = (struct nlattr *)(((char *)&req)
 487                                 + NLMSG_ALIGN(req.nh.nlmsg_len));
 488         nla->nla_type = NLA_F_NESTED | IFLA_XDP;
 489         nla->nla_len = NLA_HDRLEN;
 490
 491         /* add XDP fd */
 492         nla_xdp = (struct nlattr *)((char *)nla + nla->nla_len);
 493         nla_xdp->nla_type = IFLA_XDP_FD;
 494         nla_xdp->nla_len = NLA_HDRLEN + sizeof(int);
 495         memcpy((char *)nla_xdp + NLA_HDRLEN, &fd, sizeof(fd));
 496         nla->nla_len += nla_xdp->nla_len;
 497
 498         /* if user passed in any flags, add those too */
 499         if (flags) {
 500                 nla_xdp = (struct nlattr *)((char *)nla + nla->nla_len);
 501                 nla_xdp->nla_type = IFLA_XDP_FLAGS;
 502                 nla_xdp->nla_len = NLA_HDRLEN + sizeof(flags);
 503                 memcpy((char *)nla_xdp + NLA_HDRLEN, &flags, sizeof(flags));
 504                 nla->nla_len += nla_xdp->nla_len;
 505         }
 506
 507         req.nh.nlmsg_len += NLA_ALIGN(nla->nla_len);
 508
 509         if (send(sock, &req, req.nh.nlmsg_len, 0) < 0) {
 510                 ret = -errno;
 511                 goto cleanup;
 512         }
 513
 514         len = recv(sock, buf, sizeof(buf), 0);
 515         if (len < 0) {
 516                 ret = -errno;
 517                 goto cleanup;
 518         }
 519
 520         for (nh = (struct nlmsghdr *)buf; NLMSG_OK(nh, len);
 521              nh = NLMSG_NEXT(nh, len)) {
 522                 if (nh->nlmsg_pid != sa.nl_pid) {
 523                         ret = -LIBBPF_ERRNO__WRNGPID;
 524                         goto cleanup;
 525                 }
 526                 if (nh->nlmsg_seq != seq) {
 527                         ret = -LIBBPF_ERRNO__INVSEQ;
 528                         goto cleanup;
 529                 }
 530                 switch (nh->nlmsg_type) {
 531                 case NLMSG_ERROR:
 532                         err = (struct nlmsgerr *)NLMSG_DATA(nh);
 533                         if (!err->error)
 534                                 continue;
 535                         ret = err->error;
 536                         nla_dump_errormsg(nh);
 537                         goto cleanup;
 538                 case NLMSG_DONE:
 539                         break;
 540                 default:
 541                         break;
 542                 }
 543         }
 544
 545         ret = 0;
 546
 547 cleanup:
 548         close(sock);
 549         return ret;
 550 }