tools/lib/bpf/netlink.c

   1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
   2 /* Copyright (c) 2018 Facebook */
   3
   4 #include <stdlib.h>
   5 #include <memory.h>
   6 #include <unistd.h>
   7 #include <arpa/inet.h>
   8 #include <linux/bpf.h>
   9 #include <linux/if_ether.h>
  10 #include <linux/pkt_cls.h>
  11 #include <linux/rtnetlink.h>
  12 #include <linux/netdev.h>
  13 #include <sys/socket.h>
  14 #include <errno.h>
  15 #include <time.h>
  16
  17 #include "bpf.h"
  18 #include "libbpf.h"
  19 #include "libbpf_internal.h"
  20 #include "nlattr.h"
  21
  22 #ifndef SOL_NETLINK
  23 #define SOL_NETLINK 270
  24 #endif
  25
  26 typedef int (*libbpf_dump_nlmsg_t)(void *cookie, void *msg, struct nlattr **tb);
  27
  28 typedef int (*__dump_nlmsg_t)(struct nlmsghdr *nlmsg, libbpf_dump_nlmsg_t,
  29                               void *cookie);
  30
  31 struct xdp_link_info {
  32         __u32 prog_id;
  33         __u32 drv_prog_id;
  34         __u32 hw_prog_id;
  35         __u32 skb_prog_id;
  36         __u8 attach_mode;
  37 };
  38
  39 struct xdp_id_md {
  40         int ifindex;
  41         __u32 flags;
  42         struct xdp_link_info info;
  43         __u64 feature_flags;
  44 };
  45
  46 struct xdp_features_md {
  47         int ifindex;
  48         __u32 xdp_zc_max_segs;
  49         __u64 flags;
  50 };
  51
  52 static int libbpf_netlink_open(__u32 *nl_pid, int proto)
  53 {
  54         struct sockaddr_nl sa;
  55         socklen_t addrlen;
  56         int one = 1, ret;
  57         int sock;
  58
  59         memset(&sa, 0, sizeof(sa));
  60         sa.nl_family = AF_NETLINK;
  61
  62         sock = socket(AF_NETLINK, SOCK_RAW | SOCK_CLOEXEC, proto);
  63         if (sock < 0)
  64                 return -errno;
  65
  66         if (setsockopt(sock, SOL_NETLINK, NETLINK_EXT_ACK,
  67                        &one, sizeof(one)) < 0) {
  68                 pr_warn("Netlink error reporting not supported\n");
  69         }
  70
  71         if (bind(sock, (struct sockaddr *)&sa, sizeof(sa)) < 0) {
  72                 ret = -errno;
  73                 goto cleanup;
  74         }
  75
  76         addrlen = sizeof(sa);
  77         if (getsockname(sock, (struct sockaddr *)&sa, &addrlen) < 0) {
  78                 ret = -errno;
  79                 goto cleanup;
  80         }
  81
  82         if (addrlen != sizeof(sa)) {
  83                 ret = -LIBBPF_ERRNO__INTERNAL;
  84                 goto cleanup;
  85         }
  86
  87         *nl_pid = sa.nl_pid;
  88         return sock;
  89
  90 cleanup:
  91         close(sock);
  92         return ret;
  93 }
  94
  95 static void libbpf_netlink_close(int sock)
  96 {
  97         close(sock);
  98 }
  99
 100 enum {
 101         NL_CONT,
 102         NL_NEXT,
 103         NL_DONE,
 104 };
 105
 106 static int netlink_recvmsg(int sock, struct msghdr *mhdr, int flags)
 107 {
 108         int len;
 109
 110         do {
 111                 len = recvmsg(sock, mhdr, flags);
 112         } while (len < 0 && (errno == EINTR || errno == EAGAIN));
 113
 114         if (len < 0)
 115                 return -errno;
 116         return len;
 117 }
 118
 119 static int alloc_iov(struct iovec *iov, int len)
 120 {
 121         void *nbuf;
 122
 123         nbuf = realloc(iov->iov_base, len);
 124         if (!nbuf)
 125                 return -ENOMEM;
 126
 127         iov->iov_base = nbuf;
 128         iov->iov_len = len;
 129         return 0;
 130 }
 131
 132 static int libbpf_netlink_recv(int sock, __u32 nl_pid, int seq,
 133                                __dump_nlmsg_t _fn, libbpf_dump_nlmsg_t fn,
 134                                void *cookie)
 135 {
 136         struct iovec iov = {};
 137         struct msghdr mhdr = {
 138                 .msg_iov = &iov,
 139                 .msg_iovlen = 1,
 140         };
 141         bool multipart = true;
 142         struct nlmsgerr *err;
 143         struct nlmsghdr *nh;
 144         int len, ret;
 145
 146         ret = alloc_iov(&iov, 4096);
 147         if (ret)
 148                 goto done;
 149
 150         while (multipart) {
 151 start:
 152                 multipart = false;
 153                 len = netlink_recvmsg(sock, &mhdr, MSG_PEEK | MSG_TRUNC);
 154                 if (len < 0) {
 155                         ret = len;
 156                         goto done;
 157                 }
 158
 159                 if (len > iov.iov_len) {
 160                         ret = alloc_iov(&iov, len);
 161                         if (ret)
 162                                 goto done;
 163                 }
 164
 165                 len = netlink_recvmsg(sock, &mhdr, 0);
 166                 if (len < 0) {
 167                         ret = len;
 168                         goto done;
 169                 }
 170
 171                 if (len == 0)
 172                         break;
 173
 174                 for (nh = (struct nlmsghdr *)iov.iov_base; NLMSG_OK(nh, len);
 175                      nh = NLMSG_NEXT(nh, len)) {
 176                         if (nh->nlmsg_pid != nl_pid) {
 177                                 ret = -LIBBPF_ERRNO__WRNGPID;
 178                                 goto done;
 179                         }
 180                         if (nh->nlmsg_seq != seq) {
 181                                 ret = -LIBBPF_ERRNO__INVSEQ;
 182                                 goto done;
 183                         }
 184                         if (nh->nlmsg_flags & NLM_F_MULTI)
 185                                 multipart = true;
 186                         switch (nh->nlmsg_type) {
 187                         case NLMSG_ERROR:
 188                                 err = (struct nlmsgerr *)NLMSG_DATA(nh);
 189                                 if (!err->error)
 190                                         continue;
 191                                 ret = err->error;
 192                                 libbpf_nla_dump_errormsg(nh);
 193                                 goto done;
 194                         case NLMSG_DONE:
 195                                 ret = 0;
 196                                 goto done;
 197                         default:
 198                                 break;
 199                         }
 200                         if (_fn) {
 201                                 ret = _fn(nh, fn, cookie);
 202                                 switch (ret) {
 203                                 case NL_CONT:
 204                                         break;
 205                                 case NL_NEXT:
 206                                         goto start;
 207                                 case NL_DONE:
 208                                         ret = 0;
 209                                         goto done;
 210                                 default:
 211                                         goto done;
 212                                 }
 213                         }
 214                 }
 215         }
 216         ret = 0;
 217 done:
 218         free(iov.iov_base);
 219         return ret;
 220 }
 221
 222 static int libbpf_netlink_send_recv(struct libbpf_nla_req *req,
 223                                     int proto, __dump_nlmsg_t parse_msg,
 224                                     libbpf_dump_nlmsg_t parse_attr,
 225                                     void *cookie)
 226 {
 227         __u32 nl_pid = 0;
 228         int sock, ret;
 229
 230         sock = libbpf_netlink_open(&nl_pid, proto);
 231         if (sock < 0)
 232                 return sock;
 233
 234         req->nh.nlmsg_pid = 0;
 235         req->nh.nlmsg_seq = time(NULL);
 236
 237         if (send(sock, req, req->nh.nlmsg_len, 0) < 0) {
 238                 ret = -errno;
 239                 goto out;
 240         }
 241
 242         ret = libbpf_netlink_recv(sock, nl_pid, req->nh.nlmsg_seq,
 243                                   parse_msg, parse_attr, cookie);
 244 out:
 245         libbpf_netlink_close(sock);
 246         return ret;
 247 }
 248
 249 static int parse_genl_family_id(struct nlmsghdr *nh, libbpf_dump_nlmsg_t fn,
 250                                 void *cookie)
 251 {
 252         struct genlmsghdr *gnl = NLMSG_DATA(nh);
 253         struct nlattr *na = (struct nlattr *)((void *)gnl + GENL_HDRLEN);
 254         struct nlattr *tb[CTRL_ATTR_FAMILY_ID + 1];
 255         __u16 *id = cookie;
 256
 257         libbpf_nla_parse(tb, CTRL_ATTR_FAMILY_ID, na,
 258                          NLMSG_PAYLOAD(nh, sizeof(*gnl)), NULL);
 259         if (!tb[CTRL_ATTR_FAMILY_ID])
 260                 return NL_CONT;
 261
 262         *id = libbpf_nla_getattr_u16(tb[CTRL_ATTR_FAMILY_ID]);
 263         return NL_DONE;
 264 }
 265
 266 static int libbpf_netlink_resolve_genl_family_id(const char *name,
 267                                                  __u16 len, __u16 *id)
 268 {
 269         struct libbpf_nla_req req = {
 270                 .nh.nlmsg_len   = NLMSG_LENGTH(GENL_HDRLEN),
 271                 .nh.nlmsg_type  = GENL_ID_CTRL,
 272                 .nh.nlmsg_flags = NLM_F_REQUEST,
 273                 .gnl.cmd        = CTRL_CMD_GETFAMILY,
 274                 .gnl.version    = 2,
 275         };
 276         int err;
 277
 278         err = nlattr_add(&req, CTRL_ATTR_FAMILY_NAME, name, len);
 279         if (err < 0)
 280                 return err;
 281
 282         return libbpf_netlink_send_recv(&req, NETLINK_GENERIC,
 283                                         parse_genl_family_id, NULL, id);
 284 }
 285
 286 static int __bpf_set_link_xdp_fd_replace(int ifindex, int fd, int old_fd,
 287                                          __u32 flags)
 288 {
 289         struct nlattr *nla;
 290         int ret;
 291         struct libbpf_nla_req req;
 292
 293         memset(&req, 0, sizeof(req));
 294         req.nh.nlmsg_len      = NLMSG_LENGTH(sizeof(struct ifinfomsg));
 295         req.nh.nlmsg_flags    = NLM_F_REQUEST | NLM_F_ACK;
 296         req.nh.nlmsg_type     = RTM_SETLINK;
 297         req.ifinfo.ifi_family = AF_UNSPEC;
 298         req.ifinfo.ifi_index  = ifindex;
 299
 300         nla = nlattr_begin_nested(&req, IFLA_XDP);
 301         if (!nla)
 302                 return -EMSGSIZE;
 303         ret = nlattr_add(&req, IFLA_XDP_FD, &fd, sizeof(fd));
 304         if (ret < 0)
 305                 return ret;
 306         if (flags) {
 307                 ret = nlattr_add(&req, IFLA_XDP_FLAGS, &flags, sizeof(flags));
 308                 if (ret < 0)
 309                         return ret;
 310         }
 311         if (flags & XDP_FLAGS_REPLACE) {
 312                 ret = nlattr_add(&req, IFLA_XDP_EXPECTED_FD, &old_fd,
 313                                  sizeof(old_fd));
 314                 if (ret < 0)
 315                         return ret;
 316         }
 317         nlattr_end_nested(&req, nla);
 318
 319         return libbpf_netlink_send_recv(&req, NETLINK_ROUTE, NULL, NULL, NULL);
 320 }
 321
 322 int bpf_xdp_attach(int ifindex, int prog_fd, __u32 flags, const struct bpf_xdp_attach_opts *opts)
 323 {
 324         int old_prog_fd, err;
 325
 326         if (!OPTS_VALID(opts, bpf_xdp_attach_opts))
 327                 return libbpf_err(-EINVAL);
 328
 329         old_prog_fd = OPTS_GET(opts, old_prog_fd, 0);
 330         if (old_prog_fd)
 331                 flags |= XDP_FLAGS_REPLACE;
 332         else
 333                 old_prog_fd = -1;
 334
 335         err = __bpf_set_link_xdp_fd_replace(ifindex, prog_fd, old_prog_fd, flags);
 336         return libbpf_err(err);
 337 }
 338
 339 int bpf_xdp_detach(int ifindex, __u32 flags, const struct bpf_xdp_attach_opts *opts)
 340 {
 341         return bpf_xdp_attach(ifindex, -1, flags, opts);
 342 }
 343
 344 static int __dump_link_nlmsg(struct nlmsghdr *nlh,
 345                              libbpf_dump_nlmsg_t dump_link_nlmsg, void *cookie)
 346 {
 347         struct nlattr *tb[IFLA_MAX + 1], *attr;
 348         struct ifinfomsg *ifi = NLMSG_DATA(nlh);
 349         int len;
 350
 351         len = nlh->nlmsg_len - NLMSG_LENGTH(sizeof(*ifi));
 352         attr = (struct nlattr *) ((void *) ifi + NLMSG_ALIGN(sizeof(*ifi)));
 353
 354         if (libbpf_nla_parse(tb, IFLA_MAX, attr, len, NULL) != 0)
 355                 return -LIBBPF_ERRNO__NLPARSE;
 356
 357         return dump_link_nlmsg(cookie, ifi, tb);
 358 }
 359
 360 static int get_xdp_info(void *cookie, void *msg, struct nlattr **tb)
 361 {
 362         struct nlattr *xdp_tb[IFLA_XDP_MAX + 1];
 363         struct xdp_id_md *xdp_id = cookie;
 364         struct ifinfomsg *ifinfo = msg;
 365         int ret;
 366
 367         if (xdp_id->ifindex && xdp_id->ifindex != ifinfo->ifi_index)
 368                 return 0;
 369
 370         if (!tb[IFLA_XDP])
 371                 return 0;
 372
 373         ret = libbpf_nla_parse_nested(xdp_tb, IFLA_XDP_MAX, tb[IFLA_XDP], NULL);
 374         if (ret)
 375                 return ret;
 376
 377         if (!xdp_tb[IFLA_XDP_ATTACHED])
 378                 return 0;
 379
 380         xdp_id->info.attach_mode = libbpf_nla_getattr_u8(
 381                 xdp_tb[IFLA_XDP_ATTACHED]);
 382
 383         if (xdp_id->info.attach_mode == XDP_ATTACHED_NONE)
 384                 return 0;
 385
 386         if (xdp_tb[IFLA_XDP_PROG_ID])
 387                 xdp_id->info.prog_id = libbpf_nla_getattr_u32(
 388                         xdp_tb[IFLA_XDP_PROG_ID]);
 389
 390         if (xdp_tb[IFLA_XDP_SKB_PROG_ID])
 391                 xdp_id->info.skb_prog_id = libbpf_nla_getattr_u32(
 392                         xdp_tb[IFLA_XDP_SKB_PROG_ID]);
 393
 394         if (xdp_tb[IFLA_XDP_DRV_PROG_ID])
 395                 xdp_id->info.drv_prog_id = libbpf_nla_getattr_u32(
 396                         xdp_tb[IFLA_XDP_DRV_PROG_ID]);
 397
 398         if (xdp_tb[IFLA_XDP_HW_PROG_ID])
 399                 xdp_id->info.hw_prog_id = libbpf_nla_getattr_u32(
 400                         xdp_tb[IFLA_XDP_HW_PROG_ID]);
 401
 402         return 0;
 403 }
 404
 405 static int parse_xdp_features(struct nlmsghdr *nh, libbpf_dump_nlmsg_t fn,
 406                               void *cookie)
 407 {
 408         struct genlmsghdr *gnl = NLMSG_DATA(nh);
 409         struct nlattr *na = (struct nlattr *)((void *)gnl + GENL_HDRLEN);
 410         struct nlattr *tb[NETDEV_CMD_MAX + 1];
 411         struct xdp_features_md *md = cookie;
 412         __u32 ifindex;
 413
 414         libbpf_nla_parse(tb, NETDEV_CMD_MAX, na,
 415                          NLMSG_PAYLOAD(nh, sizeof(*gnl)), NULL);
 416
 417         if (!tb[NETDEV_A_DEV_IFINDEX] || !tb[NETDEV_A_DEV_XDP_FEATURES])
 418                 return NL_CONT;
 419
 420         ifindex = libbpf_nla_getattr_u32(tb[NETDEV_A_DEV_IFINDEX]);
 421         if (ifindex != md->ifindex)
 422                 return NL_CONT;
 423
 424         md->flags = libbpf_nla_getattr_u64(tb[NETDEV_A_DEV_XDP_FEATURES]);
 425         if (tb[NETDEV_A_DEV_XDP_ZC_MAX_SEGS])
 426                 md->xdp_zc_max_segs =
 427                         libbpf_nla_getattr_u32(tb[NETDEV_A_DEV_XDP_ZC_MAX_SEGS]);
 428         return NL_DONE;
 429 }
 430
 431 int bpf_xdp_query(int ifindex, int xdp_flags, struct bpf_xdp_query_opts *opts)
 432 {
 433         struct libbpf_nla_req req = {
 434                 .nh.nlmsg_len      = NLMSG_LENGTH(sizeof(struct ifinfomsg)),
 435                 .nh.nlmsg_type     = RTM_GETLINK,
 436                 .nh.nlmsg_flags    = NLM_F_DUMP | NLM_F_REQUEST,
 437                 .ifinfo.ifi_family = AF_PACKET,
 438         };
 439         struct xdp_id_md xdp_id = {};
 440         struct xdp_features_md md = {
 441                 .ifindex = ifindex,
 442         };
 443         __u16 id;
 444         int err;
 445
 446         if (!OPTS_VALID(opts, bpf_xdp_query_opts))
 447                 return libbpf_err(-EINVAL);
 448
 449         if (xdp_flags & ~XDP_FLAGS_MASK)
 450                 return libbpf_err(-EINVAL);
 451
 452         /* Check whether the single {HW,DRV,SKB} mode is set */
 453         xdp_flags &= XDP_FLAGS_SKB_MODE | XDP_FLAGS_DRV_MODE | XDP_FLAGS_HW_MODE;
 454         if (xdp_flags & (xdp_flags - 1))
 455                 return libbpf_err(-EINVAL);
 456
 457         xdp_id.ifindex = ifindex;
 458         xdp_id.flags = xdp_flags;
 459
 460         err = libbpf_netlink_send_recv(&req, NETLINK_ROUTE, __dump_link_nlmsg,
 461                                        get_xdp_info, &xdp_id);
 462         if (err)
 463                 return libbpf_err(err);
 464
 465         OPTS_SET(opts, prog_id, xdp_id.info.prog_id);
 466         OPTS_SET(opts, drv_prog_id, xdp_id.info.drv_prog_id);
 467         OPTS_SET(opts, hw_prog_id, xdp_id.info.hw_prog_id);
 468         OPTS_SET(opts, skb_prog_id, xdp_id.info.skb_prog_id);
 469         OPTS_SET(opts, attach_mode, xdp_id.info.attach_mode);
 470
 471         if (!OPTS_HAS(opts, feature_flags))
 472                 return 0;
 473
 474         err = libbpf_netlink_resolve_genl_family_id("netdev", sizeof("netdev"), &id);
 475         if (err < 0) {
 476                 if (err == -ENOENT) {
 477                         opts->feature_flags = 0;
 478                         goto skip_feature_flags;
 479                 }
 480                 return libbpf_err(err);
 481         }
 482
 483         memset(&req, 0, sizeof(req));
 484         req.nh.nlmsg_len = NLMSG_LENGTH(GENL_HDRLEN);
 485         req.nh.nlmsg_flags = NLM_F_REQUEST;
 486         req.nh.nlmsg_type = id;
 487         req.gnl.cmd = NETDEV_CMD_DEV_GET;
 488         req.gnl.version = 2;
 489
 490         err = nlattr_add(&req, NETDEV_A_DEV_IFINDEX, &ifindex, sizeof(ifindex));
 491         if (err < 0)
 492                 return libbpf_err(err);
 493
 494         err = libbpf_netlink_send_recv(&req, NETLINK_GENERIC,
 495                                        parse_xdp_features, NULL, &md);
 496         if (err)
 497                 return libbpf_err(err);
 498
 499         OPTS_SET(opts, feature_flags, md.flags);
 500         OPTS_SET(opts, xdp_zc_max_segs, md.xdp_zc_max_segs);
 501
 502 skip_feature_flags:
 503         return 0;
 504 }
 505
 506 int bpf_xdp_query_id(int ifindex, int flags, __u32 *prog_id)
 507 {
 508         LIBBPF_OPTS(bpf_xdp_query_opts, opts);
 509         int ret;
 510
 511         ret = bpf_xdp_query(ifindex, flags, &opts);
 512         if (ret)
 513                 return libbpf_err(ret);
 514
 515         flags &= XDP_FLAGS_MODES;
 516
 517         if (opts.attach_mode != XDP_ATTACHED_MULTI && !flags)
 518                 *prog_id = opts.prog_id;
 519         else if (flags & XDP_FLAGS_DRV_MODE)
 520                 *prog_id = opts.drv_prog_id;
 521         else if (flags & XDP_FLAGS_HW_MODE)
 522                 *prog_id = opts.hw_prog_id;
 523         else if (flags & XDP_FLAGS_SKB_MODE)
 524                 *prog_id = opts.skb_prog_id;
 525         else
 526                 *prog_id = 0;
 527
 528         return 0;
 529 }
 530
 531
 532 typedef int (*qdisc_config_t)(struct libbpf_nla_req *req);
 533
 534 static int clsact_config(struct libbpf_nla_req *req)
 535 {
 536         req->tc.tcm_parent = TC_H_CLSACT;
 537         req->tc.tcm_handle = TC_H_MAKE(TC_H_CLSACT, 0);
 538
 539         return nlattr_add(req, TCA_KIND, "clsact", sizeof("clsact"));
 540 }
 541
 542 static int attach_point_to_config(struct bpf_tc_hook *hook,
 543                                   qdisc_config_t *config)
 544 {
 545         switch (OPTS_GET(hook, attach_point, 0)) {
 546         case BPF_TC_INGRESS:
 547         case BPF_TC_EGRESS:
 548         case BPF_TC_INGRESS | BPF_TC_EGRESS:
 549                 if (OPTS_GET(hook, parent, 0))
 550                         return -EINVAL;
 551                 *config = &clsact_config;
 552                 return 0;
 553         case BPF_TC_CUSTOM:
 554                 return -EOPNOTSUPP;
 555         default:
 556                 return -EINVAL;
 557         }
 558 }
 559
 560 static int tc_get_tcm_parent(enum bpf_tc_attach_point attach_point,
 561                              __u32 *parent)
 562 {
 563         switch (attach_point) {
 564         case BPF_TC_INGRESS:
 565         case BPF_TC_EGRESS:
 566                 if (*parent)
 567                         return -EINVAL;
 568                 *parent = TC_H_MAKE(TC_H_CLSACT,
 569                                     attach_point == BPF_TC_INGRESS ?
 570                                     TC_H_MIN_INGRESS : TC_H_MIN_EGRESS);
 571                 break;
 572         case BPF_TC_CUSTOM:
 573                 if (!*parent)
 574                         return -EINVAL;
 575                 break;
 576         default:
 577                 return -EINVAL;
 578         }
 579         return 0;
 580 }
 581
 582 static int tc_qdisc_modify(struct bpf_tc_hook *hook, int cmd, int flags)
 583 {
 584         qdisc_config_t config;
 585         int ret;
 586         struct libbpf_nla_req req;
 587
 588         ret = attach_point_to_config(hook, &config);
 589         if (ret < 0)
 590                 return ret;
 591
 592         memset(&req, 0, sizeof(req));
 593         req.nh.nlmsg_len   = NLMSG_LENGTH(sizeof(struct tcmsg));
 594         req.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK | flags;
 595         req.nh.nlmsg_type  = cmd;
 596         req.tc.tcm_family  = AF_UNSPEC;
 597         req.tc.tcm_ifindex = OPTS_GET(hook, ifindex, 0);
 598
 599         ret = config(&req);
 600         if (ret < 0)
 601                 return ret;
 602
 603         return libbpf_netlink_send_recv(&req, NETLINK_ROUTE, NULL, NULL, NULL);
 604 }
 605
 606 static int tc_qdisc_create_excl(struct bpf_tc_hook *hook)
 607 {
 608         return tc_qdisc_modify(hook, RTM_NEWQDISC, NLM_F_CREATE | NLM_F_EXCL);
 609 }
 610
 611 static int tc_qdisc_delete(struct bpf_tc_hook *hook)
 612 {
 613         return tc_qdisc_modify(hook, RTM_DELQDISC, 0);
 614 }
 615
 616 int bpf_tc_hook_create(struct bpf_tc_hook *hook)
 617 {
 618         int ret;
 619
 620         if (!hook || !OPTS_VALID(hook, bpf_tc_hook) ||
 621             OPTS_GET(hook, ifindex, 0) <= 0)
 622                 return libbpf_err(-EINVAL);
 623
 624         ret = tc_qdisc_create_excl(hook);
 625         return libbpf_err(ret);
 626 }
 627
 628 static int __bpf_tc_detach(const struct bpf_tc_hook *hook,
 629                            const struct bpf_tc_opts *opts,
 630                            const bool flush);
 631
 632 int bpf_tc_hook_destroy(struct bpf_tc_hook *hook)
 633 {
 634         if (!hook || !OPTS_VALID(hook, bpf_tc_hook) ||
 635             OPTS_GET(hook, ifindex, 0) <= 0)
 636                 return libbpf_err(-EINVAL);
 637
 638         switch (OPTS_GET(hook, attach_point, 0)) {
 639         case BPF_TC_INGRESS:
 640         case BPF_TC_EGRESS:
 641                 return libbpf_err(__bpf_tc_detach(hook, NULL, true));
 642         case BPF_TC_INGRESS | BPF_TC_EGRESS:
 643                 return libbpf_err(tc_qdisc_delete(hook));
 644         case BPF_TC_CUSTOM:
 645                 return libbpf_err(-EOPNOTSUPP);
 646         default:
 647                 return libbpf_err(-EINVAL);
 648         }
 649 }
 650
 651 struct bpf_cb_ctx {
 652         struct bpf_tc_opts *opts;
 653         bool processed;
 654 };
 655
 656 static int __get_tc_info(void *cookie, struct tcmsg *tc, struct nlattr **tb,
 657                          bool unicast)
 658 {
 659         struct nlattr *tbb[TCA_BPF_MAX + 1];
 660         struct bpf_cb_ctx *info = cookie;
 661
 662         if (!info || !info->opts)
 663                 return -EINVAL;
 664         if (unicast && info->processed)
 665                 return -EINVAL;
 666         if (!tb[TCA_OPTIONS])
 667                 return NL_CONT;
 668
 669         libbpf_nla_parse_nested(tbb, TCA_BPF_MAX, tb[TCA_OPTIONS], NULL);
 670         if (!tbb[TCA_BPF_ID])
 671                 return -EINVAL;
 672
 673         OPTS_SET(info->opts, prog_id, libbpf_nla_getattr_u32(tbb[TCA_BPF_ID]));
 674         OPTS_SET(info->opts, handle, tc->tcm_handle);
 675         OPTS_SET(info->opts, priority, TC_H_MAJ(tc->tcm_info) >> 16);
 676
 677         info->processed = true;
 678         return unicast ? NL_NEXT : NL_DONE;
 679 }
 680
 681 static int get_tc_info(struct nlmsghdr *nh, libbpf_dump_nlmsg_t fn,
 682                        void *cookie)
 683 {
 684         struct tcmsg *tc = NLMSG_DATA(nh);
 685         struct nlattr *tb[TCA_MAX + 1];
 686
 687         libbpf_nla_parse(tb, TCA_MAX,
 688                          (struct nlattr *)((void *)tc + NLMSG_ALIGN(sizeof(*tc))),
 689                          NLMSG_PAYLOAD(nh, sizeof(*tc)), NULL);
 690         if (!tb[TCA_KIND])
 691                 return NL_CONT;
 692         return __get_tc_info(cookie, tc, tb, nh->nlmsg_flags & NLM_F_ECHO);
 693 }
 694
 695 static int tc_add_fd_and_name(struct libbpf_nla_req *req, int fd)
 696 {
 697         struct bpf_prog_info info;
 698         __u32 info_len = sizeof(info);
 699         char name[256];
 700         int len, ret;
 701
 702         memset(&info, 0, info_len);
 703         ret = bpf_prog_get_info_by_fd(fd, &info, &info_len);
 704         if (ret < 0)
 705                 return ret;
 706
 707         ret = nlattr_add(req, TCA_BPF_FD, &fd, sizeof(fd));
 708         if (ret < 0)
 709                 return ret;
 710         len = snprintf(name, sizeof(name), "%s:[%u]", info.name, info.id);
 711         if (len < 0)
 712                 return -errno;
 713         if (len >= sizeof(name))
 714                 return -ENAMETOOLONG;
 715         return nlattr_add(req, TCA_BPF_NAME, name, len + 1);
 716 }
 717
 718 int bpf_tc_attach(const struct bpf_tc_hook *hook, struct bpf_tc_opts *opts)
 719 {
 720         __u32 protocol, bpf_flags, handle, priority, parent, prog_id, flags;
 721         int ret, ifindex, attach_point, prog_fd;
 722         struct bpf_cb_ctx info = {};
 723         struct libbpf_nla_req req;
 724         struct nlattr *nla;
 725
 726         if (!hook || !opts ||
 727             !OPTS_VALID(hook, bpf_tc_hook) ||
 728             !OPTS_VALID(opts, bpf_tc_opts))
 729                 return libbpf_err(-EINVAL);
 730
 731         ifindex      = OPTS_GET(hook, ifindex, 0);
 732         parent       = OPTS_GET(hook, parent, 0);
 733         attach_point = OPTS_GET(hook, attach_point, 0);
 734
 735         handle       = OPTS_GET(opts, handle, 0);
 736         priority     = OPTS_GET(opts, priority, 0);
 737         prog_fd      = OPTS_GET(opts, prog_fd, 0);
 738         prog_id      = OPTS_GET(opts, prog_id, 0);
 739         flags        = OPTS_GET(opts, flags, 0);
 740
 741         if (ifindex <= 0 || !prog_fd || prog_id)
 742                 return libbpf_err(-EINVAL);
 743         if (priority > UINT16_MAX)
 744                 return libbpf_err(-EINVAL);
 745         if (flags & ~BPF_TC_F_REPLACE)
 746                 return libbpf_err(-EINVAL);
 747
 748         flags = (flags & BPF_TC_F_REPLACE) ? NLM_F_REPLACE : NLM_F_EXCL;
 749         protocol = ETH_P_ALL;
 750
 751         memset(&req, 0, sizeof(req));
 752         req.nh.nlmsg_len   = NLMSG_LENGTH(sizeof(struct tcmsg));
 753         req.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK | NLM_F_CREATE |
 754                              NLM_F_ECHO | flags;
 755         req.nh.nlmsg_type  = RTM_NEWTFILTER;
 756         req.tc.tcm_family  = AF_UNSPEC;
 757         req.tc.tcm_ifindex = ifindex;
 758         req.tc.tcm_handle  = handle;
 759         req.tc.tcm_info    = TC_H_MAKE(priority << 16, htons(protocol));
 760
 761         ret = tc_get_tcm_parent(attach_point, &parent);
 762         if (ret < 0)
 763                 return libbpf_err(ret);
 764         req.tc.tcm_parent = parent;
 765
 766         ret = nlattr_add(&req, TCA_KIND, "bpf", sizeof("bpf"));
 767         if (ret < 0)
 768                 return libbpf_err(ret);
 769         nla = nlattr_begin_nested(&req, TCA_OPTIONS);
 770         if (!nla)
 771                 return libbpf_err(-EMSGSIZE);
 772         ret = tc_add_fd_and_name(&req, prog_fd);
 773         if (ret < 0)
 774                 return libbpf_err(ret);
 775         bpf_flags = TCA_BPF_FLAG_ACT_DIRECT;
 776         ret = nlattr_add(&req, TCA_BPF_FLAGS, &bpf_flags, sizeof(bpf_flags));
 777         if (ret < 0)
 778                 return libbpf_err(ret);
 779         nlattr_end_nested(&req, nla);
 780
 781         info.opts = opts;
 782
 783         ret = libbpf_netlink_send_recv(&req, NETLINK_ROUTE, get_tc_info, NULL,
 784                                        &info);
 785         if (ret < 0)
 786                 return libbpf_err(ret);
 787         if (!info.processed)
 788                 return libbpf_err(-ENOENT);
 789         return ret;
 790 }
 791
 792 static int __bpf_tc_detach(const struct bpf_tc_hook *hook,
 793                            const struct bpf_tc_opts *opts,
 794                            const bool flush)
 795 {
 796         __u32 protocol = 0, handle, priority, parent, prog_id, flags;
 797         int ret, ifindex, attach_point, prog_fd;
 798         struct libbpf_nla_req req;
 799
 800         if (!hook ||
 801             !OPTS_VALID(hook, bpf_tc_hook) ||
 802             !OPTS_VALID(opts, bpf_tc_opts))
 803                 return -EINVAL;
 804
 805         ifindex      = OPTS_GET(hook, ifindex, 0);
 806         parent       = OPTS_GET(hook, parent, 0);
 807         attach_point = OPTS_GET(hook, attach_point, 0);
 808
 809         handle       = OPTS_GET(opts, handle, 0);
 810         priority     = OPTS_GET(opts, priority, 0);
 811         prog_fd      = OPTS_GET(opts, prog_fd, 0);
 812         prog_id      = OPTS_GET(opts, prog_id, 0);
 813         flags        = OPTS_GET(opts, flags, 0);
 814
 815         if (ifindex <= 0 || flags || prog_fd || prog_id)
 816                 return -EINVAL;
 817         if (priority > UINT16_MAX)
 818                 return -EINVAL;
 819         if (!flush) {
 820                 if (!handle || !priority)
 821                         return -EINVAL;
 822                 protocol = ETH_P_ALL;
 823         } else {
 824                 if (handle || priority)
 825                         return -EINVAL;
 826         }
 827
 828         memset(&req, 0, sizeof(req));
 829         req.nh.nlmsg_len   = NLMSG_LENGTH(sizeof(struct tcmsg));
 830         req.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK;
 831         req.nh.nlmsg_type  = RTM_DELTFILTER;
 832         req.tc.tcm_family  = AF_UNSPEC;
 833         req.tc.tcm_ifindex = ifindex;
 834         if (!flush) {
 835                 req.tc.tcm_handle = handle;
 836                 req.tc.tcm_info   = TC_H_MAKE(priority << 16, htons(protocol));
 837         }
 838
 839         ret = tc_get_tcm_parent(attach_point, &parent);
 840         if (ret < 0)
 841                 return ret;
 842         req.tc.tcm_parent = parent;
 843
 844         if (!flush) {
 845                 ret = nlattr_add(&req, TCA_KIND, "bpf", sizeof("bpf"));
 846                 if (ret < 0)
 847                         return ret;
 848         }
 849
 850         return libbpf_netlink_send_recv(&req, NETLINK_ROUTE, NULL, NULL, NULL);
 851 }
 852
 853 int bpf_tc_detach(const struct bpf_tc_hook *hook,
 854                   const struct bpf_tc_opts *opts)
 855 {
 856         int ret;
 857
 858         if (!opts)
 859                 return libbpf_err(-EINVAL);
 860
 861         ret = __bpf_tc_detach(hook, opts, false);
 862         return libbpf_err(ret);
 863 }
 864
 865 int bpf_tc_query(const struct bpf_tc_hook *hook, struct bpf_tc_opts *opts)
 866 {
 867         __u32 protocol, handle, priority, parent, prog_id, flags;
 868         int ret, ifindex, attach_point, prog_fd;
 869         struct bpf_cb_ctx info = {};
 870         struct libbpf_nla_req req;
 871
 872         if (!hook || !opts ||
 873             !OPTS_VALID(hook, bpf_tc_hook) ||
 874             !OPTS_VALID(opts, bpf_tc_opts))
 875                 return libbpf_err(-EINVAL);
 876
 877         ifindex      = OPTS_GET(hook, ifindex, 0);
 878         parent       = OPTS_GET(hook, parent, 0);
 879         attach_point = OPTS_GET(hook, attach_point, 0);
 880
 881         handle       = OPTS_GET(opts, handle, 0);
 882         priority     = OPTS_GET(opts, priority, 0);
 883         prog_fd      = OPTS_GET(opts, prog_fd, 0);
 884         prog_id      = OPTS_GET(opts, prog_id, 0);
 885         flags        = OPTS_GET(opts, flags, 0);
 886
 887         if (ifindex <= 0 || flags || prog_fd || prog_id ||
 888             !handle || !priority)
 889                 return libbpf_err(-EINVAL);
 890         if (priority > UINT16_MAX)
 891                 return libbpf_err(-EINVAL);
 892
 893         protocol = ETH_P_ALL;
 894
 895         memset(&req, 0, sizeof(req));
 896         req.nh.nlmsg_len   = NLMSG_LENGTH(sizeof(struct tcmsg));
 897         req.nh.nlmsg_flags = NLM_F_REQUEST;
 898         req.nh.nlmsg_type  = RTM_GETTFILTER;
 899         req.tc.tcm_family  = AF_UNSPEC;
 900         req.tc.tcm_ifindex = ifindex;
 901         req.tc.tcm_handle  = handle;
 902         req.tc.tcm_info    = TC_H_MAKE(priority << 16, htons(protocol));
 903
 904         ret = tc_get_tcm_parent(attach_point, &parent);
 905         if (ret < 0)
 906                 return libbpf_err(ret);
 907         req.tc.tcm_parent = parent;
 908
 909         ret = nlattr_add(&req, TCA_KIND, "bpf", sizeof("bpf"));
 910         if (ret < 0)
 911                 return libbpf_err(ret);
 912
 913         info.opts = opts;
 914
 915         ret = libbpf_netlink_send_recv(&req, NETLINK_ROUTE, get_tc_info, NULL,
 916                                        &info);
 917         if (ret < 0)
 918                 return libbpf_err(ret);
 919         if (!info.processed)
 920                 return libbpf_err(-ENOENT);
 921         return ret;
 922 }