drivers/net/netkit.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright (c) 2023 Isovalent */
   3
   4 #include <linux/netdevice.h>
   5 #include <linux/ethtool.h>
   6 #include <linux/etherdevice.h>
   7 #include <linux/filter.h>
   8 #include <linux/netfilter_netdev.h>
   9 #include <linux/bpf_mprog.h>
  10 #include <linux/indirect_call_wrapper.h>
  11
  12 #include <net/netkit.h>
  13 #include <net/dst.h>
  14 #include <net/tcx.h>
  15
  16 #define DRV_NAME "netkit"
  17
  18 struct netkit {
  19         /* Needed in fast-path */
  20         struct net_device __rcu *peer;
  21         struct bpf_mprog_entry __rcu *active;
  22         enum netkit_action policy;
  23         enum netkit_scrub scrub;
  24         struct bpf_mprog_bundle bundle;
  25
  26         /* Needed in slow-path */
  27         enum netkit_mode mode;
  28         bool primary;
  29         u32 headroom;
  30 };
  31
  32 struct netkit_link {
  33         struct bpf_link link;
  34         struct net_device *dev;
  35         u32 location;
  36 };
  37
  38 static __always_inline int
  39 netkit_run(const struct bpf_mprog_entry *entry, struct sk_buff *skb,
  40            enum netkit_action ret)
  41 {
  42         const struct bpf_mprog_fp *fp;
  43         const struct bpf_prog *prog;
  44
  45         bpf_mprog_foreach_prog(entry, fp, prog) {
  46                 bpf_compute_data_pointers(skb);
  47                 ret = bpf_prog_run(prog, skb);
  48                 if (ret != NETKIT_NEXT)
  49                         break;
  50         }
  51         return ret;
  52 }
  53
  54 static void netkit_xnet(struct sk_buff *skb)
  55 {
  56         skb->priority = 0;
  57         skb->mark = 0;
  58 }
  59
  60 static void netkit_prep_forward(struct sk_buff *skb,
  61                                 bool xnet, bool xnet_scrub)
  62 {
  63         skb_scrub_packet(skb, false);
  64         nf_skip_egress(skb, true);
  65         skb_reset_mac_header(skb);
  66         if (!xnet)
  67                 return;
  68         ipvs_reset(skb);
  69         skb_clear_tstamp(skb);
  70         if (xnet_scrub)
  71                 netkit_xnet(skb);
  72 }
  73
  74 static struct netkit *netkit_priv(const struct net_device *dev)
  75 {
  76         return netdev_priv(dev);
  77 }
  78
  79 static netdev_tx_t netkit_xmit(struct sk_buff *skb, struct net_device *dev)
  80 {
  81         struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
  82         struct netkit *nk = netkit_priv(dev);
  83         enum netkit_action ret = READ_ONCE(nk->policy);
  84         netdev_tx_t ret_dev = NET_XMIT_SUCCESS;
  85         const struct bpf_mprog_entry *entry;
  86         struct net_device *peer;
  87         int len = skb->len;
  88
  89         bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
  90         rcu_read_lock();
  91         peer = rcu_dereference(nk->peer);
  92         if (unlikely(!peer || !(peer->flags & IFF_UP) ||
  93                      !pskb_may_pull(skb, ETH_HLEN) ||
  94                      skb_orphan_frags(skb, GFP_ATOMIC)))
  95                 goto drop;
  96         netkit_prep_forward(skb, !net_eq(dev_net(dev), dev_net(peer)),
  97                             nk->scrub);
  98         eth_skb_pkt_type(skb, peer);
  99         skb->dev = peer;
 100         entry = rcu_dereference(nk->active);
 101         if (entry)
 102                 ret = netkit_run(entry, skb, ret);
 103         switch (ret) {
 104         case NETKIT_NEXT:
 105         case NETKIT_PASS:
 106                 eth_skb_pull_mac(skb);
 107                 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
 108                 if (likely(__netif_rx(skb) == NET_RX_SUCCESS)) {
 109                         dev_sw_netstats_tx_add(dev, 1, len);
 110                         dev_sw_netstats_rx_add(peer, len);
 111                 } else {
 112                         goto drop_stats;
 113                 }
 114                 break;
 115         case NETKIT_REDIRECT:
 116                 dev_sw_netstats_tx_add(dev, 1, len);
 117                 skb_do_redirect(skb);
 118                 break;
 119         case NETKIT_DROP:
 120         default:
 121 drop:
 122                 kfree_skb(skb);
 123 drop_stats:
 124                 dev_core_stats_tx_dropped_inc(dev);
 125                 ret_dev = NET_XMIT_DROP;
 126                 break;
 127         }
 128         rcu_read_unlock();
 129         bpf_net_ctx_clear(bpf_net_ctx);
 130         return ret_dev;
 131 }
 132
 133 static int netkit_open(struct net_device *dev)
 134 {
 135         struct netkit *nk = netkit_priv(dev);
 136         struct net_device *peer = rtnl_dereference(nk->peer);
 137
 138         if (!peer)
 139                 return -ENOTCONN;
 140         if (peer->flags & IFF_UP) {
 141                 netif_carrier_on(dev);
 142                 netif_carrier_on(peer);
 143         }
 144         return 0;
 145 }
 146
 147 static int netkit_close(struct net_device *dev)
 148 {
 149         struct netkit *nk = netkit_priv(dev);
 150         struct net_device *peer = rtnl_dereference(nk->peer);
 151
 152         netif_carrier_off(dev);
 153         if (peer)
 154                 netif_carrier_off(peer);
 155         return 0;
 156 }
 157
 158 static int netkit_get_iflink(const struct net_device *dev)
 159 {
 160         struct netkit *nk = netkit_priv(dev);
 161         struct net_device *peer;
 162         int iflink = 0;
 163
 164         rcu_read_lock();
 165         peer = rcu_dereference(nk->peer);
 166         if (peer)
 167                 iflink = READ_ONCE(peer->ifindex);
 168         rcu_read_unlock();
 169         return iflink;
 170 }
 171
 172 static void netkit_set_multicast(struct net_device *dev)
 173 {
 174         /* Nothing to do, we receive whatever gets pushed to us! */
 175 }
 176
 177 static int netkit_set_macaddr(struct net_device *dev, void *sa)
 178 {
 179         struct netkit *nk = netkit_priv(dev);
 180
 181         if (nk->mode != NETKIT_L2)
 182                 return -EOPNOTSUPP;
 183
 184         return eth_mac_addr(dev, sa);
 185 }
 186
 187 static void netkit_set_headroom(struct net_device *dev, int headroom)
 188 {
 189         struct netkit *nk = netkit_priv(dev), *nk2;
 190         struct net_device *peer;
 191
 192         if (headroom < 0)
 193                 headroom = NET_SKB_PAD;
 194
 195         rcu_read_lock();
 196         peer = rcu_dereference(nk->peer);
 197         if (unlikely(!peer))
 198                 goto out;
 199
 200         nk2 = netkit_priv(peer);
 201         nk->headroom = headroom;
 202         headroom = max(nk->headroom, nk2->headroom);
 203
 204         peer->needed_headroom = headroom;
 205         dev->needed_headroom = headroom;
 206 out:
 207         rcu_read_unlock();
 208 }
 209
 210 INDIRECT_CALLABLE_SCOPE struct net_device *netkit_peer_dev(struct net_device *dev)
 211 {
 212         return rcu_dereference(netkit_priv(dev)->peer);
 213 }
 214
 215 static void netkit_get_stats(struct net_device *dev,
 216                              struct rtnl_link_stats64 *stats)
 217 {
 218         dev_fetch_sw_netstats(stats, dev->tstats);
 219         stats->tx_dropped = DEV_STATS_READ(dev, tx_dropped);
 220 }
 221
 222 static void netkit_uninit(struct net_device *dev);
 223
 224 static const struct net_device_ops netkit_netdev_ops = {
 225         .ndo_open               = netkit_open,
 226         .ndo_stop               = netkit_close,
 227         .ndo_start_xmit         = netkit_xmit,
 228         .ndo_set_rx_mode        = netkit_set_multicast,
 229         .ndo_set_rx_headroom    = netkit_set_headroom,
 230         .ndo_set_mac_address    = netkit_set_macaddr,
 231         .ndo_get_iflink         = netkit_get_iflink,
 232         .ndo_get_peer_dev       = netkit_peer_dev,
 233         .ndo_get_stats64        = netkit_get_stats,
 234         .ndo_uninit             = netkit_uninit,
 235         .ndo_features_check     = passthru_features_check,
 236 };
 237
 238 static void netkit_get_drvinfo(struct net_device *dev,
 239                                struct ethtool_drvinfo *info)
 240 {
 241         strscpy(info->driver, DRV_NAME, sizeof(info->driver));
 242 }
 243
 244 static const struct ethtool_ops netkit_ethtool_ops = {
 245         .get_drvinfo            = netkit_get_drvinfo,
 246 };
 247
 248 static void netkit_setup(struct net_device *dev)
 249 {
 250         static const netdev_features_t netkit_features_hw_vlan =
 251                 NETIF_F_HW_VLAN_CTAG_TX |
 252                 NETIF_F_HW_VLAN_CTAG_RX |
 253                 NETIF_F_HW_VLAN_STAG_TX |
 254                 NETIF_F_HW_VLAN_STAG_RX;
 255         static const netdev_features_t netkit_features =
 256                 netkit_features_hw_vlan |
 257                 NETIF_F_SG |
 258                 NETIF_F_FRAGLIST |
 259                 NETIF_F_HW_CSUM |
 260                 NETIF_F_RXCSUM |
 261                 NETIF_F_SCTP_CRC |
 262                 NETIF_F_HIGHDMA |
 263                 NETIF_F_GSO_SOFTWARE |
 264                 NETIF_F_GSO_ENCAP_ALL;
 265
 266         ether_setup(dev);
 267         dev->max_mtu = ETH_MAX_MTU;
 268         dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS;
 269
 270         dev->flags |= IFF_NOARP;
 271         dev->priv_flags &= ~IFF_TX_SKB_SHARING;
 272         dev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
 273         dev->priv_flags |= IFF_PHONY_HEADROOM;
 274         dev->priv_flags |= IFF_NO_QUEUE;
 275         dev->priv_flags |= IFF_DISABLE_NETPOLL;
 276         dev->lltx = true;
 277
 278         dev->ethtool_ops = &netkit_ethtool_ops;
 279         dev->netdev_ops  = &netkit_netdev_ops;
 280
 281         dev->features |= netkit_features;
 282         dev->hw_features = netkit_features;
 283         dev->hw_enc_features = netkit_features;
 284         dev->mpls_features = NETIF_F_HW_CSUM | NETIF_F_GSO_SOFTWARE;
 285         dev->vlan_features = dev->features & ~netkit_features_hw_vlan;
 286
 287         dev->needs_free_netdev = true;
 288
 289         netif_set_tso_max_size(dev, GSO_MAX_SIZE);
 290 }
 291
 292 static struct net *netkit_get_link_net(const struct net_device *dev)
 293 {
 294         struct netkit *nk = netkit_priv(dev);
 295         struct net_device *peer = rtnl_dereference(nk->peer);
 296
 297         return peer ? dev_net(peer) : dev_net(dev);
 298 }
 299
 300 static int netkit_check_policy(int policy, struct nlattr *tb,
 301                                struct netlink_ext_ack *extack)
 302 {
 303         switch (policy) {
 304         case NETKIT_PASS:
 305         case NETKIT_DROP:
 306                 return 0;
 307         default:
 308                 NL_SET_ERR_MSG_ATTR(extack, tb,
 309                                     "Provided default xmit policy not supported");
 310                 return -EINVAL;
 311         }
 312 }
 313
 314 static int netkit_validate(struct nlattr *tb[], struct nlattr *data[],
 315                            struct netlink_ext_ack *extack)
 316 {
 317         struct nlattr *attr = tb[IFLA_ADDRESS];
 318
 319         if (!attr)
 320                 return 0;
 321         if (nla_len(attr) != ETH_ALEN)
 322                 return -EINVAL;
 323         if (!is_valid_ether_addr(nla_data(attr)))
 324                 return -EADDRNOTAVAIL;
 325         return 0;
 326 }
 327
 328 static struct rtnl_link_ops netkit_link_ops;
 329
 330 static int netkit_new_link(struct net *src_net, struct net_device *dev,
 331                            struct nlattr *tb[], struct nlattr *data[],
 332                            struct netlink_ext_ack *extack)
 333 {
 334         struct nlattr *peer_tb[IFLA_MAX + 1], **tbp = tb, *attr;
 335         enum netkit_action policy_prim = NETKIT_PASS;
 336         enum netkit_action policy_peer = NETKIT_PASS;
 337         enum netkit_scrub scrub_prim = NETKIT_SCRUB_DEFAULT;
 338         enum netkit_scrub scrub_peer = NETKIT_SCRUB_DEFAULT;
 339         enum netkit_mode mode = NETKIT_L3;
 340         unsigned char ifname_assign_type;
 341         struct ifinfomsg *ifmp = NULL;
 342         struct net_device *peer;
 343         char ifname[IFNAMSIZ];
 344         struct netkit *nk;
 345         struct net *net;
 346         int err;
 347
 348         if (data) {
 349                 if (data[IFLA_NETKIT_MODE])
 350                         mode = nla_get_u32(data[IFLA_NETKIT_MODE]);
 351                 if (data[IFLA_NETKIT_PEER_INFO]) {
 352                         attr = data[IFLA_NETKIT_PEER_INFO];
 353                         ifmp = nla_data(attr);
 354                         rtnl_nla_parse_ifinfomsg(peer_tb, attr, extack);
 355                         tbp = peer_tb;
 356                 }
 357                 if (data[IFLA_NETKIT_SCRUB])
 358                         scrub_prim = nla_get_u32(data[IFLA_NETKIT_SCRUB]);
 359                 if (data[IFLA_NETKIT_PEER_SCRUB])
 360                         scrub_peer = nla_get_u32(data[IFLA_NETKIT_PEER_SCRUB]);
 361                 if (data[IFLA_NETKIT_POLICY]) {
 362                         attr = data[IFLA_NETKIT_POLICY];
 363                         policy_prim = nla_get_u32(attr);
 364                         err = netkit_check_policy(policy_prim, attr, extack);
 365                         if (err < 0)
 366                                 return err;
 367                 }
 368                 if (data[IFLA_NETKIT_PEER_POLICY]) {
 369                         attr = data[IFLA_NETKIT_PEER_POLICY];
 370                         policy_peer = nla_get_u32(attr);
 371                         err = netkit_check_policy(policy_peer, attr, extack);
 372                         if (err < 0)
 373                                 return err;
 374                 }
 375         }
 376
 377         if (ifmp && tbp[IFLA_IFNAME]) {
 378                 nla_strscpy(ifname, tbp[IFLA_IFNAME], IFNAMSIZ);
 379                 ifname_assign_type = NET_NAME_USER;
 380         } else {
 381                 strscpy(ifname, "nk%d", IFNAMSIZ);
 382                 ifname_assign_type = NET_NAME_ENUM;
 383         }
 384         if (mode != NETKIT_L2 &&
 385             (tb[IFLA_ADDRESS] || tbp[IFLA_ADDRESS]))
 386                 return -EOPNOTSUPP;
 387
 388         net = rtnl_link_get_net(src_net, tbp);
 389         peer = rtnl_create_link(net, ifname, ifname_assign_type,
 390                                 &netkit_link_ops, tbp, extack);
 391         if (IS_ERR(peer)) {
 392                 put_net(net);
 393                 return PTR_ERR(peer);
 394         }
 395
 396         netif_inherit_tso_max(peer, dev);
 397
 398         if (mode == NETKIT_L2 && !(ifmp && tbp[IFLA_ADDRESS]))
 399                 eth_hw_addr_random(peer);
 400         if (ifmp && dev->ifindex)
 401                 peer->ifindex = ifmp->ifi_index;
 402
 403         nk = netkit_priv(peer);
 404         nk->primary = false;
 405         nk->policy = policy_peer;
 406         nk->scrub = scrub_peer;
 407         nk->mode = mode;
 408         bpf_mprog_bundle_init(&nk->bundle);
 409
 410         err = register_netdevice(peer);
 411         put_net(net);
 412         if (err < 0)
 413                 goto err_register_peer;
 414         netif_carrier_off(peer);
 415         if (mode == NETKIT_L2)
 416                 dev_change_flags(peer, peer->flags & ~IFF_NOARP, NULL);
 417
 418         err = rtnl_configure_link(peer, NULL, 0, NULL);
 419         if (err < 0)
 420                 goto err_configure_peer;
 421
 422         if (mode == NETKIT_L2 && !tb[IFLA_ADDRESS])
 423                 eth_hw_addr_random(dev);
 424         if (tb[IFLA_IFNAME])
 425                 nla_strscpy(dev->name, tb[IFLA_IFNAME], IFNAMSIZ);
 426         else
 427                 strscpy(dev->name, "nk%d", IFNAMSIZ);
 428
 429         nk = netkit_priv(dev);
 430         nk->primary = true;
 431         nk->policy = policy_prim;
 432         nk->scrub = scrub_prim;
 433         nk->mode = mode;
 434         bpf_mprog_bundle_init(&nk->bundle);
 435
 436         err = register_netdevice(dev);
 437         if (err < 0)
 438                 goto err_configure_peer;
 439         netif_carrier_off(dev);
 440         if (mode == NETKIT_L2)
 441                 dev_change_flags(dev, dev->flags & ~IFF_NOARP, NULL);
 442
 443         rcu_assign_pointer(netkit_priv(dev)->peer, peer);
 444         rcu_assign_pointer(netkit_priv(peer)->peer, dev);
 445         return 0;
 446 err_configure_peer:
 447         unregister_netdevice(peer);
 448         return err;
 449 err_register_peer:
 450         free_netdev(peer);
 451         return err;
 452 }
 453
 454 static struct bpf_mprog_entry *netkit_entry_fetch(struct net_device *dev,
 455                                                   bool bundle_fallback)
 456 {
 457         struct netkit *nk = netkit_priv(dev);
 458         struct bpf_mprog_entry *entry;
 459
 460         ASSERT_RTNL();
 461         entry = rcu_dereference_rtnl(nk->active);
 462         if (entry)
 463                 return entry;
 464         if (bundle_fallback)
 465                 return &nk->bundle.a;
 466         return NULL;
 467 }
 468
 469 static void netkit_entry_update(struct net_device *dev,
 470                                 struct bpf_mprog_entry *entry)
 471 {
 472         struct netkit *nk = netkit_priv(dev);
 473
 474         ASSERT_RTNL();
 475         rcu_assign_pointer(nk->active, entry);
 476 }
 477
 478 static void netkit_entry_sync(void)
 479 {
 480         synchronize_rcu();
 481 }
 482
 483 static struct net_device *netkit_dev_fetch(struct net *net, u32 ifindex, u32 which)
 484 {
 485         struct net_device *dev;
 486         struct netkit *nk;
 487
 488         ASSERT_RTNL();
 489
 490         switch (which) {
 491         case BPF_NETKIT_PRIMARY:
 492         case BPF_NETKIT_PEER:
 493                 break;
 494         default:
 495                 return ERR_PTR(-EINVAL);
 496         }
 497
 498         dev = __dev_get_by_index(net, ifindex);
 499         if (!dev)
 500                 return ERR_PTR(-ENODEV);
 501         if (dev->netdev_ops != &netkit_netdev_ops)
 502                 return ERR_PTR(-ENXIO);
 503
 504         nk = netkit_priv(dev);
 505         if (!nk->primary)
 506                 return ERR_PTR(-EACCES);
 507         if (which == BPF_NETKIT_PEER) {
 508                 dev = rcu_dereference_rtnl(nk->peer);
 509                 if (!dev)
 510                         return ERR_PTR(-ENODEV);
 511         }
 512         return dev;
 513 }
 514
 515 int netkit_prog_attach(const union bpf_attr *attr, struct bpf_prog *prog)
 516 {
 517         struct bpf_mprog_entry *entry, *entry_new;
 518         struct bpf_prog *replace_prog = NULL;
 519         struct net_device *dev;
 520         int ret;
 521
 522         rtnl_lock();
 523         dev = netkit_dev_fetch(current->nsproxy->net_ns, attr->target_ifindex,
 524                                attr->attach_type);
 525         if (IS_ERR(dev)) {
 526                 ret = PTR_ERR(dev);
 527                 goto out;
 528         }
 529         entry = netkit_entry_fetch(dev, true);
 530         if (attr->attach_flags & BPF_F_REPLACE) {
 531                 replace_prog = bpf_prog_get_type(attr->replace_bpf_fd,
 532                                                  prog->type);
 533                 if (IS_ERR(replace_prog)) {
 534                         ret = PTR_ERR(replace_prog);
 535                         replace_prog = NULL;
 536                         goto out;
 537                 }
 538         }
 539         ret = bpf_mprog_attach(entry, &entry_new, prog, NULL, replace_prog,
 540                                attr->attach_flags, attr->relative_fd,
 541                                attr->expected_revision);
 542         if (!ret) {
 543                 if (entry != entry_new) {
 544                         netkit_entry_update(dev, entry_new);
 545                         netkit_entry_sync();
 546                 }
 547                 bpf_mprog_commit(entry);
 548         }
 549 out:
 550         if (replace_prog)
 551                 bpf_prog_put(replace_prog);
 552         rtnl_unlock();
 553         return ret;
 554 }
 555
 556 int netkit_prog_detach(const union bpf_attr *attr, struct bpf_prog *prog)
 557 {
 558         struct bpf_mprog_entry *entry, *entry_new;
 559         struct net_device *dev;
 560         int ret;
 561
 562         rtnl_lock();
 563         dev = netkit_dev_fetch(current->nsproxy->net_ns, attr->target_ifindex,
 564                                attr->attach_type);
 565         if (IS_ERR(dev)) {
 566                 ret = PTR_ERR(dev);
 567                 goto out;
 568         }
 569         entry = netkit_entry_fetch(dev, false);
 570         if (!entry) {
 571                 ret = -ENOENT;
 572                 goto out;
 573         }
 574         ret = bpf_mprog_detach(entry, &entry_new, prog, NULL, attr->attach_flags,
 575                                attr->relative_fd, attr->expected_revision);
 576         if (!ret) {
 577                 if (!bpf_mprog_total(entry_new))
 578                         entry_new = NULL;
 579                 netkit_entry_update(dev, entry_new);
 580                 netkit_entry_sync();
 581                 bpf_mprog_commit(entry);
 582         }
 583 out:
 584         rtnl_unlock();
 585         return ret;
 586 }
 587
 588 int netkit_prog_query(const union bpf_attr *attr, union bpf_attr __user *uattr)
 589 {
 590         struct net_device *dev;
 591         int ret;
 592
 593         rtnl_lock();
 594         dev = netkit_dev_fetch(current->nsproxy->net_ns,
 595                                attr->query.target_ifindex,
 596                                attr->query.attach_type);
 597         if (IS_ERR(dev)) {
 598                 ret = PTR_ERR(dev);
 599                 goto out;
 600         }
 601         ret = bpf_mprog_query(attr, uattr, netkit_entry_fetch(dev, false));
 602 out:
 603         rtnl_unlock();
 604         return ret;
 605 }
 606
 607 static struct netkit_link *netkit_link(const struct bpf_link *link)
 608 {
 609         return container_of(link, struct netkit_link, link);
 610 }
 611
 612 static int netkit_link_prog_attach(struct bpf_link *link, u32 flags,
 613                                    u32 id_or_fd, u64 revision)
 614 {
 615         struct netkit_link *nkl = netkit_link(link);
 616         struct bpf_mprog_entry *entry, *entry_new;
 617         struct net_device *dev = nkl->dev;
 618         int ret;
 619
 620         ASSERT_RTNL();
 621         entry = netkit_entry_fetch(dev, true);
 622         ret = bpf_mprog_attach(entry, &entry_new, link->prog, link, NULL, flags,
 623                                id_or_fd, revision);
 624         if (!ret) {
 625                 if (entry != entry_new) {
 626                         netkit_entry_update(dev, entry_new);
 627                         netkit_entry_sync();
 628                 }
 629                 bpf_mprog_commit(entry);
 630         }
 631         return ret;
 632 }
 633
 634 static void netkit_link_release(struct bpf_link *link)
 635 {
 636         struct netkit_link *nkl = netkit_link(link);
 637         struct bpf_mprog_entry *entry, *entry_new;
 638         struct net_device *dev;
 639         int ret = 0;
 640
 641         rtnl_lock();
 642         dev = nkl->dev;
 643         if (!dev)
 644                 goto out;
 645         entry = netkit_entry_fetch(dev, false);
 646         if (!entry) {
 647                 ret = -ENOENT;
 648                 goto out;
 649         }
 650         ret = bpf_mprog_detach(entry, &entry_new, link->prog, link, 0, 0, 0);
 651         if (!ret) {
 652                 if (!bpf_mprog_total(entry_new))
 653                         entry_new = NULL;
 654                 netkit_entry_update(dev, entry_new);
 655                 netkit_entry_sync();
 656                 bpf_mprog_commit(entry);
 657                 nkl->dev = NULL;
 658         }
 659 out:
 660         WARN_ON_ONCE(ret);
 661         rtnl_unlock();
 662 }
 663
 664 static int netkit_link_update(struct bpf_link *link, struct bpf_prog *nprog,
 665                               struct bpf_prog *oprog)
 666 {
 667         struct netkit_link *nkl = netkit_link(link);
 668         struct bpf_mprog_entry *entry, *entry_new;
 669         struct net_device *dev;
 670         int ret = 0;
 671
 672         rtnl_lock();
 673         dev = nkl->dev;
 674         if (!dev) {
 675                 ret = -ENOLINK;
 676                 goto out;
 677         }
 678         if (oprog && link->prog != oprog) {
 679                 ret = -EPERM;
 680                 goto out;
 681         }
 682         oprog = link->prog;
 683         if (oprog == nprog) {
 684                 bpf_prog_put(nprog);
 685                 goto out;
 686         }
 687         entry = netkit_entry_fetch(dev, false);
 688         if (!entry) {
 689                 ret = -ENOENT;
 690                 goto out;
 691         }
 692         ret = bpf_mprog_attach(entry, &entry_new, nprog, link, oprog,
 693                                BPF_F_REPLACE | BPF_F_ID,
 694                                link->prog->aux->id, 0);
 695         if (!ret) {
 696                 WARN_ON_ONCE(entry != entry_new);
 697                 oprog = xchg(&link->prog, nprog);
 698                 bpf_prog_put(oprog);
 699                 bpf_mprog_commit(entry);
 700         }
 701 out:
 702         rtnl_unlock();
 703         return ret;
 704 }
 705
 706 static void netkit_link_dealloc(struct bpf_link *link)
 707 {
 708         kfree(netkit_link(link));
 709 }
 710
 711 static void netkit_link_fdinfo(const struct bpf_link *link, struct seq_file *seq)
 712 {
 713         const struct netkit_link *nkl = netkit_link(link);
 714         u32 ifindex = 0;
 715
 716         rtnl_lock();
 717         if (nkl->dev)
 718                 ifindex = nkl->dev->ifindex;
 719         rtnl_unlock();
 720
 721         seq_printf(seq, "ifindex:\t%u\n", ifindex);
 722         seq_printf(seq, "attach_type:\t%u (%s)\n",
 723                    nkl->location,
 724                    nkl->location == BPF_NETKIT_PRIMARY ? "primary" : "peer");
 725 }
 726
 727 static int netkit_link_fill_info(const struct bpf_link *link,
 728                                  struct bpf_link_info *info)
 729 {
 730         const struct netkit_link *nkl = netkit_link(link);
 731         u32 ifindex = 0;
 732
 733         rtnl_lock();
 734         if (nkl->dev)
 735                 ifindex = nkl->dev->ifindex;
 736         rtnl_unlock();
 737
 738         info->netkit.ifindex = ifindex;
 739         info->netkit.attach_type = nkl->location;
 740         return 0;
 741 }
 742
 743 static int netkit_link_detach(struct bpf_link *link)
 744 {
 745         netkit_link_release(link);
 746         return 0;
 747 }
 748
 749 static const struct bpf_link_ops netkit_link_lops = {
 750         .release        = netkit_link_release,
 751         .detach         = netkit_link_detach,
 752         .dealloc        = netkit_link_dealloc,
 753         .update_prog    = netkit_link_update,
 754         .show_fdinfo    = netkit_link_fdinfo,
 755         .fill_link_info = netkit_link_fill_info,
 756 };
 757
 758 static int netkit_link_init(struct netkit_link *nkl,
 759                             struct bpf_link_primer *link_primer,
 760                             const union bpf_attr *attr,
 761                             struct net_device *dev,
 762                             struct bpf_prog *prog)
 763 {
 764         bpf_link_init(&nkl->link, BPF_LINK_TYPE_NETKIT,
 765                       &netkit_link_lops, prog);
 766         nkl->location = attr->link_create.attach_type;
 767         nkl->dev = dev;
 768         return bpf_link_prime(&nkl->link, link_primer);
 769 }
 770
 771 int netkit_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
 772 {
 773         struct bpf_link_primer link_primer;
 774         struct netkit_link *nkl;
 775         struct net_device *dev;
 776         int ret;
 777
 778         rtnl_lock();
 779         dev = netkit_dev_fetch(current->nsproxy->net_ns,
 780                                attr->link_create.target_ifindex,
 781                                attr->link_create.attach_type);
 782         if (IS_ERR(dev)) {
 783                 ret = PTR_ERR(dev);
 784                 goto out;
 785         }
 786         nkl = kzalloc(sizeof(*nkl), GFP_KERNEL_ACCOUNT);
 787         if (!nkl) {
 788                 ret = -ENOMEM;
 789                 goto out;
 790         }
 791         ret = netkit_link_init(nkl, &link_primer, attr, dev, prog);
 792         if (ret) {
 793                 kfree(nkl);
 794                 goto out;
 795         }
 796         ret = netkit_link_prog_attach(&nkl->link,
 797                                       attr->link_create.flags,
 798                                       attr->link_create.netkit.relative_fd,
 799                                       attr->link_create.netkit.expected_revision);
 800         if (ret) {
 801                 nkl->dev = NULL;
 802                 bpf_link_cleanup(&link_primer);
 803                 goto out;
 804         }
 805         ret = bpf_link_settle(&link_primer);
 806 out:
 807         rtnl_unlock();
 808         return ret;
 809 }
 810
 811 static void netkit_release_all(struct net_device *dev)
 812 {
 813         struct bpf_mprog_entry *entry;
 814         struct bpf_tuple tuple = {};
 815         struct bpf_mprog_fp *fp;
 816         struct bpf_mprog_cp *cp;
 817
 818         entry = netkit_entry_fetch(dev, false);
 819         if (!entry)
 820                 return;
 821         netkit_entry_update(dev, NULL);
 822         netkit_entry_sync();
 823         bpf_mprog_foreach_tuple(entry, fp, cp, tuple) {
 824                 if (tuple.link)
 825                         netkit_link(tuple.link)->dev = NULL;
 826                 else
 827                         bpf_prog_put(tuple.prog);
 828         }
 829 }
 830
 831 static void netkit_uninit(struct net_device *dev)
 832 {
 833         netkit_release_all(dev);
 834 }
 835
 836 static void netkit_del_link(struct net_device *dev, struct list_head *head)
 837 {
 838         struct netkit *nk = netkit_priv(dev);
 839         struct net_device *peer = rtnl_dereference(nk->peer);
 840
 841         RCU_INIT_POINTER(nk->peer, NULL);
 842         unregister_netdevice_queue(dev, head);
 843         if (peer) {
 844                 nk = netkit_priv(peer);
 845                 RCU_INIT_POINTER(nk->peer, NULL);
 846                 unregister_netdevice_queue(peer, head);
 847         }
 848 }
 849
 850 static int netkit_change_link(struct net_device *dev, struct nlattr *tb[],
 851                               struct nlattr *data[],
 852                               struct netlink_ext_ack *extack)
 853 {
 854         struct netkit *nk = netkit_priv(dev);
 855         struct net_device *peer = rtnl_dereference(nk->peer);
 856         enum netkit_action policy;
 857         struct nlattr *attr;
 858         int err;
 859
 860         if (!nk->primary) {
 861                 NL_SET_ERR_MSG(extack,
 862                                "netkit link settings can be changed only through the primary device");
 863                 return -EACCES;
 864         }
 865
 866         if (data[IFLA_NETKIT_MODE]) {
 867                 NL_SET_ERR_MSG_ATTR(extack, data[IFLA_NETKIT_MODE],
 868                                     "netkit link operating mode cannot be changed after device creation");
 869                 return -EACCES;
 870         }
 871
 872         if (data[IFLA_NETKIT_SCRUB]) {
 873                 NL_SET_ERR_MSG_ATTR(extack, data[IFLA_NETKIT_SCRUB],
 874                                     "netkit scrubbing cannot be changed after device creation");
 875                 return -EACCES;
 876         }
 877
 878         if (data[IFLA_NETKIT_PEER_SCRUB]) {
 879                 NL_SET_ERR_MSG_ATTR(extack, data[IFLA_NETKIT_PEER_SCRUB],
 880                                     "netkit scrubbing cannot be changed after device creation");
 881                 return -EACCES;
 882         }
 883
 884         if (data[IFLA_NETKIT_PEER_INFO]) {
 885                 NL_SET_ERR_MSG_ATTR(extack, data[IFLA_NETKIT_PEER_INFO],
 886                                     "netkit peer info cannot be changed after device creation");
 887                 return -EINVAL;
 888         }
 889
 890         if (data[IFLA_NETKIT_POLICY]) {
 891                 attr = data[IFLA_NETKIT_POLICY];
 892                 policy = nla_get_u32(attr);
 893                 err = netkit_check_policy(policy, attr, extack);
 894                 if (err)
 895                         return err;
 896                 WRITE_ONCE(nk->policy, policy);
 897         }
 898
 899         if (data[IFLA_NETKIT_PEER_POLICY]) {
 900                 err = -EOPNOTSUPP;
 901                 attr = data[IFLA_NETKIT_PEER_POLICY];
 902                 policy = nla_get_u32(attr);
 903                 if (peer)
 904                         err = netkit_check_policy(policy, attr, extack);
 905                 if (err)
 906                         return err;
 907                 nk = netkit_priv(peer);
 908                 WRITE_ONCE(nk->policy, policy);
 909         }
 910
 911         return 0;
 912 }
 913
 914 static size_t netkit_get_size(const struct net_device *dev)
 915 {
 916         return nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_POLICY */
 917                nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_PEER_POLICY */
 918                nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_SCRUB */
 919                nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_PEER_SCRUB */
 920                nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_MODE */
 921                nla_total_size(sizeof(u8))  + /* IFLA_NETKIT_PRIMARY */
 922                0;
 923 }
 924
 925 static int netkit_fill_info(struct sk_buff *skb, const struct net_device *dev)
 926 {
 927         struct netkit *nk = netkit_priv(dev);
 928         struct net_device *peer = rtnl_dereference(nk->peer);
 929
 930         if (nla_put_u8(skb, IFLA_NETKIT_PRIMARY, nk->primary))
 931                 return -EMSGSIZE;
 932         if (nla_put_u32(skb, IFLA_NETKIT_POLICY, nk->policy))
 933                 return -EMSGSIZE;
 934         if (nla_put_u32(skb, IFLA_NETKIT_MODE, nk->mode))
 935                 return -EMSGSIZE;
 936         if (nla_put_u32(skb, IFLA_NETKIT_SCRUB, nk->scrub))
 937                 return -EMSGSIZE;
 938
 939         if (peer) {
 940                 nk = netkit_priv(peer);
 941                 if (nla_put_u32(skb, IFLA_NETKIT_PEER_POLICY, nk->policy))
 942                         return -EMSGSIZE;
 943                 if (nla_put_u32(skb, IFLA_NETKIT_PEER_SCRUB, nk->scrub))
 944                         return -EMSGSIZE;
 945         }
 946
 947         return 0;
 948 }
 949
 950 static const struct nla_policy netkit_policy[IFLA_NETKIT_MAX + 1] = {
 951         [IFLA_NETKIT_PEER_INFO]         = { .len = sizeof(struct ifinfomsg) },
 952         [IFLA_NETKIT_MODE]              = NLA_POLICY_MAX(NLA_U32, NETKIT_L3),
 953         [IFLA_NETKIT_POLICY]            = { .type = NLA_U32 },
 954         [IFLA_NETKIT_PEER_POLICY]       = { .type = NLA_U32 },
 955         [IFLA_NETKIT_SCRUB]             = NLA_POLICY_MAX(NLA_U32, NETKIT_SCRUB_DEFAULT),
 956         [IFLA_NETKIT_PEER_SCRUB]        = NLA_POLICY_MAX(NLA_U32, NETKIT_SCRUB_DEFAULT),
 957         [IFLA_NETKIT_PRIMARY]           = { .type = NLA_REJECT,
 958                                             .reject_message = "Primary attribute is read-only" },
 959 };
 960
 961 static struct rtnl_link_ops netkit_link_ops = {
 962         .kind           = DRV_NAME,
 963         .priv_size      = sizeof(struct netkit),
 964         .setup          = netkit_setup,
 965         .newlink        = netkit_new_link,
 966         .dellink        = netkit_del_link,
 967         .changelink     = netkit_change_link,
 968         .get_link_net   = netkit_get_link_net,
 969         .get_size       = netkit_get_size,
 970         .fill_info      = netkit_fill_info,
 971         .policy         = netkit_policy,
 972         .validate       = netkit_validate,
 973         .peer_type      = IFLA_NETKIT_PEER_INFO,
 974         .maxtype        = IFLA_NETKIT_MAX,
 975 };
 976
 977 static __init int netkit_init(void)
 978 {
 979         BUILD_BUG_ON((int)NETKIT_NEXT != (int)TCX_NEXT ||
 980                      (int)NETKIT_PASS != (int)TCX_PASS ||
 981                      (int)NETKIT_DROP != (int)TCX_DROP ||
 982                      (int)NETKIT_REDIRECT != (int)TCX_REDIRECT);
 983
 984         return rtnl_link_register(&netkit_link_ops);
 985 }
 986
 987 static __exit void netkit_exit(void)
 988 {
 989         rtnl_link_unregister(&netkit_link_ops);
 990 }
 991
 992 module_init(netkit_init);
 993 module_exit(netkit_exit);
 994
 995 MODULE_DESCRIPTION("BPF-programmable network device");
 996 MODULE_AUTHOR("Daniel Borkmann <daniel@iogearbox.net>");
 997 MODULE_AUTHOR("Nikolay Aleksandrov <razor@blackwall.org>");
 998 MODULE_LICENSE("GPL");
 999 MODULE_ALIAS_RTNL_LINK(DRV_NAME);