net/core/netdev-genl.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2
   3 #include <linux/netdevice.h>
   4 #include <linux/notifier.h>
   5 #include <linux/rtnetlink.h>
   6 #include <net/busy_poll.h>
   7 #include <net/net_namespace.h>
   8 #include <net/netdev_queues.h>
   9 #include <net/netdev_rx_queue.h>
  10 #include <net/sock.h>
  11 #include <net/xdp.h>
  12 #include <net/xdp_sock.h>
  13
  14 #include "dev.h"
  15 #include "devmem.h"
  16 #include "netdev-genl-gen.h"
  17
  18 struct netdev_nl_dump_ctx {
  19         unsigned long   ifindex;
  20         unsigned int    rxq_idx;
  21         unsigned int    txq_idx;
  22         unsigned int    napi_id;
  23 };
  24
  25 static struct netdev_nl_dump_ctx *netdev_dump_ctx(struct netlink_callback *cb)
  26 {
  27         NL_ASSERT_CTX_FITS(struct netdev_nl_dump_ctx);
  28
  29         return (struct netdev_nl_dump_ctx *)cb->ctx;
  30 }
  31
  32 static int
  33 netdev_nl_dev_fill(struct net_device *netdev, struct sk_buff *rsp,
  34                    const struct genl_info *info)
  35 {
  36         u64 xsk_features = 0;
  37         u64 xdp_rx_meta = 0;
  38         void *hdr;
  39
  40         hdr = genlmsg_iput(rsp, info);
  41         if (!hdr)
  42                 return -EMSGSIZE;
  43
  44 #define XDP_METADATA_KFUNC(_, flag, __, xmo) \
  45         if (netdev->xdp_metadata_ops && netdev->xdp_metadata_ops->xmo) \
  46                 xdp_rx_meta |= flag;
  47 XDP_METADATA_KFUNC_xxx
  48 #undef XDP_METADATA_KFUNC
  49
  50         if (netdev->xsk_tx_metadata_ops) {
  51                 if (netdev->xsk_tx_metadata_ops->tmo_fill_timestamp)
  52                         xsk_features |= NETDEV_XSK_FLAGS_TX_TIMESTAMP;
  53                 if (netdev->xsk_tx_metadata_ops->tmo_request_checksum)
  54                         xsk_features |= NETDEV_XSK_FLAGS_TX_CHECKSUM;
  55         }
  56
  57         if (nla_put_u32(rsp, NETDEV_A_DEV_IFINDEX, netdev->ifindex) ||
  58             nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_FEATURES,
  59                               netdev->xdp_features, NETDEV_A_DEV_PAD) ||
  60             nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_RX_METADATA_FEATURES,
  61                               xdp_rx_meta, NETDEV_A_DEV_PAD) ||
  62             nla_put_u64_64bit(rsp, NETDEV_A_DEV_XSK_FEATURES,
  63                               xsk_features, NETDEV_A_DEV_PAD))
  64                 goto err_cancel_msg;
  65
  66         if (netdev->xdp_features & NETDEV_XDP_ACT_XSK_ZEROCOPY) {
  67                 if (nla_put_u32(rsp, NETDEV_A_DEV_XDP_ZC_MAX_SEGS,
  68                                 netdev->xdp_zc_max_segs))
  69                         goto err_cancel_msg;
  70         }
  71
  72         genlmsg_end(rsp, hdr);
  73
  74         return 0;
  75
  76 err_cancel_msg:
  77         genlmsg_cancel(rsp, hdr);
  78         return -EMSGSIZE;
  79 }
  80
  81 static void
  82 netdev_genl_dev_notify(struct net_device *netdev, int cmd)
  83 {
  84         struct genl_info info;
  85         struct sk_buff *ntf;
  86
  87         if (!genl_has_listeners(&netdev_nl_family, dev_net(netdev),
  88                                 NETDEV_NLGRP_MGMT))
  89                 return;
  90
  91         genl_info_init_ntf(&info, &netdev_nl_family, cmd);
  92
  93         ntf = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
  94         if (!ntf)
  95                 return;
  96
  97         if (netdev_nl_dev_fill(netdev, ntf, &info)) {
  98                 nlmsg_free(ntf);
  99                 return;
 100         }
 101
 102         genlmsg_multicast_netns(&netdev_nl_family, dev_net(netdev), ntf,
 103                                 0, NETDEV_NLGRP_MGMT, GFP_KERNEL);
 104 }
 105
 106 int netdev_nl_dev_get_doit(struct sk_buff *skb, struct genl_info *info)
 107 {
 108         struct net_device *netdev;
 109         struct sk_buff *rsp;
 110         u32 ifindex;
 111         int err;
 112
 113         if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_DEV_IFINDEX))
 114                 return -EINVAL;
 115
 116         ifindex = nla_get_u32(info->attrs[NETDEV_A_DEV_IFINDEX]);
 117
 118         rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
 119         if (!rsp)
 120                 return -ENOMEM;
 121
 122         rtnl_lock();
 123
 124         netdev = __dev_get_by_index(genl_info_net(info), ifindex);
 125         if (netdev)
 126                 err = netdev_nl_dev_fill(netdev, rsp, info);
 127         else
 128                 err = -ENODEV;
 129
 130         rtnl_unlock();
 131
 132         if (err)
 133                 goto err_free_msg;
 134
 135         return genlmsg_reply(rsp, info);
 136
 137 err_free_msg:
 138         nlmsg_free(rsp);
 139         return err;
 140 }
 141
 142 int netdev_nl_dev_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
 143 {
 144         struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb);
 145         struct net *net = sock_net(skb->sk);
 146         struct net_device *netdev;
 147         int err = 0;
 148
 149         rtnl_lock();
 150         for_each_netdev_dump(net, netdev, ctx->ifindex) {
 151                 err = netdev_nl_dev_fill(netdev, skb, genl_info_dump(cb));
 152                 if (err < 0)
 153                         break;
 154         }
 155         rtnl_unlock();
 156
 157         return err;
 158 }
 159
 160 static int
 161 netdev_nl_napi_fill_one(struct sk_buff *rsp, struct napi_struct *napi,
 162                         const struct genl_info *info)
 163 {
 164         unsigned long irq_suspend_timeout;
 165         unsigned long gro_flush_timeout;
 166         u32 napi_defer_hard_irqs;
 167         void *hdr;
 168         pid_t pid;
 169
 170         if (!napi->dev->up)
 171                 return 0;
 172
 173         hdr = genlmsg_iput(rsp, info);
 174         if (!hdr)
 175                 return -EMSGSIZE;
 176
 177         if (nla_put_u32(rsp, NETDEV_A_NAPI_ID, napi->napi_id))
 178                 goto nla_put_failure;
 179
 180         if (nla_put_u32(rsp, NETDEV_A_NAPI_IFINDEX, napi->dev->ifindex))
 181                 goto nla_put_failure;
 182
 183         if (napi->irq >= 0 && nla_put_u32(rsp, NETDEV_A_NAPI_IRQ, napi->irq))
 184                 goto nla_put_failure;
 185
 186         if (napi->thread) {
 187                 pid = task_pid_nr(napi->thread);
 188                 if (nla_put_u32(rsp, NETDEV_A_NAPI_PID, pid))
 189                         goto nla_put_failure;
 190         }
 191
 192         napi_defer_hard_irqs = napi_get_defer_hard_irqs(napi);
 193         if (nla_put_s32(rsp, NETDEV_A_NAPI_DEFER_HARD_IRQS,
 194                         napi_defer_hard_irqs))
 195                 goto nla_put_failure;
 196
 197         irq_suspend_timeout = napi_get_irq_suspend_timeout(napi);
 198         if (nla_put_uint(rsp, NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT,
 199                          irq_suspend_timeout))
 200                 goto nla_put_failure;
 201
 202         gro_flush_timeout = napi_get_gro_flush_timeout(napi);
 203         if (nla_put_uint(rsp, NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT,
 204                          gro_flush_timeout))
 205                 goto nla_put_failure;
 206
 207         genlmsg_end(rsp, hdr);
 208
 209         return 0;
 210
 211 nla_put_failure:
 212         genlmsg_cancel(rsp, hdr);
 213         return -EMSGSIZE;
 214 }
 215
 216 int netdev_nl_napi_get_doit(struct sk_buff *skb, struct genl_info *info)
 217 {
 218         struct napi_struct *napi;
 219         struct sk_buff *rsp;
 220         u32 napi_id;
 221         int err;
 222
 223         if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_NAPI_ID))
 224                 return -EINVAL;
 225
 226         napi_id = nla_get_u32(info->attrs[NETDEV_A_NAPI_ID]);
 227
 228         rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
 229         if (!rsp)
 230                 return -ENOMEM;
 231
 232         napi = netdev_napi_by_id_lock(genl_info_net(info), napi_id);
 233         if (napi) {
 234                 err = netdev_nl_napi_fill_one(rsp, napi, info);
 235                 netdev_unlock(napi->dev);
 236         } else {
 237                 NL_SET_BAD_ATTR(info->extack, info->attrs[NETDEV_A_NAPI_ID]);
 238                 err = -ENOENT;
 239         }
 240
 241         if (err) {
 242                 goto err_free_msg;
 243         } else if (!rsp->len) {
 244                 err = -ENOENT;
 245                 goto err_free_msg;
 246         }
 247
 248         return genlmsg_reply(rsp, info);
 249
 250 err_free_msg:
 251         nlmsg_free(rsp);
 252         return err;
 253 }
 254
 255 static int
 256 netdev_nl_napi_dump_one(struct net_device *netdev, struct sk_buff *rsp,
 257                         const struct genl_info *info,
 258                         struct netdev_nl_dump_ctx *ctx)
 259 {
 260         struct napi_struct *napi;
 261         unsigned int prev_id;
 262         int err = 0;
 263
 264         if (!netdev->up)
 265                 return err;
 266
 267         prev_id = UINT_MAX;
 268         list_for_each_entry(napi, &netdev->napi_list, dev_list) {
 269                 if (napi->napi_id < MIN_NAPI_ID)
 270                         continue;
 271
 272                 /* Dump continuation below depends on the list being sorted */
 273                 WARN_ON_ONCE(napi->napi_id >= prev_id);
 274                 prev_id = napi->napi_id;
 275
 276                 if (ctx->napi_id && napi->napi_id >= ctx->napi_id)
 277                         continue;
 278
 279                 err = netdev_nl_napi_fill_one(rsp, napi, info);
 280                 if (err)
 281                         return err;
 282                 ctx->napi_id = napi->napi_id;
 283         }
 284         return err;
 285 }
 286
 287 int netdev_nl_napi_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
 288 {
 289         struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb);
 290         const struct genl_info *info = genl_info_dump(cb);
 291         struct net *net = sock_net(skb->sk);
 292         struct net_device *netdev;
 293         u32 ifindex = 0;
 294         int err = 0;
 295
 296         if (info->attrs[NETDEV_A_NAPI_IFINDEX])
 297                 ifindex = nla_get_u32(info->attrs[NETDEV_A_NAPI_IFINDEX]);
 298
 299         if (ifindex) {
 300                 netdev = netdev_get_by_index_lock(net, ifindex);
 301                 if (netdev) {
 302                         err = netdev_nl_napi_dump_one(netdev, skb, info, ctx);
 303                         netdev_unlock(netdev);
 304                 } else {
 305                         err = -ENODEV;
 306                 }
 307         } else {
 308                 for_each_netdev_lock_scoped(net, netdev, ctx->ifindex) {
 309                         err = netdev_nl_napi_dump_one(netdev, skb, info, ctx);
 310                         if (err < 0)
 311                                 break;
 312                         ctx->napi_id = 0;
 313                 }
 314         }
 315
 316         return err;
 317 }
 318
 319 static int
 320 netdev_nl_napi_set_config(struct napi_struct *napi, struct genl_info *info)
 321 {
 322         u64 irq_suspend_timeout = 0;
 323         u64 gro_flush_timeout = 0;
 324         u32 defer = 0;
 325
 326         if (info->attrs[NETDEV_A_NAPI_DEFER_HARD_IRQS]) {
 327                 defer = nla_get_u32(info->attrs[NETDEV_A_NAPI_DEFER_HARD_IRQS]);
 328                 napi_set_defer_hard_irqs(napi, defer);
 329         }
 330
 331         if (info->attrs[NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT]) {
 332                 irq_suspend_timeout = nla_get_uint(info->attrs[NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT]);
 333                 napi_set_irq_suspend_timeout(napi, irq_suspend_timeout);
 334         }
 335
 336         if (info->attrs[NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT]) {
 337                 gro_flush_timeout = nla_get_uint(info->attrs[NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT]);
 338                 napi_set_gro_flush_timeout(napi, gro_flush_timeout);
 339         }
 340
 341         return 0;
 342 }
 343
 344 int netdev_nl_napi_set_doit(struct sk_buff *skb, struct genl_info *info)
 345 {
 346         struct napi_struct *napi;
 347         unsigned int napi_id;
 348         int err;
 349
 350         if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_NAPI_ID))
 351                 return -EINVAL;
 352
 353         napi_id = nla_get_u32(info->attrs[NETDEV_A_NAPI_ID]);
 354
 355         napi = netdev_napi_by_id_lock(genl_info_net(info), napi_id);
 356         if (napi) {
 357                 err = netdev_nl_napi_set_config(napi, info);
 358                 netdev_unlock(napi->dev);
 359         } else {
 360                 NL_SET_BAD_ATTR(info->extack, info->attrs[NETDEV_A_NAPI_ID]);
 361                 err = -ENOENT;
 362         }
 363
 364         return err;
 365 }
 366
 367 static int
 368 netdev_nl_queue_fill_one(struct sk_buff *rsp, struct net_device *netdev,
 369                          u32 q_idx, u32 q_type, const struct genl_info *info)
 370 {
 371         struct net_devmem_dmabuf_binding *binding;
 372         struct netdev_rx_queue *rxq;
 373         struct netdev_queue *txq;
 374         void *hdr;
 375
 376         hdr = genlmsg_iput(rsp, info);
 377         if (!hdr)
 378                 return -EMSGSIZE;
 379
 380         if (nla_put_u32(rsp, NETDEV_A_QUEUE_ID, q_idx) ||
 381             nla_put_u32(rsp, NETDEV_A_QUEUE_TYPE, q_type) ||
 382             nla_put_u32(rsp, NETDEV_A_QUEUE_IFINDEX, netdev->ifindex))
 383                 goto nla_put_failure;
 384
 385         switch (q_type) {
 386         case NETDEV_QUEUE_TYPE_RX:
 387                 rxq = __netif_get_rx_queue(netdev, q_idx);
 388                 if (rxq->napi && nla_put_u32(rsp, NETDEV_A_QUEUE_NAPI_ID,
 389                                              rxq->napi->napi_id))
 390                         goto nla_put_failure;
 391
 392                 binding = rxq->mp_params.mp_priv;
 393                 if (binding &&
 394                     nla_put_u32(rsp, NETDEV_A_QUEUE_DMABUF, binding->id))
 395                         goto nla_put_failure;
 396
 397                 break;
 398         case NETDEV_QUEUE_TYPE_TX:
 399                 txq = netdev_get_tx_queue(netdev, q_idx);
 400                 if (txq->napi && nla_put_u32(rsp, NETDEV_A_QUEUE_NAPI_ID,
 401                                              txq->napi->napi_id))
 402                         goto nla_put_failure;
 403         }
 404
 405         genlmsg_end(rsp, hdr);
 406
 407         return 0;
 408
 409 nla_put_failure:
 410         genlmsg_cancel(rsp, hdr);
 411         return -EMSGSIZE;
 412 }
 413
 414 static int netdev_nl_queue_validate(struct net_device *netdev, u32 q_id,
 415                                     u32 q_type)
 416 {
 417         switch (q_type) {
 418         case NETDEV_QUEUE_TYPE_RX:
 419                 if (q_id >= netdev->real_num_rx_queues)
 420                         return -EINVAL;
 421                 return 0;
 422         case NETDEV_QUEUE_TYPE_TX:
 423                 if (q_id >= netdev->real_num_tx_queues)
 424                         return -EINVAL;
 425         }
 426         return 0;
 427 }
 428
 429 static int
 430 netdev_nl_queue_fill(struct sk_buff *rsp, struct net_device *netdev, u32 q_idx,
 431                      u32 q_type, const struct genl_info *info)
 432 {
 433         int err;
 434
 435         if (!netdev->up)
 436                 return -ENOENT;
 437
 438         err = netdev_nl_queue_validate(netdev, q_idx, q_type);
 439         if (err)
 440                 return err;
 441
 442         return netdev_nl_queue_fill_one(rsp, netdev, q_idx, q_type, info);
 443 }
 444
 445 int netdev_nl_queue_get_doit(struct sk_buff *skb, struct genl_info *info)
 446 {
 447         u32 q_id, q_type, ifindex;
 448         struct net_device *netdev;
 449         struct sk_buff *rsp;
 450         int err;
 451
 452         if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_ID) ||
 453             GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_TYPE) ||
 454             GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_IFINDEX))
 455                 return -EINVAL;
 456
 457         q_id = nla_get_u32(info->attrs[NETDEV_A_QUEUE_ID]);
 458         q_type = nla_get_u32(info->attrs[NETDEV_A_QUEUE_TYPE]);
 459         ifindex = nla_get_u32(info->attrs[NETDEV_A_QUEUE_IFINDEX]);
 460
 461         rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
 462         if (!rsp)
 463                 return -ENOMEM;
 464
 465         rtnl_lock();
 466
 467         netdev = netdev_get_by_index_lock(genl_info_net(info), ifindex);
 468         if (netdev) {
 469                 err = netdev_nl_queue_fill(rsp, netdev, q_id, q_type, info);
 470                 netdev_unlock(netdev);
 471         } else {
 472                 err = -ENODEV;
 473         }
 474
 475         rtnl_unlock();
 476
 477         if (err)
 478                 goto err_free_msg;
 479
 480         return genlmsg_reply(rsp, info);
 481
 482 err_free_msg:
 483         nlmsg_free(rsp);
 484         return err;
 485 }
 486
 487 static int
 488 netdev_nl_queue_dump_one(struct net_device *netdev, struct sk_buff *rsp,
 489                          const struct genl_info *info,
 490                          struct netdev_nl_dump_ctx *ctx)
 491 {
 492         int err = 0;
 493
 494         if (!netdev->up)
 495                 return err;
 496
 497         for (; ctx->rxq_idx < netdev->real_num_rx_queues; ctx->rxq_idx++) {
 498                 err = netdev_nl_queue_fill_one(rsp, netdev, ctx->rxq_idx,
 499                                                NETDEV_QUEUE_TYPE_RX, info);
 500                 if (err)
 501                         return err;
 502         }
 503         for (; ctx->txq_idx < netdev->real_num_tx_queues; ctx->txq_idx++) {
 504                 err = netdev_nl_queue_fill_one(rsp, netdev, ctx->txq_idx,
 505                                                NETDEV_QUEUE_TYPE_TX, info);
 506                 if (err)
 507                         return err;
 508         }
 509
 510         return err;
 511 }
 512
 513 int netdev_nl_queue_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
 514 {
 515         struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb);
 516         const struct genl_info *info = genl_info_dump(cb);
 517         struct net *net = sock_net(skb->sk);
 518         struct net_device *netdev;
 519         u32 ifindex = 0;
 520         int err = 0;
 521
 522         if (info->attrs[NETDEV_A_QUEUE_IFINDEX])
 523                 ifindex = nla_get_u32(info->attrs[NETDEV_A_QUEUE_IFINDEX]);
 524
 525         rtnl_lock();
 526         if (ifindex) {
 527                 netdev = netdev_get_by_index_lock(net, ifindex);
 528                 if (netdev) {
 529                         err = netdev_nl_queue_dump_one(netdev, skb, info, ctx);
 530                         netdev_unlock(netdev);
 531                 } else {
 532                         err = -ENODEV;
 533                 }
 534         } else {
 535                 for_each_netdev_lock_scoped(net, netdev, ctx->ifindex) {
 536                         err = netdev_nl_queue_dump_one(netdev, skb, info, ctx);
 537                         if (err < 0)
 538                                 break;
 539                         ctx->rxq_idx = 0;
 540                         ctx->txq_idx = 0;
 541                 }
 542         }
 543         rtnl_unlock();
 544
 545         return err;
 546 }
 547
 548 #define NETDEV_STAT_NOT_SET             (~0ULL)
 549
 550 static void netdev_nl_stats_add(void *_sum, const void *_add, size_t size)
 551 {
 552         const u64 *add = _add;
 553         u64 *sum = _sum;
 554
 555         while (size) {
 556                 if (*add != NETDEV_STAT_NOT_SET && *sum != NETDEV_STAT_NOT_SET)
 557                         *sum += *add;
 558                 sum++;
 559                 add++;
 560                 size -= 8;
 561         }
 562 }
 563
 564 static int netdev_stat_put(struct sk_buff *rsp, unsigned int attr_id, u64 value)
 565 {
 566         if (value == NETDEV_STAT_NOT_SET)
 567                 return 0;
 568         return nla_put_uint(rsp, attr_id, value);
 569 }
 570
 571 static int
 572 netdev_nl_stats_write_rx(struct sk_buff *rsp, struct netdev_queue_stats_rx *rx)
 573 {
 574         if (netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_PACKETS, rx->packets) ||
 575             netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_BYTES, rx->bytes) ||
 576             netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_ALLOC_FAIL, rx->alloc_fail) ||
 577             netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROPS, rx->hw_drops) ||
 578             netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROP_OVERRUNS, rx->hw_drop_overruns) ||
 579             netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_UNNECESSARY, rx->csum_unnecessary) ||
 580             netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_NONE, rx->csum_none) ||
 581             netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_BAD, rx->csum_bad) ||
 582             netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_PACKETS, rx->hw_gro_packets) ||
 583             netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_BYTES, rx->hw_gro_bytes) ||
 584             netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_WIRE_PACKETS, rx->hw_gro_wire_packets) ||
 585             netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_WIRE_BYTES, rx->hw_gro_wire_bytes) ||
 586             netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROP_RATELIMITS, rx->hw_drop_ratelimits))
 587                 return -EMSGSIZE;
 588         return 0;
 589 }
 590
 591 static int
 592 netdev_nl_stats_write_tx(struct sk_buff *rsp, struct netdev_queue_stats_tx *tx)
 593 {
 594         if (netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_PACKETS, tx->packets) ||
 595             netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_BYTES, tx->bytes) ||
 596             netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROPS, tx->hw_drops) ||
 597             netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROP_ERRORS, tx->hw_drop_errors) ||
 598             netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_CSUM_NONE, tx->csum_none) ||
 599             netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_NEEDS_CSUM, tx->needs_csum) ||
 600             netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_PACKETS, tx->hw_gso_packets) ||
 601             netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_BYTES, tx->hw_gso_bytes) ||
 602             netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_WIRE_PACKETS, tx->hw_gso_wire_packets) ||
 603             netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_WIRE_BYTES, tx->hw_gso_wire_bytes) ||
 604             netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROP_RATELIMITS, tx->hw_drop_ratelimits) ||
 605             netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_STOP, tx->stop) ||
 606             netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_WAKE, tx->wake))
 607                 return -EMSGSIZE;
 608         return 0;
 609 }
 610
 611 static int
 612 netdev_nl_stats_queue(struct net_device *netdev, struct sk_buff *rsp,
 613                       u32 q_type, int i, const struct genl_info *info)
 614 {
 615         const struct netdev_stat_ops *ops = netdev->stat_ops;
 616         struct netdev_queue_stats_rx rx;
 617         struct netdev_queue_stats_tx tx;
 618         void *hdr;
 619
 620         hdr = genlmsg_iput(rsp, info);
 621         if (!hdr)
 622                 return -EMSGSIZE;
 623         if (nla_put_u32(rsp, NETDEV_A_QSTATS_IFINDEX, netdev->ifindex) ||
 624             nla_put_u32(rsp, NETDEV_A_QSTATS_QUEUE_TYPE, q_type) ||
 625             nla_put_u32(rsp, NETDEV_A_QSTATS_QUEUE_ID, i))
 626                 goto nla_put_failure;
 627
 628         switch (q_type) {
 629         case NETDEV_QUEUE_TYPE_RX:
 630                 memset(&rx, 0xff, sizeof(rx));
 631                 ops->get_queue_stats_rx(netdev, i, &rx);
 632                 if (!memchr_inv(&rx, 0xff, sizeof(rx)))
 633                         goto nla_cancel;
 634                 if (netdev_nl_stats_write_rx(rsp, &rx))
 635                         goto nla_put_failure;
 636                 break;
 637         case NETDEV_QUEUE_TYPE_TX:
 638                 memset(&tx, 0xff, sizeof(tx));
 639                 ops->get_queue_stats_tx(netdev, i, &tx);
 640                 if (!memchr_inv(&tx, 0xff, sizeof(tx)))
 641                         goto nla_cancel;
 642                 if (netdev_nl_stats_write_tx(rsp, &tx))
 643                         goto nla_put_failure;
 644                 break;
 645         }
 646
 647         genlmsg_end(rsp, hdr);
 648         return 0;
 649
 650 nla_cancel:
 651         genlmsg_cancel(rsp, hdr);
 652         return 0;
 653 nla_put_failure:
 654         genlmsg_cancel(rsp, hdr);
 655         return -EMSGSIZE;
 656 }
 657
 658 static int
 659 netdev_nl_stats_by_queue(struct net_device *netdev, struct sk_buff *rsp,
 660                          const struct genl_info *info,
 661                          struct netdev_nl_dump_ctx *ctx)
 662 {
 663         const struct netdev_stat_ops *ops = netdev->stat_ops;
 664         int i, err;
 665
 666         if (!(netdev->flags & IFF_UP))
 667                 return 0;
 668
 669         i = ctx->rxq_idx;
 670         while (ops->get_queue_stats_rx && i < netdev->real_num_rx_queues) {
 671                 err = netdev_nl_stats_queue(netdev, rsp, NETDEV_QUEUE_TYPE_RX,
 672                                             i, info);
 673                 if (err)
 674                         return err;
 675                 ctx->rxq_idx = ++i;
 676         }
 677         i = ctx->txq_idx;
 678         while (ops->get_queue_stats_tx && i < netdev->real_num_tx_queues) {
 679                 err = netdev_nl_stats_queue(netdev, rsp, NETDEV_QUEUE_TYPE_TX,
 680                                             i, info);
 681                 if (err)
 682                         return err;
 683                 ctx->txq_idx = ++i;
 684         }
 685
 686         ctx->rxq_idx = 0;
 687         ctx->txq_idx = 0;
 688         return 0;
 689 }
 690
 691 static int
 692 netdev_nl_stats_by_netdev(struct net_device *netdev, struct sk_buff *rsp,
 693                           const struct genl_info *info)
 694 {
 695         struct netdev_queue_stats_rx rx_sum, rx;
 696         struct netdev_queue_stats_tx tx_sum, tx;
 697         const struct netdev_stat_ops *ops;
 698         void *hdr;
 699         int i;
 700
 701         ops = netdev->stat_ops;
 702         /* Netdev can't guarantee any complete counters */
 703         if (!ops->get_base_stats)
 704                 return 0;
 705
 706         memset(&rx_sum, 0xff, sizeof(rx_sum));
 707         memset(&tx_sum, 0xff, sizeof(tx_sum));
 708
 709         ops->get_base_stats(netdev, &rx_sum, &tx_sum);
 710
 711         /* The op was there, but nothing reported, don't bother */
 712         if (!memchr_inv(&rx_sum, 0xff, sizeof(rx_sum)) &&
 713             !memchr_inv(&tx_sum, 0xff, sizeof(tx_sum)))
 714                 return 0;
 715
 716         hdr = genlmsg_iput(rsp, info);
 717         if (!hdr)
 718                 return -EMSGSIZE;
 719         if (nla_put_u32(rsp, NETDEV_A_QSTATS_IFINDEX, netdev->ifindex))
 720                 goto nla_put_failure;
 721
 722         for (i = 0; i < netdev->real_num_rx_queues; i++) {
 723                 memset(&rx, 0xff, sizeof(rx));
 724                 if (ops->get_queue_stats_rx)
 725                         ops->get_queue_stats_rx(netdev, i, &rx);
 726                 netdev_nl_stats_add(&rx_sum, &rx, sizeof(rx));
 727         }
 728         for (i = 0; i < netdev->real_num_tx_queues; i++) {
 729                 memset(&tx, 0xff, sizeof(tx));
 730                 if (ops->get_queue_stats_tx)
 731                         ops->get_queue_stats_tx(netdev, i, &tx);
 732                 netdev_nl_stats_add(&tx_sum, &tx, sizeof(tx));
 733         }
 734
 735         if (netdev_nl_stats_write_rx(rsp, &rx_sum) ||
 736             netdev_nl_stats_write_tx(rsp, &tx_sum))
 737                 goto nla_put_failure;
 738
 739         genlmsg_end(rsp, hdr);
 740         return 0;
 741
 742 nla_put_failure:
 743         genlmsg_cancel(rsp, hdr);
 744         return -EMSGSIZE;
 745 }
 746
 747 static int
 748 netdev_nl_qstats_get_dump_one(struct net_device *netdev, unsigned int scope,
 749                               struct sk_buff *skb, const struct genl_info *info,
 750                               struct netdev_nl_dump_ctx *ctx)
 751 {
 752         if (!netdev->stat_ops)
 753                 return 0;
 754
 755         switch (scope) {
 756         case 0:
 757                 return netdev_nl_stats_by_netdev(netdev, skb, info);
 758         case NETDEV_QSTATS_SCOPE_QUEUE:
 759                 return netdev_nl_stats_by_queue(netdev, skb, info, ctx);
 760         }
 761
 762         return -EINVAL; /* Should not happen, per netlink policy */
 763 }
 764
 765 int netdev_nl_qstats_get_dumpit(struct sk_buff *skb,
 766                                 struct netlink_callback *cb)
 767 {
 768         struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb);
 769         const struct genl_info *info = genl_info_dump(cb);
 770         struct net *net = sock_net(skb->sk);
 771         struct net_device *netdev;
 772         unsigned int ifindex;
 773         unsigned int scope;
 774         int err = 0;
 775
 776         scope = 0;
 777         if (info->attrs[NETDEV_A_QSTATS_SCOPE])
 778                 scope = nla_get_uint(info->attrs[NETDEV_A_QSTATS_SCOPE]);
 779
 780         ifindex = 0;
 781         if (info->attrs[NETDEV_A_QSTATS_IFINDEX])
 782                 ifindex = nla_get_u32(info->attrs[NETDEV_A_QSTATS_IFINDEX]);
 783
 784         rtnl_lock();
 785         if (ifindex) {
 786                 netdev = __dev_get_by_index(net, ifindex);
 787                 if (netdev && netdev->stat_ops) {
 788                         err = netdev_nl_qstats_get_dump_one(netdev, scope, skb,
 789                                                             info, ctx);
 790                 } else {
 791                         NL_SET_BAD_ATTR(info->extack,
 792                                         info->attrs[NETDEV_A_QSTATS_IFINDEX]);
 793                         err = netdev ? -EOPNOTSUPP : -ENODEV;
 794                 }
 795         } else {
 796                 for_each_netdev_dump(net, netdev, ctx->ifindex) {
 797                         err = netdev_nl_qstats_get_dump_one(netdev, scope, skb,
 798                                                             info, ctx);
 799                         if (err < 0)
 800                                 break;
 801                 }
 802         }
 803         rtnl_unlock();
 804
 805         return err;
 806 }
 807
 808 int netdev_nl_bind_rx_doit(struct sk_buff *skb, struct genl_info *info)
 809 {
 810         struct nlattr *tb[ARRAY_SIZE(netdev_queue_id_nl_policy)];
 811         struct net_devmem_dmabuf_binding *binding;
 812         struct list_head *sock_binding_list;
 813         u32 ifindex, dmabuf_fd, rxq_idx;
 814         struct net_device *netdev;
 815         struct sk_buff *rsp;
 816         struct nlattr *attr;
 817         int rem, err = 0;
 818         void *hdr;
 819
 820         if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_DEV_IFINDEX) ||
 821             GENL_REQ_ATTR_CHECK(info, NETDEV_A_DMABUF_FD) ||
 822             GENL_REQ_ATTR_CHECK(info, NETDEV_A_DMABUF_QUEUES))
 823                 return -EINVAL;
 824
 825         ifindex = nla_get_u32(info->attrs[NETDEV_A_DEV_IFINDEX]);
 826         dmabuf_fd = nla_get_u32(info->attrs[NETDEV_A_DMABUF_FD]);
 827
 828         sock_binding_list = genl_sk_priv_get(&netdev_nl_family,
 829                                              NETLINK_CB(skb).sk);
 830         if (IS_ERR(sock_binding_list))
 831                 return PTR_ERR(sock_binding_list);
 832
 833         rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
 834         if (!rsp)
 835                 return -ENOMEM;
 836
 837         hdr = genlmsg_iput(rsp, info);
 838         if (!hdr) {
 839                 err = -EMSGSIZE;
 840                 goto err_genlmsg_free;
 841         }
 842
 843         rtnl_lock();
 844
 845         netdev = __dev_get_by_index(genl_info_net(info), ifindex);
 846         if (!netdev || !netif_device_present(netdev)) {
 847                 err = -ENODEV;
 848                 goto err_unlock;
 849         }
 850
 851         if (dev_xdp_prog_count(netdev)) {
 852                 NL_SET_ERR_MSG(info->extack, "unable to bind dmabuf to device with XDP program attached");
 853                 err = -EEXIST;
 854                 goto err_unlock;
 855         }
 856
 857         binding = net_devmem_bind_dmabuf(netdev, dmabuf_fd, info->extack);
 858         if (IS_ERR(binding)) {
 859                 err = PTR_ERR(binding);
 860                 goto err_unlock;
 861         }
 862
 863         nla_for_each_attr_type(attr, NETDEV_A_DMABUF_QUEUES,
 864                                genlmsg_data(info->genlhdr),
 865                                genlmsg_len(info->genlhdr), rem) {
 866                 err = nla_parse_nested(
 867                         tb, ARRAY_SIZE(netdev_queue_id_nl_policy) - 1, attr,
 868                         netdev_queue_id_nl_policy, info->extack);
 869                 if (err < 0)
 870                         goto err_unbind;
 871
 872                 if (NL_REQ_ATTR_CHECK(info->extack, attr, tb, NETDEV_A_QUEUE_ID) ||
 873                     NL_REQ_ATTR_CHECK(info->extack, attr, tb, NETDEV_A_QUEUE_TYPE)) {
 874                         err = -EINVAL;
 875                         goto err_unbind;
 876                 }
 877
 878                 if (nla_get_u32(tb[NETDEV_A_QUEUE_TYPE]) != NETDEV_QUEUE_TYPE_RX) {
 879                         NL_SET_BAD_ATTR(info->extack, tb[NETDEV_A_QUEUE_TYPE]);
 880                         err = -EINVAL;
 881                         goto err_unbind;
 882                 }
 883
 884                 rxq_idx = nla_get_u32(tb[NETDEV_A_QUEUE_ID]);
 885
 886                 err = net_devmem_bind_dmabuf_to_queue(netdev, rxq_idx, binding,
 887                                                       info->extack);
 888                 if (err)
 889                         goto err_unbind;
 890         }
 891
 892         list_add(&binding->list, sock_binding_list);
 893
 894         nla_put_u32(rsp, NETDEV_A_DMABUF_ID, binding->id);
 895         genlmsg_end(rsp, hdr);
 896
 897         err = genlmsg_reply(rsp, info);
 898         if (err)
 899                 goto err_unbind;
 900
 901         rtnl_unlock();
 902
 903         return 0;
 904
 905 err_unbind:
 906         net_devmem_unbind_dmabuf(binding);
 907 err_unlock:
 908         rtnl_unlock();
 909 err_genlmsg_free:
 910         nlmsg_free(rsp);
 911         return err;
 912 }
 913
 914 void netdev_nl_sock_priv_init(struct list_head *priv)
 915 {
 916         INIT_LIST_HEAD(priv);
 917 }
 918
 919 void netdev_nl_sock_priv_destroy(struct list_head *priv)
 920 {
 921         struct net_devmem_dmabuf_binding *binding;
 922         struct net_devmem_dmabuf_binding *temp;
 923
 924         list_for_each_entry_safe(binding, temp, priv, list) {
 925                 rtnl_lock();
 926                 net_devmem_unbind_dmabuf(binding);
 927                 rtnl_unlock();
 928         }
 929 }
 930
 931 static int netdev_genl_netdevice_event(struct notifier_block *nb,
 932                                        unsigned long event, void *ptr)
 933 {
 934         struct net_device *netdev = netdev_notifier_info_to_dev(ptr);
 935
 936         switch (event) {
 937         case NETDEV_REGISTER:
 938                 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_ADD_NTF);
 939                 break;
 940         case NETDEV_UNREGISTER:
 941                 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_DEL_NTF);
 942                 break;
 943         case NETDEV_XDP_FEAT_CHANGE:
 944                 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_CHANGE_NTF);
 945                 break;
 946         }
 947
 948         return NOTIFY_OK;
 949 }
 950
 951 static struct notifier_block netdev_genl_nb = {
 952         .notifier_call  = netdev_genl_netdevice_event,
 953 };
 954
 955 static int __init netdev_genl_init(void)
 956 {
 957         int err;
 958
 959         err = register_netdevice_notifier(&netdev_genl_nb);
 960         if (err)
 961                 return err;
 962
 963         err = genl_register_family(&netdev_nl_family);
 964         if (err)
 965                 goto err_unreg_ntf;
 966
 967         return 0;
 968
 969 err_unreg_ntf:
 970         unregister_netdevice_notifier(&netdev_genl_nb);
 971         return err;
 972 }
 973
 974 subsys_initcall(netdev_genl_init);