net/ipv4/udp_tunnel_nic.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 // Copyright (c) 2020 Facebook Inc.
   3
   4 #include <linux/ethtool_netlink.h>
   5 #include <linux/netdevice.h>
   6 #include <linux/slab.h>
   7 #include <linux/types.h>
   8 #include <linux/workqueue.h>
   9 #include <net/udp_tunnel.h>
  10 #include <net/vxlan.h>
  11
  12 enum udp_tunnel_nic_table_entry_flags {
  13         UDP_TUNNEL_NIC_ENTRY_ADD        = BIT(0),
  14         UDP_TUNNEL_NIC_ENTRY_DEL        = BIT(1),
  15         UDP_TUNNEL_NIC_ENTRY_OP_FAIL    = BIT(2),
  16         UDP_TUNNEL_NIC_ENTRY_FROZEN     = BIT(3),
  17 };
  18
  19 struct udp_tunnel_nic_table_entry {
  20         __be16 port;
  21         u8 type;
  22         u8 flags;
  23         u16 use_cnt;
  24 #define UDP_TUNNEL_NIC_USE_CNT_MAX      U16_MAX
  25         u8 hw_priv;
  26 };
  27
  28 /**
  29  * struct udp_tunnel_nic - UDP tunnel port offload state
  30  * @work:       async work for talking to hardware from process context
  31  * @dev:        netdev pointer
  32  * @need_sync:  at least one port start changed
  33  * @need_replay: space was freed, we need a replay of all ports
  34  * @work_pending: @work is currently scheduled
  35  * @n_tables:   number of tables under @entries
  36  * @missed:     bitmap of tables which overflown
  37  * @entries:    table of tables of ports currently offloaded
  38  */
  39 struct udp_tunnel_nic {
  40         struct work_struct work;
  41
  42         struct net_device *dev;
  43
  44         u8 need_sync:1;
  45         u8 need_replay:1;
  46         u8 work_pending:1;
  47
  48         unsigned int n_tables;
  49         unsigned long missed;
  50         struct udp_tunnel_nic_table_entry **entries;
  51 };
  52
  53 /* We ensure all work structs are done using driver state, but not the code.
  54  * We need a workqueue we can flush before module gets removed.
  55  */
  56 static struct workqueue_struct *udp_tunnel_nic_workqueue;
  57
  58 static const char *udp_tunnel_nic_tunnel_type_name(unsigned int type)
  59 {
  60         switch (type) {
  61         case UDP_TUNNEL_TYPE_VXLAN:
  62                 return "vxlan";
  63         case UDP_TUNNEL_TYPE_GENEVE:
  64                 return "geneve";
  65         case UDP_TUNNEL_TYPE_VXLAN_GPE:
  66                 return "vxlan-gpe";
  67         default:
  68                 return "unknown";
  69         }
  70 }
  71
  72 static bool
  73 udp_tunnel_nic_entry_is_free(struct udp_tunnel_nic_table_entry *entry)
  74 {
  75         return entry->use_cnt == 0 && !entry->flags;
  76 }
  77
  78 static bool
  79 udp_tunnel_nic_entry_is_present(struct udp_tunnel_nic_table_entry *entry)
  80 {
  81         return entry->use_cnt && !(entry->flags & ~UDP_TUNNEL_NIC_ENTRY_FROZEN);
  82 }
  83
  84 static bool
  85 udp_tunnel_nic_entry_is_frozen(struct udp_tunnel_nic_table_entry *entry)
  86 {
  87         return entry->flags & UDP_TUNNEL_NIC_ENTRY_FROZEN;
  88 }
  89
  90 static void
  91 udp_tunnel_nic_entry_freeze_used(struct udp_tunnel_nic_table_entry *entry)
  92 {
  93         if (!udp_tunnel_nic_entry_is_free(entry))
  94                 entry->flags |= UDP_TUNNEL_NIC_ENTRY_FROZEN;
  95 }
  96
  97 static void
  98 udp_tunnel_nic_entry_unfreeze(struct udp_tunnel_nic_table_entry *entry)
  99 {
 100         entry->flags &= ~UDP_TUNNEL_NIC_ENTRY_FROZEN;
 101 }
 102
 103 static bool
 104 udp_tunnel_nic_entry_is_queued(struct udp_tunnel_nic_table_entry *entry)
 105 {
 106         return entry->flags & (UDP_TUNNEL_NIC_ENTRY_ADD |
 107                                UDP_TUNNEL_NIC_ENTRY_DEL);
 108 }
 109
 110 static void
 111 udp_tunnel_nic_entry_queue(struct udp_tunnel_nic *utn,
 112                            struct udp_tunnel_nic_table_entry *entry,
 113                            unsigned int flag)
 114 {
 115         entry->flags |= flag;
 116         utn->need_sync = 1;
 117 }
 118
 119 static void
 120 udp_tunnel_nic_ti_from_entry(struct udp_tunnel_nic_table_entry *entry,
 121                              struct udp_tunnel_info *ti)
 122 {
 123         memset(ti, 0, sizeof(*ti));
 124         ti->port = entry->port;
 125         ti->type = entry->type;
 126         ti->hw_priv = entry->hw_priv;
 127 }
 128
 129 static bool
 130 udp_tunnel_nic_is_empty(struct net_device *dev, struct udp_tunnel_nic *utn)
 131 {
 132         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
 133         unsigned int i, j;
 134
 135         for (i = 0; i < utn->n_tables; i++)
 136                 for (j = 0; j < info->tables[i].n_entries; j++)
 137                         if (!udp_tunnel_nic_entry_is_free(&utn->entries[i][j]))
 138                                 return false;
 139         return true;
 140 }
 141
 142 static bool
 143 udp_tunnel_nic_should_replay(struct net_device *dev, struct udp_tunnel_nic *utn)
 144 {
 145         const struct udp_tunnel_nic_table_info *table;
 146         unsigned int i, j;
 147
 148         if (!utn->missed)
 149                 return false;
 150
 151         for (i = 0; i < utn->n_tables; i++) {
 152                 table = &dev->udp_tunnel_nic_info->tables[i];
 153                 if (!test_bit(i, &utn->missed))
 154                         continue;
 155
 156                 for (j = 0; j < table->n_entries; j++)
 157                         if (udp_tunnel_nic_entry_is_free(&utn->entries[i][j]))
 158                                 return true;
 159         }
 160
 161         return false;
 162 }
 163
 164 static void
 165 __udp_tunnel_nic_get_port(struct net_device *dev, unsigned int table,
 166                           unsigned int idx, struct udp_tunnel_info *ti)
 167 {
 168         struct udp_tunnel_nic_table_entry *entry;
 169         struct udp_tunnel_nic *utn;
 170
 171         utn = dev->udp_tunnel_nic;
 172         entry = &utn->entries[table][idx];
 173
 174         if (entry->use_cnt)
 175                 udp_tunnel_nic_ti_from_entry(entry, ti);
 176 }
 177
 178 static void
 179 __udp_tunnel_nic_set_port_priv(struct net_device *dev, unsigned int table,
 180                                unsigned int idx, u8 priv)
 181 {
 182         dev->udp_tunnel_nic->entries[table][idx].hw_priv = priv;
 183 }
 184
 185 static void
 186 udp_tunnel_nic_entry_update_done(struct udp_tunnel_nic_table_entry *entry,
 187                                  int err)
 188 {
 189         bool dodgy = entry->flags & UDP_TUNNEL_NIC_ENTRY_OP_FAIL;
 190
 191         WARN_ON_ONCE(entry->flags & UDP_TUNNEL_NIC_ENTRY_ADD &&
 192                      entry->flags & UDP_TUNNEL_NIC_ENTRY_DEL);
 193
 194         if (entry->flags & UDP_TUNNEL_NIC_ENTRY_ADD &&
 195             (!err || (err == -EEXIST && dodgy)))
 196                 entry->flags &= ~UDP_TUNNEL_NIC_ENTRY_ADD;
 197
 198         if (entry->flags & UDP_TUNNEL_NIC_ENTRY_DEL &&
 199             (!err || (err == -ENOENT && dodgy)))
 200                 entry->flags &= ~UDP_TUNNEL_NIC_ENTRY_DEL;
 201
 202         if (!err)
 203                 entry->flags &= ~UDP_TUNNEL_NIC_ENTRY_OP_FAIL;
 204         else
 205                 entry->flags |= UDP_TUNNEL_NIC_ENTRY_OP_FAIL;
 206 }
 207
 208 static void
 209 udp_tunnel_nic_device_sync_one(struct net_device *dev,
 210                                struct udp_tunnel_nic *utn,
 211                                unsigned int table, unsigned int idx)
 212 {
 213         struct udp_tunnel_nic_table_entry *entry;
 214         struct udp_tunnel_info ti;
 215         int err;
 216
 217         entry = &utn->entries[table][idx];
 218         if (!udp_tunnel_nic_entry_is_queued(entry))
 219                 return;
 220
 221         udp_tunnel_nic_ti_from_entry(entry, &ti);
 222         if (entry->flags & UDP_TUNNEL_NIC_ENTRY_ADD)
 223                 err = dev->udp_tunnel_nic_info->set_port(dev, table, idx, &ti);
 224         else
 225                 err = dev->udp_tunnel_nic_info->unset_port(dev, table, idx,
 226                                                            &ti);
 227         udp_tunnel_nic_entry_update_done(entry, err);
 228
 229         if (err)
 230                 netdev_warn(dev,
 231                             "UDP tunnel port sync failed port %d type %s: %d\n",
 232                             be16_to_cpu(entry->port),
 233                             udp_tunnel_nic_tunnel_type_name(entry->type),
 234                             err);
 235 }
 236
 237 static void
 238 udp_tunnel_nic_device_sync_by_port(struct net_device *dev,
 239                                    struct udp_tunnel_nic *utn)
 240 {
 241         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
 242         unsigned int i, j;
 243
 244         for (i = 0; i < utn->n_tables; i++)
 245                 for (j = 0; j < info->tables[i].n_entries; j++)
 246                         udp_tunnel_nic_device_sync_one(dev, utn, i, j);
 247 }
 248
 249 static void
 250 udp_tunnel_nic_device_sync_by_table(struct net_device *dev,
 251                                     struct udp_tunnel_nic *utn)
 252 {
 253         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
 254         unsigned int i, j;
 255         int err;
 256
 257         for (i = 0; i < utn->n_tables; i++) {
 258                 /* Find something that needs sync in this table */
 259                 for (j = 0; j < info->tables[i].n_entries; j++)
 260                         if (udp_tunnel_nic_entry_is_queued(&utn->entries[i][j]))
 261                                 break;
 262                 if (j == info->tables[i].n_entries)
 263                         continue;
 264
 265                 err = info->sync_table(dev, i);
 266                 if (err)
 267                         netdev_warn(dev, "UDP tunnel port sync failed for table %d: %d\n",
 268                                     i, err);
 269
 270                 for (j = 0; j < info->tables[i].n_entries; j++) {
 271                         struct udp_tunnel_nic_table_entry *entry;
 272
 273                         entry = &utn->entries[i][j];
 274                         if (udp_tunnel_nic_entry_is_queued(entry))
 275                                 udp_tunnel_nic_entry_update_done(entry, err);
 276                 }
 277         }
 278 }
 279
 280 static void
 281 __udp_tunnel_nic_device_sync(struct net_device *dev, struct udp_tunnel_nic *utn)
 282 {
 283         if (!utn->need_sync)
 284                 return;
 285
 286         if (dev->udp_tunnel_nic_info->sync_table)
 287                 udp_tunnel_nic_device_sync_by_table(dev, utn);
 288         else
 289                 udp_tunnel_nic_device_sync_by_port(dev, utn);
 290
 291         utn->need_sync = 0;
 292         /* Can't replay directly here, in case we come from the tunnel driver's
 293          * notification - trying to replay may deadlock inside tunnel driver.
 294          */
 295         utn->need_replay = udp_tunnel_nic_should_replay(dev, utn);
 296 }
 297
 298 static void
 299 udp_tunnel_nic_device_sync(struct net_device *dev, struct udp_tunnel_nic *utn)
 300 {
 301         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
 302         bool may_sleep;
 303
 304         if (!utn->need_sync)
 305                 return;
 306
 307         /* Drivers which sleep in the callback need to update from
 308          * the workqueue, if we come from the tunnel driver's notification.
 309          */
 310         may_sleep = info->flags & UDP_TUNNEL_NIC_INFO_MAY_SLEEP;
 311         if (!may_sleep)
 312                 __udp_tunnel_nic_device_sync(dev, utn);
 313         if (may_sleep || utn->need_replay) {
 314                 queue_work(udp_tunnel_nic_workqueue, &utn->work);
 315                 utn->work_pending = 1;
 316         }
 317 }
 318
 319 static bool
 320 udp_tunnel_nic_table_is_capable(const struct udp_tunnel_nic_table_info *table,
 321                                 struct udp_tunnel_info *ti)
 322 {
 323         return table->tunnel_types & ti->type;
 324 }
 325
 326 static bool
 327 udp_tunnel_nic_is_capable(struct net_device *dev, struct udp_tunnel_nic *utn,
 328                           struct udp_tunnel_info *ti)
 329 {
 330         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
 331         unsigned int i;
 332
 333         /* Special case IPv4-only NICs */
 334         if (info->flags & UDP_TUNNEL_NIC_INFO_IPV4_ONLY &&
 335             ti->sa_family != AF_INET)
 336                 return false;
 337
 338         for (i = 0; i < utn->n_tables; i++)
 339                 if (udp_tunnel_nic_table_is_capable(&info->tables[i], ti))
 340                         return true;
 341         return false;
 342 }
 343
 344 static int
 345 udp_tunnel_nic_has_collision(struct net_device *dev, struct udp_tunnel_nic *utn,
 346                              struct udp_tunnel_info *ti)
 347 {
 348         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
 349         struct udp_tunnel_nic_table_entry *entry;
 350         unsigned int i, j;
 351
 352         for (i = 0; i < utn->n_tables; i++)
 353                 for (j = 0; j < info->tables[i].n_entries; j++) {
 354                         entry = &utn->entries[i][j];
 355
 356                         if (!udp_tunnel_nic_entry_is_free(entry) &&
 357                             entry->port == ti->port &&
 358                             entry->type != ti->type) {
 359                                 __set_bit(i, &utn->missed);
 360                                 return true;
 361                         }
 362                 }
 363         return false;
 364 }
 365
 366 static void
 367 udp_tunnel_nic_entry_adj(struct udp_tunnel_nic *utn,
 368                          unsigned int table, unsigned int idx, int use_cnt_adj)
 369 {
 370         struct udp_tunnel_nic_table_entry *entry =  &utn->entries[table][idx];
 371         bool dodgy = entry->flags & UDP_TUNNEL_NIC_ENTRY_OP_FAIL;
 372         unsigned int from, to;
 373
 374         WARN_ON(entry->use_cnt + (u32)use_cnt_adj > U16_MAX);
 375
 376         /* If not going from used to unused or vice versa - all done.
 377          * For dodgy entries make sure we try to sync again (queue the entry).
 378          */
 379         entry->use_cnt += use_cnt_adj;
 380         if (!dodgy && !entry->use_cnt == !(entry->use_cnt - use_cnt_adj))
 381                 return;
 382
 383         /* Cancel the op before it was sent to the device, if possible,
 384          * otherwise we'd need to take special care to issue commands
 385          * in the same order the ports arrived.
 386          */
 387         if (use_cnt_adj < 0) {
 388                 from = UDP_TUNNEL_NIC_ENTRY_ADD;
 389                 to = UDP_TUNNEL_NIC_ENTRY_DEL;
 390         } else {
 391                 from = UDP_TUNNEL_NIC_ENTRY_DEL;
 392                 to = UDP_TUNNEL_NIC_ENTRY_ADD;
 393         }
 394
 395         if (entry->flags & from) {
 396                 entry->flags &= ~from;
 397                 if (!dodgy)
 398                         return;
 399         }
 400
 401         udp_tunnel_nic_entry_queue(utn, entry, to);
 402 }
 403
 404 static bool
 405 udp_tunnel_nic_entry_try_adj(struct udp_tunnel_nic *utn,
 406                              unsigned int table, unsigned int idx,
 407                              struct udp_tunnel_info *ti, int use_cnt_adj)
 408 {
 409         struct udp_tunnel_nic_table_entry *entry =  &utn->entries[table][idx];
 410
 411         if (udp_tunnel_nic_entry_is_free(entry) ||
 412             entry->port != ti->port ||
 413             entry->type != ti->type)
 414                 return false;
 415
 416         if (udp_tunnel_nic_entry_is_frozen(entry))
 417                 return true;
 418
 419         udp_tunnel_nic_entry_adj(utn, table, idx, use_cnt_adj);
 420         return true;
 421 }
 422
 423 /* Try to find existing matching entry and adjust its use count, instead of
 424  * adding a new one. Returns true if entry was found. In case of delete the
 425  * entry may have gotten removed in the process, in which case it will be
 426  * queued for removal.
 427  */
 428 static bool
 429 udp_tunnel_nic_try_existing(struct net_device *dev, struct udp_tunnel_nic *utn,
 430                             struct udp_tunnel_info *ti, int use_cnt_adj)
 431 {
 432         const struct udp_tunnel_nic_table_info *table;
 433         unsigned int i, j;
 434
 435         for (i = 0; i < utn->n_tables; i++) {
 436                 table = &dev->udp_tunnel_nic_info->tables[i];
 437                 if (!udp_tunnel_nic_table_is_capable(table, ti))
 438                         continue;
 439
 440                 for (j = 0; j < table->n_entries; j++)
 441                         if (udp_tunnel_nic_entry_try_adj(utn, i, j, ti,
 442                                                          use_cnt_adj))
 443                                 return true;
 444         }
 445
 446         return false;
 447 }
 448
 449 static bool
 450 udp_tunnel_nic_add_existing(struct net_device *dev, struct udp_tunnel_nic *utn,
 451                             struct udp_tunnel_info *ti)
 452 {
 453         return udp_tunnel_nic_try_existing(dev, utn, ti, +1);
 454 }
 455
 456 static bool
 457 udp_tunnel_nic_del_existing(struct net_device *dev, struct udp_tunnel_nic *utn,
 458                             struct udp_tunnel_info *ti)
 459 {
 460         return udp_tunnel_nic_try_existing(dev, utn, ti, -1);
 461 }
 462
 463 static bool
 464 udp_tunnel_nic_add_new(struct net_device *dev, struct udp_tunnel_nic *utn,
 465                        struct udp_tunnel_info *ti)
 466 {
 467         const struct udp_tunnel_nic_table_info *table;
 468         unsigned int i, j;
 469
 470         for (i = 0; i < utn->n_tables; i++) {
 471                 table = &dev->udp_tunnel_nic_info->tables[i];
 472                 if (!udp_tunnel_nic_table_is_capable(table, ti))
 473                         continue;
 474
 475                 for (j = 0; j < table->n_entries; j++) {
 476                         struct udp_tunnel_nic_table_entry *entry;
 477
 478                         entry = &utn->entries[i][j];
 479                         if (!udp_tunnel_nic_entry_is_free(entry))
 480                                 continue;
 481
 482                         entry->port = ti->port;
 483                         entry->type = ti->type;
 484                         entry->use_cnt = 1;
 485                         udp_tunnel_nic_entry_queue(utn, entry,
 486                                                    UDP_TUNNEL_NIC_ENTRY_ADD);
 487                         return true;
 488                 }
 489
 490                 /* The different table may still fit this port in, but there
 491                  * are no devices currently which have multiple tables accepting
 492                  * the same tunnel type, and false positives are okay.
 493                  */
 494                 __set_bit(i, &utn->missed);
 495         }
 496
 497         return false;
 498 }
 499
 500 static void
 501 __udp_tunnel_nic_add_port(struct net_device *dev, struct udp_tunnel_info *ti)
 502 {
 503         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
 504         struct udp_tunnel_nic *utn;
 505
 506         utn = dev->udp_tunnel_nic;
 507         if (!utn)
 508                 return;
 509         if (!netif_running(dev) && info->flags & UDP_TUNNEL_NIC_INFO_OPEN_ONLY)
 510                 return;
 511         if (info->flags & UDP_TUNNEL_NIC_INFO_STATIC_IANA_VXLAN &&
 512             ti->port == htons(IANA_VXLAN_UDP_PORT)) {
 513                 if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
 514                         netdev_warn(dev, "device assumes port 4789 will be used by vxlan tunnels\n");
 515                 return;
 516         }
 517
 518         if (!udp_tunnel_nic_is_capable(dev, utn, ti))
 519                 return;
 520
 521         /* It may happen that a tunnel of one type is removed and different
 522          * tunnel type tries to reuse its port before the device was informed.
 523          * Rely on utn->missed to re-add this port later.
 524          */
 525         if (udp_tunnel_nic_has_collision(dev, utn, ti))
 526                 return;
 527
 528         if (!udp_tunnel_nic_add_existing(dev, utn, ti))
 529                 udp_tunnel_nic_add_new(dev, utn, ti);
 530
 531         udp_tunnel_nic_device_sync(dev, utn);
 532 }
 533
 534 static void
 535 __udp_tunnel_nic_del_port(struct net_device *dev, struct udp_tunnel_info *ti)
 536 {
 537         struct udp_tunnel_nic *utn;
 538
 539         utn = dev->udp_tunnel_nic;
 540         if (!utn)
 541                 return;
 542
 543         if (!udp_tunnel_nic_is_capable(dev, utn, ti))
 544                 return;
 545
 546         udp_tunnel_nic_del_existing(dev, utn, ti);
 547
 548         udp_tunnel_nic_device_sync(dev, utn);
 549 }
 550
 551 static void __udp_tunnel_nic_reset_ntf(struct net_device *dev)
 552 {
 553         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
 554         struct udp_tunnel_nic *utn;
 555         unsigned int i, j;
 556
 557         ASSERT_RTNL();
 558
 559         utn = dev->udp_tunnel_nic;
 560         if (!utn)
 561                 return;
 562
 563         utn->need_sync = false;
 564         for (i = 0; i < utn->n_tables; i++)
 565                 for (j = 0; j < info->tables[i].n_entries; j++) {
 566                         struct udp_tunnel_nic_table_entry *entry;
 567
 568                         entry = &utn->entries[i][j];
 569
 570                         entry->flags &= ~(UDP_TUNNEL_NIC_ENTRY_DEL |
 571                                           UDP_TUNNEL_NIC_ENTRY_OP_FAIL);
 572                         /* We don't release rtnl across ops */
 573                         WARN_ON(entry->flags & UDP_TUNNEL_NIC_ENTRY_FROZEN);
 574                         if (!entry->use_cnt)
 575                                 continue;
 576
 577                         udp_tunnel_nic_entry_queue(utn, entry,
 578                                                    UDP_TUNNEL_NIC_ENTRY_ADD);
 579                 }
 580
 581         __udp_tunnel_nic_device_sync(dev, utn);
 582 }
 583
 584 static size_t
 585 __udp_tunnel_nic_dump_size(struct net_device *dev, unsigned int table)
 586 {
 587         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
 588         struct udp_tunnel_nic *utn;
 589         unsigned int j;
 590         size_t size;
 591
 592         utn = dev->udp_tunnel_nic;
 593         if (!utn)
 594                 return 0;
 595
 596         size = 0;
 597         for (j = 0; j < info->tables[table].n_entries; j++) {
 598                 if (!udp_tunnel_nic_entry_is_present(&utn->entries[table][j]))
 599                         continue;
 600
 601                 size += nla_total_size(0) +              /* _TABLE_ENTRY */
 602                         nla_total_size(sizeof(__be16)) + /* _ENTRY_PORT */
 603                         nla_total_size(sizeof(u32));     /* _ENTRY_TYPE */
 604         }
 605
 606         return size;
 607 }
 608
 609 static int
 610 __udp_tunnel_nic_dump_write(struct net_device *dev, unsigned int table,
 611                             struct sk_buff *skb)
 612 {
 613         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
 614         struct udp_tunnel_nic *utn;
 615         struct nlattr *nest;
 616         unsigned int j;
 617
 618         utn = dev->udp_tunnel_nic;
 619         if (!utn)
 620                 return 0;
 621
 622         for (j = 0; j < info->tables[table].n_entries; j++) {
 623                 if (!udp_tunnel_nic_entry_is_present(&utn->entries[table][j]))
 624                         continue;
 625
 626                 nest = nla_nest_start(skb, ETHTOOL_A_TUNNEL_UDP_TABLE_ENTRY);
 627
 628                 if (nla_put_be16(skb, ETHTOOL_A_TUNNEL_UDP_ENTRY_PORT,
 629                                  utn->entries[table][j].port) ||
 630                     nla_put_u32(skb, ETHTOOL_A_TUNNEL_UDP_ENTRY_TYPE,
 631                                 ilog2(utn->entries[table][j].type)))
 632                         goto err_cancel;
 633
 634                 nla_nest_end(skb, nest);
 635         }
 636
 637         return 0;
 638
 639 err_cancel:
 640         nla_nest_cancel(skb, nest);
 641         return -EMSGSIZE;
 642 }
 643
 644 static const struct udp_tunnel_nic_ops __udp_tunnel_nic_ops = {
 645         .get_port       = __udp_tunnel_nic_get_port,
 646         .set_port_priv  = __udp_tunnel_nic_set_port_priv,
 647         .add_port       = __udp_tunnel_nic_add_port,
 648         .del_port       = __udp_tunnel_nic_del_port,
 649         .reset_ntf      = __udp_tunnel_nic_reset_ntf,
 650         .dump_size      = __udp_tunnel_nic_dump_size,
 651         .dump_write     = __udp_tunnel_nic_dump_write,
 652 };
 653
 654 static void
 655 udp_tunnel_nic_flush(struct net_device *dev, struct udp_tunnel_nic *utn)
 656 {
 657         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
 658         unsigned int i, j;
 659
 660         for (i = 0; i < utn->n_tables; i++)
 661                 for (j = 0; j < info->tables[i].n_entries; j++) {
 662                         int adj_cnt = -utn->entries[i][j].use_cnt;
 663
 664                         if (adj_cnt)
 665                                 udp_tunnel_nic_entry_adj(utn, i, j, adj_cnt);
 666                 }
 667
 668         __udp_tunnel_nic_device_sync(dev, utn);
 669
 670         for (i = 0; i < utn->n_tables; i++)
 671                 memset(utn->entries[i], 0, array_size(info->tables[i].n_entries,
 672                                                       sizeof(**utn->entries)));
 673         WARN_ON(utn->need_sync);
 674         utn->need_replay = 0;
 675 }
 676
 677 static void
 678 udp_tunnel_nic_replay(struct net_device *dev, struct udp_tunnel_nic *utn)
 679 {
 680         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
 681         struct udp_tunnel_nic_shared_node *node;
 682         unsigned int i, j;
 683
 684         /* Freeze all the ports we are already tracking so that the replay
 685          * does not double up the refcount.
 686          */
 687         for (i = 0; i < utn->n_tables; i++)
 688                 for (j = 0; j < info->tables[i].n_entries; j++)
 689                         udp_tunnel_nic_entry_freeze_used(&utn->entries[i][j]);
 690         utn->missed = 0;
 691         utn->need_replay = 0;
 692
 693         if (!info->shared) {
 694                 udp_tunnel_get_rx_info(dev);
 695         } else {
 696                 list_for_each_entry(node, &info->shared->devices, list)
 697                         udp_tunnel_get_rx_info(node->dev);
 698         }
 699
 700         for (i = 0; i < utn->n_tables; i++)
 701                 for (j = 0; j < info->tables[i].n_entries; j++)
 702                         udp_tunnel_nic_entry_unfreeze(&utn->entries[i][j]);
 703 }
 704
 705 static void udp_tunnel_nic_device_sync_work(struct work_struct *work)
 706 {
 707         struct udp_tunnel_nic *utn =
 708                 container_of(work, struct udp_tunnel_nic, work);
 709
 710         rtnl_lock();
 711         utn->work_pending = 0;
 712         __udp_tunnel_nic_device_sync(utn->dev, utn);
 713
 714         if (utn->need_replay)
 715                 udp_tunnel_nic_replay(utn->dev, utn);
 716         rtnl_unlock();
 717 }
 718
 719 static struct udp_tunnel_nic *
 720 udp_tunnel_nic_alloc(const struct udp_tunnel_nic_info *info,
 721                      unsigned int n_tables)
 722 {
 723         struct udp_tunnel_nic *utn;
 724         unsigned int i;
 725
 726         utn = kzalloc(sizeof(*utn), GFP_KERNEL);
 727         if (!utn)
 728                 return NULL;
 729         utn->n_tables = n_tables;
 730         INIT_WORK(&utn->work, udp_tunnel_nic_device_sync_work);
 731
 732         utn->entries = kmalloc_array(n_tables, sizeof(void *), GFP_KERNEL);
 733         if (!utn->entries)
 734                 goto err_free_utn;
 735
 736         for (i = 0; i < n_tables; i++) {
 737                 utn->entries[i] = kcalloc(info->tables[i].n_entries,
 738                                           sizeof(*utn->entries[i]), GFP_KERNEL);
 739                 if (!utn->entries[i])
 740                         goto err_free_prev_entries;
 741         }
 742
 743         return utn;
 744
 745 err_free_prev_entries:
 746         while (i--)
 747                 kfree(utn->entries[i]);
 748         kfree(utn->entries);
 749 err_free_utn:
 750         kfree(utn);
 751         return NULL;
 752 }
 753
 754 static void udp_tunnel_nic_free(struct udp_tunnel_nic *utn)
 755 {
 756         unsigned int i;
 757
 758         for (i = 0; i < utn->n_tables; i++)
 759                 kfree(utn->entries[i]);
 760         kfree(utn->entries);
 761         kfree(utn);
 762 }
 763
 764 static int udp_tunnel_nic_register(struct net_device *dev)
 765 {
 766         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
 767         struct udp_tunnel_nic_shared_node *node = NULL;
 768         struct udp_tunnel_nic *utn;
 769         unsigned int n_tables, i;
 770
 771         BUILD_BUG_ON(sizeof(utn->missed) * BITS_PER_BYTE <
 772                      UDP_TUNNEL_NIC_MAX_TABLES);
 773         /* Expect use count of at most 2 (IPv4, IPv6) per device */
 774         BUILD_BUG_ON(UDP_TUNNEL_NIC_USE_CNT_MAX <
 775                      UDP_TUNNEL_NIC_MAX_SHARING_DEVICES * 2);
 776
 777         /* Check that the driver info is sane */
 778         if (WARN_ON(!info->set_port != !info->unset_port) ||
 779             WARN_ON(!info->set_port == !info->sync_table) ||
 780             WARN_ON(!info->tables[0].n_entries))
 781                 return -EINVAL;
 782
 783         if (WARN_ON(info->shared &&
 784                     info->flags & UDP_TUNNEL_NIC_INFO_OPEN_ONLY))
 785                 return -EINVAL;
 786
 787         n_tables = 1;
 788         for (i = 1; i < UDP_TUNNEL_NIC_MAX_TABLES; i++) {
 789                 if (!info->tables[i].n_entries)
 790                         continue;
 791
 792                 n_tables++;
 793                 if (WARN_ON(!info->tables[i - 1].n_entries))
 794                         return -EINVAL;
 795         }
 796
 797         /* Create UDP tunnel state structures */
 798         if (info->shared) {
 799                 node = kzalloc(sizeof(*node), GFP_KERNEL);
 800                 if (!node)
 801                         return -ENOMEM;
 802
 803                 node->dev = dev;
 804         }
 805
 806         if (info->shared && info->shared->udp_tunnel_nic_info) {
 807                 utn = info->shared->udp_tunnel_nic_info;
 808         } else {
 809                 utn = udp_tunnel_nic_alloc(info, n_tables);
 810                 if (!utn) {
 811                         kfree(node);
 812                         return -ENOMEM;
 813                 }
 814         }
 815
 816         if (info->shared) {
 817                 if (!info->shared->udp_tunnel_nic_info) {
 818                         INIT_LIST_HEAD(&info->shared->devices);
 819                         info->shared->udp_tunnel_nic_info = utn;
 820                 }
 821
 822                 list_add_tail(&node->list, &info->shared->devices);
 823         }
 824
 825         utn->dev = dev;
 826         dev_hold(dev);
 827         dev->udp_tunnel_nic = utn;
 828
 829         if (!(info->flags & UDP_TUNNEL_NIC_INFO_OPEN_ONLY))
 830                 udp_tunnel_get_rx_info(dev);
 831
 832         return 0;
 833 }
 834
 835 static void
 836 udp_tunnel_nic_unregister(struct net_device *dev, struct udp_tunnel_nic *utn)
 837 {
 838         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
 839
 840         /* For a shared table remove this dev from the list of sharing devices
 841          * and if there are other devices just detach.
 842          */
 843         if (info->shared) {
 844                 struct udp_tunnel_nic_shared_node *node, *first;
 845
 846                 list_for_each_entry(node, &info->shared->devices, list)
 847                         if (node->dev == dev)
 848                                 break;
 849                 if (node->dev != dev)
 850                         return;
 851
 852                 list_del(&node->list);
 853                 kfree(node);
 854
 855                 first = list_first_entry_or_null(&info->shared->devices,
 856                                                  typeof(*first), list);
 857                 if (first) {
 858                         udp_tunnel_drop_rx_info(dev);
 859                         utn->dev = first->dev;
 860                         goto release_dev;
 861                 }
 862
 863                 info->shared->udp_tunnel_nic_info = NULL;
 864         }
 865
 866         /* Flush before we check work, so we don't waste time adding entries
 867          * from the work which we will boot immediately.
 868          */
 869         udp_tunnel_nic_flush(dev, utn);
 870
 871         /* Wait for the work to be done using the state, netdev core will
 872          * retry unregister until we give up our reference on this device.
 873          */
 874         if (utn->work_pending)
 875                 return;
 876
 877         udp_tunnel_nic_free(utn);
 878 release_dev:
 879         dev->udp_tunnel_nic = NULL;
 880         dev_put(dev);
 881 }
 882
 883 static int
 884 udp_tunnel_nic_netdevice_event(struct notifier_block *unused,
 885                                unsigned long event, void *ptr)
 886 {
 887         struct net_device *dev = netdev_notifier_info_to_dev(ptr);
 888         const struct udp_tunnel_nic_info *info;
 889         struct udp_tunnel_nic *utn;
 890
 891         info = dev->udp_tunnel_nic_info;
 892         if (!info)
 893                 return NOTIFY_DONE;
 894
 895         if (event == NETDEV_REGISTER) {
 896                 int err;
 897
 898                 err = udp_tunnel_nic_register(dev);
 899                 if (err)
 900                         netdev_WARN(dev, "failed to register for UDP tunnel offloads: %d", err);
 901                 return notifier_from_errno(err);
 902         }
 903         /* All other events will need the udp_tunnel_nic state */
 904         utn = dev->udp_tunnel_nic;
 905         if (!utn)
 906                 return NOTIFY_DONE;
 907
 908         if (event == NETDEV_UNREGISTER) {
 909                 udp_tunnel_nic_unregister(dev, utn);
 910                 return NOTIFY_OK;
 911         }
 912
 913         /* All other events only matter if NIC has to be programmed open */
 914         if (!(info->flags & UDP_TUNNEL_NIC_INFO_OPEN_ONLY))
 915                 return NOTIFY_DONE;
 916
 917         if (event == NETDEV_UP) {
 918                 WARN_ON(!udp_tunnel_nic_is_empty(dev, utn));
 919                 udp_tunnel_get_rx_info(dev);
 920                 return NOTIFY_OK;
 921         }
 922         if (event == NETDEV_GOING_DOWN) {
 923                 udp_tunnel_nic_flush(dev, utn);
 924                 return NOTIFY_OK;
 925         }
 926
 927         return NOTIFY_DONE;
 928 }
 929
 930 static struct notifier_block udp_tunnel_nic_notifier_block __read_mostly = {
 931         .notifier_call = udp_tunnel_nic_netdevice_event,
 932 };
 933
 934 static int __init udp_tunnel_nic_init_module(void)
 935 {
 936         int err;
 937
 938         udp_tunnel_nic_workqueue = alloc_workqueue("udp_tunnel_nic", 0, 0);
 939         if (!udp_tunnel_nic_workqueue)
 940                 return -ENOMEM;
 941
 942         rtnl_lock();
 943         udp_tunnel_nic_ops = &__udp_tunnel_nic_ops;
 944         rtnl_unlock();
 945
 946         err = register_netdevice_notifier(&udp_tunnel_nic_notifier_block);
 947         if (err)
 948                 goto err_unset_ops;
 949
 950         return 0;
 951
 952 err_unset_ops:
 953         rtnl_lock();
 954         udp_tunnel_nic_ops = NULL;
 955         rtnl_unlock();
 956         destroy_workqueue(udp_tunnel_nic_workqueue);
 957         return err;
 958 }
 959 late_initcall(udp_tunnel_nic_init_module);
 960
 961 static void __exit udp_tunnel_nic_cleanup_module(void)
 962 {
 963         unregister_netdevice_notifier(&udp_tunnel_nic_notifier_block);
 964
 965         rtnl_lock();
 966         udp_tunnel_nic_ops = NULL;
 967         rtnl_unlock();
 968
 969         destroy_workqueue(udp_tunnel_nic_workqueue);
 970 }
 971 module_exit(udp_tunnel_nic_cleanup_module);
 972
 973 MODULE_LICENSE("GPL");