drivers/net/ethernet/netronome/nfp/flower/main.c

   1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
   2 /* Copyright (C) 2017-2018 Netronome Systems, Inc. */
   3
   4 #include <linux/etherdevice.h>
   5 #include <linux/lockdep.h>
   6 #include <linux/pci.h>
   7 #include <linux/skbuff.h>
   8 #include <linux/vmalloc.h>
   9 #include <net/devlink.h>
  10 #include <net/dst_metadata.h>
  11
  12 #include "main.h"
  13 #include "../nfpcore/nfp_cpp.h"
  14 #include "../nfpcore/nfp_nffw.h"
  15 #include "../nfpcore/nfp_nsp.h"
  16 #include "../nfp_app.h"
  17 #include "../nfp_main.h"
  18 #include "../nfp_net.h"
  19 #include "../nfp_net_repr.h"
  20 #include "../nfp_port.h"
  21 #include "./cmsg.h"
  22
  23 #define NFP_FLOWER_ALLOWED_VER 0x0001000000010000UL
  24
  25 #define NFP_MIN_INT_PORT_ID     1
  26 #define NFP_MAX_INT_PORT_ID     256
  27
  28 static const char *nfp_flower_extra_cap(struct nfp_app *app, struct nfp_net *nn)
  29 {
  30         return "FLOWER";
  31 }
  32
  33 static enum devlink_eswitch_mode eswitch_mode_get(struct nfp_app *app)
  34 {
  35         return DEVLINK_ESWITCH_MODE_SWITCHDEV;
  36 }
  37
  38 static int
  39 nfp_flower_lookup_internal_port_id(struct nfp_flower_priv *priv,
  40                                    struct net_device *netdev)
  41 {
  42         struct net_device *entry;
  43         int i, id = 0;
  44
  45         rcu_read_lock();
  46         idr_for_each_entry(&priv->internal_ports.port_ids, entry, i)
  47                 if (entry == netdev) {
  48                         id = i;
  49                         break;
  50                 }
  51         rcu_read_unlock();
  52
  53         return id;
  54 }
  55
  56 static int
  57 nfp_flower_get_internal_port_id(struct nfp_app *app, struct net_device *netdev)
  58 {
  59         struct nfp_flower_priv *priv = app->priv;
  60         int id;
  61
  62         id = nfp_flower_lookup_internal_port_id(priv, netdev);
  63         if (id > 0)
  64                 return id;
  65
  66         idr_preload(GFP_ATOMIC);
  67         spin_lock_bh(&priv->internal_ports.lock);
  68         id = idr_alloc(&priv->internal_ports.port_ids, netdev,
  69                        NFP_MIN_INT_PORT_ID, NFP_MAX_INT_PORT_ID, GFP_ATOMIC);
  70         spin_unlock_bh(&priv->internal_ports.lock);
  71         idr_preload_end();
  72
  73         return id;
  74 }
  75
  76 u32 nfp_flower_get_port_id_from_netdev(struct nfp_app *app,
  77                                        struct net_device *netdev)
  78 {
  79         struct nfp_flower_priv *priv = app->priv;
  80         int ext_port;
  81         int gid;
  82
  83         if (nfp_netdev_is_nfp_repr(netdev)) {
  84                 return nfp_repr_get_port_id(netdev);
  85         } else if (nfp_flower_internal_port_can_offload(app, netdev)) {
  86                 ext_port = nfp_flower_get_internal_port_id(app, netdev);
  87                 if (ext_port < 0)
  88                         return 0;
  89
  90                 return nfp_flower_internal_port_get_port_id(ext_port);
  91         } else if (netif_is_lag_master(netdev) &&
  92                    priv->flower_ext_feats & NFP_FL_FEATS_TUNNEL_NEIGH_LAG) {
  93                 gid = nfp_flower_lag_get_output_id(app, netdev);
  94                 if (gid < 0)
  95                         return 0;
  96
  97                 return (NFP_FL_LAG_OUT | gid);
  98         }
  99
 100         return 0;
 101 }
 102
 103 static struct net_device *
 104 nfp_flower_get_netdev_from_internal_port_id(struct nfp_app *app, int port_id)
 105 {
 106         struct nfp_flower_priv *priv = app->priv;
 107         struct net_device *netdev;
 108
 109         rcu_read_lock();
 110         netdev = idr_find(&priv->internal_ports.port_ids, port_id);
 111         rcu_read_unlock();
 112
 113         return netdev;
 114 }
 115
 116 static void
 117 nfp_flower_free_internal_port_id(struct nfp_app *app, struct net_device *netdev)
 118 {
 119         struct nfp_flower_priv *priv = app->priv;
 120         int id;
 121
 122         id = nfp_flower_lookup_internal_port_id(priv, netdev);
 123         if (!id)
 124                 return;
 125
 126         spin_lock_bh(&priv->internal_ports.lock);
 127         idr_remove(&priv->internal_ports.port_ids, id);
 128         spin_unlock_bh(&priv->internal_ports.lock);
 129 }
 130
 131 static int
 132 nfp_flower_internal_port_event_handler(struct nfp_app *app,
 133                                        struct net_device *netdev,
 134                                        unsigned long event)
 135 {
 136         if (event == NETDEV_UNREGISTER &&
 137             nfp_flower_internal_port_can_offload(app, netdev))
 138                 nfp_flower_free_internal_port_id(app, netdev);
 139
 140         return NOTIFY_OK;
 141 }
 142
 143 static void nfp_flower_internal_port_init(struct nfp_flower_priv *priv)
 144 {
 145         spin_lock_init(&priv->internal_ports.lock);
 146         idr_init(&priv->internal_ports.port_ids);
 147 }
 148
 149 static void nfp_flower_internal_port_cleanup(struct nfp_flower_priv *priv)
 150 {
 151         idr_destroy(&priv->internal_ports.port_ids);
 152 }
 153
 154 static struct nfp_flower_non_repr_priv *
 155 nfp_flower_non_repr_priv_lookup(struct nfp_app *app, struct net_device *netdev)
 156 {
 157         struct nfp_flower_priv *priv = app->priv;
 158         struct nfp_flower_non_repr_priv *entry;
 159
 160         ASSERT_RTNL();
 161
 162         list_for_each_entry(entry, &priv->non_repr_priv, list)
 163                 if (entry->netdev == netdev)
 164                         return entry;
 165
 166         return NULL;
 167 }
 168
 169 void
 170 __nfp_flower_non_repr_priv_get(struct nfp_flower_non_repr_priv *non_repr_priv)
 171 {
 172         non_repr_priv->ref_count++;
 173 }
 174
 175 struct nfp_flower_non_repr_priv *
 176 nfp_flower_non_repr_priv_get(struct nfp_app *app, struct net_device *netdev)
 177 {
 178         struct nfp_flower_priv *priv = app->priv;
 179         struct nfp_flower_non_repr_priv *entry;
 180
 181         entry = nfp_flower_non_repr_priv_lookup(app, netdev);
 182         if (entry)
 183                 goto inc_ref;
 184
 185         entry = kzalloc(sizeof(*entry), GFP_KERNEL);
 186         if (!entry)
 187                 return NULL;
 188
 189         entry->netdev = netdev;
 190         list_add(&entry->list, &priv->non_repr_priv);
 191
 192 inc_ref:
 193         __nfp_flower_non_repr_priv_get(entry);
 194         return entry;
 195 }
 196
 197 void
 198 __nfp_flower_non_repr_priv_put(struct nfp_flower_non_repr_priv *non_repr_priv)
 199 {
 200         if (--non_repr_priv->ref_count)
 201                 return;
 202
 203         list_del(&non_repr_priv->list);
 204         kfree(non_repr_priv);
 205 }
 206
 207 void
 208 nfp_flower_non_repr_priv_put(struct nfp_app *app, struct net_device *netdev)
 209 {
 210         struct nfp_flower_non_repr_priv *entry;
 211
 212         entry = nfp_flower_non_repr_priv_lookup(app, netdev);
 213         if (!entry)
 214                 return;
 215
 216         __nfp_flower_non_repr_priv_put(entry);
 217 }
 218
 219 static enum nfp_repr_type
 220 nfp_flower_repr_get_type_and_port(struct nfp_app *app, u32 port_id, u8 *port)
 221 {
 222         switch (FIELD_GET(NFP_FLOWER_CMSG_PORT_TYPE, port_id)) {
 223         case NFP_FLOWER_CMSG_PORT_TYPE_PHYS_PORT:
 224                 *port = FIELD_GET(NFP_FLOWER_CMSG_PORT_PHYS_PORT_NUM,
 225                                   port_id);
 226                 return NFP_REPR_TYPE_PHYS_PORT;
 227
 228         case NFP_FLOWER_CMSG_PORT_TYPE_PCIE_PORT:
 229                 *port = FIELD_GET(NFP_FLOWER_CMSG_PORT_VNIC, port_id);
 230                 if (FIELD_GET(NFP_FLOWER_CMSG_PORT_VNIC_TYPE, port_id) ==
 231                     NFP_FLOWER_CMSG_PORT_VNIC_TYPE_PF)
 232                         return NFP_REPR_TYPE_PF;
 233                 else
 234                         return NFP_REPR_TYPE_VF;
 235         }
 236
 237         return __NFP_REPR_TYPE_MAX;
 238 }
 239
 240 static struct net_device *
 241 nfp_flower_dev_get(struct nfp_app *app, u32 port_id, bool *redir_egress)
 242 {
 243         enum nfp_repr_type repr_type;
 244         struct nfp_reprs *reprs;
 245         u8 port = 0;
 246
 247         /* Check if the port is internal. */
 248         if (FIELD_GET(NFP_FLOWER_CMSG_PORT_TYPE, port_id) ==
 249             NFP_FLOWER_CMSG_PORT_TYPE_OTHER_PORT) {
 250                 if (redir_egress)
 251                         *redir_egress = true;
 252                 port = FIELD_GET(NFP_FLOWER_CMSG_PORT_PHYS_PORT_NUM, port_id);
 253                 return nfp_flower_get_netdev_from_internal_port_id(app, port);
 254         }
 255
 256         repr_type = nfp_flower_repr_get_type_and_port(app, port_id, &port);
 257         if (repr_type > NFP_REPR_TYPE_MAX)
 258                 return NULL;
 259
 260         reprs = rcu_dereference(app->reprs[repr_type]);
 261         if (!reprs)
 262                 return NULL;
 263
 264         if (port >= reprs->num_reprs)
 265                 return NULL;
 266
 267         return rcu_dereference(reprs->reprs[port]);
 268 }
 269
 270 static int
 271 nfp_flower_reprs_reify(struct nfp_app *app, enum nfp_repr_type type,
 272                        bool exists)
 273 {
 274         struct nfp_reprs *reprs;
 275         int i, err, count = 0;
 276
 277         reprs = rcu_dereference_protected(app->reprs[type],
 278                                           nfp_app_is_locked(app));
 279         if (!reprs)
 280                 return 0;
 281
 282         for (i = 0; i < reprs->num_reprs; i++) {
 283                 struct net_device *netdev;
 284
 285                 netdev = nfp_repr_get_locked(app, reprs, i);
 286                 if (netdev) {
 287                         struct nfp_repr *repr = netdev_priv(netdev);
 288
 289                         err = nfp_flower_cmsg_portreify(repr, exists);
 290                         if (err)
 291                                 return err;
 292                         count++;
 293                 }
 294         }
 295
 296         return count;
 297 }
 298
 299 static int
 300 nfp_flower_wait_repr_reify(struct nfp_app *app, atomic_t *replies, int tot_repl)
 301 {
 302         struct nfp_flower_priv *priv = app->priv;
 303
 304         if (!tot_repl)
 305                 return 0;
 306
 307         assert_nfp_app_locked(app);
 308         if (!wait_event_timeout(priv->reify_wait_queue,
 309                                 atomic_read(replies) >= tot_repl,
 310                                 NFP_FL_REPLY_TIMEOUT)) {
 311                 nfp_warn(app->cpp, "Not all reprs responded to reify\n");
 312                 return -EIO;
 313         }
 314
 315         return 0;
 316 }
 317
 318 static int
 319 nfp_flower_repr_netdev_open(struct nfp_app *app, struct nfp_repr *repr)
 320 {
 321         int err;
 322
 323         err = nfp_flower_cmsg_portmod(repr, true, repr->netdev->mtu, false);
 324         if (err)
 325                 return err;
 326
 327         netif_tx_wake_all_queues(repr->netdev);
 328
 329         return 0;
 330 }
 331
 332 static int
 333 nfp_flower_repr_netdev_stop(struct nfp_app *app, struct nfp_repr *repr)
 334 {
 335         netif_tx_disable(repr->netdev);
 336
 337         return nfp_flower_cmsg_portmod(repr, false, repr->netdev->mtu, false);
 338 }
 339
 340 static void
 341 nfp_flower_repr_netdev_clean(struct nfp_app *app, struct net_device *netdev)
 342 {
 343         struct nfp_repr *repr = netdev_priv(netdev);
 344
 345         kfree(repr->app_priv);
 346 }
 347
 348 static void
 349 nfp_flower_repr_netdev_preclean(struct nfp_app *app, struct net_device *netdev)
 350 {
 351         struct nfp_repr *repr = netdev_priv(netdev);
 352         struct nfp_flower_priv *priv = app->priv;
 353         atomic_t *replies = &priv->reify_replies;
 354         int err;
 355
 356         atomic_set(replies, 0);
 357         err = nfp_flower_cmsg_portreify(repr, false);
 358         if (err) {
 359                 nfp_warn(app->cpp, "Failed to notify firmware about repr destruction\n");
 360                 return;
 361         }
 362
 363         nfp_flower_wait_repr_reify(app, replies, 1);
 364 }
 365
 366 static void nfp_flower_sriov_disable(struct nfp_app *app)
 367 {
 368         struct nfp_flower_priv *priv = app->priv;
 369
 370         if (!priv->nn)
 371                 return;
 372
 373         nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_VF);
 374 }
 375
 376 static int
 377 nfp_flower_spawn_vnic_reprs(struct nfp_app *app,
 378                             enum nfp_flower_cmsg_port_vnic_type vnic_type,
 379                             enum nfp_repr_type repr_type, unsigned int cnt)
 380 {
 381         u8 nfp_pcie = nfp_cppcore_pcie_unit(app->pf->cpp);
 382         struct nfp_flower_priv *priv = app->priv;
 383         atomic_t *replies = &priv->reify_replies;
 384         struct nfp_flower_repr_priv *repr_priv;
 385         enum nfp_port_type port_type;
 386         struct nfp_repr *nfp_repr;
 387         struct nfp_reprs *reprs;
 388         int i, err, reify_cnt;
 389         const u8 queue = 0;
 390
 391         port_type = repr_type == NFP_REPR_TYPE_PF ? NFP_PORT_PF_PORT :
 392                                                     NFP_PORT_VF_PORT;
 393
 394         reprs = nfp_reprs_alloc(cnt);
 395         if (!reprs)
 396                 return -ENOMEM;
 397
 398         for (i = 0; i < cnt; i++) {
 399                 struct net_device *repr;
 400                 struct nfp_port *port;
 401                 u32 port_id;
 402
 403                 repr = nfp_repr_alloc(app);
 404                 if (!repr) {
 405                         err = -ENOMEM;
 406                         goto err_reprs_clean;
 407                 }
 408
 409                 repr_priv = kzalloc(sizeof(*repr_priv), GFP_KERNEL);
 410                 if (!repr_priv) {
 411                         err = -ENOMEM;
 412                         nfp_repr_free(repr);
 413                         goto err_reprs_clean;
 414                 }
 415
 416                 nfp_repr = netdev_priv(repr);
 417                 nfp_repr->app_priv = repr_priv;
 418                 repr_priv->nfp_repr = nfp_repr;
 419
 420                 /* For now we only support 1 PF */
 421                 WARN_ON(repr_type == NFP_REPR_TYPE_PF && i);
 422
 423                 port = nfp_port_alloc(app, port_type, repr);
 424                 if (IS_ERR(port)) {
 425                         err = PTR_ERR(port);
 426                         kfree(repr_priv);
 427                         nfp_repr_free(repr);
 428                         goto err_reprs_clean;
 429                 }
 430                 if (repr_type == NFP_REPR_TYPE_PF) {
 431                         port->pf_id = i;
 432                         port->vnic = priv->nn->dp.ctrl_bar;
 433                 } else {
 434                         port->pf_id = 0;
 435                         port->vf_id = i;
 436                         port->vnic =
 437                                 app->pf->vf_cfg_mem + i * NFP_NET_CFG_BAR_SZ;
 438                 }
 439
 440                 eth_hw_addr_random(repr);
 441
 442                 port_id = nfp_flower_cmsg_pcie_port(nfp_pcie, vnic_type,
 443                                                     i, queue);
 444                 err = nfp_repr_init(app, repr,
 445                                     port_id, port, priv->nn->dp.netdev);
 446                 if (err) {
 447                         kfree(repr_priv);
 448                         nfp_port_free(port);
 449                         nfp_repr_free(repr);
 450                         goto err_reprs_clean;
 451                 }
 452
 453                 RCU_INIT_POINTER(reprs->reprs[i], repr);
 454                 nfp_info(app->cpp, "%s%d Representor(%s) created\n",
 455                          repr_type == NFP_REPR_TYPE_PF ? "PF" : "VF", i,
 456                          repr->name);
 457         }
 458
 459         nfp_app_reprs_set(app, repr_type, reprs);
 460
 461         atomic_set(replies, 0);
 462         reify_cnt = nfp_flower_reprs_reify(app, repr_type, true);
 463         if (reify_cnt < 0) {
 464                 err = reify_cnt;
 465                 nfp_warn(app->cpp, "Failed to notify firmware about repr creation\n");
 466                 goto err_reprs_remove;
 467         }
 468
 469         err = nfp_flower_wait_repr_reify(app, replies, reify_cnt);
 470         if (err)
 471                 goto err_reprs_remove;
 472
 473         return 0;
 474 err_reprs_remove:
 475         reprs = nfp_app_reprs_set(app, repr_type, NULL);
 476 err_reprs_clean:
 477         nfp_reprs_clean_and_free(app, reprs);
 478         return err;
 479 }
 480
 481 static int nfp_flower_sriov_enable(struct nfp_app *app, int num_vfs)
 482 {
 483         struct nfp_flower_priv *priv = app->priv;
 484
 485         if (!priv->nn)
 486                 return 0;
 487
 488         return nfp_flower_spawn_vnic_reprs(app,
 489                                            NFP_FLOWER_CMSG_PORT_VNIC_TYPE_VF,
 490                                            NFP_REPR_TYPE_VF, num_vfs);
 491 }
 492
 493 static int
 494 nfp_flower_spawn_phy_reprs(struct nfp_app *app, struct nfp_flower_priv *priv)
 495 {
 496         struct nfp_eth_table *eth_tbl = app->pf->eth_tbl;
 497         atomic_t *replies = &priv->reify_replies;
 498         struct nfp_flower_repr_priv *repr_priv;
 499         struct nfp_repr *nfp_repr;
 500         struct sk_buff *ctrl_skb;
 501         struct nfp_reprs *reprs;
 502         int err, reify_cnt;
 503         unsigned int i;
 504
 505         ctrl_skb = nfp_flower_cmsg_mac_repr_start(app, eth_tbl->count);
 506         if (!ctrl_skb)
 507                 return -ENOMEM;
 508
 509         reprs = nfp_reprs_alloc(eth_tbl->max_index + 1);
 510         if (!reprs) {
 511                 err = -ENOMEM;
 512                 goto err_free_ctrl_skb;
 513         }
 514
 515         for (i = 0; i < eth_tbl->count; i++) {
 516                 unsigned int phys_port = eth_tbl->ports[i].index;
 517                 struct net_device *repr;
 518                 struct nfp_port *port;
 519                 u32 cmsg_port_id;
 520
 521                 repr = nfp_repr_alloc(app);
 522                 if (!repr) {
 523                         err = -ENOMEM;
 524                         goto err_reprs_clean;
 525                 }
 526
 527                 repr_priv = kzalloc(sizeof(*repr_priv), GFP_KERNEL);
 528                 if (!repr_priv) {
 529                         err = -ENOMEM;
 530                         nfp_repr_free(repr);
 531                         goto err_reprs_clean;
 532                 }
 533
 534                 nfp_repr = netdev_priv(repr);
 535                 nfp_repr->app_priv = repr_priv;
 536                 repr_priv->nfp_repr = nfp_repr;
 537
 538                 port = nfp_port_alloc(app, NFP_PORT_PHYS_PORT, repr);
 539                 if (IS_ERR(port)) {
 540                         err = PTR_ERR(port);
 541                         kfree(repr_priv);
 542                         nfp_repr_free(repr);
 543                         goto err_reprs_clean;
 544                 }
 545                 err = nfp_port_init_phy_port(app->pf, app, port, i);
 546                 if (err) {
 547                         kfree(repr_priv);
 548                         nfp_port_free(port);
 549                         nfp_repr_free(repr);
 550                         goto err_reprs_clean;
 551                 }
 552
 553                 SET_NETDEV_DEV(repr, &priv->nn->pdev->dev);
 554                 nfp_net_get_mac_addr(app->pf, repr, port);
 555
 556                 cmsg_port_id = nfp_flower_cmsg_phys_port(phys_port);
 557                 err = nfp_repr_init(app, repr,
 558                                     cmsg_port_id, port, priv->nn->dp.netdev);
 559                 if (err) {
 560                         kfree(repr_priv);
 561                         nfp_port_free(port);
 562                         nfp_repr_free(repr);
 563                         goto err_reprs_clean;
 564                 }
 565
 566                 nfp_flower_cmsg_mac_repr_add(ctrl_skb, i,
 567                                              eth_tbl->ports[i].nbi,
 568                                              eth_tbl->ports[i].base,
 569                                              phys_port);
 570
 571                 RCU_INIT_POINTER(reprs->reprs[phys_port], repr);
 572                 nfp_info(app->cpp, "Phys Port %d Representor(%s) created\n",
 573                          phys_port, repr->name);
 574         }
 575
 576         nfp_app_reprs_set(app, NFP_REPR_TYPE_PHYS_PORT, reprs);
 577
 578         /* The REIFY/MAC_REPR control messages should be sent after the MAC
 579          * representors are registered using nfp_app_reprs_set().  This is
 580          * because the firmware may respond with control messages for the
 581          * MAC representors, f.e. to provide the driver with information
 582          * about their state, and without registration the driver will drop
 583          * any such messages.
 584          */
 585         atomic_set(replies, 0);
 586         reify_cnt = nfp_flower_reprs_reify(app, NFP_REPR_TYPE_PHYS_PORT, true);
 587         if (reify_cnt < 0) {
 588                 err = reify_cnt;
 589                 nfp_warn(app->cpp, "Failed to notify firmware about repr creation\n");
 590                 goto err_reprs_remove;
 591         }
 592
 593         err = nfp_flower_wait_repr_reify(app, replies, reify_cnt);
 594         if (err)
 595                 goto err_reprs_remove;
 596
 597         nfp_ctrl_tx(app->ctrl, ctrl_skb);
 598
 599         return 0;
 600 err_reprs_remove:
 601         reprs = nfp_app_reprs_set(app, NFP_REPR_TYPE_PHYS_PORT, NULL);
 602 err_reprs_clean:
 603         nfp_reprs_clean_and_free(app, reprs);
 604 err_free_ctrl_skb:
 605         kfree_skb(ctrl_skb);
 606         return err;
 607 }
 608
 609 static int nfp_flower_vnic_alloc(struct nfp_app *app, struct nfp_net *nn,
 610                                  unsigned int id)
 611 {
 612         if (id > 0) {
 613                 nfp_warn(app->cpp, "FlowerNIC doesn't support more than one data vNIC\n");
 614                 goto err_invalid_port;
 615         }
 616
 617         eth_hw_addr_random(nn->dp.netdev);
 618         netif_keep_dst(nn->dp.netdev);
 619         nn->vnic_no_name = true;
 620
 621         return 0;
 622
 623 err_invalid_port:
 624         nn->port = nfp_port_alloc(app, NFP_PORT_INVALID, nn->dp.netdev);
 625         return PTR_ERR_OR_ZERO(nn->port);
 626 }
 627
 628 static void nfp_flower_vnic_clean(struct nfp_app *app, struct nfp_net *nn)
 629 {
 630         struct nfp_flower_priv *priv = app->priv;
 631
 632         if (app->pf->num_vfs)
 633                 nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_VF);
 634         nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PF);
 635         nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PHYS_PORT);
 636
 637         priv->nn = NULL;
 638 }
 639
 640 static int nfp_flower_vnic_init(struct nfp_app *app, struct nfp_net *nn)
 641 {
 642         struct nfp_flower_priv *priv = app->priv;
 643         int err;
 644
 645         priv->nn = nn;
 646
 647         err = nfp_flower_spawn_phy_reprs(app, app->priv);
 648         if (err)
 649                 goto err_clear_nn;
 650
 651         err = nfp_flower_spawn_vnic_reprs(app,
 652                                           NFP_FLOWER_CMSG_PORT_VNIC_TYPE_PF,
 653                                           NFP_REPR_TYPE_PF, 1);
 654         if (err)
 655                 goto err_destroy_reprs_phy;
 656
 657         if (app->pf->num_vfs) {
 658                 err = nfp_flower_spawn_vnic_reprs(app,
 659                                                   NFP_FLOWER_CMSG_PORT_VNIC_TYPE_VF,
 660                                                   NFP_REPR_TYPE_VF,
 661                                                   app->pf->num_vfs);
 662                 if (err)
 663                         goto err_destroy_reprs_pf;
 664         }
 665
 666         return 0;
 667
 668 err_destroy_reprs_pf:
 669         nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PF);
 670 err_destroy_reprs_phy:
 671         nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PHYS_PORT);
 672 err_clear_nn:
 673         priv->nn = NULL;
 674         return err;
 675 }
 676
 677 static void nfp_flower_wait_host_bit(struct nfp_app *app)
 678 {
 679         unsigned long err_at;
 680         u64 feat;
 681         int err;
 682
 683         /* Wait for HOST_ACK flag bit to propagate */
 684         err_at = jiffies + msecs_to_jiffies(100);
 685         do {
 686                 feat = nfp_rtsym_read_le(app->pf->rtbl,
 687                                          "_abi_flower_combined_features_global",
 688                                          &err);
 689                 if (time_is_before_eq_jiffies(err_at)) {
 690                         nfp_warn(app->cpp,
 691                                  "HOST_ACK bit not propagated in FW.\n");
 692                         break;
 693                 }
 694                 usleep_range(1000, 2000);
 695         } while (!err && !(feat & NFP_FL_FEATS_HOST_ACK));
 696
 697         if (err)
 698                 nfp_warn(app->cpp,
 699                          "Could not read global features entry from FW\n");
 700 }
 701
 702 static int nfp_flower_sync_feature_bits(struct nfp_app *app)
 703 {
 704         struct nfp_flower_priv *app_priv = app->priv;
 705         int err;
 706
 707         /* Tell the firmware of the host supported features. */
 708         err = nfp_rtsym_write_le(app->pf->rtbl, "_abi_flower_host_mask",
 709                                  app_priv->flower_ext_feats |
 710                                  NFP_FL_FEATS_HOST_ACK);
 711         if (!err)
 712                 nfp_flower_wait_host_bit(app);
 713         else if (err != -ENOENT)
 714                 return err;
 715
 716         /* Tell the firmware that the driver supports lag. */
 717         err = nfp_rtsym_write_le(app->pf->rtbl,
 718                                  "_abi_flower_balance_sync_enable", 1);
 719         if (!err) {
 720                 app_priv->flower_en_feats |= NFP_FL_ENABLE_LAG;
 721                 nfp_flower_lag_init(&app_priv->nfp_lag);
 722         } else if (err == -ENOENT) {
 723                 nfp_warn(app->cpp, "LAG not supported by FW.\n");
 724         } else {
 725                 return err;
 726         }
 727
 728         if (app_priv->flower_ext_feats & NFP_FL_FEATS_FLOW_MOD) {
 729                 /* Tell the firmware that the driver supports flow merging. */
 730                 err = nfp_rtsym_write_le(app->pf->rtbl,
 731                                          "_abi_flower_merge_hint_enable", 1);
 732                 if (!err) {
 733                         app_priv->flower_en_feats |= NFP_FL_ENABLE_FLOW_MERGE;
 734                         nfp_flower_internal_port_init(app_priv);
 735                 } else if (err == -ENOENT) {
 736                         nfp_warn(app->cpp,
 737                                  "Flow merge not supported by FW.\n");
 738                 } else {
 739                         return err;
 740                 }
 741         } else {
 742                 nfp_warn(app->cpp, "Flow mod/merge not supported by FW.\n");
 743         }
 744
 745         return 0;
 746 }
 747
 748 static int nfp_flower_init(struct nfp_app *app)
 749 {
 750         u64 version, features, ctx_count, num_mems;
 751         const struct nfp_pf *pf = app->pf;
 752         struct nfp_flower_priv *app_priv;
 753         int err;
 754
 755         if (!pf->eth_tbl) {
 756                 nfp_warn(app->cpp, "FlowerNIC requires eth table\n");
 757                 return -EINVAL;
 758         }
 759
 760         if (!pf->mac_stats_bar) {
 761                 nfp_warn(app->cpp, "FlowerNIC requires mac_stats BAR\n");
 762                 return -EINVAL;
 763         }
 764
 765         if (!pf->vf_cfg_bar) {
 766                 nfp_warn(app->cpp, "FlowerNIC requires vf_cfg BAR\n");
 767                 return -EINVAL;
 768         }
 769
 770         version = nfp_rtsym_read_le(app->pf->rtbl, "hw_flower_version", &err);
 771         if (err) {
 772                 nfp_warn(app->cpp, "FlowerNIC requires hw_flower_version memory symbol\n");
 773                 return err;
 774         }
 775
 776         num_mems = nfp_rtsym_read_le(app->pf->rtbl, "CONFIG_FC_HOST_CTX_SPLIT",
 777                                      &err);
 778         if (err) {
 779                 nfp_warn(app->cpp,
 780                          "FlowerNIC: unsupported host context memory: %d\n",
 781                          err);
 782                 err = 0;
 783                 num_mems = 1;
 784         }
 785
 786         if (!FIELD_FIT(NFP_FL_STAT_ID_MU_NUM, num_mems) || !num_mems) {
 787                 nfp_warn(app->cpp,
 788                          "FlowerNIC: invalid host context memory: %llu\n",
 789                          num_mems);
 790                 return -EINVAL;
 791         }
 792
 793         ctx_count = nfp_rtsym_read_le(app->pf->rtbl, "CONFIG_FC_HOST_CTX_COUNT",
 794                                       &err);
 795         if (err) {
 796                 nfp_warn(app->cpp,
 797                          "FlowerNIC: unsupported host context count: %d\n",
 798                          err);
 799                 err = 0;
 800                 ctx_count = BIT(17);
 801         }
 802
 803         /* We need to ensure hardware has enough flower capabilities. */
 804         if (version != NFP_FLOWER_ALLOWED_VER) {
 805                 nfp_warn(app->cpp, "FlowerNIC: unsupported firmware version\n");
 806                 return -EINVAL;
 807         }
 808
 809         app_priv = vzalloc(sizeof(struct nfp_flower_priv));
 810         if (!app_priv)
 811                 return -ENOMEM;
 812
 813         app_priv->total_mem_units = num_mems;
 814         app_priv->active_mem_unit = 0;
 815         app_priv->stats_ring_size = roundup_pow_of_two(ctx_count);
 816         app->priv = app_priv;
 817         app_priv->app = app;
 818         skb_queue_head_init(&app_priv->cmsg_skbs_high);
 819         skb_queue_head_init(&app_priv->cmsg_skbs_low);
 820         INIT_WORK(&app_priv->cmsg_work, nfp_flower_cmsg_process_rx);
 821         init_waitqueue_head(&app_priv->reify_wait_queue);
 822
 823         init_waitqueue_head(&app_priv->mtu_conf.wait_q);
 824         spin_lock_init(&app_priv->mtu_conf.lock);
 825
 826         err = nfp_flower_metadata_init(app, ctx_count, num_mems);
 827         if (err)
 828                 goto err_free_app_priv;
 829
 830         /* Extract the extra features supported by the firmware. */
 831         features = nfp_rtsym_read_le(app->pf->rtbl,
 832                                      "_abi_flower_extra_features", &err);
 833         if (err)
 834                 app_priv->flower_ext_feats = 0;
 835         else
 836                 app_priv->flower_ext_feats = features & NFP_FL_FEATS_HOST;
 837
 838         err = nfp_flower_sync_feature_bits(app);
 839         if (err)
 840                 goto err_cleanup;
 841
 842         if (app_priv->flower_ext_feats & NFP_FL_FEATS_VF_RLIM)
 843                 nfp_flower_qos_init(app);
 844
 845         INIT_LIST_HEAD(&app_priv->indr_block_cb_priv);
 846         INIT_LIST_HEAD(&app_priv->non_repr_priv);
 847         app_priv->pre_tun_rule_cnt = 0;
 848
 849         return 0;
 850
 851 err_cleanup:
 852         if (app_priv->flower_en_feats & NFP_FL_ENABLE_LAG)
 853                 nfp_flower_lag_cleanup(&app_priv->nfp_lag);
 854         nfp_flower_metadata_cleanup(app);
 855 err_free_app_priv:
 856         vfree(app->priv);
 857         return err;
 858 }
 859
 860 static void nfp_flower_clean(struct nfp_app *app)
 861 {
 862         struct nfp_flower_priv *app_priv = app->priv;
 863
 864         skb_queue_purge(&app_priv->cmsg_skbs_high);
 865         skb_queue_purge(&app_priv->cmsg_skbs_low);
 866         flush_work(&app_priv->cmsg_work);
 867
 868         if (app_priv->flower_ext_feats & NFP_FL_FEATS_VF_RLIM)
 869                 nfp_flower_qos_cleanup(app);
 870
 871         if (app_priv->flower_en_feats & NFP_FL_ENABLE_LAG)
 872                 nfp_flower_lag_cleanup(&app_priv->nfp_lag);
 873
 874         if (app_priv->flower_en_feats & NFP_FL_ENABLE_FLOW_MERGE)
 875                 nfp_flower_internal_port_cleanup(app_priv);
 876
 877         nfp_flower_metadata_cleanup(app);
 878         vfree(app->priv);
 879         app->priv = NULL;
 880 }
 881
 882 static bool nfp_flower_check_ack(struct nfp_flower_priv *app_priv)
 883 {
 884         bool ret;
 885
 886         spin_lock_bh(&app_priv->mtu_conf.lock);
 887         ret = app_priv->mtu_conf.ack;
 888         spin_unlock_bh(&app_priv->mtu_conf.lock);
 889
 890         return ret;
 891 }
 892
 893 static int
 894 nfp_flower_repr_change_mtu(struct nfp_app *app, struct net_device *netdev,
 895                            int new_mtu)
 896 {
 897         struct nfp_flower_priv *app_priv = app->priv;
 898         struct nfp_repr *repr = netdev_priv(netdev);
 899         int err;
 900
 901         /* Only need to config FW for physical port MTU change. */
 902         if (repr->port->type != NFP_PORT_PHYS_PORT)
 903                 return 0;
 904
 905         if (!(app_priv->flower_ext_feats & NFP_FL_NBI_MTU_SETTING)) {
 906                 nfp_err(app->cpp, "Physical port MTU setting not supported\n");
 907                 return -EINVAL;
 908         }
 909
 910         spin_lock_bh(&app_priv->mtu_conf.lock);
 911         app_priv->mtu_conf.ack = false;
 912         app_priv->mtu_conf.requested_val = new_mtu;
 913         app_priv->mtu_conf.portnum = repr->dst->u.port_info.port_id;
 914         spin_unlock_bh(&app_priv->mtu_conf.lock);
 915
 916         err = nfp_flower_cmsg_portmod(repr, netif_carrier_ok(netdev), new_mtu,
 917                                       true);
 918         if (err) {
 919                 spin_lock_bh(&app_priv->mtu_conf.lock);
 920                 app_priv->mtu_conf.requested_val = 0;
 921                 spin_unlock_bh(&app_priv->mtu_conf.lock);
 922                 return err;
 923         }
 924
 925         /* Wait for fw to ack the change. */
 926         if (!wait_event_timeout(app_priv->mtu_conf.wait_q,
 927                                 nfp_flower_check_ack(app_priv),
 928                                 NFP_FL_REPLY_TIMEOUT)) {
 929                 spin_lock_bh(&app_priv->mtu_conf.lock);
 930                 app_priv->mtu_conf.requested_val = 0;
 931                 spin_unlock_bh(&app_priv->mtu_conf.lock);
 932                 nfp_warn(app->cpp, "MTU change not verified with fw\n");
 933                 return -EIO;
 934         }
 935
 936         return 0;
 937 }
 938
 939 static int nfp_flower_start(struct nfp_app *app)
 940 {
 941         struct nfp_flower_priv *app_priv = app->priv;
 942         int err;
 943
 944         if (app_priv->flower_en_feats & NFP_FL_ENABLE_LAG) {
 945                 err = nfp_flower_lag_reset(&app_priv->nfp_lag);
 946                 if (err)
 947                         return err;
 948         }
 949
 950         err = flow_indr_dev_register(nfp_flower_indr_setup_tc_cb, app);
 951         if (err)
 952                 return err;
 953
 954         err = nfp_tunnel_config_start(app);
 955         if (err)
 956                 goto err_tunnel_config;
 957
 958         return 0;
 959
 960 err_tunnel_config:
 961         flow_indr_dev_unregister(nfp_flower_indr_setup_tc_cb, app,
 962                                  nfp_flower_setup_indr_tc_release);
 963         return err;
 964 }
 965
 966 static void nfp_flower_stop(struct nfp_app *app)
 967 {
 968         nfp_tunnel_config_stop(app);
 969
 970         flow_indr_dev_unregister(nfp_flower_indr_setup_tc_cb, app,
 971                                  nfp_flower_setup_indr_tc_release);
 972 }
 973
 974 static int
 975 nfp_flower_netdev_event(struct nfp_app *app, struct net_device *netdev,
 976                         unsigned long event, void *ptr)
 977 {
 978         struct nfp_flower_priv *app_priv = app->priv;
 979         int ret;
 980
 981         if (app_priv->flower_en_feats & NFP_FL_ENABLE_LAG) {
 982                 ret = nfp_flower_lag_netdev_event(app_priv, netdev, event, ptr);
 983                 if (ret & NOTIFY_STOP_MASK)
 984                         return ret;
 985         }
 986
 987         ret = nfp_flower_internal_port_event_handler(app, netdev, event);
 988         if (ret & NOTIFY_STOP_MASK)
 989                 return ret;
 990
 991         return nfp_tunnel_mac_event_handler(app, netdev, event, ptr);
 992 }
 993
 994 const struct nfp_app_type app_flower = {
 995         .id             = NFP_APP_FLOWER_NIC,
 996         .name           = "flower",
 997
 998         .ctrl_cap_mask  = ~0U,
 999         .ctrl_has_meta  = true,
1000
1001         .extra_cap      = nfp_flower_extra_cap,
1002
1003         .init           = nfp_flower_init,
1004         .clean          = nfp_flower_clean,
1005
1006         .repr_change_mtu  = nfp_flower_repr_change_mtu,
1007
1008         .vnic_alloc     = nfp_flower_vnic_alloc,
1009         .vnic_init      = nfp_flower_vnic_init,
1010         .vnic_clean     = nfp_flower_vnic_clean,
1011
1012         .repr_preclean  = nfp_flower_repr_netdev_preclean,
1013         .repr_clean     = nfp_flower_repr_netdev_clean,
1014
1015         .repr_open      = nfp_flower_repr_netdev_open,
1016         .repr_stop      = nfp_flower_repr_netdev_stop,
1017
1018         .start          = nfp_flower_start,
1019         .stop           = nfp_flower_stop,
1020
1021         .netdev_event   = nfp_flower_netdev_event,
1022
1023         .ctrl_msg_rx    = nfp_flower_cmsg_rx,
1024
1025         .sriov_enable   = nfp_flower_sriov_enable,
1026         .sriov_disable  = nfp_flower_sriov_disable,
1027
1028         .eswitch_mode_get  = eswitch_mode_get,
1029         .dev_get        = nfp_flower_dev_get,
1030
1031         .setup_tc       = nfp_flower_setup_tc,
1032 };